diff --git a/en/resources/fine_tuning/index.md b/en/resources/fine_tuning/index.md index 345df354..547a547b 100644 --- a/en/resources/fine_tuning/index.md +++ b/en/resources/fine_tuning/index.md @@ -1,190 +1,18 @@ # Fine Tuning -# Methods - -## Domain Types - -### Dpo Hyperparameters - -- `DpoHyperparameters object { batch_size, beta, learning_rate_multiplier, n_epochs }` - - The hyperparameters used for the DPO fine-tuning job. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `beta: optional "auto" or number` - - The beta value for the DPO method. A higher beta value will increase the weight of the penalty between the policy and reference model. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - -### Dpo Method - -- `DpoMethod object { hyperparameters }` - - Configuration for the DPO fine-tuning method. - - - `hyperparameters: optional DpoHyperparameters` - - The hyperparameters used for the DPO fine-tuning job. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `beta: optional "auto" or number` - - The beta value for the DPO method. A higher beta value will increase the weight of the penalty between the policy and reference model. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - -### Reinforcement Hyperparameters - -- `ReinforcementHyperparameters object { batch_size, compute_multiplier, eval_interval, 4 more }` - - The hyperparameters used for the reinforcement fine-tuning job. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `compute_multiplier: optional "auto" or number` - - Multiplier on amount of compute used for exploring search space during training. - - - `"auto"` - - - `"auto"` - - - `number` - - - `eval_interval: optional "auto" or number` - - The number of training steps between evaluation runs. - - - `"auto"` - - - `"auto"` - - - `number` - - - `eval_samples: optional "auto" or number` - - Number of evaluation samples to generate per training step. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - - - `reasoning_effort: optional "default" or "low" or "medium" or "high"` - - Level of reasoning effort. - - - `"default"` - - - `"low"` +# Alpha - - `"medium"` +# Graders - - `"high"` +## Run grader -### Reinforcement Method +**post** `/fine_tuning/alpha/graders/run` -- `ReinforcementMethod object { grader, hyperparameters }` +Run a grader. - Configuration for the reinforcement fine-tuning method. +### Body Parameters - - `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` +- `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` The grader used for the fine-tuning job. @@ -660,352 +488,300 @@ - `"multi"` - - `hyperparameters: optional ReinforcementHyperparameters` - - The hyperparameters used for the reinforcement fine-tuning job. - - - `batch_size: optional "auto" or number` +- `model_sample: string` - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. + The model sample to be evaluated. This value will be used to populate + the `sample` namespace. See [the guide](/docs/guides/graders) for more details. + The `output_json` variable will be populated if the model sample is a + valid JSON string. - - `"auto"` +- `item: optional unknown` - - `"auto"` + The dataset item provided to the grader. This will be used to populate + the `item` namespace. See [the guide](/docs/guides/graders) for more details. - - `number` +### Returns - - `compute_multiplier: optional "auto" or number` +- `metadata: object { errors, execution_time, name, 4 more }` - Multiplier on amount of compute used for exploring search space during training. + - `errors: object { formula_parse_error, invalid_variable_error, model_grader_parse_error, 11 more }` - - `"auto"` + - `formula_parse_error: boolean` - - `"auto"` + - `invalid_variable_error: boolean` - - `number` + - `model_grader_parse_error: boolean` - - `eval_interval: optional "auto" or number` + - `model_grader_refusal_error: boolean` - The number of training steps between evaluation runs. + - `model_grader_server_error: boolean` - - `"auto"` + - `model_grader_server_error_details: string` - - `"auto"` + - `other_error: boolean` - - `number` + - `python_grader_runtime_error: boolean` - - `eval_samples: optional "auto" or number` + - `python_grader_runtime_error_details: string` - Number of evaluation samples to generate per training step. + - `python_grader_server_error: boolean` - - `"auto"` + - `python_grader_server_error_type: string` - - `"auto"` + - `sample_parse_error: boolean` - - `number` + - `truncated_observation_error: boolean` - - `learning_rate_multiplier: optional "auto" or number` + - `unresponsive_reward_error: boolean` - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. + - `execution_time: number` - - `"auto"` + - `name: string` - - `"auto"` + - `sampled_model_name: string` - - `number` + - `scores: map[unknown]` - - `n_epochs: optional "auto" or number` + - `token_usage: number` - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + - `type: string` - - `"auto"` +- `model_grader_token_usage_per_model: map[unknown]` - - `"auto"` +- `reward: number` - - `number` +- `sub_rewards: map[unknown]` - - `reasoning_effort: optional "default" or "low" or "medium" or "high"` +### Example - Level of reasoning effort. - - - `"default"` - - - `"low"` - - - `"medium"` - - - `"high"` - -### Supervised Hyperparameters - -- `SupervisedHyperparameters object { batch_size, learning_rate_multiplier, n_epochs }` - - The hyperparameters used for the fine-tuning job. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - -### Supervised Method - -- `SupervisedMethod object { hyperparameters }` - - Configuration for the supervised fine-tuning method. - - - `hyperparameters: optional SupervisedHyperparameters` - - The hyperparameters used for the fine-tuning job. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - -# Jobs - -## Create fine-tuning job - -**post** `/fine_tuning/jobs` - -Creates a fine-tuning job which begins the process of creating a new model from a given dataset. - -Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete. - -[Learn more about fine-tuning](/docs/guides/model-optimization) - -### Body Parameters - -- `model: string or "babbage-002" or "davinci-002" or "gpt-3.5-turbo" or "gpt-4o-mini"` - - The name of the model to fine-tune. You can select one of the - [supported models](/docs/guides/fine-tuning#which-models-can-be-fine-tuned). - - - `string` - - - `"babbage-002" or "davinci-002" or "gpt-3.5-turbo" or "gpt-4o-mini"` - - The name of the model to fine-tune. You can select one of the - [supported models](/docs/guides/fine-tuning#which-models-can-be-fine-tuned). - - - `"babbage-002"` - - - `"davinci-002"` - - - `"gpt-3.5-turbo"` - - - `"gpt-4o-mini"` - -- `training_file: string` - - The ID of an uploaded file that contains training data. - - See [upload file](/docs/api-reference/files/create) for how to upload a file. - - Your dataset must be formatted as a JSONL file. Additionally, you must upload your file with the purpose `fine-tune`. - - The contents of the file should differ depending on if the model uses the [chat](/docs/api-reference/fine-tuning/chat-input), [completions](/docs/api-reference/fine-tuning/completions-input) format, or if the fine-tuning method uses the [preference](/docs/api-reference/fine-tuning/preference-input) format. - - See the [fine-tuning guide](/docs/guides/model-optimization) for more details. - -- `hyperparameters: optional object { batch_size, learning_rate_multiplier, n_epochs }` - - The hyperparameters used for the fine-tuning job. - This value is now deprecated in favor of `method`, and should be passed in under the `method` parameter. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters - are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid - overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle - through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - -- `integrations: optional array of object { type, wandb }` - - A list of integrations to enable for your fine-tuning job. - - - `type: "wandb"` - - The type of integration to enable. Currently, only "wandb" (Weights and Biases) is supported. - - - `"wandb"` - - - `wandb: object { project, entity, name, tags }` - - The settings for your integration with Weights and Biases. This payload specifies the project that - metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags - to your run, and set a default entity (team, username, etc) to be associated with your run. - - - `project: string` - - The name of the project that the new run will be created under. - - - `entity: optional string` - - The entity to use for the run. This allows you to set the team or username of the WandB user that you would - like associated with the run. If not set, the default entity for the registered WandB API key is used. - - - `name: optional string` - - A display name to set for the run. If not set, we will use the Job ID as the name. - - - `tags: optional array of string` - - A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some - default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". - -- `metadata: optional Metadata` - - Set of 16 key-value pairs that can be attached to an object. This can be - useful for storing additional information about the object in a structured - format, and querying for objects via API or the dashboard. - - Keys are strings with a maximum length of 64 characters. Values are strings - with a maximum length of 512 characters. - -- `method: optional object { type, dpo, reinforcement, supervised }` - - The method used for fine-tuning. - - - `type: "supervised" or "dpo" or "reinforcement"` - - The type of method. Is either `supervised`, `dpo`, or `reinforcement`. - - - `"supervised"` - - - `"dpo"` - - - `"reinforcement"` - - - `dpo: optional DpoMethod` - - Configuration for the DPO fine-tuning method. - - - `hyperparameters: optional DpoHyperparameters` - - The hyperparameters used for the DPO fine-tuning job. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `beta: optional "auto" or number` - - The beta value for the DPO method. A higher beta value will increase the weight of the penalty between the policy and reference model. - - - `"auto"` +```http +curl https://api.openai.com/v1/fine_tuning/alpha/graders/run \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "grader": { + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check" + }, + "model_sample": "model_sample" + }' +``` - - `"auto"` +#### Response - - `number` +```json +{ + "metadata": { + "errors": { + "formula_parse_error": true, + "invalid_variable_error": true, + "model_grader_parse_error": true, + "model_grader_refusal_error": true, + "model_grader_server_error": true, + "model_grader_server_error_details": "model_grader_server_error_details", + "other_error": true, + "python_grader_runtime_error": true, + "python_grader_runtime_error_details": "python_grader_runtime_error_details", + "python_grader_server_error": true, + "python_grader_server_error_type": "python_grader_server_error_type", + "sample_parse_error": true, + "truncated_observation_error": true, + "unresponsive_reward_error": true + }, + "execution_time": 0, + "name": "name", + "sampled_model_name": "sampled_model_name", + "scores": { + "foo": "bar" + }, + "token_usage": 0, + "type": "type" + }, + "model_grader_token_usage_per_model": { + "foo": "bar" + }, + "reward": 0, + "sub_rewards": { + "foo": "bar" + } +} +``` - - `learning_rate_multiplier: optional "auto" or number` +### Score an audio response - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. +```http +curl -X POST https://api.openai.com/v1/fine_tuning/alpha/graders/run \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "grader": { + "type": "score_model", + "name": "Audio clarity grader", + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Listen to the clip and return a confidence score from 0 to 1 that the speaker said: {{item.target_phrase}}" + }, + { + "type": "input_audio", + "input_audio": { + "data": "{{item.audio_clip_b64}}", + "format": "mp3" + } + } + ] + } + ], + "model": "gpt-audio", + "sampling_params": { + "temperature": 0.2, + "top_p": 1, + "seed": 123 + } + }, + "item": { + "target_phrase": "Please deliver the package on Tuesday", + "audio_clip_b64": "" + }, + "model_sample": "Please deliver the package on Tuesday" + }' +``` - - `"auto"` +### Score an image caption - - `"auto"` +```http +curl -X POST https://api.openai.com/v1/fine_tuning/alpha/graders/run \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "grader": { + "type": "score_model", + "name": "Image caption grader", + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Score how well the provided caption matches the image on a 0-1 scale. Only return the score.\n\nCaption: {{sample.output_text}}" + }, + { + "type": "input_image", + "image_url": "https://example.com/dog-catching-ball.png", + "file_id": null, + "detail": "high" + } + ] + } + ], + "model": "gpt-5-mini", + "sampling_params": { + "temperature": 0.2 + } + }, + "item": { + "expected_caption": "A golden retriever jumps to catch a tennis ball" + }, + "model_sample": "A dog leaps to grab a tennis ball mid-air" + }' +``` - - `number` +### Score text alignment - - `n_epochs: optional "auto" or number` +```http +curl -X POST https://api.openai.com/v1/fine_tuning/alpha/graders/run \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "grader": { + "type": "score_model", + "name": "Example score model grader", + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Score how close the reference answer is to the model answer on a 0-1 scale. Return only the score.\n\nReference answer: {{item.reference_answer}}\n\nModel answer: {{sample.output_text}}" + } + ] + } + ], + "model": "gpt-5-mini", + "sampling_params": { + "temperature": 1, + "top_p": 1, + "seed": 42 + } + }, + "item": { + "reference_answer": "fuzzy wuzzy was a bear" + }, + "model_sample": "fuzzy wuzzy was a bear" + }' +``` - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. +#### Response - - `"auto"` +```json +{ + "reward": 1.0, + "metadata": { + "name": "Example score model grader", + "type": "score_model", + "errors": { + "formula_parse_error": false, + "sample_parse_error": false, + "truncated_observation_error": false, + "unresponsive_reward_error": false, + "invalid_variable_error": false, + "other_error": false, + "python_grader_server_error": false, + "python_grader_server_error_type": null, + "python_grader_runtime_error": false, + "python_grader_runtime_error_details": null, + "model_grader_server_error": false, + "model_grader_refusal_error": false, + "model_grader_parse_error": false, + "model_grader_server_error_details": null + }, + "execution_time": 4.365238428115845, + "scores": {}, + "token_usage": { + "prompt_tokens": 190, + "total_tokens": 324, + "completion_tokens": 134, + "cached_tokens": 0 + }, + "sampled_model_name": "gpt-4o-2024-08-06" + }, + "sub_rewards": {}, + "model_grader_token_usage_per_model": { + "gpt-4o-2024-08-06": { + "prompt_tokens": 190, + "total_tokens": 324, + "completion_tokens": 134, + "cached_tokens": 0 + } + } +} +``` - - `"auto"` +## Validate grader - - `number` +**post** `/fine_tuning/alpha/graders/validate` - - `reinforcement: optional ReinforcementMethod` +Validate a grader. - Configuration for the reinforcement fine-tuning method. +### Body Parameters - - `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` +- `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` The grader used for the fine-tuning job. @@ -1350,33 +1126,257 @@ Response includes details of the enqueued job including job status and the name A formula to calculate the output based on grader results. - - `graders: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` + - `graders: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` + + A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. + + - `StringCheckGrader object { input, name, operation, 2 more }` + + A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. + + - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` + + A TextSimilarityGrader object which grades text based on similarity metrics. + + - `PythonGrader object { name, source, type, image_tag }` + + A PythonGrader object that runs a python script on the input. + + - `ScoreModelGrader object { input, model, name, 3 more }` + + A ScoreModelGrader object that uses a model to assign a score to the input. + + - `LabelModelGrader object { input, labels, model, 3 more }` + + A LabelModelGrader object which uses a model to assign labels to each item + in the evaluation. + + - `input: array of object { content, role, type }` + + - `content: string or ResponseInputText or object { text, type } or 3 more` + + Inputs to the model - can contain template strings. Supports text, output text, input images, and input audio, either as a single item or an array of items. + + - `TextInput = string` + + A text input to the model. + + - `ResponseInputText object { text, type, prompt_cache_breakpoint }` + + A text input to the model. + + - `OutputText object { text, type }` + + A text output from the model. + + - `text: string` + + The text output from the model. + + - `type: "output_text"` + + The type of the output text. Always `output_text`. + + - `"output_text"` + + - `InputImage object { image_url, type, detail }` + + An image input block used within EvalItem content arrays. + + - `image_url: string` + + The URL of the image input. + + - `type: "input_image"` + + The type of the image input. Always `input_image`. + + - `"input_image"` + + - `detail: optional string` + + The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. + + - `ResponseInputAudio object { input_audio, type }` + + An audio input to the model. + + - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` + + A list of inputs, each of which may be either an input text, output text, input + image, or input audio object. + + - `role: "user" or "assistant" or "system" or "developer"` + + The role of the message input. One of `user`, `assistant`, `system`, or + `developer`. + + - `"user"` + + - `"assistant"` + + - `"system"` + + - `"developer"` + + - `type: optional "message"` + + The type of the message input. Always `message`. + + - `"message"` + + - `labels: array of string` + + The labels to assign to each item in the evaluation. + + - `model: string` + + The model to use for the evaluation. Must support structured outputs. + + - `name: string` + + The name of the grader. + + - `passing_labels: array of string` + + The labels that indicate a passing result. Must be a subset of labels. + + - `type: "label_model"` + + The object type, which is always `label_model`. + + - `"label_model"` + + - `name: string` + + The name of the grader. + + - `type: "multi"` + + The object type, which is always `multi`. + + - `"multi"` + +### Returns + +- `grader: optional StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` + + The grader used for the fine-tuning job. + + - `StringCheckGrader object { input, name, operation, 2 more }` + + A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. + + - `input: string` + + The input text. This may include template strings. + + - `name: string` + + The name of the grader. + + - `operation: "eq" or "ne" or "like" or "ilike"` + + The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`. + + - `"eq"` + + - `"ne"` + + - `"like"` + + - `"ilike"` + + - `reference: string` + + The reference text. This may include template strings. + + - `type: "string_check"` + + The object type, which is always `string_check`. + + - `"string_check"` + + - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` + + A TextSimilarityGrader object which grades text based on similarity metrics. + + - `evaluation_metric: "cosine" or "fuzzy_match" or "bleu" or 8 more` + + The evaluation metric to use. One of `cosine`, `fuzzy_match`, `bleu`, + `gleu`, `meteor`, `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, + or `rouge_l`. + + - `"cosine"` + + - `"fuzzy_match"` + + - `"bleu"` + + - `"gleu"` + + - `"meteor"` + + - `"rouge_1"` + + - `"rouge_2"` + + - `"rouge_3"` + + - `"rouge_4"` + + - `"rouge_5"` + + - `"rouge_l"` + + - `input: string` + + The text being graded. + + - `name: string` + + The name of the grader. + + - `reference: string` + + The text being graded against. + + - `type: "text_similarity"` + + The type of grader. + + - `"text_similarity"` + + - `PythonGrader object { name, source, type, image_tag }` + + A PythonGrader object that runs a python script on the input. + + - `name: string` + + The name of the grader. - A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. + - `source: string` - - `StringCheckGrader object { input, name, operation, 2 more }` + The source code of the python script. - A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. + - `type: "python"` - - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` + The object type, which is always `python`. - A TextSimilarityGrader object which grades text based on similarity metrics. + - `"python"` - - `PythonGrader object { name, source, type, image_tag }` + - `image_tag: optional string` - A PythonGrader object that runs a python script on the input. + The image tag to use for the python script. - `ScoreModelGrader object { input, model, name, 3 more }` A ScoreModelGrader object that uses a model to assign a score to the input. - - `LabelModelGrader object { input, labels, model, 3 more }` - - A LabelModelGrader object which uses a model to assign labels to each item - in the evaluation. - - `input: array of object { content, role, type }` + The input messages evaluated by the grader. Supports text, output text, input image, and input audio content blocks, and may include template strings. + - `content: string or ResponseInputText or object { text, type } or 3 more` Inputs to the model - can contain template strings. Supports text, output text, input images, and input audio, either as a single item or an array of items. @@ -1389,6 +1389,26 @@ Response includes details of the enqueued job including job status and the name A text input to the model. + - `text: string` + + The text input to the model. + + - `type: "input_text"` + + The type of the input item. Always `input_text`. + + - `"input_text"` + + - `prompt_cache_breakpoint: optional object { mode }` + + Marks the exact end of a reusable prompt prefix. The breakpoint inherits its TTL from the request's `prompt_cache_options.ttl`; the boundary is not rounded to a token block. + + - `mode: "explicit"` + + The breakpoint mode. Always `explicit`. + + - `"explicit"` + - `OutputText object { text, type }` A text output from the model. @@ -1425,131 +1445,134 @@ Response includes details of the enqueued job including job status and the name An audio input to the model. - - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` - - A list of inputs, each of which may be either an input text, output text, input - image, or input audio object. - - - `role: "user" or "assistant" or "system" or "developer"` + - `input_audio: object { data, format }` - The role of the message input. One of `user`, `assistant`, `system`, or - `developer`. + - `data: string` - - `"user"` + Base64-encoded audio data. - - `"assistant"` + - `format: "mp3" or "wav"` - - `"system"` + The format of the audio data. Currently supported formats are `mp3` and + `wav`. - - `"developer"` + - `"mp3"` - - `type: optional "message"` + - `"wav"` - The type of the message input. Always `message`. + - `type: "input_audio"` - - `"message"` + The type of the input item. Always `input_audio`. - - `labels: array of string` + - `"input_audio"` - The labels to assign to each item in the evaluation. + - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` - - `model: string` + A list of inputs, each of which may be either an input text, output text, input + image, or input audio object. - The model to use for the evaluation. Must support structured outputs. + - `TextInput = string` - - `name: string` + A text input to the model. - The name of the grader. + - `ResponseInputText object { text, type, prompt_cache_breakpoint }` - - `passing_labels: array of string` + A text input to the model. - The labels that indicate a passing result. Must be a subset of labels. + - `OutputText object { text, type }` - - `type: "label_model"` + A text output from the model. - The object type, which is always `label_model`. + - `text: string` - - `"label_model"` + The text output from the model. - - `name: string` + - `type: "output_text"` - The name of the grader. + The type of the output text. Always `output_text`. - - `type: "multi"` + - `"output_text"` - The object type, which is always `multi`. + - `InputImage object { image_url, type, detail }` - - `"multi"` + An image input block used within EvalItem content arrays. - - `hyperparameters: optional ReinforcementHyperparameters` + - `image_url: string` - The hyperparameters used for the reinforcement fine-tuning job. + The URL of the image input. - - `batch_size: optional "auto" or number` + - `type: "input_image"` - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. + The type of the image input. Always `input_image`. - - `"auto"` + - `"input_image"` - - `"auto"` + - `detail: optional string` - - `number` + The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. - - `compute_multiplier: optional "auto" or number` + - `ResponseInputAudio object { input_audio, type }` - Multiplier on amount of compute used for exploring search space during training. + An audio input to the model. - - `"auto"` + - `role: "user" or "assistant" or "system" or "developer"` - - `"auto"` + The role of the message input. One of `user`, `assistant`, `system`, or + `developer`. - - `number` + - `"user"` - - `eval_interval: optional "auto" or number` + - `"assistant"` - The number of training steps between evaluation runs. + - `"system"` - - `"auto"` + - `"developer"` - - `"auto"` + - `type: optional "message"` - - `number` + The type of the message input. Always `message`. - - `eval_samples: optional "auto" or number` + - `"message"` - Number of evaluation samples to generate per training step. + - `model: string` - - `"auto"` + The model to use for the evaluation. - - `"auto"` + - `name: string` - - `number` + The name of the grader. - - `learning_rate_multiplier: optional "auto" or number` + - `type: "score_model"` - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. + The object type, which is always `score_model`. - - `"auto"` + - `"score_model"` - - `"auto"` + - `range: optional array of number` - - `number` + The range of the score. Defaults to `[0, 1]`. - - `n_epochs: optional "auto" or number` + - `sampling_params: optional object { max_completions_tokens, reasoning_effort, seed, 2 more }` - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + The sampling parameters for the model. - - `"auto"` + - `max_completions_tokens: optional number` - - `"auto"` + The maximum number of tokens the grader model may generate in its response. - - `number` + - `reasoning_effort: optional ReasoningEffort` - - `reasoning_effort: optional "default" or "low" or "medium" or "high"` + Constrains effort on reasoning for reasoning models. Currently supported + values are `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, and `max`. + Reducing reasoning effort can result in faster responses and fewer tokens + used on reasoning in a response. Not all reasoning models support every + value. See the + [reasoning guide](https://platform.openai.com/docs/guides/reasoning) + for model-specific support. - Level of reasoning effort. + - `"none"` - - `"default"` + - `"minimal"` - `"low"` @@ -1557,307 +1580,284 @@ Response includes details of the enqueued job including job status and the name - `"high"` - - `supervised: optional SupervisedMethod` - - Configuration for the supervised fine-tuning method. - - - `hyperparameters: optional SupervisedHyperparameters` - - The hyperparameters used for the fine-tuning job. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - -- `seed: optional number` - - The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. - If a seed is not specified, one will be generated for you. - -- `suffix: optional string` - - A string of up to 64 characters that will be added to your fine-tuned model name. - - For example, a `suffix` of "custom-model-name" would produce a model name like `ft:gpt-4o-mini:openai:custom-model-name:7p4lURel`. - -- `validation_file: optional string` - - The ID of an uploaded file that contains validation data. - - If you provide this file, the data is used to generate validation - metrics periodically during fine-tuning. These metrics can be viewed in - the fine-tuning results file. - The same data should not be present in both train and validation files. - - Your dataset must be formatted as a JSONL file. You must upload your file with the purpose `fine-tune`. - - See the [fine-tuning guide](/docs/guides/model-optimization) for more details. - -### Returns + - `"xhigh"` -- `FineTuningJob object { id, created_at, error, 16 more }` + - `"max"` - The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. + - `seed: optional number` - - `id: string` + A seed value to initialize the randomness, during sampling. - The object identifier, which can be referenced in the API endpoints. + - `temperature: optional number` - - `created_at: number` + A higher temperature increases randomness in the outputs. - The Unix timestamp (in seconds) for when the fine-tuning job was created. + - `top_p: optional number` - - `error: object { code, message, param }` + An alternative to temperature for nucleus sampling; 1.0 includes all tokens. - For fine-tuning jobs that have `failed`, this will contain more information on the cause of the failure. + - `MultiGrader object { calculate_output, graders, name, type }` - - `code: string` + A MultiGrader object combines the output of multiple graders to produce a single score. - A machine-readable error code. + - `calculate_output: string` - - `message: string` + A formula to calculate the output based on grader results. - A human-readable error message. + - `graders: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` - - `param: string` + A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. - The parameter that was invalid, usually `training_file` or `validation_file`. This field will be null if the failure was not parameter-specific. + - `StringCheckGrader object { input, name, operation, 2 more }` - - `fine_tuned_model: string` + A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. - The name of the fine-tuned model that is being created. The value will be null if the fine-tuning job is still running. + - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` - - `finished_at: number` + A TextSimilarityGrader object which grades text based on similarity metrics. - The Unix timestamp (in seconds) for when the fine-tuning job was finished. The value will be null if the fine-tuning job is still running. + - `PythonGrader object { name, source, type, image_tag }` - - `hyperparameters: object { batch_size, learning_rate_multiplier, n_epochs }` + A PythonGrader object that runs a python script on the input. - The hyperparameters used for the fine-tuning job. This value will only be returned when running `supervised` jobs. + - `ScoreModelGrader object { input, model, name, 3 more }` - - `batch_size: optional "auto" or number` + A ScoreModelGrader object that uses a model to assign a score to the input. - Number of examples in each batch. A larger batch size means that model parameters - are updated less frequently, but with lower variance. + - `LabelModelGrader object { input, labels, model, 3 more }` - - `"auto"` + A LabelModelGrader object which uses a model to assign labels to each item + in the evaluation. - - `"auto"` + - `input: array of object { content, role, type }` - - `number` + - `content: string or ResponseInputText or object { text, type } or 3 more` - - `learning_rate_multiplier: optional "auto" or number` + Inputs to the model - can contain template strings. Supports text, output text, input images, and input audio, either as a single item or an array of items. - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid - overfitting. + - `TextInput = string` - - `"auto"` + A text input to the model. - - `"auto"` + - `ResponseInputText object { text, type, prompt_cache_breakpoint }` - - `number` + A text input to the model. - - `n_epochs: optional "auto" or number` + - `OutputText object { text, type }` - The number of epochs to train the model for. An epoch refers to one full cycle - through the training dataset. + A text output from the model. - - `"auto"` + - `text: string` - - `"auto"` + The text output from the model. - - `number` + - `type: "output_text"` - - `model: string` + The type of the output text. Always `output_text`. - The base model that is being fine-tuned. + - `"output_text"` - - `object: "fine_tuning.job"` + - `InputImage object { image_url, type, detail }` - The object type, which is always "fine_tuning.job". + An image input block used within EvalItem content arrays. - - `"fine_tuning.job"` + - `image_url: string` - - `organization_id: string` + The URL of the image input. - The organization that owns the fine-tuning job. + - `type: "input_image"` - - `result_files: array of string` + The type of the image input. Always `input_image`. - The compiled results file ID(s) for the fine-tuning job. You can retrieve the results with the [Files API](/docs/api-reference/files/retrieve-contents). + - `"input_image"` - - `seed: number` + - `detail: optional string` - The seed used for the fine-tuning job. + The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. - - `status: "validating_files" or "queued" or "running" or 3 more` + - `ResponseInputAudio object { input_audio, type }` - The current status of the fine-tuning job, which can be either `validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`. + An audio input to the model. - - `"validating_files"` + - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` - - `"queued"` + A list of inputs, each of which may be either an input text, output text, input + image, or input audio object. - - `"running"` + - `role: "user" or "assistant" or "system" or "developer"` - - `"succeeded"` + The role of the message input. One of `user`, `assistant`, `system`, or + `developer`. - - `"failed"` + - `"user"` - - `"cancelled"` + - `"assistant"` - - `trained_tokens: number` + - `"system"` - The total number of billable tokens processed by this fine-tuning job. The value will be null if the fine-tuning job is still running. + - `"developer"` - - `training_file: string` + - `type: optional "message"` - The file ID used for training. You can retrieve the training data with the [Files API](/docs/api-reference/files/retrieve-contents). + The type of the message input. Always `message`. - - `validation_file: string` + - `"message"` - The file ID used for validation. You can retrieve the validation results with the [Files API](/docs/api-reference/files/retrieve-contents). + - `labels: array of string` - - `estimated_finish: optional number` + The labels to assign to each item in the evaluation. - The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. + - `model: string` - - `integrations: optional array of FineTuningJobWandbIntegrationObject` + The model to use for the evaluation. Must support structured outputs. - A list of integrations to enable for this fine-tuning job. + - `name: string` - - `type: "wandb"` + The name of the grader. - The type of the integration being enabled for the fine-tuning job + - `passing_labels: array of string` - - `"wandb"` + The labels that indicate a passing result. Must be a subset of labels. - - `wandb: FineTuningJobWandbIntegration` + - `type: "label_model"` - The settings for your integration with Weights and Biases. This payload specifies the project that - metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags - to your run, and set a default entity (team, username, etc) to be associated with your run. + The object type, which is always `label_model`. - - `project: string` + - `"label_model"` - The name of the project that the new run will be created under. + - `name: string` - - `entity: optional string` + The name of the grader. - The entity to use for the run. This allows you to set the team or username of the WandB user that you would - like associated with the run. If not set, the default entity for the registered WandB API key is used. + - `type: "multi"` - - `name: optional string` + The object type, which is always `multi`. - A display name to set for the run. If not set, we will use the Job ID as the name. + - `"multi"` - - `tags: optional array of string` +### Example - A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some - default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". +```http +curl https://api.openai.com/v1/fine_tuning/alpha/graders/validate \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "grader": { + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check" + } + }' +``` - - `metadata: optional Metadata` +#### Response - Set of 16 key-value pairs that can be attached to an object. This can be - useful for storing additional information about the object in a structured - format, and querying for objects via API or the dashboard. +```json +{ + "grader": { + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check" + } +} +``` - Keys are strings with a maximum length of 64 characters. Values are strings - with a maximum length of 512 characters. +### Example - - `method: optional object { type, dpo, reinforcement, supervised }` +```http +curl https://api.openai.com/v1/fine_tuning/alpha/graders/validate \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "grader": { + "type": "string_check", + "name": "Example string check grader", + "input": "{{sample.output_text}}", + "reference": "{{item.label}}", + "operation": "eq" + } + }' +``` - The method used for fine-tuning. +#### Response - - `type: "supervised" or "dpo" or "reinforcement"` +```json +{ + "grader": { + "type": "string_check", + "name": "Example string check grader", + "input": "{{sample.output_text}}", + "reference": "{{item.label}}", + "operation": "eq" + } +} +``` - The type of method. Is either `supervised`, `dpo`, or `reinforcement`. +## Domain Types - - `"supervised"` +### Grader Run Response - - `"dpo"` +- `GraderRunResponse object { metadata, model_grader_token_usage_per_model, reward, sub_rewards }` - - `"reinforcement"` + - `metadata: object { errors, execution_time, name, 4 more }` - - `dpo: optional DpoMethod` + - `errors: object { formula_parse_error, invalid_variable_error, model_grader_parse_error, 11 more }` - Configuration for the DPO fine-tuning method. + - `formula_parse_error: boolean` - - `hyperparameters: optional DpoHyperparameters` + - `invalid_variable_error: boolean` - The hyperparameters used for the DPO fine-tuning job. + - `model_grader_parse_error: boolean` - - `batch_size: optional "auto" or number` + - `model_grader_refusal_error: boolean` - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. + - `model_grader_server_error: boolean` - - `"auto"` + - `model_grader_server_error_details: string` - - `"auto"` + - `other_error: boolean` - - `number` + - `python_grader_runtime_error: boolean` - - `beta: optional "auto" or number` + - `python_grader_runtime_error_details: string` - The beta value for the DPO method. A higher beta value will increase the weight of the penalty between the policy and reference model. + - `python_grader_server_error: boolean` - - `"auto"` + - `python_grader_server_error_type: string` - - `"auto"` + - `sample_parse_error: boolean` - - `number` + - `truncated_observation_error: boolean` - - `learning_rate_multiplier: optional "auto" or number` + - `unresponsive_reward_error: boolean` - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. + - `execution_time: number` - - `"auto"` + - `name: string` - - `"auto"` + - `sampled_model_name: string` - - `number` + - `scores: map[unknown]` - - `n_epochs: optional "auto" or number` + - `token_usage: number` - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + - `type: string` - - `"auto"` + - `model_grader_token_usage_per_model: map[unknown]` - - `"auto"` + - `reward: number` - - `number` + - `sub_rewards: map[unknown]` - - `reinforcement: optional ReinforcementMethod` +### Grader Validate Response - Configuration for the reinforcement fine-tuning method. +- `GraderValidateResponse object { grader }` - - `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` + - `grader: optional StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` The grader used for the fine-tuning job. @@ -2333,593 +2333,551 @@ Response includes details of the enqueued job including job status and the name - `"multi"` - - `hyperparameters: optional ReinforcementHyperparameters` +# Checkpoints - The hyperparameters used for the reinforcement fine-tuning job. +# Permissions - - `batch_size: optional "auto" or number` +## Create checkpoint permissions - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. +**post** `/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions` - - `"auto"` +**NOTE:** Calling this endpoint requires an [admin API key](../admin-api-keys). - - `"auto"` +This enables organization owners to share fine-tuned models with other projects in their organization. - - `number` +### Path Parameters - - `compute_multiplier: optional "auto" or number` +- `fine_tuned_model_checkpoint: string` - Multiplier on amount of compute used for exploring search space during training. +### Body Parameters - - `"auto"` +- `project_ids: array of string` - - `"auto"` + The project identifiers to grant access to. - - `number` +### Returns - - `eval_interval: optional "auto" or number` +- `data: array of object { id, created_at, object, project_id }` - The number of training steps between evaluation runs. + - `id: string` - - `"auto"` + The permission identifier, which can be referenced in the API endpoints. - - `"auto"` + - `created_at: number` - - `number` + The Unix timestamp (in seconds) for when the permission was created. - - `eval_samples: optional "auto" or number` + - `object: "checkpoint.permission"` - Number of evaluation samples to generate per training step. + The object type, which is always "checkpoint.permission". - - `"auto"` + - `"checkpoint.permission"` - - `"auto"` + - `project_id: string` - - `number` + The project identifier that the permission is for. - - `learning_rate_multiplier: optional "auto" or number` +- `has_more: boolean` - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. +- `object: "list"` - - `"auto"` + - `"list"` - - `"auto"` +- `first_id: optional string` - - `number` +- `last_id: optional string` - - `n_epochs: optional "auto" or number` +### Example - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. +```http +curl https://api.openai.com/v1/fine_tuning/checkpoints/$FINE_TUNED_MODEL_CHECKPOINT/permissions \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "project_ids": [ + "string" + ] + }' +``` - - `"auto"` +#### Response - - `"auto"` +```json +{ + "data": [ + { + "id": "id", + "created_at": 0, + "object": "checkpoint.permission", + "project_id": "project_id" + } + ], + "has_more": true, + "object": "list", + "first_id": "first_id", + "last_id": "last_id" +} +``` - - `number` +### Example - - `reasoning_effort: optional "default" or "low" or "medium" or "high"` +```http +curl https://api.openai.com/v1/fine_tuning/checkpoints/ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd/permissions \ + -H "Authorization: Bearer $OPENAI_API_KEY" + -d '{"project_ids": ["proj_abGMw1llN8IrBb6SvvY5A1iH"]}' +``` - Level of reasoning effort. +#### Response - - `"default"` +```json +{ + "object": "list", + "data": [ + { + "object": "checkpoint.permission", + "id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", + "created_at": 1721764867, + "project_id": "proj_abGMw1llN8IrBb6SvvY5A1iH" + } + ], + "first_id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", + "last_id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", + "has_more": false +} +``` - - `"low"` +## Delete checkpoint permission - - `"medium"` +**delete** `/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions/{permission_id}` - - `"high"` +**NOTE:** This endpoint requires an [admin API key](../admin-api-keys). - - `supervised: optional SupervisedMethod` +Organization owners can use this endpoint to delete a permission for a fine-tuned model checkpoint. - Configuration for the supervised fine-tuning method. +### Path Parameters - - `hyperparameters: optional SupervisedHyperparameters` +- `fine_tuned_model_checkpoint: string` - The hyperparameters used for the fine-tuning job. +- `permission_id: string` - - `batch_size: optional "auto" or number` +### Returns - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. +- `id: string` - - `"auto"` + The ID of the fine-tuned model checkpoint permission that was deleted. - - `"auto"` +- `deleted: boolean` - - `number` + Whether the fine-tuned model checkpoint permission was successfully deleted. - - `learning_rate_multiplier: optional "auto" or number` +- `object: "checkpoint.permission"` - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. + The object type, which is always "checkpoint.permission". - - `"auto"` + - `"checkpoint.permission"` - - `"auto"` +### Example - - `number` +```http +curl https://api.openai.com/v1/fine_tuning/checkpoints/$FINE_TUNED_MODEL_CHECKPOINT/permissions/$PERMISSION_ID \ + -X DELETE \ + -H "Authorization: Bearer $OPENAI_API_KEY" +``` - - `n_epochs: optional "auto" or number` +#### Response - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. +```json +{ + "id": "id", + "deleted": true, + "object": "checkpoint.permission" +} +``` + +### Example + +```http +curl https://api.openai.com/v1/fine_tuning/checkpoints/ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd/permissions/cp_zc4Q7MP6XxulcVzj4MZdwsAB \ + -H "Authorization: Bearer $OPENAI_API_KEY" +``` + +#### Response + +```json +{ + "object": "checkpoint.permission", + "id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", + "deleted": true +} +``` + +## List checkpoint permissions + +**get** `/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions` + +**NOTE:** This endpoint requires an [admin API key](../admin-api-keys). + +Organization owners can use this endpoint to view all permissions for a fine-tuned model checkpoint. + +### Path Parameters + +- `fine_tuned_model_checkpoint: string` + +### Query Parameters + +- `after: optional string` + + Identifier for the last permission ID from the previous pagination request. + +- `limit: optional number` + + Number of permissions to retrieve. + +- `order: optional "ascending" or "descending"` + + The order in which to retrieve permissions. + + - `"ascending"` + + - `"descending"` + +- `project_id: optional string` + + The ID of the project to get permissions for. + +### Returns + +- `data: array of object { id, created_at, object, project_id }` + + - `id: string` + + The permission identifier, which can be referenced in the API endpoints. + + - `created_at: number` + + The Unix timestamp (in seconds) for when the permission was created. + + - `object: "checkpoint.permission"` + + The object type, which is always "checkpoint.permission". + + - `"checkpoint.permission"` + + - `project_id: string` + + The project identifier that the permission is for. + +- `has_more: boolean` + +- `object: "list"` - - `"auto"` + - `"list"` - - `"auto"` +- `first_id: optional string` - - `number` +- `last_id: optional string` ### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs \ - -H 'Content-Type: application/json' \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "model": "gpt-4o-mini", - "training_file": "file-abc123", - "seed": 42, - "validation_file": "file-abc123" - }' +curl https://api.openai.com/v1/fine_tuning/checkpoints/$FINE_TUNED_MODEL_CHECKPOINT/permissions \ + -H "Authorization: Bearer $OPENAI_API_KEY" ``` #### Response ```json { + "data": [ + { "id": "id", "created_at": 0, - "error": { - "code": "code", - "message": "message", - "param": "param" - }, - "fine_tuned_model": "fine_tuned_model", - "finished_at": 0, - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto" - }, - "model": "model", - "object": "fine_tuning.job", - "organization_id": "organization_id", - "result_files": [ - "file-abc123" - ], - "seed": 0, - "status": "validating_files", - "trained_tokens": 0, - "training_file": "training_file", - "validation_file": "validation_file", - "estimated_finish": 0, - "integrations": [ - { - "type": "wandb", - "wandb": { - "project": "my-wandb-project", - "entity": "entity", - "name": "name", - "tags": [ - "custom-tag" - ] - } + "object": "checkpoint.permission", + "project_id": "project_id" } ], - "metadata": { - "foo": "string" - }, - "method": { - "type": "supervised", - "dpo": { - "hyperparameters": { - "batch_size": "auto", - "beta": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto" - } - }, - "reinforcement": { - "grader": { - "input": "input", - "name": "name", - "operation": "eq", - "reference": "reference", - "type": "string_check" - }, - "hyperparameters": { - "batch_size": "auto", - "compute_multiplier": "auto", - "eval_interval": "auto", - "eval_samples": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto", - "reasoning_effort": "default" - } - }, - "supervised": { - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto" - } - } - } + "has_more": true, + "object": "list", + "first_id": "first_id", + "last_id": "last_id" } ``` ### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "training_file": "file-BK7bzQj3FfZFXr7DbL6xJwfo", - "model": "gpt-4o-mini" - }' +curl https://api.openai.com/v1/fine_tuning/checkpoints/ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd/permissions \ + -H "Authorization: Bearer $OPENAI_API_KEY" ``` #### Response ```json { - "object": "fine_tuning.job", - "id": "ftjob-abc123", - "model": "gpt-4o-mini-2024-07-18", + "object": "list", + "data": [ + { + "object": "checkpoint.permission", + "id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", + "created_at": 1721764867, + "project_id": "proj_abGMw1llN8IrBb6SvvY5A1iH" + }, + { + "object": "checkpoint.permission", + "id": "cp_enQCFmOTGj3syEpYVhBRLTSy", "created_at": 1721764800, - "fine_tuned_model": null, - "organization_id": "org-123", - "result_files": [], - "status": "queued", - "validation_file": null, - "training_file": "file-abc123", - "method": { - "type": "supervised", - "supervised": { - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto", - } - } + "project_id": "proj_iqGMw1llN8IrBb6SvvY5A1oF" }, - "metadata": null + ], + "first_id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", + "last_id": "cp_enQCFmOTGj3syEpYVhBRLTSy", + "has_more": false } ``` -### Epochs +## List checkpoint permissions -```http -curl https://api.openai.com/v1/fine_tuning/jobs \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "training_file": "file-abc123", - "model": "gpt-4o-mini", - "method": { - "type": "supervised", - "supervised": { - "hyperparameters": { - "n_epochs": 2 - } - } - } - }' -``` +**get** `/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions` -#### Response +**NOTE:** This endpoint requires an [admin API key](../admin-api-keys). -```json -{ - "object": "fine_tuning.job", - "id": "ftjob-abc123", - "model": "gpt-4o-mini", - "created_at": 1721764800, - "fine_tuned_model": null, - "organization_id": "org-123", - "result_files": [], - "status": "queued", - "validation_file": null, - "training_file": "file-abc123", - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": 2 - }, - "method": { - "type": "supervised", - "supervised": { - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": 2 - } - } - }, - "metadata": null, - "error": { - "code": null, - "message": null, - "param": null - }, - "finished_at": null, - "seed": 683058546, - "trained_tokens": null, - "estimated_finish": null, - "integrations": [], - "user_provided_suffix": null, - "usage_metrics": null, - "shared_with_openai": false -} -``` +Organization owners can use this endpoint to view all permissions for a fine-tuned model checkpoint. + +### Path Parameters + +- `fine_tuned_model_checkpoint: string` + +### Query Parameters + +- `after: optional string` + + Identifier for the last permission ID from the previous pagination request. + +- `limit: optional number` + + Number of permissions to retrieve. + +- `order: optional "ascending" or "descending"` + + The order in which to retrieve permissions. + + - `"ascending"` + + - `"descending"` + +- `project_id: optional string` + + The ID of the project to get permissions for. + +### Returns + +- `data: array of object { id, created_at, object, project_id }` + + - `id: string` + + The permission identifier, which can be referenced in the API endpoints. + + - `created_at: number` + + The Unix timestamp (in seconds) for when the permission was created. + + - `object: "checkpoint.permission"` + + The object type, which is always "checkpoint.permission". + + - `"checkpoint.permission"` + + - `project_id: string` + + The project identifier that the permission is for. -### DPO +- `has_more: boolean` -```http -curl https://api.openai.com/v1/fine_tuning/jobs \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "training_file": "file-abc123", - "validation_file": "file-abc123", - "model": "gpt-4o-mini", - "method": { - "type": "dpo", - "dpo": { - "hyperparameters": { - "beta": 0.1 - } - } - } - }' -``` +- `object: "list"` -#### Response + - `"list"` -```json -{ - "object": "fine_tuning.job", - "id": "ftjob-abc", - "model": "gpt-4o-mini", - "created_at": 1746130590, - "fine_tuned_model": null, - "organization_id": "org-abc", - "result_files": [], - "status": "queued", - "validation_file": "file-123", - "training_file": "file-abc", - "method": { - "type": "dpo", - "dpo": { - "hyperparameters": { - "beta": 0.1, - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto" - } - } - }, - "metadata": null, - "error": { - "code": null, - "message": null, - "param": null - }, - "finished_at": null, - "hyperparameters": null, - "seed": 1036326793, - "estimated_finish": null, - "integrations": [], - "user_provided_suffix": null, - "usage_metrics": null, - "shared_with_openai": false -} -``` +- `first_id: optional string` -### Reinforcement +- `last_id: optional string` + +### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "training_file": "file-abc", - "validation_file": "file-123", - "model": "o4-mini", - "method": { - "type": "reinforcement", - "reinforcement": { - "grader": { - "type": "string_check", - "name": "Example string check grader", - "input": "{{sample.output_text}}", - "reference": "{{item.label}}", - "operation": "eq" - }, - "hyperparameters": { - "reasoning_effort": "medium" - } - } - } - }' +curl https://api.openai.com/v1/fine_tuning/checkpoints/$FINE_TUNED_MODEL_CHECKPOINT/permissions \ + -H "Authorization: Bearer $OPENAI_API_KEY" ``` #### Response ```json { - "object": "fine_tuning.job", - "id": "ftjob-abc123", - "model": "o4-mini", - "created_at": 1721764800, - "finished_at": null, - "fine_tuned_model": null, - "organization_id": "org-123", - "result_files": [], - "status": "validating_files", - "validation_file": "file-123", - "training_file": "file-abc", - "trained_tokens": null, - "error": {}, - "user_provided_suffix": null, - "seed": 950189191, - "estimated_finish": null, - "integrations": [], - "method": { - "type": "reinforcement", - "reinforcement": { - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto", - "eval_interval": "auto", - "eval_samples": "auto", - "compute_multiplier": "auto", - "reasoning_effort": "medium" - }, - "grader": { - "type": "string_check", - "name": "Example string check grader", - "input": "{{sample.output_text}}", - "reference": "{{item.label}}", - "operation": "eq" - }, - "response_format": null + "data": [ + { + "id": "id", + "created_at": 0, + "object": "checkpoint.permission", + "project_id": "project_id" } - }, - "metadata": null, - "usage_metrics": null, - "shared_with_openai": false + ], + "has_more": true, + "object": "list", + "first_id": "first_id", + "last_id": "last_id" } - ``` -### Validation file +### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "training_file": "file-abc123", - "validation_file": "file-abc123", - "model": "gpt-4o-mini" - }' +curl https://api.openai.com/v1/fine_tuning/checkpoints/ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd/permissions \ + -H "Authorization: Bearer $OPENAI_API_KEY" ``` #### Response ```json { - "object": "fine_tuning.job", - "id": "ftjob-abc123", - "model": "gpt-4o-mini-2024-07-18", + "object": "list", + "data": [ + { + "object": "checkpoint.permission", + "id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", + "created_at": 1721764867, + "project_id": "proj_abGMw1llN8IrBb6SvvY5A1iH" + }, + { + "object": "checkpoint.permission", + "id": "cp_enQCFmOTGj3syEpYVhBRLTSy", "created_at": 1721764800, - "fine_tuned_model": null, - "organization_id": "org-123", - "result_files": [], - "status": "queued", - "validation_file": "file-abc123", - "training_file": "file-abc123", - "method": { - "type": "supervised", - "supervised": { - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto", - } - } + "project_id": "proj_iqGMw1llN8IrBb6SvvY5A1oF" }, - "metadata": null + ], + "first_id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", + "last_id": "cp_enQCFmOTGj3syEpYVhBRLTSy", + "has_more": false } ``` -### W&B Integration +## Domain Types + +### Permission Create Response + +- `PermissionCreateResponse object { id, created_at, object, project_id }` + + The `checkpoint.permission` object represents a permission for a fine-tuned model checkpoint. + + - `id: string` + + The permission identifier, which can be referenced in the API endpoints. + + - `created_at: number` + + The Unix timestamp (in seconds) for when the permission was created. + + - `object: "checkpoint.permission"` + + The object type, which is always "checkpoint.permission". + + - `"checkpoint.permission"` + + - `project_id: string` + + The project identifier that the permission is for. + +### Permission Delete Response + +- `PermissionDeleteResponse object { id, deleted, object }` + + - `id: string` + + The ID of the fine-tuned model checkpoint permission that was deleted. + + - `deleted: boolean` + + Whether the fine-tuned model checkpoint permission was successfully deleted. + + - `object: "checkpoint.permission"` + + The object type, which is always "checkpoint.permission". + + - `"checkpoint.permission"` + +### Permission List Response + +- `PermissionListResponse object { id, created_at, object, project_id }` + + The `checkpoint.permission` object represents a permission for a fine-tuned model checkpoint. + + - `id: string` + + The permission identifier, which can be referenced in the API endpoints. + + - `created_at: number` + + The Unix timestamp (in seconds) for when the permission was created. + + - `object: "checkpoint.permission"` + + The object type, which is always "checkpoint.permission". + + - `"checkpoint.permission"` + + - `project_id: string` -```http -curl https://api.openai.com/v1/fine_tuning/jobs \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "training_file": "file-abc123", - "validation_file": "file-abc123", - "model": "gpt-4o-mini", - "integrations": [ - { - "type": "wandb", - "wandb": { - "project": "my-wandb-project", - "name": "ft-run-display-name" - "tags": [ - "first-experiment", "v2" - ] - } - } - ] - }' -``` + The project identifier that the permission is for. -#### Response +### Permission Retrieve Response -```json -{ - "object": "fine_tuning.job", - "id": "ftjob-abc123", - "model": "gpt-4o-mini-2024-07-18", - "created_at": 1721764800, - "fine_tuned_model": null, - "organization_id": "org-123", - "result_files": [], - "status": "queued", - "validation_file": "file-abc123", - "training_file": "file-abc123", - "integrations": [ - { - "type": "wandb", - "wandb": { - "project": "my-wandb-project", - "entity": None, - "run_id": "ftjob-abc123" - } - } - ], - "method": { - "type": "supervised", - "supervised": { - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto", - } - } - }, - "metadata": null -} -``` +- `PermissionRetrieveResponse object { data, has_more, object, 2 more }` -## List fine-tuning jobs + - `data: array of object { id, created_at, object, project_id }` -**get** `/fine_tuning/jobs` + - `id: string` -List your organization's fine-tuning jobs + The permission identifier, which can be referenced in the API endpoints. -### Query Parameters + - `created_at: number` -- `after: optional string` + The Unix timestamp (in seconds) for when the permission was created. - Identifier for the last job from the previous pagination request. + - `object: "checkpoint.permission"` -- `limit: optional number` + The object type, which is always "checkpoint.permission". - Number of fine-tuning jobs to retrieve. + - `"checkpoint.permission"` -- `metadata: optional map[string]` + - `project_id: string` - Optional metadata filter. To filter, use the syntax `metadata[k]=v`. Alternatively, set `metadata=null` to indicate no metadata. + The project identifier that the permission is for. + + - `has_more: boolean` + + - `object: "list"` + + - `"list"` + + - `first_id: optional string` + + - `last_id: optional string` + +# Jobs + +## Cancel fine-tuning + +**post** `/fine_tuning/jobs/{fine_tuning_job_id}/cancel` + +Immediately cancel a fine-tune job. + +### Path Parameters + +- `fine_tuning_job_id: string` ### Returns -- `data: array of FineTuningJob` +- `FineTuningJob object { id, created_at, error, 16 more }` + + The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. - `id: string` @@ -3743,16 +3701,11 @@ List your organization's fine-tuning jobs - `number` -- `has_more: boolean` - -- `object: "list"` - - - `"list"` - ### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs \ +curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/cancel \ + -X POST \ -H "Authorization: Bearer $OPENAI_API_KEY" ``` @@ -3760,8 +3713,6 @@ curl https://api.openai.com/v1/fine_tuning/jobs \ ```json { - "data": [ - { "id": "id", "created_at": 0, "error": { @@ -3840,17 +3791,13 @@ curl https://api.openai.com/v1/fine_tuning/jobs \ } } } - } - ], - "has_more": true, - "object": "list" } ``` ### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs?limit=2&metadata[key]=value \ +curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/cancel \ -H "Authorization: Bearer $OPENAI_API_KEY" ``` @@ -3858,9 +3805,6 @@ curl https://api.openai.com/v1/fine_tuning/jobs?limit=2&metadata[key]=value \ ```json { - "object": "list", - "data": [ - { "object": "fine_tuning.job", "id": "ftjob-abc123", "model": "gpt-4o-mini-2024-07-18", @@ -3868,171 +3812,105 @@ curl https://api.openai.com/v1/fine_tuning/jobs?limit=2&metadata[key]=value \ "fine_tuned_model": null, "organization_id": "org-123", "result_files": [], - "status": "queued", - "validation_file": null, - "training_file": "file-abc123", - "metadata": { - "key": "value" - } - }, - { ... }, - { ... } - ], "has_more": true + "status": "cancelled", + "validation_file": "file-abc123", + "training_file": "file-abc123" } ``` -## Retrieve fine-tuning job - -**get** `/fine_tuning/jobs/{fine_tuning_job_id}` - -Get info about a fine-tuning job. - -[Learn more about fine-tuning](/docs/guides/model-optimization) - -### Path Parameters +## Create fine-tuning job -- `fine_tuning_job_id: string` +**post** `/fine_tuning/jobs` -### Returns +Creates a fine-tuning job which begins the process of creating a new model from a given dataset. -- `FineTuningJob object { id, created_at, error, 16 more }` +Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete. - The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. +[Learn more about fine-tuning](/docs/guides/model-optimization) - - `id: string` +### Body Parameters - The object identifier, which can be referenced in the API endpoints. +- `model: string or "babbage-002" or "davinci-002" or "gpt-3.5-turbo" or "gpt-4o-mini"` - - `created_at: number` + The name of the model to fine-tune. You can select one of the + [supported models](/docs/guides/fine-tuning#which-models-can-be-fine-tuned). - The Unix timestamp (in seconds) for when the fine-tuning job was created. + - `string` - - `error: object { code, message, param }` + - `"babbage-002" or "davinci-002" or "gpt-3.5-turbo" or "gpt-4o-mini"` - For fine-tuning jobs that have `failed`, this will contain more information on the cause of the failure. + The name of the model to fine-tune. You can select one of the + [supported models](/docs/guides/fine-tuning#which-models-can-be-fine-tuned). - - `code: string` + - `"babbage-002"` - A machine-readable error code. + - `"davinci-002"` - - `message: string` + - `"gpt-3.5-turbo"` - A human-readable error message. + - `"gpt-4o-mini"` - - `param: string` +- `training_file: string` - The parameter that was invalid, usually `training_file` or `validation_file`. This field will be null if the failure was not parameter-specific. + The ID of an uploaded file that contains training data. - - `fine_tuned_model: string` + See [upload file](/docs/api-reference/files/create) for how to upload a file. - The name of the fine-tuned model that is being created. The value will be null if the fine-tuning job is still running. + Your dataset must be formatted as a JSONL file. Additionally, you must upload your file with the purpose `fine-tune`. - - `finished_at: number` + The contents of the file should differ depending on if the model uses the [chat](/docs/api-reference/fine-tuning/chat-input), [completions](/docs/api-reference/fine-tuning/completions-input) format, or if the fine-tuning method uses the [preference](/docs/api-reference/fine-tuning/preference-input) format. - The Unix timestamp (in seconds) for when the fine-tuning job was finished. The value will be null if the fine-tuning job is still running. + See the [fine-tuning guide](/docs/guides/model-optimization) for more details. - - `hyperparameters: object { batch_size, learning_rate_multiplier, n_epochs }` +- `hyperparameters: optional object { batch_size, learning_rate_multiplier, n_epochs }` - The hyperparameters used for the fine-tuning job. This value will only be returned when running `supervised` jobs. + The hyperparameters used for the fine-tuning job. + This value is now deprecated in favor of `method`, and should be passed in under the `method` parameter. - `batch_size: optional "auto" or number` Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid - overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle - through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - - - `model: string` - - The base model that is being fine-tuned. - - - `object: "fine_tuning.job"` - - The object type, which is always "fine_tuning.job". - - - `"fine_tuning.job"` - - - `organization_id: string` - - The organization that owns the fine-tuning job. - - - `result_files: array of string` - - The compiled results file ID(s) for the fine-tuning job. You can retrieve the results with the [Files API](/docs/api-reference/files/retrieve-contents). - - - `seed: number` - - The seed used for the fine-tuning job. - - - `status: "validating_files" or "queued" or "running" or 3 more` - - The current status of the fine-tuning job, which can be either `validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`. - - - `"validating_files"` - - - `"queued"` + - `"auto"` - - `"running"` + - `"auto"` - - `"succeeded"` + - `number` - - `"failed"` + - `learning_rate_multiplier: optional "auto" or number` - - `"cancelled"` + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid + overfitting. - - `trained_tokens: number` + - `"auto"` - The total number of billable tokens processed by this fine-tuning job. The value will be null if the fine-tuning job is still running. + - `"auto"` - - `training_file: string` + - `number` - The file ID used for training. You can retrieve the training data with the [Files API](/docs/api-reference/files/retrieve-contents). + - `n_epochs: optional "auto" or number` - - `validation_file: string` + The number of epochs to train the model for. An epoch refers to one full cycle + through the training dataset. - The file ID used for validation. You can retrieve the validation results with the [Files API](/docs/api-reference/files/retrieve-contents). + - `"auto"` - - `estimated_finish: optional number` + - `"auto"` - The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. + - `number` - - `integrations: optional array of FineTuningJobWandbIntegrationObject` +- `integrations: optional array of object { type, wandb }` - A list of integrations to enable for this fine-tuning job. + A list of integrations to enable for your fine-tuning job. - `type: "wandb"` - The type of the integration being enabled for the fine-tuning job + The type of integration to enable. Currently, only "wandb" (Weights and Biases) is supported. - `"wandb"` - - `wandb: FineTuningJobWandbIntegration` + - `wandb: object { project, entity, name, tags }` The settings for your integration with Weights and Biases. This payload specifies the project that metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags @@ -4056,7 +3934,7 @@ Get info about a fine-tuning job. A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". - - `metadata: optional Metadata` +- `metadata: optional Metadata` Set of 16 key-value pairs that can be attached to an object. This can be useful for storing additional information about the object in a structured @@ -4065,7 +3943,7 @@ Get info about a fine-tuning job. Keys are strings with a maximum length of 64 characters. Values are strings with a maximum length of 512 characters. - - `method: optional object { type, dpo, reinforcement, supervised }` +- `method: optional object { type, dpo, reinforcement, supervised }` The method used for fine-tuning. @@ -4643,364 +4521,107 @@ Get info about a fine-tuning job. - `eval_samples: optional "auto" or number` - Number of evaluation samples to generate per training step. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - - - `reasoning_effort: optional "default" or "low" or "medium" or "high"` - - Level of reasoning effort. - - - `"default"` - - - `"low"` - - - `"medium"` - - - `"high"` - - - `supervised: optional SupervisedMethod` - - Configuration for the supervised fine-tuning method. - - - `hyperparameters: optional SupervisedHyperparameters` - - The hyperparameters used for the fine-tuning job. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - -### Example - -```http -curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` - -#### Response - -```json -{ - "id": "id", - "created_at": 0, - "error": { - "code": "code", - "message": "message", - "param": "param" - }, - "fine_tuned_model": "fine_tuned_model", - "finished_at": 0, - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto" - }, - "model": "model", - "object": "fine_tuning.job", - "organization_id": "organization_id", - "result_files": [ - "file-abc123" - ], - "seed": 0, - "status": "validating_files", - "trained_tokens": 0, - "training_file": "training_file", - "validation_file": "validation_file", - "estimated_finish": 0, - "integrations": [ - { - "type": "wandb", - "wandb": { - "project": "my-wandb-project", - "entity": "entity", - "name": "name", - "tags": [ - "custom-tag" - ] - } - } - ], - "metadata": { - "foo": "string" - }, - "method": { - "type": "supervised", - "dpo": { - "hyperparameters": { - "batch_size": "auto", - "beta": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto" - } - }, - "reinforcement": { - "grader": { - "input": "input", - "name": "name", - "operation": "eq", - "reference": "reference", - "type": "string_check" - }, - "hyperparameters": { - "batch_size": "auto", - "compute_multiplier": "auto", - "eval_interval": "auto", - "eval_samples": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto", - "reasoning_effort": "default" - } - }, - "supervised": { - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto" - } - } - } -} -``` - -### Example - -```http -curl https://api.openai.com/v1/fine_tuning/jobs/ft-AF1WoRqd3aJAHsqc9NY7iL8F \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` - -#### Response - -```json -{ - "object": "fine_tuning.job", - "id": "ftjob-abc123", - "model": "davinci-002", - "created_at": 1692661014, - "finished_at": 1692661190, - "fine_tuned_model": "ft:davinci-002:my-org:custom_suffix:7q8mpxmy", - "organization_id": "org-123", - "result_files": [ - "file-abc123" - ], - "status": "succeeded", - "validation_file": null, - "training_file": "file-abc123", - "hyperparameters": { - "n_epochs": 4, - "batch_size": 1, - "learning_rate_multiplier": 1.0 - }, - "trained_tokens": 5768, - "integrations": [], - "seed": 0, - "estimated_finish": 0, - "method": { - "type": "supervised", - "supervised": { - "hyperparameters": { - "n_epochs": 4, - "batch_size": 1, - "learning_rate_multiplier": 1.0 - } - } - } -} -``` - -## List fine-tuning events + Number of evaluation samples to generate per training step. -**get** `/fine_tuning/jobs/{fine_tuning_job_id}/events` + - `"auto"` -Get status updates for a fine-tuning job. + - `"auto"` -### Path Parameters + - `number` -- `fine_tuning_job_id: string` + - `learning_rate_multiplier: optional "auto" or number` -### Query Parameters + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. -- `after: optional string` + - `"auto"` - Identifier for the last event from the previous pagination request. + - `"auto"` -- `limit: optional number` + - `number` - Number of events to retrieve. + - `n_epochs: optional "auto" or number` -### Returns + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. -- `data: array of FineTuningJobEvent` + - `"auto"` - - `id: string` + - `"auto"` - The object identifier. + - `number` - - `created_at: number` + - `reasoning_effort: optional "default" or "low" or "medium" or "high"` - The Unix timestamp (in seconds) for when the fine-tuning job was created. + Level of reasoning effort. - - `level: "info" or "warn" or "error"` + - `"default"` - The log level of the event. + - `"low"` - - `"info"` + - `"medium"` - - `"warn"` + - `"high"` - - `"error"` + - `supervised: optional SupervisedMethod` - - `message: string` + Configuration for the supervised fine-tuning method. - The message of the event. + - `hyperparameters: optional SupervisedHyperparameters` - - `object: "fine_tuning.job.event"` + The hyperparameters used for the fine-tuning job. - The object type, which is always "fine_tuning.job.event". + - `batch_size: optional "auto" or number` - - `"fine_tuning.job.event"` + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - `data: optional unknown` + - `"auto"` - The data associated with the event. + - `"auto"` - - `type: optional "message" or "metrics"` + - `number` - The type of event. + - `learning_rate_multiplier: optional "auto" or number` - - `"message"` + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - `"metrics"` + - `"auto"` -- `has_more: boolean` + - `"auto"` -- `object: "list"` + - `number` - - `"list"` + - `n_epochs: optional "auto" or number` -### Example + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. -```http -curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/events \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` + - `"auto"` -#### Response + - `"auto"` -```json -{ - "data": [ - { - "id": "id", - "created_at": 0, - "level": "info", - "message": "message", - "object": "fine_tuning.job.event", - "data": {}, - "type": "message" - } - ], - "has_more": true, - "object": "list" -} -``` + - `number` -### Example +- `seed: optional number` -```http -curl https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/events \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` + The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. + If a seed is not specified, one will be generated for you. -#### Response +- `suffix: optional string` -```json -{ - "object": "list", - "data": [ - { - "object": "fine_tuning.job.event", - "id": "ft-event-ddTJfwuMVpfLXseO0Am0Gqjm", - "created_at": 1721764800, - "level": "info", - "message": "Fine tuning job successfully completed", - "data": null, - "type": "message" - }, - { - "object": "fine_tuning.job.event", - "id": "ft-event-tyiGuB72evQncpH87xe505Sv", - "created_at": 1721764800, - "level": "info", - "message": "New fine-tuned model created: ft:gpt-4o-mini:openai::7p4lURel", - "data": null, - "type": "message" - } - ], - "has_more": true -} -``` + A string of up to 64 characters that will be added to your fine-tuned model name. -## Cancel fine-tuning + For example, a `suffix` of "custom-model-name" would produce a model name like `ft:gpt-4o-mini:openai:custom-model-name:7p4lURel`. -**post** `/fine_tuning/jobs/{fine_tuning_job_id}/cancel` +- `validation_file: optional string` -Immediately cancel a fine-tune job. + The ID of an uploaded file that contains validation data. -### Path Parameters + If you provide this file, the data is used to generate validation + metrics periodically during fine-tuning. These metrics can be viewed in + the fine-tuning results file. + The same data should not be present in both train and validation files. -- `fine_tuning_job_id: string` + Your dataset must be formatted as a JSONL file. You must upload your file with the purpose `fine-tune`. + + See the [fine-tuning guide](/docs/guides/model-optimization) for more details. ### Returns @@ -5822,112 +5443,424 @@ Immediately cancel a fine-tune job. - `n_epochs: optional "auto" or number` - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + + - `"auto"` + + - `"auto"` + + - `number` + +### Example + +```http +curl https://api.openai.com/v1/fine_tuning/jobs \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-4o-mini", + "training_file": "file-abc123", + "seed": 42, + "validation_file": "file-abc123" + }' +``` + +#### Response + +```json +{ + "id": "id", + "created_at": 0, + "error": { + "code": "code", + "message": "message", + "param": "param" + }, + "fine_tuned_model": "fine_tuned_model", + "finished_at": 0, + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + }, + "model": "model", + "object": "fine_tuning.job", + "organization_id": "organization_id", + "result_files": [ + "file-abc123" + ], + "seed": 0, + "status": "validating_files", + "trained_tokens": 0, + "training_file": "training_file", + "validation_file": "validation_file", + "estimated_finish": 0, + "integrations": [ + { + "type": "wandb", + "wandb": { + "project": "my-wandb-project", + "entity": "entity", + "name": "name", + "tags": [ + "custom-tag" + ] + } + } + ], + "metadata": { + "foo": "string" + }, + "method": { + "type": "supervised", + "dpo": { + "hyperparameters": { + "batch_size": "auto", + "beta": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + } + }, + "reinforcement": { + "grader": { + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check" + }, + "hyperparameters": { + "batch_size": "auto", + "compute_multiplier": "auto", + "eval_interval": "auto", + "eval_samples": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + "reasoning_effort": "default" + } + }, + "supervised": { + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + } + } + } +} +``` + +### DPO - - `"auto"` +```http +curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-abc123", + "validation_file": "file-abc123", + "model": "gpt-4o-mini", + "method": { + "type": "dpo", + "dpo": { + "hyperparameters": { + "beta": 0.1 + } + } + } + }' +``` - - `"auto"` +#### Response - - `number` +```json +{ + "object": "fine_tuning.job", + "id": "ftjob-abc", + "model": "gpt-4o-mini", + "created_at": 1746130590, + "fine_tuned_model": null, + "organization_id": "org-abc", + "result_files": [], + "status": "queued", + "validation_file": "file-123", + "training_file": "file-abc", + "method": { + "type": "dpo", + "dpo": { + "hyperparameters": { + "beta": 0.1, + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + } + } + }, + "metadata": null, + "error": { + "code": null, + "message": null, + "param": null + }, + "finished_at": null, + "hyperparameters": null, + "seed": 1036326793, + "estimated_finish": null, + "integrations": [], + "user_provided_suffix": null, + "usage_metrics": null, + "shared_with_openai": false +} +``` ### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/cancel \ - -X POST \ - -H "Authorization: Bearer $OPENAI_API_KEY" +curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-BK7bzQj3FfZFXr7DbL6xJwfo", + "model": "gpt-4o-mini" + }' ``` #### Response ```json { - "id": "id", - "created_at": 0, - "error": { - "code": "code", - "message": "message", - "param": "param" - }, - "fine_tuned_model": "fine_tuned_model", - "finished_at": 0, + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-4o-mini-2024-07-18", + "created_at": 1721764800, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "queued", + "validation_file": null, + "training_file": "file-abc123", + "method": { + "type": "supervised", + "supervised": { "hyperparameters": { "batch_size": "auto", "learning_rate_multiplier": "auto", - "n_epochs": "auto" + "n_epochs": "auto", + } + } }, - "model": "model", - "object": "fine_tuning.job", - "organization_id": "organization_id", - "result_files": [ - "file-abc123" - ], - "seed": 0, - "status": "validating_files", - "trained_tokens": 0, - "training_file": "training_file", - "validation_file": "validation_file", - "estimated_finish": 0, - "integrations": [ - { - "type": "wandb", - "wandb": { - "project": "my-wandb-project", - "entity": "entity", - "name": "name", - "tags": [ - "custom-tag" - ] + "metadata": null +} +``` + +### Epochs + +```http +curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-abc123", + "model": "gpt-4o-mini", + "method": { + "type": "supervised", + "supervised": { + "hyperparameters": { + "n_epochs": 2 } } - ], - "metadata": { - "foo": "string" + } + }' +``` + +#### Response + +```json +{ + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-4o-mini", + "created_at": 1721764800, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "queued", + "validation_file": null, + "training_file": "file-abc123", + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": 2 }, "method": { "type": "supervised", - "dpo": { + "supervised": { "hyperparameters": { "batch_size": "auto", - "beta": "auto", "learning_rate_multiplier": "auto", - "n_epochs": "auto" + "n_epochs": 2 + } } }, + "metadata": null, + "error": { + "code": null, + "message": null, + "param": null + }, + "finished_at": null, + "seed": 683058546, + "trained_tokens": null, + "estimated_finish": null, + "integrations": [], + "user_provided_suffix": null, + "usage_metrics": null, + "shared_with_openai": false +} +``` + +### Reinforcement + +```http +curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-abc", + "validation_file": "file-123", + "model": "o4-mini", + "method": { + "type": "reinforcement", + "reinforcement": { + "grader": { + "type": "string_check", + "name": "Example string check grader", + "input": "{{sample.output_text}}", + "reference": "{{item.label}}", + "operation": "eq" + }, + "hyperparameters": { + "reasoning_effort": "medium" + } + } + } + }' +``` + +#### Response + +```json +{ + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "o4-mini", + "created_at": 1721764800, + "finished_at": null, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "validating_files", + "validation_file": "file-123", + "training_file": "file-abc", + "trained_tokens": null, + "error": {}, + "user_provided_suffix": null, + "seed": 950189191, + "estimated_finish": null, + "integrations": [], + "method": { + "type": "reinforcement", "reinforcement": { + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + "eval_interval": "auto", + "eval_samples": "auto", + "compute_multiplier": "auto", + "reasoning_effort": "medium" + }, "grader": { - "input": "input", - "name": "name", - "operation": "eq", - "reference": "reference", - "type": "string_check" + "type": "string_check", + "name": "Example string check grader", + "input": "{{sample.output_text}}", + "reference": "{{item.label}}", + "operation": "eq" }, - "hyperparameters": { - "batch_size": "auto", - "compute_multiplier": "auto", - "eval_interval": "auto", - "eval_samples": "auto", - "learning_rate_multiplier": "auto", - "n_epochs": "auto", - "reasoning_effort": "default" + "response_format": null } }, + "metadata": null, + "usage_metrics": null, + "shared_with_openai": false +} + +``` + +### Validation file + +```http +curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-abc123", + "validation_file": "file-abc123", + "model": "gpt-4o-mini" + }' +``` + +#### Response + +```json +{ + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-4o-mini-2024-07-18", + "created_at": 1721764800, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "queued", + "validation_file": "file-abc123", + "training_file": "file-abc123", + "method": { + "type": "supervised", "supervised": { "hyperparameters": { "batch_size": "auto", "learning_rate_multiplier": "auto", - "n_epochs": "auto" - } + "n_epochs": "auto", } } + }, + "metadata": null } ``` -### Example +### W&B Integration ```http -curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/cancel \ - -H "Authorization: Bearer $OPENAI_API_KEY" +curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-abc123", + "validation_file": "file-abc123", + "model": "gpt-4o-mini", + "integrations": [ + { + "type": "wandb", + "wandb": { + "project": "my-wandb-project", + "name": "ft-run-display-name" + "tags": [ + "first-experiment", "v2" + ] + } + } + ] + }' ``` #### Response @@ -5941,27 +5874,56 @@ curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/cancel \ "fine_tuned_model": null, "organization_id": "org-123", "result_files": [], - "status": "cancelled", + "status": "queued", "validation_file": "file-abc123", - "training_file": "file-abc123" + "training_file": "file-abc123", + "integrations": [ + { + "type": "wandb", + "wandb": { + "project": "my-wandb-project", + "entity": None, + "run_id": "ftjob-abc123" + } + } + ], + "method": { + "type": "supervised", + "supervised": { + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + } + } + }, + "metadata": null } ``` -## Pause fine-tuning +## List fine-tuning jobs -**post** `/fine_tuning/jobs/{fine_tuning_job_id}/pause` +**get** `/fine_tuning/jobs` -Pause a fine-tune job. +List your organization's fine-tuning jobs -### Path Parameters +### Query Parameters -- `fine_tuning_job_id: string` +- `after: optional string` -### Returns + Identifier for the last job from the previous pagination request. -- `FineTuningJob object { id, created_at, error, 16 more }` +- `limit: optional number` - The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. + Number of fine-tuning jobs to retrieve. + +- `metadata: optional map[string]` + + Optional metadata filter. To filter, use the syntax `metadata[k]=v`. Alternatively, set `metadata=null` to indicate no metadata. + +### Returns + +- `data: array of FineTuningJob` - `id: string` @@ -6785,11 +6747,16 @@ Pause a fine-tune job. - `number` +- `has_more: boolean` + +- `object: "list"` + + - `"list"` + ### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/pause \ - -X POST \ +curl https://api.openai.com/v1/fine_tuning/jobs \ -H "Authorization: Bearer $OPENAI_API_KEY" ``` @@ -6797,6 +6764,8 @@ curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/pause \ ```json { + "data": [ + { "id": "id", "created_at": 0, "error": { @@ -6875,13 +6844,148 @@ curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/pause \ } } } + } + ], + "has_more": true, + "object": "list" +} +``` + +### Example + +```http +curl https://api.openai.com/v1/fine_tuning/jobs?limit=2&metadata[key]=value \ + -H "Authorization: Bearer $OPENAI_API_KEY" +``` + +#### Response + +```json +{ + "object": "list", + "data": [ + { + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-4o-mini-2024-07-18", + "created_at": 1721764800, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "queued", + "validation_file": null, + "training_file": "file-abc123", + "metadata": { + "key": "value" + } + }, + { ... }, + { ... } + ], "has_more": true +} +``` + +## List fine-tuning events + +**get** `/fine_tuning/jobs/{fine_tuning_job_id}/events` + +Get status updates for a fine-tuning job. + +### Path Parameters + +- `fine_tuning_job_id: string` + +### Query Parameters + +- `after: optional string` + + Identifier for the last event from the previous pagination request. + +- `limit: optional number` + + Number of events to retrieve. + +### Returns + +- `data: array of FineTuningJobEvent` + + - `id: string` + + The object identifier. + + - `created_at: number` + + The Unix timestamp (in seconds) for when the fine-tuning job was created. + + - `level: "info" or "warn" or "error"` + + The log level of the event. + + - `"info"` + + - `"warn"` + + - `"error"` + + - `message: string` + + The message of the event. + + - `object: "fine_tuning.job.event"` + + The object type, which is always "fine_tuning.job.event". + + - `"fine_tuning.job.event"` + + - `data: optional unknown` + + The data associated with the event. + + - `type: optional "message" or "metrics"` + + The type of event. + + - `"message"` + + - `"metrics"` + +- `has_more: boolean` + +- `object: "list"` + + - `"list"` + +### Example + +```http +curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/events \ + -H "Authorization: Bearer $OPENAI_API_KEY" +``` + +#### Response + +```json +{ + "data": [ + { + "id": "id", + "created_at": 0, + "level": "info", + "message": "message", + "object": "fine_tuning.job.event", + "data": {}, + "type": "message" + } + ], + "has_more": true, + "object": "list" } ``` ### Example ```http -curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/pause \ +curl https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/events \ -H "Authorization: Bearer $OPENAI_API_KEY" ``` @@ -6889,24 +6993,36 @@ curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/pause \ ```json { - "object": "fine_tuning.job", - "id": "ftjob-abc123", - "model": "gpt-4o-mini-2024-07-18", + "object": "list", + "data": [ + { + "object": "fine_tuning.job.event", + "id": "ft-event-ddTJfwuMVpfLXseO0Am0Gqjm", "created_at": 1721764800, - "fine_tuned_model": null, - "organization_id": "org-123", - "result_files": [], - "status": "paused", - "validation_file": "file-abc123", - "training_file": "file-abc123" + "level": "info", + "message": "Fine tuning job successfully completed", + "data": null, + "type": "message" + }, + { + "object": "fine_tuning.job.event", + "id": "ft-event-tyiGuB72evQncpH87xe505Sv", + "created_at": 1721764800, + "level": "info", + "message": "New fine-tuned model created: ft:gpt-4o-mini:openai::7p4lURel", + "data": null, + "type": "message" + } + ], + "has_more": true } ``` -## Resume fine-tuning +## Pause fine-tuning -**post** `/fine_tuning/jobs/{fine_tuning_job_id}/resume` +**post** `/fine_tuning/jobs/{fine_tuning_job_id}/pause` -Resume a fine-tune job. +Pause a fine-tune job. ### Path Parameters @@ -7743,7 +7859,7 @@ Resume a fine-tune job. ### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/resume \ +curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/pause \ -X POST \ -H "Authorization: Bearer $OPENAI_API_KEY" ``` @@ -7836,7 +7952,7 @@ curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/resume \ ### Example ```http -curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/resume \ +curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/pause \ -H "Authorization: Bearer $OPENAI_API_KEY" ``` @@ -7851,15 +7967,23 @@ curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/resume \ "fine_tuned_model": null, "organization_id": "org-123", "result_files": [], - "status": "queued", + "status": "paused", "validation_file": "file-abc123", "training_file": "file-abc123" } ``` -## Domain Types +## Resume fine-tuning -### Fine Tuning Job +**post** `/fine_tuning/jobs/{fine_tuning_job_id}/resume` + +Resume a fine-tune job. + +### Path Parameters + +- `fine_tuning_job_id: string` + +### Returns - `FineTuningJob object { id, created_at, error, 16 more }` @@ -8643,240 +8767,55 @@ curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/resume \ - `"default"` - - `"low"` - - - `"medium"` - - - `"high"` - - - `supervised: optional SupervisedMethod` - - Configuration for the supervised fine-tuning method. - - - `hyperparameters: optional SupervisedHyperparameters` - - The hyperparameters used for the fine-tuning job. - - - `batch_size: optional "auto" or number` - - Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - - `"auto"` - - - `"auto"` - - - `number` - - - `learning_rate_multiplier: optional "auto" or number` - - Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - - `"auto"` - - - `"auto"` - - - `number` - - - `n_epochs: optional "auto" or number` - - The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - - `"auto"` - - - `"auto"` - - - `number` - -### Fine Tuning Job Event - -- `FineTuningJobEvent object { id, created_at, level, 4 more }` - - Fine-tuning job event object - - - `id: string` - - The object identifier. - - - `created_at: number` - - The Unix timestamp (in seconds) for when the fine-tuning job was created. - - - `level: "info" or "warn" or "error"` - - The log level of the event. - - - `"info"` - - - `"warn"` - - - `"error"` - - - `message: string` - - The message of the event. - - - `object: "fine_tuning.job.event"` - - The object type, which is always "fine_tuning.job.event". - - - `"fine_tuning.job.event"` - - - `data: optional unknown` - - The data associated with the event. - - - `type: optional "message" or "metrics"` - - The type of event. - - - `"message"` - - - `"metrics"` - -### Fine Tuning Job Wandb Integration - -- `FineTuningJobWandbIntegration object { project, entity, name, tags }` - - The settings for your integration with Weights and Biases. This payload specifies the project that - metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags - to your run, and set a default entity (team, username, etc) to be associated with your run. - - - `project: string` - - The name of the project that the new run will be created under. - - - `entity: optional string` - - The entity to use for the run. This allows you to set the team or username of the WandB user that you would - like associated with the run. If not set, the default entity for the registered WandB API key is used. - - - `name: optional string` - - A display name to set for the run. If not set, we will use the Job ID as the name. - - - `tags: optional array of string` - - A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some - default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". - -### Fine Tuning Job Wandb Integration Object - -- `FineTuningJobWandbIntegrationObject object { type, wandb }` - - - `type: "wandb"` - - The type of the integration being enabled for the fine-tuning job - - - `"wandb"` - - - `wandb: FineTuningJobWandbIntegration` - - The settings for your integration with Weights and Biases. This payload specifies the project that - metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags - to your run, and set a default entity (team, username, etc) to be associated with your run. - - - `project: string` - - The name of the project that the new run will be created under. - - - `entity: optional string` - - The entity to use for the run. This allows you to set the team or username of the WandB user that you would - like associated with the run. If not set, the default entity for the registered WandB API key is used. - - - `name: optional string` - - A display name to set for the run. If not set, we will use the Job ID as the name. - - - `tags: optional array of string` - - A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some - default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". - -# Checkpoints - -## List fine-tuning checkpoints - -**get** `/fine_tuning/jobs/{fine_tuning_job_id}/checkpoints` - -List checkpoints for a fine-tuning job. - -### Path Parameters - -- `fine_tuning_job_id: string` - -### Query Parameters - -- `after: optional string` - - Identifier for the last checkpoint ID from the previous pagination request. - -- `limit: optional number` - - Number of checkpoints to retrieve. - -### Returns - -- `data: array of FineTuningJobCheckpoint` - - - `id: string` - - The checkpoint identifier, which can be referenced in the API endpoints. - - - `created_at: number` - - The Unix timestamp (in seconds) for when the checkpoint was created. - - - `fine_tuned_model_checkpoint: string` - - The name of the fine-tuned checkpoint model that is created. + - `"low"` - - `fine_tuning_job_id: string` + - `"medium"` - The name of the fine-tuning job that this checkpoint was created from. + - `"high"` - - `metrics: object { full_valid_loss, full_valid_mean_token_accuracy, step, 4 more }` + - `supervised: optional SupervisedMethod` - Metrics at the step number during the fine-tuning job. + Configuration for the supervised fine-tuning method. - - `full_valid_loss: optional number` + - `hyperparameters: optional SupervisedHyperparameters` - - `full_valid_mean_token_accuracy: optional number` + The hyperparameters used for the fine-tuning job. - - `step: optional number` + - `batch_size: optional "auto" or number` - - `train_loss: optional number` + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - `train_mean_token_accuracy: optional number` + - `"auto"` - - `valid_loss: optional number` + - `"auto"` - - `valid_mean_token_accuracy: optional number` + - `number` - - `object: "fine_tuning.job.checkpoint"` + - `learning_rate_multiplier: optional "auto" or number` - The object type, which is always "fine_tuning.job.checkpoint". + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - `"fine_tuning.job.checkpoint"` + - `"auto"` - - `step_number: number` + - `"auto"` - The step number that the checkpoint was created at. + - `number` -- `has_more: boolean` + - `n_epochs: optional "auto" or number` -- `object: "list"` + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - `"list"` + - `"auto"` -- `first_id: optional string` + - `"auto"` -- `last_id: optional string` + - `number` ### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/checkpoints \ +curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/resume \ + -X POST \ -H "Authorization: Bearer $OPENAI_API_KEY" ``` @@ -8884,36 +8823,91 @@ curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/checkpoints ```json { - "data": [ - { "id": "id", "created_at": 0, - "fine_tuned_model_checkpoint": "fine_tuned_model_checkpoint", - "fine_tuning_job_id": "fine_tuning_job_id", - "metrics": { - "full_valid_loss": 0, - "full_valid_mean_token_accuracy": 0, - "step": 0, - "train_loss": 0, - "train_mean_token_accuracy": 0, - "valid_loss": 0, - "valid_mean_token_accuracy": 0 + "error": { + "code": "code", + "message": "message", + "param": "param" }, - "object": "fine_tuning.job.checkpoint", - "step_number": 0 + "fine_tuned_model": "fine_tuned_model", + "finished_at": 0, + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + }, + "model": "model", + "object": "fine_tuning.job", + "organization_id": "organization_id", + "result_files": [ + "file-abc123" + ], + "seed": 0, + "status": "validating_files", + "trained_tokens": 0, + "training_file": "training_file", + "validation_file": "validation_file", + "estimated_finish": 0, + "integrations": [ + { + "type": "wandb", + "wandb": { + "project": "my-wandb-project", + "entity": "entity", + "name": "name", + "tags": [ + "custom-tag" + ] + } } ], - "has_more": true, - "object": "list", - "first_id": "first_id", - "last_id": "last_id" + "metadata": { + "foo": "string" + }, + "method": { + "type": "supervised", + "dpo": { + "hyperparameters": { + "batch_size": "auto", + "beta": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + } + }, + "reinforcement": { + "grader": { + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check" + }, + "hyperparameters": { + "batch_size": "auto", + "compute_multiplier": "auto", + "eval_interval": "auto", + "eval_samples": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + "reasoning_effort": "default" + } + }, + "supervised": { + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + } + } + } } ``` ### Example ```http -curl https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/checkpoints \ +curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/resume \ -H "Authorization: Bearer $OPENAI_API_KEY" ``` @@ -8921,748 +8915,640 @@ curl https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/checkpoints \ ```json { - "object": "list", - "data": [ - { - "object": "fine_tuning.job.checkpoint", - "id": "ftckpt_zc4Q7MP6XxulcVzj4MZdwsAB", - "created_at": 1721764867, - "fine_tuned_model_checkpoint": "ft:gpt-4o-mini-2024-07-18:my-org:custom-suffix:96olL566:ckpt-step-2000", - "metrics": { - "full_valid_loss": 0.134, - "full_valid_mean_token_accuracy": 0.874 - }, - "fine_tuning_job_id": "ftjob-abc123", - "step_number": 2000 - }, - { - "object": "fine_tuning.job.checkpoint", - "id": "ftckpt_enQCFmOTGj3syEpYVhBRLTSy", + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-4o-mini-2024-07-18", "created_at": 1721764800, - "fine_tuned_model_checkpoint": "ft:gpt-4o-mini-2024-07-18:my-org:custom-suffix:7q8mpxmy:ckpt-step-1000", - "metrics": { - "full_valid_loss": 0.167, - "full_valid_mean_token_accuracy": 0.781 - }, - "fine_tuning_job_id": "ftjob-abc123", - "step_number": 1000 - } - ], - "first_id": "ftckpt_zc4Q7MP6XxulcVzj4MZdwsAB", - "last_id": "ftckpt_enQCFmOTGj3syEpYVhBRLTSy", - "has_more": true + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "queued", + "validation_file": "file-abc123", + "training_file": "file-abc123" } ``` -## Domain Types +## Retrieve fine-tuning job -### Fine Tuning Job Checkpoint +**get** `/fine_tuning/jobs/{fine_tuning_job_id}` -- `FineTuningJobCheckpoint object { id, created_at, fine_tuned_model_checkpoint, 4 more }` +Get info about a fine-tuning job. - The `fine_tuning.job.checkpoint` object represents a model checkpoint for a fine-tuning job that is ready to use. +[Learn more about fine-tuning](/docs/guides/model-optimization) + +### Path Parameters + +- `fine_tuning_job_id: string` + +### Returns + +- `FineTuningJob object { id, created_at, error, 16 more }` + + The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. - `id: string` - The checkpoint identifier, which can be referenced in the API endpoints. + The object identifier, which can be referenced in the API endpoints. - `created_at: number` - The Unix timestamp (in seconds) for when the checkpoint was created. + The Unix timestamp (in seconds) for when the fine-tuning job was created. - - `fine_tuned_model_checkpoint: string` + - `error: object { code, message, param }` - The name of the fine-tuned checkpoint model that is created. + For fine-tuning jobs that have `failed`, this will contain more information on the cause of the failure. - - `fine_tuning_job_id: string` + - `code: string` - The name of the fine-tuning job that this checkpoint was created from. + A machine-readable error code. - - `metrics: object { full_valid_loss, full_valid_mean_token_accuracy, step, 4 more }` + - `message: string` - Metrics at the step number during the fine-tuning job. + A human-readable error message. - - `full_valid_loss: optional number` + - `param: string` - - `full_valid_mean_token_accuracy: optional number` + The parameter that was invalid, usually `training_file` or `validation_file`. This field will be null if the failure was not parameter-specific. - - `step: optional number` + - `fine_tuned_model: string` - - `train_loss: optional number` + The name of the fine-tuned model that is being created. The value will be null if the fine-tuning job is still running. - - `train_mean_token_accuracy: optional number` + - `finished_at: number` - - `valid_loss: optional number` + The Unix timestamp (in seconds) for when the fine-tuning job was finished. The value will be null if the fine-tuning job is still running. - - `valid_mean_token_accuracy: optional number` + - `hyperparameters: object { batch_size, learning_rate_multiplier, n_epochs }` - - `object: "fine_tuning.job.checkpoint"` + The hyperparameters used for the fine-tuning job. This value will only be returned when running `supervised` jobs. - The object type, which is always "fine_tuning.job.checkpoint". + - `batch_size: optional "auto" or number` - - `"fine_tuning.job.checkpoint"` + Number of examples in each batch. A larger batch size means that model parameters + are updated less frequently, but with lower variance. - - `step_number: number` + - `"auto"` - The step number that the checkpoint was created at. + - `"auto"` -# Checkpoints + - `number` -# Permissions + - `learning_rate_multiplier: optional "auto" or number` + + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid + overfitting. + + - `"auto"` + + - `"auto"` + + - `number` + + - `n_epochs: optional "auto" or number` + + The number of epochs to train the model for. An epoch refers to one full cycle + through the training dataset. + + - `"auto"` -## List checkpoint permissions + - `"auto"` -**get** `/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions` + - `number` -**NOTE:** This endpoint requires an [admin API key](../admin-api-keys). + - `model: string` -Organization owners can use this endpoint to view all permissions for a fine-tuned model checkpoint. + The base model that is being fine-tuned. -### Path Parameters + - `object: "fine_tuning.job"` -- `fine_tuned_model_checkpoint: string` + The object type, which is always "fine_tuning.job". -### Query Parameters + - `"fine_tuning.job"` -- `after: optional string` + - `organization_id: string` - Identifier for the last permission ID from the previous pagination request. + The organization that owns the fine-tuning job. -- `limit: optional number` + - `result_files: array of string` - Number of permissions to retrieve. + The compiled results file ID(s) for the fine-tuning job. You can retrieve the results with the [Files API](/docs/api-reference/files/retrieve-contents). -- `order: optional "ascending" or "descending"` + - `seed: number` - The order in which to retrieve permissions. + The seed used for the fine-tuning job. - - `"ascending"` + - `status: "validating_files" or "queued" or "running" or 3 more` - - `"descending"` + The current status of the fine-tuning job, which can be either `validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`. -- `project_id: optional string` + - `"validating_files"` - The ID of the project to get permissions for. + - `"queued"` -### Returns + - `"running"` -- `data: array of object { id, created_at, object, project_id }` + - `"succeeded"` - - `id: string` + - `"failed"` - The permission identifier, which can be referenced in the API endpoints. + - `"cancelled"` - - `created_at: number` + - `trained_tokens: number` - The Unix timestamp (in seconds) for when the permission was created. + The total number of billable tokens processed by this fine-tuning job. The value will be null if the fine-tuning job is still running. - - `object: "checkpoint.permission"` + - `training_file: string` - The object type, which is always "checkpoint.permission". + The file ID used for training. You can retrieve the training data with the [Files API](/docs/api-reference/files/retrieve-contents). - - `"checkpoint.permission"` + - `validation_file: string` - - `project_id: string` + The file ID used for validation. You can retrieve the validation results with the [Files API](/docs/api-reference/files/retrieve-contents). - The project identifier that the permission is for. + - `estimated_finish: optional number` -- `has_more: boolean` + The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. -- `object: "list"` + - `integrations: optional array of FineTuningJobWandbIntegrationObject` - - `"list"` + A list of integrations to enable for this fine-tuning job. -- `first_id: optional string` + - `type: "wandb"` -- `last_id: optional string` + The type of the integration being enabled for the fine-tuning job -### Example + - `"wandb"` -```http -curl https://api.openai.com/v1/fine_tuning/checkpoints/$FINE_TUNED_MODEL_CHECKPOINT/permissions \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` + - `wandb: FineTuningJobWandbIntegration` -#### Response + The settings for your integration with Weights and Biases. This payload specifies the project that + metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags + to your run, and set a default entity (team, username, etc) to be associated with your run. -```json -{ - "data": [ - { - "id": "id", - "created_at": 0, - "object": "checkpoint.permission", - "project_id": "project_id" - } - ], - "has_more": true, - "object": "list", - "first_id": "first_id", - "last_id": "last_id" -} -``` + - `project: string` -### Example + The name of the project that the new run will be created under. -```http -curl https://api.openai.com/v1/fine_tuning/checkpoints/ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd/permissions \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` + - `entity: optional string` -#### Response + The entity to use for the run. This allows you to set the team or username of the WandB user that you would + like associated with the run. If not set, the default entity for the registered WandB API key is used. -```json -{ - "object": "list", - "data": [ - { - "object": "checkpoint.permission", - "id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", - "created_at": 1721764867, - "project_id": "proj_abGMw1llN8IrBb6SvvY5A1iH" - }, - { - "object": "checkpoint.permission", - "id": "cp_enQCFmOTGj3syEpYVhBRLTSy", - "created_at": 1721764800, - "project_id": "proj_iqGMw1llN8IrBb6SvvY5A1oF" - }, - ], - "first_id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", - "last_id": "cp_enQCFmOTGj3syEpYVhBRLTSy", - "has_more": false -} -``` + - `name: optional string` -## List checkpoint permissions + A display name to set for the run. If not set, we will use the Job ID as the name. -**get** `/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions` + - `tags: optional array of string` -**NOTE:** This endpoint requires an [admin API key](../admin-api-keys). + A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some + default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". -Organization owners can use this endpoint to view all permissions for a fine-tuned model checkpoint. + - `metadata: optional Metadata` -### Path Parameters + Set of 16 key-value pairs that can be attached to an object. This can be + useful for storing additional information about the object in a structured + format, and querying for objects via API or the dashboard. -- `fine_tuned_model_checkpoint: string` + Keys are strings with a maximum length of 64 characters. Values are strings + with a maximum length of 512 characters. -### Query Parameters + - `method: optional object { type, dpo, reinforcement, supervised }` -- `after: optional string` + The method used for fine-tuning. - Identifier for the last permission ID from the previous pagination request. + - `type: "supervised" or "dpo" or "reinforcement"` -- `limit: optional number` + The type of method. Is either `supervised`, `dpo`, or `reinforcement`. - Number of permissions to retrieve. + - `"supervised"` -- `order: optional "ascending" or "descending"` + - `"dpo"` - The order in which to retrieve permissions. + - `"reinforcement"` - - `"ascending"` + - `dpo: optional DpoMethod` - - `"descending"` + Configuration for the DPO fine-tuning method. -- `project_id: optional string` + - `hyperparameters: optional DpoHyperparameters` - The ID of the project to get permissions for. + The hyperparameters used for the DPO fine-tuning job. -### Returns + - `batch_size: optional "auto" or number` -- `data: array of object { id, created_at, object, project_id }` + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - `id: string` + - `"auto"` - The permission identifier, which can be referenced in the API endpoints. + - `"auto"` - - `created_at: number` + - `number` - The Unix timestamp (in seconds) for when the permission was created. + - `beta: optional "auto" or number` - - `object: "checkpoint.permission"` + The beta value for the DPO method. A higher beta value will increase the weight of the penalty between the policy and reference model. - The object type, which is always "checkpoint.permission". + - `"auto"` - - `"checkpoint.permission"` + - `"auto"` - - `project_id: string` + - `number` - The project identifier that the permission is for. + - `learning_rate_multiplier: optional "auto" or number` -- `has_more: boolean` + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. -- `object: "list"` + - `"auto"` - - `"list"` + - `"auto"` -- `first_id: optional string` + - `number` -- `last_id: optional string` + - `n_epochs: optional "auto" or number` -### Example + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. -```http -curl https://api.openai.com/v1/fine_tuning/checkpoints/$FINE_TUNED_MODEL_CHECKPOINT/permissions \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` + - `"auto"` -#### Response + - `"auto"` -```json -{ - "data": [ - { - "id": "id", - "created_at": 0, - "object": "checkpoint.permission", - "project_id": "project_id" - } - ], - "has_more": true, - "object": "list", - "first_id": "first_id", - "last_id": "last_id" -} -``` + - `number` -### Example + - `reinforcement: optional ReinforcementMethod` -```http -curl https://api.openai.com/v1/fine_tuning/checkpoints/ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd/permissions \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` + Configuration for the reinforcement fine-tuning method. -#### Response + - `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` -```json -{ - "object": "list", - "data": [ - { - "object": "checkpoint.permission", - "id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", - "created_at": 1721764867, - "project_id": "proj_abGMw1llN8IrBb6SvvY5A1iH" - }, - { - "object": "checkpoint.permission", - "id": "cp_enQCFmOTGj3syEpYVhBRLTSy", - "created_at": 1721764800, - "project_id": "proj_iqGMw1llN8IrBb6SvvY5A1oF" - }, - ], - "first_id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", - "last_id": "cp_enQCFmOTGj3syEpYVhBRLTSy", - "has_more": false -} -``` + The grader used for the fine-tuning job. -## Create checkpoint permissions + - `StringCheckGrader object { input, name, operation, 2 more }` -**post** `/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions` + A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. -**NOTE:** Calling this endpoint requires an [admin API key](../admin-api-keys). + - `input: string` -This enables organization owners to share fine-tuned models with other projects in their organization. + The input text. This may include template strings. -### Path Parameters + - `name: string` -- `fine_tuned_model_checkpoint: string` + The name of the grader. -### Body Parameters + - `operation: "eq" or "ne" or "like" or "ilike"` -- `project_ids: array of string` + The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`. - The project identifiers to grant access to. + - `"eq"` -### Returns + - `"ne"` -- `data: array of object { id, created_at, object, project_id }` + - `"like"` - - `id: string` + - `"ilike"` - The permission identifier, which can be referenced in the API endpoints. + - `reference: string` - - `created_at: number` + The reference text. This may include template strings. - The Unix timestamp (in seconds) for when the permission was created. + - `type: "string_check"` - - `object: "checkpoint.permission"` + The object type, which is always `string_check`. - The object type, which is always "checkpoint.permission". + - `"string_check"` - - `"checkpoint.permission"` + - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` - - `project_id: string` + A TextSimilarityGrader object which grades text based on similarity metrics. - The project identifier that the permission is for. + - `evaluation_metric: "cosine" or "fuzzy_match" or "bleu" or 8 more` -- `has_more: boolean` + The evaluation metric to use. One of `cosine`, `fuzzy_match`, `bleu`, + `gleu`, `meteor`, `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, + or `rouge_l`. -- `object: "list"` + - `"cosine"` - - `"list"` + - `"fuzzy_match"` -- `first_id: optional string` + - `"bleu"` -- `last_id: optional string` + - `"gleu"` -### Example + - `"meteor"` -```http -curl https://api.openai.com/v1/fine_tuning/checkpoints/$FINE_TUNED_MODEL_CHECKPOINT/permissions \ - -H 'Content-Type: application/json' \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "project_ids": [ - "string" - ] - }' -``` + - `"rouge_1"` -#### Response + - `"rouge_2"` -```json -{ - "data": [ - { - "id": "id", - "created_at": 0, - "object": "checkpoint.permission", - "project_id": "project_id" - } - ], - "has_more": true, - "object": "list", - "first_id": "first_id", - "last_id": "last_id" -} -``` + - `"rouge_3"` -### Example + - `"rouge_4"` -```http -curl https://api.openai.com/v1/fine_tuning/checkpoints/ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd/permissions \ - -H "Authorization: Bearer $OPENAI_API_KEY" - -d '{"project_ids": ["proj_abGMw1llN8IrBb6SvvY5A1iH"]}' -``` + - `"rouge_5"` -#### Response + - `"rouge_l"` -```json -{ - "object": "list", - "data": [ - { - "object": "checkpoint.permission", - "id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", - "created_at": 1721764867, - "project_id": "proj_abGMw1llN8IrBb6SvvY5A1iH" - } - ], - "first_id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", - "last_id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", - "has_more": false -} -``` + - `input: string` -## Delete checkpoint permission + The text being graded. -**delete** `/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions/{permission_id}` + - `name: string` -**NOTE:** This endpoint requires an [admin API key](../admin-api-keys). + The name of the grader. -Organization owners can use this endpoint to delete a permission for a fine-tuned model checkpoint. + - `reference: string` -### Path Parameters + The text being graded against. -- `fine_tuned_model_checkpoint: string` + - `type: "text_similarity"` -- `permission_id: string` + The type of grader. -### Returns + - `"text_similarity"` -- `id: string` + - `PythonGrader object { name, source, type, image_tag }` - The ID of the fine-tuned model checkpoint permission that was deleted. + A PythonGrader object that runs a python script on the input. -- `deleted: boolean` + - `name: string` - Whether the fine-tuned model checkpoint permission was successfully deleted. + The name of the grader. -- `object: "checkpoint.permission"` + - `source: string` - The object type, which is always "checkpoint.permission". + The source code of the python script. - - `"checkpoint.permission"` + - `type: "python"` -### Example + The object type, which is always `python`. -```http -curl https://api.openai.com/v1/fine_tuning/checkpoints/$FINE_TUNED_MODEL_CHECKPOINT/permissions/$PERMISSION_ID \ - -X DELETE \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` + - `"python"` -#### Response + - `image_tag: optional string` -```json -{ - "id": "id", - "deleted": true, - "object": "checkpoint.permission" -} -``` + The image tag to use for the python script. -### Example + - `ScoreModelGrader object { input, model, name, 3 more }` -```http -curl https://api.openai.com/v1/fine_tuning/checkpoints/ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd/permissions/cp_zc4Q7MP6XxulcVzj4MZdwsAB \ - -H "Authorization: Bearer $OPENAI_API_KEY" -``` + A ScoreModelGrader object that uses a model to assign a score to the input. -#### Response + - `input: array of object { content, role, type }` -```json -{ - "object": "checkpoint.permission", - "id": "cp_zc4Q7MP6XxulcVzj4MZdwsAB", - "deleted": true -} -``` + The input messages evaluated by the grader. Supports text, output text, input image, and input audio content blocks, and may include template strings. -## Domain Types + - `content: string or ResponseInputText or object { text, type } or 3 more` -### Permission Retrieve Response + Inputs to the model - can contain template strings. Supports text, output text, input images, and input audio, either as a single item or an array of items. -- `PermissionRetrieveResponse object { data, has_more, object, 2 more }` + - `TextInput = string` - - `data: array of object { id, created_at, object, project_id }` + A text input to the model. - - `id: string` + - `ResponseInputText object { text, type, prompt_cache_breakpoint }` - The permission identifier, which can be referenced in the API endpoints. + A text input to the model. - - `created_at: number` + - `text: string` - The Unix timestamp (in seconds) for when the permission was created. + The text input to the model. - - `object: "checkpoint.permission"` + - `type: "input_text"` - The object type, which is always "checkpoint.permission". + The type of the input item. Always `input_text`. - - `"checkpoint.permission"` + - `"input_text"` - - `project_id: string` + - `prompt_cache_breakpoint: optional object { mode }` - The project identifier that the permission is for. + Marks the exact end of a reusable prompt prefix. The breakpoint inherits its TTL from the request's `prompt_cache_options.ttl`; the boundary is not rounded to a token block. - - `has_more: boolean` + - `mode: "explicit"` - - `object: "list"` + The breakpoint mode. Always `explicit`. - - `"list"` + - `"explicit"` - - `first_id: optional string` + - `OutputText object { text, type }` - - `last_id: optional string` + A text output from the model. -### Permission List Response + - `text: string` -- `PermissionListResponse object { id, created_at, object, project_id }` + The text output from the model. - The `checkpoint.permission` object represents a permission for a fine-tuned model checkpoint. + - `type: "output_text"` - - `id: string` + The type of the output text. Always `output_text`. - The permission identifier, which can be referenced in the API endpoints. + - `"output_text"` - - `created_at: number` + - `InputImage object { image_url, type, detail }` - The Unix timestamp (in seconds) for when the permission was created. + An image input block used within EvalItem content arrays. - - `object: "checkpoint.permission"` + - `image_url: string` - The object type, which is always "checkpoint.permission". + The URL of the image input. - - `"checkpoint.permission"` + - `type: "input_image"` - - `project_id: string` + The type of the image input. Always `input_image`. - The project identifier that the permission is for. + - `"input_image"` -### Permission Create Response + - `detail: optional string` -- `PermissionCreateResponse object { id, created_at, object, project_id }` + The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. - The `checkpoint.permission` object represents a permission for a fine-tuned model checkpoint. + - `ResponseInputAudio object { input_audio, type }` - - `id: string` + An audio input to the model. + + - `input_audio: object { data, format }` + + - `data: string` + + Base64-encoded audio data. + + - `format: "mp3" or "wav"` + + The format of the audio data. Currently supported formats are `mp3` and + `wav`. + + - `"mp3"` - The permission identifier, which can be referenced in the API endpoints. + - `"wav"` - - `created_at: number` + - `type: "input_audio"` - The Unix timestamp (in seconds) for when the permission was created. + The type of the input item. Always `input_audio`. - - `object: "checkpoint.permission"` + - `"input_audio"` - The object type, which is always "checkpoint.permission". + - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` - - `"checkpoint.permission"` + A list of inputs, each of which may be either an input text, output text, input + image, or input audio object. - - `project_id: string` + - `TextInput = string` - The project identifier that the permission is for. + A text input to the model. -### Permission Delete Response + - `ResponseInputText object { text, type, prompt_cache_breakpoint }` -- `PermissionDeleteResponse object { id, deleted, object }` + A text input to the model. - - `id: string` + - `OutputText object { text, type }` - The ID of the fine-tuned model checkpoint permission that was deleted. + A text output from the model. - - `deleted: boolean` + - `text: string` - Whether the fine-tuned model checkpoint permission was successfully deleted. + The text output from the model. - - `object: "checkpoint.permission"` + - `type: "output_text"` - The object type, which is always "checkpoint.permission". + The type of the output text. Always `output_text`. - - `"checkpoint.permission"` + - `"output_text"` -# Alpha + - `InputImage object { image_url, type, detail }` -# Graders + An image input block used within EvalItem content arrays. -## Run grader + - `image_url: string` -**post** `/fine_tuning/alpha/graders/run` + The URL of the image input. -Run a grader. + - `type: "input_image"` -### Body Parameters + The type of the image input. Always `input_image`. -- `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` + - `"input_image"` - The grader used for the fine-tuning job. + - `detail: optional string` - - `StringCheckGrader object { input, name, operation, 2 more }` + The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. - A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. + - `ResponseInputAudio object { input_audio, type }` - - `input: string` + An audio input to the model. - The input text. This may include template strings. + - `role: "user" or "assistant" or "system" or "developer"` - - `name: string` + The role of the message input. One of `user`, `assistant`, `system`, or + `developer`. - The name of the grader. + - `"user"` - - `operation: "eq" or "ne" or "like" or "ilike"` + - `"assistant"` - The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`. + - `"system"` - - `"eq"` + - `"developer"` - - `"ne"` + - `type: optional "message"` - - `"like"` + The type of the message input. Always `message`. - - `"ilike"` + - `"message"` - - `reference: string` + - `model: string` - The reference text. This may include template strings. + The model to use for the evaluation. - - `type: "string_check"` + - `name: string` - The object type, which is always `string_check`. + The name of the grader. - - `"string_check"` + - `type: "score_model"` - - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` + The object type, which is always `score_model`. - A TextSimilarityGrader object which grades text based on similarity metrics. + - `"score_model"` - - `evaluation_metric: "cosine" or "fuzzy_match" or "bleu" or 8 more` + - `range: optional array of number` - The evaluation metric to use. One of `cosine`, `fuzzy_match`, `bleu`, - `gleu`, `meteor`, `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, - or `rouge_l`. + The range of the score. Defaults to `[0, 1]`. - - `"cosine"` + - `sampling_params: optional object { max_completions_tokens, reasoning_effort, seed, 2 more }` - - `"fuzzy_match"` + The sampling parameters for the model. - - `"bleu"` + - `max_completions_tokens: optional number` - - `"gleu"` + The maximum number of tokens the grader model may generate in its response. - - `"meteor"` + - `reasoning_effort: optional ReasoningEffort` - - `"rouge_1"` + Constrains effort on reasoning for reasoning models. Currently supported + values are `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, and `max`. + Reducing reasoning effort can result in faster responses and fewer tokens + used on reasoning in a response. Not all reasoning models support every + value. See the + [reasoning guide](https://platform.openai.com/docs/guides/reasoning) + for model-specific support. - - `"rouge_2"` + - `"none"` - - `"rouge_3"` + - `"minimal"` - - `"rouge_4"` + - `"low"` - - `"rouge_5"` + - `"medium"` - - `"rouge_l"` + - `"high"` - - `input: string` + - `"xhigh"` - The text being graded. + - `"max"` - - `name: string` + - `seed: optional number` - The name of the grader. + A seed value to initialize the randomness, during sampling. - - `reference: string` + - `temperature: optional number` - The text being graded against. + A higher temperature increases randomness in the outputs. - - `type: "text_similarity"` + - `top_p: optional number` - The type of grader. + An alternative to temperature for nucleus sampling; 1.0 includes all tokens. - - `"text_similarity"` + - `MultiGrader object { calculate_output, graders, name, type }` - - `PythonGrader object { name, source, type, image_tag }` + A MultiGrader object combines the output of multiple graders to produce a single score. - A PythonGrader object that runs a python script on the input. + - `calculate_output: string` - - `name: string` + A formula to calculate the output based on grader results. - The name of the grader. + - `graders: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` - - `source: string` + A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. - The source code of the python script. + - `StringCheckGrader object { input, name, operation, 2 more }` - - `type: "python"` + A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. - The object type, which is always `python`. + - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` - - `"python"` + A TextSimilarityGrader object which grades text based on similarity metrics. - - `image_tag: optional string` + - `PythonGrader object { name, source, type, image_tag }` - The image tag to use for the python script. + A PythonGrader object that runs a python script on the input. - `ScoreModelGrader object { input, model, name, 3 more }` A ScoreModelGrader object that uses a model to assign a score to the input. - - `input: array of object { content, role, type }` + - `LabelModelGrader object { input, labels, model, 3 more }` - The input messages evaluated by the grader. Supports text, output text, input image, and input audio content blocks, and may include template strings. + A LabelModelGrader object which uses a model to assign labels to each item + in the evaluation. + + - `input: array of object { content, role, type }` - `content: string or ResponseInputText or object { text, type } or 3 more` @@ -9676,26 +9562,6 @@ Run a grader. A text input to the model. - - `text: string` - - The text input to the model. - - - `type: "input_text"` - - The type of the input item. Always `input_text`. - - - `"input_text"` - - - `prompt_cache_breakpoint: optional object { mode }` - - Marks the exact end of a reusable prompt prefix. The breakpoint inherits its TTL from the request's `prompt_cache_options.ttl`; the boundary is not rounded to a token block. - - - `mode: "explicit"` - - The breakpoint mode. Always `explicit`. - - - `"explicit"` - - `OutputText object { text, type }` A text output from the model. @@ -9732,590 +9598,555 @@ Run a grader. An audio input to the model. - - `input_audio: object { data, format }` + - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` - - `data: string` + A list of inputs, each of which may be either an input text, output text, input + image, or input audio object. - Base64-encoded audio data. + - `role: "user" or "assistant" or "system" or "developer"` - - `format: "mp3" or "wav"` + The role of the message input. One of `user`, `assistant`, `system`, or + `developer`. - The format of the audio data. Currently supported formats are `mp3` and - `wav`. + - `"user"` - - `"mp3"` + - `"assistant"` - - `"wav"` + - `"system"` - - `type: "input_audio"` + - `"developer"` - The type of the input item. Always `input_audio`. + - `type: optional "message"` - - `"input_audio"` + The type of the message input. Always `message`. - - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` + - `"message"` - A list of inputs, each of which may be either an input text, output text, input - image, or input audio object. + - `labels: array of string` - - `TextInput = string` + The labels to assign to each item in the evaluation. - A text input to the model. + - `model: string` - - `ResponseInputText object { text, type, prompt_cache_breakpoint }` + The model to use for the evaluation. Must support structured outputs. + + - `name: string` + + The name of the grader. + + - `passing_labels: array of string` + + The labels that indicate a passing result. Must be a subset of labels. + + - `type: "label_model"` + + The object type, which is always `label_model`. + + - `"label_model"` + + - `name: string` + + The name of the grader. + + - `type: "multi"` + + The object type, which is always `multi`. + + - `"multi"` + + - `hyperparameters: optional ReinforcementHyperparameters` + + The hyperparameters used for the reinforcement fine-tuning job. + + - `batch_size: optional "auto" or number` - A text input to the model. + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - `OutputText object { text, type }` + - `"auto"` - A text output from the model. + - `"auto"` - - `text: string` + - `number` - The text output from the model. + - `compute_multiplier: optional "auto" or number` - - `type: "output_text"` + Multiplier on amount of compute used for exploring search space during training. - The type of the output text. Always `output_text`. + - `"auto"` - - `"output_text"` + - `"auto"` - - `InputImage object { image_url, type, detail }` + - `number` - An image input block used within EvalItem content arrays. + - `eval_interval: optional "auto" or number` - - `image_url: string` + The number of training steps between evaluation runs. - The URL of the image input. + - `"auto"` - - `type: "input_image"` + - `"auto"` - The type of the image input. Always `input_image`. + - `number` - - `"input_image"` + - `eval_samples: optional "auto" or number` - - `detail: optional string` + Number of evaluation samples to generate per training step. - The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. + - `"auto"` - - `ResponseInputAudio object { input_audio, type }` + - `"auto"` - An audio input to the model. + - `number` - - `role: "user" or "assistant" or "system" or "developer"` + - `learning_rate_multiplier: optional "auto" or number` - The role of the message input. One of `user`, `assistant`, `system`, or - `developer`. + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - `"user"` + - `"auto"` - - `"assistant"` + - `"auto"` - - `"system"` + - `number` - - `"developer"` + - `n_epochs: optional "auto" or number` - - `type: optional "message"` + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - The type of the message input. Always `message`. + - `"auto"` - - `"message"` + - `"auto"` - - `model: string` + - `number` - The model to use for the evaluation. + - `reasoning_effort: optional "default" or "low" or "medium" or "high"` - - `name: string` + Level of reasoning effort. - The name of the grader. + - `"default"` - - `type: "score_model"` + - `"low"` - The object type, which is always `score_model`. + - `"medium"` - - `"score_model"` + - `"high"` - - `range: optional array of number` + - `supervised: optional SupervisedMethod` - The range of the score. Defaults to `[0, 1]`. + Configuration for the supervised fine-tuning method. - - `sampling_params: optional object { max_completions_tokens, reasoning_effort, seed, 2 more }` + - `hyperparameters: optional SupervisedHyperparameters` - The sampling parameters for the model. + The hyperparameters used for the fine-tuning job. - - `max_completions_tokens: optional number` + - `batch_size: optional "auto" or number` - The maximum number of tokens the grader model may generate in its response. + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - `reasoning_effort: optional ReasoningEffort` + - `"auto"` - Constrains effort on reasoning for reasoning models. Currently supported - values are `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, and `max`. - Reducing reasoning effort can result in faster responses and fewer tokens - used on reasoning in a response. Not all reasoning models support every - value. See the - [reasoning guide](https://platform.openai.com/docs/guides/reasoning) - for model-specific support. + - `"auto"` - - `"none"` + - `number` - - `"minimal"` + - `learning_rate_multiplier: optional "auto" or number` - - `"low"` + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - `"medium"` + - `"auto"` - - `"high"` + - `"auto"` - - `"xhigh"` + - `number` - - `"max"` + - `n_epochs: optional "auto" or number` - - `seed: optional number` + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - A seed value to initialize the randomness, during sampling. + - `"auto"` - - `temperature: optional number` + - `"auto"` - A higher temperature increases randomness in the outputs. + - `number` - - `top_p: optional number` +### Example - An alternative to temperature for nucleus sampling; 1.0 includes all tokens. +```http +curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID \ + -H "Authorization: Bearer $OPENAI_API_KEY" +``` - - `MultiGrader object { calculate_output, graders, name, type }` +#### Response - A MultiGrader object combines the output of multiple graders to produce a single score. +```json +{ + "id": "id", + "created_at": 0, + "error": { + "code": "code", + "message": "message", + "param": "param" + }, + "fine_tuned_model": "fine_tuned_model", + "finished_at": 0, + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + }, + "model": "model", + "object": "fine_tuning.job", + "organization_id": "organization_id", + "result_files": [ + "file-abc123" + ], + "seed": 0, + "status": "validating_files", + "trained_tokens": 0, + "training_file": "training_file", + "validation_file": "validation_file", + "estimated_finish": 0, + "integrations": [ + { + "type": "wandb", + "wandb": { + "project": "my-wandb-project", + "entity": "entity", + "name": "name", + "tags": [ + "custom-tag" + ] + } + } + ], + "metadata": { + "foo": "string" + }, + "method": { + "type": "supervised", + "dpo": { + "hyperparameters": { + "batch_size": "auto", + "beta": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + } + }, + "reinforcement": { + "grader": { + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check" + }, + "hyperparameters": { + "batch_size": "auto", + "compute_multiplier": "auto", + "eval_interval": "auto", + "eval_samples": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + "reasoning_effort": "default" + } + }, + "supervised": { + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto" + } + } + } +} +``` - - `calculate_output: string` +### Example - A formula to calculate the output based on grader results. +```http +curl https://api.openai.com/v1/fine_tuning/jobs/ft-AF1WoRqd3aJAHsqc9NY7iL8F \ + -H "Authorization: Bearer $OPENAI_API_KEY" +``` - - `graders: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` +#### Response - A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. +```json +{ + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "davinci-002", + "created_at": 1692661014, + "finished_at": 1692661190, + "fine_tuned_model": "ft:davinci-002:my-org:custom_suffix:7q8mpxmy", + "organization_id": "org-123", + "result_files": [ + "file-abc123" + ], + "status": "succeeded", + "validation_file": null, + "training_file": "file-abc123", + "hyperparameters": { + "n_epochs": 4, + "batch_size": 1, + "learning_rate_multiplier": 1.0 + }, + "trained_tokens": 5768, + "integrations": [], + "seed": 0, + "estimated_finish": 0, + "method": { + "type": "supervised", + "supervised": { + "hyperparameters": { + "n_epochs": 4, + "batch_size": 1, + "learning_rate_multiplier": 1.0 + } + } + } +} +``` - - `StringCheckGrader object { input, name, operation, 2 more }` +## Domain Types - A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. +### Fine Tuning Job - - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` +- `FineTuningJob object { id, created_at, error, 16 more }` - A TextSimilarityGrader object which grades text based on similarity metrics. + The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. - - `PythonGrader object { name, source, type, image_tag }` + - `id: string` - A PythonGrader object that runs a python script on the input. + The object identifier, which can be referenced in the API endpoints. - - `ScoreModelGrader object { input, model, name, 3 more }` + - `created_at: number` - A ScoreModelGrader object that uses a model to assign a score to the input. + The Unix timestamp (in seconds) for when the fine-tuning job was created. - - `LabelModelGrader object { input, labels, model, 3 more }` + - `error: object { code, message, param }` - A LabelModelGrader object which uses a model to assign labels to each item - in the evaluation. + For fine-tuning jobs that have `failed`, this will contain more information on the cause of the failure. - - `input: array of object { content, role, type }` + - `code: string` - - `content: string or ResponseInputText or object { text, type } or 3 more` + A machine-readable error code. - Inputs to the model - can contain template strings. Supports text, output text, input images, and input audio, either as a single item or an array of items. + - `message: string` - - `TextInput = string` + A human-readable error message. - A text input to the model. + - `param: string` - - `ResponseInputText object { text, type, prompt_cache_breakpoint }` + The parameter that was invalid, usually `training_file` or `validation_file`. This field will be null if the failure was not parameter-specific. - A text input to the model. + - `fine_tuned_model: string` - - `OutputText object { text, type }` + The name of the fine-tuned model that is being created. The value will be null if the fine-tuning job is still running. - A text output from the model. + - `finished_at: number` - - `text: string` + The Unix timestamp (in seconds) for when the fine-tuning job was finished. The value will be null if the fine-tuning job is still running. - The text output from the model. + - `hyperparameters: object { batch_size, learning_rate_multiplier, n_epochs }` - - `type: "output_text"` + The hyperparameters used for the fine-tuning job. This value will only be returned when running `supervised` jobs. - The type of the output text. Always `output_text`. + - `batch_size: optional "auto" or number` - - `"output_text"` + Number of examples in each batch. A larger batch size means that model parameters + are updated less frequently, but with lower variance. - - `InputImage object { image_url, type, detail }` + - `"auto"` - An image input block used within EvalItem content arrays. + - `"auto"` - - `image_url: string` + - `number` - The URL of the image input. + - `learning_rate_multiplier: optional "auto" or number` - - `type: "input_image"` + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid + overfitting. - The type of the image input. Always `input_image`. + - `"auto"` - - `"input_image"` + - `"auto"` - - `detail: optional string` + - `number` - The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. + - `n_epochs: optional "auto" or number` - - `ResponseInputAudio object { input_audio, type }` + The number of epochs to train the model for. An epoch refers to one full cycle + through the training dataset. - An audio input to the model. + - `"auto"` - - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` + - `"auto"` - A list of inputs, each of which may be either an input text, output text, input - image, or input audio object. + - `number` - - `role: "user" or "assistant" or "system" or "developer"` + - `model: string` - The role of the message input. One of `user`, `assistant`, `system`, or - `developer`. + The base model that is being fine-tuned. - - `"user"` + - `object: "fine_tuning.job"` - - `"assistant"` + The object type, which is always "fine_tuning.job". - - `"system"` + - `"fine_tuning.job"` - - `"developer"` + - `organization_id: string` - - `type: optional "message"` + The organization that owns the fine-tuning job. - The type of the message input. Always `message`. + - `result_files: array of string` - - `"message"` + The compiled results file ID(s) for the fine-tuning job. You can retrieve the results with the [Files API](/docs/api-reference/files/retrieve-contents). - - `labels: array of string` + - `seed: number` - The labels to assign to each item in the evaluation. + The seed used for the fine-tuning job. - - `model: string` + - `status: "validating_files" or "queued" or "running" or 3 more` - The model to use for the evaluation. Must support structured outputs. + The current status of the fine-tuning job, which can be either `validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`. - - `name: string` + - `"validating_files"` - The name of the grader. + - `"queued"` - - `passing_labels: array of string` + - `"running"` - The labels that indicate a passing result. Must be a subset of labels. + - `"succeeded"` - - `type: "label_model"` + - `"failed"` - The object type, which is always `label_model`. + - `"cancelled"` - - `"label_model"` + - `trained_tokens: number` - - `name: string` + The total number of billable tokens processed by this fine-tuning job. The value will be null if the fine-tuning job is still running. - The name of the grader. + - `training_file: string` - - `type: "multi"` + The file ID used for training. You can retrieve the training data with the [Files API](/docs/api-reference/files/retrieve-contents). - The object type, which is always `multi`. + - `validation_file: string` - - `"multi"` + The file ID used for validation. You can retrieve the validation results with the [Files API](/docs/api-reference/files/retrieve-contents). -- `model_sample: string` + - `estimated_finish: optional number` - The model sample to be evaluated. This value will be used to populate - the `sample` namespace. See [the guide](/docs/guides/graders) for more details. - The `output_json` variable will be populated if the model sample is a - valid JSON string. + The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. -- `item: optional unknown` + - `integrations: optional array of FineTuningJobWandbIntegrationObject` - The dataset item provided to the grader. This will be used to populate - the `item` namespace. See [the guide](/docs/guides/graders) for more details. + A list of integrations to enable for this fine-tuning job. -### Returns + - `type: "wandb"` -- `metadata: object { errors, execution_time, name, 4 more }` + The type of the integration being enabled for the fine-tuning job - - `errors: object { formula_parse_error, invalid_variable_error, model_grader_parse_error, 11 more }` + - `"wandb"` - - `formula_parse_error: boolean` + - `wandb: FineTuningJobWandbIntegration` - - `invalid_variable_error: boolean` + The settings for your integration with Weights and Biases. This payload specifies the project that + metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags + to your run, and set a default entity (team, username, etc) to be associated with your run. - - `model_grader_parse_error: boolean` + - `project: string` - - `model_grader_refusal_error: boolean` + The name of the project that the new run will be created under. - - `model_grader_server_error: boolean` + - `entity: optional string` - - `model_grader_server_error_details: string` + The entity to use for the run. This allows you to set the team or username of the WandB user that you would + like associated with the run. If not set, the default entity for the registered WandB API key is used. - - `other_error: boolean` + - `name: optional string` - - `python_grader_runtime_error: boolean` + A display name to set for the run. If not set, we will use the Job ID as the name. - - `python_grader_runtime_error_details: string` + - `tags: optional array of string` - - `python_grader_server_error: boolean` + A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some + default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". - - `python_grader_server_error_type: string` + - `metadata: optional Metadata` - - `sample_parse_error: boolean` + Set of 16 key-value pairs that can be attached to an object. This can be + useful for storing additional information about the object in a structured + format, and querying for objects via API or the dashboard. - - `truncated_observation_error: boolean` + Keys are strings with a maximum length of 64 characters. Values are strings + with a maximum length of 512 characters. - - `unresponsive_reward_error: boolean` + - `method: optional object { type, dpo, reinforcement, supervised }` - - `execution_time: number` + The method used for fine-tuning. - - `name: string` + - `type: "supervised" or "dpo" or "reinforcement"` - - `sampled_model_name: string` + The type of method. Is either `supervised`, `dpo`, or `reinforcement`. - - `scores: map[unknown]` + - `"supervised"` - - `token_usage: number` + - `"dpo"` - - `type: string` + - `"reinforcement"` -- `model_grader_token_usage_per_model: map[unknown]` + - `dpo: optional DpoMethod` -- `reward: number` + Configuration for the DPO fine-tuning method. -- `sub_rewards: map[unknown]` + - `hyperparameters: optional DpoHyperparameters` -### Example + The hyperparameters used for the DPO fine-tuning job. -```http -curl https://api.openai.com/v1/fine_tuning/alpha/graders/run \ - -H 'Content-Type: application/json' \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "grader": { - "input": "input", - "name": "name", - "operation": "eq", - "reference": "reference", - "type": "string_check" - }, - "model_sample": "model_sample" - }' -``` + - `batch_size: optional "auto" or number` -#### Response + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. -```json -{ - "metadata": { - "errors": { - "formula_parse_error": true, - "invalid_variable_error": true, - "model_grader_parse_error": true, - "model_grader_refusal_error": true, - "model_grader_server_error": true, - "model_grader_server_error_details": "model_grader_server_error_details", - "other_error": true, - "python_grader_runtime_error": true, - "python_grader_runtime_error_details": "python_grader_runtime_error_details", - "python_grader_server_error": true, - "python_grader_server_error_type": "python_grader_server_error_type", - "sample_parse_error": true, - "truncated_observation_error": true, - "unresponsive_reward_error": true - }, - "execution_time": 0, - "name": "name", - "sampled_model_name": "sampled_model_name", - "scores": { - "foo": "bar" - }, - "token_usage": 0, - "type": "type" - }, - "model_grader_token_usage_per_model": { - "foo": "bar" - }, - "reward": 0, - "sub_rewards": { - "foo": "bar" - } -} -``` + - `"auto"` -### Score text alignment + - `"auto"` -```http -curl -X POST https://api.openai.com/v1/fine_tuning/alpha/graders/run \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "grader": { - "type": "score_model", - "name": "Example score model grader", - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Score how close the reference answer is to the model answer on a 0-1 scale. Return only the score.\n\nReference answer: {{item.reference_answer}}\n\nModel answer: {{sample.output_text}}" - } - ] - } - ], - "model": "gpt-5-mini", - "sampling_params": { - "temperature": 1, - "top_p": 1, - "seed": 42 - } - }, - "item": { - "reference_answer": "fuzzy wuzzy was a bear" - }, - "model_sample": "fuzzy wuzzy was a bear" - }' -``` + - `number` -#### Response + - `beta: optional "auto" or number` -```json -{ - "reward": 1.0, - "metadata": { - "name": "Example score model grader", - "type": "score_model", - "errors": { - "formula_parse_error": false, - "sample_parse_error": false, - "truncated_observation_error": false, - "unresponsive_reward_error": false, - "invalid_variable_error": false, - "other_error": false, - "python_grader_server_error": false, - "python_grader_server_error_type": null, - "python_grader_runtime_error": false, - "python_grader_runtime_error_details": null, - "model_grader_server_error": false, - "model_grader_refusal_error": false, - "model_grader_parse_error": false, - "model_grader_server_error_details": null - }, - "execution_time": 4.365238428115845, - "scores": {}, - "token_usage": { - "prompt_tokens": 190, - "total_tokens": 324, - "completion_tokens": 134, - "cached_tokens": 0 - }, - "sampled_model_name": "gpt-4o-2024-08-06" - }, - "sub_rewards": {}, - "model_grader_token_usage_per_model": { - "gpt-4o-2024-08-06": { - "prompt_tokens": 190, - "total_tokens": 324, - "completion_tokens": 134, - "cached_tokens": 0 - } - } -} -``` + The beta value for the DPO method. A higher beta value will increase the weight of the penalty between the policy and reference model. -### Score an image caption + - `"auto"` -```http -curl -X POST https://api.openai.com/v1/fine_tuning/alpha/graders/run \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "grader": { - "type": "score_model", - "name": "Image caption grader", - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Score how well the provided caption matches the image on a 0-1 scale. Only return the score.\n\nCaption: {{sample.output_text}}" - }, - { - "type": "input_image", - "image_url": "https://example.com/dog-catching-ball.png", - "file_id": null, - "detail": "high" - } - ] - } - ], - "model": "gpt-5-mini", - "sampling_params": { - "temperature": 0.2 - } - }, - "item": { - "expected_caption": "A golden retriever jumps to catch a tennis ball" - }, - "model_sample": "A dog leaps to grab a tennis ball mid-air" - }' -``` + - `"auto"` -### Score an audio response + - `number` -```http -curl -X POST https://api.openai.com/v1/fine_tuning/alpha/graders/run \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "grader": { - "type": "score_model", - "name": "Audio clarity grader", - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Listen to the clip and return a confidence score from 0 to 1 that the speaker said: {{item.target_phrase}}" - }, - { - "type": "input_audio", - "input_audio": { - "data": "{{item.audio_clip_b64}}", - "format": "mp3" - } - } - ] - } - ], - "model": "gpt-audio", - "sampling_params": { - "temperature": 0.2, - "top_p": 1, - "seed": 123 - } - }, - "item": { - "target_phrase": "Please deliver the package on Tuesday", - "audio_clip_b64": "" - }, - "model_sample": "Please deliver the package on Tuesday" - }' -``` + - `learning_rate_multiplier: optional "auto" or number` -## Validate grader + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. -**post** `/fine_tuning/alpha/graders/validate` + - `"auto"` -Validate a grader. + - `"auto"` -### Body Parameters + - `number` -- `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` + - `n_epochs: optional "auto" or number` + + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + + - `"auto"` + + - `"auto"` + + - `number` + + - `reinforcement: optional ReinforcementMethod` + + Configuration for the reinforcement fine-tuning method. + + - `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` The grader used for the fine-tuning job. @@ -10791,607 +10622,624 @@ Validate a grader. - `"multi"` -### Returns - -- `grader: optional StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` - - The grader used for the fine-tuning job. - - - `StringCheckGrader object { input, name, operation, 2 more }` + - `hyperparameters: optional ReinforcementHyperparameters` - A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. + The hyperparameters used for the reinforcement fine-tuning job. - - `input: string` + - `batch_size: optional "auto" or number` - The input text. This may include template strings. + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - `name: string` + - `"auto"` - The name of the grader. + - `"auto"` - - `operation: "eq" or "ne" or "like" or "ilike"` + - `number` - The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`. + - `compute_multiplier: optional "auto" or number` - - `"eq"` + Multiplier on amount of compute used for exploring search space during training. - - `"ne"` + - `"auto"` - - `"like"` + - `"auto"` - - `"ilike"` + - `number` - - `reference: string` + - `eval_interval: optional "auto" or number` - The reference text. This may include template strings. + The number of training steps between evaluation runs. - - `type: "string_check"` + - `"auto"` - The object type, which is always `string_check`. + - `"auto"` - - `"string_check"` + - `number` - - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` + - `eval_samples: optional "auto" or number` - A TextSimilarityGrader object which grades text based on similarity metrics. + Number of evaluation samples to generate per training step. - - `evaluation_metric: "cosine" or "fuzzy_match" or "bleu" or 8 more` + - `"auto"` - The evaluation metric to use. One of `cosine`, `fuzzy_match`, `bleu`, - `gleu`, `meteor`, `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, - or `rouge_l`. + - `"auto"` - - `"cosine"` + - `number` - - `"fuzzy_match"` + - `learning_rate_multiplier: optional "auto" or number` - - `"bleu"` + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - `"gleu"` + - `"auto"` - - `"meteor"` + - `"auto"` - - `"rouge_1"` + - `number` - - `"rouge_2"` + - `n_epochs: optional "auto" or number` - - `"rouge_3"` + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - `"rouge_4"` + - `"auto"` - - `"rouge_5"` + - `"auto"` - - `"rouge_l"` + - `number` - - `input: string` + - `reasoning_effort: optional "default" or "low" or "medium" or "high"` - The text being graded. + Level of reasoning effort. - - `name: string` + - `"default"` - The name of the grader. + - `"low"` - - `reference: string` + - `"medium"` - The text being graded against. + - `"high"` - - `type: "text_similarity"` + - `supervised: optional SupervisedMethod` - The type of grader. + Configuration for the supervised fine-tuning method. - - `"text_similarity"` + - `hyperparameters: optional SupervisedHyperparameters` - - `PythonGrader object { name, source, type, image_tag }` + The hyperparameters used for the fine-tuning job. - A PythonGrader object that runs a python script on the input. + - `batch_size: optional "auto" or number` - - `name: string` + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - The name of the grader. + - `"auto"` - - `source: string` + - `"auto"` - The source code of the python script. + - `number` - - `type: "python"` + - `learning_rate_multiplier: optional "auto" or number` - The object type, which is always `python`. + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - `"python"` + - `"auto"` - - `image_tag: optional string` + - `"auto"` - The image tag to use for the python script. + - `number` - - `ScoreModelGrader object { input, model, name, 3 more }` + - `n_epochs: optional "auto" or number` - A ScoreModelGrader object that uses a model to assign a score to the input. + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - `input: array of object { content, role, type }` + - `"auto"` - The input messages evaluated by the grader. Supports text, output text, input image, and input audio content blocks, and may include template strings. + - `"auto"` - - `content: string or ResponseInputText or object { text, type } or 3 more` + - `number` - Inputs to the model - can contain template strings. Supports text, output text, input images, and input audio, either as a single item or an array of items. +### Fine Tuning Job Event - - `TextInput = string` +- `FineTuningJobEvent object { id, created_at, level, 4 more }` - A text input to the model. + Fine-tuning job event object - - `ResponseInputText object { text, type, prompt_cache_breakpoint }` + - `id: string` - A text input to the model. + The object identifier. - - `text: string` + - `created_at: number` - The text input to the model. + The Unix timestamp (in seconds) for when the fine-tuning job was created. - - `type: "input_text"` + - `level: "info" or "warn" or "error"` - The type of the input item. Always `input_text`. + The log level of the event. - - `"input_text"` + - `"info"` - - `prompt_cache_breakpoint: optional object { mode }` + - `"warn"` - Marks the exact end of a reusable prompt prefix. The breakpoint inherits its TTL from the request's `prompt_cache_options.ttl`; the boundary is not rounded to a token block. + - `"error"` - - `mode: "explicit"` + - `message: string` - The breakpoint mode. Always `explicit`. + The message of the event. - - `"explicit"` + - `object: "fine_tuning.job.event"` - - `OutputText object { text, type }` + The object type, which is always "fine_tuning.job.event". - A text output from the model. + - `"fine_tuning.job.event"` - - `text: string` + - `data: optional unknown` - The text output from the model. + The data associated with the event. - - `type: "output_text"` + - `type: optional "message" or "metrics"` - The type of the output text. Always `output_text`. + The type of event. - - `"output_text"` + - `"message"` - - `InputImage object { image_url, type, detail }` + - `"metrics"` - An image input block used within EvalItem content arrays. +### Fine Tuning Job Wandb Integration - - `image_url: string` +- `FineTuningJobWandbIntegration object { project, entity, name, tags }` - The URL of the image input. + The settings for your integration with Weights and Biases. This payload specifies the project that + metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags + to your run, and set a default entity (team, username, etc) to be associated with your run. - - `type: "input_image"` + - `project: string` - The type of the image input. Always `input_image`. + The name of the project that the new run will be created under. - - `"input_image"` + - `entity: optional string` - - `detail: optional string` + The entity to use for the run. This allows you to set the team or username of the WandB user that you would + like associated with the run. If not set, the default entity for the registered WandB API key is used. - The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. + - `name: optional string` - - `ResponseInputAudio object { input_audio, type }` + A display name to set for the run. If not set, we will use the Job ID as the name. - An audio input to the model. + - `tags: optional array of string` - - `input_audio: object { data, format }` + A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some + default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". - - `data: string` +### Fine Tuning Job Wandb Integration Object - Base64-encoded audio data. +- `FineTuningJobWandbIntegrationObject object { type, wandb }` - - `format: "mp3" or "wav"` + - `type: "wandb"` - The format of the audio data. Currently supported formats are `mp3` and - `wav`. + The type of the integration being enabled for the fine-tuning job - - `"mp3"` + - `"wandb"` - - `"wav"` + - `wandb: FineTuningJobWandbIntegration` - - `type: "input_audio"` + The settings for your integration with Weights and Biases. This payload specifies the project that + metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags + to your run, and set a default entity (team, username, etc) to be associated with your run. - The type of the input item. Always `input_audio`. + - `project: string` - - `"input_audio"` + The name of the project that the new run will be created under. - - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` + - `entity: optional string` - A list of inputs, each of which may be either an input text, output text, input - image, or input audio object. + The entity to use for the run. This allows you to set the team or username of the WandB user that you would + like associated with the run. If not set, the default entity for the registered WandB API key is used. - - `TextInput = string` + - `name: optional string` - A text input to the model. + A display name to set for the run. If not set, we will use the Job ID as the name. - - `ResponseInputText object { text, type, prompt_cache_breakpoint }` + - `tags: optional array of string` - A text input to the model. + A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some + default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". - - `OutputText object { text, type }` +# Checkpoints - A text output from the model. +## List fine-tuning checkpoints - - `text: string` +**get** `/fine_tuning/jobs/{fine_tuning_job_id}/checkpoints` - The text output from the model. +List checkpoints for a fine-tuning job. - - `type: "output_text"` +### Path Parameters - The type of the output text. Always `output_text`. +- `fine_tuning_job_id: string` - - `"output_text"` +### Query Parameters - - `InputImage object { image_url, type, detail }` +- `after: optional string` - An image input block used within EvalItem content arrays. + Identifier for the last checkpoint ID from the previous pagination request. - - `image_url: string` +- `limit: optional number` - The URL of the image input. + Number of checkpoints to retrieve. - - `type: "input_image"` +### Returns - The type of the image input. Always `input_image`. +- `data: array of FineTuningJobCheckpoint` - - `"input_image"` + - `id: string` - - `detail: optional string` + The checkpoint identifier, which can be referenced in the API endpoints. - The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. + - `created_at: number` - - `ResponseInputAudio object { input_audio, type }` + The Unix timestamp (in seconds) for when the checkpoint was created. - An audio input to the model. + - `fine_tuned_model_checkpoint: string` - - `role: "user" or "assistant" or "system" or "developer"` + The name of the fine-tuned checkpoint model that is created. - The role of the message input. One of `user`, `assistant`, `system`, or - `developer`. + - `fine_tuning_job_id: string` - - `"user"` + The name of the fine-tuning job that this checkpoint was created from. - - `"assistant"` + - `metrics: object { full_valid_loss, full_valid_mean_token_accuracy, step, 4 more }` - - `"system"` + Metrics at the step number during the fine-tuning job. - - `"developer"` + - `full_valid_loss: optional number` - - `type: optional "message"` + - `full_valid_mean_token_accuracy: optional number` - The type of the message input. Always `message`. + - `step: optional number` - - `"message"` + - `train_loss: optional number` - - `model: string` + - `train_mean_token_accuracy: optional number` - The model to use for the evaluation. + - `valid_loss: optional number` - - `name: string` + - `valid_mean_token_accuracy: optional number` - The name of the grader. + - `object: "fine_tuning.job.checkpoint"` - - `type: "score_model"` + The object type, which is always "fine_tuning.job.checkpoint". - The object type, which is always `score_model`. + - `"fine_tuning.job.checkpoint"` - - `"score_model"` + - `step_number: number` - - `range: optional array of number` + The step number that the checkpoint was created at. - The range of the score. Defaults to `[0, 1]`. +- `has_more: boolean` - - `sampling_params: optional object { max_completions_tokens, reasoning_effort, seed, 2 more }` +- `object: "list"` - The sampling parameters for the model. + - `"list"` - - `max_completions_tokens: optional number` +- `first_id: optional string` - The maximum number of tokens the grader model may generate in its response. +- `last_id: optional string` - - `reasoning_effort: optional ReasoningEffort` +### Example - Constrains effort on reasoning for reasoning models. Currently supported - values are `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, and `max`. - Reducing reasoning effort can result in faster responses and fewer tokens - used on reasoning in a response. Not all reasoning models support every - value. See the - [reasoning guide](https://platform.openai.com/docs/guides/reasoning) - for model-specific support. +```http +curl https://api.openai.com/v1/fine_tuning/jobs/$FINE_TUNING_JOB_ID/checkpoints \ + -H "Authorization: Bearer $OPENAI_API_KEY" +``` - - `"none"` +#### Response - - `"minimal"` +```json +{ + "data": [ + { + "id": "id", + "created_at": 0, + "fine_tuned_model_checkpoint": "fine_tuned_model_checkpoint", + "fine_tuning_job_id": "fine_tuning_job_id", + "metrics": { + "full_valid_loss": 0, + "full_valid_mean_token_accuracy": 0, + "step": 0, + "train_loss": 0, + "train_mean_token_accuracy": 0, + "valid_loss": 0, + "valid_mean_token_accuracy": 0 + }, + "object": "fine_tuning.job.checkpoint", + "step_number": 0 + } + ], + "has_more": true, + "object": "list", + "first_id": "first_id", + "last_id": "last_id" +} +``` - - `"low"` +### Example - - `"medium"` +```http +curl https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/checkpoints \ + -H "Authorization: Bearer $OPENAI_API_KEY" +``` - - `"high"` +#### Response - - `"xhigh"` +```json +{ + "object": "list", + "data": [ + { + "object": "fine_tuning.job.checkpoint", + "id": "ftckpt_zc4Q7MP6XxulcVzj4MZdwsAB", + "created_at": 1721764867, + "fine_tuned_model_checkpoint": "ft:gpt-4o-mini-2024-07-18:my-org:custom-suffix:96olL566:ckpt-step-2000", + "metrics": { + "full_valid_loss": 0.134, + "full_valid_mean_token_accuracy": 0.874 + }, + "fine_tuning_job_id": "ftjob-abc123", + "step_number": 2000 + }, + { + "object": "fine_tuning.job.checkpoint", + "id": "ftckpt_enQCFmOTGj3syEpYVhBRLTSy", + "created_at": 1721764800, + "fine_tuned_model_checkpoint": "ft:gpt-4o-mini-2024-07-18:my-org:custom-suffix:7q8mpxmy:ckpt-step-1000", + "metrics": { + "full_valid_loss": 0.167, + "full_valid_mean_token_accuracy": 0.781 + }, + "fine_tuning_job_id": "ftjob-abc123", + "step_number": 1000 + } + ], + "first_id": "ftckpt_zc4Q7MP6XxulcVzj4MZdwsAB", + "last_id": "ftckpt_enQCFmOTGj3syEpYVhBRLTSy", + "has_more": true +} +``` - - `"max"` +## Domain Types - - `seed: optional number` +### Fine Tuning Job Checkpoint - A seed value to initialize the randomness, during sampling. +- `FineTuningJobCheckpoint object { id, created_at, fine_tuned_model_checkpoint, 4 more }` - - `temperature: optional number` + The `fine_tuning.job.checkpoint` object represents a model checkpoint for a fine-tuning job that is ready to use. - A higher temperature increases randomness in the outputs. + - `id: string` - - `top_p: optional number` + The checkpoint identifier, which can be referenced in the API endpoints. - An alternative to temperature for nucleus sampling; 1.0 includes all tokens. + - `created_at: number` - - `MultiGrader object { calculate_output, graders, name, type }` + The Unix timestamp (in seconds) for when the checkpoint was created. - A MultiGrader object combines the output of multiple graders to produce a single score. + - `fine_tuned_model_checkpoint: string` - - `calculate_output: string` + The name of the fine-tuned checkpoint model that is created. - A formula to calculate the output based on grader results. + - `fine_tuning_job_id: string` - - `graders: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` + The name of the fine-tuning job that this checkpoint was created from. - A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. + - `metrics: object { full_valid_loss, full_valid_mean_token_accuracy, step, 4 more }` - - `StringCheckGrader object { input, name, operation, 2 more }` + Metrics at the step number during the fine-tuning job. - A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. + - `full_valid_loss: optional number` - - `TextSimilarityGrader object { evaluation_metric, input, name, 2 more }` + - `full_valid_mean_token_accuracy: optional number` - A TextSimilarityGrader object which grades text based on similarity metrics. + - `step: optional number` - - `PythonGrader object { name, source, type, image_tag }` + - `train_loss: optional number` - A PythonGrader object that runs a python script on the input. + - `train_mean_token_accuracy: optional number` - - `ScoreModelGrader object { input, model, name, 3 more }` + - `valid_loss: optional number` - A ScoreModelGrader object that uses a model to assign a score to the input. + - `valid_mean_token_accuracy: optional number` - - `LabelModelGrader object { input, labels, model, 3 more }` + - `object: "fine_tuning.job.checkpoint"` - A LabelModelGrader object which uses a model to assign labels to each item - in the evaluation. + The object type, which is always "fine_tuning.job.checkpoint". - - `input: array of object { content, role, type }` + - `"fine_tuning.job.checkpoint"` - - `content: string or ResponseInputText or object { text, type } or 3 more` + - `step_number: number` - Inputs to the model - can contain template strings. Supports text, output text, input images, and input audio, either as a single item or an array of items. + The step number that the checkpoint was created at. - - `TextInput = string` +# Methods - A text input to the model. +## Domain Types - - `ResponseInputText object { text, type, prompt_cache_breakpoint }` +### Dpo Hyperparameters - A text input to the model. +- `DpoHyperparameters object { batch_size, beta, learning_rate_multiplier, n_epochs }` - - `OutputText object { text, type }` + The hyperparameters used for the DPO fine-tuning job. - A text output from the model. + - `batch_size: optional "auto" or number` - - `text: string` + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - The text output from the model. + - `"auto"` - - `type: "output_text"` + - `"auto"` - The type of the output text. Always `output_text`. + - `number` - - `"output_text"` + - `beta: optional "auto" or number` - - `InputImage object { image_url, type, detail }` + The beta value for the DPO method. A higher beta value will increase the weight of the penalty between the policy and reference model. - An image input block used within EvalItem content arrays. + - `"auto"` - - `image_url: string` + - `"auto"` - The URL of the image input. + - `number` - - `type: "input_image"` + - `learning_rate_multiplier: optional "auto" or number` - The type of the image input. Always `input_image`. + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - `"input_image"` + - `"auto"` - - `detail: optional string` + - `"auto"` - The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. + - `number` - - `ResponseInputAudio object { input_audio, type }` + - `n_epochs: optional "auto" or number` - An audio input to the model. + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - `GraderInputs = array of string or ResponseInputText or object { text, type } or 2 more` + - `"auto"` - A list of inputs, each of which may be either an input text, output text, input - image, or input audio object. + - `"auto"` - - `role: "user" or "assistant" or "system" or "developer"` + - `number` - The role of the message input. One of `user`, `assistant`, `system`, or - `developer`. +### Dpo Method - - `"user"` +- `DpoMethod object { hyperparameters }` - - `"assistant"` + Configuration for the DPO fine-tuning method. - - `"system"` + - `hyperparameters: optional DpoHyperparameters` - - `"developer"` + The hyperparameters used for the DPO fine-tuning job. - - `type: optional "message"` + - `batch_size: optional "auto" or number` - The type of the message input. Always `message`. + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. - - `"message"` + - `"auto"` - - `labels: array of string` + - `"auto"` - The labels to assign to each item in the evaluation. + - `number` - - `model: string` + - `beta: optional "auto" or number` - The model to use for the evaluation. Must support structured outputs. + The beta value for the DPO method. A higher beta value will increase the weight of the penalty between the policy and reference model. - - `name: string` + - `"auto"` - The name of the grader. + - `"auto"` - - `passing_labels: array of string` + - `number` - The labels that indicate a passing result. Must be a subset of labels. + - `learning_rate_multiplier: optional "auto" or number` - - `type: "label_model"` + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - The object type, which is always `label_model`. + - `"auto"` - - `"label_model"` + - `"auto"` - - `name: string` + - `number` - The name of the grader. + - `n_epochs: optional "auto" or number` - - `type: "multi"` + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - The object type, which is always `multi`. + - `"auto"` - - `"multi"` + - `"auto"` -### Example + - `number` -```http -curl https://api.openai.com/v1/fine_tuning/alpha/graders/validate \ - -H 'Content-Type: application/json' \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -d '{ - "grader": { - "input": "input", - "name": "name", - "operation": "eq", - "reference": "reference", - "type": "string_check" - } - }' -``` +### Reinforcement Hyperparameters -#### Response +- `ReinforcementHyperparameters object { batch_size, compute_multiplier, eval_interval, 4 more }` -```json -{ - "grader": { - "input": "input", - "name": "name", - "operation": "eq", - "reference": "reference", - "type": "string_check" - } -} -``` + The hyperparameters used for the reinforcement fine-tuning job. -### Example + - `batch_size: optional "auto" or number` -```http -curl https://api.openai.com/v1/fine_tuning/alpha/graders/validate \ - -H "Authorization: Bearer $OPENAI_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "grader": { - "type": "string_check", - "name": "Example string check grader", - "input": "{{sample.output_text}}", - "reference": "{{item.label}}", - "operation": "eq" - } - }' -``` + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. -#### Response + - `"auto"` -```json -{ - "grader": { - "type": "string_check", - "name": "Example string check grader", - "input": "{{sample.output_text}}", - "reference": "{{item.label}}", - "operation": "eq" - } -} -``` + - `"auto"` -## Domain Types + - `number` -### Grader Run Response + - `compute_multiplier: optional "auto" or number` -- `GraderRunResponse object { metadata, model_grader_token_usage_per_model, reward, sub_rewards }` + Multiplier on amount of compute used for exploring search space during training. - - `metadata: object { errors, execution_time, name, 4 more }` + - `"auto"` - - `errors: object { formula_parse_error, invalid_variable_error, model_grader_parse_error, 11 more }` + - `"auto"` - - `formula_parse_error: boolean` + - `number` - - `invalid_variable_error: boolean` + - `eval_interval: optional "auto" or number` - - `model_grader_parse_error: boolean` + The number of training steps between evaluation runs. - - `model_grader_refusal_error: boolean` + - `"auto"` - - `model_grader_server_error: boolean` + - `"auto"` - - `model_grader_server_error_details: string` + - `number` - - `other_error: boolean` + - `eval_samples: optional "auto" or number` - - `python_grader_runtime_error: boolean` + Number of evaluation samples to generate per training step. - - `python_grader_runtime_error_details: string` + - `"auto"` - - `python_grader_server_error: boolean` + - `"auto"` - - `python_grader_server_error_type: string` + - `number` - - `sample_parse_error: boolean` + - `learning_rate_multiplier: optional "auto" or number` - - `truncated_observation_error: boolean` + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. - - `unresponsive_reward_error: boolean` + - `"auto"` - - `execution_time: number` + - `"auto"` - - `name: string` + - `number` - - `sampled_model_name: string` + - `n_epochs: optional "auto" or number` - - `scores: map[unknown]` + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - - `token_usage: number` + - `"auto"` - - `type: string` + - `"auto"` - - `model_grader_token_usage_per_model: map[unknown]` + - `number` - - `reward: number` + - `reasoning_effort: optional "default" or "low" or "medium" or "high"` - - `sub_rewards: map[unknown]` + Level of reasoning effort. -### Grader Validate Response + - `"default"` -- `GraderValidateResponse object { grader }` + - `"low"` - - `grader: optional StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` + - `"medium"` + + - `"high"` + +### Reinforcement Method + +- `ReinforcementMethod object { grader, hyperparameters }` + + Configuration for the reinforcement fine-tuning method. + + - `grader: StringCheckGrader or TextSimilarityGrader or PythonGrader or 2 more` The grader used for the fine-tuning job. @@ -11866,3 +11714,155 @@ curl https://api.openai.com/v1/fine_tuning/alpha/graders/validate \ The object type, which is always `multi`. - `"multi"` + + - `hyperparameters: optional ReinforcementHyperparameters` + + The hyperparameters used for the reinforcement fine-tuning job. + + - `batch_size: optional "auto" or number` + + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. + + - `"auto"` + + - `"auto"` + + - `number` + + - `compute_multiplier: optional "auto" or number` + + Multiplier on amount of compute used for exploring search space during training. + + - `"auto"` + + - `"auto"` + + - `number` + + - `eval_interval: optional "auto" or number` + + The number of training steps between evaluation runs. + + - `"auto"` + + - `"auto"` + + - `number` + + - `eval_samples: optional "auto" or number` + + Number of evaluation samples to generate per training step. + + - `"auto"` + + - `"auto"` + + - `number` + + - `learning_rate_multiplier: optional "auto" or number` + + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. + + - `"auto"` + + - `"auto"` + + - `number` + + - `n_epochs: optional "auto" or number` + + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + + - `"auto"` + + - `"auto"` + + - `number` + + - `reasoning_effort: optional "default" or "low" or "medium" or "high"` + + Level of reasoning effort. + + - `"default"` + + - `"low"` + + - `"medium"` + + - `"high"` + +### Supervised Hyperparameters + +- `SupervisedHyperparameters object { batch_size, learning_rate_multiplier, n_epochs }` + + The hyperparameters used for the fine-tuning job. + + - `batch_size: optional "auto" or number` + + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. + + - `"auto"` + + - `"auto"` + + - `number` + + - `learning_rate_multiplier: optional "auto" or number` + + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. + + - `"auto"` + + - `"auto"` + + - `number` + + - `n_epochs: optional "auto" or number` + + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + + - `"auto"` + + - `"auto"` + + - `number` + +### Supervised Method + +- `SupervisedMethod object { hyperparameters }` + + Configuration for the supervised fine-tuning method. + + - `hyperparameters: optional SupervisedHyperparameters` + + The hyperparameters used for the fine-tuning job. + + - `batch_size: optional "auto" or number` + + Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. + + - `"auto"` + + - `"auto"` + + - `number` + + - `learning_rate_multiplier: optional "auto" or number` + + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. + + - `"auto"` + + - `"auto"` + + - `number` + + - `n_epochs: optional "auto" or number` + + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + + - `"auto"` + + - `"auto"` + + - `number`