Documentation 2026-10-30 23:59 UTC to 2026-10-31 23:01 UTC

120ws.send(json.dumps(event))120ws.send(json.dumps(event))

121```121```

122 122 

123```go

124// Replace pmpt_123 and 89 with an existing saved prompt and version.

125// The saved prompt must define the city variable.

126if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfSessionUpdate(realtime.RealtimeSessionCreateRequestParam{

127 Model: "gpt-realtime-2.1",

128 Prompt: responses.ResponsePromptParam{

129 ID: "pmpt_123",

130 Version: openai.String("89"),

131 Variables: map[string]responses.ResponsePromptVariableUnionParam{

132 "city": {

133 OfString: openai.String("Paris"),

134 },

135 },

136 },

137 OutputModalities: []string{"audio"},

138 Instructions: openai.String("Speak clearly and briefly. Confirm before taking action."),

139 Audio: realtime.RealtimeAudioConfigParam{

140 Input: realtime.RealtimeAudioConfigInputParam{

141 Format: realtime.RealtimeAudioFormatsUnionParam{

142 OfAudioPCM: &realtime.RealtimeAudioFormatsAudioPCMParam{

143 Rate: 24000,

144 },

145 },

146 TurnDetection: realtime.RealtimeAudioInputTurnDetectionUnionParam{

147 OfSemanticVad: &realtime.RealtimeAudioInputTurnDetectionSemanticVadParam{},

148 },

149 },

150 Output: realtime.RealtimeAudioConfigOutputParam{

151 Format: realtime.RealtimeAudioFormatsUnionParam{

152 OfAudioPCM: &realtime.RealtimeAudioFormatsAudioPCMParam{

153 Rate: 24000,

154 },

155 },

156 Voice: realtime.RealtimeAudioConfigOutputVoiceUnionParam{

157 OfString: openai.String("marin"),

158 },

159 },

160 },

161})); err != nil {

162 return err

163}

164```

165 

123```java166```java

124import com.openai.client.okhttp.OkHttpClient;167import com.openai.client.okhttp.OkHttpClient;

125import com.openai.core.ClientOptions;168import com.openai.core.ClientOptions;

267ws.send(json.dumps(event))310ws.send(json.dumps(event))

268```311```

269 312 

313```go

314if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfConversationItemCreate(realtime.RealtimeConversationItemUserMessageParam{

315 Content: []realtime.RealtimeConversationItemUserMessageContentParam{

316 {

317 Text: openai.String("What Prince album sold the most copies?"),

318 Type: "input_text",

319 },

320 },

321})); err != nil {

322 return err

323}

324```

325 

270```java326```java

271import com.openai.client.okhttp.OkHttpClient;327import com.openai.client.okhttp.OkHttpClient;

272import com.openai.core.ClientOptions;328import com.openai.core.ClientOptions;

341ws.send(json.dumps(event))397ws.send(json.dumps(event))

342```398```

343 399 

400```go

401if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{

402 OfResponseCreate: &realtime.ResponseCreateEventParam{

403 Response: realtime.RealtimeResponseCreateParams{

404 OutputModalities: []string{"text"},

405 },

406 },

407}); err != nil {

408 return err

409}

410```

411 

344```java412```java

345import com.openai.client.okhttp.OkHttpClient;413import com.openai.client.okhttp.OkHttpClient;

346import com.openai.core.ClientOptions;414import com.openai.core.ClientOptions;

420 print(server_event["response"]["output"][0])488 print(server_event["response"]["output"][0])

421```489```

422 490 

491```go

492func receiveUntil(ctx context.Context, conn *realtime.RealtimeConnection, terminal string) (realtime.RealtimeServerEventUnion, error) {

493 for {

494 event, err := conn.Recv(ctx)

495 if err != nil {

496 return event, fmt.Errorf("waiting for %s: %w", terminal, err)

497 }

498 if event.Type == "error" {

499 return event, fmt.Errorf("Realtime API error: %s", event.OfRealtimeErrorEvent.Error.Message)

500 }

501 if event.Type == "response.done" && event.OfResponseDoneEvent.Response.Status != "completed" {

502 return event, fmt.Errorf("response ended with status %s", event.OfResponseDoneEvent.Response.Status)

503 }

504 if event.Type == "response.done" {

505 response := event.OfResponseDoneEvent.Response

506 fmt.Println("Response status:", response.Status)

507 for _, output := range response.Output {

508 item := output.OfRealtimeConversationItemAssistantMessage

509 if item.Type == "message" {

510 for _, content := range item.Content {

511 if content.Type == "output_text" {

512 fmt.Print(content.Text)

513 }

514 }

515 }

516 }

517 }

518 if event.Type == terminal {

519 return event, nil

520 }

521 }

522}

523```

524 

423```java525```java

424import com.openai.client.okhttp.OkHttpClient;526import com.openai.client.okhttp.OkHttpClient;

425import com.openai.core.ClientOptions;527import com.openai.core.ClientOptions;

797 ws.send(json.dumps(event))899 ws.send(json.dumps(event))

798```900```

799 901 

902```go

903// audio contains mono PCM16 samples at 24 kHz.

904for start := 0; start < len(audio); start += 9600 {

905 end := min(start+9600, len(audio))

906 if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfInputAudioBufferAppend(base64.StdEncoding.EncodeToString(audio[start:end]))); err != nil {

907 return err

908 }

909}

910```

911 

800```java912```java

801import com.openai.client.okhttp.OkHttpClient;913import com.openai.client.okhttp.OkHttpClient;

802import com.openai.core.ClientOptions;914import com.openai.core.ClientOptions;

898ws.send(json.dumps(event))1010ws.send(json.dumps(event))

899```1011```

900 1012 

1013```go

1014if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfConversationItemCreate(realtime.RealtimeConversationItemUserMessageParam{

1015 Content: []realtime.RealtimeConversationItemUserMessageContentParam{

1016 {

1017 Audio: openai.String(base64.StdEncoding.EncodeToString(audio)),

1018 Type: "input_audio",

1019 },

1020 },

1021})); err != nil {

1022 return err

1023}

1024```

1025 

901```java1026```java

902import com.openai.client.okhttp.OkHttpClient;1027import com.openai.client.okhttp.OkHttpClient;

903import com.openai.core.ClientOptions;1028import com.openai.core.ClientOptions;

1011 print(server_event["delta"])1136 print(server_event["delta"])

1012```1137```

1013 1138 

1139```go

1140func receiveUntil(ctx context.Context, conn *realtime.RealtimeConnection, terminal string) (realtime.RealtimeServerEventUnion, error) {

1141 for {

1142 event, err := conn.Recv(ctx)

1143 if err != nil {

1144 return event, fmt.Errorf("waiting for %s: %w", terminal, err)

1145 }

1146 if event.Type == "response.output_audio.delta" {

1147 audio, err := base64.StdEncoding.DecodeString(event.OfResponseAudioDeltaEvent.Delta)

1148 if err != nil {

1149 return event, err

1150 }

1151 fmt.Printf("Received %d audio bytes\n", len(audio))

1152 }

1153 if event.Type == "error" {

1154 return event, fmt.Errorf("Realtime API error: %s", event.OfRealtimeErrorEvent.Error.Message)

1155 }

1156 if event.Type == "response.done" && event.OfResponseDoneEvent.Response.Status != "completed" {

1157 return event, fmt.Errorf("response ended with status %s", event.OfResponseDoneEvent.Response.Status)

1158 }

1159 if event.Type == terminal {

1160 return event, nil

1161 }

1162 }

1163}

1164```

1165 

1014```java1166```java

1015import com.openai.client.okhttp.OkHttpClient;1167import com.openai.client.okhttp.OkHttpClient;

1016import com.openai.core.ClientOptions;1168import com.openai.core.ClientOptions;

1080dataChannel.send(JSON.stringify(event));1232dataChannel.send(JSON.stringify(event));

1081```1233```

1082 1234 

1235```go

1236if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfConversationItemCreate(realtime.RealtimeConversationItemUserMessageParam{

1237 Content: []realtime.RealtimeConversationItemUserMessageContentParam{

1238 {

1239 ImageURL: openai.String("data:image/png;base64," + base64.StdEncoding.EncodeToString(image)),

1240 Type: "input_image",

1241 },

1242 {

1243 Text: openai.String("Describe this image."),

1244 Type: "input_text",

1245 },

1246 },

1247})); err != nil {

1248 return err

1249}

1250 

1251if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{

1252 OfResponseCreate: &realtime.ResponseCreateEventParam{

1253 Response: realtime.RealtimeResponseCreateParams{

1254 OutputModalities: []string{"text"},

1255 },

1256 },

1257}); err != nil {

1258 return err

1259}

1260```

1261 

1083```ruby1262```ruby

1084encoded_image = Base64.strict_encode64(File.binread("image.png"))1263encoded_image = Base64.strict_encode64(File.binread("image.png"))

1085 1264 

1191ws.send(json.dumps(event))1370ws.send(json.dumps(event))

1192```1371```

1193 1372 

1373```go

1374if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{

1375 OfResponseCreate: &realtime.ResponseCreateEventParam{

1376 Response: realtime.RealtimeResponseCreateParams{

1377 Instructions: openai.String("\nAnalyze the conversation so far. If it is related to support, output\n\"support\". If it is related to sales, output \"sales\".\n"),

1378 Metadata: shared.Metadata{

1379 "topic": "classification",

1380 },

1381 Conversation: "none",

1382 OutputModalities: []string{"text"},

1383 },

1384 },

1385}); err != nil {

1386 return err

1387}

1388```

1389 

1194```java1390```java

1195import com.openai.client.okhttp.OkHttpClient;1391import com.openai.client.okhttp.OkHttpClient;

1196import com.openai.core.ClientOptions;1392import com.openai.core.ClientOptions;

1303 print(response["output"][0])1499 print(response["output"][0])

1304```1500```

1305 1501 

1502```go

1503func receiveUntil(ctx context.Context, conn *realtime.RealtimeConnection, terminal string) (realtime.RealtimeServerEventUnion, error) {

1504 for {

1505 event, err := conn.Recv(ctx)

1506 if err != nil {

1507 return event, fmt.Errorf("waiting for %s: %w", terminal, err)

1508 }

1509 if event.Type == "error" {

1510 return event, fmt.Errorf("Realtime API error: %s", event.OfRealtimeErrorEvent.Error.Message)

1511 }

1512 if event.Type == "response.done" && event.OfResponseDoneEvent.Response.Metadata["topic"] != "classification" {

1513 continue

1514 }

1515 if event.Type == "response.done" && event.OfResponseDoneEvent.Response.Status != "completed" {

1516 return event, fmt.Errorf("response ended with status %s", event.OfResponseDoneEvent.Response.Status)

1517 }

1518 if event.Type == "response.done" {

1519 response := event.OfResponseDoneEvent.Response

1520 fmt.Println("Response status:", response.Status)

1521 for _, output := range response.Output {

1522 item := output.OfRealtimeConversationItemAssistantMessage

1523 if item.Type == "message" {

1524 for _, content := range item.Content {

1525 if content.Type == "output_text" {

1526 fmt.Print(content.Text)

1527 }

1528 }

1529 }

1530 }

1531 }

1532 if event.Type == terminal {

1533 return event, nil

1534 }

1535 }

1536}

1537```

1538 

1306```java1539```java

1307import com.openai.client.okhttp.OkHttpClient;1540import com.openai.client.okhttp.OkHttpClient;

1308import com.openai.core.ClientOptions;1541import com.openai.core.ClientOptions;

1552ws.send(json.dumps(event))1785ws.send(json.dumps(event))

1553```1786```

1554 1787 

1788```go

1789if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{

1790 OfResponseCreate: &realtime.ResponseCreateEventParam{

1791 Response: realtime.RealtimeResponseCreateParams{

1792 Instructions: openai.String("\nSay exactly the following:\nI'm a little teapot, short and stout!\nThis is my handle, this is my spout!\n"),

1793 Input: []realtime.ConversationItemUnionParam{},

1794 OutputModalities: []string{"text"},

1795 },

1796 },

1797}); err != nil {

1798 return err

1799}

1800```

1801 

1555```java1802```java

1556import com.openai.client.okhttp.OkHttpClient;1803import com.openai.client.okhttp.OkHttpClient;

1557import com.openai.core.ClientOptions;1804import com.openai.core.ClientOptions;

85ws.send(json.dumps(event))85ws.send(json.dumps(event))

86```86```

87 87 

88```go

89if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfSessionUpdate(realtime.RealtimeSessionCreateRequestParam{

90 Model: "gpt-realtime-2.1",

91 ToolChoice: realtime.RealtimeToolChoiceConfigUnionParam{

92 OfToolChoiceMode: param.NewOpt[responses.ToolChoiceOptions]("auto"),

93 },

94 Tools: realtime.RealtimeToolsConfigParam{

95 realtime.RealtimeToolsConfigUnionParam{

96 OfFunction: &realtime.RealtimeFunctionToolParam{

97 Description: openai.String("Look up an order by its order number."),

98 Name: openai.String("lookup_order"),

99 Parameters: map[string]any{

100 "properties": map[string]any{

101 "order_number": map[string]any{

102 "description": "The customer-facing order number.",

103 "type": "string",

104 },

105 },

106 "required": []any{"order_number"},

107 "type": "object",

108 },

109 Type: "function",

110 },

111 },

112 },

113})); err != nil {

114 return err

115}

116```

117 

88```java118```java

89import com.openai.client.okhttp.OkHttpClient;119import com.openai.client.okhttp.OkHttpClient;

90import com.openai.core.ClientOptions;120import com.openai.core.ClientOptions;

225ws.send(json.dumps({"type": "response.create"}))255ws.send(json.dumps({"type": "response.create"}))

226```256```

227 257 

258```go

259// callID comes from the model's function_call item.

260if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfConversationItemCreate(realtime.RealtimeConversationItemFunctionCallOutputParam{

261 CallID: callID,

262 Output: "{\"status\":\"shipped\",\"delivery_date\":\"2026-05-09\"}",

263})); err != nil {

264 return err

265}

266 

267if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{

268 OfResponseCreate: &realtime.ResponseCreateEventParam{

269 Response: realtime.RealtimeResponseCreateParams{

270 OutputModalities: []string{"text"},

271 ToolChoice: realtime.RealtimeResponseCreateParamsToolChoiceUnion{

272 OfToolChoiceMode: param.NewOpt[responses.ToolChoiceOptions]("none"),

273 },

274 },

275 },

276}); err != nil {

277 return err

278}

279```

280 

228```java281```java

229import com.openai.client.okhttp.OkHttpClient;282import com.openai.client.okhttp.OkHttpClient;

230import com.openai.core.ClientOptions;283import com.openai.core.ClientOptions;

374ws.send(json.dumps(event))427ws.send(json.dumps(event))

375```428```

376 429 

430```go

431if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfSessionUpdate(realtime.RealtimeSessionCreateRequestParam{

432 Model: "gpt-realtime-2.1",

433 OutputModalities: []string{"text"},

434 Tools: realtime.RealtimeToolsConfigParam{

435 {

436 OfMcp: &realtime.RealtimeToolsConfigUnionMcpParam{

437 ServerLabel: "openai_docs",

438 ServerURL: openai.String("https://developers.openai.com/mcp"),

439 AllowedTools: realtime.RealtimeToolsConfigUnionMcpAllowedToolsParam{

440 OfMcpAllowedTools: []string{"search_openai_docs", "fetch_openai_doc"},

441 },

442 RequireApproval: realtime.RealtimeToolsConfigUnionMcpRequireApprovalParam{

443 OfMcpToolApprovalSetting: openai.String("never"),

444 },

445 },

446 },

447 },

448})); err != nil {

449 return err

450}

451```

452 

377```java453```java

378import com.openai.client.okhttp.OkHttpClient;454import com.openai.client.okhttp.OkHttpClient;

379import com.openai.core.ClientOptions;455import com.openai.core.ClientOptions;

519ws.send(json.dumps(event))595ws.send(json.dumps(event))

520```596```

521 597 

598```go

599accessToken := os.Getenv("OPENAI_MCP_ACCESS_TOKEN")

600if accessToken == "" {

601 return fmt.Errorf("OPENAI_MCP_ACCESS_TOKEN is required")

602}

603 

604if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfSessionUpdate(realtime.RealtimeSessionCreateRequestParam{

605 Model: "gpt-realtime-1.5",

606 OutputModalities: []string{"text"},

607 Tools: realtime.RealtimeToolsConfigParam{

608 {

609 OfMcp: &realtime.RealtimeToolsConfigUnionMcpParam{

610 ServerLabel: "google_calendar",

611 ConnectorID: "connector_googlecalendar",

612 Authorization: param.NewOpt(accessToken),

613 AllowedTools: realtime.RealtimeToolsConfigUnionMcpAllowedToolsParam{

614 OfMcpAllowedTools: []string{"search_events", "read_event"},

615 },

616 RequireApproval: realtime.RealtimeToolsConfigUnionMcpRequireApprovalParam{

617 OfMcpToolApprovalSetting: openai.String("never"),

618 },

619 },

620 },

621 },

622})); err != nil {

623 return err

624}

625```

626 

522```java627```java

523import com.openai.client.okhttp.OkHttpClient;628import com.openai.client.okhttp.OkHttpClient;

524import com.openai.core.ClientOptions;629import com.openai.core.ClientOptions;

776 print("Realtime turn complete.")881 print("Realtime turn complete.")

777```882```

778 883 

884```go

885// Call for each received event. Tool results can arrive after response.done.

886func logMCPEvent(event realtime.RealtimeServerEventUnion) error {

887 switch e := event.AsAny().(type) {

888 case realtime.RealtimeErrorEvent:

889 return fmt.Errorf("Realtime API error: %s", e.Error.Message)

890 case realtime.McpListToolsFailed, realtime.ResponseMcpCallFailed:

891 return fmt.Errorf("MCP failed: %s", event.Type)

892 case realtime.ResponseOutputItemDoneEvent:

893 item := e.Item.OfRealtimeMcpToolCall

894 if item.Type == "mcp_call" {

895 if item.JSON.Error.Valid() || !item.JSON.Output.Valid() {

896 return fmt.Errorf("MCP call failed or has no output")

897 }

898 fmt.Fprintln(log.Writer(), "Tool output:", item.Output)

899 }

900 }

901 return nil

902}

903```

904 

779```ruby905```ruby

780completed_events = {}906completed_events = {}

781expected_calls = []907expected_calls = []

904 ws.send(json.dumps(event))1030 ws.send(json.dumps(event))

905```1031```

906 1032 

1033```go

1034// Call with the approval request received in conversation.item.done.

1035func approveDocumentationSearch(

1036 ctx context.Context,

1037 conn *realtime.RealtimeConnection,

1038 request realtime.RealtimeMcpApprovalRequest,

1039) error {

1040 // This example permits only a read-only search of the OpenAI documentation.

1041 if request.ID == "" || request.ServerLabel != "openai_docs" || request.Name != "search_openai_docs" {

1042 return fmt.Errorf("unapproved tool requested")

1043 }

1044 // Derive a distinct item ID within the API's 32-character limit.

1045 approvalID := sha256.Sum256([]byte(request.ID))

1046 return conn.Send(ctx, realtime.RealtimeClientEventParamOfConversationItemCreate(realtime.RealtimeMcpApprovalResponseParam{

1047 ID: fmt.Sprintf("%x", approvalID[:16]),

1048 ApprovalRequestID: request.ID,

1049 Approve: true,

1050 }))

1051}

1052```

1053 

907```java1054```java

908import com.openai.client.okhttp.OkHttpClient;1055import com.openai.client.okhttp.OkHttpClient;

909import com.openai.core.ClientOptions;1056import com.openai.core.ClientOptions;

1045ws.send(json.dumps(event))1192ws.send(json.dumps(event))

1046```1193```

1047 1194 

1195```go

1196// input contains the conversation items for this response.

1197if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{

1198 OfResponseCreate: &realtime.ResponseCreateEventParam{

1199 Response: realtime.RealtimeResponseCreateParams{

1200 ParallelToolCalls: openai.Bool(false),

1201 Input: input,

1202 OutputModalities: []string{"text"},

1203 ToolChoice: realtime.RealtimeResponseCreateParamsToolChoiceUnion{

1204 OfToolChoiceMode: param.NewOpt[responses.ToolChoiceOptions]("required"),

1205 },

1206 Tools: []realtime.RealtimeResponseCreateParamsToolUnion{

1207 {

1208 OfMcp: &realtime.RealtimeResponseCreateMcpToolParam{

1209 ServerLabel: "openai_docs",

1210 ServerURL: openai.String("https://developers.openai.com/mcp"),

1211 AllowedTools: realtime.RealtimeResponseCreateMcpToolAllowedToolsUnionParam{

1212 OfMcpAllowedTools: []string{"search_openai_docs", "fetch_openai_doc"},

1213 },

1214 RequireApproval: realtime.RealtimeResponseCreateMcpToolRequireApprovalUnionParam{

1215 OfMcpToolApprovalSetting: openai.String("never"),

1216 },

1217 },

1218 },

1219 },

1220 },

1221 },

1222}); err != nil {

1223 return err

1224}

1225```

1226 

1048```java1227```java

1049import com.openai.client.okhttp.OkHttpClient;1228import com.openai.client.okhttp.OkHttpClient;

1050import com.openai.core.ClientOptions;1229import com.openai.core.ClientOptions;

1231ws.send(json.dumps(event))1410ws.send(json.dumps(event))

1232```1411```

1233 1412 

1413```go

1414// input contains the conversation items for this response.

1415if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{

1416 OfResponseCreate: &realtime.ResponseCreateEventParam{

1417 Response: realtime.RealtimeResponseCreateParams{

1418 OutputModalities: []string{"text"},

1419 Input: input,

1420 Tools: []realtime.RealtimeResponseCreateParamsToolUnion{

1421 {

1422 OfMcp: &realtime.RealtimeResponseCreateMcpToolParam{

1423 ServerLabel: "google_calendar",

1424 },

1425 },

1426 },

1427 ToolChoice: realtime.RealtimeResponseCreateParamsToolChoiceUnion{

1428 OfToolChoiceMode: param.NewOpt(responses.ToolChoiceOptions("required")),

1429 },

1430 },

1431 },

1432}); err != nil {

1433 return err

1434}

1435```

1436 

1234```java1437```java

1235import com.openai.client.okhttp.OkHttpClient;1438import com.openai.client.okhttp.OkHttpClient;

1236import com.openai.core.ClientOptions;1439import com.openai.core.ClientOptions;

106 106 

107Ordering between completion events from different speech turns isn't guaranteed. Use `item_id` to match transcription events to committed input items.107Ordering between completion events from different speech turns isn't guaranteed. Use `item_id` to match transcription events to committed input items.

108 108 

109After configuring a transcription session, use this Go function to send mono PCM16 audio at 24 kHz, commit the turn, and match the final transcript to the committed item:

110 

111```go

112func transcribeTurn(ctx context.Context, conn *realtime.RealtimeConnection, audio []byte) error {

113 for start := 0; start < len(audio); start += 9600 {

114 end := min(start+9600, len(audio))

115 if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfInputAudioBufferAppend(base64.StdEncoding.EncodeToString(audio[start:end]))); err != nil {

116 return err

117 }

118 }

119 if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{

120 OfInputAudioBufferCommit: &realtime.InputAudioBufferCommitEventParam{},

121 }); err != nil {

122 return err

123 }

124 // Match the final transcript to the committed item, not event arrival order.

125 var itemID string

126 for {

127 event, err := conn.Recv(ctx)

128 if err != nil {

129 return err

130 }

131 switch event.Type {

132 case "input_audio_buffer.committed":

133 itemID = event.OfInputAudioBufferCommittedEvent.ItemID

134 case "conversation.item.input_audio_transcription.delta":

135 delta := event.OfConversationItemInputAudioTranscriptionDeltaEvent

136 if itemID != "" && delta.ItemID == itemID {

137 fmt.Fprintf(os.Stderr, "Partial transcript (%s): %s\n", delta.ItemID, delta.Delta)

138 }

139 case "conversation.item.input_audio_transcription.completed":

140 completed := event.OfConversationItemInputAudioTranscriptionCompletedEvent

141 if itemID == "" || completed.ItemID != itemID {

142 continue

143 }

144 fmt.Println("Final transcript:", completed.Transcript)

145 return nil

146 case "conversation.item.input_audio_transcription.failed":

147 if itemID != "" && event.OfConversationItemInputAudioTranscriptionFailedEvent.ItemID == itemID {

148 return fmt.Errorf("transcription failed")

149 }

150 case "error":

151 return fmt.Errorf("Realtime API error: %s", event.OfRealtimeErrorEvent.Error.Message)

152 }

153 }

154}

155```

156 

157 

109## Add transcription context158## Add transcription context

110 159 

111Add context when the audio contains specialized vocabulary or more than one expected language. Send another `session.update` event to change the transcription configuration during an existing session.160Add context when the audio contains specialized vocabulary or more than one expected language. Send another `session.update` event to change the transcription configuration during an existing session.

157)157)

158```158```

159 159 

160```go

161ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)

162defer cancel()

163client := openai.NewClient()

164session, err := client.Realtime.ConnectTranslation(ctx, "gpt-realtime-translate", realtime.TranslationConnectionOptions{}, option.WithHeader("OpenAI-Safety-Identifier", "hashed-user-id"))

165if err != nil {

166 return err

167}

168defer session.Abort()

169created, err := session.Recv(ctx)

170if err != nil {

171 return err

172}

173fmt.Println(created.Type)

174if created.Type != "session.created" {

175 return fmt.Errorf("expected session.created, got %s", created.Type)

176}

177```

178 

160```java179```java

161import com.openai.client.okhttp.OkHttpClient;180import com.openai.client.okhttp.OkHttpClient;

162import com.openai.core.ClientOptions;181import com.openai.core.ClientOptions;

261)280)

262```281```

263 282 

283```go

284if err := session.Send(ctx, realtime.RealtimeTranslationClientEventParamOfSessionUpdate(realtime.RealtimeTranslationSessionUpdateRequestParam{

285 Audio: realtime.RealtimeTranslationSessionUpdateRequestAudioParam{

286 Output: realtime.RealtimeTranslationSessionUpdateRequestAudioOutputParam{

287 Language: openai.String("es"),

288 },

289 },

290})); err != nil {

291 return err

292}

293```

294 

264```java295```java

265import com.openai.client.okhttp.OkHttpClient;296import com.openai.client.okhttp.OkHttpClient;

266import com.openai.core.ClientOptions;297import com.openai.core.ClientOptions;

314)345)

315```346```

316 347 

348```go

349// audio contains mono PCM16 samples at 24 kHz.

350for start := 0; start < len(audio); start += 4800 {

351 end := min(start+4800, len(audio))

352 if err := session.Send(ctx, realtime.RealtimeTranslationClientEventParamOfSessionInputAudioBufferAppend(base64.StdEncoding.EncodeToString(audio[start:end]))); err != nil {

353 return err

354 }

355}

356```

357 

317```java358```java

318import com.openai.client.okhttp.OkHttpClient;359import com.openai.client.okhttp.OkHttpClient;

319import com.openai.core.ClientOptions;360import com.openai.core.ClientOptions;

418 playback.result()459 playback.result()

419```460```

420 461 

462```go

463// Run this listener while another goroutine sends audio into the session.

464func listen(ctx context.Context, session *realtime.TranslationSession) error {

465 for {

466 event, err := session.Recv(ctx)

467 if err != nil {

468 return err

469 }

470 switch event.Type {

471 case "error":

472 return fmt.Errorf("translation error: %s", event.OfRealtimeErrorEvent.Error.Message)

473 case "session.output_transcript.delta":

474 fmt.Print(event.OfRealtimeTranslationOutputTranscriptDeltaEvent.Delta)

475 case "session.output_audio.delta":

476 pcm, err := base64.StdEncoding.DecodeString(event.OfRealtimeTranslationOutputAudioDeltaEvent.Delta)

477 if err != nil {

478 return err

479 }

480 // Feed pcm into your application's 24 kHz audio player.

481 fmt.Fprintln(log.Writer(), "Received audio bytes:", len(pcm))

482 case "session.closed":

483 return session.Close()

484 }

485 }

486}

487```

488 

489 

490In Go, run `listen` concurrently while sending audio. It closes the transport after receiving `session.closed`.

421 491 

422## Close a WebSocket session492## Close a WebSocket session

423 493 

568 playback.result()638 playback.result()

569```639```

570 640 

641```go

642// Stop sending audio first. The listener closes the transport after session.closed.

643if err := session.Send(ctx, realtime.RealtimeTranslationClientEventUnionParam{

644 OfSessionClose: &realtime.RealtimeTranslationSessionCloseEventParam{},

645}); err != nil {

646 return err

647}

648```

649 

571 650 

572## Build listen-along translation651## Build listen-along translation

573 652 

190app.listen(3000);190app.listen(3000);

191```191```

192 192 

193```go

194func handler(client openai.Client) http.Handler {

195 mux := http.NewServeMux()

196 mux.HandleFunc("POST /session", func(w http.ResponseWriter, r *http.Request) {

197 offer, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64*1024))

198 if err != nil || strings.TrimSpace(string(offer)) == "" {

199 http.Error(w, "A valid SDP offer is required", http.StatusBadRequest)

200 return

201 }

202 opts := []option.RequestOption{

203 option.WithHeader("OpenAI-Safety-Identifier", "hashed-user-id"),

204 }

205 // Forward the browser's negotiated data-channel ID when WARP is enabled.

206 if dcid := r.URL.Query().Get("dcid"); dcid != "" {

207 id, err := strconv.Atoi(dcid)

208 if err != nil || id < 0 || id > 65534 {

209 http.Error(w, "Invalid data-channel ID", 400)

210 return

211 }

212 opts = append(opts, option.WithQuery("dcid", dcid))

213 }

214 response, err := client.Realtime.Calls.New(r.Context(), realtime.CallNewParams{

215 Sdp: string(offer),

216 Session: realtime.RealtimeSessionCreateRequestParam{

217 Model: "gpt-realtime-2.1",

218 Audio: realtime.RealtimeAudioConfigParam{

219 Output: realtime.RealtimeAudioConfigOutputParam{

220 Voice: realtime.RealtimeAudioConfigOutputVoiceUnionParam{

221 OfString: openai.String("marin"),

222 },

223 },

224 },

225 },

226 }, opts...)

227 if err != nil {

228 http.Error(w, "Could not create Realtime call", http.StatusBadGateway)

229 return

230 }

231 defer response.Body.Close()

232 w.Header().Set("Content-Type", "application/sdp")

233 w.Header().Set("Location", response.Header.Get("Location"))

234 w.WriteHeader(response.StatusCode)

235 if _, err := io.Copy(w, response.Body); err != nil {

236 log.Print("Could not write SDP answer")

237 }

238 })

239 return mux

240}

241```

242 

193 243 

194### Connect from your browser244### Connect from your browser

195 245 

296```296```

297 297 

298 298 

299In Go, pass the call ID to your server-side session update function:

300 

301```go

302func updateSession(ctx context.Context, client openai.Client, callID string) error {

303 conn, err := client.Realtime.Connect(ctx, realtime.RealtimeConnectionOptions{}, option.WithQuery("call_id", callID))

304 if err != nil {

305 return err

306 }

307 defer conn.Close()

308 if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfSessionUpdate(realtime.RealtimeSessionCreateRequestParam{

309 Instructions: openai.String("Be extra nice today!"),

310 })); err != nil {

311 return err

312 }

313 for {

314 event, err := conn.Recv(ctx)

315 if err != nil {

316 return err

317 }

318 if event.Type == "error" {

319 return fmt.Errorf("Realtime API error: %s", event.OfRealtimeErrorEvent.Error.Message)

320 }

321 if event.Type == "session.updated" {

322 if event.OfSessionUpdatedEvent.Session.OfRealtimeSessionCreateRequest.Instructions != "Be extra nice today!" {

323 return fmt.Errorf("session update was not applied")

324 }

325 fmt.Println("session.updated")

326 return nil

327 }

328 }

329}

330```

331 

332 

299In this way, you are able to add tools, monitor sessions, and carry out business logic on the server instead of needing to configure those actions on the client.333In this way, you are able to add tools, monitor sessions, and carry out business logic on the server instead of needing to configure those actions on the client.

300 334 

301## With SIP335## With SIP

484 app.run(port=8000)484 app.run(port=8000)

485```485```

486 486 

487```go

488func handler(client openai.Client) http.Handler {

489 var mu sync.Mutex

490 type acceptance struct {

491 done chan struct{}

492 err error

493 }

494 calls := make(map[string]*acceptance)

495 mux := http.NewServeMux()

496 mux.HandleFunc("POST /webhook", func(w http.ResponseWriter, r *http.Request) {

497 body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1<<20))

498 if err != nil {

499 http.Error(w, "Invalid webhook body", 400)

500 return

501 }

502 event, err := client.Webhooks.Unwrap(body, r.Header)

503 if err != nil {

504 http.Error(w, "Invalid webhook signature", 400)

505 return

506 }

507 if event.Type != "realtime.call.incoming" {

508 w.WriteHeader(200)

509 return

510 }

511 callID := event.Data.CallID

512 mu.Lock()

513 attempt, exists := calls[callID]

514 if !exists {

515 attempt = &acceptance{done: make(chan struct{})}

516 calls[callID] = attempt

517 }

518 mu.Unlock()

519 if exists {

520 select {

521 case <-attempt.done:

522 case <-r.Context().Done():

523 http.Error(w, "Request canceled", http.StatusRequestTimeout)

524 return

525 }

526 } else {

527 // Share this attempt's result only with retries of the same call.

528 // Persist accepted call IDs in production for deduplication across restarts.

529 attempt.err = client.Realtime.Calls.Accept(r.Context(), callID, realtime.CallAcceptParams{

530 RealtimeSessionCreateRequest: realtime.RealtimeSessionCreateRequestParam{

531 Model: "gpt-realtime-2.1",

532 Instructions: openai.String("You are a helpful support agent."),

533 },

534 })

535 mu.Lock()

536 if attempt.err != nil {

537 delete(calls, callID) // Allow a later delivery to retry a failed attempt.

538 }

539 close(attempt.done)

540 mu.Unlock()

541 if attempt.err == nil {

542 go func() {

543 if err := greetCaller(client, callID); err != nil {

544 log.Print("Call greeting failed: ", err)

545 }

546 }()

547 }

548 }

549 if attempt.err != nil {

550 http.Error(w, "Could not accept call", 502)

551 return

552 }

553 w.WriteHeader(200)

554 })

555 return mux

556}

557 

558func greetCaller(client openai.Client, callID string) error {

559 ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)

560 defer cancel()

561 conn, err := client.Realtime.Connect(ctx, realtime.RealtimeConnectionOptions{}, option.WithQuery("call_id", callID))

562 if err != nil {

563 return err

564 }

565 defer conn.Close()

566 if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{

567 OfResponseCreate: &realtime.ResponseCreateEventParam{

568 Response: realtime.RealtimeResponseCreateParams{

569 Instructions: openai.String("Thank the caller and ask how you can help."),

570 OutputModalities: []string{"audio"},

571 },

572 },

573 }); err != nil {

574 return err

575 }

576 for {

577 event, err := conn.Recv(ctx)

578 if err != nil {

579 return err

580 }

581 if event.Type == "error" {

582 return fmt.Errorf("Realtime API error: %s", event.OfRealtimeErrorEvent.Error.Message)

583 }

584 if event.Type == "response.done" {

585 if event.OfResponseDoneEvent.Response.Status != "completed" {

586 return fmt.Errorf("greeting did not complete")

587 }

588 return nil

589 }

590 }

591}

592```

593 

487```ruby594```ruby

488require "openai"595require "openai"

489require "webrick"596require "webrick"

680 680 

681To create a realtime API session via the unified interface, you will need to build a small server-side application (or integrate with an existing one) to make a request to `/v1/realtime/calls`. You will use a [standard API key](https://platform.openai.com/settings/organization/api-keys) to authenticate this request on your backend server.681To create a realtime API session via the unified interface, you will need to build a small server-side application (or integrate with an existing one) to make a request to `/v1/realtime/calls`. You will use a [standard API key](https://platform.openai.com/settings/organization/api-keys) to authenticate this request on your backend server.

682 682 

683Below is an example of a simple Node.js [express](https://expressjs.com/) server which creates a realtime API session:683Create a Realtime API session from your application server:

684 684 

685```javascript685```javascript

686import express from "express";686import express from "express";

723app.listen(3000);723app.listen(3000);

724```724```

725 725 

726```go

727func handler(client openai.Client) http.Handler {

728 mux := http.NewServeMux()

729 mux.HandleFunc("POST /session", func(w http.ResponseWriter, r *http.Request) {

730 offer, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64*1024))

731 if err != nil || strings.TrimSpace(string(offer)) == "" {

732 http.Error(w, "A valid SDP offer is required", http.StatusBadRequest)

733 return

734 }

735 opts := []option.RequestOption{

736 option.WithHeader("OpenAI-Safety-Identifier", "hashed-user-id"),

737 }

738 response, err := client.Realtime.Calls.New(r.Context(), realtime.CallNewParams{

739 Sdp: string(offer),

740 Session: realtime.RealtimeSessionCreateRequestParam{

741 Model: "gpt-realtime-2.1",

742 Audio: realtime.RealtimeAudioConfigParam{

743 Output: realtime.RealtimeAudioConfigOutputParam{

744 Voice: realtime.RealtimeAudioConfigOutputVoiceUnionParam{

745 OfString: openai.String("marin"),

746 },

747 },

748 },

749 },

750 }, opts...)

751 if err != nil {

752 http.Error(w, "Could not create Realtime call", http.StatusBadGateway)

753 return

754 }

755 defer response.Body.Close()

756 w.Header().Set("Content-Type", "application/sdp")

757 w.Header().Set("Location", response.Header.Get("Location"))

758 w.WriteHeader(response.StatusCode)

759 if _, err := io.Copy(w, response.Body); err != nil {

760 log.Print("Could not write SDP answer")

761 }

762 })

763 return mux

764}

765```

766 

726 767 

727If your application assigns a [safety identifier](https://developers.openai.com/api/docs/guides/safety-best-practices#implement-safety-identifiers)768If your application assigns a [safety identifier](https://developers.openai.com/api/docs/guides/safety-best-practices#implement-safety-identifiers)

728for each end user, include it as the `OpenAI-Safety-Identifier` header in this769for each end user, include it as the `OpenAI-Safety-Identifier` header in this

732 773 

733#### Connecting to the server774#### Connecting to the server

734 775 

735In the browser, you can use standard WebRTC APIs to connect to the Realtime API via your application server. The client directly POSTs its SDP data to your server.776In the browser, you can use standard WebRTC APIs to connect to the Realtime API via your application server. The client sends its SDP data to your server in a `POST` request.

736 777 

737```javascript778```javascript

738// Create a peer connection779// Create a peer connection

786 827 

787To create an ephemeral token to use on the client-side, you will need to build a small server-side application (or integrate with an existing one) to make an [OpenAI REST API](https://developers.openai.com/api/reference/resources/realtime/subresources/client_secrets) request for an ephemeral key. You will use a [standard API key](https://platform.openai.com/settings/organization/api-keys) to authenticate this request on your backend server.828To create an ephemeral token to use on the client-side, you will need to build a small server-side application (or integrate with an existing one) to make an [OpenAI REST API](https://developers.openai.com/api/reference/resources/realtime/subresources/client_secrets) request for an ephemeral key. You will use a [standard API key](https://platform.openai.com/settings/organization/api-keys) to authenticate this request on your backend server.

788 829 

789Below is an example of a simple Node.js [express](https://expressjs.com/) server which mints an ephemeral API key using the REST API:830Create an ephemeral API key from your application server using the REST API:

790 831 

791```javascript832```javascript

792import express from "express";833import express from "express";

833app.listen(3000);874app.listen(3000);

834```875```

835 876 

877```go

878func handler(client openai.Client) http.Handler {

879 mux := http.NewServeMux()

880 mux.HandleFunc("GET /token", func(w http.ResponseWriter, r *http.Request) {

881 secret, err := client.Realtime.ClientSecrets.New(r.Context(), realtime.ClientSecretNewParams{

882 Session: realtime.ClientSecretNewParamsSessionUnion{

883 OfRealtime: &realtime.RealtimeSessionCreateRequestParam{

884 Model: "gpt-realtime-2.1",

885 Audio: realtime.RealtimeAudioConfigParam{

886 Output: realtime.RealtimeAudioConfigOutputParam{

887 Voice: realtime.RealtimeAudioConfigOutputVoiceUnionParam{

888 OfString: openai.String("marin"),

889 },

890 },

891 },

892 },

893 },

894 }, option.WithHeader("OpenAI-Safety-Identifier", "hashed-user-id"))

895 if err != nil {

896 http.Error(w, "Could not create client credential", http.StatusBadGateway)

897 return

898 }

899 w.Header().Set("Content-Type", "application/json")

900 w.Header().Set("Cache-Control", "no-store")

901 if err := json.NewEncoder(w).Encode(secret); err != nil {

902 log.Print("Could not write client credential")

903 }

904 })

905 return mux

906}

907```

908 

836 909 

837You can create a server endpoint like this one on any platform that can send and receive HTTP requests. Just ensure that **you only use standard OpenAI API keys on the server, not in the browser.**910You can create a server endpoint like this one on any platform that can send and receive HTTP requests. Just ensure that **you only use standard OpenAI API keys on the server, not in the browser.**

838 911 

843 916 

844#### Connecting to the server917#### Connecting to the server

845 918 

846In the browser, you can use standard WebRTC APIs to connect to the Realtime API with an ephemeral token. The client first fetches a token from your server endpoint, and then POSTs its SDP data (with the ephemeral token) to the Realtime API.919In the browser, you can use standard WebRTC APIs to connect to the Realtime API with an ephemeral token. The client first fetches a token from your server endpoint, and then sends its SDP data (with the ephemeral token) to the Realtime API in a `POST` request.

847 920 

848```javascript921```javascript

849// Get a session token for OpenAI Realtime API922// Get a session token for OpenAI Realtime API

710 { value: "python", label: "websocket-client (Python)" },710 { value: "python", label: "websocket-client (Python)" },

711 { value: "ruby", label: "OpenAI SDK (Ruby)" },711 { value: "ruby", label: "OpenAI SDK (Ruby)" },

712 { value: "java", label: "OpenAI SDK (Java)" },712 { value: "java", label: "OpenAI SDK (Java)" },

713 { value: "go", label: "OpenAI SDK (Go)" },

713 { value: "csharp", label: "OpenAI SDK (.NET)" },714 { value: "csharp", label: "OpenAI SDK (.NET)" },

714 { value: "websocket", label: "WebSocket (browsers)" },715 { value: "websocket", label: "WebSocket (browsers)" },

715 ]}716 ]}

717 718

718 719 

719 720

720ws module (Node.js)721 

722 `ws` module (Node.js)

723

721 724 

722 Connect using the ws module (Node.js)725 Connect using the ws module (Node.js)

723 726 

886 889

887 890 

888 891

892OpenAI SDK (Go)

893 

894

895 

896 Use Go 1.26 or later and install the SDK with

897 `go get github.com/openai/openai-go/v3@v3.85.1`. The SDK

898 manages the socket transport. Your application handles events, captures

899 and plays audio, and restores session state after a reconnect.

900

901 

902 Connect with the OpenAI SDK (Go)

903 

904```go

905ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)

906defer cancel()

907client := openai.NewClient()

908conn, err := client.Realtime.Connect(ctx, realtime.RealtimeConnectionOptions{}, option.WithQuery("model", "gpt-realtime-2.1"), option.WithHeader("OpenAI-Safety-Identifier", "hashed-user-id"))

909if err != nil {

910 return err

911}

912defer conn.Close()

913```

914 

915

916 

917

918 

919

889OpenAI SDK (.NET)920OpenAI SDK (.NET)

890 921 

891 922

173<dependency>173<dependency>

174 <groupId>com.openai</groupId>174 <groupId>com.openai</groupId>

175 <artifactId>openai-java</artifactId>175 <artifactId>openai-java</artifactId>

176 <version>4.94.0</version>176 <version>4.95.0</version>

177</dependency>177</dependency>

178```178```

179 179 

190<dependency>190<dependency>

191 <groupId>com.openai</groupId>191 <groupId>com.openai</groupId>

192 <artifactId>openai-java</artifactId>192 <artifactId>openai-java</artifactId>

193 <version>4.94.0</version>193 <version>4.95.0</version>

194</dependency>194</dependency>

195```195```

196 196