594# Terminal 1: use session_id from the creation request.594# Terminal 1: use session_id from the creation request.
595# Keep this stream open. Wait for HTTP 200 before sending input.595# Keep this stream open. Wait for HTTP 200 before sending input.
596set -o pipefail596set -o pipefail
597# Retain screenshot bytes privately even if saved history omits them.
598stream_file=$(mktemp -t openai-browser-stream.XXXXXX) || exit 1
599printf 'Saved event stream: %s\n' "$stream_file" >&2
597curl --silent --show-error --fail --no-buffer --dump-header - \600curl --silent --show-error --fail --no-buffer --dump-header - \
598 --suppress-connect-headers \601 --suppress-connect-headers \
599 "https://api.openai.com/v1/agents/sessions/$session_id/events" \602 "https://api.openai.com/v1/agents/sessions/$session_id/events" \
600 -H "OpenAI-Beta: agents=v1" \603 -H "OpenAI-Beta: agents=v1" \
601 -H "Authorization: Bearer $OPENAI_API_KEY" \604 -H "Authorization: Bearer $OPENAI_API_KEY" \
602 -H "Accept: text/event-stream" \605 -H "Accept: text/event-stream" \
606 | tee "$stream_file" \
603 | jq --raw-input --unbuffered '607 | jq --raw-input --unbuffered '
604 if startswith("HTTP/") then .608 if startswith("HTTP/") then .
605 elif startswith("data:") then609 elif startswith("data:") then
636```640```
637 641
638```javascript642```javascript
643let screenshot;
639const events = await client.beta.agents.sessions.events.stream(session.id);644const events = await client.beta.agents.sessions.events.stream(session.id);
640const handledRequests = new Set();645const handledRequests = new Set();
641let completed = false;646let completed = false;
675 }680 }
676 break;681 break;
677 }682 }
683 case "agent.session.turn.item.done": {
684 // Preserve streamed screenshots; retrieved history may omit image bytes.
685 const item = event.item;
686 if (item.type === "computer_use_call") {
687 const output = item.output;
688 if (
689 output?.type === "computer_screenshot" &&
690 output.image_url.startsWith("data:image/jpeg;base64,")
691 ) {
692 screenshot = Buffer.from(
693 output.image_url.split(",", 2)[1],
694 "base64"
695 );
696 }
697 }
698 break;
699 }
678 case "agent.session.turn.output_text.done":700 case "agent.session.turn.output_text.done":
679 console.log(event.text);701 console.log(event.text);
680 break;702 break;
709```731```
710 732
711```python733```python
734last_screenshot = None
712handled_requests = set()735handled_requests = set()
713completed = False736completed = False
714with client.beta.agents.sessions.events.stream(session.id) as events:737with client.beta.agents.sessions.events.stream(session.id) as events:
741 ):764 ):
742 respond_to_origin_approval(client, session.id, approval)765 respond_to_origin_approval(client, session.id, approval)
743 handled_requests.add(approval.request_id)766 handled_requests.add(approval.request_id)
767 elif event.type == "agent.session.turn.item.done":
768 # Preserve streamed screenshots; retrieved history may omit image bytes.
769 item = event.item
770 if item.type == "computer_use_call":
771 output = getattr(item, "output", None)
772 if output is not None and output.type == "computer_screenshot":
773 prefix = "data:image/jpeg;base64,"
774 if output.image_url.startswith(prefix):
775 last_screenshot = base64.b64decode(
776 output.image_url[len(prefix) :], validate=True
777 )
744 elif event.type == "agent.session.turn.output_text.done":778 elif event.type == "agent.session.turn.output_text.done":
745 print(event.text, flush=True)779 print(event.text, flush=True)
746 elif event.type == "agent.session.turn.completed":780 elif event.type == "agent.session.turn.completed":
766```800```
767 801
768```go802```go
769803handledRequests := map[string]bool{}var lastScreenshot []byte
804 handledRequests := map[string]bool{}
770 events := client.Beta.Agents.Sessions.Events.StreamStreaming(ctx, session.ID)805 events := client.Beta.Agents.Sessions.Events.StreamStreaming(ctx, session.ID)
771 defer events.Close()806 defer events.Close()
772 if err := events.Err(); err != nil {807 if err := events.Err(); err != nil {
812 }847 }
813 handledRequests[approval.RequestID] = true848 handledRequests[approval.RequestID] = true
814 }849 }
850 case "agent.session.turn.item.done":
851 // Preserve streamed screenshots; retrieved history may omit image bytes.
852 item := event.Item
853 if item.Type == "computer_use_call" && item.Output.Type == "computer_screenshot" {
854 const prefix = "data:image/jpeg;base64,"
855 if strings.HasPrefix(item.Output.ImageURL, prefix) {
856 lastScreenshot, err = base64.StdEncoding.DecodeString(strings.TrimPrefix(item.Output.ImageURL, prefix))
857 if err != nil {
858 return err
859 }
860 }
861 }
815 case "agent.session.turn.output_text.done":862 case "agent.session.turn.output_text.done":
816 fmt.Println(event.Text)863 fmt.Println(event.Text)
817 case "agent.session.turn.completed":864 case "agent.session.turn.completed":
838```885```
839 886
840```java887```java
888byte[] lastScreenshot = null;
841var handledRequests = new HashSet<String>();889var handledRequests = new HashSet<String>();
842try (var events = client.beta().agents().sessions().events().streamStreaming(session.id())) {890try (var events = client.beta().agents().sessions().events().streamStreaming(session.id())) {
843 client891 client
874 }922 }
875 }923 }
876 }924 }
925 // Preserve streamed screenshots; retrieved history may omit image bytes.
926 if (event.turnItemDone().isPresent()
927 && event.turnItemDone().get().item().computerUseCall().isPresent()) {
928 var activity = event.turnItemDone().get().item().computerUseCall().get();
929 var output = activity.output();
930 if (output.isPresent()) {
931 String imageUrl = output.get().imageUrl();
932 String prefix = "data:image/jpeg;base64,";
933 if (imageUrl.startsWith(prefix)) {
934 lastScreenshot = Base64.getDecoder().decode(imageUrl.substring(prefix.length()));
935 }
936 }
937 }
877 event.turnOutputTextDone().ifPresent(text -> System.out.println(text.text()));938 event.turnOutputTextDone().ifPresent(text -> System.out.println(text.text()));
878 if (event.turnCompleted().filter(e -> e.turn().subagentId().isEmpty()).isPresent()) {939 if (event.turnCompleted().filter(e -> e.turn().subagentId().isEmpty()).isPresent()) {
879 completed = true;940 completed = true;
897```958```
898 959
899```csharp960```csharp
961byte[]? lastScreenshot = null;
900HashSet<string> handledRequests = new(StringComparer.Ordinal);962HashSet<string> handledRequests = new(StringComparer.Ordinal);
901await using var events = await client.GetAgentSessionEventsAsync(session.Id);963await using var events = await client.GetAgentSessionEventsAsync(session.Id);
902await client.CreateAgentSessionEventsAsync(964await client.CreateAgentSessionEventsAsync(
932 }994 }
933 }995 }
934 }996 }
997 else if (type == "agent.session.turn.item.done")
998 {
999 // Preserve streamed screenshots; retrieved history may omit image bytes.
1000 JsonElement item = current.GetProperty("item");
1001 if (item.GetProperty("type").GetString() == "computer_use_call"
1002 && item.TryGetProperty("output", out JsonElement output)
1003 && output.ValueKind == JsonValueKind.Object
1004 && output.GetProperty("type").GetString() == "computer_screenshot")
1005 {
1006 const string prefix = "data:image/jpeg;base64,";
1007 string imageUrl = output.GetProperty("image_url").GetString()!;
1008 if (imageUrl.StartsWith(prefix, StringComparison.Ordinal))
1009 {
1010 lastScreenshot = Convert.FromBase64String(imageUrl[prefix.Length..]);
1011 }
1012 }
1013 }
935 else if (type == "agent.session.turn.output_text.done")1014 else if (type == "agent.session.turn.output_text.done")
936 {1015 {
937 Console.WriteLine(current.GetProperty("text").GetString());1016 Console.WriteLine(current.GetProperty("text").GetString());
967```1046```
968 1047
969```ruby1048```ruby
1049last_screenshot = String.new(encoding: Encoding::BINARY)
970handled_requests = Set.new1050handled_requests = Set.new
971events = client.beta.agents.sessions.events.stream_streaming(session.id)1051events = client.beta.agents.sessions.events.stream_streaming(session.id)
972begin1052begin
1001 handled_requests.add(approval.request_id)1081 handled_requests.add(approval.request_id)
1002 end1082 end
1003 false1083 false
1084 when OpenAI::Beta::AgentSessionTurnItemDoneEvent
1085 # Preserve streamed screenshots; retrieved history may omit image bytes.
1086 item = event.item
1087 if item.is_a?(OpenAI::Beta::AgentOutputItem::ComputerUseCall)
1088 output = item.output
1089 if output&.type.to_s == "computer_screenshot"
1090 prefix = "data:image/jpeg;base64,"
1091 if output.image_url.start_with?(prefix)
1092 last_screenshot.replace(Base64.strict_decode64(output.image_url.delete_prefix(prefix)))
1093 end
1094 end
1095 end
1096 false
1004 when OpenAI::Beta::AgentSessionTurnOutputTextDoneEvent1097 when OpenAI::Beta::AgentSessionTurnOutputTextDoneEvent
1005 puts event.text1098 puts event.text
1006 when OpenAI::Beta::AgentSessionTurnCompletedEvent1099 when OpenAI::Beta::AgentSessionTurnCompletedEvent
1055to `true` on the `computer_use` tool. Screenshots are excluded from API output1148to `true` on the `computer_use` tool. Screenshots are excluded from API output
1056by default; the agent can still observe them.1149by default; the agent can still observe them.
1057 1150
10581151Each browser operation returns its last emitted screenshot in `output`, whenA completed browser activity event includes its last emitted screenshot in
10591152available:`item.output`, when available:
1060 1153
1061```json1154```json
1062{1155{
1072Screenshots can contain sensitive page or account data. Show them only to1165Screenshots can contain sensitive page or account data. Show them only to
1073 authorized users and keep them out of application logs.1166 authorized users and keep them out of application logs.
1074 1167
10751168Retrieve saved browser activity after the turn completes. The SDK examples saveCapture available screenshots from `agent.session.turn.item.done` events while
10761169the latest available screenshot to `browser-screenshot.jpg`.consuming the stream. Retrieved history can omit image bytes, so retain the image
1170in your application rather than relying on a later items-list request. After the
1171turn completes, the SDK examples list browser activity and save the latest captured
1172screenshot. Most use `browser-screenshot.jpg`; Java uses a temporary file.
1077 1173
1078Read browser activity1174Read browser activity
1079 1175
1080```bash1176```bash
1177# Screenshot bytes are retained in the private event stream file printed in terminal 1.
1178# Keep that file: saved history may omit image bytes.
1081# Save the first page privately; image data stays out of terminal output.1179# Save the first page privately; image data stays out of terminal output.
1082activity_file=$(mktemp)1180activity_file=$(mktemp)
1083curl --silent --show-error --fail-with-body --get \1181curl --silent --show-error --fail-with-body --get \
1098```1196```
1099 1197
1100```javascript1198```javascript
1101let screenshot;
1102for await (const item of client.beta.agents.sessions.items.list(session.id, {1199for await (const item of client.beta.agents.sessions.items.list(session.id, {
1103 order: "asc",1200 order: "asc",
1104 limit: 100,1201 limit: 100,
1105})) {1202})) {
1106 if (item.type !== "computer_use_call") continue;1203 if (item.type !== "computer_use_call") continue;
1107 console.log(item.title ?? "Browser activity", item.status);1204 console.log(item.title ?? "Browser activity", item.status);
1108 const output = item.output;
1109 if (
1110 output?.type === "computer_screenshot" &&
1111 output.image_url.startsWith("data:image/jpeg;base64,")
1112 ) {
1113 screenshot = Buffer.from(output.image_url.split(",", 2)[1], "base64");
1114 }
1115}1205}
1116if (screenshot) {1206if (screenshot) {
1117 const file = await open("browser-screenshot.jpg", "wx", 0o600);1207 const file = await open("browser-screenshot.jpg", "wx", 0o600);
1127```1217```
1128 1218
1129```python1219```python
1130last_screenshot = None
1131for item in client.beta.agents.sessions.items.list(1220for item in client.beta.agents.sessions.items.list(
1132 session.id, order="asc", limit=1001221 session.id, order="asc", limit=100
1133):1222):
1134 if item.type != "computer_use_call":1223 if item.type != "computer_use_call":
1135 continue1224 continue
1136 print(item.title or "Browser activity", item.status)1225 print(item.title or "Browser activity", item.status)
1137 output = getattr(item, "output", None)
1138 if output is not None and output.type == "computer_screenshot":
1139 prefix = "data:image/jpeg;base64,"
1140 if output.image_url.startswith(prefix):
1141 last_screenshot = base64.b64decode(
1142 output.image_url[len(prefix) :], validate=True
1143 )
1144 1226
1145if last_screenshot is not None:1227if last_screenshot is not None:
1146 screenshot_path = Path("browser-screenshot.jpg")1228 screenshot_path = Path("browser-screenshot.jpg")
1156```1238```
1157 1239
1158```go1240```go
1159var lastScreenshot []byte
1160items := client.Beta.Agents.Sessions.Items.ListAutoPaging(ctx, session.ID, openai.BetaAgentSessionItemListParams{1241items := client.Beta.Agents.Sessions.Items.ListAutoPaging(ctx, session.ID, openai.BetaAgentSessionItemListParams{
1161 Order: "asc", Limit: openai.Int(100),1242 Order: "asc", Limit: openai.Int(100),
1162})1243})
1171 title = "Browser activity"1252 title = "Browser activity"
1172 }1253 }
1173 fmt.Println(title, activity.Status)1254 fmt.Println(title, activity.Status)
1174 if !activity.JSON.Output.Valid() || activity.Output.Type != "computer_screenshot" {
1175 continue
1176 }
1177 const prefix = "data:image/jpeg;base64,"
1178 if strings.HasPrefix(activity.Output.ImageURL, prefix) {
1179 lastScreenshot, err = base64.StdEncoding.DecodeString(strings.TrimPrefix(activity.Output.ImageURL, prefix))
1180 if err != nil {
1181 return err
1182 }
1183 }
1184}1255}
1185if err := items.Err(); err != nil {1256if err := items.Err(); err != nil {
1186 return err1257 return err
1205```1276```
1206 1277
1207```java1278```java
1208byte[] lastScreenshot = null;
1209var items =1279var items =
1210 client1280 client
1211 .beta()1281 .beta()
1222 if (item.computerUseCall().isEmpty()) continue;1292 if (item.computerUseCall().isEmpty()) continue;
1223 var activity = item.computerUseCall().get();1293 var activity = item.computerUseCall().get();
1224 System.out.println(activity.title().orElse("Browser activity") + " " + activity.status());1294 System.out.println(activity.title().orElse("Browser activity") + " " + activity.status());
1225 var output = activity.output();
1226 if (output.isPresent()) {
1227 String imageUrl = output.get().imageUrl();
1228 String prefix = "data:image/jpeg;base64,";
1229 if (imageUrl.startsWith(prefix)) {
1230 lastScreenshot = Base64.getDecoder().decode(imageUrl.substring(prefix.length()));
1231 }
1232 }
1233}1295}
1234if (lastScreenshot != null) {1296if (lastScreenshot != null) {
1235 var screenshotPath = Files.createTempFile("browser-screenshot-", ".jpg");1297 var screenshotPath = Files.createTempFile("browser-screenshot-", ".jpg");
1241```1303```
1242 1304
1243```csharp1305```csharp
1244byte[]? lastScreenshot = null;
1245await foreach (AgentSessionItem item in client.GetAgentSessionItemsAsync(1306await foreach (AgentSessionItem item in client.GetAgentSessionItemsAsync(
1246 session.Id, limit: 100, order: AgentSessionItemCollectionOrder.Ascending))1307 session.Id, limit: 100, order: AgentSessionItemCollectionOrder.Ascending))
1247{1308{
1250 continue;1311 continue;
1251 }1312 }
1252 Console.WriteLine($"{activity.Title ?? "Browser activity"}: {activity.Status}");1313 Console.WriteLine($"{activity.Title ?? "Browser activity"}: {activity.Status}");
12531314 if (activity.Output is ComputerUseOutputResourceComputerScreenshot screenshot)
1254 {
1255 const string prefix = "data:image/jpeg;base64,";
1256 if (screenshot.ImageUrl.StartsWith(prefix, StringComparison.Ordinal))
1257 {
1258 lastScreenshot = Convert.FromBase64String(screenshot.ImageUrl[prefix.Length..]);
1259 }
1260 }
1261}1315}
1262if (lastScreenshot is not null)1316if (lastScreenshot is not null)
1263{1317{
1286```1340```
1287 1341
1288```ruby1342```ruby
1289last_screenshot = String.new(encoding: Encoding::BINARY)
1290items = client.beta.agents.sessions.items.list(1343items = client.beta.agents.sessions.items.list(
1291 session.id,1344 session.id,
1292 order: "asc",1345 order: "asc",
1293 limit: 1001346 limit: 100
1294)1347)
1295items.auto_paging_each do |item|1348items.auto_paging_each do |item|
12961349 next unless item.is_a?(OpenAI::Beta::AgentComputerUseCallItem) next unless item.is_a?(OpenAI::Beta::AgentSessionItem::ComputerUseCall)
1297 1350
1298 puts "#{item.title || "Browser activity"}: #{item.status}"1351 puts "#{item.title || "Browser activity"}: #{item.status}"
1299 output = item.output
1300 if output&.type.to_s == "computer_screenshot"
1301 prefix = "data:image/jpeg;base64,"
1302 if output.image_url.start_with?(prefix)
1303 last_screenshot.replace(Base64.strict_decode64(output.image_url.delete_prefix(prefix)))
1304 end
1305 end
1306end1352end
1307 1353
1308if last_screenshot.empty?1354if last_screenshot.empty?