146```146```
147 147
148```javascript148```javascript
149import { exec } from "node:child_process";149import { execFile } from "node:child_process";
150import { promisify } from "node:util";150import { promisify } from "node:util";
151 151
152const execAsync = promisify(exec);152const execFileAsync = promisify(execFile);
153 153
154async function dockerExec(cmd, containerName, decode = true) {154async function dockerExec(
155 const safeCmd = cmd.replace(/"/g, '\\"');155 containerName,
156 const dockerCmd = `docker exec ${containerName} sh -c "${safeCmd}"`;156 executable,
157 const output = await execAsync(dockerCmd, {157 args = [],
158 { decode = true, env = {} } = {}
159) {
160 const environmentArgs = Object.entries(env).flatMap(([name, value]) => [
161 "--env",
162 `${name}=${value}`,
163 ]);
164 const output = await execFileAsync(
165 "docker",
166 [
167 "exec",
168 ...environmentArgs,
169 containerName,
170 executable,
171 ...args.map(String),
172 ],
173 {
158 encoding: decode ? "utf8" : "buffer",174 encoding: decode ? "utf8" : "buffer",
159 });175 maxBuffer: 10 * 1024 * 1024,
176 }
177 );
160 return output.stdout;178 return output.stdout;
161}179}
162 180
325 }343 }
326};344};
327 345
346// Translate API button names to Playwright's supported button names.
347const normalizePlaywrightButton = (button = "left") => {
348 const buttons = {
349 left: "left",
350 right: "right",
351 wheel: "middle",
352 };
353 const normalized = buttons[button];
354 if (!normalized) {
355 throw new Error(
356 `Unsupported Playwright mouse button: ${button}. The back and forward buttons are not supported.`
357 );
358 }
359 return normalized;
360};
361
328// Accept drag paths as either [x, y] pairs or {x, y} objects.362// Accept drag paths as either [x, y] pairs or {x, y} objects.
329const normalizeDragPath = (path) => {363const normalizeDragPath = (path) => {
330 if (!Array.isArray(path)) {364 if (!Array.isArray(path)) {
338 if (point && typeof point === "object" && "x" in point && "y" in point) {372 if (point && typeof point === "object" && "x" in point && "y" in point) {
339 return [point.x, point.y];373 return [point.x, point.y];
340 }374 }
341 throw new Error("drag path entries must be coordinate pairs or {x, y} objects");375 throw new Error(
376 "drag path entries must be coordinate pairs or {x, y} objects"
377 );
342 });378 });
343};379};
344```380```
380 return key_map.get(key, key)416 return key_map.get(key, key)
381 417
382 418
419def normalize_playwright_button(button="left"):
420 """Translate API button names to Playwright's supported button names."""
421 button_map = {
422 "left": "left",
423 "right": "right",
424 "wheel": "middle",
425 }
426 if button not in button_map:
427 raise ValueError(
428 f"Unsupported Playwright mouse button: {button}. "
429 "The back and forward buttons are not supported."
430 )
431 return button_map[button]
432
433
383def normalize_drag_path(path):434def normalize_drag_path(path):
384 """Accept drag paths as either [x, y] pairs or {x, y} objects."""435 """Accept drag paths as either [x, y] pairs or {x, y} objects."""
385 if not isinstance(path, list):436 if not isinstance(path, list):
459 }510 }
460};511};
461 512
513// Translate API button names to X11 button numbers.
514const normalizeXdotoolButton = (button = "left") => {
515 const buttons = {
516 left: 1,
517 wheel: 2,
518 right: 3,
519 back: 8,
520 forward: 9,
521 };
522 const normalized = buttons[button];
523 if (!normalized) {
524 throw new Error(`Unsupported xdotool mouse button: ${button}`);
525 }
526 return normalized;
527};
528
529// Translate API scroll deltas to vertical and horizontal X11 wheel clicks.
530const getXdotoolScrollButtons = (scrollX, scrollY) => {
531 const scrollButtons = [];
532 const appendClicks = (delta, negativeButton, positiveButton) => {
533 if (!delta) {
534 return;
535 }
536 const button = delta < 0 ? negativeButton : positiveButton;
537 const clicks = Math.max(1, Math.abs(Math.round(delta / 100)));
538 scrollButtons.push(...Array(clicks).fill(button));
539 };
540
541 appendClicks(scrollY, 4, 5);
542 appendClicks(scrollX, 6, 7);
543 return scrollButtons;
544};
545
462// Accept drag paths as either [x, y] pairs or {x, y} objects.546// Accept drag paths as either [x, y] pairs or {x, y} objects.
463const normalizeDragPath = (path) => {547const normalizeDragPath = (path) => {
464 if (!Array.isArray(path)) {548 if (!Array.isArray(path)) {
472 if (point && typeof point === "object" && "x" in point && "y" in point) {556 if (point && typeof point === "object" && "x" in point && "y" in point) {
473 return [point.x, point.y];557 return [point.x, point.y];
474 }558 }
475 throw new Error("drag path entries must be coordinate pairs or {x, y} objects");559 throw new Error(
560 "drag path entries must be coordinate pairs or {x, y} objects"
561 );
476 });562 });
477};563};
478```564```
514 return key_map.get(key, key)600 return key_map.get(key, key)
515 601
516 602
603def normalize_xdotool_button(button="left"):
604 """Translate API button names to X11 button numbers."""
605 button_map = {
606 "left": 1,
607 "wheel": 2,
608 "right": 3,
609 "back": 8,
610 "forward": 9,
611 }
612 if button not in button_map:
613 raise ValueError(f"Unsupported xdotool mouse button: {button}")
614 return button_map[button]
615
616
617def get_xdotool_scroll_buttons(scroll_x, scroll_y):
618 """Translate API scroll deltas to vertical and horizontal X11 wheel clicks."""
619 buttons = []
620 for delta, negative_button, positive_button in (
621 (scroll_y, 4, 5),
622 (scroll_x, 6, 7),
623 ):
624 if not delta:
625 continue
626 button = negative_button if delta < 0 else positive_button
627 clicks = max(1, abs(round(delta / 100)))
628 buttons.extend([button] * clicks)
629 return buttons
630
631
517def normalize_drag_path(path):632def normalize_drag_path(path):
518 """Accept drag paths as either [x, y] pairs or {x, y} objects."""633 """Accept drag paths as either [x, y] pairs or {x, y} objects."""
519 if not isinstance(path, list):634 if not isinstance(path, list):
565 680
566```javascript681```javascript
567// Reuse normalizeKey from the helper above.682// Reuse normalizeKey from the helper above.
683// Reuse normalizePlaywrightButton from the helper above.
568// Reuse normalizeDragPath from the helper above.684// Reuse normalizeDragPath from the helper above.
569 685
686function rejectModifiers(action) {
687 if (action.keys?.length) {
688 throw new Error(
689 "This handler does not support modifier keys. Use the modifier-aware handler below."
690 );
691 }
692}
693
570async function handleComputerActions(page, actions) {694async function handleComputerActions(page, actions) {
571 for (const action of actions) {695 for (const action of actions) {
572 switch (action.type) {696 switch (action.type) {
573 case "click":697 case "click": {
698 rejectModifiers(action);
574 await page.mouse.click(action.x, action.y, {699 await page.mouse.click(action.x, action.y, {
575 button: action.button ?? "left",700 button: normalizePlaywrightButton(action.button),
576 });701 });
577 break;702 break;
703 }
578 case "double_click":704 case "double_click":
579 await page.mouse.dblclick(action.x, action.y, {705 rejectModifiers(action);
580 button: action.button ?? "left",706 await page.mouse.dblclick(action.x, action.y);
581 });
582 break;707 break;
583 case "drag": {708 case "drag": {
709 rejectModifiers(action);
584 const path = normalizeDragPath(action.path);710 const path = normalizeDragPath(action.path);
585 if (path.length < 2) {711 if (path.length < 2) {
586 throw new Error("drag action requires at least two path points");712 throw new Error("drag action requires at least two path points");
595 break;721 break;
596 }722 }
597 case "move":723 case "move":
724 rejectModifiers(action);
598 await page.mouse.move(action.x, action.y);725 await page.mouse.move(action.x, action.y);
599 break;726 break;
600 case "scroll":727 case "scroll":
728 rejectModifiers(action);
601 await page.mouse.move(action.x, action.y);729 await page.mouse.move(action.x, action.y);
602 await page.mouse.wheel(action.scrollX ?? 0, action.scrollY ?? 0);730 await page.mouse.wheel(action.scroll_x, action.scroll_y);
603 break;731 break;
604 case "keypress":732 case "keypress":
605 for (const key of action.keys) {733 for (const key of action.keys) {
610 await page.keyboard.type(action.text);738 await page.keyboard.type(action.text);
611 break;739 break;
612 case "wait":740 case "wait":
741 await page.waitForTimeout(2000);
742 break;
613 case "screenshot":743 case "screenshot":
614 break;744 break;
615 default:745 default:
623import time753import time
624 754
625# Reuse normalize_key from the helper above.755# Reuse normalize_key from the helper above.
756# Reuse normalize_playwright_button from the helper above.
626# Reuse normalize_drag_path from the helper above.757# Reuse normalize_drag_path from the helper above.
627 758
628 759
760def reject_modifiers(action):
761 if getattr(action, "keys", None):
762 raise ValueError(
763 "This handler does not support modifier keys. "
764 "Use the modifier-aware handler below."
765 )
766
767
629def handle_computer_actions(page, actions):768def handle_computer_actions(page, actions):
630 for action in actions:769 for action in actions:
631 match action.type:770 match action.type:
632 case "click":771 case "click":
772 reject_modifiers(action)
633 page.mouse.click(773 page.mouse.click(
634 action.x,774 action.x,
635 action.y,775 action.y,
636 button=getattr(action, "button", "left"),776 button=normalize_playwright_button(
777 getattr(action, "button", "left")
778 ),
637 )779 )
638 case "double_click":780 case "double_click":
639 page.mouse.dblclick(781 reject_modifiers(action)
640 action.x,782 page.mouse.dblclick(action.x, action.y)
641 action.y,
642 button=getattr(action, "button", "left"),
643 )
644 case "drag":783 case "drag":
784 reject_modifiers(action)
645 path = normalize_drag_path(action.path)785 path = normalize_drag_path(action.path)
646 if len(path) < 2:786 if len(path) < 2:
647 raise ValueError("drag action requires at least two path points")787 raise ValueError("drag action requires at least two path points")
652 page.mouse.move(x, y)792 page.mouse.move(x, y)
653 page.mouse.up()793 page.mouse.up()
654 case "move":794 case "move":
795 reject_modifiers(action)
655 page.mouse.move(action.x, action.y)796 page.mouse.move(action.x, action.y)
656 case "scroll":797 case "scroll":
798 reject_modifiers(action)
657 page.mouse.move(action.x, action.y)799 page.mouse.move(action.x, action.y)
658 page.mouse.wheel(800 page.mouse.wheel(
659 getattr(action, "scrollX", 0),801 action.scroll_x,
660 getattr(action, "scrollY", 0),802 action.scroll_y,
661 )803 )
662 case "keypress":804 case "keypress":
663 for key in action.keys:805 for key in action.keys:
679 821
680```javascript822```javascript
681// Reuse normalizeXdotoolKey from the helper above.823// Reuse normalizeXdotoolKey from the helper above.
824// Reuse normalizeXdotoolButton and getXdotoolScrollButtons from the helper above.
682// Reuse normalizeDragPath from the helper above.825// Reuse normalizeDragPath from the helper above.
683 826
684async function handleComputerActions(vm, actions) {827function rejectModifiers(action) {
685 const buttonMap = { left: 1, middle: 2, right: 3 };828 if (action.keys?.length) {
829 throw new Error(
830 "This handler does not support modifier keys. Use the modifier-aware handler below."
831 );
832 }
833}
686 834
835async function handleComputerActions(vm, actions) {
687 for (const action of actions) {836 for (const action of actions) {
688 switch (action.type) {837 switch (action.type) {
689 case "click": {838 case "click": {
690 const button = buttonMap[action.button ?? "left"] ?? 1;839 rejectModifiers(action);
840 const button = normalizeXdotoolButton(action.button);
691 await dockerExec(841 await dockerExec(
692 `DISPLAY=${vm.display} xdotool mousemove ${action.x} ${action.y} click ${button}`,842 vm.containerName,
693 vm.containerName843 "xdotool",
844 ["mousemove", action.x, action.y, "click", button],
845 { env: { DISPLAY: vm.display } }
694 );846 );
695 break;847 break;
696 }848 }
697 case "double_click": {849 case "double_click": {
698 const button = buttonMap[action.button ?? "left"] ?? 1;850 rejectModifiers(action);
699 await dockerExec(851 await dockerExec(
700 `DISPLAY=${vm.display} xdotool mousemove ${action.x} ${action.y} click --repeat 2 ${button}`,852 vm.containerName,
701 vm.containerName853 "xdotool",
854 ["mousemove", action.x, action.y, "click", "--repeat", 2, 1],
855 { env: { DISPLAY: vm.display } }
702 );856 );
703 break;857 break;
704 }858 }
705 case "drag": {859 case "drag": {
860 rejectModifiers(action);
706 const path = normalizeDragPath(action.path);861 const path = normalizeDragPath(action.path);
707 if (path.length < 2) {862 if (path.length < 2) {
708 throw new Error("drag action requires at least two path points");863 throw new Error("drag action requires at least two path points");
709 }864 }
710 const [[startX, startY], ...rest] = path;865 const [[startX, startY], ...rest] = path;
711 await dockerExec(866 await dockerExec(
712 `DISPLAY=${vm.display} xdotool mousemove ${startX} ${startY} mousedown 1`,867 vm.containerName,
713 vm.containerName868 "xdotool",
869 ["mousemove", startX, startY, "mousedown", 1],
870 { env: { DISPLAY: vm.display } }
714 );871 );
715 for (const [x, y] of rest) {872 for (const [x, y] of rest) {
716 await dockerExec(873 await dockerExec(vm.containerName, "xdotool", ["mousemove", x, y], {
717 `DISPLAY=${vm.display} xdotool mousemove ${x} ${y}`,874 env: { DISPLAY: vm.display },
718 vm.containerName875 });
719 );
720 }876 }
721 await dockerExec(877 await dockerExec(vm.containerName, "xdotool", ["mouseup", 1], {
722 `DISPLAY=${vm.display} xdotool mouseup 1`,878 env: { DISPLAY: vm.display },
723 vm.containerName879 });
724 );
725 break;880 break;
726 }881 }
727 case "move":882 case "move":
883 rejectModifiers(action);
728 await dockerExec(884 await dockerExec(
729 `DISPLAY=${vm.display} xdotool mousemove ${action.x} ${action.y}`,885 vm.containerName,
730 vm.containerName886 "xdotool",
887 ["mousemove", action.x, action.y],
888 { env: { DISPLAY: vm.display } }
731 );889 );
732 break;890 break;
733 case "scroll": {891 case "scroll": {
734 const button = action.scrollY < 0 ? 4 : 5;892 rejectModifiers(action);
735 const clicks = Math.max(1, Math.abs(Math.round(action.scrollY / 100)));893 const buttons = getXdotoolScrollButtons(
736 await dockerExec(894 action.scroll_x,
737 `DISPLAY=${vm.display} xdotool mousemove ${action.x} ${action.y}`,895 action.scroll_y
738 vm.containerName
739 );896 );
740 for (let i = 0; i < clicks; i += 1) {
741 await dockerExec(897 await dockerExec(
742 `DISPLAY=${vm.display} xdotool click ${button}`,898 vm.containerName,
743 vm.containerName899 "xdotool",
900 ["mousemove", action.x, action.y],
901 { env: { DISPLAY: vm.display } }
744 );902 );
903 for (const button of buttons) {
904 await dockerExec(vm.containerName, "xdotool", ["click", button], {
905 env: { DISPLAY: vm.display },
906 });
745 }907 }
746 break;908 break;
747 }909 }
748 case "keypress":910 case "keypress":
749 for (const key of action.keys) {911 for (const key of action.keys) {
750 await dockerExec(912 await dockerExec(
751 `DISPLAY=${vm.display} xdotool key '${normalizeXdotoolKey(key)}'`,913 vm.containerName,
752 vm.containerName914 "xdotool",
915 ["key", normalizeXdotoolKey(key)],
916 { env: { DISPLAY: vm.display } }
753 );917 );
754 }918 }
755 break;919 break;
756 case "type":920 case "type":
757 await dockerExec(921 await dockerExec(
758 `DISPLAY=${vm.display} xdotool type --delay 0 '${action.text}'`,922 vm.containerName,
759 vm.containerName923 "xdotool",
924 ["type", "--delay", 0, action.text],
925 { env: { DISPLAY: vm.display } }
760 );926 );
761 break;927 break;
762 case "wait":928 case "wait":
929 await new Promise((resolve) => setTimeout(resolve, 2000));
930 break;
763 case "screenshot":931 case "screenshot":
764 break;932 break;
765 default:933 default:
773import time941import time
774 942
775# Reuse normalize_xdotool_key from the helper above.943# Reuse normalize_xdotool_key from the helper above.
944# Reuse normalize_xdotool_button and get_xdotool_scroll_buttons from the helper above.
776# Reuse normalize_drag_path from the helper above.945# Reuse normalize_drag_path from the helper above.
777 946
778 947
779def handle_computer_actions(vm, actions):948def reject_modifiers(action):
780 button_map = {"left": 1, "middle": 2, "right": 3}949 if getattr(action, "keys", None):
950 raise ValueError(
951 "This handler does not support modifier keys. "
952 "Use the modifier-aware handler below."
953 )
954
781 955
956def handle_computer_actions(vm, actions):
782 for action in actions:957 for action in actions:
783 match action.type:958 match action.type:
784 case "click":959 case "click":
785 button = button_map.get(getattr(action, "button", "left"), 1)960 reject_modifiers(action)
961 button = normalize_xdotool_button(
962 getattr(action, "button", "left")
963 )
786 docker_exec(964 docker_exec(
787 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y} click {button}",965 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y} click {button}",
788 vm.container_name,966 vm.container_name,
789 )967 )
790 case "double_click":968 case "double_click":
791 button = button_map.get(getattr(action, "button", "left"), 1)969 reject_modifiers(action)
792 docker_exec(970 docker_exec(
793 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y} click --repeat 2 {button}",971 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y} click --repeat 2 1",
794 vm.container_name,972 vm.container_name,
795 )973 )
796 case "drag":974 case "drag":
975 reject_modifiers(action)
797 path = normalize_drag_path(action.path)976 path = normalize_drag_path(action.path)
798 if len(path) < 2:977 if len(path) < 2:
799 raise ValueError("drag action requires at least two path points")978 raise ValueError("drag action requires at least two path points")
812 vm.container_name,991 vm.container_name,
813 )992 )
814 case "move":993 case "move":
994 reject_modifiers(action)
815 docker_exec(995 docker_exec(
816 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y}",996 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y}",
817 vm.container_name,997 vm.container_name,
818 )998 )
819 case "scroll":999 case "scroll":
820 button = 4 if getattr(action, "scrollY", 0) < 0 else 51000 reject_modifiers(action)
821 clicks = max(1, abs(round(getattr(action, "scrollY", 0) / 100)))1001 buttons = get_xdotool_scroll_buttons(
1002 action.scroll_x,
1003 action.scroll_y,
1004 )
822 1005
823 docker_exec(1006 docker_exec(
824 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y}",1007 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y}",
825 vm.container_name,1008 vm.container_name,
826 )1009 )
827 for _ in range(clicks):1010 for button in buttons:
828 docker_exec(1011 docker_exec(
829 f"DISPLAY={vm.display} xdotool click {button}",1012 f"DISPLAY={vm.display} xdotool click {button}",
830 vm.container_name,1013 vm.container_name,
892 1075
893```javascript1076```javascript
894// Reuse normalizeKey from the helper above.1077// Reuse normalizeKey from the helper above.
1078// Reuse normalizePlaywrightButton from the helper above.
895// Reuse normalizeDragPath from the helper above.1079// Reuse normalizeDragPath from the helper above.
896 1080
897async function withModifiers(page, keys, callback) {1081async function withModifiers(page, keys, callback) {
918 case "click":1102 case "click":
919 await withModifiers(page, action.keys, async () => {1103 await withModifiers(page, action.keys, async () => {
920 await page.mouse.click(action.x, action.y, {1104 await page.mouse.click(action.x, action.y, {
921 button: action.button ?? "left",1105 button: normalizePlaywrightButton(action.button),
922 });1106 });
923 });1107 });
924 break;1108 break;
925 case "double_click":1109 case "double_click":
926 await withModifiers(page, action.keys, async () => {1110 await withModifiers(page, action.keys, async () => {
927 await page.mouse.dblclick(action.x, action.y, {1111 await page.mouse.dblclick(action.x, action.y);
928 button: action.button ?? "left",
929 });
930 });1112 });
931 break;1113 break;
932 case "drag": {1114 case "drag": {
953 case "scroll":1135 case "scroll":
954 await withModifiers(page, action.keys, async () => {1136 await withModifiers(page, action.keys, async () => {
955 await page.mouse.move(action.x, action.y);1137 await page.mouse.move(action.x, action.y);
956 await page.mouse.wheel(action.scrollX ?? 0, action.scrollY ?? 0);1138 await page.mouse.wheel(action.scroll_x, action.scroll_y);
957 });1139 });
958 break;1140 break;
959 case "keypress":1141 case "keypress":
965 await page.keyboard.type(action.text);1147 await page.keyboard.type(action.text);
966 break;1148 break;
967 case "wait":1149 case "wait":
1150 await page.waitForTimeout(2000);
1151 break;
968 case "screenshot":1152 case "screenshot":
969 break;1153 break;
970 default:1154 default:
978import time1162import time
979 1163
980# Reuse normalize_key from the helper above.1164# Reuse normalize_key from the helper above.
1165# Reuse normalize_playwright_button from the helper above.
981# Reuse normalize_drag_path from the helper above.1166# Reuse normalize_drag_path from the helper above.
982 1167
983 1168
1006 lambda: page.mouse.click(1191 lambda: page.mouse.click(
1007 action.x,1192 action.x,
1008 action.y,1193 action.y,
1009 button=getattr(action, "button", "left"),1194 button=normalize_playwright_button(
1195 getattr(action, "button", "left")
1196 ),
1010 ),1197 ),
1011 )1198 )
1012 case "double_click":1199 case "double_click":
1013 with_modifiers(1200 with_modifiers(
1014 page,1201 page,
1015 getattr(action, "keys", None),1202 getattr(action, "keys", None),
1016 lambda: page.mouse.dblclick(1203 lambda: page.mouse.dblclick(action.x, action.y),
1017 action.x,
1018 action.y,
1019 button=getattr(action, "button", "left"),
1020 ),
1021 )1204 )
1022 case "drag":1205 case "drag":
1023 path = normalize_drag_path(action.path)1206 path = normalize_drag_path(action.path)
1050 lambda: (1233 lambda: (
1051 page.mouse.move(action.x, action.y),1234 page.mouse.move(action.x, action.y),
1052 page.mouse.wheel(1235 page.mouse.wheel(
1053 getattr(action, "scrollX", 0),1236 action.scroll_x,
1054 getattr(action, "scrollY", 0),1237 action.scroll_y,
1055 ),1238 ),
1056 ),1239 ),
1057 )1240 )
1075 1258
1076```javascript1259```javascript
1077// Reuse normalizeXdotoolKey from the helper above.1260// Reuse normalizeXdotoolKey from the helper above.
1261// Reuse normalizeXdotoolButton and getXdotoolScrollButtons from the helper above.
1078// Reuse normalizeDragPath from the helper above.1262// Reuse normalizeDragPath from the helper above.
1079 1263
1080async function withModifiers(vm, keys, callback) {1264async function withModifiers(vm, keys, callback) {
1083 1267
1084 try {1268 try {
1085 for (const key of normalizedKeys) {1269 for (const key of normalizedKeys) {
1086 await dockerExec(1270 await dockerExec(vm.containerName, "xdotool", ["keydown", key], {
1087 `DISPLAY=${vm.display} xdotool keydown '${key}'`,1271 env: { DISPLAY: vm.display },
1088 vm.containerName1272 });
1089 );
1090 pressedKeys.push(key);1273 pressedKeys.push(key);
1091 }1274 }
1092 1275
1093 await callback();1276 await callback();
1094 } finally {1277 } finally {
1095 for (const key of [...pressedKeys].reverse()) {1278 for (const key of [...pressedKeys].reverse()) {
1096 await dockerExec(1279 await dockerExec(vm.containerName, "xdotool", ["keyup", key], {
1097 `DISPLAY=${vm.display} xdotool keyup '${key}'`,1280 env: { DISPLAY: vm.display },
1098 vm.containerName1281 });
1099 );
1100 }1282 }
1101 }1283 }
1102}1284}
1103 1285
1104async function handleComputerActions(vm, actions) {1286async function handleComputerActions(vm, actions) {
1105 const buttonMap = { left: 1, middle: 2, right: 3 };
1106
1107 for (const action of actions) {1287 for (const action of actions) {
1108 switch (action.type) {1288 switch (action.type) {
1109 case "click": {1289 case "click": {
1110 const button = buttonMap[action.button ?? "left"] ?? 1;1290 const button = normalizeXdotoolButton(action.button);
1111 await withModifiers(vm, action.keys, async () => {1291 await withModifiers(vm, action.keys, async () => {
1112 await dockerExec(1292 await dockerExec(
1113 `DISPLAY=${vm.display} xdotool mousemove ${action.x} ${action.y} click ${button}`,1293 vm.containerName,
1114 vm.containerName1294 "xdotool",
1295 ["mousemove", action.x, action.y, "click", button],
1296 { env: { DISPLAY: vm.display } }
1115 );1297 );
1116 });1298 });
1117 break;1299 break;
1118 }1300 }
1119 case "double_click": {1301 case "double_click": {
1120 const button = buttonMap[action.button ?? "left"] ?? 1;
1121 await withModifiers(vm, action.keys, async () => {1302 await withModifiers(vm, action.keys, async () => {
1122 await dockerExec(1303 await dockerExec(
1123 `DISPLAY=${vm.display} xdotool mousemove ${action.x} ${action.y} click --repeat 2 ${button}`,1304 vm.containerName,
1124 vm.containerName1305 "xdotool",
1306 ["mousemove", action.x, action.y, "click", "--repeat", 2, 1],
1307 { env: { DISPLAY: vm.display } }
1125 );1308 );
1126 });1309 });
1127 break;1310 break;
1134 await withModifiers(vm, action.keys, async () => {1317 await withModifiers(vm, action.keys, async () => {
1135 const [[startX, startY], ...rest] = path;1318 const [[startX, startY], ...rest] = path;
1136 await dockerExec(1319 await dockerExec(
1137 `DISPLAY=${vm.display} xdotool mousemove ${startX} ${startY} mousedown 1`,1320 vm.containerName,
1138 vm.containerName1321 "xdotool",
1322 ["mousemove", startX, startY, "mousedown", 1],
1323 { env: { DISPLAY: vm.display } }
1139 );1324 );
1140 for (const [x, y] of rest) {1325 for (const [x, y] of rest) {
1141 await dockerExec(1326 await dockerExec(vm.containerName, "xdotool", ["mousemove", x, y], {
1142 `DISPLAY=${vm.display} xdotool mousemove ${x} ${y}`,1327 env: { DISPLAY: vm.display },
1143 vm.containerName1328 });
1144 );
1145 }1329 }
1146 await dockerExec(1330 await dockerExec(vm.containerName, "xdotool", ["mouseup", 1], {
1147 `DISPLAY=${vm.display} xdotool mouseup 1`,1331 env: { DISPLAY: vm.display },
1148 vm.containerName1332 });
1149 );
1150 });1333 });
1151 break;1334 break;
1152 }1335 }
1153 case "move": {1336 case "move": {
1154 await withModifiers(vm, action.keys, async () => {1337 await withModifiers(vm, action.keys, async () => {
1155 await dockerExec(1338 await dockerExec(
1156 `DISPLAY=${vm.display} xdotool mousemove ${action.x} ${action.y}`,1339 vm.containerName,
1157 vm.containerName1340 "xdotool",
1341 ["mousemove", action.x, action.y],
1342 { env: { DISPLAY: vm.display } }
1158 );1343 );
1159 });1344 });
1160 break;1345 break;
1161 }1346 }
1162 case "scroll": {1347 case "scroll": {
1163 const button = action.scrollY < 0 ? 4 : 5;1348 const buttons = getXdotoolScrollButtons(
1164 const clicks = Math.max(1, Math.abs(Math.round(action.scrollY / 100)));1349 action.scroll_x,
1165 await withModifiers(vm, action.keys, async () => {1350 action.scroll_y
1166 await dockerExec(
1167 `DISPLAY=${vm.display} xdotool mousemove ${action.x} ${action.y}`,
1168 vm.containerName
1169 );1351 );
1170 for (let i = 0; i < clicks; i += 1) {1352 await withModifiers(vm, action.keys, async () => {
1171 await dockerExec(1353 await dockerExec(
1172 `DISPLAY=${vm.display} xdotool click ${button}`,1354 vm.containerName,
1173 vm.containerName1355 "xdotool",
1356 ["mousemove", action.x, action.y],
1357 { env: { DISPLAY: vm.display } }
1174 );1358 );
1359 for (const button of buttons) {
1360 await dockerExec(vm.containerName, "xdotool", ["click", button], {
1361 env: { DISPLAY: vm.display },
1362 });
1175 }1363 }
1176 });1364 });
1177 break;1365 break;
1179 case "keypress":1367 case "keypress":
1180 for (const key of action.keys) {1368 for (const key of action.keys) {
1181 await dockerExec(1369 await dockerExec(
1182 `DISPLAY=${vm.display} xdotool key '${normalizeXdotoolKey(key)}'`,1370 vm.containerName,
1183 vm.containerName1371 "xdotool",
1372 ["key", normalizeXdotoolKey(key)],
1373 { env: { DISPLAY: vm.display } }
1184 );1374 );
1185 }1375 }
1186 break;1376 break;
1187 case "type":1377 case "type":
1188 await dockerExec(1378 await dockerExec(
1189 `DISPLAY=${vm.display} xdotool type --delay 0 '${action.text}'`,1379 vm.containerName,
1190 vm.containerName1380 "xdotool",
1381 ["type", "--delay", 0, action.text],
1382 { env: { DISPLAY: vm.display } }
1191 );1383 );
1192 break;1384 break;
1193 case "wait":1385 case "wait":
1386 await new Promise((resolve) => setTimeout(resolve, 2000));
1387 break;
1194 case "screenshot":1388 case "screenshot":
1195 break;1389 break;
1196 default:1390 default:
1204import time1398import time
1205 1399
1206# Reuse normalize_xdotool_key from the helper above.1400# Reuse normalize_xdotool_key from the helper above.
1401# Reuse normalize_xdotool_button and get_xdotool_scroll_buttons from the helper above.
1207# Reuse normalize_drag_path from the helper above.1402# Reuse normalize_drag_path from the helper above.
1208 1403
1209 1404
1229 1424
1230 1425
1231def handle_computer_actions(vm, actions):1426def handle_computer_actions(vm, actions):
1232 button_map = {"left": 1, "middle": 2, "right": 3}
1233
1234 for action in actions:1427 for action in actions:
1235 match action.type:1428 match action.type:
1236 case "click":1429 case "click":
1237 button = button_map.get(getattr(action, "button", "left"), 1)1430 button = normalize_xdotool_button(
1431 getattr(action, "button", "left")
1432 )
1238 with_modifiers(1433 with_modifiers(
1239 vm,1434 vm,
1240 getattr(action, "keys", None),1435 getattr(action, "keys", None),
1244 ),1439 ),
1245 )1440 )
1246 case "double_click":1441 case "double_click":
1247 button = button_map.get(getattr(action, "button", "left"), 1)
1248 with_modifiers(1442 with_modifiers(
1249 vm,1443 vm,
1250 getattr(action, "keys", None),1444 getattr(action, "keys", None),
1251 lambda: docker_exec(1445 lambda: docker_exec(
1252 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y} click --repeat 2 {button}",1446 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y} click --repeat 2 1",
1253 vm.container_name,1447 vm.container_name,
1254 ),1448 ),
1255 )1449 )
1285 ),1479 ),
1286 )1480 )
1287 case "scroll":1481 case "scroll":
1288 button = 4 if getattr(action, "scrollY", 0) < 0 else 51482 buttons = get_xdotool_scroll_buttons(
1289 clicks = max(1, abs(round(getattr(action, "scrollY", 0) / 100)))1483 action.scroll_x,
1484 action.scroll_y,
1485 )
1290 1486
1291 def do_scroll():1487 def do_scroll():
1292 docker_exec(1488 docker_exec(
1293 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y}",1489 f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y}",
1294 vm.container_name,1490 vm.container_name,
1295 )1491 )
1296 for _ in range(clicks):1492 for button in buttons:
1297 docker_exec(1493 docker_exec(
1298 f"DISPLAY={vm.display} xdotool click {button}",1494 f"DISPLAY={vm.display} xdotool click {button}",
1299 vm.container_name,1495 vm.container_name,
1352```javascript1548```javascript
1353async function captureScreenshot(vm) {1549async function captureScreenshot(vm) {
1354 return await dockerExec(1550 return await dockerExec(
1355 `export DISPLAY=${vm.display} && import -window root png:-`,
1356 vm.containerName,1551 vm.containerName,
1357 false1552 "import",
1553 ["-window", "root", "png:-"],
1554 { decode: false, env: { DISPLAY: vm.display } }
1358 );1555 );
1359}1556}
1360```1557```
1384const client = new OpenAI();1581const client = new OpenAI();
1385 1582
1386async function sendComputerScreenshot(response, callId, screenshotBase64) {1583async function sendComputerScreenshot(response, callId, screenshotBase64) {
1584 const output = /** @type {const} */ ({
1585 type: "computer_screenshot",
1586 image_url: `data:image/png;base64,${screenshotBase64}`,
1587 detail: "original",
1588 });
1589
1387 return await client.responses.create({1590 return await client.responses.create({
1388 model: "gpt-5.6",1591 model: "gpt-5.6",
1389 tools: [{ type: "computer" }],1592 tools: [{ type: "computer" }],
1392 {1595 {
1393 type: "computer_call_output",1596 type: "computer_call_output",
1394 call_id: callId,1597 call_id: callId,
1395 output: {1598 output,
1396 type: "computer_screenshot",
1397 image_url: `data:image/png;base64,${screenshotBase64}`,
1398 detail: "original",
1399 },
1400 },1599 },
1401 ],1600 ],
1402 });1601 });
1442 1641
1443async function computerUseLoop(target, response) {1642async function computerUseLoop(target, response) {
1444 while (true) {1643 while (true) {
1445 const computerCall = response.output.find((item) => item.type === "computer_call");1644 const computerCall = response.output.find(
1645 (item) => item.type === "computer_call"
1646 );
1446 if (!computerCall) {1647 if (!computerCall) {
1447 return response;1648 return response;
1448 }1649 }
1451 1652
1452 const screenshot = await captureScreenshot(target);1653 const screenshot = await captureScreenshot(target);
1453 const screenshotBase64 = Buffer.from(screenshot).toString("base64");1654 const screenshotBase64 = Buffer.from(screenshot).toString("base64");
1655 const output = /** @type {const} */ ({
1656 type: "computer_screenshot",
1657 image_url: `data:image/png;base64,${screenshotBase64}`,
1658 detail: "original",
1659 });
1454 1660
1455 response = await client.responses.create({1661 response = await client.responses.create({
1456 model: "gpt-5.6",1662 model: "gpt-5.6",
1460 {1666 {
1461 type: "computer_call_output",1667 type: "computer_call_output",
1462 call_id: computerCall.call_id,1668 call_id: computerCall.call_id,
1463 output: {1669 output,
1464 type: "computer_screenshot",
1465 image_url: `data:image/png;base64,${screenshotBase64}`,
1466 detail: "original",
1467 },
1468 },1670 },
1469 ],1671 ],
1470 });1672 });
1565 1767
1566These minimal JavaScript and Python implementations demonstrate a code-execution harness. They give the model a code-execution tool, keep Playwright objects available to the runtime, return text and screenshots back to the model, and let the model ask the user clarifying questions when it gets blocked.1768These minimal JavaScript and Python implementations demonstrate a code-execution harness. They give the model a code-execution tool, keep Playwright objects available to the runtime, return text and screenshots back to the model, and let the model ask the user clarifying questions when it gets blocked.
1567 1769
1770Run model-generated code only inside a disposable, least-privilege container or VM with resource and network limits. Language-level sandboxes such as Node.js `vm` and restricted Python global variables are not security boundaries. Keep the sandbox in a separate process and security boundary from the API client, with no shared credentials or host mounts. Enforce time and resource limits inside the sandbox, and terminate the runtime when it exceeds them.
1771
1772The examples below do not run generated code in the API client. They send each approved snippet to the separately isolated service configured by `OPENAI_EXAMPLE_CODE_EXECUTION_URL`, with an optional `OPENAI_EXAMPLE_CODE_EXECUTION_TOKEN`. The service accepts `{ session_id, language, code }` and returns `{ output }`, where `output` contains Responses API `input_text` or `input_image` items. It owns the persistent Playwright objects and must validate requests, authenticate callers, enforce its own execution deadline, and return only validated output. The client-side timeout only limits how long the example waits for a response.
1773
1568 1774
1569 1775
1570<div data-content-switcher-pane data-value="javascript">1776<div data-content-switcher-pane data-value="javascript">
1573 1779
1574```javascript1780```javascript
1575// Run with:1781// Run with:
1576// bun run -i cua_code_mode.ts1782// pnpm example -- tools/cua/015-code-execution-harness-example.ts
1577// Override the user prompt with:1783// Override the user prompt with:
1578// bun run -i cua_code_mode.ts --prompt "Go to example.com and summarize the page."1784// pnpm example -- tools/cua/015-code-execution-harness-example.ts --prompt "Go to example.com and summarize the page."
1579// Note: this script intentionally leaves the Playwright browser open after the1785//
1580// model reaches a final answer. Because the browser/context are not closed,1786// Requires OPENAI_EXAMPLE_CODE_EXECUTION_URL to point to a separately isolated
1581// Bun stays alive until you close the browser or stop the process manually.1787// sandbox service. The service keeps a browser, context, and page alive for each
1788// session and returns text or image outputs. Do not run model-generated code in
1789// this API client process.
1790
1791import { randomUUID } from "node:crypto";
1792import readline from "node:readline/promises";
1582 1793
1583import OpenAI from "openai";1794import OpenAI from "openai";
1584import readline from "node:readline/promises";1795
1585import vm from "node:vm";1796const EXECUTION_TIMEOUT_MS = 30_000;
1586import { chromium } from "playwright";1797
1587import util from "node:util";1798type ExecutionOutput =
1799 | { type: "input_text"; text: string }
1800 | {
1801 type: "input_image";
1802 image_url: string;
1803 detail: "original";
1804 };
1805
1806function isExecutionOutput(value: unknown): value is ExecutionOutput {
1807 if (typeof value !== "object" || value === null || !("type" in value)) {
1808 return false;
1809 }
1810 if (
1811 value.type === "input_text" &&
1812 "text" in value &&
1813 typeof value.text === "string"
1814 ) {
1815 return true;
1816 }
1817 return (
1818 value.type === "input_image" &&
1819 "image_url" in value &&
1820 typeof value.image_url === "string" &&
1821 "detail" in value &&
1822 value.detail === "original"
1823 );
1824}
1825
1826async function executeInSandbox(
1827 code: string,
1828 sessionId: string
1829): Promise<ExecutionOutput[]> {
1830 const endpoint = process.env.OPENAI_EXAMPLE_CODE_EXECUTION_URL;
1831 if (!endpoint) {
1832 return [
1833 {
1834 type: "input_text",
1835 text: "Execution blocked. Configure OPENAI_EXAMPLE_CODE_EXECUTION_URL with a separately isolated sandbox service.",
1836 },
1837 ];
1838 }
1839
1840 const headers: Record<string, string> = {
1841 "content-type": "application/json",
1842 };
1843 const token = process.env.OPENAI_EXAMPLE_CODE_EXECUTION_TOKEN;
1844 if (token) headers.authorization = `Bearer ${token}`;
1845
1846 const response = await fetch(endpoint, {
1847 method: "POST",
1848 headers,
1849 body: JSON.stringify({
1850 session_id: sessionId,
1851 language: "javascript",
1852 code,
1853 }),
1854 signal: AbortSignal.timeout(EXECUTION_TIMEOUT_MS),
1855 });
1856 if (!response.ok) {
1857 throw new Error(
1858 `Sandbox request failed with ${response.status} ${response.statusText}`
1859 );
1860 }
1861
1862 const payload: unknown = await response.json();
1863 if (
1864 typeof payload !== "object" ||
1865 payload === null ||
1866 !("output" in payload) ||
1867 !Array.isArray(payload.output) ||
1868 !payload.output.every(isExecutionOutput)
1869 ) {
1870 throw new Error("Sandbox returned an invalid output payload.");
1871 }
1872 return payload.output;
1873}
1588 1874
1589async function main(1875async function main(
1590 prompt: string = "Go to Hacker News, click on the most interesting link (be prepared to justify your choice), take a screenshot, and give me a critique of the visual layout.",1876 prompt: string = "Go to Hacker News, click on the most interesting link (be prepared to justify your choice), take a screenshot, and give me a critique of the visual layout.",
1591 max_steps: number = 50,1877 maxSteps: number = 50,
1592 model: string = "gpt-5.6"1878 model: string = "gpt-5.6"
1593) {1879) {
1594 type Phase = null | "commentary" | "final_answer";1880 type Phase = null | "commentary" | "final_answer";
1597 input: process.stdin,1883 input: process.stdin,
1598 output: process.stdout,1884 output: process.stdout,
1599 });1885 });
1600 const browser = await chromium.launch({1886 const sessionId = randomUUID();
1601 headless: false,1887 const conversation: any[] = [{ role: "user", content: prompt }];
1602 args: ["--window-size=1440,900"],
1603 });
1604 const context = await browser.newContext({
1605 viewport: { width: 1440, height: 900 },
1606 });
1607 const page = await context.newPage();
1608
1609 const conversation: any[] = [];
1610 const js_output: any[] = [];
1611 const sandbox: Record<string, any> = {
1612 console: {
1613 log: (...xs: any[]) => {
1614 js_output.push({
1615 type: "input_text",
1616 text: util.formatWithOptions(
1617 { showHidden: false, getters: false, maxStringLength: 2000 },
1618 ...xs
1619 ),
1620 });
1621 },
1622 },
1623 browser: browser,
1624 context: context,
1625 page: page,
1626 display: (base64_image: string) => {
1627 js_output.push({
1628 type: "input_image",
1629 image_url: `data:image/png;base64,${base64_image}`,
1630 detail: "original",
1631 });
1632 },
1633 };
1634 const ctx = vm.createContext(sandbox);
1635
1636 conversation.push({
1637 role: "user",
1638 content: prompt,
1639 });
1640 1888
1641 for (let i = 0; i < max_steps; i++) {1889 try {
1642 const resp = await client.responses.create({1890 for (let i = 0; i < maxSteps; i++) {
1891 const response = await client.responses.create({
1643 model,1892 model,
1644 tools: [1893 tools: [
1645 {1894 {
1646 type: "function" as const,1895 type: "function" as const,
1647 name: "exec_js",1896 name: "exec_js",
1648 description:1897 description:
1649 "Execute provided interactive JavaScript in a persistent REPL context.",1898 "Execute provided interactive JavaScript in a persistent, isolated browser runtime.",
1650 parameters: {1899 parameters: {
1651 type: "object",1900 type: "object",
1652 properties: {1901 properties: {
1653 code: {1902 code: {
1654 type: "string",1903 type: "string",
1655 description: `1904 description: `
1656JavaScript to execute. Write small snippets of interactive code. To persist variables or functions across tool calls, you must save them to globalThis. Code is executed in an async node:vm context, so you can use await. You have access to ONLY the following:1905JavaScript to execute. Write small snippets of interactive code. To persist variables or functions across tool calls, save them to globalThis. The isolated runtime supports await and provides only these helpers and Playwright objects:
1657- console.log(x): Use this to read contents back to you. But be minimal: otherwise the output may be too long. Avoid using console.log() for large base64 payloads like screenshots or buffer. If you create an image or screenshot, pass the base64 string to display().1906- console.log(x): Return concise text. Do not log large base64 payloads, screenshots, buffers, page HTML, or other large blobs.
1658- display(base64_image_string): Use this to view a base64-encoded image.1907- display(base64_image_string): Return a base64-encoded image.
1659- Do not write screenshots or image data to temporary files or disk just to pass them back. Keep image data in memory and send it directly to display().1908- browser: A Playwright Chromium browser instance.
1660- Do not assume package globals like Bun.file are available unless they are explicitly provided.1909- context: A Playwright browser context with viewport 1440x900.
1661- browser: A playwright chromium browser instance.1910- page: A Playwright page already created in that context.
1662- context: A playwright browser context with viewport 1440x900.1911Keep screenshots and image data in memory and pass them directly to display(). Do not assume other globals or packages are available.
1663- page: A playwright page already created in that context.
1664`,1912`,
1665 },1913 },
1666 },1914 },
1667 required: ["code"],1915 required: ["code"],
1668 additionalProperties: false,1916 additionalProperties: false,
1669 },1917 },
1918 strict: true,
1670 },1919 },
1671 {1920 {
1672 type: "function" as const,1921 type: "function" as const,
1685 required: ["question"],1934 required: ["question"],
1686 additionalProperties: false,1935 additionalProperties: false,
1687 },1936 },
1937 strict: true,
1688 },1938 },
1689 ],1939 ],
1690 input: conversation,1940 input: conversation,
1693 },1943 },
1694 });1944 });
1695 1945
1696 // Save model outputs into the running conversation1946 conversation.push(...response.output);
1697 conversation.push(...resp.output);
1698
1699 let hadToolCall = false;1947 let hadToolCall = false;
1700 let latestPhase: Phase = null;1948 let latestPhase: Phase = null;
1701 1949
1702 // Handle tool calls1950 for (const item of response.output) {
1703 for (const item of resp.output) {
1704 if (item.type === "function_call" && item.name === "exec_js") {1951 if (item.type === "function_call" && item.name === "exec_js") {
1705 hadToolCall = true;1952 hadToolCall = true;
1706 const parsed = JSON.parse(item.arguments ?? "{}") as {1953 const parsed = JSON.parse(item.arguments ?? "{}") as {
1709 const code = parsed.code ?? "";1956 const code = parsed.code ?? "";
1710 console.log(code);1957 console.log(code);
1711 console.log("----");1958 console.log("----");
1712 const wrappedCode = `
1713 (async () => {
1714 ${code}
1715 })();
1716 `;
1717 1959
1960 let executionOutput: ExecutionOutput[];
1961 const endpoint = process.env.OPENAI_EXAMPLE_CODE_EXECUTION_URL;
1962 if (!endpoint) {
1963 executionOutput = await executeInSandbox(code, sessionId);
1964 } else {
1965 const approval = await rl.question(
1966 "Send this generated JavaScript to the isolated runtime? Type yes to continue: "
1967 );
1968 if (approval.trim().toLowerCase() !== "yes") {
1969 executionOutput = [
1970 {
1971 type: "input_text",
1972 text: "The user declined this code execution.",
1973 },
1974 ];
1975 } else {
1718 try {1976 try {
1719 await new vm.Script(wrappedCode, {1977 executionOutput = await executeInSandbox(code, sessionId);
1720 filename: "exec_js.js",1978 } catch (error) {
1721 }).runInContext(ctx);1979 executionOutput = [
1722 } catch (e: any) {1980 {
1723 sandbox.console.log(e, e?.message, e?.stack);1981 type: "input_text",
1982 text:
1983 error instanceof Error ? error.message : String(error),
1984 },
1985 ];
1986 }
1987 }
1724 }1988 }
1725 1989
1726 // Send tool output back to the model, keyed by call_id
1727 conversation.push({1990 conversation.push({
1728 type: "function_call_output",1991 type: "function_call_output",
1729 call_id: item.call_id,1992 call_id: item.call_id,
1730 output: js_output.slice(),1993 output: executionOutput,
1731 });1994 });
1732 1995
1733 for (const out of js_output) {1996 for (const output of executionOutput) {
1734 if (out.type === "input_text") {1997 if (output.type === "input_text") {
1735 console.log("JS LOG:", out.text);1998 console.log("JS LOG:", output.text);
1736 } else if (out.type === "input_image") {1999 } else {
1737 console.log("JS IMAGE: [base64 string omitted]");2000 console.log("JS IMAGE: [base64 string omitted]");
1738 }2001 }
1739 }2002 }
1740 console.log("=====");2003 console.log("=====");
1741
1742 js_output.length = 0;
1743 } else if (item.type === "function_call" && item.name === "ask_user") {2004 } else if (item.type === "function_call" && item.name === "ask_user") {
1744 hadToolCall = true;2005 hadToolCall = true;
1745 const parsed = JSON.parse(item.arguments ?? "{}") as {2006 const parsed = JSON.parse(item.arguments ?? "{}") as {
1746 question?: string;2007 question?: string;
1747 };2008 };
1748 const question = parsed.question ?? "Please provide more information.";2009 const question =
2010 parsed.question ?? "Please provide more information.";
1749 console.log(`MODEL QUESTION: ${question}`);2011 console.log(`MODEL QUESTION: ${question}`);
1750 const answer = await rl.question("> ");2012 const answer = await rl.question("> ");
1751 conversation.push({2013 conversation.push({
1754 output: answer,2016 output: answer,
1755 });2017 });
1756 } else if (item.type === "message") {2018 } else if (item.type === "message") {
1757 console.log(item.content[0]?.text ?? item.content);2019 const text = item.content.find((part) => part.type === "output_text");
2020 console.log(text?.text ?? item.content);
1758 if ("phase" in item) {2021 if ("phase" in item) {
1759 latestPhase = (item.phase as Phase) ?? null;2022 latestPhase = (item.phase as Phase) ?? null;
1760 }2023 }
1761 } else if (item.type === "output_item.done" && "phase" in item) {
1762 latestPhase = (item.phase as Phase) ?? null;
1763 }2024 }
1764 }2025 }
1765 2026
1766 // Stop only when the model explicitly marks the turn as a final answer
1767 // and there were no tool calls in the same turn.
1768 if (!hadToolCall && latestPhase === "final_answer") return;2027 if (!hadToolCall && latestPhase === "final_answer") return;
1769 }2028 }
2029 } finally {
2030 rl.close();
2031 }
1770}2032}
1771 2033
1772function getCliPrompt(): string | undefined {2034function getCliPrompt(): string | undefined {
1773 const args = Bun.argv.slice(2);2035 const args = process.argv.slice(2);
1774 for (let i = 0; i < args.length; i++) {2036 for (let i = 0; i < args.length; i++) {
1775 if (args[i] === "--prompt") {2037 if (args[i] === "--prompt") return args[i + 1];
1776 return args[i + 1];
1777 }
1778 }2038 }
1779 return undefined;2039 return undefined;
1780}2040}
1781 2041
1782main(getCliPrompt());2042await main(getCliPrompt());
1783```2043```
1784 2044
2045 </div>
2046 <div data-content-switcher-pane data-value="python" hidden>
2047 <div class="hidden">Python</div>
2048 Code-execution harness
2049
1785```python2050```python
1786# /// script2051# /// script
1787# requires-python = ">=3.10"2052# requires-python = ">=3.10"
1788# dependencies = [2053# dependencies = [
1789# "openai",2054# "openai",
1790# "playwright",
1791# ]2055# ]
1792# ///2056# ///
1793# Run with: `uv run cua_code_mode_py_async.py`2057# Run with: `uv run cua_code_mode_py_async.py`
1794# Override the user prompt with:2058# Override the user prompt with:
1795# `uv run cua_code_mode_py_async.py --prompt "Go to example.com and summarize the page."`2059# `uv run cua_code_mode_py_async.py --prompt "Go to example.com and summarize the page."`
1796# Install Chromium once first: `uv run --with playwright python -m playwright install chromium`2060# Requires `OPENAI_API_KEY` and `OPENAI_EXAMPLE_CODE_EXECUTION_URL`.
1797# Requires `OPENAI_API_KEY` in the environment.
1798 2061
1799"""Async Python analogue of cua_code_mode.ts.2062"""Async Python analogue of cua_code_mode.ts.
1800 2063
1801Runs a Responses API loop with one persistent Playwright browser/context/page,2064The API client sends approved snippets to a separately isolated sandbox service.
1802and tools that let the model execute short async Python snippets and ask the2065The sandbox keeps a Playwright browser, context, and page alive for each session
1803user clarifying questions.2066and returns text or image outputs. Never run model-generated code in this API
1804 2067client process.
1805The model can return visual observations by calling:
1806 display(base64_png_string)
1807"""2068"""
1808 2069
1809from __future__ import annotations2070from __future__ import annotations
1811import argparse2072import argparse
1812import asyncio2073import asyncio
1813import json2074import json
1814import traceback2075import os
2076import uuid
1815from typing import Any2077from typing import Any
2078from urllib import request
1816 2079
1817from openai import OpenAI2080from openai import OpenAI
1818from playwright.async_api import async_playwright
1819 2081
1820Phase = str | None2082Phase = str | None
2083EXECUTION_TIMEOUT_SECONDS = 30
1821 2084
1822 2085
1823def _message_text(item: Any) -> str:2086def _message_text(item: Any) -> str:
1840 return await asyncio.to_thread(input, prompt)2103 return await asyncio.to_thread(input, prompt)
1841 2104
1842 2105
2106def _is_execution_output(value: Any) -> bool:
2107 if not isinstance(value, dict):
2108 return False
2109 if value.get("type") == "input_text":
2110 return isinstance(value.get("text"), str)
2111 return (
2112 value.get("type") == "input_image"
2113 and isinstance(value.get("image_url"), str)
2114 and value.get("detail") == "original"
2115 )
2116
2117
2118def _execute_in_sandbox(code: str, session_id: str) -> list[dict[str, Any]]:
2119 endpoint = os.environ.get("OPENAI_EXAMPLE_CODE_EXECUTION_URL")
2120 if not endpoint:
2121 return [
2122 {
2123 "type": "input_text",
2124 "text": (
2125 "Execution blocked. Configure OPENAI_EXAMPLE_CODE_EXECUTION_URL "
2126 "with a separately isolated sandbox service."
2127 ),
2128 }
2129 ]
2130
2131 headers = {"Content-Type": "application/json"}
2132 token = os.environ.get("OPENAI_EXAMPLE_CODE_EXECUTION_TOKEN")
2133 if token:
2134 headers["Authorization"] = f"Bearer {token}"
2135
2136 body = json.dumps(
2137 {
2138 "session_id": session_id,
2139 "language": "python",
2140 "code": code,
2141 }
2142 ).encode()
2143 sandbox_request = request.Request(
2144 endpoint,
2145 data=body,
2146 headers=headers,
2147 method="POST",
2148 )
2149 with request.urlopen(
2150 sandbox_request,
2151 timeout=EXECUTION_TIMEOUT_SECONDS,
2152 ) as response:
2153 payload = json.loads(response.read())
2154
2155 output = payload.get("output") if isinstance(payload, dict) else None
2156 if not isinstance(output, list) or not all(
2157 _is_execution_output(item) for item in output
2158 ):
2159 raise ValueError("Sandbox returned an invalid output payload.")
2160 return output
2161
2162
1843async def main(2163async def main(
1844 prompt: str = "Go to Hacker News, click on the most interesting link (be prepared to justify your choice), take a screenshot, and give me a critique of the visual layout.",2164 prompt: str = "Go to Hacker News, click on the most interesting link (be prepared to justify your choice), take a screenshot, and give me a critique of the visual layout.",
1845 max_steps: int = 20,2165 max_steps: int = 20,
1846 model: str = "gpt-5.6",2166 model: str = "gpt-5.6",
1847) -> None:2167) -> None:
1848 client = OpenAI()2168 client = OpenAI()
2169 session_id = str(uuid.uuid4())
1849 2170
1850 async with async_playwright() as p:2171 async def run_loop() -> None:
1851 browser = await p.chromium.launch(
1852 headless=False,
1853 args=["--window-size=1440,900"],
1854 )
1855 context = await browser.new_context(viewport={"width": 1440, "height": 900})
1856 page = await context.new_page()
1857
1858 conversation: list[dict[str, Any]] = [{"role": "user", "content": prompt}]2172 conversation: list[dict[str, Any]] = [{"role": "user", "content": prompt}]
1859 py_output: list[dict[str, Any]] = []
1860
1861 def log(*xs: Any) -> None:
1862 text = " ".join(str(x) for x in xs)
1863 py_output.append({"type": "input_text", "text": text[:5000]})
1864
1865 def display(base64_image: str) -> None:
1866 py_output.append(
1867 {
1868 "type": "input_image",
1869 "image_url": f"data:image/png;base64,{base64_image}",
1870 "detail": "original",
1871 }
1872 )
1873
1874 runtime_globals: dict[str, Any] = {
1875 "__builtins__": __builtins__,
1876 "asyncio": asyncio,
1877 "browser": browser,
1878 "context": context,
1879 "page": page,
1880 "display": display,
1881 "log": log,
1882 }
1883 2173
1884 for _ in range(max_steps):2174 for _ in range(max_steps):
1885 resp = client.responses.create(2175 resp = client.responses.create(
1888 {2178 {
1889 "type": "function",2179 "type": "function",
1890 "name": "exec_py",2180 "name": "exec_py",
1891 "description": "Execute provided interactive async Python in a persistent runtime context.",2181 "description": "Execute provided interactive async Python in a persistent, isolated browser runtime.",
1892 "parameters": {2182 "parameters": {
1893 "type": "object",2183 "type": "object",
1894 "properties": {2184 "properties": {
1897 "description": (2187 "description": (
1898 "Python code to execute. Write small snippets. "2188 "Python code to execute. Write small snippets. "
1899 "State persists across tool calls via globals(). "2189 "State persists across tool calls via globals(). "
1900 "This runtime uses Playwright's async Python API, so you may use await directly. "2190 "The isolated runtime supports await and provides only these helpers and Playwright objects: "
1901 "Do not call asyncio.run(...), loop.run_until_complete(...), or manage the event loop yourself. "2191 "log(x) for concise text output, display(base64_png_string) for image output, "
1902 "You can use ONLY these prebound objects/helpers: "2192 "browser (async Playwright browser), context (viewport 1440x900), and page. "
1903 "log(x) for text output, display(base64_png_string) for image output, "2193 "Keep screenshots and image data in memory and pass them directly to display(). "
1904 "browser (async Playwright browser), context (viewport 1440x900), page (already created), "2194 "Do not assume other globals or packages are available."
1905 "asyncio (module). "
1906 "Be concise with log(x): do not send large base64 payloads, screenshots, buffers, page HTML, "
1907 "or other large blobs through log(). If you create an image or screenshot, pass the base64 PNG "
1908 "string to display(). Do not write screenshots or image data to temporary files or disk just "
1909 "to pass them back; keep image data in memory and send it directly to display(). "
1910 "Do not assume extra globals or helpers are available unless they are explicitly listed here. "
1911 "Do not close browser/context/page unless explicitly asked."
1912 ),2195 ),
1913 }2196 }
1914 },2197 },
1915 "required": ["code"],2198 "required": ["code"],
1916 "additionalProperties": False,2199 "additionalProperties": False,
1917 },2200 },
2201 "strict": True,
1918 },2202 },
1919 {2203 {
1920 "type": "function",2204 "type": "function",
1931 "required": ["question"],2215 "required": ["question"],
1932 "additionalProperties": False,2216 "additionalProperties": False,
1933 },2217 },
2218 "strict": True,
1934 },2219 },
1935 ],2220 ],
1936 input=conversation,2221 input=conversation,
1956 print(code)2241 print(code)
1957 print("----")2242 print("----")
1958 2243
1959 wrapped = (2244 if not os.environ.get("OPENAI_EXAMPLE_CODE_EXECUTION_URL"):
1960 "async def __codex_exec__():\n"2245 py_output = _execute_in_sandbox(code, session_id)
1961 + "".join(2246 else:
1962 f" {line}\n" if line else " \n"2247 approval = await _ainput(
1963 for line in (code or "pass").splitlines()2248 "Send this generated Python to the isolated runtime? "
1964 )2249 "Type yes to continue: "
1965 )2250 )
1966 2251 if approval.strip().lower() != "yes":
2252 py_output = [
2253 {
2254 "type": "input_text",
2255 "text": "The user declined this code execution.",
2256 }
2257 ]
2258 else:
1967 try:2259 try:
1968 exec(wrapped, runtime_globals, runtime_globals)2260 py_output = await asyncio.wait_for(
1969 await runtime_globals["__codex_exec__"]()2261 asyncio.to_thread(
1970 except Exception:2262 _execute_in_sandbox,
1971 log(traceback.format_exc())2263 code,
2264 session_id,
2265 ),
2266 timeout=EXECUTION_TIMEOUT_SECONDS,
2267 )
2268 except Exception as exc:
2269 py_output = [
2270 {
2271 "type": "input_text",
2272 "text": str(exc),
2273 }
2274 ]
1972 2275
1973 conversation.append(2276 conversation.append(
1974 {2277 {
1975 "type": "function_call_output",2278 "type": "function_call_output",
1976 "call_id": getattr(item, "call_id", None),2279 "call_id": getattr(item, "call_id", None),
1977 "output": py_output[:],2280 "output": py_output,
1978 }2281 }
1979 )2282 )
1980 2283
1985 print("PY IMAGE: [base64 string omitted]")2288 print("PY IMAGE: [base64 string omitted]")
1986 print("=====")2289 print("=====")
1987 2290
1988 py_output.clear()
1989
1990 elif item_type == "function_call" and getattr(item, "name", None) == "ask_user":2291 elif item_type == "function_call" and getattr(item, "name", None) == "ask_user":
1991 had_tool_call = True2292 had_tool_call = True
1992 raw_args = getattr(item, "arguments", "{}") or "{}"2293 raw_args = getattr(item, "arguments", "{}") or "{}"
2024 if not had_tool_call and latest_phase == "final_answer":2325 if not had_tool_call and latest_phase == "final_answer":
2025 return2326 return
2026 2327
2027 2328 await run_loop()
2028if __name__ == "__main__":
2029 parser = argparse.ArgumentParser()
2030 parser.add_argument("--prompt", help="Override the default user prompt.")
2031 args = parser.parse_args()
2032 asyncio.run(main(prompt=args.prompt) if args.prompt is not None else main())
2033```
2034
2035 </div>
2036 <div data-content-switcher-pane data-value="python" hidden>
2037 <div class="hidden">Python</div>
2038 Code-execution harness
2039
2040```javascript
2041// Run with:
2042// bun run -i cua_code_mode.ts
2043// Override the user prompt with:
2044// bun run -i cua_code_mode.ts --prompt "Go to example.com and summarize the page."
2045// Note: this script intentionally leaves the Playwright browser open after the
2046// model reaches a final answer. Because the browser/context are not closed,
2047// Bun stays alive until you close the browser or stop the process manually.
2048
2049import OpenAI from "openai";
2050import readline from "node:readline/promises";
2051import vm from "node:vm";
2052import { chromium } from "playwright";
2053import util from "node:util";
2054
2055async function main(
2056 prompt: string = "Go to Hacker News, click on the most interesting link (be prepared to justify your choice), take a screenshot, and give me a critique of the visual layout.",
2057 max_steps: number = 50,
2058 model: string = "gpt-5.6"
2059) {
2060 type Phase = null | "commentary" | "final_answer";
2061 const client = new OpenAI();
2062 const rl = readline.createInterface({
2063 input: process.stdin,
2064 output: process.stdout,
2065 });
2066 const browser = await chromium.launch({
2067 headless: false,
2068 args: ["--window-size=1440,900"],
2069 });
2070 const context = await browser.newContext({
2071 viewport: { width: 1440, height: 900 },
2072 });
2073 const page = await context.newPage();
2074
2075 const conversation: any[] = [];
2076 const js_output: any[] = [];
2077 const sandbox: Record<string, any> = {
2078 console: {
2079 log: (...xs: any[]) => {
2080 js_output.push({
2081 type: "input_text",
2082 text: util.formatWithOptions(
2083 { showHidden: false, getters: false, maxStringLength: 2000 },
2084 ...xs
2085 ),
2086 });
2087 },
2088 },
2089 browser: browser,
2090 context: context,
2091 page: page,
2092 display: (base64_image: string) => {
2093 js_output.push({
2094 type: "input_image",
2095 image_url: `data:image/png;base64,${base64_image}`,
2096 detail: "original",
2097 });
2098 },
2099 };
2100 const ctx = vm.createContext(sandbox);
2101
2102 conversation.push({
2103 role: "user",
2104 content: prompt,
2105 });
2106
2107 for (let i = 0; i < max_steps; i++) {
2108 const resp = await client.responses.create({
2109 model,
2110 tools: [
2111 {
2112 type: "function" as const,
2113 name: "exec_js",
2114 description:
2115 "Execute provided interactive JavaScript in a persistent REPL context.",
2116 parameters: {
2117 type: "object",
2118 properties: {
2119 code: {
2120 type: "string",
2121 description: `
2122JavaScript to execute. Write small snippets of interactive code. To persist variables or functions across tool calls, you must save them to globalThis. Code is executed in an async node:vm context, so you can use await. You have access to ONLY the following:
2123- console.log(x): Use this to read contents back to you. But be minimal: otherwise the output may be too long. Avoid using console.log() for large base64 payloads like screenshots or buffer. If you create an image or screenshot, pass the base64 string to display().
2124- display(base64_image_string): Use this to view a base64-encoded image.
2125- Do not write screenshots or image data to temporary files or disk just to pass them back. Keep image data in memory and send it directly to display().
2126- Do not assume package globals like Bun.file are available unless they are explicitly provided.
2127- browser: A playwright chromium browser instance.
2128- context: A playwright browser context with viewport 1440x900.
2129- page: A playwright page already created in that context.
2130`,
2131 },
2132 },
2133 required: ["code"],
2134 additionalProperties: false,
2135 },
2136 },
2137 {
2138 type: "function" as const,
2139 name: "ask_user",
2140 description:
2141 "Ask the user a clarification question and wait for their response.",
2142 parameters: {
2143 type: "object",
2144 properties: {
2145 question: {
2146 type: "string",
2147 description:
2148 "The exact question to show the human. Use this instead of answering with a freeform clarifying question in a final answer.",
2149 },
2150 },
2151 required: ["question"],
2152 additionalProperties: false,
2153 },
2154 },
2155 ],
2156 input: conversation,
2157 reasoning: {
2158 effort: "low",
2159 },
2160 });
2161
2162 // Save model outputs into the running conversation
2163 conversation.push(...resp.output);
2164
2165 let hadToolCall = false;
2166 let latestPhase: Phase = null;
2167
2168 // Handle tool calls
2169 for (const item of resp.output) {
2170 if (item.type === "function_call" && item.name === "exec_js") {
2171 hadToolCall = true;
2172 const parsed = JSON.parse(item.arguments ?? "{}") as {
2173 code?: string;
2174 };
2175 const code = parsed.code ?? "";
2176 console.log(code);
2177 console.log("----");
2178 const wrappedCode = `
2179 (async () => {
2180 ${code}
2181 })();
2182 `;
2183
2184 try {
2185 await new vm.Script(wrappedCode, {
2186 filename: "exec_js.js",
2187 }).runInContext(ctx);
2188 } catch (e: any) {
2189 sandbox.console.log(e, e?.message, e?.stack);
2190 }
2191
2192 // Send tool output back to the model, keyed by call_id
2193 conversation.push({
2194 type: "function_call_output",
2195 call_id: item.call_id,
2196 output: js_output.slice(),
2197 });
2198
2199 for (const out of js_output) {
2200 if (out.type === "input_text") {
2201 console.log("JS LOG:", out.text);
2202 } else if (out.type === "input_image") {
2203 console.log("JS IMAGE: [base64 string omitted]");
2204 }
2205 }
2206 console.log("=====");
2207
2208 js_output.length = 0;
2209 } else if (item.type === "function_call" && item.name === "ask_user") {
2210 hadToolCall = true;
2211 const parsed = JSON.parse(item.arguments ?? "{}") as {
2212 question?: string;
2213 };
2214 const question = parsed.question ?? "Please provide more information.";
2215 console.log(`MODEL QUESTION: ${question}`);
2216 const answer = await rl.question("> ");
2217 conversation.push({
2218 type: "function_call_output",
2219 call_id: item.call_id,
2220 output: answer,
2221 });
2222 } else if (item.type === "message") {
2223 console.log(item.content[0]?.text ?? item.content);
2224 if ("phase" in item) {
2225 latestPhase = (item.phase as Phase) ?? null;
2226 }
2227 } else if (item.type === "output_item.done" && "phase" in item) {
2228 latestPhase = (item.phase as Phase) ?? null;
2229 }
2230 }
2231
2232 // Stop only when the model explicitly marks the turn as a final answer
2233 // and there were no tool calls in the same turn.
2234 if (!hadToolCall && latestPhase === "final_answer") return;
2235 }
2236}
2237
2238function getCliPrompt(): string | undefined {
2239 const args = Bun.argv.slice(2);
2240 for (let i = 0; i < args.length; i++) {
2241 if (args[i] === "--prompt") {
2242 return args[i + 1];
2243 }
2244 }
2245 return undefined;
2246}
2247
2248main(getCliPrompt());
2249```
2250
2251```python
2252# /// script
2253# requires-python = ">=3.10"
2254# dependencies = [
2255# "openai",
2256# "playwright",
2257# ]
2258# ///
2259# Run with: `uv run cua_code_mode_py_async.py`
2260# Override the user prompt with:
2261# `uv run cua_code_mode_py_async.py --prompt "Go to example.com and summarize the page."`
2262# Install Chromium once first: `uv run --with playwright python -m playwright install chromium`
2263# Requires `OPENAI_API_KEY` in the environment.
2264
2265"""Async Python analogue of cua_code_mode.ts.
2266
2267Runs a Responses API loop with one persistent Playwright browser/context/page,
2268and tools that let the model execute short async Python snippets and ask the
2269user clarifying questions.
2270
2271The model can return visual observations by calling:
2272 display(base64_png_string)
2273"""
2274
2275from __future__ import annotations
2276
2277import argparse
2278import asyncio
2279import json
2280import traceback
2281from typing import Any
2282
2283from openai import OpenAI
2284from playwright.async_api import async_playwright
2285
2286Phase = str | None
2287
2288
2289def _message_text(item: Any) -> str:
2290 try:
2291 parts = getattr(item, "content", None)
2292 if isinstance(parts, list) and parts:
2293 out: list[str] = []
2294 for p in parts:
2295 t = getattr(p, "text", None)
2296 if isinstance(t, str) and t:
2297 out.append(t)
2298 if out:
2299 return "\n".join(out)
2300 except Exception:
2301 pass
2302 return str(item)
2303
2304
2305async def _ainput(prompt: str) -> str:
2306 return await asyncio.to_thread(input, prompt)
2307
2308
2309async def main(
2310 prompt: str = "Go to Hacker News, click on the most interesting link (be prepared to justify your choice), take a screenshot, and give me a critique of the visual layout.",
2311 max_steps: int = 20,
2312 model: str = "gpt-5.6",
2313) -> None:
2314 client = OpenAI()
2315
2316 async with async_playwright() as p:
2317 browser = await p.chromium.launch(
2318 headless=False,
2319 args=["--window-size=1440,900"],
2320 )
2321 context = await browser.new_context(viewport={"width": 1440, "height": 900})
2322 page = await context.new_page()
2323
2324 conversation: list[dict[str, Any]] = [{"role": "user", "content": prompt}]
2325 py_output: list[dict[str, Any]] = []
2326
2327 def log(*xs: Any) -> None:
2328 text = " ".join(str(x) for x in xs)
2329 py_output.append({"type": "input_text", "text": text[:5000]})
2330
2331 def display(base64_image: str) -> None:
2332 py_output.append(
2333 {
2334 "type": "input_image",
2335 "image_url": f"data:image/png;base64,{base64_image}",
2336 "detail": "original",
2337 }
2338 )
2339
2340 runtime_globals: dict[str, Any] = {
2341 "__builtins__": __builtins__,
2342 "asyncio": asyncio,
2343 "browser": browser,
2344 "context": context,
2345 "page": page,
2346 "display": display,
2347 "log": log,
2348 }
2349
2350 for _ in range(max_steps):
2351 resp = client.responses.create(
2352 model=model,
2353 tools=[
2354 {
2355 "type": "function",
2356 "name": "exec_py",
2357 "description": "Execute provided interactive async Python in a persistent runtime context.",
2358 "parameters": {
2359 "type": "object",
2360 "properties": {
2361 "code": {
2362 "type": "string",
2363 "description": (
2364 "Python code to execute. Write small snippets. "
2365 "State persists across tool calls via globals(). "
2366 "This runtime uses Playwright's async Python API, so you may use await directly. "
2367 "Do not call asyncio.run(...), loop.run_until_complete(...), or manage the event loop yourself. "
2368 "You can use ONLY these prebound objects/helpers: "
2369 "log(x) for text output, display(base64_png_string) for image output, "
2370 "browser (async Playwright browser), context (viewport 1440x900), page (already created), "
2371 "asyncio (module). "
2372 "Be concise with log(x): do not send large base64 payloads, screenshots, buffers, page HTML, "
2373 "or other large blobs through log(). If you create an image or screenshot, pass the base64 PNG "
2374 "string to display(). Do not write screenshots or image data to temporary files or disk just "
2375 "to pass them back; keep image data in memory and send it directly to display(). "
2376 "Do not assume extra globals or helpers are available unless they are explicitly listed here. "
2377 "Do not close browser/context/page unless explicitly asked."
2378 ),
2379 }
2380 },
2381 "required": ["code"],
2382 "additionalProperties": False,
2383 },
2384 },
2385 {
2386 "type": "function",
2387 "name": "ask_user",
2388 "description": "Ask the user a clarification question and wait for their response.",
2389 "parameters": {
2390 "type": "object",
2391 "properties": {
2392 "question": {
2393 "type": "string",
2394 "description": "The exact question to show the user. Use this instead of asking a freeform clarifying question in a final answer.",
2395 }
2396 },
2397 "required": ["question"],
2398 "additionalProperties": False,
2399 },
2400 },
2401 ],
2402 input=conversation,
2403 )
2404
2405 conversation.extend(resp.output)
2406
2407 had_tool_call = False
2408 latest_phase: Phase = None
2409
2410 for item in resp.output:
2411 item_type = getattr(item, "type", None)
2412
2413 if item_type == "function_call" and getattr(item, "name", None) == "exec_py":
2414 had_tool_call = True
2415 raw_args = getattr(item, "arguments", "{}") or "{}"
2416 try:
2417 args = json.loads(raw_args)
2418 except json.JSONDecodeError:
2419 args = {}
2420 code = args.get("code", "") if isinstance(args, dict) else ""
2421
2422 print(code)
2423 print("----")
2424
2425 wrapped = (
2426 "async def __codex_exec__():\n"
2427 + "".join(
2428 f" {line}\n" if line else " \n"
2429 for line in (code or "pass").splitlines()
2430 )
2431 )
2432
2433 try:
2434 exec(wrapped, runtime_globals, runtime_globals)
2435 await runtime_globals["__codex_exec__"]()
2436 except Exception:
2437 log(traceback.format_exc())
2438
2439 conversation.append(
2440 {
2441 "type": "function_call_output",
2442 "call_id": getattr(item, "call_id", None),
2443 "output": py_output[:],
2444 }
2445 )
2446
2447 for out in py_output:
2448 if out.get("type") == "input_text":
2449 print("PY LOG:", out.get("text", ""))
2450 elif out.get("type") == "input_image":
2451 print("PY IMAGE: [base64 string omitted]")
2452 print("=====")
2453
2454 py_output.clear()
2455
2456 elif item_type == "function_call" and getattr(item, "name", None) == "ask_user":
2457 had_tool_call = True
2458 raw_args = getattr(item, "arguments", "{}") or "{}"
2459 try:
2460 args = json.loads(raw_args)
2461 except json.JSONDecodeError:
2462 args = {}
2463 question = (
2464 args.get("question", "Please provide more information.")
2465 if isinstance(args, dict)
2466 else "Please provide more information."
2467 )
2468
2469 print(f"MODEL QUESTION: {question}")
2470 answer = await _ainput("> ")
2471
2472 conversation.append(
2473 {
2474 "type": "function_call_output",
2475 "call_id": getattr(item, "call_id", None),
2476 "output": answer,
2477 }
2478 )
2479
2480 elif item_type == "message":
2481 print(_message_text(item))
2482 phase = getattr(item, "phase", None)
2483 if isinstance(phase, str) or phase is None:
2484 latest_phase = phase
2485 elif item_type == "output_item.done":
2486 phase = getattr(item, "phase", None)
2487 if isinstance(phase, str) or phase is None:
2488 latest_phase = phase
2489
2490 if not had_tool_call and latest_phase == "final_answer":
2491 return
2492 2329
2493 2330
2494if __name__ == "__main__":2331if __name__ == "__main__":
