mirror of
https://github.com/harivansh-afk/sandbox-agent.git
synced 2026-04-17 09:02:12 +00:00
feat: desktop computer-use APIs with neko-based streaming
Add desktop runtime management (Xvfb, openbox, dbus), screen capture, mouse/keyboard input, and video streaming via neko binary extracted from the m1k1o/neko container. Includes Docker test rig, TypeScript SDK desktop support, and inspector Desktop tab. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
3895e34bdb
commit
33821d8660
66 changed files with 13190 additions and 1135 deletions
57
.context/docker-test-zgvGyf/bin/xdotool
Executable file
57
.context/docker-test-zgvGyf/bin/xdotool
Executable file
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
state_dir="${SANDBOX_AGENT_DESKTOP_FAKE_STATE_DIR:?missing fake state dir}"
|
||||
state_file="${state_dir}/mouse"
|
||||
mkdir -p "$state_dir"
|
||||
if [ ! -f "$state_file" ]; then
|
||||
printf '0 0\n' > "$state_file"
|
||||
fi
|
||||
|
||||
read_state() {
|
||||
read -r x y < "$state_file"
|
||||
}
|
||||
|
||||
write_state() {
|
||||
printf '%s %s\n' "$1" "$2" > "$state_file"
|
||||
}
|
||||
|
||||
command="${1:-}"
|
||||
case "$command" in
|
||||
getmouselocation)
|
||||
read_state
|
||||
printf 'X=%s\nY=%s\nSCREEN=0\nWINDOW=0\n' "$x" "$y"
|
||||
;;
|
||||
mousemove)
|
||||
shift
|
||||
x="${1:-0}"
|
||||
y="${2:-0}"
|
||||
shift 2 || true
|
||||
while [ "$#" -gt 0 ]; do
|
||||
token="$1"
|
||||
shift
|
||||
case "$token" in
|
||||
mousemove)
|
||||
x="${1:-0}"
|
||||
y="${2:-0}"
|
||||
shift 2 || true
|
||||
;;
|
||||
mousedown|mouseup)
|
||||
shift 1 || true
|
||||
;;
|
||||
click)
|
||||
if [ "${1:-}" = "--repeat" ]; then
|
||||
shift 2 || true
|
||||
fi
|
||||
shift 1 || true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
write_state "$x" "$y"
|
||||
;;
|
||||
type|key)
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
Loading…
Add table
Add a link
Reference in a new issue