Compare commits

..
4712 changed files with 798939 additions and 148228 deletions

No files matched your search

+34 -171
View File
@@ -15,26 +15,13 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: false
- name: Checkout submodules (retry)
run: |
set -euo pipefail
git submodule sync --recursive
for attempt in 1 2 3 4 5; do
if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then
exit 0
fi
echo "Submodule update failed (attempt $attempt/5). Retrying…"
sleep $((attempt * 10))
done
exit 1
submodules: recursive
- name: Setup Node.js
if: matrix.runtime == 'node'
uses: actions/setup-node@v4
with:
node-version: 24
node-version: 22
check-latest: true
- name: Setup Bun
@@ -49,7 +36,7 @@ jobs:
if: matrix.runtime == 'bun'
uses: actions/setup-node@v4
with:
node-version: 24
node-version: 22
check-latest: true
- name: Runtime versions
@@ -106,26 +93,12 @@ jobs:
run: bunx tsc -p tsconfig.json
macos-app:
if: github.event_name == 'pull_request'
runs-on: macos-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: false
- name: Checkout submodules (retry)
run: |
set -euo pipefail
git submodule sync --recursive
for attempt in 1 2 3 4 5; do
if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then
exit 0
fi
echo "Submodule update failed (attempt $attempt/5). Retrying…"
sleep $((attempt * 10))
done
exit 1
submodules: recursive
- name: Select Xcode 26.1
run: |
@@ -149,66 +122,10 @@ jobs:
run: swiftformat --lint apps/macos/Sources --config .swiftformat
- name: Swift build (release)
run: |
set -euo pipefail
for attempt in 1 2 3; do
if swift build --package-path apps/macos --configuration release; then
exit 0
fi
echo "swift build failed (attempt $attempt/3). Retrying…"
sleep $((attempt * 20))
done
exit 1
run: swift build --package-path apps/macos --configuration release
- name: Swift tests (coverage)
run: |
set -euo pipefail
for attempt in 1 2 3; do
if swift test --package-path apps/macos --parallel --enable-code-coverage --show-codecov-path; then
exit 0
fi
echo "swift test failed (attempt $attempt/3). Retrying…"
sleep $((attempt * 20))
done
exit 1
ios:
if: false # ignore iOS in CI for now
runs-on: macos-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: false
- name: Checkout submodules (retry)
run: |
set -euo pipefail
git submodule sync --recursive
for attempt in 1 2 3 4 5; do
if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then
exit 0
fi
echo "Submodule update failed (attempt $attempt/5). Retrying…"
sleep $((attempt * 10))
done
exit 1
- name: Select Xcode 26.1
run: |
sudo xcode-select -s /Applications/Xcode_26.1.app
xcodebuild -version
- name: Install XcodeGen
run: brew install xcodegen
- name: Install SwiftLint / SwiftFormat
run: brew install swiftlint swiftformat
- name: Show toolchain
run: |
sw_vers
xcodebuild -version
swift --version
run: swift test --package-path apps/macos --parallel --enable-code-coverage --show-codecov-path
- name: Generate iOS project
run: |
@@ -222,77 +139,36 @@ jobs:
DEST_ID="$(
python3 - <<'PY'
import json
import re
import subprocess
import sys
import uuid
def sh(args: list[str]) -> str:
return subprocess.check_output(args, text=True).strip()
# Prefer an already-created iPhone simulator if it exists.
devices = json.loads(sh(["xcrun", "simctl", "list", "devices", "-j"]))
candidates: list[tuple[str, str]] = []
for runtime, devs in (devices.get("devices") or {}).items():
for dev in devs or []:
if not dev.get("isAvailable"):
continue
name = str(dev.get("name") or "")
udid = str(dev.get("udid") or "")
if not udid or not name.startswith("iPhone"):
continue
candidates.append((name, udid))
candidates.sort(key=lambda it: (0 if "iPhone 16" in it[0] else 1, it[0]))
if candidates:
print(candidates[0][1])
sys.exit(0)
# Otherwise, create one from the newest available iOS runtime.
runtimes = json.loads(sh(["xcrun", "simctl", "list", "runtimes", "-j"])).get("runtimes") or []
ios = [rt for rt in runtimes if rt.get("platform") == "iOS" and rt.get("isAvailable")]
if not ios:
print("No available iOS runtimes found.", file=sys.stderr)
sys.exit(1)
def version_key(rt: dict) -> tuple[int, ...]:
parts: list[int] = []
for p in str(rt.get("version") or "0").split("."):
try:
parts.append(int(p))
except ValueError:
parts.append(0)
return tuple(parts)
ios.sort(key=version_key, reverse=True)
runtime = ios[0]
runtime_id = str(runtime.get("identifier") or "")
if not runtime_id:
print("Missing iOS runtime identifier.", file=sys.stderr)
sys.exit(1)
supported = runtime.get("supportedDeviceTypes") or []
iphones = [dt for dt in supported if dt.get("productFamily") == "iPhone"]
if not iphones:
print("No iPhone device types for iOS runtime.", file=sys.stderr)
sys.exit(1)
iphones.sort(
key=lambda dt: (
0 if "iPhone 16" in str(dt.get("name") or "") else 1,
str(dt.get("name") or ""),
)
data = json.loads(
subprocess.check_output(["xcrun", "simctl", "list", "devices", "available", "-j"], text=True)
)
device_type_id = str(iphones[0].get("identifier") or "")
if not device_type_id:
print("Missing iPhone device type identifier.", file=sys.stderr)
sys.exit(1)
runtimes = []
for runtime in data.get("devices", {}).keys():
m = re.search(r"\\.iOS-(\\d+)-(\\d+)$", runtime)
if m:
runtimes.append((int(m.group(1)), int(m.group(2)), runtime))
sim_name = f"CI iPhone {uuid.uuid4().hex[:8]}"
udid = sh(["xcrun", "simctl", "create", sim_name, device_type_id, runtime_id])
if not udid:
print("Failed to create iPhone simulator.", file=sys.stderr)
sys.exit(1)
print(udid)
runtimes.sort(reverse=True)
def pick_device(devices):
iphones = [d for d in devices if d.get("isAvailable") and d.get("name", "").startswith("iPhone")]
if not iphones:
return None
prefer = [d for d in iphones if "iPhone 16" in d.get("name", "")]
return (prefer[0] if prefer else iphones[0]).get("udid")
for _, __, runtime in runtimes:
udid = pick_device(data["devices"].get(runtime, []))
if udid:
print(udid)
sys.exit(0)
print("No available iPhone simulators found.", file=sys.stderr)
sys.exit(1)
PY
)"
echo "Using iOS Simulator id: $DEST_ID"
@@ -309,7 +185,7 @@ jobs:
RESULT_BUNDLE_PATH="$RUNNER_TEMP/Clawdis-iOS.xcresult"
xcrun xccov view --report --only-targets "$RESULT_BUNDLE_PATH"
- name: iOS coverage gate (43%)
- name: iOS coverage gate (50%)
run: |
set -euo pipefail
RESULT_BUNDLE_PATH="$RUNNER_TEMP/Clawdis-iOS.xcresult"
@@ -320,7 +196,7 @@ jobs:
import sys
target_name = "Clawdis.app"
minimum = 0.43
minimum = 0.50
report = json.loads(
subprocess.check_output(
@@ -350,20 +226,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: false
- name: Checkout submodules (retry)
run: |
set -euo pipefail
git submodule sync --recursive
for attempt in 1 2 3 4 5; do
if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then
exit 0
fi
echo "Submodule update failed (attempt $attempt/5). Retrying…"
sleep $((attempt * 10))
done
exit 1
submodules: recursive
- name: Setup Java
uses: actions/setup-java@v4
-12
View File
@@ -1,32 +1,20 @@
node_modules
.env
dist
*.bun-build
pnpm-lock.yaml
coverage
.pnpm-store
.worktrees/
.DS_Store
**/.DS_Store
# Bun build artifacts
*.bun-build
apps/macos/.build/
apps/shared/ClawdisKit/.build/
bin/
bin/clawdis-mac
bin/docs-list
apps/macos/.build-local/
apps/macos/.swiftpm/
apps/shared/ClawdisKit/.swiftpm/
Core/
apps/ios/*.xcodeproj/
apps/ios/*.xcworkspace/
apps/ios/.swiftpm/
vendor/
# Vendor build artifacts
vendor/a2ui/renderers/lit/dist/
# fastlane (iOS)
apps/ios/fastlane/README.md
-1
View File
@@ -1,4 +1,3 @@
[submodule "Peekaboo"]
path = Peekaboo
url = https://github.com/steipete/Peekaboo.git
branch = main
+5 -10
View File
@@ -1,3 +1,5 @@
READ ~/Projects/agent-scripts/AGENTS.MD BEFORE ANYTHING (skip if missing).
# Repository Guidelines
## Project Structure & Module Organization
@@ -16,7 +18,7 @@
- Language: TypeScript (ESM). Prefer strict typing; avoid `any`.
- Formatting/linting via Biome; run `pnpm lint` before commits.
- Keep files concise; extract helpers instead of “V2” copies. Use existing patterns for CLI options and dependency injection via `createDefaultDeps`.
- Aim to keep files under ~700 LOC; guideline only (not a hard guardrail). Split/refactor when it improves clarity or testability.
- Keep every file ≤ 500 LOC; refactor or split before exceeding and check frequently.
## Testing Guidelines
- Framework: Vitest with V8 coverage thresholds (70% lines/branches/functions/statements).
@@ -39,15 +41,8 @@
- Gateway currently runs only as the menubar app (launchctl shows `application.com.steipete.clawdis.debug.*`), there is no separate LaunchAgent/helper label installed. Restart via the Clawdis Mac app or `scripts/restart-mac.sh`; to verify/kill use `launchctl print gui/$UID | grep clawdis` rather than expecting `com.steipete.clawdis`. **When debugging on macOS, start/stop the gateway via the app, not ad-hoc tmux sessions; kill any temporary tunnels before handoff.**
- macOS logs: use `./scripts/clawlog.sh` (aka `vtlog`) to query unified logs for subsystem `com.steipete.clawdis`; it supports follow/tail/category filters and expects passwordless sudo for `/usr/bin/log`.
- Also read the shared guardrails at `~/Projects/oracle/AGENTS.md` and `~/Projects/agent-scripts/AGENTS.MD` before making changes; align with any cross-repo rules noted there.
- SwiftUI state management (iOS/macOS): prefer the `Observation` framework (`@Observable`, `@Bindable`) over `ObservableObject`/`@StateObject`; don’t introduce new `ObservableObject` unless required for compatibility, and migrate existing usages when touching related code.
- **Restart apps:** “restart iOS/Android apps” means rebuild (recompile/install) and relaunch, not just kill/launch.
- Notary key file lives at `~/Library/CloudStorage/Dropbox/Backup/AppStore/AuthKey_NJF3NFGTS3.p8` (Sparkle keys live under `~/Library/CloudStorage/Dropbox/Backup/Sparkle`).
- **Multi-agent safety:** do **not** create/apply/drop `git stash` entries unless Peter explicitly asks (this includes `git pull --rebase --autostash`). Assume other agents may be working; keep unrelated WIP untouched and avoid cross-cutting state changes.
- **Multi-agent safety:** when Peter says "push", you may `git pull --rebase` to integrate latest changes (never discard other agents' work). When Peter says "commit", scope to your changes only. When Peter says "commit all", commit everything in grouped chunks.
- **Multi-agent safety:** do **not** create/remove/modify `git worktree` checkouts (or edit `.worktrees/*`) unless Peter explicitly asks.
- **Multi-agent safety:** do **not** switch branches / check out a different branch unless Peter explicitly asks.
- **Multi-agent safety:** running multiple agents is OK as long as each agent has its own session.
- When asked to open a “session” file, open the Pi session logs under `~/.clawdis/sessions/*.jsonl` (newest unless a specific ID is given), not the default `sessions.json`. If logs are needed from Mac Studio, SSH via Tailscale and read the same path there.
- **Multi-agent safety:** do **not** create/apply/drop `git stash` entries unless Peter explicitly asks. Assume other agents may be working; keep unrelated WIP untouched and avoid cross-cutting state changes.
- When asked to open a “session” file, open the Pi session logs under `~/.clawdis/sessions/*.jsonl` (newest unless a specific ID is given), not the default `sessions.json`.
- Menubar dimming + restart flow mirrors Trimmy: use `scripts/restart-mac.sh` (kills all Clawdis variants, runs `swift build`, packages, relaunches). Icon dimming depends on MenuBarExtraAccess wiring in AppMain; keep `appearsDisabled` updates intact when touching the status item.
- Never send streaming/partial replies to external messaging surfaces (WhatsApp, Telegram); only final replies should be delivered there. Streaming/tool events may still go to internal UIs/control channel.
- Voice wake forwarding tips:
+15 -156
View File
@@ -1,157 +1,16 @@
# Changelog
## 2.0.0-beta4 — 2025-12-27
## 2.0.0 — Unreleased
### Fixes
- Package contents: include Discord/hooks build outputs in the npm tarball to avoid missing module errors.
_No changes since 2.0.0-beta1._
## 2.0.0-beta3 — 2025-12-27
## 2.0.0-beta1 — 2025-12-14
### Highlights
- First-class Clawdis tools (browser, canvas, nodes, cron) replace the old `clawdis-*` skills; tool schemas are now injected directly into the agent runtime.
- Per-session model selection + custom model providers: `models.providers` merges into `~/.clawdis/agent/models.json` (merge/replace modes) for LiteLLM, local OpenAI-compatible servers, Anthropic proxies, etc.
- Group chat activation modes: per-group `/activation mention|always` command with status visibility.
- Discord bot transport for DMs and guild text channels, with allowlists + mention gating.
- Gateway webhooks: external `wake` and isolated `agent` hooks with dedicated token auth.
- Hook mappings + Gmail Pub/Sub helper (`clawdis hooks gmail setup/run`) with auto-renew + Tailscale Funnel support.
- Command queue modes + per-session overrides (`/queue ...`) and new `agent.maxConcurrent` cap for safe parallelism across sessions.
- Background bash tasks: `bash` auto-yields after 20s (or on demand) with a `process` tool to list/poll/log/write/kill sessions.
- Gateway in-process restart: `clawdis_gateway` tool action triggers a SIGUSR1 restart without needing a supervisor.
### Breaking
- Config refactor: `inbound.*` removed; use top-level `routing` (allowlists + group rules + transcription), `messages` (prefixes/timestamps), and `session` (scoping/store/mainKey). No legacy keys read.
- Heartbeat config moved to `agent.heartbeat`: set `every: "30m"` (duration string) and optional `model`. `agent.heartbeatMinutes` is removed, and heartbeats are disabled unless `agent.heartbeat.every` is set.
- Heartbeats now run via the gateway runner (main session) and deliver to the last used channel by default. WhatsApp reply-heartbeat behavior is removed; use `agent.heartbeat.target`/`to` (or `target: "none"`) to control delivery.
- Browser `act` no longer accepts CSS `selector`; use `snapshot` refs (default `ai`) or `evaluate` as an escape hatch.
### Fixes
- Heartbeat replies now strip repeated `HEARTBEAT_OK` tails to avoid accidental “OK OK” spam.
- Heartbeat delivery now uses the last non-empty payload, preventing tool preambles from swallowing the final reply.
- Heartbeat failure logs now include the error reason instead of `[object Object]`.
- Duration strings now accept `h` (hours) where durations are parsed (e.g., heartbeat intervals).
- WhatsApp inbound now normalizes more wrapper types so quoted reply bodies are extracted reliably.
- WhatsApp send now preserves existing JIDs (including group `@g.us`) instead of coercing to `@s.whatsapp.net`. (Thanks @arun-8687.)
- Telegram/WhatsApp: reply context stays in `Body`/`ReplyTo*`, but outbound replies no longer thread to the original message. (Thanks @joshp123 for the PR and follow-up question.)
- Suppressed libsignal session cleanup spam from console logs unless verbose mode is enabled.
- WhatsApp web creds persistence hardened; credentials are restored before auth checks and QR login auto-restarts if it stalls.
- Group chats now honor `routing.groupChat.requireMention=false` as the default activation when no per-group override exists.
- Gateway auth no longer supports PAM/system mode; use token or shared password.
- Tailscale Funnel now requires password auth (no token-only public exposure).
- Group `/new` resets now work with @mentions so activation guidance appears on fresh sessions.
- Group chat activation context is now injected into the system prompt at session start (and after activation changes), including /new greetings.
- Typing indicators now start only once a reply payload is produced (no "thinking" typing for silent runs).
- WhatsApp group typing now starts immediately only when the bot is mentioned; otherwise it waits until real output exists.
- Streamed `<think>` segments are stripped before partial replies are emitted.
- System prompt now tags allowlisted owner numbers as the user identity to avoid mistaken “friend” assumptions.
- LM Studio/Ollama replies now require <final> tags; streaming ignores content until <final> begins.
- LM Studio responses API: tools payloads no longer include `strict: null`, and LM Studio no longer gets forced `<think>/<final>` tags.
- Identity emoji no longer auto-prefixes replies (set `messages.responsePrefix` explicitly if desired).
- Model switches now enqueue a system event so the next run knows the active model.
- `/model status` now lists available models (same as `/model`).
- `process log` pagination is now line-based (omit `offset` to grab the last N lines).
- macOS WebChat: assistant bubbles now update correctly when toggling light/dark mode.
- macOS: avoid spawning a duplicate gateway process when an external listener already exists.
- Node bridge: when binding to a non-loopback host (e.g. Tailnet IP), also listens on `127.0.0.1` for local connections (without creating duplicate loopback listeners for `0.0.0.0`/`127.0.0.1` binds).
- UI perf: pause repeat animations when scenes are inactive (typing dots, onboarding glow, iOS status pulse), throttle voice overlay level updates, and reduce overlay focus churn.
- Canvas defaults/A2UI auto-nav aligned; debug status overlay centered; redundant await removed in `CanvasManager`.
- Gateway launchd loop fixed by removing redundant `kickstart -k`.
- CLI now hints when Peekaboo is unauthorized.
- WhatsApp web inbox listeners now clean up on close to avoid duplicate handlers.
- Gateway startup now brings up browser control before external providers; WhatsApp/Telegram/Discord auto-start can be disabled with `web.enabled`, `telegram.enabled`, or `discord.enabled`.
### Providers & Routing
- New Discord provider for DMs + guild text channels with allowlists and mention-gated replies by default.
- `routing.queue` now controls queue vs interrupt behavior globally + per surface (defaults: WhatsApp/Telegram interrupt, Discord/WebChat queue).
- `/queue <mode>` supports one-shot or per-session overrides; `/queue reset|default` clears overrides.
- `agent.maxConcurrent` caps global parallel runs while keeping per-session serialization.
### macOS app
- Update-ready state surfaced in the menu; menu sections regrouped with session submenus.
- Menu bar now shows a dedicated Nodes section under Context with inline rows, overflow submenu, and iconized actions.
- Nodes now expose consistent inline details with per-node submenus for quick copy of key fields.
- Node rows now show compact app versions (build numbers moved to submenus) and offer SSH launch from Bonjour when available.
- Menu actions are grouped below toggles; Open Canvas hides when disabled and Voice Wake now anchors the mic picker.
- Connections now include Discord provider status + configuration UI.
- Menu bar gains an Allow Camera toggle alongside Canvas.
- Session list polish: sleeping/disconnected/error states, usage bar restored, padding + bar sizing tuned, syncing menu removed, header hidden when disconnected.
- Chat UI polish: tool call cards + merged tool results, glass background, tighter composer spacing, visual effect host tweaks.
- OAuth storage moved; legacy session syncing metadata removed.
- Remote SSH tunnels now get health checks; Debug → Ports highlights unhealthy tunnels and offers Reset SSH tunnel.
- Menu bar session/node sections no longer reflow while open, keeping hover highlights aligned.
- Menu hover highlights now span the full width (including submenu arrows).
- Menu session rows now refresh while open without width changes (no more stuck “Loading sessions…”).
- Menu width no longer grows on hover when moving the mouse across rows.
- Context usage bars now have higher contrast in light mode.
- macOS node timeouts now share a single async timeout helper for consistent behavior.
- WebChat window defaults tightened (narrower width, edge-to-edge layout) and the SwiftUI tag removed from the title.
### Nodes & Canvas
- Debug status overlay gated and toggleable on macOS/iOS/Android nodes.
- Gateway now derives the canvas host URL via a shared helper for bridge + WS handshakes (avoids loopback pitfalls).
- `canvas a2ui push` validates JSONL with line errors, rejects v0.9 payloads, and supports `--text` quick renders.
- `nodes rename` lets you override paired node display names without editing JSON.
- Android scaffold asset cleanup; iOS canvas/voice wake adjustments.
### Logging & Observability
- New subsystem console formatter with color modes, shortened prefixes, and TTY detection; browser/gateway logs route through the subsystem logger.
- WhatsApp console output streamlined; chalk/tslog typing fixes.
### Web UI
- Chat is now the dashboard landing view; health status simplified; initial scroll animation removed.
### Build, Dev, Docs
- Notarization flow added for macOS release artifacts; packaging scripts updated.
- macOS signing auto-selects Developer ID → Apple Distribution → Apple Development; no ad-hoc fallback.
- Added type-aware oxlint; docs list resolves from cwd; formatting/lint cleanup and dependency bumps (Peekaboo).
- Docs refreshed for tools, custom model providers, Discord, queue/routing, group activation commands, logging, restart semantics, release notes, GitHub pages CTAs, and npm pitfalls.
- `pnpm build` now skips A2UI bundling for faster builds (run `pnpm canvas:a2ui:bundle` when needed).
### Tests
- Coverage added for models config merging, WhatsApp reply context, QR login flows, auto-reply behavior, and gateway SIGTERM timeouts.
- Added gateway webhook coverage (auth, validation, and summary posting).
- Vitest now isolates HOME/XDG config roots so tests never touch a real `~/.clawdis` install.
## 2.0.0-beta2 — 2025-12-21
Second beta focused on bundled gateway packaging, skills management, onboarding polish, and provider reliability.
### Highlights
- Bundled gateway packaging: bun-compiled embedded gateway, new `gateway-daemon` command, launchd support, DMG packaging (zip+DMG).
- Skills platform: managed/bundled skills, install metadata + installers (uv), skill search + website, media/transcription helpers.
- macOS app: new Connections settings w/ provider status + QR login, skills settings redesign w/ install targets, models list loaded from the Gateway, clearer local/remote gateway choices.
- Web/agent UX: tool summary streaming + runtime toggle, WhatsApp QR login tool, agent steering queue, voice wake routes to main session, workspace bootstrap ritual.
### Gateway & providers
- Gateway: `models.list`, provider status events + RPC coverage, tailscale auth + PAM, bind-mode config, enriched agent WS logs, safer upgrade socket handling, fixed handshake auth crash.
- WhatsApp Web: QR login flow improvements (logged-out clearing, wait flow), self-chat mode handling, removed batching delay, web inbox made non-blocking.
- Telegram: normalized chat IDs with clearer error reporting.
### Canvas & browser control
- Canvas host served on Gateway port; removed standalone canvasHost port config; restored action bridge; refreshed A2UI bundle + message context; bridge canvas host for nodes.
- A2UI full-screen gutters + status clearance after successful load to avoid overlay collisions.
- Browser control API simplified; added MCP tool dispatch + native actions; control server can start without Playwright; hook timeouts extended.
### macOS UI polish
- Onboarding chat UI: kickoff flow, bubble tails, spacing + bottom bar refinements, window sizing tweaks, show Dock icon during onboarding.
- Skills UI: stabilized action column, fixed install target access, refined list layout and sizing, always show CLI installer.
- Remote/local gateway: auto-enable local gateway, clearer labels, re-ensure remote tunnel, hide local bridge discovery in remote mode.
### Build, CI, deps
- Bundled playwright-core + chromium-bidi/long; bun gateway bytecode builds; swiftformat/biome CI fixes; iOS lint script updates; Android icon/compiler updates; ignored new ClawdisKit `.swiftpm` path.
### Docs
- README architecture refresh + npm header image fix; onboarding/bootstrap steps; skills install guidance + new skills; browser/canvas control docs; bundled gateway + DMG packaging notes.
## 2.0.0-beta1 — 2025-12-19
First Clawdis release post rebrand. This is a semver-major because we dropped legacy providers/agents and moved defaults to new paths while adding a full macOS companion app, a WebSocket Gateway, and an iOS node.
### Bug Fixes
- macOS: Voice Wake / push-to-talk no longer initialize `AVAudioEngine` at app launch, preventing Bluetooth headphones from switching into headset profile when voice features are unused. (Thanks @Nachx639)
First Clawdis release post rebrand. This is a semver-major because we dropped legacy providers/agents and moved defaults to new paths while adding a full macOS companion app, a WebSocket Gateway, and an iOS node (Iris).
### Breaking
- Renamed to **Clawdis**: defaults now live under `~/.clawdis` (sessions in `~/.clawdis/sessions/`, IPC at `~/.clawdis/clawdis.sock`, logs in `/tmp/clawdis`). Launchd labels and config filenames follow the new name; legacy stores are copied forward on first run.
- Pi only: only the embedded Pi runtime remains, and the agent CLI/CLI flags for Claude/Codex/Gemini were removed. The Pi CLI runs in RPC mode with a persistent worker.
- Pi only: `inbound.reply.agent.kind` accepts only `"pi"`, and the agent CLI/CLI flags for Claude/Codex/Gemini were removed. The Pi CLI runs in RPC mode with a persistent worker.
- WhatsApp Web is the only transport; Twilio support and related CLI flags/tests were removed.
- Direct chats now collapse into a single `main` session by default (no config needed); groups stay isolated as `group:<jid>`.
- Gateway is now a loopback-only WebSocket daemon (`ws://127.0.0.1:18789`) that owns all providers/state; clients (CLI, WebChat, macOS app, nodes) connect to it. Start it explicitly (`clawdis gateway …`) or via Clawdis.app; helper subcommands no longer auto-spawn a gateway.
@@ -159,7 +18,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
### Gateway, nodes, and automation
- New typed Gateway WS protocol (JSON schema validated) with `clawdis gateway {health,status,send,agent,call}` helpers and structured presence/instance updates for all clients.
- Optional LAN-facing bridge (`tcp://0.0.0.0:18790`) keeps the Gateway loopback-only while enabling direct Bonjour-discovered connections for paired nodes.
- Node pairing + management via `clawdis nodes {pending,approve,reject,invoke}` (used by the iOS node and future remote nodes).
- Node pairing + management via `clawdis nodes {pending,approve,reject,invoke}` (used by the iOS node “Iris” and future remote nodes).
- Cron jobs are Gateway-owned (`clawdis cron …`) with run history stored as JSONL and support for “isolated summary” posting into the main session.
### macOS companion app
@@ -169,10 +28,10 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
- **Browser control**: manage clawd’s dedicated Chrome/Chromium with tab listing/open/focus/close, screenshots, DOM query/dump, and “AI snapshots” (aria/domSnapshot/ai) via `clawdis browser …` and UI controls.
- **Remote gateway control**: Bonjour discovery for local masters plus SSH-tunnel fallback for remote control when multicast is unavailable.
### iOS node
### iOS node (Iris)
- New iOS companion app that pairs to the Gateway bridge, reports presence as a node, and exposes a WKWebView “Canvas” for agent-driven UI.
- `clawdis nodes invoke` supports `canvas.eval` and `canvas.snapshot` to drive and verify the iOS Canvas (fails fast when the iOS node is backgrounded).
- Voice wake words are configurable in-app; the iOS node reconnects to the last bridge when credentials are still present in Keychain.
- `clawdis nodes invoke` supports `screen.eval` and `screen.snapshot` to drive and verify the iOS Canvas (fails fast when Iris is backgrounded).
- Voice wake words are configurable in-app; Iris reconnects to the last bridge when credentials are still present in Keychain.
### WhatsApp & agent experience
- Group chats fully supported: mention-gated triggers (including media-only captions), sender attribution, session primer with subject/member roster, allowlist bypass when you’re @‑mentioned, and safer handling of view-once/ephemeral media.
@@ -199,7 +58,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.5.0 — 2025-12-05
### Breaking
- Dropped all non-Pi agents (Claude, Codex, Gemini, Opencode); only the embedded Pi runtime remains and related CLI helpers have been removed.
- Dropped all non-Pi agents (Claude, Codex, Gemini, Opencode); `inbound.reply.agent.kind` now only accepts `"pi"` and related CLI helpers have been removed.
- Removed Twilio support and all related commands/options (webhook/up/provider flags/wait-poll); CLAWDIS is Baileys Web-only.
### Changes
@@ -226,7 +85,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.4.0 — 2025-12-03
### Highlights
- **Thinking directives & state:** `/t|/think|/thinking <level>` (aliases off|minimal|low|medium|high|max/highest). Inline applies to that message; directive-only message pins the level for the session; `/think:off` clears. Resolution: inline > session override > `agent.thinkingDefault` > off. Pi gets `--thinking <level>` (except off); other agents append cue words (`think` → `think hard` → `think harder` → `ultrathink`). Heartbeat probe uses `HEARTBEAT /think:high`.
- **Thinking directives & state:** `/t|/think|/thinking <level>` (aliases off|minimal|low|medium|high|max/highest). Inline applies to that message; directive-only message pins the level for the session; `/think:off` clears. Resolution: inline > session override > `inbound.reply.thinkingDefault` > off. Pi gets `--thinking <level>` (except off); other agents append cue words (`think` → `think hard` → `think harder` → `ultrathink`). Heartbeat probe uses `HEARTBEAT /think:high`.
- **Group chats (web provider):** Clawdis now fully supports WhatsApp groups: mention-gated triggers (including image-only @ mentions), recent group history injection, per-group sessions, sender attribution, and a first-turn primer with group subject/member roster; heartbeats are skipped for groups.
- **Group session primer:** The first turn of a group session now tells the agent it is in a WhatsApp group and lists known members/subject so it can address the right speaker.
- **Media failures are surfaced:** When a web auto-reply media fetch/send fails (e.g., HTTP 404), we now append a warning to the fallback text so you know the attachment was skipped.
@@ -268,7 +127,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.3.0 — 2025-12-02
### Highlights
- **Pluggable agents (Claude, Pi, Codex, Opencode):** agent selection via config/CLI plus per-agent argv builders and NDJSON parsers enable swapping without template changes.
- **Pluggable agents (Claude, Pi, Codex, Opencode):** `inbound.reply.agent` selects CLI/parser; per-agent argv builders and NDJSON parsers enable swapping without template changes.
- **Safety stop words:** `stop|esc|abort|wait|exit` immediately reply “Agent was aborted.” and mark the session so the next prompt is prefixed with an abort reminder.
- **Agent session reliability:** Only Claude returns a stable `session_id`; others may reset between runs.
@@ -285,7 +144,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
- Batched inbound messages with timestamps; typing indicator after sends.
- Watchdog restarts WhatsApp after long inactivity; heartbeat logging includes minutes since last message.
- Early `allowFrom` filtering before decryption.
- Same-phone mode with echo detection and optional message prefix marker.
- Same-phone mode with echo detection and optional `inbound.samePhoneMarker`.
## 1.2.2 — 2025-11-28
@@ -315,10 +174,10 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.1.0 — 2025-11-26
### Changes
- Web auto-replies resize/recompress media and honor `agent.mediaMaxMb`.
- Web auto-replies resize/recompress media and honor `inbound.reply.mediaMaxMb`.
- Detect media kind, enforce provider caps (images ≤6MB, audio/video ≤16MB, docs ≤100MB).
- `session.sendSystemOnce` and optional `sessionIntro`.
- Typing indicator refresh during commands; configurable via `agent.typingIntervalSeconds`.
- Typing indicator refresh during commands; configurable via `inbound.reply.typingIntervalSeconds`.
- Optional audio transcription via external CLI.
- Command replies return structured payload/meta; respect `mediaMaxMb`; log Claude metadata; include `cwd` in timeout messages.
- Web provider refactor; logout command; web-only gateway start helper.
Submodule Peekaboo updated: 9db365b73c...9bff10d088.
+173 -154
View File
@@ -1,7 +1,7 @@
# 🦞 CLAWDIS — Personal AI Assistant
# 🦞 CLAWDIS — WhatsApp & Telegram Gateway for AI Agents
<p align="center">
<img src="https://raw.githubusercontent.com/steipete/clawdis/main/docs/whatsapp-clawd.jpg" alt="CLAWDIS" width="400">
<img src="docs/whatsapp-clawd.jpg" alt="CLAWDIS" width="400">
</p>
<p align="center">
@@ -14,176 +14,131 @@
<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg?style=for-the-badge" alt="MIT License"></a>
</p>
**Clawdis** is a *personal AI assistant* you run on your own devices.
It answers you on the surfaces you already use (WhatsApp, Telegram, Discord, WebChat), can speak and listen on macOS/iOS, and can render a live Canvas you control. The Gateway is just the control plane — the product is the assistant.
If you want a private, single-user assistant that feels local, fast, and always-on, this is it.
**CLAWDIS** is a TypeScript/Node gateway that bridges WhatsApp (Web/Baileys) and Telegram (Bot API/grammY) to a local coding agent (**Pi**).
It’s like having a genius lobster in your pocket 24/7 — but with a real control plane, companion apps, and a network model that won’t corrupt sessions.
```
Your surfaces
│
▼
┌───────────────────────────────┐
│ Gateway │ ws://127.0.0.1:18789
│ (control plane) │ tcp://0.0.0.0:18790 (optional Bridge)
└──────────────┬────────────────┘
│
├─ Pi agent (RPC)
├─ CLI (clawdis …)
├─ WebChat (browser)
├─ macOS app (Clawdis.app)
└─ iOS node (Canvas + voice)
WhatsApp / Telegram
│
▼
┌──────────────────────────┐
│ Gateway │ ws://127.0.0.1:18789 (loopback-only)
│ (single source) │ tcp://0.0.0.0:18790 (optional Bridge)
└───────────┬───────────────┘
│
├─ Pi agent (RPC)
├─ CLI (clawdis …)
├─ WebChat (loopback UI)
├─ macOS app (Clawdis.app)
└─ iOS node (Iris) via Bridge + pairing
```
## What Clawdis does
## Why "CLAWDIS"?
- **Personal assistant** — one user, one identity, one memory surface.
- **Multi-surface inbox** — WhatsApp, Telegram, Discord, WebChat, macOS, iOS.
- **Voice wake + push-to-talk** — local speech recognition on macOS/iOS.
- **Canvas** — a live visual workspace you can drive from the agent.
- **Automation-ready** — browser control, media handling, and tool streaming.
- **Local-first control plane** — the Gateway owns state, everything else connects.
- **Group chats** — mention-based by default, `/activation always|mention` per group (owner-only).
**CLAWDIS** = CLAW + TARDIS
## How it works (short)
Because every space lobster needs a time-and-space machine. The Doctor has a TARDIS. [Clawd](https://clawd.me) has a CLAWDIS. Both are blue. Both are chaotic. Both are loved.
- **Gateway** is the single source of truth for sessions/providers.
- **Loopback-first**: `ws://127.0.0.1:18789` by default.
- **Bridge** (optional) exposes a paired-node port for iOS/Android.
- **Agent runtime** is **Pi** in RPC mode.
## Features
## Quick start (from source)
- 📱 **WhatsApp Integration** — Personal WhatsApp Web (Baileys)
- ✈️ **Telegram (Bot API)** — DMs and groups via grammY
- 🛰️ **Gateway control plane** — One long-lived gateway owns provider state; clients connect over WebSocket
- 🤖 **Agent runtime** — Pi only (Pi CLI in RPC mode), with tool streaming
- 💬 **Sessions** — Direct chats collapse into `main` by default; groups are isolated
- 🔔 **Heartbeats** — Periodic check-ins for proactive AI
- 🧭 **Clawd Browser** — Dedicated Chrome/Chromium profile with tabs + screenshot control (no interference with your daily browser)
- 👥 **Group Chat Support** — Mention-based triggering
- 📎 **Media Support** — Images, audio, documents, voice notes
- 🎤 **Voice & transcription hooks** — Voice Wake (macOS/iOS) + optional transcription pipeline
- 🔧 **Tool Streaming** — Real-time display (💻📄✍️📝)
- 🖥️ **macOS Companion (Clawdis.app)** — Menu bar controls, Voice Wake, WebChat, onboarding, remote gateway control
- 📱 **iOS Node (Iris)** — Pairs as a node, exposes a Canvas surface, forwards voice wake transcripts
Runtime: **Node ≥22** + **pnpm**.
Only the Pi CLI is supported now; legacy Claude/Codex/Gemini paths have been removed.
## Network model (the “new reality”)
- **One Gateway per host**. The Gateway is the only process allowed to own the WhatsApp Web session.
- **Loopback-first**: the Gateway WebSocket listens on `ws://127.0.0.1:18789` and is not exposed on the LAN.
- **Bridge for nodes**: when enabled, the Gateway also exposes a LAN/tailnet-facing bridge on `tcp://0.0.0.0:18790` for paired nodes (Bonjour-discoverable).
- **Remote control**: use a VPN/tailnet or an SSH tunnel (`ssh -N -L 18789:127.0.0.1:18789 user@host`). The macOS app can drive this flow.
## Codebase
- **TypeScript (ESM)**: CLI + Gateway live in `src/` and run on Node ≥ 22.
- **macOS app (Swift)**: menu bar companion lives in `apps/macos/`.
- **iOS app (Swift)**: Iris node prototype lives in `apps/ios/`.
## Quick Start
Runtime requirement: **Node ≥22.0.0** (not bundled). The macOS app and CLI both use the host runtime; install via Homebrew or official installers before running `clawdis`.
```bash
# From source (recommended while the npm package is still settling)
pnpm install
pnpm build
pnpm ui:build
# Link WhatsApp (stores creds in ~/.clawdis/credentials)
# Link your WhatsApp (stores creds under ~/.clawdis/credentials)
pnpm clawdis login
# Start the gateway
# Start the gateway (WebSocket control plane)
pnpm clawdis gateway --port 18789 --verbose
# Dev loop (auto-reload on TS changes)
pnpm gateway:watch
# Send a WhatsApp message (WhatsApp sends go through the Gateway)
pnpm clawdis send --to +1234567890 --message "Hello from the CLAWDIS!"
# Send a message
pnpm clawdis send --to +1234567890 --message "Hello from Clawdis"
# Talk to the assistant (optionally deliver back to WhatsApp/Telegram/Discord)
# Talk to the agent (optionally deliver back to WhatsApp/Telegram)
pnpm clawdis agent --message "Ship checklist" --thinking high
# If the port is busy, force-kill listeners then start
pnpm clawdis gateway --force
```
If you run from source, prefer `pnpm clawdis …` (not global `clawdis`).
## Companion Apps
## Chat commands
### macOS Companion (Clawdis.app)
Send these in WhatsApp/Telegram/WebChat (group commands are owner-only):
- A menu bar app that can start/stop the Gateway, show health/presence, and provide a local ops UI.
- **Voice Wake** (on-device speech recognition) and Push-to-talk overlay.
- **WebChat** embed + debug tooling (logs, status, heartbeats, sessions).
- Hosts **PeekabooBridge** for UI automation brokering (for clawd workflows).
- `/status` — health + session info (group shows activation mode)
- `/new` or `/reset` — reset the session
- `/think <level>` — off|minimal|low|medium|high
- `/verbose on|off`
- `/restart` — restart the gateway (owner-only in groups)
- `/activation mention|always` — group activation toggle (groups only)
### Voice Wake reply routing
## Architecture
Voice Wake sends messages into the `main` session and replies on the **last used surface**:
### TypeScript Gateway (src/gateway/server.ts)
- **Single HTTP+WS server** on `ws://127.0.0.1:18789` (bind policy: loopback/lan/tailnet/auto). The first frame must be `connect`; AJV validates frames against TypeBox schemas (`src/gateway/protocol`).
- **Single source of truth** for sessions, providers, cron, voice wake, and presence. Methods cover `send`, `agent`, `chat.*`, `sessions.*`, `config.*`, `cron.*`, `voicewake.*`, `node.*`, `system-*`, `wake`.
- **Events + snapshot**: handshake returns a snapshot (presence/health) and declares event types; runtime events include `agent`, `chat`, `presence`, `tick`, `health`, `heartbeat`, `cron`, `node.pair.*`, `voicewake.changed`, `shutdown`.
- **Idempotency & safety**: `send`/`agent`/`chat.send` require idempotency keys with a TTL cache (5 min, cap 1000) to avoid double‑sends on reconnects; payload sizes are capped per connection.
- **Bridge for nodes**: optional TCP bridge (`src/infra/bridge/server.ts`) is newline‑delimited JSON frames (`hello`, pairing, RPC, `invoke`); node connect/disconnect is surfaced into presence.
- **Control UI + Canvas Host**: HTTP serves `/ui` assets (if built) and can host a live‑reload Canvas host for nodes (`src/canvas-host/server.ts`), injecting the A2UI postMessage bridge.
- WhatsApp: last direct message you sent/received.
- Telegram: last DM chat id (bot mode).
- WebChat: last WebChat thread you used.
### iOS app (apps/ios)
- **Discovery + pairing**: Bonjour discovery via `BridgeDiscoveryModel` (NWBrowser). `BridgeConnectionController` auto‑connects using Keychain token or allows manual host/port.
- **Node runtime**: `BridgeSession` (actor) maintains the `NWConnection`, hello handshake, ping/pong, RPC requests, and `invoke` callbacks.
- **Capabilities + commands**: advertises `canvas`, `screen`, `camera`, `voiceWake` (settings‑driven) and executes `canvas.*`, `canvas.a2ui.*`, `camera.*`, `screen.record` (`NodeAppModel.handleInvoke`).
- **Canvas**: `WKWebView` with bundled Canvas scaffold + A2UI, JS eval, snapshot capture, and `clawdis://` deep‑link interception (`ScreenController`).
- **Voice + deep links**: voice wake sends `voice.transcript` events; `clawdis://agent` links emit `agent.request`. Voice wake triggers sync via `voicewake.get` + `voicewake.changed`.
If delivery fails (e.g. WhatsApp disconnected / Telegram token missing), Clawdis logs the error and you can still inspect the run via WebChat/session logs.
## Companion apps
Build/run the mac app with `./scripts/restart-mac.sh` (packages, installs, and launches), or `swift build --package-path apps/macos && open dist/Clawdis.app`.
The **macOS app is critical**: it runs the menu‑bar control plane, owns local permissions (TCC), hosts Voice Wake, exposes WebChat/debug tools, and coordinates local/remote gateway mode. Most “assistant” UX lives here.
### iOS Node (Iris) (internal)
### macOS (Clawdis.app)
Iris is an internal/prototype iOS app that connects as a **remote node**:
- Menu bar control for the Gateway and health.
- Voice Wake + push-to-talk overlay.
- WebChat + debug tools.
- Remote gateway control over SSH.
- **Voice trigger:** forwards transcripts into the Gateway (agent runs + wakeups).
- **Canvas screen:** a WKWebView + `<canvas>` surface the agent can control (via `screen.eval` / `screen.snapshot` over `node.invoke`).
- **Discovery + pairing:** finds the bridge via Bonjour (`_clawdis-bridge._tcp`) and uses Gateway-owned pairing (`clawdis nodes pending|approve`).
Build/run: `./scripts/restart-mac.sh` (packages + launches).
### iOS node (internal)
- Pairs as a node via the Bridge.
- Voice trigger forwarding + Canvas surface.
- Controlled via `clawdis nodes …`.
Runbook: `docs/ios/connect.md`.
### Android node (internal)
- Pairs via the same Bridge + pairing flow as iOS.
- Exposes Canvas, Camera, and Screen capture commands.
- Runbook: `docs/android/connect.md`.
## Agent workspace + skills
- Workspace root: `~/clawd` (configurable via `agent.workspace`).
- Injected prompt files: `AGENTS.md`, `SOUL.md`, `TOOLS.md`.
- Skills: `~/clawd/skills/<skill>/SKILL.md`.
Runbook: `docs/ios/connect.md`
## Configuration
Minimal `~/.clawdis/clawdis.json`:
Create `~/.clawdis/clawdis.json`:
```json5
{
routing: {
inbound: {
allowFrom: ["+1234567890"]
}
}
```
### WhatsApp
- Link the device: `pnpm clawdis login` (stores creds in `~/.clawdis/credentials`).
- Allowlist who can talk to the assistant via `routing.allowFrom`.
### Telegram
- Set `TELEGRAM_BOT_TOKEN` or `telegram.botToken` (env wins).
- Optional: set `telegram.requireMention`, `telegram.allowFrom`, or `telegram.webhookUrl` as needed.
```json5
{
telegram: {
botToken: "123456:ABCDEF"
}
}
```
### Discord
- Set `DISCORD_BOT_TOKEN` or `discord.token` (env wins).
- Optional: set `discord.requireMention`, `discord.allowFrom`, or `discord.mediaMaxMb` as needed.
```json5
{
discord: {
token: "1234abcd"
}
}
```
Browser control (optional):
Optional: enable/configure clawd’s dedicated browser control (defaults are already on):
```json5
{
@@ -195,34 +150,98 @@ Browser control (optional):
}
```
## Docs
## Documentation
- [`docs/index.md`](docs/index.md) (overview)
- [`docs/configuration.md`](docs/configuration.md)
- [`docs/group-messages.md`](docs/group-messages.md)
- [`docs/gateway.md`](docs/gateway.md)
- [`docs/web.md`](docs/web.md)
- [`docs/discovery.md`](docs/discovery.md)
- [`docs/agent.md`](docs/agent.md)
- [`docs/discord.md`](docs/discord.md)
- Webhooks + external triggers: [`docs/webhook.md`](docs/webhook.md)
- Gmail hooks (email → wake): [`docs/gmail-pubsub.md`](docs/gmail-pubsub.md)
## Email hooks (Gmail)
```bash
clawdis hooks gmail setup --account you@gmail.com
clawdis hooks gmail run
```
- [`docs/security.md`](docs/security.md)
- [`docs/troubleshooting.md`](docs/troubleshooting.md)
- [`docs/ios/connect.md`](docs/ios/connect.md)
- [`docs/clawdis-mac.md`](docs/clawdis-mac.md)
- [Configuration Guide](./docs/configuration.md)
- [Gateway runbook](./docs/gateway.md)
- [Discovery + transports](./docs/discovery.md)
- [Agent Integration](./docs/agents.md)
- [Group Chats](./docs/group-messages.md)
- [Security](./docs/security.md)
- [Troubleshooting](./docs/troubleshooting.md)
- [The Lore](./docs/lore.md) 🦞
- [Telegram (Bot API)](./docs/telegram.md)
- [iOS node runbook (Iris)](./docs/ios/connect.md)
- [macOS app spec](./docs/clawdis-mac.md)
## Clawd
Clawdis was built for **Clawd**, a space lobster AI assistant.
CLAWDIS was built for **Clawd**, a space lobster AI assistant. See the full setup in [`docs/clawd.md`](./docs/clawd.md).
- https://clawd.me
- https://soul.md
- https://steipete.me
- 🦞 **Clawd's Home:** [clawd.me](https://clawd.me)
- 📜 **Clawd's Soul:** [soul.md](https://soul.md)
- 👨‍💻 **Peter's Blog:** [steipete.me](https://steipete.me)
- 🐦 **Twitter:** [@steipete](https://twitter.com/steipete)
## Provider
If you’re running from source, use `pnpm clawdis …` instead of `clawdis …`.
### WhatsApp Web
```bash
clawdis login # scan QR, store creds
clawdis gateway # run Gateway (WS on 127.0.0.1:18789)
```
### Telegram (Bot API)
Bot-mode support (grammY only) shares the same `main` session as WhatsApp/WebChat, with groups kept isolated. Text/media sends work via `clawdis send --provider telegram` (reads `TELEGRAM_BOT_TOKEN` or `telegram.botToken`). Webhook mode is supported; see `docs/telegram.md` for setup and limits.
## Commands
| Command | Description |
|---------|-------------|
| `clawdis login` | Link WhatsApp Web via QR |
| `clawdis send` | Send a message (WhatsApp default; `--provider telegram` for bot mode). WhatsApp sends go via the Gateway WS; Telegram sends are direct. |
| `clawdis agent` | Talk directly to the agent (no WhatsApp send) |
| `clawdis browser ...` | Manage clawd’s dedicated browser (status/tabs/open/screenshot). |
| `clawdis gateway` | Start the Gateway server (WS control plane). Params: `--port`, `--token`, `--force`, `--verbose`. |
| `clawdis gateway health|status|send|agent|call` | Gateway WS clients; assume a running gateway. |
| `clawdis wake` | Enqueue a system event and optionally trigger a heartbeat via the Gateway. |
| `clawdis cron ...` | Manage scheduled jobs (via Gateway). |
| `clawdis nodes ...` | Manage Gateway-owned node pairing. |
| `clawdis status` | Web session health + session store summary |
| `clawdis health` | Reports cached provider state from the running gateway. |
| `clawdis webchat` | Start the loopback-only WebChat HTTP server |
#### Gateway client params (WS only)
- `--url` (default `ws://127.0.0.1:18789`)
- `--token` (shared secret if set on the gateway)
- `--timeout <ms>` (WS call timeout)
#### Send
- `--provider whatsapp|telegram` (default whatsapp)
- `--media <path-or-url>`
- `--json` for machine-readable output
#### Health
- Reads gateway/provider state (no direct Baileys socket from the CLI).
In chat, send `/status` to see if the agent is reachable, how much context the session has used, and the current thinking/verbose toggles—no agent call required.
`/status` also shows whether your WhatsApp web session is linked and how long ago the creds were refreshed so you know when to re-scan the QR.
### Sessions, surfaces, and WebChat
- Direct chats now share a canonical session key `main` by default (configurable via `inbound.reply.session.mainKey`). Groups stay isolated as `group:<jid>`.
- WebChat attaches to `main` and hydrates history from `~/.clawdis/sessions/<SessionId>.jsonl`, so desktop view mirrors WhatsApp/Telegram turns.
- Inbound contexts carry a `Surface` hint (e.g., `whatsapp`, `webchat`, `telegram`) for logging; replies still go back to the originating surface deterministically.
- Every inbound message is wrapped for the agent as `[Surface FROM HOST/IP TIMESTAMP] body`:
- WhatsApp: `[WhatsApp +15551234567 2025-12-09 12:34] …`
- Telegram: `[Telegram Ada Lovelace (@ada_bot) id:123456789 2025-12-09 12:34] …`
- WebChat: `[WebChat my-mac.local 10.0.0.5 2025-12-09 12:34] …`
This keeps the model aware of the transport, sender, host, and time without relying on implicit context.
## Credits
- **Peter Steinberger** ([@steipete](https://twitter.com/steipete)) — Creator
- **Mario Zechner** ([@badlogicgames](https://twitter.com/badlogicgames)) — Pi, security testing
- **Clawd** 🦞 — The space lobster who demanded a better name
## License
MIT — Free as a lobster in the ocean.
---
*"We're all just playing with our own prompts."*
🦞💙
-11
View File
@@ -1,11 +0,0 @@
# Changelog
## 0.2.0 — 2025-12-23
### Highlights
- Added `SwabbleKit` (multi-platform wake-word gate utilities with segment-aware gap detection).
- Swabble package now supports iOS + macOS consumers; CLI remains macOS 26-only.
### Changes
- CLI wake-word matching/stripping routed through `SwabbleKit` helpers.
- Speech pipeline types now explicitly gated to macOS 26 / iOS 26 availability.
+10 -1
View File
@@ -1,6 +1,15 @@
{
"originHash" : "2012d083159d375d07febbc184c592c569d7ab48247045e35a762e3269d4cadc",
"originHash" : "3018b2c8c183d55b57ad0c4526b2380ac3b957d13a3a86e1b2845e81323c443a",
"pins" : [
{
"identity" : "commander",
"kind" : "remoteSourceControl",
"location" : "https://github.com/steipete/Commander.git",
"state" : {
"revision" : "8b8cb4f34315ce9e5307b3a2bcd77ff73f586a02",
"version" : "0.2.0"
}
},
{
"identity" : "swift-syntax",
"kind" : "remoteSourceControl",
+2 -21
View File
@@ -4,16 +4,14 @@ import PackageDescription
let package = Package(
name: "swabble",
platforms: [
.macOS(.v15),
.iOS(.v17),
.macOS(.v26),
],
products: [
.library(name: "Swabble", targets: ["Swabble"]),
.library(name: "SwabbleKit", targets: ["SwabbleKit"]),
.executable(name: "swabble", targets: ["SwabbleCLI"]),
],
dependencies: [
.package(path: "../Peekaboo/Commander"),
.package(url: "https://github.com/steipete/Commander.git", from: "0.2.0"),
.package(url: "https://github.com/apple/swift-testing", from: "0.99.0"),
],
targets: [
@@ -21,30 +19,13 @@ let package = Package(
name: "Swabble",
path: "Sources/SwabbleCore",
swiftSettings: []),
.target(
name: "SwabbleKit",
path: "Sources/SwabbleKit",
swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"),
]),
.executableTarget(
name: "SwabbleCLI",
dependencies: [
"Swabble",
"SwabbleKit",
.product(name: "Commander", package: "Commander"),
],
path: "Sources/swabble"),
.testTarget(
name: "SwabbleKitTests",
dependencies: [
"SwabbleKit",
.product(name: "Testing", package: "swift-testing"),
],
swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"),
.enableExperimentalFeature("SwiftTesting"),
]),
.testTarget(
name: "swabbleTests",
dependencies: [
+4 -8
View File
@@ -1,10 +1,9 @@
# 🎙️ swabble — Speech.framework wake-word hook daemon (macOS 26)
swabble is a Swift 6.2 wake-word hook daemon. The CLI targets macOS 26 (SpeechAnalyzer + SpeechTranscriber). The shared `SwabbleKit` target is multi-platform and exposes wake-word gating utilities for iOS/macOS apps.
swabble is a Swift 6.2, macOS 26-only rewrite of the brabble voice daemon. It listens on your mic, gates on a wake word, transcribes locally using Apple's new SpeechAnalyzer + SpeechTranscriber, then fires a shell hook with the transcript. No cloud calls, no Whisper binaries.
- **Local-only**: Speech.framework on-device models; zero network usage.
- **Wake word**: Default `clawd` (aliases `claude`), optional `--no-wake` bypass.
- **SwabbleKit**: Shared wake gate utilities (gap-based gating when you provide speech segments).
- **Hooks**: Run any command with prefix/env, cooldown, min_chars, timeout.
- **Services**: launchd helper stubs for start/stop/install.
- **File transcribe**: TXT or SRT with time ranges (using AttributedString splits).
@@ -31,7 +30,7 @@ swift run swabble transcribe /path/to/audio.m4a --format srt --output out.srt
```
## Use as a library
Add swabble as a SwiftPM dependency and import the `Swabble` or `SwabbleKit` product:
Add swabble as a SwiftPM dependency and import the `Swabble` product to reuse the Speech pipeline, config loader, hook executor, and transcript store in your own app:
```swift
// Package.swift
@@ -39,10 +38,7 @@ dependencies: [
.package(url: "https://github.com/steipete/swabble.git", branch: "main"),
],
targets: [
.target(name: "MyApp", dependencies: [
.product(name: "Swabble", package: "swabble"), // Speech pipeline (macOS 26+ / iOS 26+)
.product(name: "SwabbleKit", package: "swabble"), // Wake-word gate utilities (iOS 17+ / macOS 15+)
]),
.target(name: "MyApp", dependencies: [.product(name: "Swabble", package: "swabble")]),
]
```
@@ -97,7 +93,7 @@ Environment variables:
## Speech pipeline
- `AVAudioEngine` tap → `BufferConverter` → `AnalyzerInput` → `SpeechAnalyzer` with a `SpeechTranscriber` module.
- Requests volatile + final results; the CLI uses text-only wake gating today.
- Requests volatile + final results; wake gating is string match on partial/final.
- Authorization requested at first start; requires macOS 26 + new Speech.framework APIs.
## Development
@@ -2,13 +2,11 @@ import AVFoundation
import Foundation
import Speech
@available(macOS 26.0, iOS 26.0, *)
public struct SpeechSegment: Sendable {
public let text: String
public let isFinal: Bool
}
@available(macOS 26.0, iOS 26.0, *)
public enum SpeechPipelineError: Error {
case authorizationDenied
case analyzerFormatUnavailable
@@ -16,7 +14,6 @@ public enum SpeechPipelineError: Error {
}
/// Live microphone → SpeechAnalyzer → SpeechTranscriber pipeline.
@available(macOS 26.0, iOS 26.0, *)
public actor SpeechPipeline {
private struct UnsafeBuffer: @unchecked Sendable { let buffer: AVAudioPCMBuffer }
@@ -1,192 +0,0 @@
import Foundation
public struct WakeWordSegment: Sendable, Equatable {
public let text: String
public let start: TimeInterval
public let duration: TimeInterval
public let range: Range<String.Index>?
public init(text: String, start: TimeInterval, duration: TimeInterval, range: Range<String.Index>? = nil) {
self.text = text
self.start = start
self.duration = duration
self.range = range
}
public var end: TimeInterval { self.start + self.duration }
}
public struct WakeWordGateConfig: Sendable, Equatable {
public var triggers: [String]
public var minPostTriggerGap: TimeInterval
public var minCommandLength: Int
public init(
triggers: [String],
minPostTriggerGap: TimeInterval = 0.45,
minCommandLength: Int = 1)
{
self.triggers = triggers
self.minPostTriggerGap = minPostTriggerGap
self.minCommandLength = minCommandLength
}
}
public struct WakeWordGateMatch: Sendable, Equatable {
public let triggerEndTime: TimeInterval
public let postGap: TimeInterval
public let command: String
public init(triggerEndTime: TimeInterval, postGap: TimeInterval, command: String) {
self.triggerEndTime = triggerEndTime
self.postGap = postGap
self.command = command
}
}
public enum WakeWordGate {
private struct Token {
let normalized: String
let start: TimeInterval
let end: TimeInterval
let range: Range<String.Index>?
let text: String
}
private struct TriggerTokens {
let tokens: [String]
}
public static func match(
transcript: String,
segments: [WakeWordSegment],
config: WakeWordGateConfig)
-> WakeWordGateMatch? {
let triggerTokens = self.normalizeTriggers(config.triggers)
guard !triggerTokens.isEmpty else { return nil }
let tokens = self.normalizeSegments(segments)
guard !tokens.isEmpty else { return nil }
var best: (index: Int, triggerEnd: TimeInterval, gap: TimeInterval)?
for trigger in triggerTokens {
let count = trigger.tokens.count
guard count > 0, tokens.count > count else { continue }
for i in 0...(tokens.count - count - 1) {
let matched = (0..<count).allSatisfy { tokens[i + $0].normalized == trigger.tokens[$0] }
if !matched { continue }
let triggerEnd = tokens[i + count - 1].end
let nextToken = tokens[i + count]
let gap = nextToken.start - triggerEnd
if gap < config.minPostTriggerGap { continue }
if let best, i <= best.index { continue }
best = (i, triggerEnd, gap)
}
}
guard let best else { return nil }
let command = self.commandText(transcript: transcript, segments: segments, triggerEndTime: best.triggerEnd)
.trimmingCharacters(in: Self.whitespaceAndPunctuation)
guard command.count >= config.minCommandLength else { return nil }
return WakeWordGateMatch(triggerEndTime: best.triggerEnd, postGap: best.gap, command: command)
}
public static func commandText(
transcript: String,
segments: [WakeWordSegment],
triggerEndTime: TimeInterval)
-> String {
let threshold = triggerEndTime + 0.001
for segment in segments where segment.start >= threshold {
if normalizeToken(segment.text).isEmpty { continue }
if let range = segment.range {
let slice = transcript[range.lowerBound...]
return String(slice).trimmingCharacters(in: Self.whitespaceAndPunctuation)
}
break
}
let text = segments
.filter { $0.start >= threshold && !self.normalizeToken($0.text).isEmpty }
.map(\.text)
.joined(separator: " ")
return text.trimmingCharacters(in: Self.whitespaceAndPunctuation)
}
public static func matchesTextOnly(text: String, triggers: [String]) -> Bool {
guard !text.isEmpty else { return false }
let normalized = text.lowercased()
for trigger in triggers {
let token = trigger.trimmingCharacters(in: self.whitespaceAndPunctuation).lowercased()
if token.isEmpty { continue }
if normalized.contains(token) { return true }
}
return false
}
public static func stripWake(text: String, triggers: [String]) -> String {
var out = text
for trigger in triggers {
let token = trigger.trimmingCharacters(in: self.whitespaceAndPunctuation)
guard !token.isEmpty else { continue }
out = out.replacingOccurrences(of: token, with: "", options: [.caseInsensitive])
}
return out.trimmingCharacters(in: self.whitespaceAndPunctuation)
}
private static func normalizeTriggers(_ triggers: [String]) -> [TriggerTokens] {
var output: [TriggerTokens] = []
for trigger in triggers {
let tokens = trigger
.split(whereSeparator: { $0.isWhitespace })
.map { self.normalizeToken(String($0)) }
.filter { !$0.isEmpty }
if tokens.isEmpty { continue }
output.append(TriggerTokens(tokens: tokens))
}
return output
}
private static func normalizeSegments(_ segments: [WakeWordSegment]) -> [Token] {
segments.compactMap { segment in
let normalized = self.normalizeToken(segment.text)
guard !normalized.isEmpty else { return nil }
return Token(
normalized: normalized,
start: segment.start,
end: segment.end,
range: segment.range,
text: segment.text)
}
}
private static func normalizeToken(_ token: String) -> String {
token
.trimmingCharacters(in: self.whitespaceAndPunctuation)
.lowercased()
}
private static let whitespaceAndPunctuation = CharacterSet.whitespacesAndNewlines
.union(.punctuationCharacters)
}
#if canImport(Speech)
import Speech
public enum WakeWordSpeechSegments {
public static func from(transcription: SFTranscription, transcript: String) -> [WakeWordSegment] {
transcription.segments.map { segment in
let range = Range(segment.substringRange, in: transcript)
return WakeWordSegment(
text: segment.substring,
start: segment.timestamp,
duration: segment.duration,
range: range)
}
}
}
#endif
@@ -1,7 +1,6 @@
import Commander
import Foundation
@available(macOS 26.0, *)
@MainActor
enum CLIRegistry {
static var descriptors: [CommandDescriptor] {
@@ -1,9 +1,7 @@
import Commander
import Foundation
import Swabble
import SwabbleKit
@available(macOS 26.0, *)
@MainActor
struct ServeCommand: ParsableCommand {
@Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
@@ -70,12 +68,17 @@ struct ServeCommand: ParsableCommand {
}
private static func matchesWake(text: String, cfg: SwabbleConfig) -> Bool {
let triggers = [cfg.wake.word] + cfg.wake.aliases
return WakeWordGate.matchesTextOnly(text: text, triggers: triggers)
let lowered = text.lowercased()
if lowered.contains(cfg.wake.word.lowercased()) { return true }
return cfg.wake.aliases.contains(where: { lowered.contains($0.lowercased()) })
}
private static func stripWake(text: String, cfg: SwabbleConfig) -> String {
let triggers = [cfg.wake.word] + cfg.wake.aliases
return WakeWordGate.stripWake(text: text, triggers: triggers)
var out = text
out = out.replacingOccurrences(of: cfg.wake.word, with: "", options: [.caseInsensitive])
for alias in cfg.wake.aliases {
out = out.replacingOccurrences(of: alias, with: "", options: [.caseInsensitive])
}
return out.trimmingCharacters(in: .whitespacesAndNewlines)
}
}
+2 -9
View File
@@ -1,7 +1,6 @@
import Commander
import Foundation
@available(macOS 26.0, *)
@MainActor
private func runCLI() async -> Int32 {
do {
@@ -16,7 +15,6 @@ private func runCLI() async -> Int32 {
}
}
@available(macOS 26.0, *)
@MainActor
private func dispatch(invocation: CommandInvocation) async throws {
let parsed = invocation.parsedValues
@@ -97,10 +95,5 @@ private func dispatch(invocation: CommandInvocation) async throws {
}
}
if #available(macOS 26.0, *) {
let exitCode = await runCLI()
exit(exitCode)
} else {
fputs("error: swabble requires macOS 26 or newer\n", stderr)
exit(1)
}
let exitCode = await runCLI()
exit(exitCode)
@@ -1,63 +0,0 @@
import Foundation
import Testing
import SwabbleKit
@Suite struct WakeWordGateTests {
@Test func matchRequiresGapAfterTrigger() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.35, 0.1),
("thing", 0.5, 0.1),
])
let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
#expect(WakeWordGate.match(transcript: transcript, segments: segments, config: config) == nil)
}
@Test func matchAllowsGapAndExtractsCommand() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.9, 0.1),
("thing", 1.1, 0.1),
])
let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
#expect(match?.command == "do thing")
}
@Test func matchHandlesMultiWordTriggers() {
let transcript = "hey clawd do it"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.8, 0.1),
("it", 1.0, 0.1),
])
let config = WakeWordGateConfig(triggers: ["hey clawd"], minPostTriggerGap: 0.3)
let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
#expect(match?.command == "do it")
}
}
private func makeSegments(
transcript: String,
words: [(String, TimeInterval, TimeInterval)])
-> [WakeWordSegment] {
var searchStart = transcript.startIndex
var output: [WakeWordSegment] = []
for (word, start, duration) in words {
let range = transcript.range(of: word, range: searchStart..<transcript.endIndex)
output.append(WakeWordSegment(text: word, start: start, duration: duration, range: range))
if let range { searchStart = range.upperBound }
}
return output
}
+3 -4
View File
@@ -1,12 +1,11 @@
# swabble — macOS 26 speech hook daemon (Swift 6.2)
Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framework (SpeechAnalyzer + SpeechTranscriber) instead of whisper.cpp. Local-only, wake word gated, dispatches a shell hook with the transcript. Shared wake-gate utilities live in `SwabbleKit` for reuse by other apps (iOS/macOS).
Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framework (SpeechAnalyzer + SpeechTranscriber) instead of whisper.cpp. Local-only, wake word gated, dispatches a shell hook with the transcript.
## Requirements
- macOS 26+, Swift 6.2, Speech.framework with on-device assets.
- Local only; no network calls during transcription.
- Wake word gating (default "clawd" plus aliases) with bypass flag `--no-wake`.
- `SwabbleKit` target (multi-platform) providing wake-word gating helpers that can use speech segment timing to require a post-trigger gap.
- Hook execution with cooldown, min_chars, timeout, prefix, env vars.
- Simple config at `~/.config/swabble/config.json` (JSON, Codable) — no TOML.
- CLI implemented with Commander (SwiftPM package `steipete/Commander`); core types are available via the SwiftPM library product `Swabble` for embedding.
@@ -18,7 +17,7 @@ Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framewo
- **CLI layer (Commander)**: Root command `swabble` with subcommands `serve`, `transcribe`, `test-hook`, `mic list|set`, `doctor`, `health`, `tail-log`. Runtime flags from Commander (`-v/--verbose`, `--json-output`, `--log-level`). Custom `--config` path applies everywhere.
- **Config**: `SwabbleConfig` Codable. Fields: audio device name/index, wake (enabled/word/aliases/sensitivity placeholder), hook (command/args/prefix/cooldown/min_chars/timeout/env), logging (level, format), transcripts (enabled, max kept), speech (locale, enableEtiquetteReplacements flag). Stored JSON; default written by `setup`.
- **Audio + Speech pipeline**: `SpeechPipeline` wraps `AVAudioEngine` input → `SpeechAnalyzer` with `SpeechTranscriber` module. Emits partial/final transcripts via async stream. Requests `.audioTimeRange` when transcripts enabled. Handles Speech permission and asset download prompts ahead of capture.
- **Wake gate**: CLI currently uses text-only keyword match; shared `SwabbleKit` gate can enforce a minimum pause between the wake word and the next token when speech segments are available. `--no-wake` disables gating.
- **Wake gate**: text-based keyword match against latest partial/final; strips wake term before hook dispatch. `--no-wake` disables.
- **Hook executor**: async `HookExecutor` spawns `Process` with configured args, prefix substitution `${hostname}`. Enforces cooldown + timeout; injects env `SWABBLE_TEXT`, `SWABBLE_PREFIX` plus user env map.
- **Transcripts store**: in-memory ring buffer; optional persisted JSON lines under `~/Library/Application Support/swabble/transcripts.log`.
- **Logging**: simple structured logger to stderr; respects log level.
@@ -26,7 +25,7 @@ Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framewo
## Out of scope (initial cut)
- Model management (Speech handles assets).
- Launchd helper (planned follow-up).
- Advanced wake-word detector (segment-aware gate now lives in `SwabbleKit`; CLI still text-only until segment timing is plumbed through).
- Advanced wake-word detector (text match only for now).
## Open decisions
- Whether to expose a UNIX control socket for `status`/`health` (currently planned as stdin/out direct calls).
+10 -31
View File
@@ -1,31 +1,10 @@
<?xml version="1.0" standalone="yes"?>
<rss xmlns:sparkle="http://www.andymatuschak.org/xml-namespaces/sparkle" version="2.0">
<channel>
<title>Clawdis</title>
<item>
<title>2.0.0-beta3</title>
<pubDate>Sat, 27 Dec 2025 19:02:02 +0100</pubDate>
<link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link>
<sparkle:version>2.0.0-beta3</sparkle:version>
<sparkle:shortVersionString>2.0.0-beta3</sparkle:shortVersionString>
<sparkle:minimumSystemVersion>15.0</sparkle:minimumSystemVersion>
<enclosure url="https://github.com/steipete/clawdis/releases/download/v2.0.0-beta4/Clawdis-2.0.0-beta3.zip" length="70407960" type="application/octet-stream" sparkle:edSignature="A8ySMmbLRrpIkqkrmc9QrC+6om8Iyqray/6x/YNiJxDoJeXjp2T5t8XT0CKJeNBUlDkzIj/fwiK53v0qQ59cDQ=="/>
</item>
<item>
<title>2.0.0-beta4</title>
<pubDate>Sat, 27 Dec 2025 19:43:22 +0100</pubDate>
<link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link>
<sparkle:version>2.0.0-beta4</sparkle:version>
<sparkle:shortVersionString>2.0.0-beta4</sparkle:shortVersionString>
<sparkle:minimumSystemVersion>15.0</sparkle:minimumSystemVersion>
<description><![CDATA[<h2>Clawdis 2.0.0-beta4</h2>
<h3>Fixes</h3>
<ul>
<li>Package contents: include Discord/hooks build outputs in the npm tarball to avoid missing module errors.</li>
</ul>
<p><a href="https://github.com/steipete/clawdis/blob/main/CHANGELOG.md">View full changelog</a></p>
]]></description>
<enclosure url="https://github.com/steipete/clawdis/releases/download/v2.0.0-beta4/Clawdis-2.0.0-beta4.zip" length="70407894" type="application/octet-stream" sparkle:edSignature="HmuiC7TnUn80ZApnKfb6w+JGSrjc3uUOndMrtbTp42bkBSVifbttNVazqvJueGBo4MgoJV8CP+zQNzVmtVihAQ=="/>
</item>
</channel>
</rss>
<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0"
xmlns:sparkle="http://www.andymatuschak.org/xml-namespaces/sparkle"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Clawdis Updates</title>
<link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link>
<description>Signed update feed for the Clawdis macOS companion app.</description>
</channel>
</rss>
+1 -1
View File
@@ -1,6 +1,6 @@
## Clawdis Node (Android) (internal)
Modern Android node app: connects to the **Gateway-owned bridge** (`_clawdis-bridge._tcp`) over TCP and exposes **Canvas + Chat + Camera**.
Modern Android “node” app (Iris parity): connects to the **Gateway-owned bridge** (`_clawdis-bridge._tcp`) over TCP and exposes **Canvas + Chat + Camera**.
Notes:
- The node keeps the connection alive via a **foreground service** (persistent notification with a Disconnect action).
+6 -23
View File
@@ -9,18 +9,12 @@ android {
namespace = "com.steipete.clawdis.node"
compileSdk = 36
sourceSets {
getByName("main") {
assets.srcDir(file("../../shared/ClawdisKit/Sources/ClawdisKit/Resources"))
}
}
defaultConfig {
applicationId = "com.steipete.clawdis.node"
minSdk = 31
targetSdk = 36
versionCode = 1
versionName = "2.0.0-beta3"
versionName = "0.1"
}
buildTypes {
@@ -31,7 +25,6 @@ android {
buildFeatures {
compose = true
buildConfig = true
}
compileOptions {
@@ -39,21 +32,15 @@ android {
targetCompatibility = JavaVersion.VERSION_17
}
kotlinOptions {
jvmTarget = "17"
}
packaging {
resources {
excludes += "/META-INF/{AL2.0,LGPL2.1}"
}
}
lint {
disable += setOf("IconLauncherShape")
}
}
kotlin {
compilerOptions {
jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17)
}
}
dependencies {
@@ -63,12 +50,11 @@ dependencies {
implementation("androidx.core:core-ktx:1.17.0")
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.10.0")
implementation("androidx.activity:activity-compose:1.12.2")
implementation("androidx.activity:activity-compose:1.12.1")
implementation("androidx.compose.ui:ui")
implementation("androidx.compose.ui:ui-tooling-preview")
implementation("androidx.compose.material3:material3")
implementation("androidx.compose.material:material-icons-extended")
implementation("androidx.navigation:navigation-compose:2.9.6")
debugImplementation("androidx.compose.ui:ui-tooling")
@@ -88,9 +74,6 @@ dependencies {
implementation("androidx.camera:camera-video:1.5.2")
implementation("androidx.camera:camera-view:1.5.2")
// Unicast DNS-SD (Wide-Area Bonjour) for tailnet discovery domains.
implementation("dnsjava:dnsjava:3.6.3")
testImplementation("junit:junit:4.13.2")
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
}
+1 -14
View File
@@ -3,8 +3,6 @@
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<uses-permission
android:name="android.permission.NEARBY_WIFI_DEVICES"
@@ -12,30 +10,19 @@
<uses-permission
android:name="android.permission.ACCESS_FINE_LOCATION"
android:maxSdkVersion="32" />
<uses-permission
android:name="android.permission.ACCESS_COARSE_LOCATION"
android:maxSdkVersion="32" />
<uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-feature
android:name="android.hardware.camera"
android:required="false" />
<application
android:name=".NodeApp"
android:allowBackup="true"
android:dataExtractionRules="@xml/data_extraction_rules"
android:fullBackupContent="@xml/backup_rules"
android:icon="@mipmap/ic_launcher"
android:roundIcon="@mipmap/ic_launcher_round"
android:label="@string/app_name"
android:supportsRtl="true"
android:networkSecurityConfig="@xml/network_security_config"
android:theme="@style/Theme.ClawdisNode">
<service
android:name=".NodeForegroundService"
android:exported="false"
android:foregroundServiceType="dataSync|microphone|mediaProjection" />
android:foregroundServiceType="dataSync" />
<activity
android:name=".MainActivity"
android:exported="true">
@@ -1,15 +0,0 @@
package com.steipete.clawdis.node
enum class CameraHudKind {
Photo,
Recording,
Success,
Error,
}
data class CameraHudState(
val token: Long,
val kind: CameraHudKind,
val message: String,
)
@@ -1,26 +0,0 @@
package com.steipete.clawdis.node
import android.content.Context
import android.os.Build
import android.provider.Settings
object DeviceNames {
fun bestDefaultNodeName(context: Context): String {
val deviceName =
runCatching {
Settings.Global.getString(context.contentResolver, "device_name")
}
.getOrNull()
?.trim()
.orEmpty()
if (deviceName.isNotEmpty()) return deviceName
val model =
listOfNotNull(Build.MANUFACTURER?.takeIf { it.isNotBlank() }, Build.MODEL?.takeIf { it.isNotBlank() })
.joinToString(" ")
.trim()
return model.ifEmpty { "Android Node" }
}
}
@@ -1,61 +1,28 @@
package com.steipete.clawdis.node
import android.Manifest
import android.content.pm.ApplicationInfo
import android.os.Bundle
import android.os.Build
import android.view.WindowManager
import android.webkit.WebView
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.activity.viewModels
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.ui.Modifier
import androidx.core.content.ContextCompat
import androidx.core.view.WindowCompat
import androidx.core.view.WindowInsetsCompat
import androidx.core.view.WindowInsetsControllerCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle
import com.steipete.clawdis.node.ui.RootScreen
import com.steipete.clawdis.node.ui.ClawdisTheme
import kotlinx.coroutines.launch
class MainActivity : ComponentActivity() {
private val viewModel: MainViewModel by viewModels()
private lateinit var permissionRequester: PermissionRequester
private lateinit var screenCaptureRequester: ScreenCaptureRequester
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
val isDebuggable = (applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE) != 0
WebView.setWebContentsDebuggingEnabled(isDebuggable)
applyImmersiveMode()
requestDiscoveryPermissionsIfNeeded()
requestNotificationPermissionIfNeeded()
NodeForegroundService.start(this)
permissionRequester = PermissionRequester(this)
screenCaptureRequester = ScreenCaptureRequester(this)
viewModel.camera.attachLifecycleOwner(this)
viewModel.camera.attachPermissionRequester(permissionRequester)
viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester)
viewModel.screenRecorder.attachPermissionRequester(permissionRequester)
lifecycleScope.launch {
repeatOnLifecycle(Lifecycle.State.STARTED) {
viewModel.preventSleep.collect { enabled ->
if (enabled) {
window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
} else {
window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
}
}
}
}
setContent {
ClawdisTheme {
MaterialTheme {
Surface(modifier = Modifier) {
RootScreen(viewModel = viewModel)
}
@@ -63,18 +30,6 @@ class MainActivity : ComponentActivity() {
}
}
override fun onResume() {
super.onResume()
applyImmersiveMode()
}
override fun onWindowFocusChanged(hasFocus: Boolean) {
super.onWindowFocusChanged(hasFocus)
if (hasFocus) {
applyImmersiveMode()
}
}
override fun onStart() {
super.onStart()
viewModel.setForeground(true)
@@ -85,14 +40,6 @@ class MainActivity : ComponentActivity() {
super.onStop()
}
private fun applyImmersiveMode() {
WindowCompat.setDecorFitsSystemWindows(window, false)
val controller = WindowInsetsControllerCompat(window, window.decorView)
controller.systemBarsBehavior =
WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE
controller.hide(WindowInsetsCompat.Type.systemBars())
}
private fun requestDiscoveryPermissionsIfNeeded() {
if (Build.VERSION.SDK_INT >= 33) {
val ok =
@@ -3,10 +3,8 @@ package com.steipete.clawdis.node
import android.app.Application
import androidx.lifecycle.AndroidViewModel
import com.steipete.clawdis.node.bridge.BridgeEndpoint
import com.steipete.clawdis.node.chat.OutgoingAttachment
import com.steipete.clawdis.node.node.CameraCaptureManager
import com.steipete.clawdis.node.node.CanvasController
import com.steipete.clawdis.node.node.ScreenRecordManager
import kotlinx.coroutines.flow.StateFlow
class MainViewModel(app: Application) : AndroidViewModel(app) {
@@ -14,41 +12,23 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val canvas: CanvasController = runtime.canvas
val camera: CameraCaptureManager = runtime.camera
val screenRecorder: ScreenRecordManager = runtime.screenRecorder
val bridges: StateFlow<List<BridgeEndpoint>> = runtime.bridges
val discoveryStatusText: StateFlow<String> = runtime.discoveryStatusText
val isConnected: StateFlow<Boolean> = runtime.isConnected
val statusText: StateFlow<String> = runtime.statusText
val serverName: StateFlow<String?> = runtime.serverName
val remoteAddress: StateFlow<String?> = runtime.remoteAddress
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
val instanceId: StateFlow<String> = runtime.instanceId
val displayName: StateFlow<String> = runtime.displayName
val cameraEnabled: StateFlow<Boolean> = runtime.cameraEnabled
val preventSleep: StateFlow<Boolean> = runtime.preventSleep
val wakeWords: StateFlow<List<String>> = runtime.wakeWords
val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode
val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening
val manualEnabled: StateFlow<Boolean> = runtime.manualEnabled
val manualHost: StateFlow<String> = runtime.manualHost
val manualPort: StateFlow<Int> = runtime.manualPort
val canvasDebugStatusEnabled: StateFlow<Boolean> = runtime.canvasDebugStatusEnabled
val chatSessionKey: StateFlow<String> = runtime.chatSessionKey
val chatSessionId: StateFlow<String?> = runtime.chatSessionId
val chatMessages = runtime.chatMessages
val chatMessages: StateFlow<List<NodeRuntime.ChatMessage>> = runtime.chatMessages
val chatError: StateFlow<String?> = runtime.chatError
val chatHealthOk: StateFlow<Boolean> = runtime.chatHealthOk
val chatThinkingLevel: StateFlow<String> = runtime.chatThinkingLevel
val chatStreamingAssistantText: StateFlow<String?> = runtime.chatStreamingAssistantText
val chatPendingToolCalls = runtime.chatPendingToolCalls
val chatSessions = runtime.chatSessions
val pendingRunCount: StateFlow<Int> = runtime.pendingRunCount
fun setForeground(value: Boolean) {
@@ -63,10 +43,6 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.setCameraEnabled(value)
}
fun setPreventSleep(value: Boolean) {
runtime.setPreventSleep(value)
}
fun setManualEnabled(value: Boolean) {
runtime.setManualEnabled(value)
}
@@ -79,22 +55,6 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.setManualPort(value)
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
runtime.setCanvasDebugStatusEnabled(value)
}
fun setWakeWords(words: List<String>) {
runtime.setWakeWords(words)
}
fun resetWakeWordsDefaults() {
runtime.resetWakeWordsDefaults()
}
fun setVoiceWakeMode(mode: VoiceWakeMode) {
runtime.setVoiceWakeMode(mode)
}
fun connect(endpoint: BridgeEndpoint) {
runtime.connect(endpoint)
}
@@ -107,35 +67,12 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.disconnect()
}
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
runtime.handleCanvasA2UIActionFromWebView(payloadJson)
}
fun loadChat(sessionKey: String = "main") {
runtime.loadChat(sessionKey)
}
fun refreshChat() {
runtime.refreshChat()
}
fun refreshChatSessions(limit: Int? = null) {
runtime.refreshChatSessions(limit = limit)
}
fun setChatThinkingLevel(level: String) {
runtime.setChatThinkingLevel(level)
}
fun switchChatSession(sessionKey: String) {
runtime.switchChatSession(sessionKey)
}
fun abortChat() {
runtime.abortChat()
}
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
runtime.sendChat(message = message, thinking = thinking, attachments = attachments)
fun sendChat(sessionKey: String = "main", message: String) {
runtime.sendChat(sessionKey, message)
}
}
@@ -5,13 +5,11 @@ import android.app.NotificationChannel
import android.app.NotificationManager
import android.app.Service
import android.app.PendingIntent
import android.Manifest
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.content.pm.ServiceInfo
import android.os.Build
import androidx.core.app.NotificationCompat
import androidx.core.content.ContextCompat
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
@@ -23,42 +21,26 @@ import kotlinx.coroutines.launch
class NodeForegroundService : Service() {
private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
private var notificationJob: Job? = null
private var lastRequiresMic = false
private var didStartForeground = false
override fun onCreate() {
super.onCreate()
ensureChannel()
val initial = buildNotification(title = "Clawdis Node", text = "Starting…")
startForegroundWithTypes(notification = initial, requiresMic = false)
if (Build.VERSION.SDK_INT >= 29) {
startForeground(NOTIFICATION_ID, initial, ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
} else {
startForeground(NOTIFICATION_ID, initial)
}
val runtime = (application as NodeApp).runtime
notificationJob =
scope.launch {
combine(
runtime.statusText,
runtime.serverName,
runtime.isConnected,
runtime.voiceWakeMode,
runtime.voiceWakeIsListening,
) { status, server, connected, voiceMode, voiceListening ->
Quint(status, server, connected, voiceMode, voiceListening)
}.collect { (status, server, connected, voiceMode, voiceListening) ->
combine(runtime.statusText, runtime.serverName, runtime.isConnected) { status, server, connected ->
Triple(status, server, connected)
}.collect { (status, server, connected) ->
val title = if (connected) "Clawdis Node · Connected" else "Clawdis Node"
val voiceSuffix =
if (voiceMode == VoiceWakeMode.Always) {
if (voiceListening) " · Voice Wake: Listening" else " · Voice Wake: Paused"
} else {
""
}
val text = (server?.let { "$status · $it" } ?: status) + voiceSuffix
val requiresMic =
voiceMode == VoiceWakeMode.Always && hasRecordAudioPermission()
startForegroundWithTypes(
notification = buildNotification(title = title, text = text),
requiresMic = requiresMic,
)
val text = server?.let { "$status · $it" } ?: status
updateNotification(buildNotification(title = title, text = text))
}
}
}
@@ -84,6 +66,7 @@ class NodeForegroundService : Service() {
override fun onBind(intent: Intent?) = null
private fun ensureChannel() {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return
val mgr = getSystemService(NotificationManager::class.java)
val channel =
NotificationChannel(
@@ -99,11 +82,16 @@ class NodeForegroundService : Service() {
private fun buildNotification(title: String, text: String): Notification {
val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP)
val flags = PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
val flags =
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
} else {
PendingIntent.FLAG_UPDATE_CURRENT
}
val stopPending = PendingIntent.getService(this, 2, stopIntent, flags)
return NotificationCompat.Builder(this, CHANNEL_ID)
.setSmallIcon(R.mipmap.ic_launcher)
.setSmallIcon(android.R.drawable.stat_sys_upload)
.setContentTitle(title)
.setContentText(text)
.setOngoing(true)
@@ -118,30 +106,6 @@ class NodeForegroundService : Service() {
mgr.notify(NOTIFICATION_ID, notification)
}
private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) {
if (didStartForeground && requiresMic == lastRequiresMic) {
updateNotification(notification)
return
}
lastRequiresMic = requiresMic
val types =
if (requiresMic) {
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
} else {
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
}
startForeground(NOTIFICATION_ID, notification, types)
didStartForeground = true
}
private fun hasRecordAudioPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
)
}
companion object {
private const val CHANNEL_ID = "connection"
private const val NOTIFICATION_ID = 1
@@ -150,7 +114,11 @@ class NodeForegroundService : Service() {
fun start(context: Context) {
val intent = Intent(context, NodeForegroundService::class.java)
context.startForegroundService(intent)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
context.startForegroundService(intent)
} else {
context.startService(intent)
}
}
fun stop(context: Context) {
@@ -159,5 +127,3 @@ class NodeForegroundService : Service() {
}
}
}
private data class Quint<A, B, C, D, E>(val first: A, val second: B, val third: C, val fourth: D, val fifth: E)
@@ -1,42 +1,19 @@
package com.steipete.clawdis.node
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.os.Build
import android.os.SystemClock
import androidx.core.content.ContextCompat
import com.steipete.clawdis.node.chat.ChatController
import com.steipete.clawdis.node.chat.ChatMessage
import com.steipete.clawdis.node.chat.ChatPendingToolCall
import com.steipete.clawdis.node.chat.ChatSessionEntry
import com.steipete.clawdis.node.chat.OutgoingAttachment
import com.steipete.clawdis.node.bridge.BridgeDiscovery
import com.steipete.clawdis.node.bridge.BridgeEndpoint
import com.steipete.clawdis.node.bridge.BridgePairingClient
import com.steipete.clawdis.node.bridge.BridgeSession
import com.steipete.clawdis.node.node.CameraCaptureManager
import com.steipete.clawdis.node.BuildConfig
import com.steipete.clawdis.node.node.CanvasController
import com.steipete.clawdis.node.node.ScreenRecordManager
import com.steipete.clawdis.node.protocol.ClawdisCapability
import com.steipete.clawdis.node.protocol.ClawdisCameraCommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UIAction
import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UICommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasCommand
import com.steipete.clawdis.node.protocol.ClawdisScreenCommand
import com.steipete.clawdis.node.voice.VoiceWakeManager
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
@@ -44,8 +21,6 @@ import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import java.util.concurrent.atomic.AtomicLong
class NodeRuntime(context: Context) {
private val appContext = context.applicationContext
@@ -54,53 +29,17 @@ class NodeRuntime(context: Context) {
val prefs = SecurePrefs(appContext)
val canvas = CanvasController()
val camera = CameraCaptureManager(appContext)
val screenRecorder = ScreenRecordManager(appContext)
private val json = Json { ignoreUnknownKeys = true }
private val externalAudioCaptureActive = MutableStateFlow(false)
private val voiceWake: VoiceWakeManager by lazy {
VoiceWakeManager(
context = appContext,
scope = scope,
onCommand = { command ->
session.sendEvent(
event = "agent.request",
payloadJson =
buildJsonObject {
put("message", JsonPrimitive(command))
put("sessionKey", JsonPrimitive("main"))
put("thinking", JsonPrimitive(chatThinkingLevel.value))
put("deliver", JsonPrimitive(false))
}.toString(),
)
},
)
}
val voiceWakeIsListening: StateFlow<Boolean>
get() = voiceWake.isListening
val voiceWakeStatusText: StateFlow<String>
get() = voiceWake.statusText
private val discovery = BridgeDiscovery(appContext, scope = scope)
private val discovery = BridgeDiscovery(appContext)
val bridges: StateFlow<List<BridgeEndpoint>> = discovery.bridges
val discoveryStatusText: StateFlow<String> = discovery.statusText
private val _isConnected = MutableStateFlow(false)
val isConnected: StateFlow<Boolean> = _isConnected.asStateFlow()
private val _statusText = MutableStateFlow("Offline")
private val _statusText = MutableStateFlow("Not connected")
val statusText: StateFlow<String> = _statusText.asStateFlow()
private val cameraHudSeq = AtomicLong(0)
private val _cameraHud = MutableStateFlow<CameraHudState?>(null)
val cameraHud: StateFlow<CameraHudState?> = _cameraHud.asStateFlow()
private val _cameraFlashToken = MutableStateFlow(0L)
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
private val _serverName = MutableStateFlow<String?>(null)
val serverName: StateFlow<String?> = _serverName.asStateFlow()
@@ -110,8 +49,6 @@ class NodeRuntime(context: Context) {
private val _isForeground = MutableStateFlow(true)
val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow()
private var lastAutoA2uiUrl: String? = null
private val session =
BridgeSession(
scope = scope,
@@ -120,10 +57,13 @@ class NodeRuntime(context: Context) {
_serverName.value = name
_remoteAddress.value = remote
_isConnected.value = true
scope.launch { refreshWakeWordsFromGateway() }
maybeNavigateToA2uiOnConnect()
},
onDisconnected = { message -> handleSessionDisconnected(message) },
onDisconnected = { message ->
_statusText.value = message
_serverName.value = null
_remoteAddress.value = null
_isConnected.value = false
},
onEvent = { event, payloadJson ->
handleBridgeEvent(event, payloadJson)
},
@@ -132,92 +72,29 @@ class NodeRuntime(context: Context) {
},
)
private val chat = ChatController(scope = scope, session = session, json = json)
private fun handleSessionDisconnected(message: String) {
_statusText.value = message
_serverName.value = null
_remoteAddress.value = null
_isConnected.value = false
chat.onDisconnected(message)
showLocalCanvasOnDisconnect()
}
private fun maybeNavigateToA2uiOnConnect() {
val a2uiUrl = resolveA2uiHostUrl() ?: return
val current = canvas.currentUrl()?.trim().orEmpty()
if (current.isEmpty() || current == lastAutoA2uiUrl) {
lastAutoA2uiUrl = a2uiUrl
canvas.navigate(a2uiUrl)
}
}
private fun showLocalCanvasOnDisconnect() {
lastAutoA2uiUrl = null
canvas.navigate("")
}
val instanceId: StateFlow<String> = prefs.instanceId
val displayName: StateFlow<String> = prefs.displayName
val cameraEnabled: StateFlow<Boolean> = prefs.cameraEnabled
val preventSleep: StateFlow<Boolean> = prefs.preventSleep
val wakeWords: StateFlow<List<String>> = prefs.wakeWords
val voiceWakeMode: StateFlow<VoiceWakeMode> = prefs.voiceWakeMode
val manualEnabled: StateFlow<Boolean> = prefs.manualEnabled
val manualHost: StateFlow<String> = prefs.manualHost
val manualPort: StateFlow<Int> = prefs.manualPort
val lastDiscoveredStableId: StateFlow<String> = prefs.lastDiscoveredStableId
val canvasDebugStatusEnabled: StateFlow<Boolean> = prefs.canvasDebugStatusEnabled
private var didAutoConnect = false
private var suppressWakeWordsSync = false
private var wakeWordsSyncJob: Job? = null
val chatSessionKey: StateFlow<String> = chat.sessionKey
val chatSessionId: StateFlow<String?> = chat.sessionId
val chatMessages: StateFlow<List<ChatMessage>> = chat.messages
val chatError: StateFlow<String?> = chat.errorText
val chatHealthOk: StateFlow<Boolean> = chat.healthOk
val chatThinkingLevel: StateFlow<String> = chat.thinkingLevel
val chatStreamingAssistantText: StateFlow<String?> = chat.streamingAssistantText
val chatPendingToolCalls: StateFlow<List<ChatPendingToolCall>> = chat.pendingToolCalls
val chatSessions: StateFlow<List<ChatSessionEntry>> = chat.sessions
val pendingRunCount: StateFlow<Int> = chat.pendingRunCount
data class ChatMessage(val id: String, val role: String, val text: String, val timestampMs: Long?)
private val _chatMessages = MutableStateFlow<List<ChatMessage>>(emptyList())
val chatMessages: StateFlow<List<ChatMessage>> = _chatMessages.asStateFlow()
private val _chatError = MutableStateFlow<String?>(null)
val chatError: StateFlow<String?> = _chatError.asStateFlow()
private val pendingRuns = mutableSetOf<String>()
private val _pendingRunCount = MutableStateFlow(0)
val pendingRunCount: StateFlow<Int> = _pendingRunCount.asStateFlow()
init {
scope.launch {
combine(
voiceWakeMode,
isForeground,
externalAudioCaptureActive,
wakeWords,
) { mode, foreground, externalAudio, words ->
Quad(mode, foreground, externalAudio, words)
}.distinctUntilChanged()
.collect { (mode, foreground, externalAudio, words) ->
voiceWake.setTriggerWords(words)
val shouldListen =
when (mode) {
VoiceWakeMode.Off -> false
VoiceWakeMode.Foreground -> foreground
VoiceWakeMode.Always -> true
} && !externalAudio
if (!shouldListen) {
voiceWake.stop(statusText = if (mode == VoiceWakeMode.Off) "Off" else "Paused")
return@collect
}
if (!hasRecordAudioPermission()) {
voiceWake.stop(statusText = "Microphone permission required")
return@collect
}
voiceWake.start()
}
}
scope.launch(Dispatchers.Default) {
bridges.collect { list ->
if (list.isNotEmpty()) {
@@ -248,22 +125,6 @@ class NodeRuntime(context: Context) {
connect(target)
}
}
scope.launch {
combine(
canvasDebugStatusEnabled,
statusText,
serverName,
remoteAddress,
) { debugEnabled, status, server, remote ->
Quad(debugEnabled, status, server, remote)
}.distinctUntilChanged()
.collect { (debugEnabled, status, server, remote) ->
canvas.setDebugStatusEnabled(debugEnabled)
if (!debugEnabled) return@collect
canvas.setDebugStatus(status, server ?: remote)
}
}
}
fun setForeground(value: Boolean) {
@@ -278,10 +139,6 @@ class NodeRuntime(context: Context) {
prefs.setCameraEnabled(value)
}
fun setPreventSleep(value: Boolean) {
prefs.setPreventSleep(value)
}
fun setManualEnabled(value: Boolean) {
prefs.setManualEnabled(value)
}
@@ -294,66 +151,13 @@ class NodeRuntime(context: Context) {
prefs.setManualPort(value)
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
prefs.setCanvasDebugStatusEnabled(value)
}
fun setWakeWords(words: List<String>) {
prefs.setWakeWords(words)
scheduleWakeWordsSyncIfNeeded()
}
fun resetWakeWordsDefaults() {
setWakeWords(SecurePrefs.defaultWakeWords)
}
fun setVoiceWakeMode(mode: VoiceWakeMode) {
prefs.setVoiceWakeMode(mode)
}
fun connect(endpoint: BridgeEndpoint) {
scope.launch {
_statusText.value = "Connecting…"
val storedToken = prefs.loadBridgeToken()
val modelIdentifier = listOfNotNull(Build.MANUFACTURER, Build.MODEL)
.joinToString(" ")
.trim()
.ifEmpty { null }
val invokeCommands =
buildList {
add(ClawdisCanvasCommand.Present.rawValue)
add(ClawdisCanvasCommand.Hide.rawValue)
add(ClawdisCanvasCommand.Navigate.rawValue)
add(ClawdisCanvasCommand.Eval.rawValue)
add(ClawdisCanvasCommand.Snapshot.rawValue)
add(ClawdisCanvasA2UICommand.Push.rawValue)
add(ClawdisCanvasA2UICommand.PushJSONL.rawValue)
add(ClawdisCanvasA2UICommand.Reset.rawValue)
add(ClawdisScreenCommand.Record.rawValue)
if (cameraEnabled.value) {
add(ClawdisCameraCommand.Snap.rawValue)
add(ClawdisCameraCommand.Clip.rawValue)
}
}
val resolved =
if (storedToken.isNullOrBlank()) {
_statusText.value = "Pairing…"
val caps = buildList {
add(ClawdisCapability.Canvas.rawValue)
add(ClawdisCapability.Screen.rawValue)
if (cameraEnabled.value) add(ClawdisCapability.Camera.rawValue)
if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) {
add(ClawdisCapability.VoiceWake.rawValue)
}
}
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
val advertisedVersion =
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
BridgePairingClient().pairAndHello(
endpoint = endpoint,
hello =
@@ -362,11 +166,7 @@ class NodeRuntime(context: Context) {
displayName = displayName.value,
token = null,
platform = "Android",
version = advertisedVersion,
deviceFamily = "Android",
modelIdentifier = modelIdentifier,
caps = caps,
commands = invokeCommands,
version = "dev",
),
)
} else {
@@ -380,13 +180,6 @@ class NodeRuntime(context: Context) {
val authToken = requireNotNull(resolved.token).trim()
prefs.saveBridgeToken(authToken)
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
val advertisedVersion =
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
session.connect(
endpoint = endpoint,
hello =
@@ -395,31 +188,12 @@ class NodeRuntime(context: Context) {
displayName = displayName.value,
token = authToken,
platform = "Android",
version = advertisedVersion,
deviceFamily = "Android",
modelIdentifier = modelIdentifier,
caps =
buildList {
add(ClawdisCapability.Canvas.rawValue)
add(ClawdisCapability.Screen.rawValue)
if (cameraEnabled.value) add(ClawdisCapability.Camera.rawValue)
if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) {
add(ClawdisCapability.VoiceWake.rawValue)
}
},
commands = invokeCommands,
version = "dev",
),
)
}
}
private fun hasRecordAudioPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
)
}
fun connectManual() {
val host = manualHost.value.trim()
val port = manualPort.value
@@ -434,176 +208,140 @@ class NodeRuntime(context: Context) {
session.disconnect()
}
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
fun loadChat(sessionKey: String = "main") {
scope.launch {
val trimmed = payloadJson.trim()
if (trimmed.isEmpty()) return@launch
val root =
try {
json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch
} catch (_: Throwable) {
return@launch
}
val userActionObj = (root["userAction"] as? JsonObject) ?: root
val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty {
java.util.UUID.randomUUID().toString()
}
val name = ClawdisCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch
val surfaceId =
(userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" }
val sourceComponentId =
(userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" }
val contextJson = (userActionObj["context"] as? JsonObject)?.toString()
val sessionKey = "main"
val message =
ClawdisCanvasA2UIAction.formatAgentMessage(
actionName = name,
sessionKey = sessionKey,
surfaceId = surfaceId,
sourceComponentId = sourceComponentId,
host = displayName.value,
instanceId = instanceId.value.lowercase(),
contextJson = contextJson,
)
val connected = isConnected.value
var error: String? = null
if (connected) {
try {
session.sendEvent(
event = "agent.request",
payloadJson =
buildJsonObject {
put("message", JsonPrimitive(message))
put("sessionKey", JsonPrimitive(sessionKey))
put("thinking", JsonPrimitive("low"))
put("deliver", JsonPrimitive(false))
put("key", JsonPrimitive(actionId))
}.toString(),
)
} catch (e: Throwable) {
error = e.message ?: "send failed"
}
} else {
error = "bridge not connected"
_chatError.value = null
try {
// Best-effort; push events are optional, but improve latency.
session.sendEvent("chat.subscribe", """{"sessionKey":"$sessionKey"}""")
} catch (_: Throwable) {
// ignore
}
try {
canvas.eval(
ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(
actionId = actionId,
ok = connected && error == null,
error = error,
),
)
} catch (_: Throwable) {
// ignore
val res = session.request("chat.history", """{"sessionKey":"$sessionKey"}""")
_chatMessages.value = parseHistory(res)
} catch (e: Exception) {
_chatError.value = e.message
}
}
}
fun loadChat(sessionKey: String = "main") {
chat.load(sessionKey)
}
fun sendChat(sessionKey: String = "main", message: String, thinking: String = "off") {
val trimmed = message.trim()
if (trimmed.isEmpty()) return
scope.launch {
_chatError.value = null
val idem = java.util.UUID.randomUUID().toString()
fun refreshChat() {
chat.refresh()
}
_chatMessages.value =
_chatMessages.value +
ChatMessage(
id = java.util.UUID.randomUUID().toString(),
role = "user",
text = trimmed,
timestampMs = System.currentTimeMillis(),
)
fun refreshChatSessions(limit: Int? = null) {
chat.refreshSessions(limit = limit)
}
fun setChatThinkingLevel(level: String) {
chat.setThinkingLevel(level)
}
fun switchChatSession(sessionKey: String) {
chat.switchSession(sessionKey)
}
fun abortChat() {
chat.abort()
}
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
chat.sendMessage(message = message, thinkingLevel = thinking, attachments = attachments)
try {
val params =
"""{"sessionKey":"$sessionKey","message":${trimmed.toJsonString()},"thinking":"$thinking","timeoutMs":30000,"idempotencyKey":"$idem"}"""
val res = session.request("chat.send", params)
val runId = parseRunId(res) ?: idem
pendingRuns.add(runId)
_pendingRunCount.value = pendingRuns.size
} catch (e: Exception) {
_chatError.value = e.message
}
}
}
private fun handleBridgeEvent(event: String, payloadJson: String?) {
if (event == "voicewake.changed") {
if (payloadJson.isNullOrBlank()) return
try {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val array = payload["triggers"] as? JsonArray ?: return
val triggers = array.mapNotNull { it.asStringOrNull() }
applyWakeWordsFromGateway(triggers)
} catch (_: Throwable) {
// ignore
if (event != "chat" || payloadJson.isNullOrBlank()) return
try {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val state = payload["state"].asStringOrNull()
val runId = payload["runId"].asStringOrNull()
if (!runId.isNullOrBlank()) {
pendingRuns.remove(runId)
_pendingRunCount.value = pendingRuns.size
}
return
}
chat.handleBridgeEvent(event, payloadJson)
}
private fun applyWakeWordsFromGateway(words: List<String>) {
suppressWakeWordsSync = true
prefs.setWakeWords(words)
suppressWakeWordsSync = false
}
private fun scheduleWakeWordsSyncIfNeeded() {
if (suppressWakeWordsSync) return
if (!_isConnected.value) return
val snapshot = prefs.wakeWords.value
wakeWordsSyncJob?.cancel()
wakeWordsSyncJob =
scope.launch {
delay(650)
val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() }
val params = """{"triggers":[$jsonList]}"""
try {
session.request("voicewake.set", params)
} catch (_: Throwable) {
// ignore
when (state) {
"final" -> {
val msgObj = payload["message"].asObjectOrNull()
val role = msgObj?.get("role").asStringOrNull() ?: "assistant"
val text = extractTextFromMessage(msgObj)
if (!text.isNullOrBlank()) {
_chatMessages.value =
_chatMessages.value +
ChatMessage(
id = java.util.UUID.randomUUID().toString(),
role = role,
text = text,
timestampMs = System.currentTimeMillis(),
)
}
}
"error" -> {
_chatError.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed"
}
}
}
private suspend fun refreshWakeWordsFromGateway() {
if (!_isConnected.value) return
try {
val res = session.request("voicewake.get", "{}")
val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return
val array = payload["triggers"] as? JsonArray ?: return
val triggers = array.mapNotNull { it.asStringOrNull() }
applyWakeWordsFromGateway(triggers)
} catch (_: Throwable) {
// ignore
}
}
private fun parseHistory(historyJson: String): List<ChatMessage> {
val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return emptyList()
val raw = root["messages"] ?: return emptyList()
val array = raw as? JsonArray ?: return emptyList()
return array.mapNotNull { item ->
val obj = item as? JsonObject ?: return@mapNotNull null
val role = obj["role"].asStringOrNull() ?: return@mapNotNull null
val text = extractTextFromMessage(obj) ?: return@mapNotNull null
ChatMessage(
id = java.util.UUID.randomUUID().toString(),
role = role,
text = text,
timestampMs = null,
)
}
}
private fun extractTextFromMessage(msgObj: JsonObject?): String? {
if (msgObj == null) return null
val content = msgObj["content"] ?: return null
return when (content) {
is JsonPrimitive -> content.asStringOrNull()
else -> {
val arr = (content as? JsonArray) ?: return null
arr.mapNotNull { part ->
val p = part as? JsonObject ?: return@mapNotNull null
p["text"].asStringOrNull()
}.joinToString("\n").trim().ifBlank { null }
}
}
}
private fun parseRunId(resJson: String): String? {
return try {
json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull()
} catch (_: Throwable) {
null
}
}
private suspend fun handleInvoke(command: String, paramsJson: String?): BridgeSession.InvokeResult {
if (
command.startsWith(ClawdisCanvasCommand.NamespacePrefix) ||
command.startsWith(ClawdisCanvasA2UICommand.NamespacePrefix) ||
command.startsWith(ClawdisCameraCommand.NamespacePrefix) ||
command.startsWith(ClawdisScreenCommand.NamespacePrefix)
) {
if (command.startsWith("screen.") || command.startsWith("camera.")) {
if (!isForeground.value) {
return BridgeSession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground",
message = "NODE_BACKGROUND_UNAVAILABLE: screen/camera commands require foreground",
)
}
}
if (command.startsWith(ClawdisCameraCommand.NamespacePrefix) && !cameraEnabled.value) {
if (command.startsWith("camera.") && !cameraEnabled.value) {
return BridgeSession.InvokeResult.error(
code = "CAMERA_DISABLED",
message = "CAMERA_DISABLED: enable Camera in Settings",
@@ -611,18 +349,19 @@ class NodeRuntime(context: Context) {
}
return when (command) {
ClawdisCanvasCommand.Present.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
"screen.show" -> BridgeSession.InvokeResult.ok(null)
"screen.hide" -> BridgeSession.InvokeResult.ok(null)
"screen.setMode" -> {
val mode = CanvasController.parseMode(paramsJson)
canvas.setMode(mode)
BridgeSession.InvokeResult.ok(null)
}
ClawdisCanvasCommand.Hide.rawValue -> BridgeSession.InvokeResult.ok(null)
ClawdisCanvasCommand.Navigate.rawValue -> {
"screen.navigate" -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
if (url != null) canvas.navigate(url)
BridgeSession.InvokeResult.ok(null)
}
ClawdisCanvasCommand.Eval.rawValue -> {
"screen.eval" -> {
val js =
CanvasController.parseEvalJs(paramsJson)
?: return BridgeSession.InvokeResult.error(
@@ -640,103 +379,25 @@ class NodeRuntime(context: Context) {
}
BridgeSession.InvokeResult.ok("""{"result":${result.toJsonString()}}""")
}
ClawdisCanvasCommand.Snapshot.rawValue -> {
val snapshotParams = CanvasController.parseSnapshotParams(paramsJson)
"screen.snapshot" -> {
val maxWidth = CanvasController.parseSnapshotMaxWidth(paramsJson)
val base64 =
try {
canvas.snapshotBase64(
format = snapshotParams.format,
quality = snapshotParams.quality,
maxWidth = snapshotParams.maxWidth,
)
canvas.snapshotPngBase64(maxWidth = maxWidth)
} catch (err: Throwable) {
return BridgeSession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
)
}
BridgeSession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""")
BridgeSession.InvokeResult.ok("""{"format":"png","base64":"$base64"}""")
}
ClawdisCanvasA2UICommand.Reset.rawValue -> {
val a2uiUrl = resolveA2uiHostUrl()
?: return BridgeSession.InvokeResult.error(
code = "A2UI_HOST_NOT_CONFIGURED",
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
)
val ready = ensureA2uiReady(a2uiUrl)
if (!ready) {
return BridgeSession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI host not reachable",
)
}
val res = canvas.eval(a2uiResetJS)
BridgeSession.InvokeResult.ok(res)
}
ClawdisCanvasA2UICommand.Push.rawValue, ClawdisCanvasA2UICommand.PushJSONL.rawValue -> {
val messages =
try {
decodeA2uiMessages(command, paramsJson)
} catch (err: Throwable) {
return BridgeSession.InvokeResult.error(code = "INVALID_REQUEST", message = err.message ?: "invalid A2UI payload")
}
val a2uiUrl = resolveA2uiHostUrl()
?: return BridgeSession.InvokeResult.error(
code = "A2UI_HOST_NOT_CONFIGURED",
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
)
val ready = ensureA2uiReady(a2uiUrl)
if (!ready) {
return BridgeSession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI host not reachable",
)
}
val js = a2uiApplyMessagesJS(messages)
val res = canvas.eval(js)
BridgeSession.InvokeResult.ok(res)
}
ClawdisCameraCommand.Snap.rawValue -> {
showCameraHud(message = "Taking photo…", kind = CameraHudKind.Photo)
triggerCameraFlash()
val res =
try {
camera.snap(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2200)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
showCameraHud(message = "Photo captured", kind = CameraHudKind.Success, autoHideMs = 1600)
"camera.snap" -> {
val res = camera.snap(paramsJson)
BridgeSession.InvokeResult.ok(res.payloadJson)
}
ClawdisCameraCommand.Clip.rawValue -> {
val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false
if (includeAudio) externalAudioCaptureActive.value = true
try {
showCameraHud(message = "Recording…", kind = CameraHudKind.Recording)
val res =
try {
camera.clip(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2400)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
showCameraHud(message = "Clip captured", kind = CameraHudKind.Success, autoHideMs = 1800)
BridgeSession.InvokeResult.ok(res.payloadJson)
} finally {
if (includeAudio) externalAudioCaptureActive.value = false
}
}
ClawdisScreenCommand.Record.rawValue -> {
val res =
try {
screenRecorder.record(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
"camera.clip" -> {
val res = camera.clip(paramsJson)
BridgeSession.InvokeResult.ok(res.payloadJson)
}
else ->
@@ -746,161 +407,6 @@ class NodeRuntime(context: Context) {
)
}
}
private fun triggerCameraFlash() {
// Token is used as a pulse trigger; value doesn't matter as long as it changes.
_cameraFlashToken.value = SystemClock.elapsedRealtimeNanos()
}
private fun showCameraHud(message: String, kind: CameraHudKind, autoHideMs: Long? = null) {
val token = cameraHudSeq.incrementAndGet()
_cameraHud.value = CameraHudState(token = token, kind = kind, message = message)
if (autoHideMs != null && autoHideMs > 0) {
scope.launch {
delay(autoHideMs)
if (_cameraHud.value?.token == token) _cameraHud.value = null
}
}
}
private fun invokeErrorFromThrowable(err: Throwable): Pair<String, String> {
val raw = (err.message ?: "").trim()
if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: camera error"
val idx = raw.indexOf(':')
if (idx <= 0) return "UNAVAILABLE" to raw
val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" }
val message = raw.substring(idx + 1).trim().ifEmpty { raw }
// Preserve full string for callers/logging, but keep the returned message human-friendly.
return code to "$code: $message"
}
private fun resolveA2uiHostUrl(): String? {
val raw = session.currentCanvasHostUrl()?.trim().orEmpty()
if (raw.isBlank()) return null
val base = raw.trimEnd('/')
return "${base}/__clawdis__/a2ui/"
}
private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
try {
val already = canvas.eval(a2uiReadyCheckJS)
if (already == "true") return true
} catch (_: Throwable) {
// ignore
}
canvas.navigate(a2uiUrl)
repeat(50) {
try {
val ready = canvas.eval(a2uiReadyCheckJS)
if (ready == "true") return true
} catch (_: Throwable) {
// ignore
}
delay(120)
}
return false
}
private fun decodeA2uiMessages(command: String, paramsJson: String?): String {
val raw = paramsJson?.trim().orEmpty()
if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
val hasMessagesArray = obj["messages"] is JsonArray
if (command == ClawdisCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) {
val jsonl = jsonlField
if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
val messages =
jsonl
.lineSequence()
.map { it.trim() }
.filter { it.isNotBlank() }
.mapIndexed { idx, line ->
val el = json.parseToJsonElement(line)
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
.toList()
return JsonArray(messages).toString()
}
val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
val out =
arr.mapIndexed { idx, el ->
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
return JsonArray(out).toString()
}
private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) {
if (msg.containsKey("createSurface")) {
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.",
)
}
val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface")
val matched = msg.keys.filter { allowed.contains(it) }
if (matched.size != 1) {
val found = msg.keys.sorted().joinToString(", ")
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found",
)
}
}
}
private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
private const val a2uiReadyCheckJS: String =
"""
(() => {
try {
return !!globalThis.clawdisA2UI && typeof globalThis.clawdisA2UI.applyMessages === 'function';
} catch (_) {
return false;
}
})()
"""
private const val a2uiResetJS: String =
"""
(() => {
try {
if (!globalThis.clawdisA2UI) return { ok: false, error: "missing clawdisA2UI" };
return globalThis.clawdisA2UI.reset();
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
"""
private fun a2uiApplyMessagesJS(messagesJson: String): String {
return """
(() => {
try {
if (!globalThis.clawdisA2UI) return { ok: false, error: "missing clawdisA2UI" };
const messages = $messagesJson;
return globalThis.clawdisA2UI.applyMessages(messages);
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
""".trimIndent()
}
private fun String.toJsonString(): String {
@@ -1,132 +0,0 @@
package com.steipete.clawdis.node
import android.content.pm.PackageManager
import android.content.Intent
import android.Manifest
import android.net.Uri
import android.provider.Settings
import androidx.appcompat.app.AlertDialog
import androidx.activity.ComponentActivity
import androidx.activity.result.ActivityResultLauncher
import androidx.activity.result.contract.ActivityResultContracts
import androidx.core.content.ContextCompat
import androidx.core.app.ActivityCompat
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlin.coroutines.resume
class PermissionRequester(private val activity: ComponentActivity) {
private val mutex = Mutex()
private var pending: CompletableDeferred<Map<String, Boolean>>? = null
private val launcher: ActivityResultLauncher<Array<String>> =
activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result ->
val p = pending
pending = null
p?.complete(result)
}
suspend fun requestIfMissing(
permissions: List<String>,
timeoutMs: Long = 20_000,
): Map<String, Boolean> =
mutex.withLock {
val missing =
permissions.filter { perm ->
ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED
}
if (missing.isEmpty()) {
return permissions.associateWith { true }
}
val needsRationale =
missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) }
if (needsRationale) {
val proceed = showRationaleDialog(missing)
if (!proceed) {
return permissions.associateWith { perm ->
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
}
}
}
val deferred = CompletableDeferred<Map<String, Boolean>>()
pending = deferred
withContext(Dispatchers.Main) {
launcher.launch(missing.toTypedArray())
}
val result =
withContext(Dispatchers.Default) {
kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() }
}
// Merge: if something was already granted, treat it as granted even if launcher omitted it.
val merged =
permissions.associateWith { perm ->
val nowGranted =
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
result[perm] == true || nowGranted
}
val denied =
merged.filterValues { !it }.keys.filter {
!ActivityCompat.shouldShowRequestPermissionRationale(activity, it)
}
if (denied.isNotEmpty()) {
showSettingsDialog(denied)
}
return merged
}
private suspend fun showRationaleDialog(permissions: List<String>): Boolean =
withContext(Dispatchers.Main) {
suspendCancellableCoroutine { cont ->
AlertDialog.Builder(activity)
.setTitle("Permission required")
.setMessage(buildRationaleMessage(permissions))
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
.setOnCancelListener { cont.resume(false) }
.show()
}
}
private fun showSettingsDialog(permissions: List<String>) {
AlertDialog.Builder(activity)
.setTitle("Enable permission in Settings")
.setMessage(buildSettingsMessage(permissions))
.setPositiveButton("Open Settings") { _, _ ->
val intent =
Intent(
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
Uri.fromParts("package", activity.packageName, null),
)
activity.startActivity(intent)
}
.setNegativeButton("Cancel", null)
.show()
}
private fun buildRationaleMessage(permissions: List<String>): String {
val labels = permissions.map { permissionLabel(it) }
return "Clawdis needs ${labels.joinToString(", ")} to capture camera media."
}
private fun buildSettingsMessage(permissions: List<String>): String {
val labels = permissions.map { permissionLabel(it) }
return "Please enable ${labels.joinToString(", ")} in Android Settings to continue."
}
private fun permissionLabel(permission: String): String =
when (permission) {
Manifest.permission.CAMERA -> "Camera"
Manifest.permission.RECORD_AUDIO -> "Microphone"
else -> permission
}
}
@@ -1,65 +0,0 @@
package com.steipete.clawdis.node
import android.app.Activity
import android.content.Context
import android.content.Intent
import android.media.projection.MediaProjectionManager
import androidx.activity.ComponentActivity
import androidx.activity.result.ActivityResultLauncher
import androidx.activity.result.contract.ActivityResultContracts
import androidx.appcompat.app.AlertDialog
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeout
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlin.coroutines.resume
class ScreenCaptureRequester(private val activity: ComponentActivity) {
data class CaptureResult(val resultCode: Int, val data: Intent)
private val mutex = Mutex()
private var pending: CompletableDeferred<CaptureResult?>? = null
private val launcher: ActivityResultLauncher<Intent> =
activity.registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
val p = pending
pending = null
val data = result.data
if (result.resultCode == Activity.RESULT_OK && data != null) {
p?.complete(CaptureResult(result.resultCode, data))
} else {
p?.complete(null)
}
}
suspend fun requestCapture(timeoutMs: Long = 20_000): CaptureResult? =
mutex.withLock {
val proceed = showRationaleDialog()
if (!proceed) return null
val mgr = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
val intent = mgr.createScreenCaptureIntent()
val deferred = CompletableDeferred<CaptureResult?>()
pending = deferred
withContext(Dispatchers.Main) { launcher.launch(intent) }
withContext(Dispatchers.Default) { withTimeout(timeoutMs) { deferred.await() } }
}
private suspend fun showRationaleDialog(): Boolean =
withContext(Dispatchers.Main) {
suspendCancellableCoroutine { cont ->
AlertDialog.Builder(activity)
.setTitle("Screen recording required")
.setMessage("Clawdis needs to record the screen for this command.")
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
.setOnCancelListener { cont.resume(false) }
.show()
}
}
}
@@ -1,28 +1,13 @@
@file:Suppress("DEPRECATION")
package com.steipete.clawdis.node
import android.content.Context
import androidx.core.content.edit
import androidx.security.crypto.EncryptedSharedPreferences
import androidx.security.crypto.MasterKey
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonPrimitive
import java.util.UUID
class SecurePrefs(context: Context) {
companion object {
val defaultWakeWords: List<String> = listOf("clawd", "claude")
private const val displayNameKey = "node.displayName"
private const val voiceWakeModeKey = "voiceWake.mode"
}
private val json = Json { ignoreUnknownKeys = true }
private val masterKey =
MasterKey.Builder(context)
.setKeyScheme(MasterKey.KeyScheme.AES256_GCM)
@@ -40,16 +25,12 @@ class SecurePrefs(context: Context) {
private val _instanceId = MutableStateFlow(loadOrCreateInstanceId())
val instanceId: StateFlow<String> = _instanceId
private val _displayName =
MutableStateFlow(loadOrMigrateDisplayName(context = context))
private val _displayName = MutableStateFlow(prefs.getString("node.displayName", "Android Node")!!)
val displayName: StateFlow<String> = _displayName
private val _cameraEnabled = MutableStateFlow(prefs.getBoolean("camera.enabled", true))
val cameraEnabled: StateFlow<Boolean> = _cameraEnabled
private val _preventSleep = MutableStateFlow(prefs.getBoolean("screen.preventSleep", true))
val preventSleep: StateFlow<Boolean> = _preventSleep
private val _manualEnabled = MutableStateFlow(prefs.getBoolean("bridge.manual.enabled", false))
val manualEnabled: StateFlow<Boolean> = _manualEnabled
@@ -63,59 +44,39 @@ class SecurePrefs(context: Context) {
MutableStateFlow(prefs.getString("bridge.lastDiscoveredStableId", "")!!)
val lastDiscoveredStableId: StateFlow<String> = _lastDiscoveredStableId
private val _canvasDebugStatusEnabled =
MutableStateFlow(prefs.getBoolean("canvas.debugStatusEnabled", false))
val canvasDebugStatusEnabled: StateFlow<Boolean> = _canvasDebugStatusEnabled
private val _wakeWords = MutableStateFlow(loadWakeWords())
val wakeWords: StateFlow<List<String>> = _wakeWords
private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode())
val voiceWakeMode: StateFlow<VoiceWakeMode> = _voiceWakeMode
fun setLastDiscoveredStableId(value: String) {
val trimmed = value.trim()
prefs.edit { putString("bridge.lastDiscoveredStableId", trimmed) }
prefs.edit().putString("bridge.lastDiscoveredStableId", trimmed).apply()
_lastDiscoveredStableId.value = trimmed
}
fun setDisplayName(value: String) {
val trimmed = value.trim()
prefs.edit { putString(displayNameKey, trimmed) }
prefs.edit().putString("node.displayName", trimmed).apply()
_displayName.value = trimmed
}
fun setCameraEnabled(value: Boolean) {
prefs.edit { putBoolean("camera.enabled", value) }
prefs.edit().putBoolean("camera.enabled", value).apply()
_cameraEnabled.value = value
}
fun setPreventSleep(value: Boolean) {
prefs.edit { putBoolean("screen.preventSleep", value) }
_preventSleep.value = value
}
fun setManualEnabled(value: Boolean) {
prefs.edit { putBoolean("bridge.manual.enabled", value) }
prefs.edit().putBoolean("bridge.manual.enabled", value).apply()
_manualEnabled.value = value
}
fun setManualHost(value: String) {
val trimmed = value.trim()
prefs.edit { putString("bridge.manual.host", trimmed) }
prefs.edit().putString("bridge.manual.host", trimmed).apply()
_manualHost.value = trimmed
}
fun setManualPort(value: Int) {
prefs.edit { putInt("bridge.manual.port", value) }
prefs.edit().putInt("bridge.manual.port", value).apply()
_manualPort.value = value
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
prefs.edit { putBoolean("canvas.debugStatusEnabled", value) }
_canvasDebugStatusEnabled.value = value
}
fun loadBridgeToken(): String? {
val key = "bridge.token.${_instanceId.value}"
return prefs.getString(key, null)
@@ -123,70 +84,14 @@ class SecurePrefs(context: Context) {
fun saveBridgeToken(token: String) {
val key = "bridge.token.${_instanceId.value}"
prefs.edit { putString(key, token.trim()) }
prefs.edit().putString(key, token.trim()).apply()
}
private fun loadOrCreateInstanceId(): String {
val existing = prefs.getString("node.instanceId", null)?.trim()
if (!existing.isNullOrBlank()) return existing
val fresh = UUID.randomUUID().toString()
prefs.edit { putString("node.instanceId", fresh) }
prefs.edit().putString("node.instanceId", fresh).apply()
return fresh
}
private fun loadOrMigrateDisplayName(context: Context): String {
val existing = prefs.getString(displayNameKey, null)?.trim().orEmpty()
if (existing.isNotEmpty() && existing != "Android Node") return existing
val candidate = DeviceNames.bestDefaultNodeName(context).trim()
val resolved = candidate.ifEmpty { "Android Node" }
prefs.edit { putString(displayNameKey, resolved) }
return resolved
}
fun setWakeWords(words: List<String>) {
val sanitized = WakeWords.sanitize(words, defaultWakeWords)
val encoded =
JsonArray(sanitized.map { JsonPrimitive(it) }).toString()
prefs.edit { putString("voiceWake.triggerWords", encoded) }
_wakeWords.value = sanitized
}
fun setVoiceWakeMode(mode: VoiceWakeMode) {
prefs.edit { putString(voiceWakeModeKey, mode.rawValue) }
_voiceWakeMode.value = mode
}
private fun loadVoiceWakeMode(): VoiceWakeMode {
val raw = prefs.getString(voiceWakeModeKey, null)
val resolved = VoiceWakeMode.fromRawValue(raw)
// Default ON (foreground) when unset.
if (raw.isNullOrBlank()) {
prefs.edit { putString(voiceWakeModeKey, resolved.rawValue) }
}
return resolved
}
private fun loadWakeWords(): List<String> {
val raw = prefs.getString("voiceWake.triggerWords", null)?.trim()
if (raw.isNullOrEmpty()) return defaultWakeWords
return try {
val element = json.parseToJsonElement(raw)
val array = element as? JsonArray ?: return defaultWakeWords
val decoded =
array.mapNotNull { item ->
when (item) {
is JsonNull -> null
is JsonPrimitive -> item.content.trim().takeIf { it.isNotEmpty() }
else -> null
}
}
WakeWords.sanitize(decoded, defaultWakeWords)
} catch (_: Throwable) {
defaultWakeWords
}
}
}
@@ -1,15 +0,0 @@
package com.steipete.clawdis.node
enum class VoiceWakeMode(val rawValue: String) {
Off("off"),
Foreground("foreground"),
Always("always"),
;
companion object {
fun fromRawValue(raw: String?): VoiceWakeMode {
return entries.firstOrNull { it.rawValue == raw?.trim()?.lowercase() } ?: Foreground
}
}
}
@@ -1,17 +0,0 @@
package com.steipete.clawdis.node
object WakeWords {
const val maxWords: Int = 32
const val maxWordLength: Int = 64
fun parseCommaSeparated(input: String): List<String> {
return input.split(",").map { it.trim() }.filter { it.isNotEmpty() }
}
fun sanitize(words: List<String>, defaults: List<String>): List<String> {
val cleaned =
words.map { it.trim() }.filter { it.isNotEmpty() }.take(maxWords).map { it.take(maxWordLength) }
return cleaned.ifEmpty { defaults }
}
}
@@ -1,35 +0,0 @@
package com.steipete.clawdis.node.bridge
object BonjourEscapes {
fun decode(input: String): String {
if (input.isEmpty()) return input
val bytes = mutableListOf<Byte>()
var i = 0
while (i < input.length) {
if (input[i] == '\\' && i + 3 < input.length) {
val d0 = input[i + 1]
val d1 = input[i + 2]
val d2 = input[i + 3]
if (d0.isDigit() && d1.isDigit() && d2.isDigit()) {
val value =
((d0.code - '0'.code) * 100) + ((d1.code - '0'.code) * 10) + (d2.code - '0'.code)
if (value in 0..255) {
bytes.add(value.toByte())
i += 4
continue
}
}
}
val codePoint = Character.codePointAt(input, i)
val charBytes = String(Character.toChars(codePoint)).toByteArray(Charsets.UTF_8)
for (b in charBytes) {
bytes.add(b)
}
i += Character.charCount(codePoint)
}
return String(bytes.toByteArray(), Charsets.UTF_8)
}
}
@@ -1,74 +1,22 @@
package com.steipete.clawdis.node.bridge
import android.content.Context
import android.net.ConnectivityManager
import android.net.DnsResolver
import android.net.NetworkCapabilities
import android.net.nsd.NsdManager
import android.net.nsd.NsdServiceInfo
import android.os.CancellationSignal
import android.util.Log
import java.io.IOException
import java.net.InetSocketAddress
import java.nio.ByteBuffer
import java.nio.charset.CodingErrorAction
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Executor
import java.util.concurrent.Executors
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import android.os.Build
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import org.xbill.DNS.AAAARecord
import org.xbill.DNS.ARecord
import org.xbill.DNS.DClass
import org.xbill.DNS.ExtendedResolver
import org.xbill.DNS.Message
import org.xbill.DNS.Name
import org.xbill.DNS.PTRRecord
import org.xbill.DNS.Record
import org.xbill.DNS.Rcode
import org.xbill.DNS.Resolver
import org.xbill.DNS.SRVRecord
import org.xbill.DNS.Section
import org.xbill.DNS.SimpleResolver
import org.xbill.DNS.TextParseException
import org.xbill.DNS.TXTRecord
import org.xbill.DNS.Type
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
import java.util.concurrent.ConcurrentHashMap
@Suppress("DEPRECATION")
class BridgeDiscovery(
context: Context,
private val scope: CoroutineScope,
) {
class BridgeDiscovery(context: Context) {
private val nsd = context.getSystemService(NsdManager::class.java)
private val connectivity = context.getSystemService(ConnectivityManager::class.java)
private val dns = DnsResolver.getInstance()
private val serviceType = "_clawdis-bridge._tcp."
private val wideAreaDomain = "clawdis.internal."
private val logTag = "Clawdis/BridgeDiscovery"
private val localById = ConcurrentHashMap<String, BridgeEndpoint>()
private val unicastById = ConcurrentHashMap<String, BridgeEndpoint>()
private val byId = ConcurrentHashMap<String, BridgeEndpoint>()
private val _bridges = MutableStateFlow<List<BridgeEndpoint>>(emptyList())
val bridges: StateFlow<List<BridgeEndpoint>> = _bridges.asStateFlow()
private val _statusText = MutableStateFlow("Searching…")
val statusText: StateFlow<String> = _statusText.asStateFlow()
private var unicastJob: Job? = null
private val dnsExecutor: Executor = Executors.newCachedThreadPool()
@Volatile private var lastWideAreaRcode: Int? = null
@Volatile private var lastWideAreaCount: Int = 0
private val discoveryListener =
object : NsdManager.DiscoveryListener {
override fun onStartDiscoveryFailed(serviceType: String, errorCode: Int) {}
@@ -82,19 +30,13 @@ class BridgeDiscovery(
}
override fun onServiceLost(serviceInfo: NsdServiceInfo) {
val serviceName = BonjourEscapes.decode(serviceInfo.serviceName)
val id = stableId(serviceName, "local.")
localById.remove(id)
val id = stableId(serviceInfo)
byId.remove(id)
publish()
}
}
init {
startLocalDiscovery()
startUnicastDiscovery(wideAreaDomain)
}
private fun startLocalDiscovery() {
try {
nsd.discoverServices(serviceType, NsdManager.PROTOCOL_DNS_SD, discoveryListener)
} catch (_: Throwable) {
@@ -102,28 +44,6 @@ class BridgeDiscovery(
}
}
private fun stopLocalDiscovery() {
try {
nsd.stopServiceDiscovery(discoveryListener)
} catch (_: Throwable) {
// ignore (best-effort)
}
}
private fun startUnicastDiscovery(domain: String) {
unicastJob =
scope.launch(Dispatchers.IO) {
while (true) {
try {
refreshUnicast(domain)
} catch (_: Throwable) {
// ignore (best-effort)
}
delay(5000)
}
}
}
private fun resolve(serviceInfo: NsdServiceInfo) {
nsd.resolveService(
serviceInfo,
@@ -135,11 +55,9 @@ class BridgeDiscovery(
val port = resolved.port
if (port <= 0) return
val rawServiceName = resolved.serviceName
val serviceName = BonjourEscapes.decode(rawServiceName)
val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName)
val id = stableId(serviceName, "local.")
localById[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port)
val displayName = txt(resolved, "displayName") ?: resolved.serviceName
val id = stableId(resolved)
byId[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port)
publish()
}
},
@@ -147,33 +65,11 @@ class BridgeDiscovery(
}
private fun publish() {
_bridges.value =
(localById.values + unicastById.values).sortedBy { it.name.lowercase() }
_statusText.value = buildStatusText()
_bridges.value = byId.values.sortedBy { it.name.lowercase() }
}
private fun buildStatusText(): String {
val localCount = localById.size
val wideRcode = lastWideAreaRcode
val wideCount = lastWideAreaCount
val wide =
when (wideRcode) {
null -> "Wide: ?"
Rcode.NOERROR -> "Wide: $wideCount"
Rcode.NXDOMAIN -> "Wide: NXDOMAIN"
else -> "Wide: ${Rcode.string(wideRcode)}"
}
return when {
localCount == 0 && wideRcode == null -> "Searching for bridges…"
localCount == 0 -> "$wide"
else -> "Local: $localCount • $wide"
}
}
private fun stableId(serviceName: String, domain: String): String {
return "${serviceType}|${domain}|${normalizeName(serviceName)}"
private fun stableId(info: NsdServiceInfo): String {
return "${info.serviceType}|local.|${normalizeName(info.serviceName)}"
}
private fun normalizeName(raw: String): String {
@@ -181,6 +77,7 @@ class BridgeDiscovery(
}
private fun txt(info: NsdServiceInfo, key: String): String? {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) return null
val bytes = info.attributes[key] ?: return null
return try {
String(bytes, Charsets.UTF_8).trim().ifEmpty { null }
@@ -188,278 +85,4 @@ class BridgeDiscovery(
null
}
}
private suspend fun refreshUnicast(domain: String) {
val ptrName = "${serviceType}${domain}"
val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return
val ptrRecords = records(ptrMsg, Section.ANSWER).mapNotNull { it as? PTRRecord }
val next = LinkedHashMap<String, BridgeEndpoint>()
for (ptr in ptrRecords) {
val instanceFqdn = ptr.target.toString()
val srv =
recordByName(ptrMsg, instanceFqdn, Type.SRV) as? SRVRecord
?: run {
val msg = lookupUnicastMessage(instanceFqdn, Type.SRV) ?: return@run null
recordByName(msg, instanceFqdn, Type.SRV) as? SRVRecord
}
?: continue
val port = srv.port
if (port <= 0) continue
val targetFqdn = srv.target.toString()
val host =
resolveHostFromMessage(ptrMsg, targetFqdn)
?: resolveHostFromMessage(lookupUnicastMessage(instanceFqdn, Type.SRV), targetFqdn)
?: resolveHostUnicast(targetFqdn)
?: continue
val txtFromPtr =
recordsByName(ptrMsg, Section.ADDITIONAL)[keyName(instanceFqdn)]
.orEmpty()
.mapNotNull { it as? TXTRecord }
val txt =
if (txtFromPtr.isNotEmpty()) {
txtFromPtr
} else {
val msg = lookupUnicastMessage(instanceFqdn, Type.TXT)
records(msg, Section.ANSWER).mapNotNull { it as? TXTRecord }
}
val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain))
val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName)
val id = stableId(instanceName, domain)
next[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port)
}
unicastById.clear()
unicastById.putAll(next)
lastWideAreaRcode = ptrMsg.header.rcode
lastWideAreaCount = next.size
publish()
if (next.isEmpty()) {
Log.d(
logTag,
"wide-area discovery: 0 results for $ptrName (rcode=${Rcode.string(ptrMsg.header.rcode)})",
)
}
}
private fun decodeInstanceName(instanceFqdn: String, domain: String): String {
val suffix = "${serviceType}${domain}"
val withoutSuffix =
if (instanceFqdn.endsWith(suffix)) {
instanceFqdn.removeSuffix(suffix)
} else {
instanceFqdn.substringBefore(serviceType)
}
return normalizeName(stripTrailingDot(withoutSuffix))
}
private fun stripTrailingDot(raw: String): String {
return raw.removeSuffix(".")
}
private suspend fun lookupUnicastMessage(name: String, type: Int): Message? {
val query =
try {
Message.newQuery(
org.xbill.DNS.Record.newRecord(
Name.fromString(name),
type,
DClass.IN,
),
)
} catch (_: TextParseException) {
return null
}
val system = queryViaSystemDns(query)
if (records(system, Section.ANSWER).any { it.type == type }) return system
val direct = createDirectResolver() ?: return system
return try {
val msg = direct.send(query)
if (records(msg, Section.ANSWER).any { it.type == type }) msg else system
} catch (_: Throwable) {
system
}
}
private suspend fun queryViaSystemDns(query: Message): Message? {
val network = preferredDnsNetwork()
val bytes =
try {
rawQuery(network, query.toWire())
} catch (_: Throwable) {
return null
}
return try {
Message(bytes)
} catch (_: IOException) {
null
}
}
private fun records(msg: Message?, section: Int): List<Record> {
return msg?.getSectionArray(section)?.toList() ?: emptyList()
}
private fun keyName(raw: String): String {
return raw.trim().lowercase()
}
private fun recordsByName(msg: Message, section: Int): Map<String, List<Record>> {
val next = LinkedHashMap<String, MutableList<Record>>()
for (r in records(msg, section)) {
val name = r.name?.toString() ?: continue
next.getOrPut(keyName(name)) { mutableListOf() }.add(r)
}
return next
}
private fun recordByName(msg: Message, fqdn: String, type: Int): Record? {
val key = keyName(fqdn)
val byNameAnswer = recordsByName(msg, Section.ANSWER)
val fromAnswer = byNameAnswer[key].orEmpty().firstOrNull { it.type == type }
if (fromAnswer != null) return fromAnswer
val byNameAdditional = recordsByName(msg, Section.ADDITIONAL)
return byNameAdditional[key].orEmpty().firstOrNull { it.type == type }
}
private fun resolveHostFromMessage(msg: Message?, hostname: String): String? {
val m = msg ?: return null
val key = keyName(hostname)
val additional = recordsByName(m, Section.ADDITIONAL)[key].orEmpty()
val a = additional.mapNotNull { it as? ARecord }.mapNotNull { it.address?.hostAddress }
val aaaa = additional.mapNotNull { it as? AAAARecord }.mapNotNull { it.address?.hostAddress }
return a.firstOrNull() ?: aaaa.firstOrNull()
}
private fun preferredDnsNetwork(): android.net.Network? {
val cm = connectivity ?: return null
// Prefer VPN (Tailscale) when present; otherwise use the active network.
cm.allNetworks.firstOrNull { n ->
val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false
caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN)
}?.let { return it }
return cm.activeNetwork
}
private fun createDirectResolver(): Resolver? {
val cm = connectivity ?: return null
val candidateNetworks =
buildList {
cm.allNetworks
.firstOrNull { n ->
val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false
caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN)
}?.let(::add)
cm.activeNetwork?.let(::add)
}.distinct()
val servers =
candidateNetworks
.asSequence()
.flatMap { n ->
cm.getLinkProperties(n)?.dnsServers?.asSequence() ?: emptySequence()
}
.distinctBy { it.hostAddress ?: it.toString() }
.toList()
if (servers.isEmpty()) return null
return try {
val resolvers =
servers.mapNotNull { addr ->
try {
SimpleResolver().apply {
setAddress(InetSocketAddress(addr, 53))
setTimeout(3)
}
} catch (_: Throwable) {
null
}
}
if (resolvers.isEmpty()) return null
ExtendedResolver(resolvers.toTypedArray()).apply { setTimeout(3) }
} catch (_: Throwable) {
null
}
}
private suspend fun rawQuery(network: android.net.Network?, wireQuery: ByteArray): ByteArray =
suspendCancellableCoroutine { cont ->
val signal = CancellationSignal()
cont.invokeOnCancellation { signal.cancel() }
dns.rawQuery(
network,
wireQuery,
DnsResolver.FLAG_EMPTY,
dnsExecutor,
signal,
object : DnsResolver.Callback<ByteArray> {
override fun onAnswer(answer: ByteArray, rcode: Int) {
cont.resume(answer)
}
override fun onError(error: DnsResolver.DnsException) {
cont.resumeWithException(error)
}
},
)
}
private fun txtValue(records: List<TXTRecord>, key: String): String? {
val prefix = "$key="
for (r in records) {
val strings: List<String> =
try {
r.strings.mapNotNull { it as? String }
} catch (_: Throwable) {
emptyList()
}
for (s in strings) {
val trimmed = decodeDnsTxtString(s).trim()
if (trimmed.startsWith(prefix)) {
return trimmed.removePrefix(prefix).trim().ifEmpty { null }
}
}
}
return null
}
private fun decodeDnsTxtString(raw: String): String {
// dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes.
// Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible.
val bytes = raw.toByteArray(Charsets.ISO_8859_1)
val decoder =
Charsets.UTF_8
.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT)
return try {
decoder.decode(ByteBuffer.wrap(bytes)).toString()
} catch (_: Throwable) {
raw
}
}
private suspend fun resolveHostUnicast(hostname: String): String? {
val a =
records(lookupUnicastMessage(hostname, Type.A), Section.ANSWER)
.mapNotNull { it as? ARecord }
.mapNotNull { it.address?.hostAddress }
val aaaa =
records(lookupUnicastMessage(hostname, Type.AAAA), Section.ANSWER)
.mapNotNull { it as? AAAARecord }
.mapNotNull { it.address?.hostAddress }
return a.firstOrNull() ?: aaaa.firstOrNull()
}
}
@@ -3,7 +3,6 @@ package com.steipete.clawdis.node.bridge
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.JsonNull
@@ -25,10 +24,6 @@ class BridgePairingClient {
val token: String?,
val platform: String?,
val version: String?,
val deviceFamily: String?,
val modelIdentifier: String?,
val caps: List<String>?,
val commands: List<String>?,
)
data class PairResult(val ok: Boolean, val token: String?, val error: String? = null)
@@ -60,10 +55,6 @@ class BridgePairingClient {
hello.token?.let { put("token", JsonPrimitive(it)) }
hello.platform?.let { put("platform", JsonPrimitive(it)) }
hello.version?.let { put("version", JsonPrimitive(it)) }
hello.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
hello.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
hello.caps?.let { put("caps", JsonArray(it.map(::JsonPrimitive))) }
hello.commands?.let { put("commands", JsonArray(it.map(::JsonPrimitive))) }
},
)
@@ -85,10 +76,6 @@ class BridgePairingClient {
hello.displayName?.let { put("displayName", JsonPrimitive(it)) }
hello.platform?.let { put("platform", JsonPrimitive(it)) }
hello.version?.let { put("version", JsonPrimitive(it)) }
hello.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
hello.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
hello.caps?.let { put("caps", JsonArray(it.map(::JsonPrimitive))) }
hello.commands?.let { put("commands", JsonArray(it.map(::JsonPrimitive))) }
},
)
@@ -12,7 +12,6 @@ import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonElement
@@ -40,10 +39,6 @@ class BridgeSession(
val token: String?,
val platform: String?,
val version: String?,
val deviceFamily: String?,
val modelIdentifier: String?,
val caps: List<String>?,
val commands: List<String>?,
)
data class InvokeRequest(val id: String, val command: String, val paramsJson: String?)
@@ -61,7 +56,6 @@ class BridgeSession(
private val json = Json { ignoreUnknownKeys = true }
private val writeLock = Mutex()
private val pending = ConcurrentHashMap<String, CompletableDeferred<RpcResponse>>()
@Volatile private var canvasHostUrl: String? = null
private var desired: Pair<BridgeEndpoint, Hello>? = null
private var job: Job? = null
@@ -78,13 +72,10 @@ class BridgeSession(
scope.launch(Dispatchers.IO) {
job?.cancelAndJoin()
job = null
canvasHostUrl = null
onDisconnected("Offline")
onDisconnected("Disconnected")
}
}
fun currentCanvasHostUrl(): String? = canvasHostUrl
suspend fun sendEvent(event: String, payloadJson: String?) {
val conn = currentConnection ?: return
conn.sendJson(
@@ -200,10 +191,6 @@ class BridgeSession(
hello.token?.let { put("token", JsonPrimitive(it)) }
hello.platform?.let { put("platform", JsonPrimitive(it)) }
hello.version?.let { put("version", JsonPrimitive(it)) }
hello.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
hello.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
hello.caps?.let { put("caps", JsonArray(it.map(::JsonPrimitive))) }
hello.commands?.let { put("commands", JsonArray(it.map(::JsonPrimitive))) }
},
)
@@ -213,7 +200,6 @@ class BridgeSession(
when (first["type"].asStringOrNull()) {
"hello-ok" -> {
val name = first["serverName"].asStringOrNull() ?: "Bridge"
canvasHostUrl = first["canvasHostUrl"].asStringOrNull()?.trim()?.ifEmpty { null }
onConnected(name, conn.remoteAddress)
}
"error" -> {
@@ -1,509 +0,0 @@
package com.steipete.clawdis.node.chat
import com.steipete.clawdis.node.bridge.BridgeSession
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
class ChatController(
private val scope: CoroutineScope,
private val session: BridgeSession,
private val json: Json,
) {
private val _sessionKey = MutableStateFlow("main")
val sessionKey: StateFlow<String> = _sessionKey.asStateFlow()
private val _sessionId = MutableStateFlow<String?>(null)
val sessionId: StateFlow<String?> = _sessionId.asStateFlow()
private val _messages = MutableStateFlow<List<ChatMessage>>(emptyList())
val messages: StateFlow<List<ChatMessage>> = _messages.asStateFlow()
private val _errorText = MutableStateFlow<String?>(null)
val errorText: StateFlow<String?> = _errorText.asStateFlow()
private val _healthOk = MutableStateFlow(false)
val healthOk: StateFlow<Boolean> = _healthOk.asStateFlow()
private val _thinkingLevel = MutableStateFlow("off")
val thinkingLevel: StateFlow<String> = _thinkingLevel.asStateFlow()
private val _pendingRunCount = MutableStateFlow(0)
val pendingRunCount: StateFlow<Int> = _pendingRunCount.asStateFlow()
private val _streamingAssistantText = MutableStateFlow<String?>(null)
val streamingAssistantText: StateFlow<String?> = _streamingAssistantText.asStateFlow()
private val pendingToolCallsById = ConcurrentHashMap<String, ChatPendingToolCall>()
private val _pendingToolCalls = MutableStateFlow<List<ChatPendingToolCall>>(emptyList())
val pendingToolCalls: StateFlow<List<ChatPendingToolCall>> = _pendingToolCalls.asStateFlow()
private val _sessions = MutableStateFlow<List<ChatSessionEntry>>(emptyList())
val sessions: StateFlow<List<ChatSessionEntry>> = _sessions.asStateFlow()
private val pendingRuns = mutableSetOf<String>()
private val pendingRunTimeoutJobs = ConcurrentHashMap<String, Job>()
private val pendingRunTimeoutMs = 120_000L
private var lastHealthPollAtMs: Long? = null
fun onDisconnected(message: String) {
_healthOk.value = false
// Not an error; keep connection status in the UI pill.
_errorText.value = null
clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
_sessionId.value = null
}
fun load(sessionKey: String = "main") {
val key = sessionKey.trim().ifEmpty { "main" }
_sessionKey.value = key
scope.launch { bootstrap(forceHealth = true) }
}
fun refresh() {
scope.launch { bootstrap(forceHealth = true) }
}
fun refreshSessions(limit: Int? = null) {
scope.launch { fetchSessions(limit = limit) }
}
fun setThinkingLevel(thinkingLevel: String) {
val normalized = normalizeThinking(thinkingLevel)
if (normalized == _thinkingLevel.value) return
_thinkingLevel.value = normalized
}
fun switchSession(sessionKey: String) {
val key = sessionKey.trim()
if (key.isEmpty()) return
if (key == _sessionKey.value) return
_sessionKey.value = key
scope.launch { bootstrap(forceHealth = true) }
}
fun sendMessage(
message: String,
thinkingLevel: String,
attachments: List<OutgoingAttachment>,
) {
val trimmed = message.trim()
if (trimmed.isEmpty() && attachments.isEmpty()) return
if (!_healthOk.value) {
_errorText.value = "Gateway health not OK; cannot send"
return
}
val runId = UUID.randomUUID().toString()
val text = if (trimmed.isEmpty() && attachments.isNotEmpty()) "See attached." else trimmed
val sessionKey = _sessionKey.value
val thinking = normalizeThinking(thinkingLevel)
// Optimistic user message.
val userContent =
buildList {
add(ChatMessageContent(type = "text", text = text))
for (att in attachments) {
add(
ChatMessageContent(
type = att.type,
mimeType = att.mimeType,
fileName = att.fileName,
base64 = att.base64,
),
)
}
}
_messages.value =
_messages.value +
ChatMessage(
id = UUID.randomUUID().toString(),
role = "user",
content = userContent,
timestampMs = System.currentTimeMillis(),
)
armPendingRunTimeout(runId)
synchronized(pendingRuns) {
pendingRuns.add(runId)
_pendingRunCount.value = pendingRuns.size
}
_errorText.value = null
_streamingAssistantText.value = null
pendingToolCallsById.clear()
publishPendingToolCalls()
scope.launch {
try {
val params =
buildJsonObject {
put("sessionKey", JsonPrimitive(sessionKey))
put("message", JsonPrimitive(text))
put("thinking", JsonPrimitive(thinking))
put("timeoutMs", JsonPrimitive(30_000))
put("idempotencyKey", JsonPrimitive(runId))
if (attachments.isNotEmpty()) {
put(
"attachments",
JsonArray(
attachments.map { att ->
buildJsonObject {
put("type", JsonPrimitive(att.type))
put("mimeType", JsonPrimitive(att.mimeType))
put("fileName", JsonPrimitive(att.fileName))
put("content", JsonPrimitive(att.base64))
}
},
),
)
}
}
val res = session.request("chat.send", params.toString())
val actualRunId = parseRunId(res) ?: runId
if (actualRunId != runId) {
clearPendingRun(runId)
armPendingRunTimeout(actualRunId)
synchronized(pendingRuns) {
pendingRuns.add(actualRunId)
_pendingRunCount.value = pendingRuns.size
}
}
} catch (err: Throwable) {
clearPendingRun(runId)
_errorText.value = err.message
}
}
}
fun abort() {
val runIds =
synchronized(pendingRuns) {
pendingRuns.toList()
}
if (runIds.isEmpty()) return
scope.launch {
for (runId in runIds) {
try {
val params =
buildJsonObject {
put("sessionKey", JsonPrimitive(_sessionKey.value))
put("runId", JsonPrimitive(runId))
}
session.request("chat.abort", params.toString())
} catch (_: Throwable) {
// best-effort
}
}
}
}
fun handleBridgeEvent(event: String, payloadJson: String?) {
when (event) {
"tick" -> {
scope.launch { pollHealthIfNeeded(force = false) }
}
"health" -> {
// If we receive a health snapshot, the gateway is reachable.
_healthOk.value = true
}
"seqGap" -> {
_errorText.value = "Event stream interrupted; try refreshing."
clearPendingRuns()
}
"chat" -> {
if (payloadJson.isNullOrBlank()) return
handleChatEvent(payloadJson)
}
"agent" -> {
if (payloadJson.isNullOrBlank()) return
handleAgentEvent(payloadJson)
}
}
}
private suspend fun bootstrap(forceHealth: Boolean) {
_errorText.value = null
_healthOk.value = false
clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
_sessionId.value = null
val key = _sessionKey.value
try {
try {
session.sendEvent("chat.subscribe", """{"sessionKey":"$key"}""")
} catch (_: Throwable) {
// best-effort
}
val historyJson = session.request("chat.history", """{"sessionKey":"$key"}""")
val history = parseHistory(historyJson, sessionKey = key)
_messages.value = history.messages
_sessionId.value = history.sessionId
history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it }
pollHealthIfNeeded(force = forceHealth)
fetchSessions(limit = 50)
} catch (err: Throwable) {
_errorText.value = err.message
}
}
private suspend fun fetchSessions(limit: Int?) {
try {
val params =
buildJsonObject {
put("includeGlobal", JsonPrimitive(true))
put("includeUnknown", JsonPrimitive(false))
if (limit != null && limit > 0) put("limit", JsonPrimitive(limit))
}
val res = session.request("sessions.list", params.toString())
_sessions.value = parseSessions(res)
} catch (_: Throwable) {
// best-effort
}
}
private suspend fun pollHealthIfNeeded(force: Boolean) {
val now = System.currentTimeMillis()
val last = lastHealthPollAtMs
if (!force && last != null && now - last < 10_000) return
lastHealthPollAtMs = now
try {
session.request("health", null)
_healthOk.value = true
} catch (_: Throwable) {
_healthOk.value = false
}
}
private fun handleChatEvent(payloadJson: String) {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val sessionKey = payload["sessionKey"].asStringOrNull()?.trim()
if (!sessionKey.isNullOrEmpty() && sessionKey != _sessionKey.value) return
val runId = payload["runId"].asStringOrNull()
if (runId != null) {
val isPending =
synchronized(pendingRuns) {
pendingRuns.contains(runId)
}
if (!isPending) return
}
val state = payload["state"].asStringOrNull()
when (state) {
"final", "aborted", "error" -> {
if (state == "error") {
_errorText.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed"
}
if (runId != null) clearPendingRun(runId) else clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
scope.launch {
try {
val historyJson =
session.request("chat.history", """{"sessionKey":"${_sessionKey.value}"}""")
val history = parseHistory(historyJson, sessionKey = _sessionKey.value)
_messages.value = history.messages
_sessionId.value = history.sessionId
history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it }
} catch (_: Throwable) {
// best-effort
}
}
}
}
}
private fun handleAgentEvent(payloadJson: String) {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val runId = payload["runId"].asStringOrNull()
val sessionId = _sessionId.value
if (sessionId != null && runId != sessionId) return
val stream = payload["stream"].asStringOrNull()
val data = payload["data"].asObjectOrNull()
when (stream) {
"assistant" -> {
val text = data?.get("text")?.asStringOrNull()
if (!text.isNullOrEmpty()) {
_streamingAssistantText.value = text
}
}
"tool" -> {
val phase = data?.get("phase")?.asStringOrNull()
val name = data?.get("name")?.asStringOrNull()
val toolCallId = data?.get("toolCallId")?.asStringOrNull()
if (phase.isNullOrEmpty() || name.isNullOrEmpty() || toolCallId.isNullOrEmpty()) return
val ts = payload["ts"].asLongOrNull() ?: System.currentTimeMillis()
if (phase == "start") {
pendingToolCallsById[toolCallId] =
ChatPendingToolCall(
toolCallId = toolCallId,
name = name,
startedAtMs = ts,
isError = null,
)
publishPendingToolCalls()
} else if (phase == "result") {
pendingToolCallsById.remove(toolCallId)
publishPendingToolCalls()
}
}
"error" -> {
_errorText.value = "Event stream interrupted; try refreshing."
clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
}
}
}
private fun publishPendingToolCalls() {
_pendingToolCalls.value =
pendingToolCallsById.values.sortedBy { it.startedAtMs }
}
private fun armPendingRunTimeout(runId: String) {
pendingRunTimeoutJobs[runId]?.cancel()
pendingRunTimeoutJobs[runId] =
scope.launch {
delay(pendingRunTimeoutMs)
val stillPending =
synchronized(pendingRuns) {
pendingRuns.contains(runId)
}
if (!stillPending) return@launch
clearPendingRun(runId)
_errorText.value = "Timed out waiting for a reply; try again or refresh."
}
}
private fun clearPendingRun(runId: String) {
pendingRunTimeoutJobs.remove(runId)?.cancel()
synchronized(pendingRuns) {
pendingRuns.remove(runId)
_pendingRunCount.value = pendingRuns.size
}
}
private fun clearPendingRuns() {
for ((_, job) in pendingRunTimeoutJobs) {
job.cancel()
}
pendingRunTimeoutJobs.clear()
synchronized(pendingRuns) {
pendingRuns.clear()
_pendingRunCount.value = 0
}
}
private fun parseHistory(historyJson: String, sessionKey: String): ChatHistory {
val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return ChatHistory(sessionKey, null, null, emptyList())
val sid = root["sessionId"].asStringOrNull()
val thinkingLevel = root["thinkingLevel"].asStringOrNull()
val array = root["messages"].asArrayOrNull() ?: JsonArray(emptyList())
val messages =
array.mapNotNull { item ->
val obj = item.asObjectOrNull() ?: return@mapNotNull null
val role = obj["role"].asStringOrNull() ?: return@mapNotNull null
val content = obj["content"].asArrayOrNull()?.mapNotNull(::parseMessageContent) ?: emptyList()
val ts = obj["timestamp"].asLongOrNull()
ChatMessage(
id = UUID.randomUUID().toString(),
role = role,
content = content,
timestampMs = ts,
)
}
return ChatHistory(sessionKey = sessionKey, sessionId = sid, thinkingLevel = thinkingLevel, messages = messages)
}
private fun parseMessageContent(el: JsonElement): ChatMessageContent? {
val obj = el.asObjectOrNull() ?: return null
val type = obj["type"].asStringOrNull() ?: "text"
return if (type == "text") {
ChatMessageContent(type = "text", text = obj["text"].asStringOrNull())
} else {
ChatMessageContent(
type = type,
mimeType = obj["mimeType"].asStringOrNull(),
fileName = obj["fileName"].asStringOrNull(),
base64 = obj["content"].asStringOrNull(),
)
}
}
private fun parseSessions(jsonString: String): List<ChatSessionEntry> {
val root = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return emptyList()
val sessions = root["sessions"].asArrayOrNull() ?: return emptyList()
return sessions.mapNotNull { item ->
val obj = item.asObjectOrNull() ?: return@mapNotNull null
val key = obj["key"].asStringOrNull()?.trim().orEmpty()
if (key.isEmpty()) return@mapNotNull null
val updatedAt = obj["updatedAt"].asLongOrNull()
ChatSessionEntry(key = key, updatedAtMs = updatedAt)
}
}
private fun parseRunId(resJson: String): String? {
return try {
json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull()
} catch (_: Throwable) {
null
}
}
private fun normalizeThinking(raw: String): String {
return when (raw.trim().lowercase()) {
"low" -> "low"
"medium" -> "medium"
"high" -> "high"
else -> "off"
}
}
}
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
private fun JsonElement?.asArrayOrNull(): JsonArray? = this as? JsonArray
private fun JsonElement?.asStringOrNull(): String? =
when (this) {
is JsonNull -> null
is JsonPrimitive -> content
else -> null
}
private fun JsonElement?.asLongOrNull(): Long? =
when (this) {
is JsonPrimitive -> content.toLongOrNull()
else -> null
}
@@ -1,42 +0,0 @@
package com.steipete.clawdis.node.chat
data class ChatMessage(
val id: String,
val role: String,
val content: List<ChatMessageContent>,
val timestampMs: Long?,
)
data class ChatMessageContent(
val type: String = "text",
val text: String? = null,
val mimeType: String? = null,
val fileName: String? = null,
val base64: String? = null,
)
data class ChatPendingToolCall(
val toolCallId: String,
val name: String,
val startedAtMs: Long,
val isError: Boolean? = null,
)
data class ChatSessionEntry(
val key: String,
val updatedAtMs: Long?,
)
data class ChatHistory(
val sessionKey: String,
val sessionId: String?,
val thinkingLevel: String?,
val messages: List<ChatMessage>,
)
data class OutgoingAttachment(
val type: String,
val mimeType: String,
val fileName: String,
val base64: String,
)
@@ -2,7 +2,6 @@ package com.steipete.clawdis.node.node
import android.Manifest
import android.content.Context
import android.annotation.SuppressLint
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.util.Base64
@@ -19,8 +18,6 @@ import androidx.camera.video.VideoCapture
import androidx.camera.video.VideoRecordEvent
import androidx.core.content.ContextCompat
import androidx.core.content.ContextCompat.checkSelfPermission
import androidx.core.graphics.scale
import com.steipete.clawdis.node.PermissionRequester
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withTimeout
@@ -35,43 +32,24 @@ class CameraCaptureManager(private val context: Context) {
data class Payload(val payloadJson: String)
@Volatile private var lifecycleOwner: LifecycleOwner? = null
@Volatile private var permissionRequester: PermissionRequester? = null
fun attachLifecycleOwner(owner: LifecycleOwner) {
lifecycleOwner = owner
}
fun attachPermissionRequester(requester: PermissionRequester) {
permissionRequester = requester
}
private suspend fun ensureCameraPermission() {
private fun requireCameraPermission() {
val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED
if (granted) return
val requester = permissionRequester
?: throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
val results = requester.requestIfMissing(listOf(Manifest.permission.CAMERA))
if (results[Manifest.permission.CAMERA] != true) {
throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
}
if (!granted) throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
}
private suspend fun ensureMicPermission() {
private fun requireMicPermission() {
val granted = checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
if (granted) return
val requester = permissionRequester
?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
val results = requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO))
if (results[Manifest.permission.RECORD_AUDIO] != true) {
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
}
if (!granted) throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
}
suspend fun snap(paramsJson: String?): Payload =
withContext(Dispatchers.Main) {
ensureCameraPermission()
requireCameraPermission()
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
val facing = parseFacing(paramsJson) ?: "front"
val quality = (parseQuality(paramsJson) ?: 0.9).coerceIn(0.1, 1.0)
@@ -94,7 +72,7 @@ class CameraCaptureManager(private val context: Context) {
(decoded.height.toDouble() * (maxWidth.toDouble() / decoded.width.toDouble()))
.toInt()
.coerceAtLeast(1)
decoded.scale(maxWidth, h)
Bitmap.createScaledBitmap(decoded, maxWidth, h, true)
} else {
decoded
}
@@ -110,15 +88,14 @@ class CameraCaptureManager(private val context: Context) {
)
}
@SuppressLint("MissingPermission")
suspend fun clip(paramsJson: String?): Payload =
withContext(Dispatchers.Main) {
ensureCameraPermission()
requireCameraPermission()
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
val facing = parseFacing(paramsJson) ?: "front"
val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 60_000)
val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 45_000)
val includeAudio = parseIncludeAudio(paramsJson) ?: true
if (includeAudio) ensureMicPermission()
if (includeAudio) requireMicPermission()
val provider = context.cameraProvider()
val recorder = Recorder.Builder().build()
@@ -1,116 +1,43 @@
package com.steipete.clawdis.node.node
import android.graphics.Bitmap
import android.os.Build
import android.graphics.Canvas
import android.os.Looper
import android.webkit.WebView
import androidx.core.graphics.createBitmap
import androidx.core.graphics.scale
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import java.io.ByteArrayOutputStream
import android.util.Base64
import org.json.JSONObject
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlin.coroutines.resume
class CanvasController {
enum class SnapshotFormat(val rawValue: String) {
Png("png"),
Jpeg("jpeg"),
}
enum class Mode { CANVAS, WEB }
@Volatile private var webView: WebView? = null
@Volatile private var url: String? = null
@Volatile private var debugStatusEnabled: Boolean = false
@Volatile private var debugStatusTitle: String? = null
@Volatile private var debugStatusSubtitle: String? = null
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
private fun clampJpegQuality(quality: Double?): Int {
val q = (quality ?: 0.82).coerceIn(0.1, 1.0)
return (q * 100.0).toInt().coerceIn(1, 100)
}
@Volatile private var mode: Mode = Mode.CANVAS
@Volatile private var url: String = ""
fun attach(webView: WebView) {
this.webView = webView
reload()
applyDebugStatus()
}
fun navigate(url: String) {
val trimmed = url.trim()
this.url = if (trimmed.isBlank() || trimmed == "/") null else trimmed
fun setMode(mode: Mode) {
this.mode = mode
reload()
}
fun currentUrl(): String? = url
fun isDefaultCanvas(): Boolean = url == null
fun setDebugStatusEnabled(enabled: Boolean) {
debugStatusEnabled = enabled
applyDebugStatus()
}
fun setDebugStatus(title: String?, subtitle: String?) {
debugStatusTitle = title
debugStatusSubtitle = subtitle
applyDebugStatus()
}
fun onPageFinished() {
applyDebugStatus()
}
private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) {
val wv = webView ?: return
if (Looper.myLooper() == Looper.getMainLooper()) {
block(wv)
} else {
wv.post { block(wv) }
}
fun navigate(url: String) {
this.url = url
reload()
}
private fun reload() {
val currentUrl = url
withWebViewOnMain { wv ->
if (currentUrl == null) {
wv.loadUrl(scaffoldAssetUrl)
} else {
wv.loadUrl(currentUrl)
}
}
}
private fun applyDebugStatus() {
val enabled = debugStatusEnabled
val title = debugStatusTitle
val subtitle = debugStatusSubtitle
withWebViewOnMain { wv ->
val titleJs = title?.let { JSONObject.quote(it) } ?: "null"
val subtitleJs = subtitle?.let { JSONObject.quote(it) } ?: "null"
val js = """
(() => {
try {
const api = globalThis.__clawdis;
if (!api) return;
if (typeof api.setDebugStatusEnabled === 'function') {
api.setDebugStatusEnabled(${if (enabled) "true" else "false"});
}
if (!${if (enabled) "true" else "false"}) return;
if (typeof api.setStatus === 'function') {
api.setStatus($titleJs, $subtitleJs);
}
} catch (_) {}
})();
""".trimIndent()
wv.evaluateJavascript(js, null)
val wv = webView ?: return
when (mode) {
Mode.WEB -> wv.loadUrl(url.trim())
Mode.CANVAS -> wv.loadDataWithBaseURL(null, canvasHtml, "text/html", "utf-8", null)
}
}
@@ -131,7 +58,7 @@ class CanvasController {
val scaled =
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
bmp.scale(maxWidth, h)
Bitmap.createScaledBitmap(bmp, maxWidth, h, true)
} else {
bmp
}
@@ -141,33 +68,11 @@ class CanvasController {
Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP)
}
suspend fun snapshotBase64(format: SnapshotFormat, quality: Double?, maxWidth: Int?): String =
withContext(Dispatchers.Main) {
val wv = webView ?: throw IllegalStateException("no webview")
val bmp = wv.captureBitmap()
val scaled =
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
bmp.scale(maxWidth, h)
} else {
bmp
}
val out = ByteArrayOutputStream()
val (compressFormat, compressQuality) =
when (format) {
SnapshotFormat.Png -> Bitmap.CompressFormat.PNG to 100
SnapshotFormat.Jpeg -> Bitmap.CompressFormat.JPEG to clampJpegQuality(quality)
}
scaled.compress(compressFormat, compressQuality, out)
Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP)
}
private suspend fun WebView.captureBitmap(): Bitmap =
suspendCancellableCoroutine { cont ->
val width = width.coerceAtLeast(1)
val height = height.coerceAtLeast(1)
val bitmap = createBitmap(width, height, Bitmap.Config.ARGB_8888)
val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
// WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable
// cross-version snapshot for this lightweight "canvas" use-case.
@@ -176,81 +81,162 @@ class CanvasController {
}
companion object {
data class SnapshotParams(val format: SnapshotFormat, val quality: Double?, val maxWidth: Int?)
fun parseMode(paramsJson: String?): Mode {
val raw = paramsJson ?: return Mode.CANVAS
return if (raw.contains("\"web\"")) Mode.WEB else Mode.CANVAS
}
fun parseNavigateUrl(paramsJson: String?): String {
val obj = parseParamsObject(paramsJson) ?: return ""
return obj.string("url").trim()
fun parseNavigateUrl(paramsJson: String?): String? {
val raw = paramsJson ?: return null
val key = "\"url\""
val idx = raw.indexOf(key)
if (idx < 0) return null
val start = raw.indexOf('"', idx + key.length)
if (start < 0) return null
val end = raw.indexOf('"', start + 1)
if (end < 0) return null
return raw.substring(start + 1, end)
}
fun parseEvalJs(paramsJson: String?): String? {
val obj = parseParamsObject(paramsJson) ?: return null
val js = obj.string("javaScript").trim()
return js.takeIf { it.isNotBlank() }
val raw = paramsJson ?: return null
val key = "\"javaScript\""
val idx = raw.indexOf(key)
if (idx < 0) return null
val start = raw.indexOf('"', idx + key.length)
if (start < 0) return null
val end = raw.lastIndexOf('"')
if (end <= start) return null
return raw.substring(start + 1, end)
.replace("\\n", "\n")
.replace("\\\"", "\"")
.replace("\\\\", "\\")
}
fun parseSnapshotMaxWidth(paramsJson: String?): Int? {
val obj = parseParamsObject(paramsJson) ?: return null
if (!obj.containsKey("maxWidth")) return null
val width = obj.int("maxWidth") ?: 0
return width.takeIf { it > 0 }
}
fun parseSnapshotFormat(paramsJson: String?): SnapshotFormat {
val obj = parseParamsObject(paramsJson) ?: return SnapshotFormat.Jpeg
val raw = obj.string("format").trim().lowercase()
return when (raw) {
"png" -> SnapshotFormat.Png
"jpeg", "jpg" -> SnapshotFormat.Jpeg
"" -> SnapshotFormat.Jpeg
else -> SnapshotFormat.Jpeg
}
}
fun parseSnapshotQuality(paramsJson: String?): Double? {
val obj = parseParamsObject(paramsJson) ?: return null
if (!obj.containsKey("quality")) return null
val q = obj.double("quality") ?: Double.NaN
if (!q.isFinite()) return null
return q.coerceIn(0.1, 1.0)
}
fun parseSnapshotParams(paramsJson: String?): SnapshotParams {
return SnapshotParams(
format = parseSnapshotFormat(paramsJson),
quality = parseSnapshotQuality(paramsJson),
maxWidth = parseSnapshotMaxWidth(paramsJson),
)
}
private val json = Json { ignoreUnknownKeys = true }
private fun parseParamsObject(paramsJson: String?): JsonObject? {
val raw = paramsJson?.trim().orEmpty()
if (raw.isEmpty()) return null
return try {
json.parseToJsonElement(raw).asObjectOrNull()
} catch (_: Throwable) {
null
}
}
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
private fun JsonObject.string(key: String): String {
val prim = this[key] as? JsonPrimitive ?: return ""
val raw = prim.content
return raw.takeIf { it != "null" }.orEmpty()
}
private fun JsonObject.int(key: String): Int? {
val prim = this[key] as? JsonPrimitive ?: return null
return prim.content.toIntOrNull()
}
private fun JsonObject.double(key: String): Double? {
val prim = this[key] as? JsonPrimitive ?: return null
return prim.content.toDoubleOrNull()
val raw = paramsJson ?: return null
val key = "\"maxWidth\""
val idx = raw.indexOf(key)
if (idx < 0) return null
val colon = raw.indexOf(':', idx + key.length)
if (colon < 0) return null
val tail = raw.substring(colon + 1).trimStart()
val num = tail.takeWhile { it.isDigit() }
return num.toIntOrNull()
}
}
}
private val canvasHtml =
"""
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
<title>Canvas</title>
<style>
:root { color-scheme: dark; }
html,body { height:100%; margin:0; }
body {
background: radial-gradient(1200px 900px at 15% 20%, rgba(42, 113, 255, 0.18), rgba(0,0,0,0) 55%),
radial-gradient(900px 700px at 85% 30%, rgba(255, 0, 138, 0.14), rgba(0,0,0,0) 60%),
radial-gradient(1000px 900px at 60% 90%, rgba(0, 209, 255, 0.10), rgba(0,0,0,0) 60%),
#000;
overflow: hidden;
}
body::before {
content:"";
position: fixed;
inset: -20%;
background:
repeating-linear-gradient(0deg, rgba(255,255,255,0.02) 0, rgba(255,255,255,0.02) 1px,
transparent 1px, transparent 48px),
repeating-linear-gradient(90deg, rgba(255,255,255,0.02) 0, rgba(255,255,255,0.02) 1px,
transparent 1px, transparent 48px);
transform: rotate(-7deg);
opacity: 0.55;
pointer-events: none;
}
canvas {
display:block;
width:100vw;
height:100vh;
touch-action: none;
}
#clawdis-status {
position: fixed;
inset: 0;
display: grid;
place-items: center;
pointer-events: none;
}
#clawdis-status .card {
text-align: center;
padding: 16px 18px;
border-radius: 14px;
background: rgba(18, 18, 22, 0.42);
border: 1px solid rgba(255,255,255,0.08);
box-shadow: 0 18px 60px rgba(0,0,0,0.55);
backdrop-filter: blur(14px);
}
#clawdis-status .title {
font: 600 20px system-ui, sans-serif;
letter-spacing: 0.2px;
color: rgba(255,255,255,0.92);
text-shadow: 0 0 22px rgba(42, 113, 255, 0.35);
}
#clawdis-status .subtitle {
margin-top: 6px;
font: 500 12px system-ui, sans-serif;
color: rgba(255,255,255,0.58);
}
</style>
</head>
<body>
<canvas id="clawdis-canvas"></canvas>
<div id="clawdis-status">
<div class="card">
<div class="title" id="clawdis-status-title">Ready</div>
<div class="subtitle" id="clawdis-status-subtitle">Waiting for agent</div>
</div>
</div>
<script>
(() => {
const canvas = document.getElementById('clawdis-canvas');
const ctx = canvas.getContext('2d');
const statusEl = document.getElementById('clawdis-status');
const titleEl = document.getElementById('clawdis-status-title');
const subtitleEl = document.getElementById('clawdis-status-subtitle');
function resize() {
const dpr = window.devicePixelRatio || 1;
const w = Math.max(1, Math.floor(window.innerWidth * dpr));
const h = Math.max(1, Math.floor(window.innerHeight * dpr));
canvas.width = w;
canvas.height = h;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
}
window.addEventListener('resize', resize);
resize();
window.__clawdis = {
canvas,
ctx,
setStatus: (title, subtitle) => {
if (!statusEl) return;
if (!title && !subtitle) {
statusEl.style.display = 'none';
return;
}
statusEl.style.display = 'grid';
if (titleEl && typeof title === 'string') titleEl.textContent = title;
if (subtitleEl && typeof subtitle === 'string') subtitleEl.textContent = subtitle;
}
};
})();
</script>
</body>
</html>
""".trimIndent()
@@ -1,196 +0,0 @@
package com.steipete.clawdis.node.node
import android.content.Context
import android.hardware.display.DisplayManager
import android.media.MediaRecorder
import android.media.projection.MediaProjectionManager
import android.util.Base64
import com.steipete.clawdis.node.ScreenCaptureRequester
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import java.io.File
import kotlin.math.roundToInt
class ScreenRecordManager(private val context: Context) {
data class Payload(val payloadJson: String)
@Volatile private var screenCaptureRequester: ScreenCaptureRequester? = null
@Volatile private var permissionRequester: com.steipete.clawdis.node.PermissionRequester? = null
fun attachScreenCaptureRequester(requester: ScreenCaptureRequester) {
screenCaptureRequester = requester
}
fun attachPermissionRequester(requester: com.steipete.clawdis.node.PermissionRequester) {
permissionRequester = requester
}
suspend fun record(paramsJson: String?): Payload =
withContext(Dispatchers.Default) {
val requester =
screenCaptureRequester
?: throw IllegalStateException(
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
)
val durationMs = (parseDurationMs(paramsJson) ?: 10_000).coerceIn(250, 60_000)
val fps = (parseFps(paramsJson) ?: 10.0).coerceIn(1.0, 60.0)
val fpsInt = fps.roundToInt().coerceIn(1, 60)
val screenIndex = parseScreenIndex(paramsJson)
val includeAudio = parseIncludeAudio(paramsJson) ?: true
val format = parseString(paramsJson, key = "format")
if (format != null && format.lowercase() != "mp4") {
throw IllegalArgumentException("INVALID_REQUEST: screen format must be mp4")
}
if (screenIndex != null && screenIndex != 0) {
throw IllegalArgumentException("INVALID_REQUEST: screenIndex must be 0 on Android")
}
val capture = requester.requestCapture()
?: throw IllegalStateException(
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
)
val mgr =
context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
val projection = mgr.getMediaProjection(capture.resultCode, capture.data)
?: throw IllegalStateException("UNAVAILABLE: screen capture unavailable")
val metrics = context.resources.displayMetrics
val width = metrics.widthPixels
val height = metrics.heightPixels
val densityDpi = metrics.densityDpi
val file = File.createTempFile("clawdis-screen-", ".mp4")
if (includeAudio) ensureMicPermission()
val recorder = MediaRecorder()
var virtualDisplay: android.hardware.display.VirtualDisplay? = null
try {
if (includeAudio) {
recorder.setAudioSource(MediaRecorder.AudioSource.MIC)
}
recorder.setVideoSource(MediaRecorder.VideoSource.SURFACE)
recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
recorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264)
if (includeAudio) {
recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
recorder.setAudioChannels(1)
recorder.setAudioSamplingRate(44_100)
recorder.setAudioEncodingBitRate(96_000)
}
recorder.setVideoSize(width, height)
recorder.setVideoFrameRate(fpsInt)
recorder.setVideoEncodingBitRate(estimateBitrate(width, height, fpsInt))
recorder.setOutputFile(file.absolutePath)
recorder.prepare()
val surface = recorder.surface
virtualDisplay =
projection.createVirtualDisplay(
"clawdis-screen",
width,
height,
densityDpi,
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
surface,
null,
null,
)
recorder.start()
delay(durationMs.toLong())
} finally {
try {
recorder.stop()
} catch (_: Throwable) {
// ignore
}
recorder.reset()
recorder.release()
virtualDisplay?.release()
projection.stop()
}
val bytes = withContext(Dispatchers.IO) { file.readBytes() }
file.delete()
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
Payload(
"""{"format":"mp4","base64":"$base64","durationMs":$durationMs,"fps":$fpsInt,"screenIndex":0,"hasAudio":$includeAudio}""",
)
}
private suspend fun ensureMicPermission() {
val granted =
androidx.core.content.ContextCompat.checkSelfPermission(
context,
android.Manifest.permission.RECORD_AUDIO,
) == android.content.pm.PackageManager.PERMISSION_GRANTED
if (granted) return
val requester =
permissionRequester
?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
val results = requester.requestIfMissing(listOf(android.Manifest.permission.RECORD_AUDIO))
if (results[android.Manifest.permission.RECORD_AUDIO] != true) {
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
}
}
private fun parseDurationMs(paramsJson: String?): Int? =
parseNumber(paramsJson, key = "durationMs")?.toIntOrNull()
private fun parseFps(paramsJson: String?): Double? =
parseNumber(paramsJson, key = "fps")?.toDoubleOrNull()
private fun parseScreenIndex(paramsJson: String?): Int? =
parseNumber(paramsJson, key = "screenIndex")?.toIntOrNull()
private fun parseIncludeAudio(paramsJson: String?): Boolean? {
val raw = paramsJson ?: return null
val key = "\"includeAudio\""
val idx = raw.indexOf(key)
if (idx < 0) return null
val colon = raw.indexOf(':', idx + key.length)
if (colon < 0) return null
val tail = raw.substring(colon + 1).trimStart()
return when {
tail.startsWith("true") -> true
tail.startsWith("false") -> false
else -> null
}
}
private fun parseNumber(paramsJson: String?, key: String): String? {
val raw = paramsJson ?: return null
val needle = "\"$key\""
val idx = raw.indexOf(needle)
if (idx < 0) return null
val colon = raw.indexOf(':', idx + needle.length)
if (colon < 0) return null
val tail = raw.substring(colon + 1).trimStart()
return tail.takeWhile { it.isDigit() || it == '.' || it == '-' }
}
private fun parseString(paramsJson: String?, key: String): String? {
val raw = paramsJson ?: return null
val needle = "\"$key\""
val idx = raw.indexOf(needle)
if (idx < 0) return null
val colon = raw.indexOf(':', idx + needle.length)
if (colon < 0) return null
val tail = raw.substring(colon + 1).trimStart()
if (!tail.startsWith('\"')) return null
val rest = tail.drop(1)
val end = rest.indexOf('\"')
if (end < 0) return null
return rest.substring(0, end)
}
private fun estimateBitrate(width: Int, height: Int, fps: Int): Int {
val pixels = width.toLong() * height.toLong()
val raw = (pixels * fps.toLong() * 2L).toInt()
return raw.coerceIn(1_000_000, 12_000_000)
}
}
@@ -1,66 +0,0 @@
package com.steipete.clawdis.node.protocol
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
object ClawdisCanvasA2UIAction {
fun extractActionName(userAction: JsonObject): String? {
val name =
(userAction["name"] as? JsonPrimitive)
?.content
?.trim()
.orEmpty()
if (name.isNotEmpty()) return name
val action =
(userAction["action"] as? JsonPrimitive)
?.content
?.trim()
.orEmpty()
return action.ifEmpty { null }
}
fun sanitizeTagValue(value: String): String {
val trimmed = value.trim().ifEmpty { "-" }
val normalized = trimmed.replace(" ", "_")
val out = StringBuilder(normalized.length)
for (c in normalized) {
val ok =
c.isLetterOrDigit() ||
c == '_' ||
c == '-' ||
c == '.' ||
c == ':'
out.append(if (ok) c else '_')
}
return out.toString()
}
fun formatAgentMessage(
actionName: String,
sessionKey: String,
surfaceId: String,
sourceComponentId: String,
host: String,
instanceId: String,
contextJson: String?,
): String {
val ctxSuffix = contextJson?.takeIf { it.isNotBlank() }?.let { " ctx=$it" }.orEmpty()
return listOf(
"CANVAS_A2UI",
"action=${sanitizeTagValue(actionName)}",
"session=${sanitizeTagValue(sessionKey)}",
"surface=${sanitizeTagValue(surfaceId)}",
"component=${sanitizeTagValue(sourceComponentId)}",
"host=${sanitizeTagValue(host)}",
"instance=${sanitizeTagValue(instanceId)}$ctxSuffix",
"default=update_canvas",
).joinToString(separator = " ")
}
fun jsDispatchA2UIActionStatus(actionId: String, ok: Boolean, error: String?): String {
val err = (error ?: "").replace("\\", "\\\\").replace("\"", "\\\"")
val okLiteral = if (ok) "true" else "false"
val idEscaped = actionId.replace("\\", "\\\\").replace("\"", "\\\"")
return "window.dispatchEvent(new CustomEvent('clawdis:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));"
}
}
@@ -1,51 +0,0 @@
package com.steipete.clawdis.node.protocol
enum class ClawdisCapability(val rawValue: String) {
Canvas("canvas"),
Camera("camera"),
Screen("screen"),
VoiceWake("voiceWake"),
}
enum class ClawdisCanvasCommand(val rawValue: String) {
Present("canvas.present"),
Hide("canvas.hide"),
Navigate("canvas.navigate"),
Eval("canvas.eval"),
Snapshot("canvas.snapshot"),
;
companion object {
const val NamespacePrefix: String = "canvas."
}
}
enum class ClawdisCanvasA2UICommand(val rawValue: String) {
Push("canvas.a2ui.push"),
PushJSONL("canvas.a2ui.pushJSONL"),
Reset("canvas.a2ui.reset"),
;
companion object {
const val NamespacePrefix: String = "canvas.a2ui."
}
}
enum class ClawdisCameraCommand(val rawValue: String) {
Snap("camera.snap"),
Clip("camera.clip"),
;
companion object {
const val NamespacePrefix: String = "camera."
}
}
enum class ClawdisScreenCommand(val rawValue: String) {
Record("screen.record"),
;
companion object {
const val NamespacePrefix: String = "screen."
}
}
@@ -1,123 +0,0 @@
package com.steipete.clawdis.node.ui
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
import androidx.compose.animation.slideInVertically
import androidx.compose.animation.slideOutVertically
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.statusBarsPadding
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.CheckCircle
import androidx.compose.material.icons.filled.Error
import androidx.compose.material.icons.filled.FiberManualRecord
import androidx.compose.material.icons.filled.PhotoCamera
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.CameraHudKind
import com.steipete.clawdis.node.CameraHudState
import kotlinx.coroutines.delay
@Composable
fun CameraHudOverlay(
hud: CameraHudState?,
flashToken: Long,
modifier: Modifier = Modifier,
) {
Box(modifier = modifier.fillMaxSize()) {
CameraFlash(token = flashToken)
AnimatedVisibility(
visible = hud != null,
enter = slideInVertically(initialOffsetY = { -it / 2 }) + fadeIn(),
exit = slideOutVertically(targetOffsetY = { -it / 2 }) + fadeOut(),
modifier = Modifier.align(Alignment.TopStart).statusBarsPadding().padding(start = 12.dp, top = 58.dp),
) {
if (hud != null) {
Toast(hud = hud)
}
}
}
}
@Composable
private fun CameraFlash(token: Long) {
var alpha by remember { mutableFloatStateOf(0f) }
LaunchedEffect(token) {
if (token == 0L) return@LaunchedEffect
alpha = 0.85f
delay(110)
alpha = 0f
}
Box(
modifier =
Modifier
.fillMaxSize()
.alpha(alpha)
.background(Color.White),
)
}
@Composable
private fun Toast(hud: CameraHudState) {
Surface(
shape = RoundedCornerShape(14.dp),
color = MaterialTheme.colorScheme.surface.copy(alpha = 0.85f),
tonalElevation = 2.dp,
shadowElevation = 8.dp,
) {
Row(
modifier = Modifier.padding(vertical = 10.dp, horizontal = 12.dp),
verticalAlignment = Alignment.CenterVertically,
) {
when (hud.kind) {
CameraHudKind.Photo -> {
Icon(Icons.Default.PhotoCamera, contentDescription = null)
Spacer(Modifier.size(10.dp))
CircularProgressIndicator(modifier = Modifier.size(14.dp), strokeWidth = 2.dp)
}
CameraHudKind.Recording -> {
Icon(Icons.Default.FiberManualRecord, contentDescription = null, tint = Color.Red)
}
CameraHudKind.Success -> {
Icon(Icons.Default.CheckCircle, contentDescription = null)
}
CameraHudKind.Error -> {
Icon(Icons.Default.Error, contentDescription = null)
}
}
Spacer(Modifier.size(10.dp))
Text(
text = hud.message,
style = MaterialTheme.typography.bodyMedium,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
}
}
@@ -1,10 +1,73 @@
package com.steipete.clawdis.node.ui
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.material3.Button
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.MainViewModel
import com.steipete.clawdis.node.ui.chat.ChatSheetContent
@Composable
fun ChatSheet(viewModel: MainViewModel) {
ChatSheetContent(viewModel = viewModel)
val messages by viewModel.chatMessages.collectAsState()
val error by viewModel.chatError.collectAsState()
val pendingRunCount by viewModel.pendingRunCount.collectAsState()
var input by remember { mutableStateOf("") }
LaunchedEffect(Unit) {
viewModel.loadChat("main")
}
Column(modifier = Modifier.fillMaxSize().padding(16.dp), verticalArrangement = Arrangement.spacedBy(12.dp)) {
Text("Clawd Chat · session main")
if (!error.isNullOrBlank()) {
Text("Error: $error")
}
LazyColumn(modifier = Modifier.fillMaxWidth().weight(1f, fill = true)) {
items(messages) { msg ->
Text("${msg.role}: ${msg.text}")
}
if (pendingRunCount > 0) {
item { Text("assistant: …") }
}
}
Row(horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.fillMaxWidth()) {
OutlinedTextField(
value = input,
onValueChange = { input = it },
modifier = Modifier.weight(1f),
label = { Text("Message") },
)
Button(
onClick = {
val text = input
input = ""
viewModel.sendChat("main", text)
},
enabled = input.trim().isNotEmpty(),
) {
Text("Send")
}
}
}
}
@@ -1,32 +0,0 @@
package com.steipete.clawdis.node.ui
import androidx.compose.foundation.isSystemInDarkTheme
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.dynamicDarkColorScheme
import androidx.compose.material3.dynamicLightColorScheme
import androidx.compose.runtime.Composable
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalContext
@Composable
fun ClawdisTheme(content: @Composable () -> Unit) {
val context = LocalContext.current
val isDark = isSystemInDarkTheme()
val colorScheme = if (isDark) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
MaterialTheme(colorScheme = colorScheme, content = content)
}
@Composable
fun overlayContainerColor(): Color {
val scheme = MaterialTheme.colorScheme
val isDark = isSystemInDarkTheme()
val base = if (isDark) scheme.surfaceContainerLow else scheme.surfaceContainerHigh
// Light mode: background stays dark (canvas), so clamp overlays away from pure-white glare.
return if (isDark) base else base.copy(alpha = 0.88f)
}
@Composable
fun overlayIconColor(): Color {
return MaterialTheme.colorScheme.onSurfaceVariant
}
@@ -1,40 +1,22 @@
package com.steipete.clawdis.node.ui
import android.annotation.SuppressLint
import android.Manifest
import android.content.pm.PackageManager
import android.graphics.Color
import android.util.Log
import android.view.View
import android.webkit.JavascriptInterface
import android.webkit.WebView
import android.webkit.WebSettings
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebViewClient
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.WindowInsetsSides
import androidx.compose.foundation.layout.only
import androidx.compose.foundation.layout.statusBars
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.safeDrawing
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.material3.Button
import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButtonDefaults
import androidx.compose.material3.ModalBottomSheet
import androidx.compose.material3.Text
import androidx.compose.material3.rememberModalBottomSheetState
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ChatBubble
import androidx.compose.material.icons.filled.Settings
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
@@ -44,9 +26,7 @@ import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.compose.ui.window.Popup
import androidx.compose.ui.window.PopupProperties
import androidx.core.content.ContextCompat
import androidx.compose.ui.zIndex
import com.steipete.clawdis.node.MainViewModel
@OptIn(ExperimentalMaterial3Api::class)
@@ -54,74 +34,29 @@ import com.steipete.clawdis.node.MainViewModel
fun RootScreen(viewModel: MainViewModel) {
var sheet by remember { mutableStateOf<Sheet?>(null) }
val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true)
val safeOverlayInsets = WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal)
val context = LocalContext.current
val serverName by viewModel.serverName.collectAsState()
val statusText by viewModel.statusText.collectAsState()
val cameraHud by viewModel.cameraHud.collectAsState()
val cameraFlashToken by viewModel.cameraFlashToken.collectAsState()
val bridgeState =
remember(serverName, statusText) {
when {
serverName != null -> BridgeState.Connected
statusText.contains("connecting", ignoreCase = true) ||
statusText.contains("reconnecting", ignoreCase = true) -> BridgeState.Connecting
statusText.contains("error", ignoreCase = true) -> BridgeState.Error
else -> BridgeState.Disconnected
}
}
val voiceEnabled =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
val safeButtonInsets = WindowInsets.statusBars.only(WindowInsetsSides.Top)
Box(modifier = Modifier.fillMaxSize()) {
CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize())
}
CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize().zIndex(0f))
// Camera HUD (flash + toast) must be in a Popup to render above the WebView.
Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) {
CameraHudOverlay(hud = cameraHud, flashToken = cameraFlashToken, modifier = Modifier.fillMaxSize())
}
Box(modifier = Modifier.align(Alignment.TopEnd).zIndex(1f).windowInsetsPadding(safeButtonInsets).padding(12.dp)) {
Button(onClick = { sheet = Sheet.Settings }) { Text("Settings") }
}
// Keep the overlay buttons above the WebView canvas (AndroidView), otherwise they may not receive touches.
Popup(alignment = Alignment.TopStart, properties = PopupProperties(focusable = false)) {
StatusPill(
bridge = bridgeState,
voiceEnabled = voiceEnabled,
onClick = { sheet = Sheet.Settings },
modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(start = 12.dp, top = 12.dp),
)
}
Popup(alignment = Alignment.TopEnd, properties = PopupProperties(focusable = false)) {
Column(
modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(end = 12.dp, top = 12.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
horizontalAlignment = Alignment.End,
) {
OverlayIconButton(
onClick = { sheet = Sheet.Chat },
icon = { Icon(Icons.Default.ChatBubble, contentDescription = "Chat") },
)
OverlayIconButton(
onClick = { sheet = Sheet.Settings },
icon = { Icon(Icons.Default.Settings, contentDescription = "Settings") },
)
Box(modifier = Modifier.align(Alignment.TopStart).zIndex(1f).windowInsetsPadding(safeButtonInsets).padding(12.dp)) {
Button(onClick = { sheet = Sheet.Chat }) { Text("Chat") }
}
}
val currentSheet = sheet
if (currentSheet != null) {
if (sheet != null) {
ModalBottomSheet(
onDismissRequest = { sheet = null },
sheetState = sheetState,
) {
when (currentSheet) {
when (sheet) {
Sheet.Chat -> ChatSheet(viewModel = viewModel)
Sheet.Settings -> SettingsSheet(viewModel = viewModel)
null -> {}
}
}
}
@@ -132,109 +67,20 @@ private enum class Sheet {
Settings,
}
@Composable
private fun OverlayIconButton(
onClick: () -> Unit,
icon: @Composable () -> Unit,
) {
FilledTonalIconButton(
onClick = onClick,
modifier = Modifier.size(44.dp),
colors =
IconButtonDefaults.filledTonalIconButtonColors(
containerColor = overlayContainerColor(),
contentColor = overlayIconColor(),
),
) {
icon()
}
}
@SuppressLint("SetJavaScriptEnabled")
@Composable
private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier) {
val context = LocalContext.current
val isDebuggable = (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0
AndroidView(
modifier = modifier,
factory = {
WebView(context).apply {
settings.javaScriptEnabled = true
// Some embedded web UIs (incl. the "background website") use localStorage/sessionStorage.
settings.domStorageEnabled = true
settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE
webViewClient =
object : WebViewClient() {
override fun onReceivedError(
view: WebView,
request: WebResourceRequest,
error: WebResourceError,
) {
if (!isDebuggable) return
if (!request.isForMainFrame) return
Log.e("ClawdisWebView", "onReceivedError: ${error.errorCode} ${error.description} ${request.url}")
}
override fun onReceivedHttpError(
view: WebView,
request: WebResourceRequest,
errorResponse: WebResourceResponse,
) {
if (!isDebuggable) return
if (!request.isForMainFrame) return
Log.e(
"ClawdisWebView",
"onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}",
)
}
override fun onPageFinished(view: WebView, url: String?) {
viewModel.canvas.onPageFinished()
}
}
setBackgroundColor(Color.BLACK)
setLayerType(View.LAYER_TYPE_HARDWARE, null)
val a2uiBridge =
CanvasA2UIActionBridge { payload ->
viewModel.handleCanvasA2UIActionFromWebView(payload)
}
addJavascriptInterface(a2uiBridge, CanvasA2UIActionBridge.interfaceName)
addJavascriptInterface(
CanvasA2UIActionLegacyBridge(a2uiBridge),
CanvasA2UIActionLegacyBridge.interfaceName,
)
settings.domStorageEnabled = false
webViewClient = WebViewClient()
setBackgroundColor(0x00000000)
viewModel.canvas.attach(this)
}
},
)
}
private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) {
@JavascriptInterface
fun postMessage(payload: String?) {
val msg = payload?.trim().orEmpty()
if (msg.isEmpty()) return
onMessage(msg)
}
companion object {
const val interfaceName: String = "clawdisCanvasA2UIAction"
}
}
private class CanvasA2UIActionLegacyBridge(private val bridge: CanvasA2UIActionBridge) {
@JavascriptInterface
fun canvasAction(payload: String?) {
bridge.postMessage(payload)
}
@JavascriptInterface
fun postMessage(payload: String?) {
bridge.postMessage(payload)
}
companion object {
const val interfaceName: String = "Android"
}
}
@@ -4,51 +4,31 @@ import android.Manifest
import android.content.pm.PackageManager
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.foundation.clickable
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.WindowInsetsSides
import androidx.compose.foundation.layout.fillMaxHeight
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.imePadding
import androidx.compose.foundation.layout.only
import androidx.compose.foundation.layout.safeDrawing
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.foundation.lazy.rememberLazyListState
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ExpandLess
import androidx.compose.material.icons.filled.ExpandMore
import androidx.compose.foundation.verticalScroll
import androidx.compose.material3.Button
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.Icon
import androidx.compose.material3.ListItem
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.RadioButton
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import com.steipete.clawdis.node.MainViewModel
import com.steipete.clawdis.node.NodeForegroundService
import com.steipete.clawdis.node.VoiceWakeMode
@Composable
fun SettingsSheet(viewModel: MainViewModel) {
@@ -56,26 +36,14 @@ fun SettingsSheet(viewModel: MainViewModel) {
val instanceId by viewModel.instanceId.collectAsState()
val displayName by viewModel.displayName.collectAsState()
val cameraEnabled by viewModel.cameraEnabled.collectAsState()
val preventSleep by viewModel.preventSleep.collectAsState()
val wakeWords by viewModel.wakeWords.collectAsState()
val voiceWakeMode by viewModel.voiceWakeMode.collectAsState()
val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState()
val isConnected by viewModel.isConnected.collectAsState()
val manualEnabled by viewModel.manualEnabled.collectAsState()
val manualHost by viewModel.manualHost.collectAsState()
val manualPort by viewModel.manualPort.collectAsState()
val canvasDebugStatusEnabled by viewModel.canvasDebugStatusEnabled.collectAsState()
val statusText by viewModel.statusText.collectAsState()
val serverName by viewModel.serverName.collectAsState()
val remoteAddress by viewModel.remoteAddress.collectAsState()
val bridges by viewModel.bridges.collectAsState()
val discoveryStatusText by viewModel.discoveryStatusText.collectAsState()
val listState = rememberLazyListState()
val (wakeWordsText, setWakeWordsText) = remember { mutableStateOf("") }
val (advancedExpanded, setAdvancedExpanded) = remember { mutableStateOf(false) }
LaunchedEffect(wakeWords) { setWakeWordsText(wakeWords.joinToString(", ")) }
val scrollState = rememberScrollState()
val permissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms ->
@@ -83,335 +51,125 @@ fun SettingsSheet(viewModel: MainViewModel) {
viewModel.setCameraEnabled(cameraOk)
}
val audioPermissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { _ ->
// Status text is handled by NodeRuntime.
}
fun setCameraEnabledChecked(checked: Boolean) {
if (!checked) {
viewModel.setCameraEnabled(false)
return
}
val cameraOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) ==
PackageManager.PERMISSION_GRANTED
if (cameraOk) {
viewModel.setCameraEnabled(true)
} else {
permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO))
}
}
val visibleBridges =
if (isConnected && remoteAddress != null) {
bridges.filterNot { "${it.host}:${it.port}" == remoteAddress }
} else {
bridges
}
val bridgeDiscoveryFooterText =
if (visibleBridges.isEmpty()) {
discoveryStatusText
} else if (isConnected) {
"Discovery active • ${visibleBridges.size} other bridge${if (visibleBridges.size == 1) "" else "s"} found"
} else {
"Discovery active • ${visibleBridges.size} bridge${if (visibleBridges.size == 1) "" else "s"} found"
}
LazyColumn(
state = listState,
modifier =
Modifier
.fillMaxWidth()
.fillMaxHeight()
.imePadding()
.windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Bottom)),
contentPadding = PaddingValues(16.dp),
verticalArrangement = Arrangement.spacedBy(6.dp),
Column(
modifier = Modifier.fillMaxWidth().verticalScroll(scrollState).padding(16.dp),
verticalArrangement = Arrangement.spacedBy(14.dp),
) {
// Order parity: Node → Bridge → Voice → Camera → Screen.
item { Text("Node", style = MaterialTheme.typography.titleSmall) }
item {
OutlinedTextField(
value = displayName,
onValueChange = viewModel::setDisplayName,
label = { Text("Name") },
modifier = Modifier.fillMaxWidth(),
)
}
item { Text("Instance ID: $instanceId", color = MaterialTheme.colorScheme.onSurfaceVariant) }
Text("Node")
OutlinedTextField(
value = displayName,
onValueChange = viewModel::setDisplayName,
label = { Text("Name") },
modifier = Modifier.fillMaxWidth(),
)
Text("Instance ID: $instanceId")
item { HorizontalDivider() }
HorizontalDivider()
// Bridge
item { Text("Bridge", style = MaterialTheme.typography.titleSmall) }
item { ListItem(headlineContent = { Text("Status") }, supportingContent = { Text(statusText) }) }
if (serverName != null) {
item { ListItem(headlineContent = { Text("Server") }, supportingContent = { Text(serverName!!) }) }
}
if (remoteAddress != null) {
item { ListItem(headlineContent = { Text("Address") }, supportingContent = { Text(remoteAddress!!) }) }
}
item {
// UI sanity: "Disconnect" only when we have an active remote.
if (isConnected && remoteAddress != null) {
Button(
onClick = {
viewModel.disconnect()
NodeForegroundService.stop(context)
},
) {
Text("Disconnect")
}
}
}
item { HorizontalDivider() }
if (!isConnected || visibleBridges.isNotEmpty()) {
item {
Text(
if (isConnected) "Other Bridges" else "Discovered Bridges",
style = MaterialTheme.typography.titleSmall,
)
}
if (!isConnected && visibleBridges.isEmpty()) {
item { Text("No bridges found yet.", color = MaterialTheme.colorScheme.onSurfaceVariant) }
} else {
items(items = visibleBridges, key = { it.stableId }) { bridge ->
ListItem(
headlineContent = { Text(bridge.name) },
supportingContent = { Text("${bridge.host}:${bridge.port}") },
trailingContent = {
Button(
onClick = {
NodeForegroundService.start(context)
viewModel.connect(bridge)
},
) {
Text("Connect")
}
},
)
}
}
item {
Text(
bridgeDiscoveryFooterText,
modifier = Modifier.fillMaxWidth(),
textAlign = TextAlign.Center,
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
item { HorizontalDivider() }
item {
ListItem(
headlineContent = { Text("Advanced") },
supportingContent = { Text("Manual bridge connection") },
trailingContent = {
Icon(
imageVector = if (advancedExpanded) Icons.Filled.ExpandLess else Icons.Filled.ExpandMore,
contentDescription = if (advancedExpanded) "Collapse" else "Expand",
)
},
modifier =
Modifier.clickable {
setAdvancedExpanded(!advancedExpanded)
},
)
}
item {
AnimatedVisibility(visible = advancedExpanded) {
Column(verticalArrangement = Arrangement.spacedBy(10.dp), modifier = Modifier.fillMaxWidth()) {
ListItem(
headlineContent = { Text("Use Manual Bridge") },
supportingContent = { Text("Use this when discovery is blocked.") },
trailingContent = { Switch(checked = manualEnabled, onCheckedChange = viewModel::setManualEnabled) },
)
OutlinedTextField(
value = manualHost,
onValueChange = viewModel::setManualHost,
label = { Text("Host") },
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
)
OutlinedTextField(
value = manualPort.toString(),
onValueChange = { v -> viewModel.setManualPort(v.toIntOrNull() ?: 0) },
label = { Text("Port") },
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
)
val hostOk = manualHost.trim().isNotEmpty()
val portOk = manualPort in 1..65535
Button(
onClick = {
NodeForegroundService.start(context)
viewModel.connectManual()
},
enabled = manualEnabled && hostOk && portOk,
) {
Text("Connect (Manual)")
Text("Camera")
Row(horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.fillMaxWidth()) {
Switch(
checked = cameraEnabled,
onCheckedChange = { enabled ->
if (!enabled) {
viewModel.setCameraEnabled(false)
return@Switch
}
val cameraOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) ==
PackageManager.PERMISSION_GRANTED
if (cameraOk) {
viewModel.setCameraEnabled(true)
} else {
permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO))
}
},
)
Text(if (cameraEnabled) "Allow Camera" else "Camera Disabled")
}
Text("Tip: grant Microphone permission for video clips with audio.")
HorizontalDivider()
Text("Bridge")
Text("Status: $statusText")
if (serverName != null) Text("Server: $serverName")
if (remoteAddress != null) Text("Address: $remoteAddress")
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
Button(
onClick = {
viewModel.disconnect()
NodeForegroundService.stop(context)
},
) {
Text("Disconnect")
}
}
HorizontalDivider()
Text("Advanced")
Row(horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.fillMaxWidth()) {
Switch(checked = manualEnabled, onCheckedChange = viewModel::setManualEnabled)
Text(if (manualEnabled) "Manual Bridge Enabled" else "Manual Bridge Disabled")
}
OutlinedTextField(
value = manualHost,
onValueChange = viewModel::setManualHost,
label = { Text("Host") },
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
)
OutlinedTextField(
value = manualPort.toString(),
onValueChange = { v -> viewModel.setManualPort(v.toIntOrNull() ?: 0) },
label = { Text("Port") },
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
)
Button(
onClick = {
NodeForegroundService.start(context)
viewModel.connectManual()
},
enabled = manualEnabled,
) {
Text("Connect (Manual)")
}
HorizontalDivider()
Text("Discovered Bridges")
if (bridges.isEmpty()) {
Text("No bridges found yet.")
} else {
LazyColumn(modifier = Modifier.fillMaxWidth().height(240.dp)) {
items(bridges) { bridge ->
Row(
modifier = Modifier.fillMaxWidth().padding(vertical = 8.dp),
horizontalArrangement = Arrangement.SpaceBetween,
) {
Column(modifier = Modifier.weight(1f)) {
Text(bridge.name)
Text("${bridge.host}:${bridge.port}")
}
Spacer(modifier = Modifier.padding(4.dp))
Button(
onClick = {
NodeForegroundService.start(context)
viewModel.connect(bridge)
},
) {
Text("Connect")
}
}
HorizontalDivider()
}
}
}
item { HorizontalDivider() }
// Voice
item { Text("Voice", style = MaterialTheme.typography.titleSmall) }
item {
val enabled = voiceWakeMode != VoiceWakeMode.Off
ListItem(
headlineContent = { Text("Voice Wake") },
supportingContent = { Text(voiceWakeStatusText) },
trailingContent = {
Switch(
checked = enabled,
onCheckedChange = { on ->
if (on) {
val micOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground)
} else {
viewModel.setVoiceWakeMode(VoiceWakeMode.Off)
}
},
)
},
)
}
item {
AnimatedVisibility(visible = voiceWakeMode != VoiceWakeMode.Off) {
Column(verticalArrangement = Arrangement.spacedBy(6.dp), modifier = Modifier.fillMaxWidth()) {
ListItem(
headlineContent = { Text("Foreground Only") },
supportingContent = { Text("Listens only while Clawdis is open.") },
trailingContent = {
RadioButton(
selected = voiceWakeMode == VoiceWakeMode.Foreground,
onClick = {
val micOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground)
},
)
},
)
ListItem(
headlineContent = { Text("Always") },
supportingContent = { Text("Keeps listening in the background (shows a persistent notification).") },
trailingContent = {
RadioButton(
selected = voiceWakeMode == VoiceWakeMode.Always,
onClick = {
val micOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
viewModel.setVoiceWakeMode(VoiceWakeMode.Always)
},
)
},
)
}
}
}
item {
OutlinedTextField(
value = wakeWordsText,
onValueChange = setWakeWordsText,
label = { Text("Wake Words (comma-separated)") },
modifier = Modifier.fillMaxWidth(),
singleLine = true,
)
}
item {
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
Button(
onClick = {
val parsed = com.steipete.clawdis.node.WakeWords.parseCommaSeparated(wakeWordsText)
viewModel.setWakeWords(parsed)
},
enabled = isConnected,
) {
Text("Save + Sync")
}
Button(onClick = viewModel::resetWakeWordsDefaults) { Text("Reset defaults") }
}
}
item {
Text(
if (isConnected) {
"Any node can edit wake words. Changes sync via the gateway bridge."
} else {
"Connect to a gateway to sync wake words globally."
},
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
item { HorizontalDivider() }
// Camera
item { Text("Camera", style = MaterialTheme.typography.titleSmall) }
item {
ListItem(
headlineContent = { Text("Allow Camera") },
supportingContent = { Text("Allows the bridge to request photos or short video clips (foreground only).") },
trailingContent = { Switch(checked = cameraEnabled, onCheckedChange = ::setCameraEnabledChecked) },
)
}
item {
Text(
"Tip: grant Microphone permission for video clips with audio.",
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
item { HorizontalDivider() }
// Screen
item { Text("Screen", style = MaterialTheme.typography.titleSmall) }
item {
ListItem(
headlineContent = { Text("Prevent Sleep") },
supportingContent = { Text("Keeps the screen awake while Clawdis is open.") },
trailingContent = { Switch(checked = preventSleep, onCheckedChange = viewModel::setPreventSleep) },
)
}
item { HorizontalDivider() }
// Debug
item { Text("Debug", style = MaterialTheme.typography.titleSmall) }
item {
ListItem(
headlineContent = { Text("Debug Canvas Status") },
supportingContent = { Text("Show status text in the canvas when debug is enabled.") },
trailingContent = {
Switch(
checked = canvasDebugStatusEnabled,
onCheckedChange = viewModel::setCanvasDebugStatusEnabled,
)
},
)
}
item { Spacer(modifier = Modifier.height(20.dp)) }
Spacer(modifier = Modifier.height(20.dp))
}
}
@@ -1,87 +0,0 @@
package com.steipete.clawdis.node.ui
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Mic
import androidx.compose.material.icons.filled.MicOff
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.material3.VerticalDivider
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.unit.dp
@Composable
fun StatusPill(
bridge: BridgeState,
voiceEnabled: Boolean,
onClick: () -> Unit,
modifier: Modifier = Modifier,
) {
Surface(
onClick = onClick,
modifier = modifier,
shape = RoundedCornerShape(14.dp),
color = overlayContainerColor(),
tonalElevation = 3.dp,
shadowElevation = 0.dp,
) {
Row(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp),
horizontalArrangement = Arrangement.spacedBy(10.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Row(horizontalArrangement = Arrangement.spacedBy(8.dp), verticalAlignment = Alignment.CenterVertically) {
Surface(
modifier = Modifier.size(9.dp),
shape = CircleShape,
color = bridge.color,
) {}
Text(
text = bridge.title,
style = MaterialTheme.typography.labelLarge,
)
}
VerticalDivider(
modifier = Modifier.height(14.dp).alpha(0.35f),
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
Icon(
imageVector = if (voiceEnabled) Icons.Default.Mic else Icons.Default.MicOff,
contentDescription = if (voiceEnabled) "Voice enabled" else "Voice disabled",
tint =
if (voiceEnabled) {
overlayIconColor()
} else {
MaterialTheme.colorScheme.onSurfaceVariant
},
modifier = Modifier.size(18.dp),
)
Spacer(modifier = Modifier.width(2.dp))
}
}
}
enum class BridgeState(val title: String, val color: Color) {
Connected("Connected", Color(0xFF2ECC71)),
Connecting("Connecting…", Color(0xFFF1C40F)),
Error("Error", Color(0xFFE74C3C)),
Disconnected("Offline", Color(0xFF9E9E9E)),
}
@@ -1,254 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.foundation.horizontalScroll
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ArrowUpward
import androidx.compose.material.icons.filled.AttachFile
import androidx.compose.material.icons.filled.FolderOpen
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Stop
import androidx.compose.material3.ButtonDefaults
import androidx.compose.material3.DropdownMenu
import androidx.compose.material3.DropdownMenuItem
import androidx.compose.material3.FilledTonalButton
import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButtonDefaults
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.unit.dp
@Composable
fun ChatComposer(
sessionKey: String,
healthOk: Boolean,
thinkingLevel: String,
pendingRunCount: Int,
errorText: String?,
attachments: List<PendingImageAttachment>,
onPickImages: () -> Unit,
onRemoveAttachment: (id: String) -> Unit,
onSetThinkingLevel: (level: String) -> Unit,
onShowSessions: () -> Unit,
onRefresh: () -> Unit,
onAbort: () -> Unit,
onSend: (text: String) -> Unit,
) {
var input by rememberSaveable { mutableStateOf("") }
var showThinkingMenu by remember { mutableStateOf(false) }
val canSend = pendingRunCount == 0 && (input.trim().isNotEmpty() || attachments.isNotEmpty()) && healthOk
Surface(
shape = MaterialTheme.shapes.large,
color = MaterialTheme.colorScheme.surfaceContainer,
tonalElevation = 0.dp,
shadowElevation = 0.dp,
) {
Column(modifier = Modifier.padding(10.dp), verticalArrangement = Arrangement.spacedBy(8.dp)) {
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Box {
FilledTonalButton(
onClick = { showThinkingMenu = true },
contentPadding = ButtonDefaults.ContentPadding,
) {
Text("Thinking: ${thinkingLabel(thinkingLevel)}")
}
DropdownMenu(expanded = showThinkingMenu, onDismissRequest = { showThinkingMenu = false }) {
ThinkingMenuItem("off", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
ThinkingMenuItem("low", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
ThinkingMenuItem("medium", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
ThinkingMenuItem("high", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
}
}
Spacer(modifier = Modifier.weight(1f))
FilledTonalIconButton(onClick = onShowSessions, modifier = Modifier.size(42.dp)) {
Icon(Icons.Default.FolderOpen, contentDescription = "Sessions")
}
FilledTonalIconButton(onClick = onRefresh, modifier = Modifier.size(42.dp)) {
Icon(Icons.Default.Refresh, contentDescription = "Refresh")
}
FilledTonalIconButton(onClick = onPickImages, modifier = Modifier.size(42.dp)) {
Icon(Icons.Default.AttachFile, contentDescription = "Add image")
}
}
if (attachments.isNotEmpty()) {
AttachmentsStrip(attachments = attachments, onRemoveAttachment = onRemoveAttachment)
}
OutlinedTextField(
value = input,
onValueChange = { input = it },
modifier = Modifier.fillMaxWidth(),
placeholder = { Text("Message Clawd…") },
minLines = 2,
maxLines = 6,
)
Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) {
ConnectionPill(sessionKey = sessionKey, healthOk = healthOk)
Spacer(modifier = Modifier.weight(1f))
if (pendingRunCount > 0) {
FilledTonalIconButton(
onClick = onAbort,
colors =
IconButtonDefaults.filledTonalIconButtonColors(
containerColor = Color(0x33E74C3C),
contentColor = Color(0xFFE74C3C),
),
) {
Icon(Icons.Default.Stop, contentDescription = "Abort")
}
} else {
FilledTonalIconButton(onClick = {
val text = input
input = ""
onSend(text)
}, enabled = canSend) {
Icon(Icons.Default.ArrowUpward, contentDescription = "Send")
}
}
}
if (!errorText.isNullOrBlank()) {
Text(
text = errorText,
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.error,
maxLines = 2,
)
}
}
}
}
@Composable
private fun ConnectionPill(sessionKey: String, healthOk: Boolean) {
Surface(
shape = RoundedCornerShape(999.dp),
color = MaterialTheme.colorScheme.surfaceContainerHighest,
) {
Row(
modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Surface(
modifier = Modifier.size(7.dp),
shape = androidx.compose.foundation.shape.CircleShape,
color = if (healthOk) Color(0xFF2ECC71) else Color(0xFFF39C12),
) {}
Text(sessionKey, style = MaterialTheme.typography.labelSmall)
Text(
if (healthOk) "Connected" else "Connecting…",
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
}
@Composable
private fun ThinkingMenuItem(
value: String,
current: String,
onSet: (String) -> Unit,
onDismiss: () -> Unit,
) {
DropdownMenuItem(
text = { Text(thinkingLabel(value)) },
onClick = {
onSet(value)
onDismiss()
},
trailingIcon = {
if (value == current.trim().lowercase()) {
Text("✓")
} else {
Spacer(modifier = Modifier.width(10.dp))
}
},
)
}
private fun thinkingLabel(raw: String): String {
return when (raw.trim().lowercase()) {
"low" -> "Low"
"medium" -> "Medium"
"high" -> "High"
else -> "Off"
}
}
@Composable
private fun AttachmentsStrip(
attachments: List<PendingImageAttachment>,
onRemoveAttachment: (id: String) -> Unit,
) {
Row(
modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()),
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
for (att in attachments) {
AttachmentChip(
fileName = att.fileName,
onRemove = { onRemoveAttachment(att.id) },
)
}
}
}
@Composable
private fun AttachmentChip(fileName: String, onRemove: () -> Unit) {
Surface(
shape = RoundedCornerShape(999.dp),
color = MaterialTheme.colorScheme.primary.copy(alpha = 0.10f),
) {
Row(
modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Text(text = fileName, style = MaterialTheme.typography.bodySmall, maxLines = 1)
FilledTonalIconButton(
onClick = onRemove,
modifier = Modifier.size(30.dp),
) {
Text("×")
}
}
}
}
@@ -1,214 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import android.graphics.BitmapFactory
import android.util.Base64
import androidx.compose.foundation.Image
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.text.selection.SelectionContainer
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.AnnotatedString
import androidx.compose.ui.text.SpanStyle
import androidx.compose.ui.text.buildAnnotatedString
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontStyle
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.withStyle
import androidx.compose.ui.unit.dp
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
@Composable
fun ChatMarkdown(text: String) {
val blocks = remember(text) { splitMarkdown(text) }
val inlineCodeBg = MaterialTheme.colorScheme.surfaceContainerLow
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
for (b in blocks) {
when (b) {
is ChatMarkdownBlock.Text -> {
val trimmed = b.text.trimEnd()
if (trimmed.isEmpty()) continue
Text(
text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg),
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurface,
)
}
is ChatMarkdownBlock.Code -> {
SelectionContainer(modifier = Modifier.fillMaxWidth()) {
ChatCodeBlock(code = b.code, language = b.language)
}
}
is ChatMarkdownBlock.InlineImage -> {
InlineBase64Image(base64 = b.base64, mimeType = b.mimeType)
}
}
}
}
}
private sealed interface ChatMarkdownBlock {
data class Text(val text: String) : ChatMarkdownBlock
data class Code(val code: String, val language: String?) : ChatMarkdownBlock
data class InlineImage(val mimeType: String?, val base64: String) : ChatMarkdownBlock
}
private fun splitMarkdown(raw: String): List<ChatMarkdownBlock> {
if (raw.isEmpty()) return emptyList()
val out = ArrayList<ChatMarkdownBlock>()
var idx = 0
while (idx < raw.length) {
val fenceStart = raw.indexOf("```", startIndex = idx)
if (fenceStart < 0) {
out.addAll(splitInlineImages(raw.substring(idx)))
break
}
if (fenceStart > idx) {
out.addAll(splitInlineImages(raw.substring(idx, fenceStart)))
}
val langLineStart = fenceStart + 3
val langLineEnd = raw.indexOf('\n', startIndex = langLineStart).let { if (it < 0) raw.length else it }
val language = raw.substring(langLineStart, langLineEnd).trim().ifEmpty { null }
val codeStart = if (langLineEnd < raw.length && raw[langLineEnd] == '\n') langLineEnd + 1 else langLineEnd
val fenceEnd = raw.indexOf("```", startIndex = codeStart)
if (fenceEnd < 0) {
out.addAll(splitInlineImages(raw.substring(fenceStart)))
break
}
val code = raw.substring(codeStart, fenceEnd)
out.add(ChatMarkdownBlock.Code(code = code, language = language))
idx = fenceEnd + 3
}
return out
}
private fun splitInlineImages(text: String): List<ChatMarkdownBlock> {
if (text.isEmpty()) return emptyList()
val regex = Regex("data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)")
val out = ArrayList<ChatMarkdownBlock>()
var idx = 0
while (idx < text.length) {
val m = regex.find(text, startIndex = idx) ?: break
val start = m.range.first
val end = m.range.last + 1
if (start > idx) out.add(ChatMarkdownBlock.Text(text.substring(idx, start)))
val mime = "image/" + (m.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png")
val b64 = m.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty()
if (b64.isNotEmpty()) {
out.add(ChatMarkdownBlock.InlineImage(mimeType = mime, base64 = b64))
}
idx = end
}
if (idx < text.length) out.add(ChatMarkdownBlock.Text(text.substring(idx)))
return out
}
private fun parseInlineMarkdown(text: String, inlineCodeBg: androidx.compose.ui.graphics.Color): AnnotatedString {
if (text.isEmpty()) return AnnotatedString("")
val out = buildAnnotatedString {
var i = 0
while (i < text.length) {
if (text.startsWith("**", startIndex = i)) {
val end = text.indexOf("**", startIndex = i + 2)
if (end > i + 2) {
withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) {
append(text.substring(i + 2, end))
}
i = end + 2
continue
}
}
if (text[i] == '`') {
val end = text.indexOf('`', startIndex = i + 1)
if (end > i + 1) {
withStyle(
SpanStyle(
fontFamily = FontFamily.Monospace,
background = inlineCodeBg,
),
) {
append(text.substring(i + 1, end))
}
i = end + 1
continue
}
}
if (text[i] == '*' && (i + 1 < text.length && text[i + 1] != '*')) {
val end = text.indexOf('*', startIndex = i + 1)
if (end > i + 1) {
withStyle(SpanStyle(fontStyle = FontStyle.Italic)) {
append(text.substring(i + 1, end))
}
i = end + 1
continue
}
}
append(text[i])
i += 1
}
}
return out
}
@Composable
private fun InlineBase64Image(base64: String, mimeType: String?) {
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
var failed by remember(base64) { mutableStateOf(false) }
LaunchedEffect(base64) {
failed = false
image =
withContext(Dispatchers.Default) {
try {
val bytes = Base64.decode(base64, Base64.DEFAULT)
val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null
bitmap.asImageBitmap()
} catch (_: Throwable) {
null
}
}
if (image == null) failed = true
}
if (image != null) {
Image(
bitmap = image!!,
contentDescription = mimeType ?: "image",
contentScale = ContentScale.Fit,
modifier = Modifier.fillMaxWidth(),
)
} else if (failed) {
Text(
text = "Image unavailable",
modifier = Modifier.padding(vertical = 2.dp),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
@@ -1,111 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.rememberLazyListState
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ArrowCircleDown
import androidx.compose.material3.Card
import androidx.compose.material3.CardDefaults
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.chat.ChatMessage
import com.steipete.clawdis.node.chat.ChatPendingToolCall
@Composable
fun ChatMessageListCard(
messages: List<ChatMessage>,
pendingRunCount: Int,
pendingToolCalls: List<ChatPendingToolCall>,
streamingAssistantText: String?,
modifier: Modifier = Modifier,
) {
val listState = rememberLazyListState()
LaunchedEffect(messages.size, pendingRunCount, pendingToolCalls.size, streamingAssistantText) {
val total =
messages.size +
(if (pendingRunCount > 0) 1 else 0) +
(if (pendingToolCalls.isNotEmpty()) 1 else 0) +
(if (!streamingAssistantText.isNullOrBlank()) 1 else 0)
if (total <= 0) return@LaunchedEffect
listState.animateScrollToItem(index = total - 1)
}
Card(
modifier = modifier.fillMaxWidth(),
shape = MaterialTheme.shapes.large,
colors =
CardDefaults.cardColors(
containerColor = MaterialTheme.colorScheme.surfaceContainer,
),
elevation = CardDefaults.cardElevation(defaultElevation = 0.dp),
) {
Box(modifier = Modifier.fillMaxSize()) {
LazyColumn(
modifier = Modifier.fillMaxSize(),
state = listState,
verticalArrangement = Arrangement.spacedBy(14.dp),
contentPadding = androidx.compose.foundation.layout.PaddingValues(top = 12.dp, bottom = 12.dp, start = 12.dp, end = 12.dp),
) {
items(count = messages.size, key = { idx -> messages[idx].id }) { idx ->
ChatMessageBubble(message = messages[idx])
}
if (pendingRunCount > 0) {
item(key = "typing") {
ChatTypingIndicatorBubble()
}
}
if (pendingToolCalls.isNotEmpty()) {
item(key = "tools") {
ChatPendingToolsBubble(toolCalls = pendingToolCalls)
}
}
val stream = streamingAssistantText?.trim()
if (!stream.isNullOrEmpty()) {
item(key = "stream") {
ChatStreamingAssistantBubble(text = stream)
}
}
}
if (messages.isEmpty() && pendingRunCount == 0 && pendingToolCalls.isEmpty() && streamingAssistantText.isNullOrBlank()) {
EmptyChatHint(modifier = Modifier.align(Alignment.Center))
}
}
}
}
@Composable
private fun EmptyChatHint(modifier: Modifier = Modifier) {
Row(
modifier = modifier.alpha(0.7f),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Icon(
imageVector = Icons.Default.ArrowCircleDown,
contentDescription = null,
tint = MaterialTheme.colorScheme.onSurfaceVariant,
)
Text(
text = "Message Clawd…",
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
@@ -1,218 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import android.graphics.BitmapFactory
import android.util.Base64
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.unit.dp
import androidx.compose.foundation.Image
import com.steipete.clawdis.node.chat.ChatMessage
import com.steipete.clawdis.node.chat.ChatMessageContent
import com.steipete.clawdis.node.chat.ChatPendingToolCall
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
@Composable
fun ChatMessageBubble(message: ChatMessage) {
val isUser = message.role.lowercase() == "user"
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start,
) {
Surface(
shape = RoundedCornerShape(16.dp),
tonalElevation = 0.dp,
shadowElevation = 0.dp,
color = Color.Transparent,
modifier = Modifier.fillMaxWidth(0.92f),
) {
Box(
modifier =
Modifier
.background(bubbleBackground(isUser))
.padding(horizontal = 12.dp, vertical = 10.dp),
) {
ChatMessageBody(content = message.content)
}
}
}
}
@Composable
private fun ChatMessageBody(content: List<ChatMessageContent>) {
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
for (part in content) {
when (part.type) {
"text" -> {
val text = part.text ?: continue
ChatMarkdown(text = text)
}
else -> {
val b64 = part.base64 ?: continue
ChatBase64Image(base64 = b64, mimeType = part.mimeType)
}
}
}
}
}
@Composable
fun ChatTypingIndicatorBubble() {
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) {
Surface(
shape = RoundedCornerShape(16.dp),
color = MaterialTheme.colorScheme.surfaceContainer,
) {
Row(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
DotPulse()
Text("Thinking…", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
}
}
@Composable
fun ChatPendingToolsBubble(toolCalls: List<ChatPendingToolCall>) {
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) {
Surface(
shape = RoundedCornerShape(16.dp),
color = MaterialTheme.colorScheme.surfaceContainer,
) {
Column(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), verticalArrangement = Arrangement.spacedBy(6.dp)) {
Text("Tools", style = MaterialTheme.typography.labelLarge, color = MaterialTheme.colorScheme.onSurface)
for (t in toolCalls.take(6)) {
Text("· ${t.name}", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
if (toolCalls.size > 6) {
Text("… +${toolCalls.size - 6} more", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
}
}
}
@Composable
fun ChatStreamingAssistantBubble(text: String) {
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) {
Surface(
shape = RoundedCornerShape(16.dp),
color = MaterialTheme.colorScheme.surfaceContainer,
) {
Box(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp)) {
ChatMarkdown(text = text)
}
}
}
}
@Composable
private fun bubbleBackground(isUser: Boolean): Brush {
return if (isUser) {
Brush.linearGradient(
colors = listOf(MaterialTheme.colorScheme.primary, MaterialTheme.colorScheme.primary.copy(alpha = 0.78f)),
)
} else {
Brush.linearGradient(
colors = listOf(MaterialTheme.colorScheme.surfaceContainer, MaterialTheme.colorScheme.surfaceContainerHigh),
)
}
}
@Composable
private fun ChatBase64Image(base64: String, mimeType: String?) {
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
var failed by remember(base64) { mutableStateOf(false) }
LaunchedEffect(base64) {
failed = false
image =
withContext(Dispatchers.Default) {
try {
val bytes = Base64.decode(base64, Base64.DEFAULT)
val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null
bitmap.asImageBitmap()
} catch (_: Throwable) {
null
}
}
if (image == null) failed = true
}
if (image != null) {
Image(
bitmap = image!!,
contentDescription = mimeType ?: "attachment",
contentScale = ContentScale.Fit,
modifier = Modifier.fillMaxWidth(),
)
} else if (failed) {
Text("Unsupported attachment", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
@Composable
private fun DotPulse() {
Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) {
PulseDot(alpha = 0.38f)
PulseDot(alpha = 0.62f)
PulseDot(alpha = 0.90f)
}
}
@Composable
private fun PulseDot(alpha: Float) {
Surface(
modifier = Modifier.size(6.dp).alpha(alpha),
shape = CircleShape,
color = MaterialTheme.colorScheme.onSurfaceVariant,
) {}
}
@Composable
fun ChatCodeBlock(code: String, language: String?) {
Surface(
shape = RoundedCornerShape(12.dp),
color = MaterialTheme.colorScheme.surfaceContainerLowest,
modifier = Modifier.fillMaxWidth(),
) {
Text(
text = code.trimEnd(),
modifier = Modifier.padding(10.dp),
fontFamily = FontFamily.Monospace,
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurface,
)
}
}
@@ -1,93 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material3.AlertDialog
import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.chat.ChatSessionEntry
@Composable
fun ChatSessionsDialog(
currentSessionKey: String,
sessions: List<ChatSessionEntry>,
onDismiss: () -> Unit,
onRefresh: () -> Unit,
onSelect: (sessionKey: String) -> Unit,
) {
AlertDialog(
onDismissRequest = onDismiss,
confirmButton = {},
title = {
Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth()) {
Text("Sessions", style = MaterialTheme.typography.titleMedium)
Spacer(modifier = Modifier.weight(1f))
FilledTonalIconButton(onClick = onRefresh) {
Icon(Icons.Default.Refresh, contentDescription = "Refresh")
}
}
},
text = {
if (sessions.isEmpty()) {
Text("No sessions", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
} else {
LazyColumn(verticalArrangement = Arrangement.spacedBy(8.dp)) {
items(sessions, key = { it.key }) { entry ->
SessionRow(
entry = entry,
isCurrent = entry.key == currentSessionKey,
onClick = { onSelect(entry.key) },
)
}
}
}
},
)
}
@Composable
private fun SessionRow(
entry: ChatSessionEntry,
isCurrent: Boolean,
onClick: () -> Unit,
) {
Surface(
onClick = onClick,
shape = MaterialTheme.shapes.medium,
color =
if (isCurrent) {
MaterialTheme.colorScheme.primary.copy(alpha = 0.14f)
} else {
MaterialTheme.colorScheme.surfaceContainer
},
modifier = Modifier.fillMaxWidth(),
) {
Row(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(10.dp),
) {
Text(entry.key, style = MaterialTheme.typography.bodyMedium)
Spacer(modifier = Modifier.weight(1f))
if (isCurrent) {
Text("Current", style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
}
}
@@ -1,157 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import android.content.ContentResolver
import android.net.Uri
import android.util.Base64
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateListOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.MainViewModel
import com.steipete.clawdis.node.chat.OutgoingAttachment
import java.io.ByteArrayOutputStream
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
@Composable
fun ChatSheetContent(viewModel: MainViewModel) {
val messages by viewModel.chatMessages.collectAsState()
val errorText by viewModel.chatError.collectAsState()
val pendingRunCount by viewModel.pendingRunCount.collectAsState()
val healthOk by viewModel.chatHealthOk.collectAsState()
val sessionKey by viewModel.chatSessionKey.collectAsState()
val thinkingLevel by viewModel.chatThinkingLevel.collectAsState()
val streamingAssistantText by viewModel.chatStreamingAssistantText.collectAsState()
val pendingToolCalls by viewModel.chatPendingToolCalls.collectAsState()
val sessions by viewModel.chatSessions.collectAsState()
var showSessions by remember { mutableStateOf(false) }
LaunchedEffect(Unit) {
viewModel.loadChat("main")
}
val context = LocalContext.current
val resolver = context.contentResolver
val scope = rememberCoroutineScope()
val attachments = remember { mutableStateListOf<PendingImageAttachment>() }
val pickImages =
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult
scope.launch(Dispatchers.IO) {
val next =
uris.take(8).mapNotNull { uri ->
try {
loadImageAttachment(resolver, uri)
} catch (_: Throwable) {
null
}
}
withContext(Dispatchers.Main) {
attachments.addAll(next)
}
}
}
Column(
modifier =
Modifier
.fillMaxSize()
.padding(horizontal = 12.dp, vertical = 12.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
ChatMessageListCard(
messages = messages,
pendingRunCount = pendingRunCount,
pendingToolCalls = pendingToolCalls,
streamingAssistantText = streamingAssistantText,
modifier = Modifier.weight(1f, fill = true),
)
ChatComposer(
sessionKey = sessionKey,
healthOk = healthOk,
thinkingLevel = thinkingLevel,
pendingRunCount = pendingRunCount,
errorText = errorText,
attachments = attachments,
onPickImages = { pickImages.launch("image/*") },
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) },
onShowSessions = { showSessions = true },
onRefresh = { viewModel.refreshChat() },
onAbort = { viewModel.abortChat() },
onSend = { text ->
val outgoing =
attachments.map { att ->
OutgoingAttachment(
type = "image",
mimeType = att.mimeType,
fileName = att.fileName,
base64 = att.base64,
)
}
viewModel.sendChat(message = text, thinking = thinkingLevel, attachments = outgoing)
attachments.clear()
},
)
}
if (showSessions) {
ChatSessionsDialog(
currentSessionKey = sessionKey,
sessions = sessions,
onDismiss = { showSessions = false },
onRefresh = { viewModel.refreshChatSessions(limit = 50) },
onSelect = { key ->
viewModel.switchChatSession(key)
showSessions = false
},
)
}
}
data class PendingImageAttachment(
val id: String,
val fileName: String,
val mimeType: String,
val base64: String,
)
private suspend fun loadImageAttachment(resolver: ContentResolver, uri: Uri): PendingImageAttachment {
val mimeType = resolver.getType(uri) ?: "image/*"
val fileName = (uri.lastPathSegment ?: "image").substringAfterLast('/')
val bytes =
withContext(Dispatchers.IO) {
resolver.openInputStream(uri)?.use { input ->
val out = ByteArrayOutputStream()
input.copyTo(out)
out.toByteArray()
} ?: ByteArray(0)
}
if (bytes.isEmpty()) throw IllegalStateException("empty attachment")
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
return PendingImageAttachment(
id = uri.toString() + "#" + System.currentTimeMillis().toString(),
fileName = fileName,
mimeType = mimeType,
base64 = base64,
)
}
@@ -1,40 +0,0 @@
package com.steipete.clawdis.node.voice
object VoiceWakeCommandExtractor {
fun extractCommand(text: String, triggerWords: List<String>): String? {
val raw = text.trim()
if (raw.isEmpty()) return null
val triggers =
triggerWords
.map { it.trim().lowercase() }
.filter { it.isNotEmpty() }
.distinct()
if (triggers.isEmpty()) return null
val alternation = triggers.joinToString("|") { Regex.escape(it) }
// Match: "<anything> <trigger><punct/space> <command>"
val regex = Regex("(?i)(?:^|\\s)($alternation)\\b[\\s\\p{Punct}]*([\\s\\S]+)$")
val match = regex.find(raw) ?: return null
val extracted = match.groupValues.getOrNull(2)?.trim().orEmpty()
if (extracted.isEmpty()) return null
val cleaned = extracted.trimStart { it.isWhitespace() || it.isPunctuation() }.trim()
if (cleaned.isEmpty()) return null
return cleaned
}
}
private fun Char.isPunctuation(): Boolean {
return when (Character.getType(this)) {
Character.CONNECTOR_PUNCTUATION.toInt(),
Character.DASH_PUNCTUATION.toInt(),
Character.START_PUNCTUATION.toInt(),
Character.END_PUNCTUATION.toInt(),
Character.INITIAL_QUOTE_PUNCTUATION.toInt(),
Character.FINAL_QUOTE_PUNCTUATION.toInt(),
Character.OTHER_PUNCTUATION.toInt(),
-> true
else -> false
}
}
@@ -1,173 +0,0 @@
package com.steipete.clawdis.node.voice
import android.content.Context
import android.content.Intent
import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.speech.RecognitionListener
import android.speech.RecognizerIntent
import android.speech.SpeechRecognizer
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch
class VoiceWakeManager(
private val context: Context,
private val scope: CoroutineScope,
private val onCommand: suspend (String) -> Unit,
) {
private val mainHandler = Handler(Looper.getMainLooper())
private val _isListening = MutableStateFlow(false)
val isListening: StateFlow<Boolean> = _isListening
private val _statusText = MutableStateFlow("Off")
val statusText: StateFlow<String> = _statusText
var triggerWords: List<String> = emptyList()
private set
private var recognizer: SpeechRecognizer? = null
private var restartJob: Job? = null
private var lastDispatched: String? = null
private var stopRequested = false
fun setTriggerWords(words: List<String>) {
triggerWords = words
}
fun start() {
mainHandler.post {
if (_isListening.value) return@post
stopRequested = false
if (!SpeechRecognizer.isRecognitionAvailable(context)) {
_isListening.value = false
_statusText.value = "Speech recognizer unavailable"
return@post
}
try {
recognizer?.destroy()
recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) }
startListeningInternal()
} catch (err: Throwable) {
_isListening.value = false
_statusText.value = "Start failed: ${err.message ?: err::class.simpleName}"
}
}
}
fun stop(statusText: String = "Off") {
stopRequested = true
restartJob?.cancel()
restartJob = null
mainHandler.post {
_isListening.value = false
_statusText.value = statusText
recognizer?.cancel()
recognizer?.destroy()
recognizer = null
}
}
private fun startListeningInternal() {
val r = recognizer ?: return
val intent =
Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3)
putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName)
}
_statusText.value = "Listening"
_isListening.value = true
r.startListening(intent)
}
private fun scheduleRestart(delayMs: Long = 350) {
if (stopRequested) return
restartJob?.cancel()
restartJob =
scope.launch {
delay(delayMs)
mainHandler.post {
if (stopRequested) return@post
try {
recognizer?.cancel()
startListeningInternal()
} catch (_: Throwable) {
// Will be picked up by onError and retry again.
}
}
}
}
private fun handleTranscription(text: String) {
val command = VoiceWakeCommandExtractor.extractCommand(text, triggerWords) ?: return
if (command == lastDispatched) return
lastDispatched = command
scope.launch { onCommand(command) }
_statusText.value = "Triggered"
scheduleRestart(delayMs = 650)
}
private val listener =
object : RecognitionListener {
override fun onReadyForSpeech(params: Bundle?) {
_statusText.value = "Listening"
}
override fun onBeginningOfSpeech() {}
override fun onRmsChanged(rmsdB: Float) {}
override fun onBufferReceived(buffer: ByteArray?) {}
override fun onEndOfSpeech() {
scheduleRestart()
}
override fun onError(error: Int) {
if (stopRequested) return
_isListening.value = false
if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) {
_statusText.value = "Microphone permission required"
return
}
_statusText.value =
when (error) {
SpeechRecognizer.ERROR_AUDIO -> "Audio error"
SpeechRecognizer.ERROR_CLIENT -> "Client error"
SpeechRecognizer.ERROR_NETWORK -> "Network error"
SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout"
SpeechRecognizer.ERROR_NO_MATCH -> "Listening"
SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy"
SpeechRecognizer.ERROR_SERVER -> "Server error"
SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening"
else -> "Speech error ($error)"
}
scheduleRestart(delayMs = 600)
}
override fun onResults(results: Bundle?) {
val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
list.firstOrNull()?.let(::handleTranscription)
scheduleRestart()
}
override fun onPartialResults(partialResults: Bundle?) {
val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
list.firstOrNull()?.let(::handleTranscription)
}
override fun onEvent(eventType: Int, params: Bundle?) {}
}
}
@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@color/ic_launcher_background" />
<foreground android:drawable="@mipmap/ic_launcher_foreground" />
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
</adaptive-icon>
@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@color/ic_launcher_background" />
<foreground android:drawable="@mipmap/ic_launcher_foreground" />
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
</adaptive-icon>
Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 67 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 32 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 148 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 267 KiB

@@ -1,4 +0,0 @@
<resources>
<color name="ic_launcher_background">#0A0A0A</color>
</resources>
@@ -1,7 +1,8 @@
<resources>
<resources xmlns:tools="http://schemas.android.com/tools">
<style name="Theme.ClawdisNode" parent="Theme.Material3.DayNight.NoActionBar">
<item name="android:statusBarColor">@android:color/transparent</item>
<item name="android:navigationBarColor">@android:color/transparent</item>
<item name="android:windowLightStatusBar">false</item>
<item name="android:windowLightStatusBar" tools:targetApi="m">false</item>
</style>
</resources>
@@ -1,4 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<full-backup-content>
<include domain="file" path="." />
</full-backup-content>
@@ -1,9 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<data-extraction-rules>
<cloud-backup>
<include domain="file" path="." />
</cloud-backup>
<device-transfer>
<include domain="file" path="." />
</device-transfer>
</data-extraction-rules>
@@ -1,12 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<network-security-config xmlns:tools="http://schemas.android.com/tools">
<!-- This app is primarily used on a trusted tailnet; allow cleartext for IP-based endpoints too. -->
<base-config cleartextTrafficPermitted="true" tools:ignore="InsecureBaseConfiguration" />
<!-- Allow HTTP for tailnet/local dev endpoints (e.g. canvas/background web). -->
<domain-config cleartextTrafficPermitted="true">
<domain includeSubdomains="true">clawdis.internal</domain>
</domain-config>
<domain-config cleartextTrafficPermitted="true">
<domain includeSubdomains="true">ts.net</domain>
</domain-config>
</network-security-config>
@@ -1,36 +0,0 @@
package com.steipete.clawdis.node
import org.junit.Assert.assertEquals
import org.junit.Test
class WakeWordsTest {
@Test
fun parseCommaSeparatedTrimsAndDropsEmpty() {
assertEquals(listOf("clawd", "claude"), WakeWords.parseCommaSeparated(" clawd , claude, , "))
}
@Test
fun sanitizeTrimsCapsAndFallsBack() {
val defaults = listOf("clawd", "claude")
val long = "x".repeat(WakeWords.maxWordLength + 10)
val words = listOf(" ", " hello ", long)
val sanitized = WakeWords.sanitize(words, defaults)
assertEquals(2, sanitized.size)
assertEquals("hello", sanitized[0])
assertEquals("x".repeat(WakeWords.maxWordLength), sanitized[1])
assertEquals(defaults, WakeWords.sanitize(listOf(" ", ""), defaults))
}
@Test
fun sanitizeLimitsWordCount() {
val defaults = listOf("clawd")
val words = (1..(WakeWords.maxWords + 5)).map { "w$it" }
val sanitized = WakeWords.sanitize(words, defaults)
assertEquals(WakeWords.maxWords, sanitized.size)
assertEquals("w1", sanitized.first())
assertEquals("w${WakeWords.maxWords}", sanitized.last())
}
}
@@ -1,19 +0,0 @@
package com.steipete.clawdis.node.bridge
import org.junit.Assert.assertEquals
import org.junit.Test
class BonjourEscapesTest {
@Test
fun decodeNoop() {
assertEquals("", BonjourEscapes.decode(""))
assertEquals("hello", BonjourEscapes.decode("hello"))
}
@Test
fun decodeDecodesDecimalEscapes() {
assertEquals("Clawdis Gateway", BonjourEscapes.decode("Clawdis\\032Gateway"))
assertEquals("A B", BonjourEscapes.decode("A\\032B"))
assertEquals("Peter\u2019s Mac", BonjourEscapes.decode("Peter\\226\\128\\153s Mac"))
}
}
@@ -46,10 +46,6 @@ class BridgePairingClientTest {
token = "token-123",
platform = "Android",
version = "test",
deviceFamily = "Android",
modelIdentifier = "SM-X000",
caps = null,
commands = null,
),
)
assertTrue(res.ok)
@@ -95,10 +91,6 @@ class BridgePairingClientTest {
token = null,
platform = "Android",
version = "test",
deviceFamily = "Android",
modelIdentifier = "SM-X000",
caps = null,
commands = null,
),
)
assertTrue(res.ok)
@@ -106,3 +98,4 @@ class BridgePairingClientTest {
server.await()
}
}
@@ -15,8 +15,6 @@ import java.io.BufferedWriter
import java.io.InputStreamReader
import java.io.OutputStreamWriter
import java.net.ServerSocket
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
class BridgeSessionTest {
@Test
@@ -46,7 +44,7 @@ class BridgeSessionTest {
val hello = reader.readLine()
assertTrue(hello.contains("\"type\":\"hello\""))
writer.write("""{"type":"hello-ok","serverName":"Test Bridge","canvasHostUrl":"http://127.0.0.1:18789"}""")
writer.write("""{"type":"hello-ok","serverName":"Test Bridge"}""")
writer.write("\n")
writer.flush()
@@ -69,15 +67,10 @@ class BridgeSessionTest {
token = null,
platform = "Android",
version = "test",
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
connected.await()
assertEquals("http://127.0.0.1:18789", session.currentCanvasHostUrl())
val payload = session.request(method = "health", paramsJson = null)
assertEquals("""{"value":123}""", payload)
server.await()
@@ -136,10 +129,6 @@ class BridgeSessionTest {
token = null,
platform = "Android",
version = "test",
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
connected.await()
@@ -188,7 +177,7 @@ class BridgeSessionTest {
writer.flush()
// Ask the node to invoke something; handler will throw.
writer.write("""{"type":"invoke","id":"i1","command":"canvas.snapshot","paramsJSON":null}""")
writer.write("""{"type":"invoke","id":"i1","command":"screen.snapshot","paramsJSON":null}""")
writer.write("\n")
writer.flush()
@@ -207,10 +196,6 @@ class BridgeSessionTest {
token = null,
platform = "Android",
version = "test",
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
connected.await()
@@ -226,74 +211,6 @@ class BridgeSessionTest {
session.disconnect()
scope.cancel()
}
@Test(timeout = 12_000)
fun reconnectsAfterBridgeClosesDuringHello() = runBlocking {
val serverSocket = ServerSocket(0)
val port = serverSocket.localPort
val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
val connected = CountDownLatch(1)
val connectionsSeen = CountDownLatch(2)
val session =
BridgeSession(
scope = scope,
onConnected = { _, _ -> connected.countDown() },
onDisconnected = { /* ignore */ },
onEvent = { _, _ -> /* ignore */ },
onInvoke = { BridgeSession.InvokeResult.ok(null) },
)
val server =
async(Dispatchers.IO) {
serverSocket.use { ss ->
// First connection: read hello, then close (no response).
val sock1 = ss.accept()
sock1.use { s ->
val reader = BufferedReader(InputStreamReader(s.getInputStream(), Charsets.UTF_8))
reader.readLine() // hello
connectionsSeen.countDown()
}
// Second connection: complete hello.
val sock2 = ss.accept()
sock2.use { s ->
val reader = BufferedReader(InputStreamReader(s.getInputStream(), Charsets.UTF_8))
val writer = BufferedWriter(OutputStreamWriter(s.getOutputStream(), Charsets.UTF_8))
reader.readLine() // hello
writer.write("""{"type":"hello-ok","serverName":"Test Bridge"}""")
writer.write("\n")
writer.flush()
connectionsSeen.countDown()
Thread.sleep(200)
}
}
}
session.connect(
endpoint = BridgeEndpoint.manual(host = "127.0.0.1", port = port),
hello =
BridgeSession.Hello(
nodeId = "node-1",
displayName = "Android Node",
token = null,
platform = "Android",
version = "test",
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
assertTrue("expected two connection attempts", connectionsSeen.await(8, TimeUnit.SECONDS))
assertTrue("expected session to connect", connected.await(8, TimeUnit.SECONDS))
session.disconnect()
scope.cancel()
server.await()
}
}
private fun extractJsonString(raw: String, key: String): String {
@@ -1,43 +0,0 @@
package com.steipete.clawdis.node.node
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
class CanvasControllerSnapshotParamsTest {
@Test
fun parseSnapshotParamsDefaultsToJpeg() {
val params = CanvasController.parseSnapshotParams(null)
assertEquals(CanvasController.SnapshotFormat.Jpeg, params.format)
assertNull(params.quality)
assertNull(params.maxWidth)
}
@Test
fun parseSnapshotParamsParsesPng() {
val params = CanvasController.parseSnapshotParams("""{"format":"png","maxWidth":900}""")
assertEquals(CanvasController.SnapshotFormat.Png, params.format)
assertEquals(900, params.maxWidth)
}
@Test
fun parseSnapshotParamsParsesJpegAliases() {
assertEquals(
CanvasController.SnapshotFormat.Jpeg,
CanvasController.parseSnapshotParams("""{"format":"jpeg"}""").format,
)
assertEquals(
CanvasController.SnapshotFormat.Jpeg,
CanvasController.parseSnapshotParams("""{"format":"jpg"}""").format,
)
}
@Test
fun parseSnapshotParamsClampsQuality() {
val low = CanvasController.parseSnapshotParams("""{"quality":0.01}""")
assertEquals(0.1, low.quality)
val high = CanvasController.parseSnapshotParams("""{"quality":5}""")
assertEquals(1.0, high.quality)
}
}
@@ -1,49 +0,0 @@
package com.steipete.clawdis.node.protocol
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonObject
import org.junit.Assert.assertEquals
import org.junit.Test
class ClawdisCanvasA2UIActionTest {
@Test
fun extractActionNameAcceptsNameOrAction() {
val nameObj = Json.parseToJsonElement("{\"name\":\"Hello\"}").jsonObject
assertEquals("Hello", ClawdisCanvasA2UIAction.extractActionName(nameObj))
val actionObj = Json.parseToJsonElement("{\"action\":\"Wave\"}").jsonObject
assertEquals("Wave", ClawdisCanvasA2UIAction.extractActionName(actionObj))
val fallbackObj =
Json.parseToJsonElement("{\"name\":\" \",\"action\":\"Fallback\"}").jsonObject
assertEquals("Fallback", ClawdisCanvasA2UIAction.extractActionName(fallbackObj))
}
@Test
fun formatAgentMessageMatchesSharedSpec() {
val msg =
ClawdisCanvasA2UIAction.formatAgentMessage(
actionName = "Get Weather",
sessionKey = "main",
surfaceId = "main",
sourceComponentId = "btnWeather",
host = "Peter’s iPad",
instanceId = "ipad16,6",
contextJson = "{\"city\":\"Vienna\"}",
)
assertEquals(
"CANVAS_A2UI action=Get_Weather session=main surface=main component=btnWeather host=Peter_s_iPad instance=ipad16_6 ctx={\"city\":\"Vienna\"} default=update_canvas",
msg,
)
}
@Test
fun jsDispatchA2uiStatusIsStable() {
val js = ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId = "a1", ok = true, error = null)
assertEquals(
"window.dispatchEvent(new CustomEvent('clawdis:a2ui-action-status', { detail: { id: \"a1\", ok: true, error: \"\" } }));",
js,
)
}
}
@@ -1,35 +0,0 @@
package com.steipete.clawdis.node.protocol
import org.junit.Assert.assertEquals
import org.junit.Test
class ClawdisProtocolConstantsTest {
@Test
fun canvasCommandsUseStableStrings() {
assertEquals("canvas.present", ClawdisCanvasCommand.Present.rawValue)
assertEquals("canvas.hide", ClawdisCanvasCommand.Hide.rawValue)
assertEquals("canvas.navigate", ClawdisCanvasCommand.Navigate.rawValue)
assertEquals("canvas.eval", ClawdisCanvasCommand.Eval.rawValue)
assertEquals("canvas.snapshot", ClawdisCanvasCommand.Snapshot.rawValue)
}
@Test
fun a2uiCommandsUseStableStrings() {
assertEquals("canvas.a2ui.push", ClawdisCanvasA2UICommand.Push.rawValue)
assertEquals("canvas.a2ui.pushJSONL", ClawdisCanvasA2UICommand.PushJSONL.rawValue)
assertEquals("canvas.a2ui.reset", ClawdisCanvasA2UICommand.Reset.rawValue)
}
@Test
fun capabilitiesUseStableStrings() {
assertEquals("canvas", ClawdisCapability.Canvas.rawValue)
assertEquals("camera", ClawdisCapability.Camera.rawValue)
assertEquals("screen", ClawdisCapability.Screen.rawValue)
assertEquals("voiceWake", ClawdisCapability.VoiceWake.rawValue)
}
@Test
fun screenCommandsUseStableStrings() {
assertEquals("screen.record", ClawdisScreenCommand.Record.rawValue)
}
}
@@ -1,26 +0,0 @@
package com.steipete.clawdis.node.voice
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
class VoiceWakeCommandExtractorTest {
@Test
fun extractsCommandAfterTriggerWord() {
val res = VoiceWakeCommandExtractor.extractCommand("Claude take a photo", listOf("clawd", "claude"))
assertEquals("take a photo", res)
}
@Test
fun extractsCommandWithPunctuation() {
val res = VoiceWakeCommandExtractor.extractCommand("hey clawd, what's the weather?", listOf("clawd"))
assertEquals("what's the weather?", res)
}
@Test
fun returnsNullWhenNoCommandProvided() {
assertNull(VoiceWakeCommandExtractor.extractCommand("claude", listOf("claude")))
assertNull(VoiceWakeCommandExtractor.extractCommand("hey claude!", listOf("claude")))
}
}
+2 -2
View File
@@ -1,4 +1,4 @@
org.gradle.jvmargs=-Xmx3g -Dfile.encoding=UTF-8 --enable-native-access=ALL-UNNAMED
org.gradle.warning.mode=none
org.gradle.jvmargs=-Xmx3g -Dfile.encoding=UTF-8
android.useAndroidX=true
android.nonTransitiveRClass=true
+1 -1
View File
@@ -200,7 +200,7 @@ fi
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m" "--enable-native-access=ALL-UNNAMED"'
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+1 -1
View File
@@ -34,7 +34,7 @@ set APP_HOME=%DIRNAME%
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" "--enable-native-access=ALL-UNNAMED"
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
+1 -5
View File
@@ -51,11 +51,7 @@ actor BridgeClient {
nodeId: hello.nodeId,
displayName: hello.displayName,
platform: hello.platform,
version: hello.version,
deviceFamily: hello.deviceFamily,
modelIdentifier: hello.modelIdentifier,
caps: hello.caps,
commands: hello.commands),
version: hello.version),
over: connection)
onStatus?("Waiting for approval…")
@@ -1,20 +1,17 @@
import ClawdisKit
import Darwin
import Combine
import Foundation
import Network
import Observation
import SwiftUI
import UIKit
@MainActor
@Observable
final class BridgeConnectionController {
private(set) var bridges: [BridgeDiscoveryModel.DiscoveredBridge] = []
private(set) var discoveryStatusText: String = "Idle"
private(set) var discoveryDebugLog: [BridgeDiscoveryModel.DebugLogEntry] = []
final class BridgeConnectionController: ObservableObject {
@Published private(set) var bridges: [BridgeDiscoveryModel.DiscoveredBridge] = []
@Published private(set) var discoveryStatusText: String = "Idle"
private let discovery = BridgeDiscoveryModel()
private weak var appModel: NodeAppModel?
private var cancellables = Set<AnyCancellable>()
private var didAutoConnect = false
private var seenStableIDs = Set<String>()
@@ -22,21 +19,24 @@ final class BridgeConnectionController {
self.appModel = appModel
BridgeSettingsStore.bootstrapPersistence()
let defaults = UserDefaults.standard
self.discovery.setDebugLoggingEnabled(defaults.bool(forKey: "bridge.discovery.debugLogs"))
self.updateFromDiscovery()
self.observeDiscovery()
self.discovery.$bridges
.sink { [weak self] newValue in
guard let self else { return }
self.bridges = newValue
self.updateLastDiscoveredBridge(from: newValue)
self.maybeAutoConnect()
}
.store(in: &self.cancellables)
self.discovery.$statusText
.assign(to: &self.$discoveryStatusText)
if startDiscovery {
self.discovery.start()
}
}
func setDiscoveryDebugLoggingEnabled(_ enabled: Bool) {
self.discovery.setDebugLoggingEnabled(enabled)
}
func setScenePhase(_ phase: ScenePhase) {
switch phase {
case .background:
@@ -48,29 +48,6 @@ final class BridgeConnectionController {
}
}
private func updateFromDiscovery() {
let newBridges = self.discovery.bridges
self.bridges = newBridges
self.discoveryStatusText = self.discovery.statusText
self.discoveryDebugLog = self.discovery.debugLog
self.updateLastDiscoveredBridge(from: newBridges)
self.maybeAutoConnect()
}
private func observeDiscovery() {
withObservationTracking {
_ = self.discovery.bridges
_ = self.discovery.statusText
_ = self.discovery.debugLog
} onChange: { [weak self] in
Task { @MainActor in
guard let self else { return }
self.updateFromDiscovery()
self.observeDiscovery()
}
}
}
private func maybeAutoConnect() {
guard !self.didAutoConnect else { return }
guard let appModel = self.appModel else { return }
@@ -126,144 +103,22 @@ final class BridgeConnectionController {
private func makeHello(token: String) -> BridgeHello {
let defaults = UserDefaults.standard
let nodeId = defaults.string(forKey: "node.instanceId") ?? "ios-node"
let displayName = self.resolvedDisplayName(defaults: defaults)
let displayName = defaults.string(forKey: "node.displayName") ?? "iOS Node"
return BridgeHello(
nodeId: nodeId,
displayName: displayName,
token: token,
platform: self.platformString(),
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands())
}
private func resolvedDisplayName(defaults: UserDefaults) -> String {
let key = "node.displayName"
let existing = defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if !existing.isEmpty, existing != "iOS Node" { return existing }
let deviceName = UIDevice.current.name.trimmingCharacters(in: .whitespacesAndNewlines)
let candidate = deviceName.isEmpty ? "iOS Node" : deviceName
if existing.isEmpty || existing == "iOS Node" {
defaults.set(candidate, forKey: key)
}
return candidate
}
private func currentCaps() -> [String] {
var caps = [ClawdisCapability.canvas.rawValue, ClawdisCapability.screen.rawValue]
// Default-on: if the key doesn't exist yet, treat it as enabled.
let cameraEnabled =
UserDefaults.standard.object(forKey: "camera.enabled") == nil
? true
: UserDefaults.standard.bool(forKey: "camera.enabled")
if cameraEnabled { caps.append(ClawdisCapability.camera.rawValue) }
let voiceWakeEnabled = UserDefaults.standard.bool(forKey: VoiceWakePreferences.enabledKey)
if voiceWakeEnabled { caps.append(ClawdisCapability.voiceWake.rawValue) }
return caps
}
private func currentCommands() -> [String] {
var commands: [String] = [
ClawdisCanvasCommand.present.rawValue,
ClawdisCanvasCommand.hide.rawValue,
ClawdisCanvasCommand.navigate.rawValue,
ClawdisCanvasCommand.evalJS.rawValue,
ClawdisCanvasCommand.snapshot.rawValue,
ClawdisCanvasA2UICommand.push.rawValue,
ClawdisCanvasA2UICommand.pushJSONL.rawValue,
ClawdisCanvasA2UICommand.reset.rawValue,
ClawdisScreenCommand.record.rawValue,
]
let caps = Set(self.currentCaps())
if caps.contains(ClawdisCapability.camera.rawValue) {
commands.append(ClawdisCameraCommand.snap.rawValue)
commands.append(ClawdisCameraCommand.clip.rawValue)
}
return commands
version: self.appVersion())
}
private func platformString() -> String {
let v = ProcessInfo.processInfo.operatingSystemVersion
let name = switch UIDevice.current.userInterfaceIdiom {
case .pad:
"iPadOS"
case .phone:
"iOS"
default:
"iOS"
}
return "\(name) \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"
}
private func deviceFamily() -> String {
switch UIDevice.current.userInterfaceIdiom {
case .pad:
"iPad"
case .phone:
"iPhone"
default:
"iOS"
}
}
private func modelIdentifier() -> String {
var systemInfo = utsname()
uname(&systemInfo)
let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in
String(bytes: ptr.prefix { $0 != 0 }, encoding: .utf8)
}
let trimmed = machine?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return trimmed.isEmpty ? "unknown" : trimmed
return "iOS \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"
}
private func appVersion() -> String {
Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"
}
}
#if DEBUG
extension BridgeConnectionController {
func _test_makeHello(token: String) -> BridgeHello {
self.makeHello(token: token)
}
func _test_resolvedDisplayName(defaults: UserDefaults) -> String {
self.resolvedDisplayName(defaults: defaults)
}
func _test_currentCaps() -> [String] {
self.currentCaps()
}
func _test_currentCommands() -> [String] {
self.currentCommands()
}
func _test_platformString() -> String {
self.platformString()
}
func _test_deviceFamily() -> String {
self.deviceFamily()
}
func _test_modelIdentifier() -> String {
self.modelIdentifier()
}
func _test_appVersion() -> String {
self.appVersion()
}
}
#endif
@@ -1,68 +0,0 @@
import SwiftUI
import UIKit
struct BridgeDiscoveryDebugLogView: View {
@Environment(BridgeConnectionController.self) private var bridgeController
@AppStorage("bridge.discovery.debugLogs") private var debugLogsEnabled: Bool = false
var body: some View {
List {
if !self.debugLogsEnabled {
Text("Enable “Discovery Debug Logs” to start collecting events.")
.foregroundStyle(.secondary)
}
if self.bridgeController.discoveryDebugLog.isEmpty {
Text("No log entries yet.")
.foregroundStyle(.secondary)
} else {
ForEach(self.bridgeController.discoveryDebugLog) { entry in
VStack(alignment: .leading, spacing: 2) {
Text(Self.formatTime(entry.ts))
.font(.caption)
.foregroundStyle(.secondary)
Text(entry.message)
.font(.callout)
.textSelection(.enabled)
}
.padding(.vertical, 4)
}
}
}
.navigationTitle("Discovery Logs")
.toolbar {
ToolbarItem(placement: .topBarTrailing) {
Button("Copy") {
UIPasteboard.general.string = self.formattedLog()
}
.disabled(self.bridgeController.discoveryDebugLog.isEmpty)
}
}
}
private func formattedLog() -> String {
self.bridgeController.discoveryDebugLog
.map { "\(Self.formatISO($0.ts)) \($0.message)" }
.joined(separator: "\n")
}
private static let timeFormatter: DateFormatter = {
let formatter = DateFormatter()
formatter.dateFormat = "HH:mm:ss"
return formatter
}()
private static let isoFormatter: ISO8601DateFormatter = {
let formatter = ISO8601DateFormatter()
formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
return formatter
}()
private static func formatTime(_ date: Date) -> String {
self.timeFormatter.string(from: date)
}
private static func formatISO(_ date: Date) -> String {
self.isoFormatter.string(from: date)
}
}
@@ -1,17 +1,9 @@
import ClawdisKit
import Foundation
import Network
import Observation
@MainActor
@Observable
final class BridgeDiscoveryModel {
struct DebugLogEntry: Identifiable, Equatable {
var id = UUID()
var ts: Date
var message: String
}
final class BridgeDiscoveryModel: ObservableObject {
struct DiscoveredBridge: Identifiable, Equatable {
var id: String { self.stableID }
var name: String
@@ -20,171 +12,75 @@ final class BridgeDiscoveryModel {
var debugID: String
}
var bridges: [DiscoveredBridge] = []
var statusText: String = "Idle"
private(set) var debugLog: [DebugLogEntry] = []
@Published var bridges: [DiscoveredBridge] = []
@Published var statusText: String = "Idle"
private var browsers: [String: NWBrowser] = [:]
private var bridgesByDomain: [String: [DiscoveredBridge]] = [:]
private var statesByDomain: [String: NWBrowser.State] = [:]
private var debugLoggingEnabled = false
private var lastStableIDs = Set<String>()
func setDebugLoggingEnabled(_ enabled: Bool) {
let wasEnabled = self.debugLoggingEnabled
self.debugLoggingEnabled = enabled
if !enabled {
self.debugLog = []
} else if !wasEnabled {
self.appendDebugLog("debug logging enabled")
self.appendDebugLog("snapshot: status=\(self.statusText) bridges=\(self.bridges.count)")
}
}
private var browser: NWBrowser?
func start() {
if !self.browsers.isEmpty { return }
self.appendDebugLog("start()")
if self.browser != nil { return }
let params = NWParameters.tcp
params.includePeerToPeer = true
let browser = NWBrowser(
for: .bonjour(type: ClawdisBonjour.bridgeServiceType, domain: ClawdisBonjour.bridgeServiceDomain),
using: params)
for domain in ClawdisBonjour.bridgeServiceDomains {
let params = NWParameters.tcp
params.includePeerToPeer = true
let browser = NWBrowser(
for: .bonjour(type: ClawdisBonjour.bridgeServiceType, domain: domain),
using: params)
browser.stateUpdateHandler = { [weak self] state in
Task { @MainActor in
guard let self else { return }
self.statesByDomain[domain] = state
self.updateStatusText()
self.appendDebugLog("state[\(domain)]: \(Self.prettyState(state))")
browser.stateUpdateHandler = { [weak self] state in
Task { @MainActor in
guard let self else { return }
switch state {
case .setup:
self.statusText = "Setup"
case .ready:
self.statusText = "Searching…"
case let .failed(err):
self.statusText = "Failed: \(err)"
case .cancelled:
self.statusText = "Stopped"
case let .waiting(err):
self.statusText = "Waiting: \(err)"
@unknown default:
self.statusText = "Unknown"
}
}
browser.browseResultsChangedHandler = { [weak self] results, _ in
Task { @MainActor in
guard let self else { return }
self.bridgesByDomain[domain] = results.compactMap { result -> DiscoveredBridge? in
switch result.endpoint {
case let .service(name, _, _, _):
let decodedName = BonjourEscapes.decode(name)
let advertisedName = result.endpoint.txtRecord?.dictionary["displayName"]
let prettyAdvertised = advertisedName
.map(Self.prettifyInstanceName)
.flatMap { $0.isEmpty ? nil : $0 }
let prettyName = prettyAdvertised ?? Self.prettifyInstanceName(decodedName)
return DiscoveredBridge(
name: prettyName,
endpoint: result.endpoint,
stableID: BridgeEndpointID.stableID(result.endpoint),
debugID: BridgeEndpointID.prettyDescription(result.endpoint))
default:
return nil
}
}
.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
self.recomputeBridges()
}
}
self.browsers[domain] = browser
browser.start(queue: DispatchQueue(label: "com.steipete.clawdis.ios.bridge-discovery.\(domain)"))
}
browser.browseResultsChangedHandler = { [weak self] results, _ in
Task { @MainActor in
guard let self else { return }
self.bridges = results.compactMap { result -> DiscoveredBridge? in
switch result.endpoint {
case let .service(name, _, _, _):
let decodedName = BonjourEscapes.decode(name)
let advertisedName = result.endpoint.txtRecord?.dictionary["displayName"]
let prettyAdvertised = advertisedName
.map(Self.prettifyInstanceName)
.flatMap { $0.isEmpty ? nil : $0 }
let prettyName = prettyAdvertised ?? Self.prettifyInstanceName(decodedName)
return DiscoveredBridge(
name: prettyName,
endpoint: result.endpoint,
stableID: BridgeEndpointID.stableID(result.endpoint),
debugID: BridgeEndpointID.prettyDescription(result.endpoint))
default:
return nil
}
}
.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
}
}
self.browser = browser
browser.start(queue: DispatchQueue(label: "com.steipete.clawdis.ios.bridge-discovery"))
}
func stop() {
self.appendDebugLog("stop()")
for browser in self.browsers.values {
browser.cancel()
}
self.browsers = [:]
self.bridgesByDomain = [:]
self.statesByDomain = [:]
self.browser?.cancel()
self.browser = nil
self.bridges = []
self.statusText = "Stopped"
}
private func recomputeBridges() {
let next = self.bridgesByDomain.values
.flatMap(\.self)
.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
let nextIDs = Set(next.map(\.stableID))
let added = nextIDs.subtracting(self.lastStableIDs)
let removed = self.lastStableIDs.subtracting(nextIDs)
if !added.isEmpty || !removed.isEmpty {
self.appendDebugLog("results: total=\(next.count) added=\(added.count) removed=\(removed.count)")
}
self.lastStableIDs = nextIDs
self.bridges = next
}
private func updateStatusText() {
let states = Array(self.statesByDomain.values)
if states.isEmpty {
self.statusText = self.browsers.isEmpty ? "Idle" : "Setup"
return
}
if let failed = states.first(where: { state in
if case .failed = state { return true }
return false
}) {
if case let .failed(err) = failed {
self.statusText = "Failed: \(err)"
return
}
}
if let waiting = states.first(where: { state in
if case .waiting = state { return true }
return false
}) {
if case let .waiting(err) = waiting {
self.statusText = "Waiting: \(err)"
return
}
}
if states.contains(where: { if case .ready = $0 { true } else { false } }) {
self.statusText = "Searching…"
return
}
if states.contains(where: { if case .setup = $0 { true } else { false } }) {
self.statusText = "Setup"
return
}
self.statusText = "Searching…"
}
private static func prettyState(_ state: NWBrowser.State) -> String {
switch state {
case .setup:
"setup"
case .ready:
"ready"
case let .failed(err):
"failed (\(err))"
case .cancelled:
"cancelled"
case let .waiting(err):
"waiting (\(err))"
@unknown default:
"unknown"
}
}
private func appendDebugLog(_ message: String) {
guard self.debugLoggingEnabled else { return }
self.debugLog.append(DebugLogEntry(ts: Date(), message: message))
if self.debugLog.count > 200 {
self.debugLog.removeFirst(self.debugLog.count - 200)
}
}
private static func prettifyInstanceName(_ decodedName: String) -> String {
let normalized = decodedName.split(whereSeparator: \.isWhitespace).joined(separator: " ")
let stripped = normalized.replacingOccurrences(of: " (Clawdis)", with: "")
@@ -25,11 +25,6 @@ actor BridgeSession {
private var serverEventSubscribers: [UUID: AsyncStream<BridgeEventFrame>.Continuation] = [:]
private(set) var state: State = .idle
private var canvasHostUrl: String?
func currentCanvasHostUrl() -> String? {
self.canvasHostUrl
}
func currentRemoteAddress() -> String? {
guard let endpoint = self.connection?.currentPath?.remoteEndpoint else { return nil }
@@ -106,7 +101,6 @@ actor BridgeSession {
if base.type == "hello-ok" {
let ok = try self.decoder.decode(BridgeHelloOk.self, from: data)
self.state = .connected(serverName: ok.serverName)
self.canvasHostUrl = ok.canvasHostUrl?.trimmingCharacters(in: .whitespacesAndNewlines)
await onConnected?(ok.serverName)
} else if base.type == "error" {
let err = try self.decoder.decode(BridgeErrorFrame.self, from: data)
@@ -216,7 +210,6 @@ actor BridgeSession {
self.connection = nil
self.queue = nil
self.buffer = Data()
self.canvasHostUrl = nil
let pending = self.pendingRPC.values
self.pendingRPC.removeAll()
+19 -59
View File
@@ -36,7 +36,6 @@ actor CameraController {
height: Int)
{
let facing = params.facing ?? .front
let format = params.format ?? .jpg
// Default to a reasonable max width to keep bridge payload sizes manageable.
// If you need the full-res photo, explicitly request a larger maxWidth.
let maxWidth = params.maxWidth.flatMap { $0 > 0 ? $0 : nil } ?? 1600
@@ -66,7 +65,6 @@ actor CameraController {
session.startRunning()
defer { session.stopRunning() }
await Self.warmUpCaptureSession()
let settings: AVCapturePhotoSettings = {
if output.availablePhotoCodecTypes.contains(.jpeg) {
@@ -76,13 +74,9 @@ actor CameraController {
}()
settings.photoQualityPrioritization = .quality
var delegate: PhotoCaptureDelegate?
let rawData: Data = try await withCheckedThrowingContinuation { cont in
let d = PhotoCaptureDelegate(cont)
delegate = d
output.capturePhoto(with: settings, delegate: d)
output.capturePhoto(with: settings, delegate: PhotoCaptureDelegate(cont))
}
withExtendedLifetime(delegate) {}
let res = try JPEGTranscoder.transcodeToJPEG(
imageData: rawData,
@@ -90,7 +84,7 @@ actor CameraController {
quality: quality)
return (
format: format.rawValue,
format: "jpg",
base64: res.data.base64EncodedString(),
width: res.widthPx,
height: res.heightPx)
@@ -105,7 +99,6 @@ actor CameraController {
let facing = params.facing ?? .front
let durationMs = Self.clampDurationMs(params.durationMs)
let includeAudio = params.includeAudio ?? true
let format = params.format ?? .mp4
try await self.ensureAccess(for: .video)
if includeAudio {
@@ -145,7 +138,6 @@ actor CameraController {
session.startRunning()
defer { session.stopRunning() }
await Self.warmUpCaptureSession()
let movURL = FileManager.default.temporaryDirectory
.appendingPathComponent("clawdis-camera-\(UUID().uuidString).mov")
@@ -157,23 +149,16 @@ actor CameraController {
try? FileManager.default.removeItem(at: mp4URL)
}
var delegate: MovieFileDelegate?
let recordedURL: URL = try await withCheckedThrowingContinuation { cont in
let d = MovieFileDelegate(cont)
delegate = d
output.startRecording(to: movURL, recordingDelegate: d)
let delegate = MovieFileDelegate(cont)
output.startRecording(to: movURL, recordingDelegate: delegate)
}
withExtendedLifetime(delegate) {}
// Transcode .mov -> .mp4 for easier downstream handling.
try await Self.exportToMP4(inputURL: recordedURL, outputURL: mp4URL)
let data = try Data(contentsOf: mp4URL)
return (
format: format.rawValue,
base64: data.base64EncodedString(),
durationMs: durationMs,
hasAudio: includeAudio)
return (format: "mp4", base64: data.base64EncodedString(), durationMs: durationMs, hasAudio: includeAudio)
}
private func ensureAccess(for mediaType: AVMediaType) async throws {
@@ -199,11 +184,7 @@ actor CameraController {
private nonisolated static func pickCamera(facing: ClawdisCameraFacing) -> AVCaptureDevice? {
let position: AVCaptureDevice.Position = (facing == .front) ? .front : .back
if let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) {
return device
}
// Fall back to any default camera (e.g. simulator / unusual device configurations).
return AVCaptureDevice.default(for: .video)
return AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position)
}
nonisolated static func clampQuality(_ quality: Double?) -> Double {
@@ -214,12 +195,12 @@ actor CameraController {
nonisolated static func clampDurationMs(_ ms: Int?) -> Int {
let v = ms ?? 3000
// Keep clips short by default; avoid huge base64 payloads on the bridge.
return min(60000, max(250, v))
return min(15000, max(250, v))
}
private nonisolated static func exportToMP4(inputURL: URL, outputURL: URL) async throws {
let asset = AVURLAsset(url: inputURL)
guard let exporter = AVAssetExportSession(asset: asset, presetName: AVAssetExportPresetMediumQuality) else {
guard let exporter = AVAssetExportSession(asset: asset, presetName: AVAssetExportPresetHighestQuality) else {
throw CameraError.exportFailed("Failed to create export session")
}
exporter.shouldOptimizeForNetworkUse = true
@@ -235,29 +216,22 @@ actor CameraController {
exporter.outputURL = outputURL
exporter.outputFileType = .mp4
try await withCheckedThrowingContinuation(isolation: nil) { (cont: CheckedContinuation<Void, Error>) in
try await withCheckedThrowingContinuation(isolation: nil) { cont in
exporter.exportAsynchronously {
cont.resume(returning: ())
switch exporter.status {
case .completed:
cont.resume(returning: ())
case .failed:
cont.resume(throwing: exporter.error ?? CameraError.exportFailed("Export failed"))
case .cancelled:
cont.resume(throwing: CameraError.exportFailed("Export cancelled"))
default:
cont.resume(throwing: CameraError.exportFailed("Export did not complete"))
}
}
}
switch exporter.status {
case .completed:
return
case .failed:
throw CameraError.exportFailed(exporter.error?.localizedDescription ?? "export failed")
case .cancelled:
throw CameraError.exportFailed("export cancelled")
default:
throw CameraError.exportFailed("export did not complete")
}
}
}
private nonisolated static func warmUpCaptureSession() async {
// A short delay after `startRunning()` significantly reduces "blank first frame" captures on some devices.
try? await Task.sleep(nanoseconds: 150_000_000) // 150ms
}
}
private final class PhotoCaptureDelegate: NSObject, AVCapturePhotoCaptureDelegate {
@@ -287,13 +261,6 @@ private final class PhotoCaptureDelegate: NSObject, AVCapturePhotoCaptureDelegat
]))
return
}
if data.isEmpty {
self.continuation.resume(
throwing: NSError(domain: "Camera", code: 2, userInfo: [
NSLocalizedDescriptionKey: "photo data empty",
]))
return
}
self.continuation.resume(returning: data)
}
@@ -327,13 +294,6 @@ private final class MovieFileDelegate: NSObject, AVCaptureFileOutputRecordingDel
self.didResume = true
if let error {
let ns = error as NSError
if ns.domain == AVFoundationErrorDomain,
ns.code == AVError.maximumDurationReached.rawValue
{
self.continuation.resume(returning: outputFileURL)
return
}
self.continuation.resume(throwing: error)
return
}
+3 -3
View File
@@ -3,12 +3,12 @@ import SwiftUI
struct ChatSheet: View {
@Environment(\.dismiss) private var dismiss
@State private var viewModel: ClawdisChatViewModel
@StateObject private var viewModel: ClawdisChatViewModel
init(bridge: BridgeSession, sessionKey: String = "main") {
let transport = IOSBridgeChatTransport(bridge: bridge)
self._viewModel = State(
initialValue: ClawdisChatViewModel(
self._viewModel = StateObject(
wrappedValue: ClawdisChatViewModel(
sessionKey: sessionKey,
transport: transport))
}
@@ -9,28 +9,6 @@ struct IOSBridgeChatTransport: ClawdisChatTransport, Sendable {
self.bridge = bridge
}
func abortRun(sessionKey: String, runId: String) async throws {
struct Params: Codable {
var sessionKey: String
var runId: String
}
let data = try JSONEncoder().encode(Params(sessionKey: sessionKey, runId: runId))
let json = String(data: data, encoding: .utf8)
_ = try await self.bridge.request(method: "chat.abort", paramsJSON: json, timeoutSeconds: 10)
}
func listSessions(limit: Int?) async throws -> ClawdisChatSessionsListResponse {
struct Params: Codable {
var includeGlobal: Bool
var includeUnknown: Bool
var limit: Int?
}
let data = try JSONEncoder().encode(Params(includeGlobal: true, includeUnknown: false, limit: limit))
let json = String(data: data, encoding: .utf8)
let res = try await self.bridge.request(method: "sessions.list", paramsJSON: json, timeoutSeconds: 15)
return try JSONDecoder().decode(ClawdisChatSessionsListResponse.self, from: res)
}
func setActiveSessionKey(_ sessionKey: String) async throws {
struct Subscribe: Codable { var sessionKey: String }
let data = try JSONEncoder().encode(Subscribe(sessionKey: sessionKey))
@@ -101,11 +79,6 @@ struct IOSBridgeChatTransport: ClawdisChatTransport, Sendable {
if let payload = try? JSONDecoder().decode(ClawdisChatEventPayload.self, from: data) {
continuation.yield(.chat(payload))
}
case "agent":
guard let json = evt.payloadJSON, let data = json.data(using: .utf8) else { break }
if let payload = try? JSONDecoder().decode(ClawdisAgentEventPayload.self, from: data) {
continuation.yield(.agent(payload))
}
default:
break
}
+7 -7
View File
@@ -2,23 +2,23 @@ import SwiftUI
@main
struct ClawdisApp: App {
@State private var appModel: NodeAppModel
@State private var bridgeController: BridgeConnectionController
@StateObject private var appModel: NodeAppModel
@StateObject private var bridgeController: BridgeConnectionController
@Environment(\.scenePhase) private var scenePhase
init() {
BridgeSettingsStore.bootstrapPersistence()
let appModel = NodeAppModel()
_appModel = State(initialValue: appModel)
_bridgeController = State(initialValue: BridgeConnectionController(appModel: appModel))
_appModel = StateObject(wrappedValue: appModel)
_bridgeController = StateObject(wrappedValue: BridgeConnectionController(appModel: appModel))
}
var body: some Scene {
WindowGroup {
RootCanvas()
.environment(self.appModel)
.environment(self.appModel.voiceWake)
.environment(self.bridgeController)
.environmentObject(self.appModel)
.environmentObject(self.appModel.voiceWake)
.environmentObject(self.bridgeController)
.onOpenURL { url in
Task { await self.appModel.handleDeepLink(url: url) }
}
-19
View File
@@ -22,11 +22,6 @@
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>NSAppTransportSecurity</key>
<dict>
<key>NSAllowsArbitraryLoadsInWebContent</key>
<true/>
</dict>
<key>NSBonjourServices</key>
<array>
<string>_clawdis-bridge._tcp</string>
@@ -50,19 +45,5 @@
</array>
<key>UILaunchScreen</key>
<dict/>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
</dict>
</plist>
+30 -419
View File
@@ -1,45 +1,28 @@
import ClawdisKit
import Network
import Observation
import SwiftUI
import UIKit
@MainActor
@Observable
final class NodeAppModel {
enum CameraHUDKind {
case photo
case recording
case success
case error
}
var isBackgrounded: Bool = false
final class NodeAppModel: ObservableObject {
@Published var isBackgrounded: Bool = false
let screen = ScreenController()
let camera = CameraController()
private let screenRecorder = ScreenRecordService()
var bridgeStatusText: String = "Offline"
var bridgeServerName: String?
var bridgeRemoteAddress: String?
var connectedBridgeID: String?
@Published var bridgeStatusText: String = "Not connected"
@Published var bridgeServerName: String?
@Published var bridgeRemoteAddress: String?
@Published var connectedBridgeID: String?
private let bridge = BridgeSession()
private var bridgeTask: Task<Void, Never>?
private var voiceWakeSyncTask: Task<Void, Never>?
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
let voiceWake = VoiceWakeManager()
private var lastAutoA2uiURL: String?
var bridgeSession: BridgeSession { self.bridge }
var cameraHUDText: String?
var cameraHUDKind: CameraHUDKind?
var cameraFlashNonce: Int = 0
init() {
self.voiceWake.configure { [weak self] cmd in
guard let self else { return }
let sessionKey = "main"
let nodeId = UserDefaults.standard.string(forKey: "node.instanceId") ?? "ios-node"
let sessionKey = "node-\(nodeId)"
do {
try await self.sendVoiceTranscript(text: cmd, sessionKey: sessionKey)
} catch {
@@ -49,117 +32,6 @@ final class NodeAppModel {
let enabled = UserDefaults.standard.bool(forKey: "voiceWake.enabled")
self.voiceWake.setEnabled(enabled)
// Wire up deep links from canvas taps
self.screen.onDeepLink = { [weak self] url in
guard let self else { return }
Task { @MainActor in
await self.handleDeepLink(url: url)
}
}
// Wire up A2UI action clicks (buttons, etc.)
self.screen.onA2UIAction = { [weak self] body in
guard let self else { return }
Task { @MainActor in
await self.handleCanvasA2UIAction(body: body)
}
}
}
private func handleCanvasA2UIAction(body: [String: Any]) async {
let userActionAny = body["userAction"] ?? body
let userAction: [String: Any] = {
if let dict = userActionAny as? [String: Any] { return dict }
if let dict = userActionAny as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !userAction.isEmpty else { return }
guard let name = ClawdisCanvasA2UIAction.extractActionName(userAction) else { return }
let actionId: String = {
let id = (userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return id.isEmpty ? UUID().uuidString : id
}()
let surfaceId: String = {
let raw = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return raw.isEmpty ? "main" : raw
}()
let sourceComponentId: String = {
let raw = (userAction[
"sourceComponentId",
] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return raw.isEmpty ? "-" : raw
}()
let host = UserDefaults.standard.string(forKey: "node.displayName") ?? UIDevice.current.name
let instanceId = (UserDefaults.standard.string(forKey: "node.instanceId") ?? "ios-node").lowercased()
let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"])
let sessionKey = "main"
let messageContext = ClawdisCanvasA2UIAction.AgentMessageContext(
actionName: name,
session: .init(key: sessionKey, surfaceId: surfaceId),
component: .init(id: sourceComponentId, host: host, instanceId: instanceId),
contextJSON: contextJSON)
let message = ClawdisCanvasA2UIAction.formatAgentMessage(messageContext)
let ok: Bool
var errorText: String?
if await !self.isBridgeConnected() {
ok = false
errorText = "bridge not connected"
} else {
do {
try await self.sendAgentRequest(link: AgentDeepLink(
message: message,
sessionKey: sessionKey,
thinking: "low",
deliver: false,
to: nil,
channel: nil,
timeoutSeconds: nil,
key: actionId))
ok = true
} catch {
ok = false
errorText = error.localizedDescription
}
}
let js = ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId: actionId, ok: ok, error: errorText)
do {
_ = try await self.screen.eval(javaScript: js)
} catch {
// ignore
}
}
private func resolveA2UIHostURL() async -> String? {
guard let raw = await self.bridge.currentCanvasHostUrl() else { return nil }
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
return base.appendingPathComponent("__clawdis__/a2ui/").absoluteString
}
private func showA2UIOnConnectIfNeeded() async {
guard let a2uiUrl = await self.resolveA2UIHostURL() else { return }
let current = self.screen.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
if current.isEmpty || current == self.lastAutoA2uiURL {
self.screen.navigate(to: a2uiUrl)
self.lastAutoA2uiURL = a2uiUrl
}
}
private func showLocalCanvasOnDisconnect() {
self.lastAutoA2uiURL = nil
self.screen.showDefaultCanvas()
}
func setScenePhase(_ phase: ScenePhase) {
@@ -185,8 +57,6 @@ final class NodeAppModel {
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeID = BridgeEndpointID.stableID(endpoint)
self.voiceWakeSyncTask?.cancel()
self.voiceWakeSyncTask = nil
self.bridgeTask = Task {
var attempt = 0
@@ -216,8 +86,6 @@ final class NodeAppModel {
self.bridgeRemoteAddress = addr
}
}
await self.startVoiceWakeSync()
await self.showA2UIOnConnectIfNeeded()
},
onInvoke: { [weak self] req in
guard let self else {
@@ -230,9 +98,6 @@ final class NodeAppModel {
})
if Task.isCancelled { break }
await MainActor.run {
self.showLocalCanvasOnDisconnect()
}
attempt += 1
let sleepSeconds = min(6.0, 0.35 * pow(1.7, Double(attempt)))
try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000))
@@ -243,7 +108,6 @@ final class NodeAppModel {
self.bridgeStatusText = "Bridge error: \(error.localizedDescription)"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.showLocalCanvasOnDisconnect()
}
let sleepSeconds = min(8.0, 0.5 * pow(1.7, Double(attempt)))
try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000))
@@ -251,11 +115,10 @@ final class NodeAppModel {
}
await MainActor.run {
self.bridgeStatusText = "Offline"
self.bridgeStatusText = "Disconnected"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeID = nil
self.showLocalCanvasOnDisconnect()
}
}
}
@@ -263,60 +126,11 @@ final class NodeAppModel {
func disconnectBridge() {
self.bridgeTask?.cancel()
self.bridgeTask = nil
self.voiceWakeSyncTask?.cancel()
self.voiceWakeSyncTask = nil
Task { await self.bridge.disconnect() }
self.bridgeStatusText = "Offline"
self.bridgeStatusText = "Disconnected"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeID = nil
self.showLocalCanvasOnDisconnect()
}
func setGlobalWakeWords(_ words: [String]) async {
let sanitized = VoiceWakePreferences.sanitizeTriggerWords(words)
struct Payload: Codable {
var triggers: [String]
}
let payload = Payload(triggers: sanitized)
guard let data = try? JSONEncoder().encode(payload),
let json = String(data: data, encoding: .utf8)
else { return }
do {
_ = try await self.bridge.request(method: "voicewake.set", paramsJSON: json, timeoutSeconds: 12)
} catch {
// Best-effort only.
}
}
private func startVoiceWakeSync() async {
self.voiceWakeSyncTask?.cancel()
self.voiceWakeSyncTask = Task { [weak self] in
guard let self else { return }
await self.refreshWakeWordsFromGateway()
let stream = await self.bridge.subscribeServerEvents(bufferingNewest: 200)
for await evt in stream {
if Task.isCancelled { return }
guard evt.event == "voicewake.changed" else { continue }
guard let payloadJSON = evt.payloadJSON else { continue }
guard let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: payloadJSON) else { continue }
VoiceWakePreferences.saveTriggerWords(triggers)
}
}
}
private func refreshWakeWordsFromGateway() async {
do {
let data = try await self.bridge.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8)
guard let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: data) else { return }
VoiceWakePreferences.saveTriggerWords(triggers)
} catch {
// Best-effort only.
}
}
func sendVoiceTranscript(text: String, sessionKey: String?) async throws {
@@ -388,22 +202,17 @@ final class NodeAppModel {
return false
}
// swiftlint:disable:next function_body_length cyclomatic_complexity
private func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
let command = req.command
if command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen."),
self.isBackgrounded
{
if req.command.hasPrefix("screen.") || req.command.hasPrefix("camera."), self.isBackgrounded {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(
code: .backgroundUnavailable,
message: "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground"))
message: "NODE_BACKGROUND_UNAVAILABLE: screen/camera commands require foreground"))
}
if command.hasPrefix("camera."), !self.isCameraEnabled() {
if req.command.hasPrefix("camera."), !self.isCameraEnabled() {
return BridgeInvokeResponse(
id: req.id,
ok: false,
@@ -413,133 +222,37 @@ final class NodeAppModel {
}
do {
switch command {
case ClawdisCanvasCommand.present.rawValue:
let params = (try? Self.decodeParams(ClawdisCanvasPresentParams.self, from: req.paramsJSON)) ??
ClawdisCanvasPresentParams()
let url = params.url?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if url.isEmpty {
self.screen.showDefaultCanvas()
} else {
self.screen.navigate(to: url)
}
switch req.command {
case ClawdisScreenCommand.show.rawValue:
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisCanvasCommand.hide.rawValue:
case ClawdisScreenCommand.hide.rawValue:
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisCanvasCommand.navigate.rawValue:
let params = try Self.decodeParams(ClawdisCanvasNavigateParams.self, from: req.paramsJSON)
case ClawdisScreenCommand.setMode.rawValue:
let params = try Self.decodeParams(ClawdisScreenSetModeParams.self, from: req.paramsJSON)
self.screen.setMode(params.mode)
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisScreenCommand.navigate.rawValue:
let params = try Self.decodeParams(ClawdisScreenNavigateParams.self, from: req.paramsJSON)
self.screen.navigate(to: params.url)
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisCanvasCommand.evalJS.rawValue:
let params = try Self.decodeParams(ClawdisCanvasEvalParams.self, from: req.paramsJSON)
case ClawdisScreenCommand.evalJS.rawValue:
let params = try Self.decodeParams(ClawdisScreenEvalParams.self, from: req.paramsJSON)
let result = try await self.screen.eval(javaScript: params.javaScript)
let payload = try Self.encodePayload(["result": result])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisCanvasCommand.snapshot.rawValue:
let params = try? Self.decodeParams(ClawdisCanvasSnapshotParams.self, from: req.paramsJSON)
let format = params?.format ?? .jpeg
let maxWidth: CGFloat? = {
if let raw = params?.maxWidth, raw > 0 { return CGFloat(raw) }
// Keep default snapshots comfortably below the gateway client's maxPayload.
// For full-res, clients should explicitly request a larger maxWidth.
return switch format {
case .png: 900
case .jpeg: 1600
}
}()
let base64 = try await self.screen.snapshotBase64(
maxWidth: maxWidth,
format: format,
quality: params?.quality)
let payload = try Self.encodePayload([
"format": format == .jpeg ? "jpeg" : "png",
"base64": base64,
])
case ClawdisScreenCommand.snapshot.rawValue:
let params = try? Self.decodeParams(ClawdisScreenSnapshotParams.self, from: req.paramsJSON)
let maxWidth = params?.maxWidth.map { CGFloat($0) }
let base64 = try await self.screen.snapshotPNGBase64(maxWidth: maxWidth)
let payload = try Self.encodePayload(["format": "png", "base64": base64])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisCanvasA2UICommand.reset.rawValue:
guard let a2uiUrl = await self.resolveA2UIHostURL() else {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(
code: .unavailable,
message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host"))
}
self.screen.navigate(to: a2uiUrl)
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(
code: .unavailable,
message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable"))
}
let json = try await self.screen.eval(javaScript: """
(() => {
if (!globalThis.clawdisA2UI) return JSON.stringify({ ok: false, error: "missing clawdisA2UI" });
return JSON.stringify(globalThis.clawdisA2UI.reset());
})()
""")
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
case ClawdisCanvasA2UICommand.push.rawValue, ClawdisCanvasA2UICommand.pushJSONL.rawValue:
let messages: [AnyCodable]
if command == ClawdisCanvasA2UICommand.pushJSONL.rawValue {
let params = try Self.decodeParams(ClawdisCanvasA2UIPushJSONLParams.self, from: req.paramsJSON)
messages = try ClawdisCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl)
} else {
do {
let params = try Self.decodeParams(ClawdisCanvasA2UIPushParams.self, from: req.paramsJSON)
messages = params.messages
} catch {
// Be forgiving: some clients still send JSONL payloads to `canvas.a2ui.push`.
let params = try Self.decodeParams(ClawdisCanvasA2UIPushJSONLParams.self, from: req.paramsJSON)
messages = try ClawdisCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl)
}
}
guard let a2uiUrl = await self.resolveA2UIHostURL() else {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(
code: .unavailable,
message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host"))
}
self.screen.navigate(to: a2uiUrl)
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(
code: .unavailable,
message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable"))
}
let messagesJSON = try ClawdisCanvasA2UIJSONL.encodeMessagesJSONArray(messages)
let js = """
(() => {
try {
if (!globalThis.clawdisA2UI) return JSON.stringify({ ok: false, error: "missing clawdisA2UI" });
const messages = \(messagesJSON);
return JSON.stringify(globalThis.clawdisA2UI.applyMessages(messages));
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e) });
}
})()
"""
let resultJSON = try await self.screen.eval(javaScript: js)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON)
case ClawdisCameraCommand.snap.rawValue:
self.showCameraHUD(text: "Taking photo…", kind: .photo)
self.triggerCameraFlash()
let params = (try? Self.decodeParams(ClawdisCameraSnapParams.self, from: req.paramsJSON)) ??
ClawdisCameraSnapParams()
let res = try await self.camera.snap(params: params)
@@ -555,7 +268,6 @@ final class NodeAppModel {
base64: res.base64,
width: res.width,
height: res.height))
self.showCameraHUD(text: "Photo captured", kind: .success, autoHideSeconds: 1.6)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisCameraCommand.clip.rawValue:
@@ -565,7 +277,6 @@ final class NodeAppModel {
let suspended = (params.includeAudio ?? true) ? self.voiceWake.suspendForExternalAudioCapture() : false
defer { self.voiceWake.resumeAfterExternalAudioCapture(wasSuspended: suspended) }
self.showCameraHUD(text: "Recording…", kind: .recording)
let res = try await self.camera.clip(params: params)
struct Payload: Codable {
@@ -579,40 +290,6 @@ final class NodeAppModel {
base64: res.base64,
durationMs: res.durationMs,
hasAudio: res.hasAudio))
self.showCameraHUD(text: "Clip captured", kind: .success, autoHideSeconds: 1.8)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisScreenCommand.record.rawValue:
let params = (try? Self.decodeParams(ClawdisScreenRecordParams.self, from: req.paramsJSON)) ??
ClawdisScreenRecordParams()
if let format = params.format, format.lowercased() != "mp4" {
throw NSError(domain: "Screen", code: 30, userInfo: [
NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4",
])
}
let path = try await self.screenRecorder.record(
screenIndex: params.screenIndex,
durationMs: params.durationMs,
fps: params.fps,
includeAudio: params.includeAudio,
outPath: nil)
defer { try? FileManager.default.removeItem(atPath: path) }
let data = try Data(contentsOf: URL(fileURLWithPath: path))
struct Payload: Codable {
var format: String
var base64: String
var durationMs: Int?
var fps: Double?
var screenIndex: Int?
var hasAudio: Bool
}
let payload = try Self.encodePayload(Payload(
format: "mp4",
base64: data.base64EncodedString(),
durationMs: params.durationMs,
fps: params.fps,
screenIndex: params.screenIndex,
hasAudio: params.includeAudio ?? true))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
default:
@@ -622,10 +299,6 @@ final class NodeAppModel {
error: ClawdisNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
} catch {
if command.hasPrefix("camera.") {
let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2)
}
return BridgeInvokeResponse(
id: req.id,
ok: false,
@@ -657,66 +330,4 @@ final class NodeAppModel {
if UserDefaults.standard.object(forKey: "camera.enabled") == nil { return true }
return UserDefaults.standard.bool(forKey: "camera.enabled")
}
private func triggerCameraFlash() {
self.cameraFlashNonce &+= 1
}
private func showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) {
self.cameraHUDDismissTask?.cancel()
withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) {
self.cameraHUDText = text
self.cameraHUDKind = kind
}
guard let autoHideSeconds else { return }
self.cameraHUDDismissTask = Task { @MainActor in
try? await Task.sleep(nanoseconds: UInt64(autoHideSeconds * 1_000_000_000))
withAnimation(.easeOut(duration: 0.25)) {
self.cameraHUDText = nil
self.cameraHUDKind = nil
}
}
}
}
#if DEBUG
extension NodeAppModel {
func _test_handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
await self.handleInvoke(req)
}
static func _test_decodeParams<T: Decodable>(_ type: T.Type, from json: String?) throws -> T {
try self.decodeParams(type, from: json)
}
static func _test_encodePayload(_ obj: some Encodable) throws -> String {
try self.encodePayload(obj)
}
func _test_isCameraEnabled() -> Bool {
self.isCameraEnabled()
}
func _test_triggerCameraFlash() {
self.triggerCameraFlash()
}
func _test_showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) {
self.showCameraHUD(text: text, kind: kind, autoHideSeconds: autoHideSeconds)
}
func _test_handleCanvasA2UIAction(body: [String: Any]) async {
await self.handleCanvasA2UIAction(body: body)
}
func _test_resolveA2UIHostURL() async -> String? {
await self.resolveA2UIHostURL()
}
func _test_showLocalCanvasOnDisconnect() {
self.showLocalCanvasOnDisconnect()
}
}
#endif
+36 -211
View File
@@ -1,14 +1,9 @@
import SwiftUI
import UIKit
struct RootCanvas: View {
@Environment(NodeAppModel.self) private var appModel
@Environment(VoiceWakeManager.self) private var voiceWake
@Environment(\.colorScheme) private var systemColorScheme
@Environment(\.scenePhase) private var scenePhase
@EnvironmentObject private var appModel: NodeAppModel
@EnvironmentObject private var voiceWake: VoiceWakeManager
@AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true
@AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false
@State private var presentedSheet: PresentedSheet?
@State private var voiceWakeToastText: String?
@State private var toastDismissTask: Task<Void, Never>?
@@ -26,24 +21,37 @@ struct RootCanvas: View {
}
var body: some View {
ZStack {
CanvasContent(
systemColorScheme: self.systemColorScheme,
bridgeStatus: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
voiceWakeToastText: self.voiceWakeToastText,
cameraHUDText: self.appModel.cameraHUDText,
cameraHUDKind: self.appModel.cameraHUDKind,
openChat: {
self.presentedSheet = .chat
},
openSettings: {
self.presentedSheet = .settings
})
.preferredColorScheme(.dark)
ZStack(alignment: .topTrailing) {
ScreenTab()
if self.appModel.cameraFlashNonce != 0 {
CameraFlashOverlay(nonce: self.appModel.cameraFlashNonce)
VStack(spacing: 10) {
OverlayButton(systemImage: "text.bubble.fill") {
self.presentedSheet = .chat
}
.accessibilityLabel("Chat")
OverlayButton(systemImage: "gearshape.fill") {
self.presentedSheet = .settings
}
.accessibilityLabel("Settings")
}
.padding(.top, 10)
.padding(.trailing, 10)
}
.overlay(alignment: .topLeading) {
StatusPill(
bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
onTap: { self.presentedSheet = .settings })
.padding(.leading, 10)
.safeAreaPadding(.top, 10)
}
.overlay(alignment: .topLeading) {
if let voiceWakeToastText, !voiceWakeToastText.isEmpty {
VoiceWakeToast(command: voiceWakeToastText)
.padding(.leading, 10)
.safeAreaPadding(.top, 58)
.transition(.move(edge: .top).combined(with: .opacity))
}
}
.sheet(item: self.$presentedSheet) { sheet in
@@ -54,14 +62,7 @@ struct RootCanvas: View {
ChatSheet(bridge: self.appModel.bridgeSession)
}
}
.onAppear { self.updateIdleTimer() }
.onChange(of: self.preventSleep) { _, _ in self.updateIdleTimer() }
.onChange(of: self.scenePhase) { _, _ in self.updateIdleTimer() }
.onAppear { self.updateCanvasDebugStatus() }
.onChange(of: self.canvasDebugStatusEnabled) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.bridgeStatusText) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.bridgeServerName) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.bridgeRemoteAddress) { _, _ in self.updateCanvasDebugStatus() }
.preferredColorScheme(.dark)
.onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in
guard let newValue else { return }
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -82,7 +83,6 @@ struct RootCanvas: View {
}
}
.onDisappear {
UIApplication.shared.isIdleTimerDisabled = false
self.toastDismissTask?.cancel()
self.toastDismissTask = nil
}
@@ -104,103 +104,10 @@ struct RootCanvas: View {
return .disconnected
}
private func updateIdleTimer() {
UIApplication.shared.isIdleTimerDisabled = (self.scenePhase == .active && self.preventSleep)
}
private func updateCanvasDebugStatus() {
self.appModel.screen.setDebugStatusEnabled(self.canvasDebugStatusEnabled)
guard self.canvasDebugStatusEnabled else { return }
let title = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let subtitle = self.appModel.bridgeServerName ?? self.appModel.bridgeRemoteAddress
self.appModel.screen.updateDebugStatus(title: title, subtitle: subtitle)
}
}
private struct CanvasContent: View {
var systemColorScheme: ColorScheme
var bridgeStatus: StatusPill.BridgeState
var voiceWakeEnabled: Bool
var voiceWakeToastText: String?
var cameraHUDText: String?
var cameraHUDKind: NodeAppModel.CameraHUDKind?
var openChat: () -> Void
var openSettings: () -> Void
private var brightenButtons: Bool { self.systemColorScheme == .light }
var body: some View {
ZStack(alignment: .topTrailing) {
ScreenTab()
VStack(spacing: 10) {
OverlayButton(systemImage: "text.bubble.fill", brighten: self.brightenButtons) {
self.openChat()
}
.accessibilityLabel("Chat")
OverlayButton(systemImage: "gearshape.fill", brighten: self.brightenButtons) {
self.openSettings()
}
.accessibilityLabel("Settings")
}
.padding(.top, 10)
.padding(.trailing, 10)
}
.overlay(alignment: .topLeading) {
StatusPill(
bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
brighten: self.brightenButtons,
onTap: {
self.openSettings()
})
.padding(.leading, 10)
.safeAreaPadding(.top, 10)
}
.overlay(alignment: .topLeading) {
if let voiceWakeToastText, !voiceWakeToastText.isEmpty {
VoiceWakeToast(
command: voiceWakeToastText,
brighten: self.brightenButtons)
.padding(.leading, 10)
.safeAreaPadding(.top, 58)
.transition(.move(edge: .top).combined(with: .opacity))
}
}
.overlay(alignment: .topLeading) {
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
CameraCaptureToast(
text: cameraHUDText,
kind: self.mapCameraKind(cameraHUDKind),
brighten: self.brightenButtons)
.padding(SwiftUI.Edge.Set.leading, 10)
.safeAreaPadding(SwiftUI.Edge.Set.top, 106)
.transition(
AnyTransition.move(edge: SwiftUI.Edge.top)
.combined(with: AnyTransition.opacity))
}
}
}
private func mapCameraKind(_ kind: NodeAppModel.CameraHUDKind) -> CameraCaptureToast.Kind {
switch kind {
case .photo:
.photo
case .recording:
.recording
case .success:
.success
case .error:
.error
}
}
}
private struct OverlayButton: View {
let systemImage: String
let brighten: Bool
let action: () -> Void
var body: some View {
@@ -217,8 +124,8 @@ private struct OverlayButton: View {
.fill(
LinearGradient(
colors: [
.white.opacity(self.brighten ? 0.26 : 0.18),
.white.opacity(self.brighten ? 0.08 : 0.04),
.white.opacity(0.18),
.white.opacity(0.04),
.clear,
],
startPoint: .topLeading,
@@ -227,7 +134,7 @@ private struct OverlayButton: View {
}
.overlay {
RoundedRectangle(cornerRadius: 12, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
.strokeBorder(.white.opacity(0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.35), radius: 12, y: 6)
}
@@ -235,85 +142,3 @@ private struct OverlayButton: View {
.buttonStyle(.plain)
}
}
private struct CameraFlashOverlay: View {
var nonce: Int
@State private var opacity: CGFloat = 0
@State private var task: Task<Void, Never>?
var body: some View {
Color.white
.opacity(self.opacity)
.ignoresSafeArea()
.allowsHitTesting(false)
.onChange(of: self.nonce) { _, _ in
self.task?.cancel()
self.task = Task { @MainActor in
withAnimation(.easeOut(duration: 0.08)) {
self.opacity = 0.85
}
try? await Task.sleep(nanoseconds: 110_000_000)
withAnimation(.easeOut(duration: 0.32)) {
self.opacity = 0
}
}
}
}
}
private struct CameraCaptureToast: View {
enum Kind {
case photo
case recording
case success
case error
}
var text: String
var kind: Kind
var brighten: Bool = false
var body: some View {
HStack(spacing: 10) {
self.icon
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
Text(self.text)
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
.lineLimit(1)
.truncationMode(.tail)
}
.padding(.vertical, 10)
.padding(.horizontal, 12)
.background {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.fill(.ultraThinMaterial)
.overlay {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
}
.accessibilityLabel("Camera")
.accessibilityValue(self.text)
}
@ViewBuilder
private var icon: some View {
switch self.kind {
case .photo:
Image(systemName: "camera.fill")
case .recording:
Image(systemName: "record.circle.fill")
.symbolRenderingMode(.palette)
.foregroundStyle(.red, .primary)
case .success:
Image(systemName: "checkmark.circle.fill")
case .error:
Image(systemName: "exclamationmark.triangle.fill")
}
}
}
+2 -2
View File
@@ -1,8 +1,8 @@
import SwiftUI
struct RootTabs: View {
@Environment(NodeAppModel.self) private var appModel
@Environment(VoiceWakeManager.self) private var voiceWake
@EnvironmentObject private var appModel: NodeAppModel
@EnvironmentObject private var voiceWake: VoiceWakeManager
@AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false
@State private var selectedTab: Int = 0
@State private var voiceWakeToastText: String?
+128 -328
View File
@@ -1,144 +1,49 @@
import ClawdisKit
import Observation
import SwiftUI
import WebKit
@MainActor
@Observable
final class ScreenController {
final class ScreenController: ObservableObject {
let webView: WKWebView
private let navigationDelegate: ScreenNavigationDelegate
private let a2uiActionHandler: CanvasA2UIActionMessageHandler
var urlString: String = ""
var errorText: String?
/// Callback invoked when a clawdis:// deep link is tapped in the canvas
var onDeepLink: ((URL) -> Void)?
/// Callback invoked when the user clicks an A2UI action (e.g. button) inside the canvas web UI.
var onA2UIAction: (([String: Any]) -> Void)?
private var debugStatusEnabled: Bool = false
private var debugStatusTitle: String?
private var debugStatusSubtitle: String?
@Published var mode: ClawdisScreenMode = .canvas
@Published var urlString: String = ""
@Published var errorText: String?
init() {
let config = WKWebViewConfiguration()
config.websiteDataStore = .nonPersistent()
let a2uiActionHandler = CanvasA2UIActionMessageHandler()
let userContentController = WKUserContentController()
for name in CanvasA2UIActionMessageHandler.handlerNames {
userContentController.add(a2uiActionHandler, name: name)
}
config.userContentController = userContentController
self.navigationDelegate = ScreenNavigationDelegate()
self.a2uiActionHandler = a2uiActionHandler
self.webView = WKWebView(frame: .zero, configuration: config)
// Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays.
self.webView.isOpaque = true
self.webView.backgroundColor = .black
self.webView.scrollView.backgroundColor = .black
self.webView.isOpaque = false
self.webView.backgroundColor = .clear
self.webView.scrollView.backgroundColor = .clear
self.webView.scrollView.contentInsetAdjustmentBehavior = .never
self.webView.scrollView.contentInset = .zero
self.webView.scrollView.scrollIndicatorInsets = .zero
self.webView.scrollView.automaticallyAdjustsScrollIndicatorInsets = false
// Disable scroll to allow touch events to pass through to canvas
self.webView.scrollView.isScrollEnabled = false
self.webView.scrollView.bounces = false
self.webView.navigationDelegate = self.navigationDelegate
self.navigationDelegate.controller = self
a2uiActionHandler.controller = self
self.reload()
}
func setMode(_ mode: ClawdisScreenMode) {
self.mode = mode
self.reload()
}
func navigate(to urlString: String) {
let trimmed = urlString.trimmingCharacters(in: .whitespacesAndNewlines)
self.urlString = (trimmed == "/" ? "" : trimmed)
self.urlString = urlString
self.reload()
}
func reload() {
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
guard let url = Self.canvasScaffoldURL else { return }
self.errorText = nil
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
return
} else {
guard let url = URL(string: trimmed) else {
self.errorText = "Invalid URL: \(trimmed)"
return
}
self.errorText = nil
if url.isFileURL {
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
} else {
self.webView.load(URLRequest(url: url))
}
switch self.mode {
case .web:
guard let url = URL(string: self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)) else { return }
self.webView.load(URLRequest(url: url))
case .canvas:
self.webView.loadHTMLString(Self.canvasScaffoldHTML, baseURL: nil)
}
}
func showDefaultCanvas() {
self.urlString = ""
self.reload()
}
func setDebugStatusEnabled(_ enabled: Bool) {
self.debugStatusEnabled = enabled
self.applyDebugStatusIfNeeded()
}
func updateDebugStatus(title: String?, subtitle: String?) {
self.debugStatusTitle = title
self.debugStatusSubtitle = subtitle
self.applyDebugStatusIfNeeded()
}
fileprivate func applyDebugStatusIfNeeded() {
let enabled = self.debugStatusEnabled
let title = self.debugStatusTitle
let subtitle = self.debugStatusSubtitle
let js = """
(() => {
try {
const api = globalThis.__clawdis;
if (!api) return;
if (typeof api.setDebugStatusEnabled === 'function') {
api.setDebugStatusEnabled(\(enabled ? "true" : "false"));
}
if (!\(enabled ? "true" : "false")) return;
if (typeof api.setStatus === 'function') {
api.setStatus(\(Self.jsValue(title)), \(Self.jsValue(subtitle)));
}
} catch (_) {}
})()
"""
self.webView.evaluateJavaScript(js) { _, _ in }
}
func waitForA2UIReady(timeoutMs: Int) async -> Bool {
let clock = ContinuousClock()
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))
while clock.now < deadline {
do {
let res = try await self.eval(javaScript: """
(() => {
try {
return !!globalThis.clawdisA2UI && typeof globalThis.clawdisA2UI.applyMessages === 'function';
} catch (_) { return false; }
})()
""")
let trimmed = res.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
if trimmed == "true" || trimmed == "1" { return true }
} catch {
// ignore; page likely still loading
}
try? await Task.sleep(nanoseconds: 120_000_000)
}
return false
}
func eval(javaScript: String) async throws -> String {
try await withCheckedThrowingContinuation { cont in
self.webView.evaluateJavaScript(javaScript) { result, error in
@@ -183,221 +88,116 @@ final class ScreenController {
return data.base64EncodedString()
}
func snapshotBase64(
maxWidth: CGFloat? = nil,
format: ClawdisCanvasSnapshotFormat,
quality: Double? = nil) async throws -> String
{
let config = WKSnapshotConfiguration()
if let maxWidth {
config.snapshotWidth = NSNumber(value: Double(maxWidth))
}
let image: UIImage = try await withCheckedThrowingContinuation { cont in
self.webView.takeSnapshot(with: config) { image, error in
if let error {
cont.resume(throwing: error)
return
}
guard let image else {
cont.resume(throwing: NSError(domain: "Screen", code: 2, userInfo: [
NSLocalizedDescriptionKey: "snapshot failed",
]))
return
}
cont.resume(returning: image)
private static let canvasScaffoldHTML = """
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
<title>Canvas</title>
<style>
:root { color-scheme: dark; }
html,body { height:100%; margin:0; }
body {
background: radial-gradient(1200px 900px at 15% 20%, rgba(42, 113, 255, 0.18), rgba(0,0,0,0) 55%),
radial-gradient(900px 700px at 85% 30%, rgba(255, 0, 138, 0.14), rgba(0,0,0,0) 60%),
radial-gradient(1000px 900px at 60% 90%, rgba(0, 209, 255, 0.10), rgba(0,0,0,0) 60%),
#000;
overflow: hidden;
}
body::before {
content:"";
position: fixed;
inset: -20%;
background:
repeating-linear-gradient(0deg, rgba(255,255,255,0.02) 0, rgba(255,255,255,0.02) 1px,
transparent 1px, transparent 48px),
repeating-linear-gradient(90deg, rgba(255,255,255,0.02) 0, rgba(255,255,255,0.02) 1px,
transparent 1px, transparent 48px);
transform: rotate(-7deg);
opacity: 0.55;
pointer-events: none;
}
canvas {
display:block;
width:100vw;
height:100vh;
touch-action: none;
}
#clawdis-status {
position: fixed;
inset: 0;
display: grid;
place-items: center;
pointer-events: none;
}
#clawdis-status .card {
text-align: center;
padding: 16px 18px;
border-radius: 14px;
background: rgba(18, 18, 22, 0.42);
border: 1px solid rgba(255,255,255,0.08);
box-shadow: 0 18px 60px rgba(0,0,0,0.55);
backdrop-filter: blur(14px);
}
#clawdis-status .title {
font: 600 20px -apple-system, BlinkMacSystemFont, "SF Pro Display", "SF Pro Text", system-ui, sans-serif;
letter-spacing: 0.2px;
color: rgba(255,255,255,0.92);
text-shadow: 0 0 22px rgba(42, 113, 255, 0.35);
}
#clawdis-status .subtitle {
margin-top: 6px;
font: 500 12px -apple-system, BlinkMacSystemFont, "SF Pro Text", system-ui, sans-serif;
color: rgba(255,255,255,0.58);
}
</style>
</head>
<body>
<canvas id="clawdis-canvas"></canvas>
<div id="clawdis-status">
<div class="card">
<div class="title" id="clawdis-status-title">Ready</div>
<div class="subtitle" id="clawdis-status-subtitle">Waiting for agent</div>
</div>
</div>
<script>
(() => {
const canvas = document.getElementById('clawdis-canvas');
const ctx = canvas.getContext('2d');
const statusEl = document.getElementById('clawdis-status');
const titleEl = document.getElementById('clawdis-status-title');
const subtitleEl = document.getElementById('clawdis-status-subtitle');
function resize() {
const dpr = window.devicePixelRatio || 1;
const w = Math.max(1, Math.floor(window.innerWidth * dpr));
const h = Math.max(1, Math.floor(window.innerHeight * dpr));
canvas.width = w;
canvas.height = h;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
}
}
let data: Data?
switch format {
case .png:
data = image.pngData()
case .jpeg:
let q = (quality ?? 0.82).clamped(to: 0.1...1.0)
data = image.jpegData(compressionQuality: q)
}
guard let data else {
throw NSError(domain: "Screen", code: 1, userInfo: [
NSLocalizedDescriptionKey: "snapshot encode failed",
])
}
return data.base64EncodedString()
}
window.addEventListener('resize', resize);
resize();
private static func bundledResourceURL(
name: String,
ext: String,
subdirectory: String)
-> URL?
{
let bundle = ClawdisKitResources.bundle
return bundle.url(forResource: name, withExtension: ext, subdirectory: subdirectory)
?? bundle.url(forResource: name, withExtension: ext)
}
private static let canvasScaffoldURL: URL? = ScreenController.bundledResourceURL(
name: "scaffold",
ext: "html",
subdirectory: "CanvasScaffold")
func isTrustedCanvasUIURL(_ url: URL) -> Bool {
guard url.isFileURL else { return false }
let std = url.standardizedFileURL
if let expected = Self.canvasScaffoldURL,
std == expected.standardizedFileURL
{
return true
}
return false
}
private static func jsValue(_ value: String?) -> String {
guard let value else { return "null" }
if let data = try? JSONSerialization.data(withJSONObject: [value]),
let encoded = String(data: data, encoding: .utf8),
encoded.count >= 2
{
return String(encoded.dropFirst().dropLast())
}
return "null"
}
func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return false
}
guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else {
return false
}
if host == "localhost" { return true }
if host.hasSuffix(".local") { return true }
if host.hasSuffix(".ts.net") { return true }
if host.hasSuffix(".tailscale.net") { return true }
// Allow MagicDNS / LAN hostnames like "peters-mac-studio-1".
if !host.contains("."), !host.contains(":") { return true }
if let ipv4 = Self.parseIPv4(host) {
return Self.isLocalNetworkIPv4(ipv4)
}
return false
}
private static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? {
let parts = host.split(separator: ".", omittingEmptySubsequences: false)
guard parts.count == 4 else { return nil }
let bytes: [UInt8] = parts.compactMap { UInt8($0) }
guard bytes.count == 4 else { return nil }
return (bytes[0], bytes[1], bytes[2], bytes[3])
}
private static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool {
let (a, b, _, _) = ip
// 10.0.0.0/8
if a == 10 { return true }
// 172.16.0.0/12
if a == 172, (16...31).contains(Int(b)) { return true }
// 192.168.0.0/16
if a == 192, b == 168 { return true }
// 127.0.0.0/8
if a == 127 { return true }
// 169.254.0.0/16 (link-local)
if a == 169, b == 254 { return true }
// Tailscale: 100.64.0.0/10
if a == 100, (64...127).contains(Int(b)) { return true }
return false
}
nonisolated static func parseA2UIActionBody(_ body: Any) -> [String: Any]? {
if let dict = body as? [String: Any] { return dict.isEmpty ? nil : dict }
if let str = body as? String,
let data = str.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
{
return json.isEmpty ? nil : json
}
if let dict = body as? [AnyHashable: Any] {
let mapped = dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
return mapped.isEmpty ? nil : mapped
}
return nil
}
}
extension Double {
fileprivate func clamped(to range: ClosedRange<Double>) -> Double {
if self < range.lowerBound { return range.lowerBound }
if self > range.upperBound { return range.upperBound }
return self
}
}
// MARK: - Navigation Delegate
/// Handles navigation policy to intercept clawdis:// deep links from canvas
@MainActor
private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
weak var controller: ScreenController?
func webView(
_ webView: WKWebView,
decidePolicyFor navigationAction: WKNavigationAction,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
{
guard let url = navigationAction.request.url else {
decisionHandler(.allow)
return
}
// Intercept clawdis:// deep links
if url.scheme == "clawdis" {
decisionHandler(.cancel)
self.controller?.onDeepLink?(url)
return
}
decisionHandler(.allow)
}
func webView(
_: WKWebView,
didFailProvisionalNavigation _: WKNavigation?,
withError error: any Error)
{
self.controller?.errorText = error.localizedDescription
}
func webView(_: WKWebView, didFinish _: WKNavigation?) {
self.controller?.errorText = nil
self.controller?.applyDebugStatusIfNeeded()
}
func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) {
self.controller?.errorText = error.localizedDescription
}
}
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "clawdisCanvasA2UIAction"
static let legacyMessageNames = ["canvas", "a2ui", "userAction", "action"]
static let handlerNames = [messageName] + legacyMessageNames
weak var controller: ScreenController?
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
guard message.name == Self.messageName else { return }
guard let controller else { return }
guard let url = message.webView?.url else { return }
if url.isFileURL {
guard controller.isTrustedCanvasUIURL(url) else { return }
} else {
// For security, only accept actions from local-network pages (e.g. the canvas host).
guard controller.isLocalNetworkCanvasURL(url) else { return }
}
guard let body = ScreenController.parseA2UIActionBody(message.body) else { return }
controller.onA2UIAction?(body)
}
window.__clawdis = {
canvas,
ctx,
setStatus: (title, subtitle) => {
if (!statusEl) return;
if (!title && !subtitle) {
statusEl.style.display = 'none';
return;
}
statusEl.style.display = 'grid';
if (titleEl && typeof title === 'string') titleEl.textContent = title;
if (subtitleEl && typeof subtitle === 'string') subtitleEl.textContent = subtitle;
}
};
})();
</script>
</body>
</html>
"""
}
Loaded 100 of 4712 files, more files were not shown because too many files have changed in this diff. Show more