Compare commits

..
575 changed files with 13913 additions and 50508 deletions

No files matched your search

+2 -41
View File
@@ -34,7 +34,7 @@ jobs:
if: matrix.runtime == 'node'
uses: actions/setup-node@v4
with:
node-version: 24
node-version: 22
check-latest: true
- name: Setup Bun
@@ -49,7 +49,7 @@ jobs:
if: matrix.runtime == 'bun'
uses: actions/setup-node@v4
with:
node-version: 24
node-version: 22
check-latest: true
- name: Runtime versions
@@ -106,7 +106,6 @@ jobs:
run: bunx tsc -p tsconfig.json
macos-app:
if: github.event_name == 'pull_request'
runs-on: macos-latest
steps:
- name: Checkout
@@ -171,44 +170,6 @@ jobs:
sleep $((attempt * 20))
done
exit 1
ios:
if: false # ignore iOS in CI for now
runs-on: macos-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: false
- name: Checkout submodules (retry)
run: |
set -euo pipefail
git submodule sync --recursive
for attempt in 1 2 3 4 5; do
if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then
exit 0
fi
echo "Submodule update failed (attempt $attempt/5). Retrying…"
sleep $((attempt * 10))
done
exit 1
- name: Select Xcode 26.1
run: |
sudo xcode-select -s /Applications/Xcode_26.1.app
xcodebuild -version
- name: Install XcodeGen
run: brew install xcodegen
- name: Install SwiftLint / SwiftFormat
run: brew install swiftlint swiftformat
- name: Show toolchain
run: |
sw_vers
xcodebuild -version
swift --version
- name: Generate iOS project
run: |
-5
View File
@@ -1,30 +1,25 @@
node_modules
.env
dist
*.bun-build
pnpm-lock.yaml
coverage
.pnpm-store
.worktrees/
.DS_Store
**/.DS_Store
ui/src/ui/__screenshots__/
# Bun build artifacts
*.bun-build
apps/macos/.build/
apps/shared/ClawdisKit/.build/
bin/
bin/clawdis-mac
bin/docs-list
apps/macos/.build-local/
apps/macos/.swiftpm/
apps/shared/ClawdisKit/.swiftpm/
Core/
apps/ios/*.xcodeproj/
apps/ios/*.xcworkspace/
apps/ios/.swiftpm/
vendor/
# Vendor build artifacts
vendor/a2ui/renderers/lit/dist/
+2 -10
View File
@@ -16,14 +16,13 @@
- Language: TypeScript (ESM). Prefer strict typing; avoid `any`.
- Formatting/linting via Biome; run `pnpm lint` before commits.
- Keep files concise; extract helpers instead of “V2” copies. Use existing patterns for CLI options and dependency injection via `createDefaultDeps`.
- Aim to keep files under ~700 LOC; guideline only (not a hard guardrail). Split/refactor when it improves clarity or testability.
- Keep every file ≤ 500 LOC; refactor or split before exceeding and check frequently.
## Testing Guidelines
- Framework: Vitest with V8 coverage thresholds (70% lines/branches/functions/statements).
- Naming: match source names with `*.test.ts`; e2e in `*.e2e.test.ts`.
- Run `pnpm test` (or `pnpm test:coverage`) before pushing when you touch logic.
- Pure test additions/fixes generally do **not** need a changelog entry unless they alter user-facing behavior or the user asks for one.
- Mobile: before using a simulator, check for connected real devices (iOS + Android) and prefer them when available.
## Commit & Pull Request Guidelines
- Create commits with `scripts/committer "<msg>" <file...>`; avoid manual `git add`/`git commit` so staging stays scoped.
@@ -41,17 +40,10 @@
- macOS logs: use `./scripts/clawlog.sh` (aka `vtlog`) to query unified logs for subsystem `com.steipete.clawdis`; it supports follow/tail/category filters and expects passwordless sudo for `/usr/bin/log`.
- Also read the shared guardrails at `~/Projects/oracle/AGENTS.md` and `~/Projects/agent-scripts/AGENTS.MD` before making changes; align with any cross-repo rules noted there.
- SwiftUI state management (iOS/macOS): prefer the `Observation` framework (`@Observable`, `@Bindable`) over `ObservableObject`/`@StateObject`; don’t introduce new `ObservableObject` unless required for compatibility, and migrate existing usages when touching related code.
- **Restart apps:** “restart iOS/Android apps” means rebuild (recompile/install) and relaunch, not just kill/launch.
- **Device checks:** before testing, verify connected real devices (iOS/Android) before reaching for simulators/emulators.
- iOS Team ID lookup: `security find-identity -p codesigning -v` → use Apple Development (…) TEAMID. Fallback: `defaults read com.apple.dt.Xcode IDEProvisioningTeamIdentifiers`.
- A2UI bundle hash: `src/canvas-host/a2ui/.bundle.hash` is auto-generated; regenerate via `pnpm canvas:a2ui:bundle` (or `scripts/bundle-a2ui.sh`) instead of manual conflict resolution.
- Notary key file lives at `~/Library/CloudStorage/Dropbox/Backup/AppStore/AuthKey_NJF3NFGTS3.p8` (Sparkle keys live under `~/Library/CloudStorage/Dropbox/Backup/Sparkle`).
- **Multi-agent safety:** do **not** create/apply/drop `git stash` entries unless Peter explicitly asks (this includes `git pull --rebase --autostash`). Assume other agents may be working; keep unrelated WIP untouched and avoid cross-cutting state changes.
- **Multi-agent safety:** when Peter says "push", you may `git pull --rebase` to integrate latest changes (never discard other agents' work). When Peter says "commit", scope to your changes only. When Peter says "commit all", commit everything in grouped chunks.
- **Multi-agent safety:** do **not** create/remove/modify `git worktree` checkouts (or edit `.worktrees/*`) unless Peter explicitly asks.
- **Multi-agent safety:** do **not** switch branches / check out a different branch unless Peter explicitly asks.
- **Multi-agent safety:** running multiple agents is OK as long as each agent has its own session.
- When asked to open a “session” file, open the Pi session logs under `~/.clawdis/sessions/*.jsonl` (newest unless a specific ID is given), not the default `sessions.json`. If logs are needed from Mac Studio, SSH via Tailscale and read the same path there.
- When asked to open a “session” file, open the Pi session logs under `~/.clawdis/sessions/*.jsonl` (newest unless a specific ID is given), not the default `sessions.json`.
- Menubar dimming + restart flow mirrors Trimmy: use `scripts/restart-mac.sh` (kills all Clawdis variants, runs `swift build`, packages, relaunches). Icon dimming depends on MenuBarExtraAccess wiring in AppMain; keep `appearsDisabled` updates intact when touching the status item.
- Never send streaming/partial replies to external messaging surfaces (WhatsApp, Telegram); only final replies should be delivered there. Streaming/tool events may still go to internal UIs/control channel.
- Voice wake forwarding tips:
+7 -229
View File
@@ -1,227 +1,5 @@
# Changelog
## 2.0.0-beta5 — Unreleased
### Breaking
- Skills config schema moved under `skills.*`:
- `skillsLoad.extraDirs` → `skills.load.extraDirs`
- `skillsInstall.*` → `skills.install.*`
- per-skill config map moved to `skills.entries` (e.g. `skills.peekaboo.enabled` → `skills.entries.peekaboo.enabled`)
- new optional bundled allowlist: `skills.allowBundled` (only affects bundled skills)
### Features
- Talk mode: continuous speech conversations (macOS/iOS/Android) with ElevenLabs TTS, reply directives, and optional interrupt-on-speech.
- UI: add optional `ui.seamColor` accent to tint the Talk Mode side bubble (macOS/iOS/Android).
- Nix mode: opt-in declarative config + read-only settings UI when `CLAWDIS_NIX_MODE=1` (thanks @joshp123 for the persistence — earned my trust; I'll merge these going forward).
- Agent runtime: accept legacy `Z_AI_API_KEY` for Z.AI provider auth (maps to `ZAI_API_KEY`).
- Tests: add a Z.AI live test gate for smoke validation when keys are present.
- macOS Debug: add app log verbosity and rolling file log toggle for swift-log-backed app logs.
### Fixes
- macOS codesign: skip hardened runtime for ad-hoc signing and avoid empty options args (#70) — thanks @petter-b
- Docs/agent tools: clarify that browser `wait` should be avoided by default and used only in exceptional cases.
- Browser tools: `upload` supports auto-click refs, direct `inputRef`/`element` file inputs, and emits input/change after `setFiles` so JS-heavy sites pick up attachments.
- macOS: Voice Wake now fully tears down the Speech pipeline when disabled (cancel pending restarts, drop stale callbacks) to avoid high CPU in the background.
- macOS menu: add a Talk Mode action alongside the Open Dashboard/Chat/Canvas entries.
- macOS Debug: hide “Restart Gateway” when the app won’t start a local gateway (remote mode / attach-only).
- macOS Talk Mode: orb overlay refresh, ElevenLabs request logging, API key status in settings, and auto-select first voice when none is configured.
- macOS Talk Mode: add hard timeout around ElevenLabs TTS synthesis to avoid getting stuck “speaking” forever on hung requests.
- macOS Talk Mode: avoid stuck playback when the audio player never starts (fail-fast + watchdog).
- macOS Talk Mode: fix audio stop ordering so disabling Talk Mode always stops in-flight playback.
- macOS Talk Mode: throttle audio-level updates (avoid per-buffer task creation) to reduce CPU/task churn.
- macOS Talk Mode: increase overlay window size so wave rings don’t clip; close button is hover-only and closer to the orb.
- Talk Mode: fall back to system TTS when ElevenLabs is unavailable, returns non-audio, or playback fails (macOS/iOS/Android).
- Talk Mode: stream PCM on macOS/iOS for lower latency (incremental playback); Android continues MP3 streaming.
- Talk Mode: validate ElevenLabs v3 stability and latency tier directives before sending requests.
- iOS/Android Talk Mode: auto-select the first ElevenLabs voice when none is configured.
- ElevenLabs: add retry/backoff for 429/5xx and include content-type in errors for debugging.
- Talk Mode: align to the gateway’s main session key and fall back to history polling when chat events drop (prevents stuck “thinking” / missing messages).
- Talk Mode: treat history timestamps as seconds or milliseconds to avoid stale assistant picks (macOS/iOS/Android).
- Chat UI: clear streaming/tool bubbles when external runs finish, preventing duplicate assistant bubbles.
- Chat UI: user bubbles use `ui.seamColor` (fallback to a calmer default blue).
- Android Chat UI: use `onPrimary` for user bubble text to preserve contrast (thanks @Syhids).
- Control UI: sync sidebar navigation with the URL for deep-linking, and auto-scroll chat to the latest message.
- Control UI: disable Web Chat + Talk when no iOS/Android node is connected; refreshed Web Chat styling and keyboard send.
- macOS Web Chat: fix composer layout so the connection pill and send button stay inside the input field.
- macOS: bundle Control UI assets into the app relay so the packaged app can serve them (thanks @mbelinky).
- Talk Mode: wait for chat history to surface the assistant reply before starting TTS (macOS/iOS/Android).
- iOS Talk Mode: fix chat completion wait to time out even if no events arrive (prevents “Thinking…” hangs).
- iOS Talk Mode: keep recognition running during playback to support interrupt-on-speech.
- iOS Talk Mode: preserve directive voice/model overrides across config reloads and add ElevenLabs request timeouts.
- iOS/Android Talk Mode: explicitly `chat.subscribe` when Talk Mode is active, so completion events arrive even if the Chat UI isn’t open.
- Chat UI: refresh history when another client finishes a run in the same session, so Talk Mode + Voice Wake transcripts appear consistently.
- Gateway: `voice.transcript` now also maps agent bus output to `chat` events, ensuring chat UIs refresh for voice-triggered runs.
- iOS/Android: show a centered Talk Mode orb overlay while Talk Mode is enabled.
- Gateway config: inject `talk.apiKey` from `ELEVENLABS_API_KEY`/shell profile so nodes can fetch it on demand.
- Canvas A2UI: tag requests with `platform=android|ios|macos` and boost Android canvas background contrast.
- iOS/Android nodes: enable scrolling for loaded web pages in the Canvas WebView (default scaffold stays touch-first).
- macOS menu: device list now uses `node.list` (devices only; no agent/tool presence entries).
- macOS menu: device list now shows connected nodes only.
- macOS menu: device rows now pack platform/version on the first line, and command lists wrap in submenus.
- macOS menu: split device platform/version across first and second rows for better fit.
- iOS node: fix ReplayKit screen recording crash caused by queue isolation assertions during capture.
- iOS Talk Mode: avoid audio tap queue assertions when starting recognition.
- macOS: use $HOME/Library/pnpm for SSH PATH exports (thanks @mbelinky).
- iOS/Android nodes: bridge auto-connect refreshes stale tokens and settings now show richer bridge/device details.
- macOS: bundle device model resources to prevent Instances crashes (thanks @mbelinky).
- iOS/Android nodes: status pill now surfaces camera activity instead of overlay toasts.
- iOS/Android/macOS nodes: camera snaps recompress to keep base64 payloads under 5 MB.
- iOS/Android nodes: status pill now surfaces pairing, screen recording, voice wake, and foreground-required states.
- iOS/Android nodes: avoid duplicating “Gateway reconnecting…” when the bridge is already connecting.
- iOS/Android nodes: Talk Mode now lives on a side bubble (with an iOS toggle to hide it), and Android settings no longer show the Talk Mode switch.
- macOS menu: top status line now shows pending node pairing approvals (incl. repairs).
- CLI: avoid spurious gateway close errors after successful request/response cycles.
- Agent runtime: clamp tool-result images to the 5MB Anthropic limit to avoid hard request rejections.
- Tests: add Swift Testing coverage for camera errors and Kotest coverage for Android bridge endpoints.
## 2.0.0-beta4 — 2025-12-27
### Fixes
- Package contents: include Discord/hooks build outputs in the npm tarball to avoid missing module errors.
- Heartbeat replies now drop any output containing `HEARTBEAT_OK`, preventing stray emoji/text from being delivered.
- macOS menu now refreshes the control channel after the gateway starts and shows “Connecting to gateway…” while the gateway is coming up.
- macOS local mode now waits for the gateway to be ready before configuring the control channel, avoiding false “no connection” flashes.
- WhatsApp watchdog now forces a reconnect even if the socket close event stalls (force-close to unblock reconnect loop).
- Gateway presence now reports macOS product version (via `sw_vers`) instead of Darwin kernel version.
## 2.0.0-beta3 — 2025-12-27
### Highlights
- First-class Clawdis tools (browser, canvas, nodes, cron) replace the old `clawdis-*` skills; tool schemas are now injected directly into the agent runtime.
- Per-session model selection + custom model providers: `models.providers` merges into `~/.clawdis/agent/models.json` (merge/replace modes) for LiteLLM, local OpenAI-compatible servers, Anthropic proxies, etc.
- Group chat activation modes: per-group `/activation mention|always` command with status visibility.
- Discord bot transport for DMs and guild text channels, with allowlists + mention gating.
- Gateway webhooks: external `wake` and isolated `agent` hooks with dedicated token auth.
- Hook mappings + Gmail Pub/Sub helper (`clawdis hooks gmail setup/run`) with auto-renew + Tailscale Funnel support.
- Command queue modes + per-session overrides (`/queue ...`) and new `agent.maxConcurrent` cap for safe parallelism across sessions.
- Background bash tasks: `bash` auto-yields after 20s (or on demand) with a `process` tool to list/poll/log/write/kill sessions.
- Gateway in-process restart: `clawdis_gateway` tool action triggers a SIGUSR1 restart without needing a supervisor.
### Breaking
- Config refactor: `inbound.*` removed; use top-level `routing` (allowlists + group rules + transcription), `messages` (prefixes/timestamps), and `session` (scoping/store/mainKey). No legacy keys read.
- Heartbeat config moved to `agent.heartbeat`: set `every: "30m"` (duration string) and optional `model`. `agent.heartbeatMinutes` is removed, and heartbeats are disabled unless `agent.heartbeat.every` is set.
- Heartbeats now run via the gateway runner (main session) and deliver to the last used channel by default. WhatsApp reply-heartbeat behavior is removed; use `agent.heartbeat.target`/`to` (or `target: "none"`) to control delivery.
- Browser `act` no longer accepts CSS `selector`; use `snapshot` refs (default `ai`) or `evaluate` as an escape hatch.
### Fixes
- Heartbeat replies now strip repeated `HEARTBEAT_OK` tails to avoid accidental “OK OK” spam.
- Heartbeat delivery now uses the last non-empty payload, preventing tool preambles from swallowing the final reply.
- Heartbeats now skip WhatsApp delivery when the web provider is inactive or unlinked (instead of logging “no active gateway listener”).
- Heartbeat failure logs now include the error reason instead of `[object Object]`.
- Duration strings now accept `h` (hours) where durations are parsed (e.g., heartbeat intervals).
- WhatsApp inbound now normalizes more wrapper types so quoted reply bodies are extracted reliably.
- WhatsApp send now preserves existing JIDs (including group `@g.us`) instead of coercing to `@s.whatsapp.net`. (Thanks @arun-8687.)
- Telegram/WhatsApp: reply context stays in `Body`/`ReplyTo*`, but outbound replies no longer thread to the original message. (Thanks @joshp123 for the PR and follow-up question.)
- Suppressed libsignal session cleanup spam from console logs unless verbose mode is enabled.
- WhatsApp web creds persistence hardened; credentials are restored before auth checks and QR login auto-restarts if it stalls.
- Group chats now honor `routing.groupChat.requireMention=false` as the default activation when no per-group override exists.
- Gateway auth no longer supports PAM/system mode; use token or shared password.
- Tailscale Funnel now requires password auth (no token-only public exposure).
- Group `/new` resets now work with @mentions so activation guidance appears on fresh sessions.
- Group chat activation context is now injected into the system prompt at session start (and after activation changes), including /new greetings.
- Typing indicators now start only once a reply payload is produced (no "thinking" typing for silent runs).
- WhatsApp group typing now starts immediately only when the bot is mentioned; otherwise it waits until real output exists.
- Streamed `<think>` segments are stripped before partial replies are emitted.
- System prompt now tags allowlisted owner numbers as the user identity to avoid mistaken “friend” assumptions.
- LM Studio/Ollama replies now require <final> tags; streaming ignores content until <final> begins.
- LM Studio responses API: tools payloads no longer include `strict: null`, and LM Studio no longer gets forced `<think>/<final>` tags.
- Identity emoji no longer auto-prefixes replies (set `messages.responsePrefix` explicitly if desired).
- Model switches now enqueue a system event so the next run knows the active model.
- `/model status` now lists available models (same as `/model`).
- `process log` pagination is now line-based (omit `offset` to grab the last N lines).
- macOS WebChat: assistant bubbles now update correctly when toggling light/dark mode.
- macOS: avoid spawning a duplicate gateway process when an external listener already exists.
- Node bridge: when binding to a non-loopback host (e.g. Tailnet IP), also listens on `127.0.0.1` for local connections (without creating duplicate loopback listeners for `0.0.0.0`/`127.0.0.1` binds).
- UI perf: pause repeat animations when scenes are inactive (typing dots, onboarding glow, iOS status pulse), throttle voice overlay level updates, and reduce overlay focus churn.
- Canvas defaults/A2UI auto-nav aligned; debug status overlay centered; redundant await removed in `CanvasManager`.
- Gateway launchd loop fixed by removing redundant `kickstart -k`.
- CLI now hints when Peekaboo is unauthorized.
- WhatsApp web inbox listeners now clean up on close to avoid duplicate handlers.
- Gateway startup now brings up browser control before external providers; WhatsApp/Telegram/Discord auto-start can be disabled with `web.enabled`, `telegram.enabled`, or `discord.enabled`.
### Providers & Routing
- New Discord provider for DMs + guild text channels with allowlists and mention-gated replies by default.
- `routing.queue` now controls queue vs interrupt behavior globally + per surface (defaults: WhatsApp/Telegram interrupt, Discord/WebChat queue).
- `/queue <mode>` supports one-shot or per-session overrides; `/queue reset|default` clears overrides.
- `agent.maxConcurrent` caps global parallel runs while keeping per-session serialization.
### macOS app
- Update-ready state surfaced in the menu; menu sections regrouped with session submenus.
- Menu bar now shows a dedicated Nodes section under Context with inline rows, overflow submenu, and iconized actions.
- Nodes now expose consistent inline details with per-node submenus for quick copy of key fields.
- Node rows now show compact app versions (build numbers moved to submenus) and offer SSH launch from Bonjour when available.
- Menu actions are grouped below toggles; Open Canvas hides when disabled and Voice Wake now anchors the mic picker.
- Connections now include Discord provider status + configuration UI.
- Menu bar gains an Allow Camera toggle alongside Canvas.
- Session list polish: sleeping/disconnected/error states, usage bar restored, padding + bar sizing tuned, syncing menu removed, header hidden when disconnected.
- Chat UI polish: tool call cards + merged tool results, glass background, tighter composer spacing, visual effect host tweaks.
- OAuth storage moved; legacy session syncing metadata removed.
- Remote SSH tunnels now get health checks; Debug → Ports highlights unhealthy tunnels and offers Reset SSH tunnel.
- Menu bar session/node sections no longer reflow while open, keeping hover highlights aligned.
- Menu hover highlights now span the full width (including submenu arrows).
- Menu session rows now refresh while open without width changes (no more stuck “Loading sessions…”).
- Menu width no longer grows on hover when moving the mouse across rows.
- Context usage bars now have higher contrast in light mode.
- macOS node timeouts now share a single async timeout helper for consistent behavior.
- WebChat window defaults tightened (narrower width, edge-to-edge layout) and the SwiftUI tag removed from the title.
### Nodes & Canvas
- Debug status overlay gated and toggleable on macOS/iOS/Android nodes.
- Gateway now derives the canvas host URL via a shared helper for bridge + WS handshakes (avoids loopback pitfalls).
- `canvas a2ui push` validates JSONL with line errors, rejects v0.9 payloads, and supports `--text` quick renders.
- `nodes rename` lets you override paired node display names without editing JSON.
- Android scaffold asset cleanup; iOS canvas/voice wake adjustments.
### Logging & Observability
- New subsystem console formatter with color modes, shortened prefixes, and TTY detection; browser/gateway logs route through the subsystem logger.
- WhatsApp console output streamlined; chalk/tslog typing fixes.
### Web UI
- Chat is now the dashboard landing view; health status simplified; initial scroll animation removed.
### Build, Dev, Docs
- Notarization flow added for macOS release artifacts; packaging scripts updated.
- macOS signing auto-selects Developer ID → Apple Distribution → Apple Development; no ad-hoc fallback.
- Added type-aware oxlint; docs list resolves from cwd; formatting/lint cleanup and dependency bumps (Peekaboo).
- Docs refreshed for tools, custom model providers, Discord, queue/routing, group activation commands, logging, restart semantics, release notes, GitHub pages CTAs, and npm pitfalls.
- `pnpm build` now skips A2UI bundling for faster builds (run `pnpm canvas:a2ui:bundle` when needed).
### Tests
- Coverage added for models config merging, WhatsApp reply context, QR login flows, auto-reply behavior, and gateway SIGTERM timeouts.
- Added gateway webhook coverage (auth, validation, and summary posting).
- Vitest now isolates HOME/XDG config roots so tests never touch a real `~/.clawdis` install.
## 2.0.0-beta2 — 2025-12-21
Second beta focused on bundled gateway packaging, skills management, onboarding polish, and provider reliability.
### Highlights
- Bundled gateway packaging: bun-compiled embedded gateway, new `gateway-daemon` command, launchd support, DMG packaging (zip+DMG).
- Skills platform: managed/bundled skills, install metadata + installers (uv), skill search + website, media/transcription helpers.
- macOS app: new Connections settings w/ provider status + QR login, skills settings redesign w/ install targets, models list loaded from the Gateway, clearer local/remote gateway choices.
- Web/agent UX: tool summary streaming + runtime toggle, WhatsApp QR login tool, agent steering queue, voice wake routes to main session, workspace bootstrap ritual.
### Gateway & providers
- Gateway: `models.list`, provider status events + RPC coverage, tailscale auth + PAM, bind-mode config, enriched agent WS logs, safer upgrade socket handling, fixed handshake auth crash.
- WhatsApp Web: QR login flow improvements (logged-out clearing, wait flow), self-chat mode handling, removed batching delay, web inbox made non-blocking.
- Telegram: normalized chat IDs with clearer error reporting.
### Canvas & browser control
- Canvas host served on Gateway port; removed standalone canvasHost port config; restored action bridge; refreshed A2UI bundle + message context; bridge canvas host for nodes.
- A2UI full-screen gutters + status clearance after successful load to avoid overlay collisions.
- Browser control API simplified; added MCP tool dispatch + native actions; control server can start without Playwright; hook timeouts extended.
### macOS UI polish
- Onboarding chat UI: kickoff flow, bubble tails, spacing + bottom bar refinements, window sizing tweaks, show Dock icon during onboarding.
- Skills UI: stabilized action column, fixed install target access, refined list layout and sizing, always show CLI installer.
- Remote/local gateway: auto-enable local gateway, clearer labels, re-ensure remote tunnel, hide local bridge discovery in remote mode.
### Build, CI, deps
- Bundled playwright-core + chromium-bidi/long; bun gateway bytecode builds; swiftformat/biome CI fixes; iOS lint script updates; Android icon/compiler updates; ignored new ClawdisKit `.swiftpm` path.
### Docs
- README architecture refresh + npm header image fix; onboarding/bootstrap steps; skills install guidance + new skills; browser/canvas control docs; bundled gateway + DMG packaging notes.
## 2.0.0-beta1 — 2025-12-19
First Clawdis release post rebrand. This is a semver-major because we dropped legacy providers/agents and moved defaults to new paths while adding a full macOS companion app, a WebSocket Gateway, and an iOS node.
@@ -231,7 +9,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
### Breaking
- Renamed to **Clawdis**: defaults now live under `~/.clawdis` (sessions in `~/.clawdis/sessions/`, IPC at `~/.clawdis/clawdis.sock`, logs in `/tmp/clawdis`). Launchd labels and config filenames follow the new name; legacy stores are copied forward on first run.
- Pi only: only the embedded Pi runtime remains, and the agent CLI/CLI flags for Claude/Codex/Gemini were removed. The Pi CLI runs in RPC mode with a persistent worker.
- Pi only: `inbound.reply.agent.kind` accepts only `"pi"`, and the agent CLI/CLI flags for Claude/Codex/Gemini were removed. The Pi CLI runs in RPC mode with a persistent worker.
- WhatsApp Web is the only transport; Twilio support and related CLI flags/tests were removed.
- Direct chats now collapse into a single `main` session by default (no config needed); groups stay isolated as `group:<jid>`.
- Gateway is now a loopback-only WebSocket daemon (`ws://127.0.0.1:18789`) that owns all providers/state; clients (CLI, WebChat, macOS app, nodes) connect to it. Start it explicitly (`clawdis gateway …`) or via Clawdis.app; helper subcommands no longer auto-spawn a gateway.
@@ -279,7 +57,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.5.0 — 2025-12-05
### Breaking
- Dropped all non-Pi agents (Claude, Codex, Gemini, Opencode); only the embedded Pi runtime remains and related CLI helpers have been removed.
- Dropped all non-Pi agents (Claude, Codex, Gemini, Opencode); `inbound.reply.agent.kind` now only accepts `"pi"` and related CLI helpers have been removed.
- Removed Twilio support and all related commands/options (webhook/up/provider flags/wait-poll); CLAWDIS is Baileys Web-only.
### Changes
@@ -306,7 +84,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.4.0 — 2025-12-03
### Highlights
- **Thinking directives & state:** `/t|/think|/thinking <level>` (aliases off|minimal|low|medium|high|max/highest). Inline applies to that message; directive-only message pins the level for the session; `/think:off` clears. Resolution: inline > session override > `agent.thinkingDefault` > off. Pi gets `--thinking <level>` (except off); other agents append cue words (`think` → `think hard` → `think harder` → `ultrathink`). Heartbeat probe uses `HEARTBEAT /think:high`.
- **Thinking directives & state:** `/t|/think|/thinking <level>` (aliases off|minimal|low|medium|high|max/highest). Inline applies to that message; directive-only message pins the level for the session; `/think:off` clears. Resolution: inline > session override > `inbound.reply.thinkingDefault` > off. Pi gets `--thinking <level>` (except off); other agents append cue words (`think` → `think hard` → `think harder` → `ultrathink`). Heartbeat probe uses `HEARTBEAT /think:high`.
- **Group chats (web provider):** Clawdis now fully supports WhatsApp groups: mention-gated triggers (including image-only @ mentions), recent group history injection, per-group sessions, sender attribution, and a first-turn primer with group subject/member roster; heartbeats are skipped for groups.
- **Group session primer:** The first turn of a group session now tells the agent it is in a WhatsApp group and lists known members/subject so it can address the right speaker.
- **Media failures are surfaced:** When a web auto-reply media fetch/send fails (e.g., HTTP 404), we now append a warning to the fallback text so you know the attachment was skipped.
@@ -348,7 +126,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.3.0 — 2025-12-02
### Highlights
- **Pluggable agents (Claude, Pi, Codex, Opencode):** agent selection via config/CLI plus per-agent argv builders and NDJSON parsers enable swapping without template changes.
- **Pluggable agents (Claude, Pi, Codex, Opencode):** `inbound.reply.agent` selects CLI/parser; per-agent argv builders and NDJSON parsers enable swapping without template changes.
- **Safety stop words:** `stop|esc|abort|wait|exit` immediately reply “Agent was aborted.” and mark the session so the next prompt is prefixed with an abort reminder.
- **Agent session reliability:** Only Claude returns a stable `session_id`; others may reset between runs.
@@ -365,7 +143,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
- Batched inbound messages with timestamps; typing indicator after sends.
- Watchdog restarts WhatsApp after long inactivity; heartbeat logging includes minutes since last message.
- Early `allowFrom` filtering before decryption.
- Same-phone mode with echo detection and optional message prefix marker.
- Same-phone mode with echo detection and optional `inbound.samePhoneMarker`.
## 1.2.2 — 2025-11-28
@@ -395,10 +173,10 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.1.0 — 2025-11-26
### Changes
- Web auto-replies resize/recompress media and honor `agent.mediaMaxMb`.
- Web auto-replies resize/recompress media and honor `inbound.reply.mediaMaxMb`.
- Detect media kind, enforce provider caps (images ≤6MB, audio/video ≤16MB, docs ≤100MB).
- `session.sendSystemOnce` and optional `sessionIntro`.
- Typing indicator refresh during commands; configurable via `agent.typingIntervalSeconds`.
- Typing indicator refresh during commands; configurable via `inbound.reply.typingIntervalSeconds`.
- Optional audio transcription via external CLI.
- Command replies return structured payload/meta; respect `mediaMaxMb`; log Claude metadata; include `cwd` in timeout messages.
- Web provider refactor; logout command; web-only gateway start helper.
Submodule Peekaboo updated: 9db365b73c...be03d6e3df.
+7 -49
View File
@@ -1,7 +1,7 @@
# 🦞 CLAWDIS — Personal AI Assistant
<p align="center">
<img src="https://raw.githubusercontent.com/steipete/clawdis/main/docs/whatsapp-clawd.jpg" alt="CLAWDIS" width="400">
<img src="docs/whatsapp-clawd.jpg" alt="CLAWDIS" width="400">
</p>
<p align="center">
@@ -15,12 +15,10 @@
</p>
**Clawdis** is a *personal AI assistant* you run on your own devices.
It answers you on the surfaces you already use (WhatsApp, Telegram, Discord, WebChat), can speak and listen on macOS/iOS, and can render a live Canvas you control. The Gateway is just the control plane — the product is the assistant.
It answers you on the surfaces you already use (WhatsApp, Telegram, WebChat), can speak and listen on macOS/iOS, and can render a live Canvas you control. The Gateway is just the control plane — the product is the assistant.
If you want a private, single-user assistant that feels local, fast, and always-on, this is it.
Using Claude Pro/Max subscription? See `docs/onboarding.md` for the Anthropic OAuth setup.
```
Your surfaces
│
@@ -40,13 +38,11 @@ Your surfaces
## What Clawdis does
- **Personal assistant** — one user, one identity, one memory surface.
- **Multi-surface inbox** — WhatsApp, Telegram, Discord, WebChat, macOS, iOS.
- **Multi-surface inbox** — WhatsApp, Telegram, WebChat, macOS, iOS.
- **Voice wake + push-to-talk** — local speech recognition on macOS/iOS.
- **Canvas** — a live visual workspace you can drive from the agent.
- **Automation-ready** — browser control, media handling, and tool streaming.
- **Local-first control plane** — the Gateway owns state, everything else connects.
- **Group chats** — mention-based by default, `/activation always|mention` per group (owner-only).
- **Nix mode** — opt-in declarative config + read-only UI when `CLAWDIS_NIX_MODE=1`.
## How it works (short)
@@ -70,29 +66,15 @@ pnpm clawdis login
# Start the gateway
pnpm clawdis gateway --port 18789 --verbose
# Dev loop (auto-reload on TS changes)
pnpm gateway:watch
# Send a message
pnpm clawdis send --to +1234567890 --message "Hello from Clawdis"
# Talk to the assistant (optionally deliver back to WhatsApp/Telegram/Discord)
# Talk to the assistant (optionally deliver back to WhatsApp/Telegram)
pnpm clawdis agent --message "Ship checklist" --thinking high
```
If you run from source, prefer `pnpm clawdis …` (not global `clawdis`).
## Chat commands
Send these in WhatsApp/Telegram/WebChat (group commands are owner-only):
- `/status` — health + session info (group shows activation mode)
- `/new` or `/reset` — reset the session
- `/think <level>` — off|minimal|low|medium|high
- `/verbose on|off`
- `/restart` — restart the gateway (owner-only in groups)
- `/activation mention|always` — group activation toggle (groups only)
## Architecture
### TypeScript Gateway (src/gateway/server.ts)
@@ -139,7 +121,7 @@ Runbook: `docs/ios/connect.md`.
## Agent workspace + skills
- Workspace root: `~/clawd` (configurable via `agent.workspace`).
- Workspace root: `~/clawd` (configurable via `inbound.workspace`).
- Injected prompt files: `AGENTS.md`, `SOUL.md`, `TOOLS.md`.
- Skills: `~/clawd/skills/<skill>/SKILL.md`.
@@ -149,7 +131,7 @@ Minimal `~/.clawdis/clawdis.json`:
```json5
{
routing: {
inbound: {
allowFrom: ["+1234567890"]
}
}
@@ -158,7 +140,7 @@ Minimal `~/.clawdis/clawdis.json`:
### WhatsApp
- Link the device: `pnpm clawdis login` (stores creds in `~/.clawdis/credentials`).
- Allowlist who can talk to the assistant via `routing.allowFrom`.
- Allowlist who can talk to the assistant via `inbound.allowFrom`.
### Telegram
@@ -173,19 +155,6 @@ Minimal `~/.clawdis/clawdis.json`:
}
```
### Discord
- Set `DISCORD_BOT_TOKEN` or `discord.token` (env wins).
- Optional: set `discord.requireMention`, `discord.allowFrom`, or `discord.mediaMaxMb` as needed.
```json5
{
discord: {
token: "1234abcd"
}
}
```
Browser control (optional):
```json5
@@ -202,21 +171,10 @@ Browser control (optional):
- [`docs/index.md`](docs/index.md) (overview)
- [`docs/configuration.md`](docs/configuration.md)
- [`docs/group-messages.md`](docs/group-messages.md)
- [`docs/gateway.md`](docs/gateway.md)
- [`docs/web.md`](docs/web.md)
- [`docs/discovery.md`](docs/discovery.md)
- [`docs/agent.md`](docs/agent.md)
- [`docs/discord.md`](docs/discord.md)
- Webhooks + external triggers: [`docs/webhook.md`](docs/webhook.md)
- Gmail hooks (email → wake): [`docs/gmail-pubsub.md`](docs/gmail-pubsub.md)
## Email hooks (Gmail)
```bash
clawdis hooks gmail setup --account you@gmail.com
clawdis hooks gmail run
```
- [`docs/security.md`](docs/security.md)
- [`docs/troubleshooting.md`](docs/troubleshooting.md)
- [`docs/ios/connect.md`](docs/ios/connect.md)
-11
View File
@@ -1,11 +0,0 @@
# Changelog
## 0.2.0 — 2025-12-23
### Highlights
- Added `SwabbleKit` (multi-platform wake-word gate utilities with segment-aware gap detection).
- Swabble package now supports iOS + macOS consumers; CLI remains macOS 26-only.
### Changes
- CLI wake-word matching/stripping routed through `SwabbleKit` helpers.
- Speech pipeline types now explicitly gated to macOS 26 / iOS 26 availability.
+10 -1
View File
@@ -1,6 +1,15 @@
{
"originHash" : "2012d083159d375d07febbc184c592c569d7ab48247045e35a762e3269d4cadc",
"originHash" : "3018b2c8c183d55b57ad0c4526b2380ac3b957d13a3a86e1b2845e81323c443a",
"pins" : [
{
"identity" : "commander",
"kind" : "remoteSourceControl",
"location" : "https://github.com/steipete/Commander.git",
"state" : {
"revision" : "8b8cb4f34315ce9e5307b3a2bcd77ff73f586a02",
"version" : "0.2.0"
}
},
{
"identity" : "swift-syntax",
"kind" : "remoteSourceControl",
+2 -21
View File
@@ -4,16 +4,14 @@ import PackageDescription
let package = Package(
name: "swabble",
platforms: [
.macOS(.v15),
.iOS(.v17),
.macOS(.v26),
],
products: [
.library(name: "Swabble", targets: ["Swabble"]),
.library(name: "SwabbleKit", targets: ["SwabbleKit"]),
.executable(name: "swabble", targets: ["SwabbleCLI"]),
],
dependencies: [
.package(path: "../Peekaboo/Commander"),
.package(url: "https://github.com/steipete/Commander.git", from: "0.2.0"),
.package(url: "https://github.com/apple/swift-testing", from: "0.99.0"),
],
targets: [
@@ -21,30 +19,13 @@ let package = Package(
name: "Swabble",
path: "Sources/SwabbleCore",
swiftSettings: []),
.target(
name: "SwabbleKit",
path: "Sources/SwabbleKit",
swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"),
]),
.executableTarget(
name: "SwabbleCLI",
dependencies: [
"Swabble",
"SwabbleKit",
.product(name: "Commander", package: "Commander"),
],
path: "Sources/swabble"),
.testTarget(
name: "SwabbleKitTests",
dependencies: [
"SwabbleKit",
.product(name: "Testing", package: "swift-testing"),
],
swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"),
.enableExperimentalFeature("SwiftTesting"),
]),
.testTarget(
name: "swabbleTests",
dependencies: [
+4 -8
View File
@@ -1,10 +1,9 @@
# 🎙️ swabble — Speech.framework wake-word hook daemon (macOS 26)
swabble is a Swift 6.2 wake-word hook daemon. The CLI targets macOS 26 (SpeechAnalyzer + SpeechTranscriber). The shared `SwabbleKit` target is multi-platform and exposes wake-word gating utilities for iOS/macOS apps.
swabble is a Swift 6.2, macOS 26-only rewrite of the brabble voice daemon. It listens on your mic, gates on a wake word, transcribes locally using Apple's new SpeechAnalyzer + SpeechTranscriber, then fires a shell hook with the transcript. No cloud calls, no Whisper binaries.
- **Local-only**: Speech.framework on-device models; zero network usage.
- **Wake word**: Default `clawd` (aliases `claude`), optional `--no-wake` bypass.
- **SwabbleKit**: Shared wake gate utilities (gap-based gating when you provide speech segments).
- **Hooks**: Run any command with prefix/env, cooldown, min_chars, timeout.
- **Services**: launchd helper stubs for start/stop/install.
- **File transcribe**: TXT or SRT with time ranges (using AttributedString splits).
@@ -31,7 +30,7 @@ swift run swabble transcribe /path/to/audio.m4a --format srt --output out.srt
```
## Use as a library
Add swabble as a SwiftPM dependency and import the `Swabble` or `SwabbleKit` product:
Add swabble as a SwiftPM dependency and import the `Swabble` product to reuse the Speech pipeline, config loader, hook executor, and transcript store in your own app:
```swift
// Package.swift
@@ -39,10 +38,7 @@ dependencies: [
.package(url: "https://github.com/steipete/swabble.git", branch: "main"),
],
targets: [
.target(name: "MyApp", dependencies: [
.product(name: "Swabble", package: "swabble"), // Speech pipeline (macOS 26+ / iOS 26+)
.product(name: "SwabbleKit", package: "swabble"), // Wake-word gate utilities (iOS 17+ / macOS 15+)
]),
.target(name: "MyApp", dependencies: [.product(name: "Swabble", package: "swabble")]),
]
```
@@ -97,7 +93,7 @@ Environment variables:
## Speech pipeline
- `AVAudioEngine` tap → `BufferConverter` → `AnalyzerInput` → `SpeechAnalyzer` with a `SpeechTranscriber` module.
- Requests volatile + final results; the CLI uses text-only wake gating today.
- Requests volatile + final results; wake gating is string match on partial/final.
- Authorization requested at first start; requires macOS 26 + new Speech.framework APIs.
## Development
@@ -2,13 +2,11 @@ import AVFoundation
import Foundation
import Speech
@available(macOS 26.0, iOS 26.0, *)
public struct SpeechSegment: Sendable {
public let text: String
public let isFinal: Bool
}
@available(macOS 26.0, iOS 26.0, *)
public enum SpeechPipelineError: Error {
case authorizationDenied
case analyzerFormatUnavailable
@@ -16,7 +14,6 @@ public enum SpeechPipelineError: Error {
}
/// Live microphone → SpeechAnalyzer → SpeechTranscriber pipeline.
@available(macOS 26.0, iOS 26.0, *)
public actor SpeechPipeline {
private struct UnsafeBuffer: @unchecked Sendable { let buffer: AVAudioPCMBuffer }
@@ -1,192 +0,0 @@
import Foundation
public struct WakeWordSegment: Sendable, Equatable {
public let text: String
public let start: TimeInterval
public let duration: TimeInterval
public let range: Range<String.Index>?
public init(text: String, start: TimeInterval, duration: TimeInterval, range: Range<String.Index>? = nil) {
self.text = text
self.start = start
self.duration = duration
self.range = range
}
public var end: TimeInterval { self.start + self.duration }
}
public struct WakeWordGateConfig: Sendable, Equatable {
public var triggers: [String]
public var minPostTriggerGap: TimeInterval
public var minCommandLength: Int
public init(
triggers: [String],
minPostTriggerGap: TimeInterval = 0.45,
minCommandLength: Int = 1)
{
self.triggers = triggers
self.minPostTriggerGap = minPostTriggerGap
self.minCommandLength = minCommandLength
}
}
public struct WakeWordGateMatch: Sendable, Equatable {
public let triggerEndTime: TimeInterval
public let postGap: TimeInterval
public let command: String
public init(triggerEndTime: TimeInterval, postGap: TimeInterval, command: String) {
self.triggerEndTime = triggerEndTime
self.postGap = postGap
self.command = command
}
}
public enum WakeWordGate {
private struct Token {
let normalized: String
let start: TimeInterval
let end: TimeInterval
let range: Range<String.Index>?
let text: String
}
private struct TriggerTokens {
let tokens: [String]
}
public static func match(
transcript: String,
segments: [WakeWordSegment],
config: WakeWordGateConfig)
-> WakeWordGateMatch? {
let triggerTokens = self.normalizeTriggers(config.triggers)
guard !triggerTokens.isEmpty else { return nil }
let tokens = self.normalizeSegments(segments)
guard !tokens.isEmpty else { return nil }
var best: (index: Int, triggerEnd: TimeInterval, gap: TimeInterval)?
for trigger in triggerTokens {
let count = trigger.tokens.count
guard count > 0, tokens.count > count else { continue }
for i in 0...(tokens.count - count - 1) {
let matched = (0..<count).allSatisfy { tokens[i + $0].normalized == trigger.tokens[$0] }
if !matched { continue }
let triggerEnd = tokens[i + count - 1].end
let nextToken = tokens[i + count]
let gap = nextToken.start - triggerEnd
if gap < config.minPostTriggerGap { continue }
if let best, i <= best.index { continue }
best = (i, triggerEnd, gap)
}
}
guard let best else { return nil }
let command = self.commandText(transcript: transcript, segments: segments, triggerEndTime: best.triggerEnd)
.trimmingCharacters(in: Self.whitespaceAndPunctuation)
guard command.count >= config.minCommandLength else { return nil }
return WakeWordGateMatch(triggerEndTime: best.triggerEnd, postGap: best.gap, command: command)
}
public static func commandText(
transcript: String,
segments: [WakeWordSegment],
triggerEndTime: TimeInterval)
-> String {
let threshold = triggerEndTime + 0.001
for segment in segments where segment.start >= threshold {
if normalizeToken(segment.text).isEmpty { continue }
if let range = segment.range {
let slice = transcript[range.lowerBound...]
return String(slice).trimmingCharacters(in: Self.whitespaceAndPunctuation)
}
break
}
let text = segments
.filter { $0.start >= threshold && !self.normalizeToken($0.text).isEmpty }
.map(\.text)
.joined(separator: " ")
return text.trimmingCharacters(in: Self.whitespaceAndPunctuation)
}
public static func matchesTextOnly(text: String, triggers: [String]) -> Bool {
guard !text.isEmpty else { return false }
let normalized = text.lowercased()
for trigger in triggers {
let token = trigger.trimmingCharacters(in: self.whitespaceAndPunctuation).lowercased()
if token.isEmpty { continue }
if normalized.contains(token) { return true }
}
return false
}
public static func stripWake(text: String, triggers: [String]) -> String {
var out = text
for trigger in triggers {
let token = trigger.trimmingCharacters(in: self.whitespaceAndPunctuation)
guard !token.isEmpty else { continue }
out = out.replacingOccurrences(of: token, with: "", options: [.caseInsensitive])
}
return out.trimmingCharacters(in: self.whitespaceAndPunctuation)
}
private static func normalizeTriggers(_ triggers: [String]) -> [TriggerTokens] {
var output: [TriggerTokens] = []
for trigger in triggers {
let tokens = trigger
.split(whereSeparator: { $0.isWhitespace })
.map { self.normalizeToken(String($0)) }
.filter { !$0.isEmpty }
if tokens.isEmpty { continue }
output.append(TriggerTokens(tokens: tokens))
}
return output
}
private static func normalizeSegments(_ segments: [WakeWordSegment]) -> [Token] {
segments.compactMap { segment in
let normalized = self.normalizeToken(segment.text)
guard !normalized.isEmpty else { return nil }
return Token(
normalized: normalized,
start: segment.start,
end: segment.end,
range: segment.range,
text: segment.text)
}
}
private static func normalizeToken(_ token: String) -> String {
token
.trimmingCharacters(in: self.whitespaceAndPunctuation)
.lowercased()
}
private static let whitespaceAndPunctuation = CharacterSet.whitespacesAndNewlines
.union(.punctuationCharacters)
}
#if canImport(Speech)
import Speech
public enum WakeWordSpeechSegments {
public static func from(transcription: SFTranscription, transcript: String) -> [WakeWordSegment] {
transcription.segments.map { segment in
let range = Range(segment.substringRange, in: transcript)
return WakeWordSegment(
text: segment.substring,
start: segment.timestamp,
duration: segment.duration,
range: range)
}
}
}
#endif
@@ -1,7 +1,6 @@
import Commander
import Foundation
@available(macOS 26.0, *)
@MainActor
enum CLIRegistry {
static var descriptors: [CommandDescriptor] {
@@ -1,9 +1,7 @@
import Commander
import Foundation
import Swabble
import SwabbleKit
@available(macOS 26.0, *)
@MainActor
struct ServeCommand: ParsableCommand {
@Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
@@ -70,12 +68,17 @@ struct ServeCommand: ParsableCommand {
}
private static func matchesWake(text: String, cfg: SwabbleConfig) -> Bool {
let triggers = [cfg.wake.word] + cfg.wake.aliases
return WakeWordGate.matchesTextOnly(text: text, triggers: triggers)
let lowered = text.lowercased()
if lowered.contains(cfg.wake.word.lowercased()) { return true }
return cfg.wake.aliases.contains(where: { lowered.contains($0.lowercased()) })
}
private static func stripWake(text: String, cfg: SwabbleConfig) -> String {
let triggers = [cfg.wake.word] + cfg.wake.aliases
return WakeWordGate.stripWake(text: text, triggers: triggers)
var out = text
out = out.replacingOccurrences(of: cfg.wake.word, with: "", options: [.caseInsensitive])
for alias in cfg.wake.aliases {
out = out.replacingOccurrences(of: alias, with: "", options: [.caseInsensitive])
}
return out.trimmingCharacters(in: .whitespacesAndNewlines)
}
}
+2 -9
View File
@@ -1,7 +1,6 @@
import Commander
import Foundation
@available(macOS 26.0, *)
@MainActor
private func runCLI() async -> Int32 {
do {
@@ -16,7 +15,6 @@ private func runCLI() async -> Int32 {
}
}
@available(macOS 26.0, *)
@MainActor
private func dispatch(invocation: CommandInvocation) async throws {
let parsed = invocation.parsedValues
@@ -97,10 +95,5 @@ private func dispatch(invocation: CommandInvocation) async throws {
}
}
if #available(macOS 26.0, *) {
let exitCode = await runCLI()
exit(exitCode)
} else {
fputs("error: swabble requires macOS 26 or newer\n", stderr)
exit(1)
}
let exitCode = await runCLI()
exit(exitCode)
@@ -1,63 +0,0 @@
import Foundation
import Testing
import SwabbleKit
@Suite struct WakeWordGateTests {
@Test func matchRequiresGapAfterTrigger() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.35, 0.1),
("thing", 0.5, 0.1),
])
let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
#expect(WakeWordGate.match(transcript: transcript, segments: segments, config: config) == nil)
}
@Test func matchAllowsGapAndExtractsCommand() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.9, 0.1),
("thing", 1.1, 0.1),
])
let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
#expect(match?.command == "do thing")
}
@Test func matchHandlesMultiWordTriggers() {
let transcript = "hey clawd do it"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.8, 0.1),
("it", 1.0, 0.1),
])
let config = WakeWordGateConfig(triggers: ["hey clawd"], minPostTriggerGap: 0.3)
let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
#expect(match?.command == "do it")
}
}
private func makeSegments(
transcript: String,
words: [(String, TimeInterval, TimeInterval)])
-> [WakeWordSegment] {
var searchStart = transcript.startIndex
var output: [WakeWordSegment] = []
for (word, start, duration) in words {
let range = transcript.range(of: word, range: searchStart..<transcript.endIndex)
output.append(WakeWordSegment(text: word, start: start, duration: duration, range: range))
if let range { searchStart = range.upperBound }
}
return output
}
+3 -4
View File
@@ -1,12 +1,11 @@
# swabble — macOS 26 speech hook daemon (Swift 6.2)
Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framework (SpeechAnalyzer + SpeechTranscriber) instead of whisper.cpp. Local-only, wake word gated, dispatches a shell hook with the transcript. Shared wake-gate utilities live in `SwabbleKit` for reuse by other apps (iOS/macOS).
Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framework (SpeechAnalyzer + SpeechTranscriber) instead of whisper.cpp. Local-only, wake word gated, dispatches a shell hook with the transcript.
## Requirements
- macOS 26+, Swift 6.2, Speech.framework with on-device assets.
- Local only; no network calls during transcription.
- Wake word gating (default "clawd" plus aliases) with bypass flag `--no-wake`.
- `SwabbleKit` target (multi-platform) providing wake-word gating helpers that can use speech segment timing to require a post-trigger gap.
- Hook execution with cooldown, min_chars, timeout, prefix, env vars.
- Simple config at `~/.config/swabble/config.json` (JSON, Codable) — no TOML.
- CLI implemented with Commander (SwiftPM package `steipete/Commander`); core types are available via the SwiftPM library product `Swabble` for embedding.
@@ -18,7 +17,7 @@ Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framewo
- **CLI layer (Commander)**: Root command `swabble` with subcommands `serve`, `transcribe`, `test-hook`, `mic list|set`, `doctor`, `health`, `tail-log`. Runtime flags from Commander (`-v/--verbose`, `--json-output`, `--log-level`). Custom `--config` path applies everywhere.
- **Config**: `SwabbleConfig` Codable. Fields: audio device name/index, wake (enabled/word/aliases/sensitivity placeholder), hook (command/args/prefix/cooldown/min_chars/timeout/env), logging (level, format), transcripts (enabled, max kept), speech (locale, enableEtiquetteReplacements flag). Stored JSON; default written by `setup`.
- **Audio + Speech pipeline**: `SpeechPipeline` wraps `AVAudioEngine` input → `SpeechAnalyzer` with `SpeechTranscriber` module. Emits partial/final transcripts via async stream. Requests `.audioTimeRange` when transcripts enabled. Handles Speech permission and asset download prompts ahead of capture.
- **Wake gate**: CLI currently uses text-only keyword match; shared `SwabbleKit` gate can enforce a minimum pause between the wake word and the next token when speech segments are available. `--no-wake` disables gating.
- **Wake gate**: text-based keyword match against latest partial/final; strips wake term before hook dispatch. `--no-wake` disables.
- **Hook executor**: async `HookExecutor` spawns `Process` with configured args, prefix substitution `${hostname}`. Enforces cooldown + timeout; injects env `SWABBLE_TEXT`, `SWABBLE_PREFIX` plus user env map.
- **Transcripts store**: in-memory ring buffer; optional persisted JSON lines under `~/Library/Application Support/swabble/transcripts.log`.
- **Logging**: simple structured logger to stderr; respects log level.
@@ -26,7 +25,7 @@ Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framewo
## Out of scope (initial cut)
- Model management (Speech handles assets).
- Launchd helper (planned follow-up).
- Advanced wake-word detector (segment-aware gate now lives in `SwabbleKit`; CLI still text-only until segment timing is plumbed through).
- Advanced wake-word detector (text match only for now).
## Open decisions
- Whether to expose a UNIX control socket for `status`/`health` (currently planned as stdin/out direct calls).
+20 -30
View File
@@ -1,31 +1,21 @@
<?xml version="1.0" standalone="yes"?>
<rss xmlns:sparkle="http://www.andymatuschak.org/xml-namespaces/sparkle" version="2.0">
<channel>
<title>Clawdis</title>
<item>
<title>2.0.0-beta3</title>
<pubDate>Sat, 27 Dec 2025 19:02:02 +0100</pubDate>
<link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link>
<sparkle:version>2.0.0-beta3</sparkle:version>
<sparkle:shortVersionString>2.0.0-beta3</sparkle:shortVersionString>
<sparkle:minimumSystemVersion>15.0</sparkle:minimumSystemVersion>
<enclosure url="https://github.com/steipete/clawdis/releases/download/v2.0.0-beta3/Clawdis-2.0.0-beta3.zip" length="70407960" type="application/octet-stream" sparkle:edSignature="A8ySMmbLRrpIkqkrmc9QrC+6om8Iyqray/6x/YNiJxDoJeXjp2T5t8XT0CKJeNBUlDkzIj/fwiK53v0qQ59cDQ=="/>
</item>
<item>
<title>2.0.0-beta4</title>
<pubDate>Sat, 27 Dec 2025 19:43:22 +0100</pubDate>
<link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link>
<sparkle:version>2.0.0-beta4</sparkle:version>
<sparkle:shortVersionString>2.0.0-beta4</sparkle:shortVersionString>
<sparkle:minimumSystemVersion>15.0</sparkle:minimumSystemVersion>
<description><![CDATA[<h2>Clawdis 2.0.0-beta4</h2>
<h3>Fixes</h3>
<ul>
<li>Package contents: include Discord/hooks build outputs in the npm tarball to avoid missing module errors.</li>
</ul>
<p><a href="https://github.com/steipete/clawdis/blob/main/CHANGELOG.md">View full changelog</a></p>
]]></description>
<enclosure url="https://github.com/steipete/clawdis/releases/download/v2.0.0-beta4/Clawdis-2.0.0-beta4.zip" length="70407894" type="application/octet-stream" sparkle:edSignature="HmuiC7TnUn80ZApnKfb6w+JGSrjc3uUOndMrtbTp42bkBSVifbttNVazqvJueGBo4MgoJV8CP+zQNzVmtVihAQ=="/>
</item>
</channel>
<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0"
xmlns:sparkle="http://www.andymatuschak.org/xml-namespaces/sparkle"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Clawdis Updates</title>
<link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link>
<description>Signed update feed for the Clawdis macOS companion app.</description>
<item>
<title>Clawdis 2.0.0-beta1</title>
<sparkle:releaseNotesLink>https://github.com/steipete/clawdis/releases/tag/v2.0.0-beta1</sparkle:releaseNotesLink>
<pubDate>Fri, 19 Dec 2025 17:19:50 +0000</pubDate>
<enclosure url="https://github.com/steipete/clawdis/releases/download/v2.0.0-beta1/Clawdis-2.0.0-beta1.zip"
sparkle:edSignature="oEpGD46U4ZyBBSY9/piUIFDJU+KlFB751JIWOW2yS0sRNHKszyG5khDHg9o9bV9Zo8DOCNF/HOi88jmtHJAaCQ=="
sparkle:version="2.0.0-beta1"
sparkle:shortVersionString="2.0.0-beta1"
length="72410016"
type="application/octet-stream" />
</item>
</channel>
</rss>
+6 -21
View File
@@ -20,7 +20,7 @@ android {
minSdk = 31
targetSdk = 36
versionCode = 1
versionName = "2.0.0-beta3"
versionName = "0.1"
}
buildTypes {
@@ -31,7 +31,6 @@ android {
buildFeatures {
compose = true
buildConfig = true
}
compileOptions {
@@ -39,21 +38,15 @@ android {
targetCompatibility = JavaVersion.VERSION_17
}
kotlinOptions {
jvmTarget = "17"
}
packaging {
resources {
excludes += "/META-INF/{AL2.0,LGPL2.1}"
}
}
lint {
disable += setOf("IconLauncherShape")
}
}
kotlin {
compilerOptions {
jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17)
}
}
dependencies {
@@ -63,8 +56,7 @@ dependencies {
implementation("androidx.core:core-ktx:1.17.0")
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.10.0")
implementation("androidx.activity:activity-compose:1.12.2")
implementation("androidx.webkit:webkit:1.14.0")
implementation("androidx.activity:activity-compose:1.12.1")
implementation("androidx.compose.ui:ui")
implementation("androidx.compose.ui:ui-tooling-preview")
@@ -94,11 +86,4 @@ dependencies {
testImplementation("junit:junit:4.13.2")
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
testImplementation("io.kotest:kotest-runner-junit5-jvm:6.0.7")
testImplementation("io.kotest:kotest-assertions-core-jvm:6.0.7")
testRuntimeOnly("org.junit.vintage:junit-vintage-engine:5.13.3")
}
tasks.withType<Test>().configureEach {
useJUnitPlatform()
}
@@ -12,20 +12,12 @@
<uses-permission
android:name="android.permission.ACCESS_FINE_LOCATION"
android:maxSdkVersion="32" />
<uses-permission
android:name="android.permission.ACCESS_COARSE_LOCATION"
android:maxSdkVersion="32" />
<uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-feature
android:name="android.hardware.camera"
android:required="false" />
<application
android:name=".NodeApp"
android:allowBackup="true"
android:dataExtractionRules="@xml/data_extraction_rules"
android:fullBackupContent="@xml/backup_rules"
android:icon="@mipmap/ic_launcher"
android:roundIcon="@mipmap/ic_launcher_round"
android:label="@string/app_name"
@@ -23,12 +23,9 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val statusText: StateFlow<String> = runtime.statusText
val serverName: StateFlow<String?> = runtime.serverName
val remoteAddress: StateFlow<String?> = runtime.remoteAddress
val isForeground: StateFlow<Boolean> = runtime.isForeground
val seamColorArgb: StateFlow<Long> = runtime.seamColorArgb
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
val screenRecordActive: StateFlow<Boolean> = runtime.screenRecordActive
val instanceId: StateFlow<String> = runtime.instanceId
val displayName: StateFlow<String> = runtime.displayName
@@ -38,14 +35,9 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode
val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening
val talkEnabled: StateFlow<Boolean> = runtime.talkEnabled
val talkStatusText: StateFlow<String> = runtime.talkStatusText
val talkIsListening: StateFlow<Boolean> = runtime.talkIsListening
val talkIsSpeaking: StateFlow<Boolean> = runtime.talkIsSpeaking
val manualEnabled: StateFlow<Boolean> = runtime.manualEnabled
val manualHost: StateFlow<String> = runtime.manualHost
val manualPort: StateFlow<Int> = runtime.manualPort
val canvasDebugStatusEnabled: StateFlow<Boolean> = runtime.canvasDebugStatusEnabled
val chatSessionKey: StateFlow<String> = runtime.chatSessionKey
val chatSessionId: StateFlow<String?> = runtime.chatSessionId
@@ -86,10 +78,6 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.setManualPort(value)
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
runtime.setCanvasDebugStatusEnabled(value)
}
fun setWakeWords(words: List<String>) {
runtime.setWakeWords(words)
}
@@ -102,10 +90,6 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.setVoiceWakeMode(mode)
}
fun setTalkEnabled(enabled: Boolean) {
runtime.setTalkEnabled(enabled)
}
fun connect(endpoint: BridgeEndpoint) {
runtime.connect(endpoint)
}
@@ -10,6 +10,7 @@ import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.content.pm.ServiceInfo
import android.os.Build
import androidx.core.app.NotificationCompat
import androidx.core.content.ContextCompat
import kotlinx.coroutines.CoroutineScope
@@ -84,6 +85,7 @@ class NodeForegroundService : Service() {
override fun onBind(intent: Intent?) = null
private fun ensureChannel() {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return
val mgr = getSystemService(NotificationManager::class.java)
val channel =
NotificationChannel(
@@ -99,7 +101,12 @@ class NodeForegroundService : Service() {
private fun buildNotification(title: String, text: String): Notification {
val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP)
val flags = PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
val flags =
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
} else {
PendingIntent.FLAG_UPDATE_CURRENT
}
val stopPending = PendingIntent.getService(this, 2, stopIntent, flags)
return NotificationCompat.Builder(this, CHANNEL_ID)
@@ -119,6 +126,11 @@ class NodeForegroundService : Service() {
}
private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) {
if (Build.VERSION.SDK_INT < 29) {
startForeground(NOTIFICATION_ID, notification)
return
}
if (didStartForeground && requiresMic == lastRequiresMic) {
updateNotification(notification)
return
@@ -150,7 +162,11 @@ class NodeForegroundService : Service() {
fun start(context: Context) {
val intent = Intent(context, NodeForegroundService::class.java)
context.startForegroundService(intent)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
context.startForegroundService(intent)
} else {
context.startService(intent)
}
}
fun stop(context: Context) {
@@ -16,7 +16,6 @@ import com.steipete.clawdis.node.bridge.BridgeEndpoint
import com.steipete.clawdis.node.bridge.BridgePairingClient
import com.steipete.clawdis.node.bridge.BridgeSession
import com.steipete.clawdis.node.node.CameraCaptureManager
import com.steipete.clawdis.node.BuildConfig
import com.steipete.clawdis.node.node.CanvasController
import com.steipete.clawdis.node.node.ScreenRecordManager
import com.steipete.clawdis.node.protocol.ClawdisCapability
@@ -25,7 +24,6 @@ import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UIAction
import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UICommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasCommand
import com.steipete.clawdis.node.protocol.ClawdisScreenCommand
import com.steipete.clawdis.node.voice.TalkModeManager
import com.steipete.clawdis.node.voice.VoiceWakeManager
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -70,7 +68,7 @@ class NodeRuntime(context: Context) {
payloadJson =
buildJsonObject {
put("message", JsonPrimitive(command))
put("sessionKey", JsonPrimitive(mainSessionKey.value))
put("sessionKey", JsonPrimitive("main"))
put("thinking", JsonPrimitive(chatThinkingLevel.value))
put("deliver", JsonPrimitive(false))
}.toString(),
@@ -85,15 +83,6 @@ class NodeRuntime(context: Context) {
val voiceWakeStatusText: StateFlow<String>
get() = voiceWake.statusText
val talkStatusText: StateFlow<String>
get() = talkMode.statusText
val talkIsListening: StateFlow<Boolean>
get() = talkMode.isListening
val talkIsSpeaking: StateFlow<Boolean>
get() = talkMode.isSpeaking
private val discovery = BridgeDiscovery(appContext, scope = scope)
val bridges: StateFlow<List<BridgeEndpoint>> = discovery.bridges
val discoveryStatusText: StateFlow<String> = discovery.statusText
@@ -104,9 +93,6 @@ class NodeRuntime(context: Context) {
private val _statusText = MutableStateFlow("Offline")
val statusText: StateFlow<String> = _statusText.asStateFlow()
private val _mainSessionKey = MutableStateFlow("main")
val mainSessionKey: StateFlow<String> = _mainSessionKey.asStateFlow()
private val cameraHudSeq = AtomicLong(0)
private val _cameraHud = MutableStateFlow<CameraHudState?>(null)
val cameraHud: StateFlow<CameraHudState?> = _cameraHud.asStateFlow()
@@ -114,23 +100,15 @@ class NodeRuntime(context: Context) {
private val _cameraFlashToken = MutableStateFlow(0L)
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
private val _screenRecordActive = MutableStateFlow(false)
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
private val _serverName = MutableStateFlow<String?>(null)
val serverName: StateFlow<String?> = _serverName.asStateFlow()
private val _remoteAddress = MutableStateFlow<String?>(null)
val remoteAddress: StateFlow<String?> = _remoteAddress.asStateFlow()
private val _seamColorArgb = MutableStateFlow(DEFAULT_SEAM_COLOR_ARGB)
val seamColorArgb: StateFlow<Long> = _seamColorArgb.asStateFlow()
private val _isForeground = MutableStateFlow(true)
val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow()
private var lastAutoA2uiUrl: String? = null
private val session =
BridgeSession(
scope = scope,
@@ -139,10 +117,7 @@ class NodeRuntime(context: Context) {
_serverName.value = name
_remoteAddress.value = remote
_isConnected.value = true
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
scope.launch { refreshBrandingFromGateway() }
scope.launch { refreshWakeWordsFromGateway() }
maybeNavigateToA2uiOnConnect()
},
onDisconnected = { message -> handleSessionDisconnected(message) },
onEvent = { event, payloadJson ->
@@ -154,33 +129,13 @@ class NodeRuntime(context: Context) {
)
private val chat = ChatController(scope = scope, session = session, json = json)
private val talkMode: TalkModeManager by lazy {
TalkModeManager(context = appContext, scope = scope).also { it.attachSession(session) }
}
private fun handleSessionDisconnected(message: String) {
_statusText.value = message
_serverName.value = null
_remoteAddress.value = null
_isConnected.value = false
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
_mainSessionKey.value = "main"
chat.onDisconnected(message)
showLocalCanvasOnDisconnect()
}
private fun maybeNavigateToA2uiOnConnect() {
val a2uiUrl = resolveA2uiHostUrl() ?: return
val current = canvas.currentUrl()?.trim().orEmpty()
if (current.isEmpty() || current == lastAutoA2uiUrl) {
lastAutoA2uiUrl = a2uiUrl
canvas.navigate(a2uiUrl)
}
}
private fun showLocalCanvasOnDisconnect() {
lastAutoA2uiUrl = null
canvas.navigate("")
}
val instanceId: StateFlow<String> = prefs.instanceId
@@ -189,12 +144,10 @@ class NodeRuntime(context: Context) {
val preventSleep: StateFlow<Boolean> = prefs.preventSleep
val wakeWords: StateFlow<List<String>> = prefs.wakeWords
val voiceWakeMode: StateFlow<VoiceWakeMode> = prefs.voiceWakeMode
val talkEnabled: StateFlow<Boolean> = prefs.talkEnabled
val manualEnabled: StateFlow<Boolean> = prefs.manualEnabled
val manualHost: StateFlow<String> = prefs.manualHost
val manualPort: StateFlow<Int> = prefs.manualPort
val lastDiscoveredStableId: StateFlow<String> = prefs.lastDiscoveredStableId
val canvasDebugStatusEnabled: StateFlow<Boolean> = prefs.canvasDebugStatusEnabled
private var didAutoConnect = false
private var suppressWakeWordsSync = false
@@ -245,13 +198,6 @@ class NodeRuntime(context: Context) {
}
}
scope.launch {
talkEnabled.collect { enabled ->
talkMode.setEnabled(enabled)
externalAudioCaptureActive.value = enabled
}
}
scope.launch(Dispatchers.Default) {
bridges.collect { list ->
if (list.isNotEmpty()) {
@@ -282,22 +228,6 @@ class NodeRuntime(context: Context) {
connect(target)
}
}
scope.launch {
combine(
canvasDebugStatusEnabled,
statusText,
serverName,
remoteAddress,
) { debugEnabled, status, server, remote ->
Quad(debugEnabled, status, server, remote)
}.distinctUntilChanged()
.collect { (debugEnabled, status, server, remote) ->
canvas.setDebugStatusEnabled(debugEnabled)
if (!debugEnabled) return@collect
canvas.setDebugStatus(status, server ?: remote)
}
}
}
fun setForeground(value: Boolean) {
@@ -328,10 +258,6 @@ class NodeRuntime(context: Context) {
prefs.setManualPort(value)
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
prefs.setCanvasDebugStatusEnabled(value)
}
fun setWakeWords(words: List<String>) {
prefs.setWakeWords(words)
scheduleWakeWordsSyncIfNeeded()
@@ -345,10 +271,6 @@ class NodeRuntime(context: Context) {
prefs.setVoiceWakeMode(mode)
}
fun setTalkEnabled(value: Boolean) {
prefs.setTalkEnabled(value)
}
fun connect(endpoint: BridgeEndpoint) {
scope.launch {
_statusText.value = "Connecting…"
@@ -385,13 +307,6 @@ class NodeRuntime(context: Context) {
add(ClawdisCapability.VoiceWake.rawValue)
}
}
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
val advertisedVersion =
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
BridgePairingClient().pairAndHello(
endpoint = endpoint,
hello =
@@ -400,7 +315,7 @@ class NodeRuntime(context: Context) {
displayName = displayName.value,
token = null,
platform = "Android",
version = advertisedVersion,
version = "dev",
deviceFamily = "Android",
modelIdentifier = modelIdentifier,
caps = caps,
@@ -418,13 +333,6 @@ class NodeRuntime(context: Context) {
val authToken = requireNotNull(resolved.token).trim()
prefs.saveBridgeToken(authToken)
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
val advertisedVersion =
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
session.connect(
endpoint = endpoint,
hello =
@@ -433,7 +341,7 @@ class NodeRuntime(context: Context) {
displayName = displayName.value,
token = authToken,
platform = "Android",
version = advertisedVersion,
version = "dev",
deviceFamily = "Android",
modelIdentifier = modelIdentifier,
caps =
@@ -488,7 +396,8 @@ class NodeRuntime(context: Context) {
val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty {
java.util.UUID.randomUUID().toString()
}
val name = ClawdisCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch
val name = (userActionObj["name"] as? JsonPrimitive)?.content?.trim().orEmpty()
if (name.isEmpty()) return@launch
val surfaceId =
(userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" }
@@ -586,7 +495,6 @@ class NodeRuntime(context: Context) {
return
}
talkMode.handleBridgeEvent(event, payloadJson)
chat.handleBridgeEvent(event, payloadJson)
}
@@ -628,25 +536,6 @@ class NodeRuntime(context: Context) {
}
}
private suspend fun refreshBrandingFromGateway() {
if (!_isConnected.value) return
try {
val res = session.request("config.get", "{}")
val root = json.parseToJsonElement(res).asObjectOrNull()
val config = root?.get("config").asObjectOrNull()
val ui = config?.get("ui").asObjectOrNull()
val raw = ui?.get("seamColor").asStringOrNull()?.trim()
val sessionCfg = config?.get("session").asObjectOrNull()
val rawMainKey = sessionCfg?.get("mainKey").asStringOrNull()?.trim()
_mainSessionKey.value = rawMainKey?.takeIf { it.isNotEmpty() } ?: "main"
val parsed = parseHexColorArgb(raw)
_seamColorArgb.value = parsed ?: DEFAULT_SEAM_COLOR_ARGB
} catch (_: Throwable) {
// ignore
}
}
private suspend fun handleInvoke(command: String, paramsJson: String?): BridgeSession.InvokeResult {
if (
command.startsWith(ClawdisCanvasCommand.NamespacePrefix) ||
@@ -788,20 +677,14 @@ class NodeRuntime(context: Context) {
}
}
ClawdisScreenCommand.Record.rawValue -> {
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
_screenRecordActive.value = true
try {
val res =
try {
screenRecorder.record(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
BridgeSession.InvokeResult.ok(res.payloadJson)
} finally {
_screenRecordActive.value = false
}
val res =
try {
screenRecorder.record(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
BridgeSession.InvokeResult.ok(res.payloadJson)
}
else ->
BridgeSession.InvokeResult.error(
@@ -844,7 +727,7 @@ class NodeRuntime(context: Context) {
val raw = session.currentCanvasHostUrl()?.trim().orEmpty()
if (raw.isBlank()) return null
val base = raw.trimEnd('/')
return "${base}/__clawdis__/a2ui/?platform=android"
return "${base}/__clawdis__/a2ui/"
}
private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
@@ -930,8 +813,6 @@ class NodeRuntime(context: Context) {
private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
private const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A
private const val a2uiReadyCheckJS: String =
"""
(() => {
@@ -986,12 +867,3 @@ private fun JsonElement?.asStringOrNull(): String? =
is JsonPrimitive -> content
else -> null
}
private fun parseHexColorArgb(raw: String?): Long? {
val trimmed = raw?.trim().orEmpty()
if (trimmed.isEmpty()) return null
val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed
if (hex.length != 6) return null
val rgb = hex.toLongOrNull(16) ?: return null
return 0xFF000000L or rgb
}
@@ -3,7 +3,6 @@
package com.steipete.clawdis.node
import android.content.Context
import androidx.core.content.edit
import androidx.security.crypto.EncryptedSharedPreferences
import androidx.security.crypto.MasterKey
import kotlinx.coroutines.flow.MutableStateFlow
@@ -63,62 +62,50 @@ class SecurePrefs(context: Context) {
MutableStateFlow(prefs.getString("bridge.lastDiscoveredStableId", "")!!)
val lastDiscoveredStableId: StateFlow<String> = _lastDiscoveredStableId
private val _canvasDebugStatusEnabled =
MutableStateFlow(prefs.getBoolean("canvas.debugStatusEnabled", false))
val canvasDebugStatusEnabled: StateFlow<Boolean> = _canvasDebugStatusEnabled
private val _wakeWords = MutableStateFlow(loadWakeWords())
val wakeWords: StateFlow<List<String>> = _wakeWords
private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode())
val voiceWakeMode: StateFlow<VoiceWakeMode> = _voiceWakeMode
private val _talkEnabled = MutableStateFlow(prefs.getBoolean("talk.enabled", false))
val talkEnabled: StateFlow<Boolean> = _talkEnabled
fun setLastDiscoveredStableId(value: String) {
val trimmed = value.trim()
prefs.edit { putString("bridge.lastDiscoveredStableId", trimmed) }
prefs.edit().putString("bridge.lastDiscoveredStableId", trimmed).apply()
_lastDiscoveredStableId.value = trimmed
}
fun setDisplayName(value: String) {
val trimmed = value.trim()
prefs.edit { putString(displayNameKey, trimmed) }
prefs.edit().putString(displayNameKey, trimmed).apply()
_displayName.value = trimmed
}
fun setCameraEnabled(value: Boolean) {
prefs.edit { putBoolean("camera.enabled", value) }
prefs.edit().putBoolean("camera.enabled", value).apply()
_cameraEnabled.value = value
}
fun setPreventSleep(value: Boolean) {
prefs.edit { putBoolean("screen.preventSleep", value) }
prefs.edit().putBoolean("screen.preventSleep", value).apply()
_preventSleep.value = value
}
fun setManualEnabled(value: Boolean) {
prefs.edit { putBoolean("bridge.manual.enabled", value) }
prefs.edit().putBoolean("bridge.manual.enabled", value).apply()
_manualEnabled.value = value
}
fun setManualHost(value: String) {
val trimmed = value.trim()
prefs.edit { putString("bridge.manual.host", trimmed) }
prefs.edit().putString("bridge.manual.host", trimmed).apply()
_manualHost.value = trimmed
}
fun setManualPort(value: Int) {
prefs.edit { putInt("bridge.manual.port", value) }
prefs.edit().putInt("bridge.manual.port", value).apply()
_manualPort.value = value
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
prefs.edit { putBoolean("canvas.debugStatusEnabled", value) }
_canvasDebugStatusEnabled.value = value
}
fun loadBridgeToken(): String? {
val key = "bridge.token.${_instanceId.value}"
return prefs.getString(key, null)
@@ -126,14 +113,14 @@ class SecurePrefs(context: Context) {
fun saveBridgeToken(token: String) {
val key = "bridge.token.${_instanceId.value}"
prefs.edit { putString(key, token.trim()) }
prefs.edit().putString(key, token.trim()).apply()
}
private fun loadOrCreateInstanceId(): String {
val existing = prefs.getString("node.instanceId", null)?.trim()
if (!existing.isNullOrBlank()) return existing
val fresh = UUID.randomUUID().toString()
prefs.edit { putString("node.instanceId", fresh) }
prefs.edit().putString("node.instanceId", fresh).apply()
return fresh
}
@@ -144,7 +131,7 @@ class SecurePrefs(context: Context) {
val candidate = DeviceNames.bestDefaultNodeName(context).trim()
val resolved = candidate.ifEmpty { "Android Node" }
prefs.edit { putString(displayNameKey, resolved) }
prefs.edit().putString(displayNameKey, resolved).apply()
return resolved
}
@@ -152,27 +139,22 @@ class SecurePrefs(context: Context) {
val sanitized = WakeWords.sanitize(words, defaultWakeWords)
val encoded =
JsonArray(sanitized.map { JsonPrimitive(it) }).toString()
prefs.edit { putString("voiceWake.triggerWords", encoded) }
prefs.edit().putString("voiceWake.triggerWords", encoded).apply()
_wakeWords.value = sanitized
}
fun setVoiceWakeMode(mode: VoiceWakeMode) {
prefs.edit { putString(voiceWakeModeKey, mode.rawValue) }
prefs.edit().putString(voiceWakeModeKey, mode.rawValue).apply()
_voiceWakeMode.value = mode
}
fun setTalkEnabled(value: Boolean) {
prefs.edit { putBoolean("talk.enabled", value) }
_talkEnabled.value = value
}
private fun loadVoiceWakeMode(): VoiceWakeMode {
val raw = prefs.getString(voiceWakeModeKey, null)
val resolved = VoiceWakeMode.fromRawValue(raw)
// Default ON (foreground) when unset.
if (raw.isNullOrBlank()) {
prefs.edit { putString(voiceWakeModeKey, resolved.rawValue) }
prefs.edit().putString(voiceWakeModeKey, resolved.rawValue).apply()
}
return resolved
@@ -6,6 +6,7 @@ import android.net.DnsResolver
import android.net.NetworkCapabilities
import android.net.nsd.NsdManager
import android.net.nsd.NsdServiceInfo
import android.os.Build
import android.os.CancellationSignal
import android.util.Log
import java.io.IOException
@@ -43,7 +44,6 @@ import org.xbill.DNS.Type
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
@Suppress("DEPRECATION")
class BridgeDiscovery(
context: Context,
private val scope: CoroutineScope,
@@ -130,36 +130,20 @@ class BridgeDiscovery(
object : NsdManager.ResolveListener {
override fun onResolveFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {}
override fun onServiceResolved(resolved: NsdServiceInfo) {
val host = resolved.host?.hostAddress ?: return
val port = resolved.port
if (port <= 0) return
override fun onServiceResolved(resolved: NsdServiceInfo) {
val host = resolved.host?.hostAddress ?: return
val port = resolved.port
if (port <= 0) return
val rawServiceName = resolved.serviceName
val serviceName = BonjourEscapes.decode(rawServiceName)
val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName)
val lanHost = txt(resolved, "lanHost")
val tailnetDns = txt(resolved, "tailnetDns")
val gatewayPort = txtInt(resolved, "gatewayPort")
val bridgePort = txtInt(resolved, "bridgePort")
val canvasPort = txtInt(resolved, "canvasPort")
val id = stableId(serviceName, "local.")
localById[id] =
BridgeEndpoint(
stableId = id,
name = displayName,
host = host,
port = port,
lanHost = lanHost,
tailnetDns = tailnetDns,
gatewayPort = gatewayPort,
bridgePort = bridgePort,
canvasPort = canvasPort,
)
publish()
}
},
)
val rawServiceName = resolved.serviceName
val serviceName = BonjourEscapes.decode(rawServiceName)
val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName)
val id = stableId(serviceName, "local.")
localById[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port)
publish()
}
},
)
}
private fun publish() {
@@ -197,6 +181,7 @@ class BridgeDiscovery(
}
private fun txt(info: NsdServiceInfo, key: String): String? {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) return null
val bytes = info.attributes[key] ?: return null
return try {
String(bytes, Charsets.UTF_8).trim().ifEmpty { null }
@@ -205,10 +190,6 @@ class BridgeDiscovery(
}
}
private fun txtInt(info: NsdServiceInfo, key: String): Int? {
return txt(info, key)?.toIntOrNull()
}
private suspend fun refreshUnicast(domain: String) {
val ptrName = "${serviceType}${domain}"
val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return
@@ -247,24 +228,8 @@ class BridgeDiscovery(
}
val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain))
val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName)
val lanHost = txtValue(txt, "lanHost")
val tailnetDns = txtValue(txt, "tailnetDns")
val gatewayPort = txtIntValue(txt, "gatewayPort")
val bridgePort = txtIntValue(txt, "bridgePort")
val canvasPort = txtIntValue(txt, "canvasPort")
val id = stableId(instanceName, domain)
next[id] =
BridgeEndpoint(
stableId = id,
name = displayName,
host = host,
port = port,
lanHost = lanHost,
tailnetDns = tailnetDns,
gatewayPort = gatewayPort,
bridgePort = bridgePort,
canvasPort = canvasPort,
)
next[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port)
}
unicastById.clear()
@@ -436,7 +401,7 @@ class BridgeDiscovery(
dns.rawQuery(
network,
wireQuery,
DnsResolver.FLAG_EMPTY,
0,
dnsExecutor,
signal,
object : DnsResolver.Callback<ByteArray> {
@@ -470,10 +435,6 @@ class BridgeDiscovery(
return null
}
private fun txtIntValue(records: List<TXTRecord>, key: String): Int? {
return txtValue(records, key)?.toIntOrNull()
}
private fun decodeDnsTxtString(raw: String): String {
// dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes.
// Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible.
@@ -5,11 +5,6 @@ data class BridgeEndpoint(
val name: String,
val host: String,
val port: Int,
val lanHost: String? = null,
val tailnetDns: String? = null,
val gatewayPort: Int? = null,
val bridgePort: Int? = null,
val canvasPort: Int? = null,
) {
companion object {
fun manual(host: String, port: Int): BridgeEndpoint =
@@ -21,3 +16,4 @@ data class BridgeEndpoint(
)
}
}
@@ -11,7 +11,6 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import com.steipete.clawdis.node.BuildConfig
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
@@ -24,7 +23,6 @@ import java.io.BufferedWriter
import java.io.InputStreamReader
import java.io.OutputStreamWriter
import java.net.InetSocketAddress
import java.net.URI
import java.net.Socket
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
@@ -77,8 +75,6 @@ class BridgeSession(
fun disconnect() {
desired = null
// Unblock connectOnce() read loop. Coroutine cancellation alone won't interrupt BufferedReader.readLine().
currentConnection?.closeQuietly()
scope.launch(Dispatchers.IO) {
job?.cancelAndJoin()
job = null
@@ -217,17 +213,7 @@ class BridgeSession(
when (first["type"].asStringOrNull()) {
"hello-ok" -> {
val name = first["serverName"].asStringOrNull() ?: "Bridge"
val rawCanvasUrl = first["canvasHostUrl"].asStringOrNull()?.trim()?.ifEmpty { null }
canvasHostUrl = normalizeCanvasHostUrl(rawCanvasUrl, endpoint)
if (BuildConfig.DEBUG) {
// Local JVM unit tests use android.jar stubs; Log.d can throw "not mocked".
runCatching {
android.util.Log.d(
"ClawdisBridge",
"canvasHostUrl resolved=${canvasHostUrl ?: "none"} (raw=${rawCanvasUrl ?: "none"})",
)
}
}
canvasHostUrl = first["canvasHostUrl"].asStringOrNull()?.trim()?.ifEmpty { null }
onConnected(name, conn.remoteAddress)
}
"error" -> {
@@ -306,37 +292,6 @@ class BridgeSession(
conn.closeQuietly()
}
}
private fun normalizeCanvasHostUrl(raw: String?, endpoint: BridgeEndpoint): String? {
val trimmed = raw?.trim().orEmpty()
val parsed = trimmed.takeIf { it.isNotBlank() }?.let { runCatching { URI(it) }.getOrNull() }
val host = parsed?.host?.trim().orEmpty()
val port = parsed?.port ?: -1
val scheme = parsed?.scheme?.trim().orEmpty().ifBlank { "http" }
if (trimmed.isNotBlank() && !isLoopbackHost(host)) {
return trimmed
}
val fallbackHost =
endpoint.tailnetDns?.trim().takeIf { !it.isNullOrEmpty() }
?: endpoint.lanHost?.trim().takeIf { !it.isNullOrEmpty() }
?: endpoint.host.trim()
if (fallbackHost.isEmpty()) return trimmed.ifBlank { null }
val fallbackPort = endpoint.canvasPort ?: if (port > 0) port else 18793
val formattedHost = if (fallbackHost.contains(":")) "[${fallbackHost}]" else fallbackHost
return "$scheme://$formattedHost:$fallbackPort"
}
private fun isLoopbackHost(raw: String?): Boolean {
val host = raw?.trim()?.lowercase().orEmpty()
if (host.isEmpty()) return false
if (host == "localhost") return true
if (host == "::1") return true
if (host == "0.0.0.0" || host == "::") return true
return host.startsWith("127.")
}
}
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
@@ -2,7 +2,6 @@ package com.steipete.clawdis.node.node
import android.Manifest
import android.content.Context
import android.annotation.SuppressLint
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.util.Base64
@@ -19,7 +18,6 @@ import androidx.camera.video.VideoCapture
import androidx.camera.video.VideoRecordEvent
import androidx.core.content.ContextCompat
import androidx.core.content.ContextCompat.checkSelfPermission
import androidx.core.graphics.scale
import com.steipete.clawdis.node.PermissionRequester
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
@@ -28,7 +26,6 @@ import kotlinx.coroutines.withContext
import java.io.ByteArrayOutputStream
import java.io.File
import java.util.concurrent.Executor
import kotlin.math.roundToInt
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
@@ -95,45 +92,22 @@ class CameraCaptureManager(private val context: Context) {
(decoded.height.toDouble() * (maxWidth.toDouble() / decoded.width.toDouble()))
.toInt()
.coerceAtLeast(1)
decoded.scale(maxWidth, h)
Bitmap.createScaledBitmap(decoded, maxWidth, h, true)
} else {
decoded
}
val maxPayloadBytes = 5 * 1024 * 1024
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
val maxEncodedBytes = (maxPayloadBytes / 4) * 3
val result =
JpegSizeLimiter.compressToLimit(
initialWidth = scaled.width,
initialHeight = scaled.height,
startQuality = (quality * 100.0).roundToInt().coerceIn(10, 100),
maxBytes = maxEncodedBytes,
encode = { width, height, q ->
val bitmap =
if (width == scaled.width && height == scaled.height) {
scaled
} else {
scaled.scale(width, height)
}
val out = ByteArrayOutputStream()
if (!bitmap.compress(Bitmap.CompressFormat.JPEG, q, out)) {
if (bitmap !== scaled) bitmap.recycle()
throw IllegalStateException("UNAVAILABLE: failed to encode JPEG")
}
if (bitmap !== scaled) {
bitmap.recycle()
}
out.toByteArray()
},
)
val base64 = Base64.encodeToString(result.bytes, Base64.NO_WRAP)
val out = ByteArrayOutputStream()
val jpegQuality = (quality * 100.0).toInt().coerceIn(10, 100)
if (!scaled.compress(Bitmap.CompressFormat.JPEG, jpegQuality, out)) {
throw IllegalStateException("UNAVAILABLE: failed to encode JPEG")
}
val base64 = Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP)
Payload(
"""{"format":"jpg","base64":"$base64","width":${result.width},"height":${result.height}}""",
"""{"format":"jpg","base64":"$base64","width":${scaled.width},"height":${scaled.height}}""",
)
}
@SuppressLint("MissingPermission")
suspend fun clip(paramsJson: String?): Payload =
withContext(Dispatchers.Main) {
ensureCameraPermission()
@@ -1,23 +1,19 @@
package com.steipete.clawdis.node.node
import android.graphics.Bitmap
import android.os.Build
import android.graphics.Canvas
import android.os.Looper
import android.util.Log
import android.webkit.WebView
import androidx.core.graphics.createBitmap
import androidx.core.graphics.scale
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import java.io.ByteArrayOutputStream
import android.util.Base64
import org.json.JSONObject
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import com.steipete.clawdis.node.BuildConfig
import kotlin.coroutines.resume
class CanvasController {
@@ -28,9 +24,6 @@ class CanvasController {
@Volatile private var webView: WebView? = null
@Volatile private var url: String? = null
@Volatile private var debugStatusEnabled: Boolean = false
@Volatile private var debugStatusTitle: String? = null
@Volatile private var debugStatusSubtitle: String? = null
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
@@ -42,7 +35,6 @@ class CanvasController {
fun attach(webView: WebView) {
this.webView = webView
reload()
applyDebugStatus()
}
fun navigate(url: String) {
@@ -51,25 +43,6 @@ class CanvasController {
reload()
}
fun currentUrl(): String? = url
fun isDefaultCanvas(): Boolean = url == null
fun setDebugStatusEnabled(enabled: Boolean) {
debugStatusEnabled = enabled
applyDebugStatus()
}
fun setDebugStatus(title: String?, subtitle: String?) {
debugStatusTitle = title
debugStatusSubtitle = subtitle
applyDebugStatus()
}
fun onPageFinished() {
applyDebugStatus()
}
private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) {
val wv = webView ?: return
if (Looper.myLooper() == Looper.getMainLooper()) {
@@ -83,45 +56,13 @@ class CanvasController {
val currentUrl = url
withWebViewOnMain { wv ->
if (currentUrl == null) {
if (BuildConfig.DEBUG) {
Log.d("ClawdisCanvas", "load scaffold: $scaffoldAssetUrl")
}
wv.loadUrl(scaffoldAssetUrl)
} else {
if (BuildConfig.DEBUG) {
Log.d("ClawdisCanvas", "load url: $currentUrl")
}
wv.loadUrl(currentUrl)
}
}
}
private fun applyDebugStatus() {
val enabled = debugStatusEnabled
val title = debugStatusTitle
val subtitle = debugStatusSubtitle
withWebViewOnMain { wv ->
val titleJs = title?.let { JSONObject.quote(it) } ?: "null"
val subtitleJs = subtitle?.let { JSONObject.quote(it) } ?: "null"
val js = """
(() => {
try {
const api = globalThis.__clawdis;
if (!api) return;
if (typeof api.setDebugStatusEnabled === 'function') {
api.setDebugStatusEnabled(${if (enabled) "true" else "false"});
}
if (!${if (enabled) "true" else "false"}) return;
if (typeof api.setStatus === 'function') {
api.setStatus($titleJs, $subtitleJs);
}
} catch (_) {}
})();
""".trimIndent()
wv.evaluateJavascript(js, null)
}
}
suspend fun eval(javaScript: String): String =
withContext(Dispatchers.Main) {
val wv = webView ?: throw IllegalStateException("no webview")
@@ -139,7 +80,7 @@ class CanvasController {
val scaled =
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
bmp.scale(maxWidth, h)
Bitmap.createScaledBitmap(bmp, maxWidth, h, true)
} else {
bmp
}
@@ -156,7 +97,7 @@ class CanvasController {
val scaled =
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
bmp.scale(maxWidth, h)
Bitmap.createScaledBitmap(bmp, maxWidth, h, true)
} else {
bmp
}
@@ -175,7 +116,7 @@ class CanvasController {
suspendCancellableCoroutine { cont ->
val width = width.coerceAtLeast(1)
val height = height.coerceAtLeast(1)
val bitmap = createBitmap(width, height, Bitmap.Config.ARGB_8888)
val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
// WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable
// cross-version snapshot for this lightweight "canvas" use-case.
@@ -1,61 +0,0 @@
package com.steipete.clawdis.node.node
import kotlin.math.max
import kotlin.math.min
import kotlin.math.roundToInt
internal data class JpegSizeLimiterResult(
val bytes: ByteArray,
val width: Int,
val height: Int,
val quality: Int,
)
internal object JpegSizeLimiter {
fun compressToLimit(
initialWidth: Int,
initialHeight: Int,
startQuality: Int,
maxBytes: Int,
minQuality: Int = 20,
minSize: Int = 256,
scaleStep: Double = 0.85,
maxScaleAttempts: Int = 6,
maxQualityAttempts: Int = 6,
encode: (width: Int, height: Int, quality: Int) -> ByteArray,
): JpegSizeLimiterResult {
require(initialWidth > 0 && initialHeight > 0) { "Invalid image size" }
require(maxBytes > 0) { "Invalid maxBytes" }
var width = initialWidth
var height = initialHeight
val clampedStartQuality = startQuality.coerceIn(minQuality, 100)
var best = JpegSizeLimiterResult(bytes = encode(width, height, clampedStartQuality), width = width, height = height, quality = clampedStartQuality)
if (best.bytes.size <= maxBytes) return best
repeat(maxScaleAttempts) {
var quality = clampedStartQuality
repeat(maxQualityAttempts) {
val bytes = encode(width, height, quality)
best = JpegSizeLimiterResult(bytes = bytes, width = width, height = height, quality = quality)
if (bytes.size <= maxBytes) return best
if (quality <= minQuality) return@repeat
quality = max(minQuality, (quality * 0.75).roundToInt())
}
val minScale = (minSize.toDouble() / min(width, height).toDouble()).coerceAtMost(1.0)
val nextScale = max(scaleStep, minScale)
val nextWidth = max(minSize, (width * nextScale).roundToInt())
val nextHeight = max(minSize, (height * nextScale).roundToInt())
if (nextWidth == width && nextHeight == height) return@repeat
width = min(nextWidth, width)
height = min(nextHeight, height)
}
if (best.bytes.size > maxBytes) {
throw IllegalStateException("CAMERA_TOO_LARGE: ${best.bytes.size} bytes > $maxBytes bytes")
}
return best
}
}
@@ -1,24 +1,6 @@
package com.steipete.clawdis.node.protocol
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
object ClawdisCanvasA2UIAction {
fun extractActionName(userAction: JsonObject): String? {
val name =
(userAction["name"] as? JsonPrimitive)
?.content
?.trim()
.orEmpty()
if (name.isNotEmpty()) return name
val action =
(userAction["action"] as? JsonPrimitive)
?.content
?.trim()
.orEmpty()
return action.ifEmpty { null }
}
fun sanitizeTagValue(value: String): String {
val trimmed = value.trim().ifEmpty { "-" }
val normalized = trimmed.replace(" ", "_")
@@ -64,3 +46,4 @@ object ClawdisCanvasA2UIAction {
return "window.dispatchEvent(new CustomEvent('clawdis:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));"
}
}
@@ -1,26 +1,64 @@
package com.steipete.clawdis.node.ui
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
import androidx.compose.animation.slideInVertically
import androidx.compose.animation.slideOutVertically
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.statusBarsPadding
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.CheckCircle
import androidx.compose.material.icons.filled.Error
import androidx.compose.material.icons.filled.FiberManualRecord
import androidx.compose.material.icons.filled.PhotoCamera
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.CameraHudKind
import com.steipete.clawdis.node.CameraHudState
import kotlinx.coroutines.delay
@Composable
fun CameraFlashOverlay(
token: Long,
fun CameraHudOverlay(
hud: CameraHudState?,
flashToken: Long,
modifier: Modifier = Modifier,
) {
Box(modifier = modifier.fillMaxSize()) {
CameraFlash(token = token)
CameraFlash(token = flashToken)
AnimatedVisibility(
visible = hud != null,
enter = slideInVertically(initialOffsetY = { -it / 2 }) + fadeIn(),
exit = slideOutVertically(targetOffsetY = { -it / 2 }) + fadeOut(),
modifier = Modifier.align(Alignment.TopStart).statusBarsPadding().padding(start = 12.dp, top = 58.dp),
) {
if (hud != null) {
Toast(hud = hud)
}
}
}
}
@@ -42,3 +80,44 @@ private fun CameraFlash(token: Long) {
.background(Color.White),
)
}
@Composable
private fun Toast(hud: CameraHudState) {
Surface(
shape = RoundedCornerShape(14.dp),
color = MaterialTheme.colorScheme.surface.copy(alpha = 0.85f),
tonalElevation = 2.dp,
shadowElevation = 8.dp,
) {
Row(
modifier = Modifier.padding(vertical = 10.dp, horizontal = 12.dp),
verticalAlignment = Alignment.CenterVertically,
) {
when (hud.kind) {
CameraHudKind.Photo -> {
Icon(Icons.Default.PhotoCamera, contentDescription = null)
Spacer(Modifier.size(10.dp))
CircularProgressIndicator(modifier = Modifier.size(14.dp), strokeWidth = 2.dp)
}
CameraHudKind.Recording -> {
Icon(Icons.Default.FiberManualRecord, contentDescription = null, tint = Color.Red)
}
CameraHudKind.Success -> {
Icon(Icons.Default.CheckCircle, contentDescription = null)
}
CameraHudKind.Error -> {
Icon(Icons.Default.Error, contentDescription = null)
}
}
Spacer(Modifier.size(10.dp))
Text(
text = hud.message,
style = MaterialTheme.typography.bodyMedium,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
}
}
@@ -7,18 +7,12 @@ import android.graphics.Color
import android.util.Log
import android.view.View
import android.webkit.JavascriptInterface
import android.webkit.ConsoleMessage
import android.webkit.WebChromeClient
import android.webkit.WebView
import android.webkit.WebSettings
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebViewClient
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.webkit.WebSettingsCompat
import androidx.webkit.WebViewFeature
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
@@ -34,20 +28,10 @@ import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButtonDefaults
import androidx.compose.material3.LocalContentColor
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.ModalBottomSheet
import androidx.compose.material3.rememberModalBottomSheetState
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ChatBubble
import androidx.compose.material.icons.filled.CheckCircle
import androidx.compose.material.icons.filled.Error
import androidx.compose.material.icons.filled.FiberManualRecord
import androidx.compose.material.icons.filled.PhotoCamera
import androidx.compose.material.icons.filled.RecordVoiceOver
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Report
import androidx.compose.material.icons.filled.ScreenShare
import androidx.compose.material.icons.filled.Settings
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
@@ -57,15 +41,12 @@ import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color as ComposeColor
import androidx.compose.ui.graphics.lerp
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.compose.ui.window.Popup
import androidx.compose.ui.window.PopupProperties
import androidx.core.content.ContextCompat
import com.steipete.clawdis.node.CameraHudKind
import com.steipete.clawdis.node.MainViewModel
@OptIn(ExperimentalMaterial3Api::class)
@@ -79,105 +60,6 @@ fun RootScreen(viewModel: MainViewModel) {
val statusText by viewModel.statusText.collectAsState()
val cameraHud by viewModel.cameraHud.collectAsState()
val cameraFlashToken by viewModel.cameraFlashToken.collectAsState()
val screenRecordActive by viewModel.screenRecordActive.collectAsState()
val isForeground by viewModel.isForeground.collectAsState()
val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState()
val talkEnabled by viewModel.talkEnabled.collectAsState()
val talkStatusText by viewModel.talkStatusText.collectAsState()
val talkIsListening by viewModel.talkIsListening.collectAsState()
val talkIsSpeaking by viewModel.talkIsSpeaking.collectAsState()
val seamColorArgb by viewModel.seamColorArgb.collectAsState()
val seamColor = remember(seamColorArgb) { ComposeColor(seamColorArgb) }
val audioPermissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
if (granted) viewModel.setTalkEnabled(true)
}
val activity =
remember(cameraHud, screenRecordActive, isForeground, statusText, voiceWakeStatusText) {
// Status pill owns transient activity state so it doesn't overlap the connection indicator.
if (!isForeground) {
return@remember StatusActivity(
title = "Foreground required",
icon = Icons.Default.Report,
contentDescription = "Foreground required",
)
}
val lowerStatus = statusText.lowercase()
if (lowerStatus.contains("repair")) {
return@remember StatusActivity(
title = "Repairing…",
icon = Icons.Default.Refresh,
contentDescription = "Repairing",
)
}
if (lowerStatus.contains("pairing") || lowerStatus.contains("approval")) {
return@remember StatusActivity(
title = "Approval pending",
icon = Icons.Default.RecordVoiceOver,
contentDescription = "Approval pending",
)
}
// Avoid duplicating the primary bridge status ("Connecting…") in the activity slot.
if (screenRecordActive) {
return@remember StatusActivity(
title = "Recording screen…",
icon = Icons.Default.ScreenShare,
contentDescription = "Recording screen",
tint = androidx.compose.ui.graphics.Color.Red,
)
}
cameraHud?.let { hud ->
return@remember when (hud.kind) {
CameraHudKind.Photo ->
StatusActivity(
title = hud.message,
icon = Icons.Default.PhotoCamera,
contentDescription = "Taking photo",
)
CameraHudKind.Recording ->
StatusActivity(
title = hud.message,
icon = Icons.Default.FiberManualRecord,
contentDescription = "Recording",
tint = androidx.compose.ui.graphics.Color.Red,
)
CameraHudKind.Success ->
StatusActivity(
title = hud.message,
icon = Icons.Default.CheckCircle,
contentDescription = "Capture finished",
)
CameraHudKind.Error ->
StatusActivity(
title = hud.message,
icon = Icons.Default.Error,
contentDescription = "Capture failed",
tint = androidx.compose.ui.graphics.Color.Red,
)
}
}
if (voiceWakeStatusText.contains("Microphone permission", ignoreCase = true)) {
return@remember StatusActivity(
title = "Mic permission",
icon = Icons.Default.Error,
contentDescription = "Mic permission required",
)
}
if (voiceWakeStatusText == "Paused") {
val suffix = if (!isForeground) " (background)" else ""
return@remember StatusActivity(
title = "Voice Wake paused$suffix",
icon = Icons.Default.RecordVoiceOver,
contentDescription = "Voice Wake paused",
)
}
null
}
val bridgeState =
remember(serverName, statusText) {
@@ -198,9 +80,9 @@ fun RootScreen(viewModel: MainViewModel) {
CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize())
}
// Camera flash must be in a Popup to render above the WebView.
// Camera HUD (flash + toast) must be in a Popup to render above the WebView.
Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) {
CameraFlashOverlay(token = cameraFlashToken, modifier = Modifier.fillMaxSize())
CameraHudOverlay(hud = cameraHud, flashToken = cameraFlashToken, modifier = Modifier.fillMaxSize())
}
// Keep the overlay buttons above the WebView canvas (AndroidView), otherwise they may not receive touches.
@@ -208,7 +90,6 @@ fun RootScreen(viewModel: MainViewModel) {
StatusPill(
bridge = bridgeState,
voiceEnabled = voiceEnabled,
activity = activity,
onClick = { sheet = Sheet.Settings },
modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(start = 12.dp, top = 12.dp),
)
@@ -225,38 +106,6 @@ fun RootScreen(viewModel: MainViewModel) {
icon = { Icon(Icons.Default.ChatBubble, contentDescription = "Chat") },
)
// Talk mode gets a dedicated side bubble instead of burying it in settings.
val baseOverlay = overlayContainerColor()
val talkContainer =
lerp(
baseOverlay,
seamColor.copy(alpha = baseOverlay.alpha),
if (talkEnabled) 0.35f else 0.22f,
)
val talkContent = if (talkEnabled) seamColor else overlayIconColor()
OverlayIconButton(
onClick = {
val next = !talkEnabled
if (next) {
val micOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
viewModel.setTalkEnabled(true)
} else {
viewModel.setTalkEnabled(false)
}
},
containerColor = talkContainer,
contentColor = talkContent,
icon = {
Icon(
Icons.Default.RecordVoiceOver,
contentDescription = "Talk Mode",
)
},
)
OverlayIconButton(
onClick = { sheet = Sheet.Settings },
icon = { Icon(Icons.Default.Settings, contentDescription = "Settings") },
@@ -264,17 +113,6 @@ fun RootScreen(viewModel: MainViewModel) {
}
}
if (talkEnabled) {
Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) {
TalkOrbOverlay(
seamColor = seamColor,
statusText = talkStatusText,
isListening = talkIsListening,
isSpeaking = talkIsSpeaking,
)
}
}
val currentSheet = sheet
if (currentSheet != null) {
ModalBottomSheet(
@@ -298,16 +136,14 @@ private enum class Sheet {
private fun OverlayIconButton(
onClick: () -> Unit,
icon: @Composable () -> Unit,
containerColor: ComposeColor? = null,
contentColor: ComposeColor? = null,
) {
FilledTonalIconButton(
onClick = onClick,
modifier = Modifier.size(44.dp),
colors =
IconButtonDefaults.filledTonalIconButtonColors(
containerColor = containerColor ?: overlayContainerColor(),
contentColor = contentColor ?: overlayIconColor(),
containerColor = overlayContainerColor(),
contentColor = overlayIconColor(),
),
) {
icon()
@@ -327,19 +163,6 @@ private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier)
// Some embedded web UIs (incl. the "background website") use localStorage/sessionStorage.
settings.domStorageEnabled = true
settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE
if (WebViewFeature.isFeatureSupported(WebViewFeature.FORCE_DARK)) {
WebSettingsCompat.setForceDark(settings, WebSettingsCompat.FORCE_DARK_OFF)
}
if (WebViewFeature.isFeatureSupported(WebViewFeature.ALGORITHMIC_DARKENING)) {
WebSettingsCompat.setAlgorithmicDarkeningAllowed(settings, false)
}
if (isDebuggable) {
Log.d("ClawdisWebView", "userAgent: ${settings.userAgentString}")
}
isScrollContainer = true
overScrollMode = View.OVER_SCROLL_IF_CONTENT_SCROLLS
isVerticalScrollBarEnabled = true
isHorizontalScrollBarEnabled = true
webViewClient =
object : WebViewClient() {
override fun onReceivedError(
@@ -364,49 +187,15 @@ private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier)
"onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}",
)
}
override fun onPageFinished(view: WebView, url: String?) {
if (isDebuggable) {
Log.d("ClawdisWebView", "onPageFinished: $url")
}
viewModel.canvas.onPageFinished()
}
override fun onRenderProcessGone(
view: WebView,
detail: android.webkit.RenderProcessGoneDetail,
): Boolean {
if (isDebuggable) {
Log.e(
"ClawdisWebView",
"onRenderProcessGone didCrash=${detail.didCrash()} priorityAtExit=${detail.rendererPriorityAtExit()}",
)
}
return true
}
}
webChromeClient =
object : WebChromeClient() {
override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean {
if (!isDebuggable) return false
val msg = consoleMessage ?: return false
Log.d(
"ClawdisWebView",
"console ${msg.messageLevel()} @ ${msg.sourceId()}:${msg.lineNumber()} ${msg.message()}",
)
return false
}
}
// Use default layer/background; avoid forcing a black fill over WebView content.
setBackgroundColor(Color.BLACK)
setLayerType(View.LAYER_TYPE_HARDWARE, null)
val a2uiBridge =
addJavascriptInterface(
CanvasA2UIActionBridge { payload ->
viewModel.handleCanvasA2UIActionFromWebView(payload)
}
addJavascriptInterface(a2uiBridge, CanvasA2UIActionBridge.interfaceName)
addJavascriptInterface(
CanvasA2UIActionLegacyBridge(a2uiBridge),
CanvasA2UIActionLegacyBridge.interfaceName,
},
CanvasA2UIActionBridge.interfaceName,
)
viewModel.canvas.attach(this)
}
@@ -426,19 +215,3 @@ private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) {
const val interfaceName: String = "clawdisCanvasA2UIAction"
}
}
private class CanvasA2UIActionLegacyBridge(private val bridge: CanvasA2UIActionBridge) {
@JavascriptInterface
fun canvasAction(payload: String?) {
bridge.postMessage(payload)
}
@JavascriptInterface
fun postMessage(payload: String?) {
bridge.postMessage(payload)
}
companion object {
const val interfaceName: String = "Android"
}
}
@@ -2,7 +2,6 @@ package com.steipete.clawdis.node.ui
import android.Manifest
import android.content.pm.PackageManager
import android.os.Build
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.animation.AnimatedVisibility
@@ -47,7 +46,6 @@ import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import com.steipete.clawdis.node.BuildConfig
import com.steipete.clawdis.node.MainViewModel
import com.steipete.clawdis.node.NodeForegroundService
import com.steipete.clawdis.node.VoiceWakeMode
@@ -66,7 +64,6 @@ fun SettingsSheet(viewModel: MainViewModel) {
val manualEnabled by viewModel.manualEnabled.collectAsState()
val manualHost by viewModel.manualHost.collectAsState()
val manualPort by viewModel.manualPort.collectAsState()
val canvasDebugStatusEnabled by viewModel.canvasDebugStatusEnabled.collectAsState()
val statusText by viewModel.statusText.collectAsState()
val serverName by viewModel.serverName.collectAsState()
val remoteAddress by viewModel.remoteAddress.collectAsState()
@@ -76,22 +73,6 @@ fun SettingsSheet(viewModel: MainViewModel) {
val listState = rememberLazyListState()
val (wakeWordsText, setWakeWordsText) = remember { mutableStateOf("") }
val (advancedExpanded, setAdvancedExpanded) = remember { mutableStateOf(false) }
val deviceModel =
remember {
listOfNotNull(Build.MANUFACTURER, Build.MODEL)
.joinToString(" ")
.trim()
.ifEmpty { "Android" }
}
val appVersion =
remember {
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
}
LaunchedEffect(wakeWords) { setWakeWordsText(wakeWords.joinToString(", ")) }
@@ -160,8 +141,6 @@ fun SettingsSheet(viewModel: MainViewModel) {
)
}
item { Text("Instance ID: $instanceId", color = MaterialTheme.colorScheme.onSurfaceVariant) }
item { Text("Device: $deviceModel", color = MaterialTheme.colorScheme.onSurfaceVariant) }
item { Text("Version: $appVersion", color = MaterialTheme.colorScheme.onSurfaceVariant) }
item { HorizontalDivider() }
@@ -201,27 +180,9 @@ fun SettingsSheet(viewModel: MainViewModel) {
item { Text("No bridges found yet.", color = MaterialTheme.colorScheme.onSurfaceVariant) }
} else {
items(items = visibleBridges, key = { it.stableId }) { bridge ->
val detailLines =
buildList {
add("IP: ${bridge.host}:${bridge.port}")
bridge.lanHost?.let { add("LAN: $it") }
bridge.tailnetDns?.let { add("Tailnet: $it") }
if (bridge.gatewayPort != null || bridge.bridgePort != null || bridge.canvasPort != null) {
val gw = bridge.gatewayPort?.toString() ?: "—"
val br = (bridge.bridgePort ?: bridge.port).toString()
val canvas = bridge.canvasPort?.toString() ?: "—"
add("Ports: gw $gw · bridge $br · canvas $canvas")
}
}
ListItem(
headlineContent = { Text(bridge.name) },
supportingContent = {
Column(verticalArrangement = Arrangement.spacedBy(2.dp)) {
detailLines.forEach { line ->
Text(line, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
},
supportingContent = { Text("${bridge.host}:${bridge.port}") },
trailingContent = {
Button(
onClick = {
@@ -433,23 +394,6 @@ fun SettingsSheet(viewModel: MainViewModel) {
)
}
item { HorizontalDivider() }
// Debug
item { Text("Debug", style = MaterialTheme.typography.titleSmall) }
item {
ListItem(
headlineContent = { Text("Debug Canvas Status") },
supportingContent = { Text("Show status text in the canvas when debug is enabled.") },
trailingContent = {
Switch(
checked = canvasDebugStatusEnabled,
onCheckedChange = viewModel::setCanvasDebugStatusEnabled,
)
},
)
}
item { Spacer(modifier = Modifier.height(20.dp)) }
}
}
@@ -28,7 +28,6 @@ import androidx.compose.ui.unit.dp
fun StatusPill(
bridge: BridgeState,
voiceEnabled: Boolean,
activity: StatusActivity? = null,
onClick: () -> Unit,
modifier: Modifier = Modifier,
) {
@@ -63,49 +62,23 @@ fun StatusPill(
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
if (activity != null) {
Row(
horizontalArrangement = Arrangement.spacedBy(6.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Icon(
imageVector = activity.icon,
contentDescription = activity.contentDescription,
tint = activity.tint ?: overlayIconColor(),
modifier = Modifier.size(18.dp),
)
Text(
text = activity.title,
style = MaterialTheme.typography.labelLarge,
maxLines = 1,
)
}
} else {
Icon(
imageVector = if (voiceEnabled) Icons.Default.Mic else Icons.Default.MicOff,
contentDescription = if (voiceEnabled) "Voice enabled" else "Voice disabled",
tint =
if (voiceEnabled) {
overlayIconColor()
} else {
MaterialTheme.colorScheme.onSurfaceVariant
},
modifier = Modifier.size(18.dp),
)
}
Icon(
imageVector = if (voiceEnabled) Icons.Default.Mic else Icons.Default.MicOff,
contentDescription = if (voiceEnabled) "Voice enabled" else "Voice disabled",
tint =
if (voiceEnabled) {
overlayIconColor()
} else {
MaterialTheme.colorScheme.onSurfaceVariant
},
modifier = Modifier.size(18.dp),
)
Spacer(modifier = Modifier.width(2.dp))
}
}
}
data class StatusActivity(
val title: String,
val icon: androidx.compose.ui.graphics.vector.ImageVector,
val contentDescription: String,
val tint: Color? = null,
)
enum class BridgeState(val title: String, val color: Color) {
Connected("Connected", Color(0xFF2ECC71)),
Connecting("Connecting…", Color(0xFFF1C40F)),
@@ -1,134 +0,0 @@
package com.steipete.clawdis.node.ui
import androidx.compose.animation.core.LinearEasing
import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
@Composable
fun TalkOrbOverlay(
seamColor: Color,
statusText: String,
isListening: Boolean,
isSpeaking: Boolean,
modifier: Modifier = Modifier,
) {
val transition = rememberInfiniteTransition(label = "talk-orb")
val t by
transition.animateFloat(
initialValue = 0f,
targetValue = 1f,
animationSpec =
infiniteRepeatable(
animation = tween(durationMillis = 1500, easing = LinearEasing),
repeatMode = RepeatMode.Restart,
),
label = "pulse",
)
val trimmed = statusText.trim()
val showStatus = trimmed.isNotEmpty() && trimmed != "Off"
val phase =
when {
isSpeaking -> "Speaking"
isListening -> "Listening"
else -> "Thinking"
}
Column(
modifier = modifier.padding(24.dp),
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.spacedBy(12.dp),
) {
Box(contentAlignment = Alignment.Center) {
Canvas(modifier = Modifier.size(360.dp)) {
val center = this.center
val baseRadius = size.minDimension * 0.30f
val ring1 = 1.05f + (t * 0.25f)
val ring2 = 1.20f + (t * 0.55f)
val ringAlpha1 = (1f - t) * 0.34f
val ringAlpha2 = (1f - t) * 0.22f
drawCircle(
color = seamColor.copy(alpha = ringAlpha1),
radius = baseRadius * ring1,
center = center,
style = Stroke(width = 3.dp.toPx()),
)
drawCircle(
color = seamColor.copy(alpha = ringAlpha2),
radius = baseRadius * ring2,
center = center,
style = Stroke(width = 3.dp.toPx()),
)
drawCircle(
brush =
Brush.radialGradient(
colors =
listOf(
seamColor.copy(alpha = 0.92f),
seamColor.copy(alpha = 0.40f),
Color.Black.copy(alpha = 0.56f),
),
center = center,
radius = baseRadius * 1.35f,
),
radius = baseRadius,
center = center,
)
drawCircle(
color = seamColor.copy(alpha = 0.34f),
radius = baseRadius,
center = center,
style = Stroke(width = 1.dp.toPx()),
)
}
}
if (showStatus) {
Surface(
color = Color.Black.copy(alpha = 0.40f),
shape = CircleShape,
) {
Text(
text = trimmed,
modifier = Modifier.padding(horizontal = 14.dp, vertical = 8.dp),
color = Color.White.copy(alpha = 0.92f),
style = MaterialTheme.typography.labelLarge,
fontWeight = FontWeight.SemiBold,
)
}
} else {
Text(
text = phase,
color = Color.White.copy(alpha = 0.80f),
style = MaterialTheme.typography.labelLarge,
fontWeight = FontWeight.SemiBold,
)
}
}
}
@@ -17,7 +17,6 @@ import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.AnnotatedString
@@ -32,7 +31,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
@Composable
fun ChatMarkdown(text: String, textColor: Color) {
fun ChatMarkdown(text: String) {
val blocks = remember(text) { splitMarkdown(text) }
val inlineCodeBg = MaterialTheme.colorScheme.surfaceContainerLow
@@ -45,7 +44,7 @@ fun ChatMarkdown(text: String, textColor: Color) {
Text(
text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg),
style = MaterialTheme.typography.bodyMedium,
color = textColor,
color = MaterialTheme.colorScheme.onSurface,
)
}
is ChatMarkdownBlock.Code -> {
@@ -7,9 +7,11 @@ import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme
@@ -58,21 +60,20 @@ fun ChatMessageBubble(message: ChatMessage) {
.background(bubbleBackground(isUser))
.padding(horizontal = 12.dp, vertical = 10.dp),
) {
val textColor = textColorOverBubble(isUser)
ChatMessageBody(content = message.content, textColor = textColor)
ChatMessageBody(content = message.content)
}
}
}
}
@Composable
private fun ChatMessageBody(content: List<ChatMessageContent>, textColor: Color) {
private fun ChatMessageBody(content: List<ChatMessageContent>) {
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
for (part in content) {
when (part.type) {
"text" -> {
val text = part.text ?: continue
ChatMarkdown(text = text, textColor = textColor)
ChatMarkdown(text = text)
}
else -> {
val b64 = part.base64 ?: continue
@@ -130,7 +131,7 @@ fun ChatStreamingAssistantBubble(text: String) {
color = MaterialTheme.colorScheme.surfaceContainer,
) {
Box(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp)) {
ChatMarkdown(text = text, textColor = MaterialTheme.colorScheme.onSurface)
ChatMarkdown(text = text)
}
}
}
@@ -149,15 +150,6 @@ private fun bubbleBackground(isUser: Boolean): Brush {
}
}
@Composable
private fun textColorOverBubble(isUser: Boolean): Color {
return if (isUser) {
MaterialTheme.colorScheme.onPrimary
} else {
MaterialTheme.colorScheme.onSurface
}
}
@Composable
private fun ChatBase64Image(base64: String, mimeType: String?) {
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
@@ -1,98 +0,0 @@
package com.steipete.clawdis.node.voice
import android.media.MediaDataSource
import kotlin.math.min
internal class StreamingMediaDataSource : MediaDataSource() {
private data class Chunk(val start: Long, val data: ByteArray)
private val lock = Object()
private val chunks = ArrayList<Chunk>()
private var totalSize: Long = 0
private var closed = false
private var finished = false
private var lastReadIndex = 0
fun append(data: ByteArray) {
if (data.isEmpty()) return
synchronized(lock) {
if (closed || finished) return
val chunk = Chunk(totalSize, data)
chunks.add(chunk)
totalSize += data.size.toLong()
lock.notifyAll()
}
}
fun finish() {
synchronized(lock) {
if (closed) return
finished = true
lock.notifyAll()
}
}
fun fail() {
synchronized(lock) {
closed = true
lock.notifyAll()
}
}
override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int {
if (position < 0) return -1
synchronized(lock) {
while (!closed && !finished && position >= totalSize) {
lock.wait()
}
if (closed) return -1
if (position >= totalSize && finished) return -1
val available = (totalSize - position).toInt()
val toRead = min(size, available)
var remaining = toRead
var destOffset = offset
var pos = position
var index = findChunkIndex(pos)
while (remaining > 0 && index < chunks.size) {
val chunk = chunks[index]
val inChunkOffset = (pos - chunk.start).toInt()
if (inChunkOffset >= chunk.data.size) {
index++
continue
}
val copyLen = min(remaining, chunk.data.size - inChunkOffset)
System.arraycopy(chunk.data, inChunkOffset, buffer, destOffset, copyLen)
remaining -= copyLen
destOffset += copyLen
pos += copyLen
if (inChunkOffset + copyLen >= chunk.data.size) {
index++
}
}
return toRead - remaining
}
}
override fun getSize(): Long = -1
override fun close() {
synchronized(lock) {
closed = true
lock.notifyAll()
}
}
private fun findChunkIndex(position: Long): Int {
var index = lastReadIndex
while (index < chunks.size) {
val chunk = chunks[index]
if (position < chunk.start + chunk.data.size) break
index++
}
lastReadIndex = index
return index
}
}
@@ -1,191 +0,0 @@
package com.steipete.clawdis.node.voice
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
private val directiveJson = Json { ignoreUnknownKeys = true }
data class TalkDirective(
val voiceId: String? = null,
val modelId: String? = null,
val speed: Double? = null,
val rateWpm: Int? = null,
val stability: Double? = null,
val similarity: Double? = null,
val style: Double? = null,
val speakerBoost: Boolean? = null,
val seed: Long? = null,
val normalize: String? = null,
val language: String? = null,
val outputFormat: String? = null,
val latencyTier: Int? = null,
val once: Boolean? = null,
)
data class TalkDirectiveParseResult(
val directive: TalkDirective?,
val stripped: String,
val unknownKeys: List<String>,
)
object TalkDirectiveParser {
fun parse(text: String): TalkDirectiveParseResult {
val normalized = text.replace("\r\n", "\n")
val lines = normalized.split("\n").toMutableList()
if (lines.isEmpty()) return TalkDirectiveParseResult(null, text, emptyList())
val firstNonEmpty = lines.indexOfFirst { it.trim().isNotEmpty() }
if (firstNonEmpty == -1) return TalkDirectiveParseResult(null, text, emptyList())
val head = lines[firstNonEmpty].trim()
if (!head.startsWith("{") || !head.endsWith("}")) {
return TalkDirectiveParseResult(null, text, emptyList())
}
val obj = parseJsonObject(head) ?: return TalkDirectiveParseResult(null, text, emptyList())
val speakerBoost =
boolValue(obj, listOf("speaker_boost", "speakerBoost"))
?: boolValue(obj, listOf("no_speaker_boost", "noSpeakerBoost"))?.not()
val directive = TalkDirective(
voiceId = stringValue(obj, listOf("voice", "voice_id", "voiceId")),
modelId = stringValue(obj, listOf("model", "model_id", "modelId")),
speed = doubleValue(obj, listOf("speed")),
rateWpm = intValue(obj, listOf("rate", "wpm")),
stability = doubleValue(obj, listOf("stability")),
similarity = doubleValue(obj, listOf("similarity", "similarity_boost", "similarityBoost")),
style = doubleValue(obj, listOf("style")),
speakerBoost = speakerBoost,
seed = longValue(obj, listOf("seed")),
normalize = stringValue(obj, listOf("normalize", "apply_text_normalization")),
language = stringValue(obj, listOf("lang", "language_code", "language")),
outputFormat = stringValue(obj, listOf("output_format", "format")),
latencyTier = intValue(obj, listOf("latency", "latency_tier", "latencyTier")),
once = boolValue(obj, listOf("once")),
)
val hasDirective = listOf(
directive.voiceId,
directive.modelId,
directive.speed,
directive.rateWpm,
directive.stability,
directive.similarity,
directive.style,
directive.speakerBoost,
directive.seed,
directive.normalize,
directive.language,
directive.outputFormat,
directive.latencyTier,
directive.once,
).any { it != null }
if (!hasDirective) return TalkDirectiveParseResult(null, text, emptyList())
val knownKeys = setOf(
"voice", "voice_id", "voiceid",
"model", "model_id", "modelid",
"speed", "rate", "wpm",
"stability", "similarity", "similarity_boost", "similarityboost",
"style",
"speaker_boost", "speakerboost",
"no_speaker_boost", "nospeakerboost",
"seed",
"normalize", "apply_text_normalization",
"lang", "language_code", "language",
"output_format", "format",
"latency", "latency_tier", "latencytier",
"once",
)
val unknownKeys = obj.keys.filter { !knownKeys.contains(it.lowercase()) }.sorted()
lines.removeAt(firstNonEmpty)
if (firstNonEmpty < lines.size) {
if (lines[firstNonEmpty].trim().isEmpty()) {
lines.removeAt(firstNonEmpty)
}
}
return TalkDirectiveParseResult(directive, lines.joinToString("\n"), unknownKeys)
}
private fun parseJsonObject(line: String): JsonObject? {
return try {
directiveJson.parseToJsonElement(line) as? JsonObject
} catch (_: Throwable) {
null
}
}
private fun stringValue(obj: JsonObject, keys: List<String>): String? {
for (key in keys) {
val value = obj[key].asStringOrNull()?.trim()
if (!value.isNullOrEmpty()) return value
}
return null
}
private fun doubleValue(obj: JsonObject, keys: List<String>): Double? {
for (key in keys) {
val value = obj[key].asDoubleOrNull()
if (value != null) return value
}
return null
}
private fun intValue(obj: JsonObject, keys: List<String>): Int? {
for (key in keys) {
val value = obj[key].asIntOrNull()
if (value != null) return value
}
return null
}
private fun longValue(obj: JsonObject, keys: List<String>): Long? {
for (key in keys) {
val value = obj[key].asLongOrNull()
if (value != null) return value
}
return null
}
private fun boolValue(obj: JsonObject, keys: List<String>): Boolean? {
for (key in keys) {
val value = obj[key].asBooleanOrNull()
if (value != null) return value
}
return null
}
}
private fun JsonElement?.asStringOrNull(): String? =
(this as? JsonPrimitive)?.takeIf { it.isString }?.content
private fun JsonElement?.asDoubleOrNull(): Double? {
val primitive = this as? JsonPrimitive ?: return null
return primitive.content.toDoubleOrNull()
}
private fun JsonElement?.asIntOrNull(): Int? {
val primitive = this as? JsonPrimitive ?: return null
return primitive.content.toIntOrNull()
}
private fun JsonElement?.asLongOrNull(): Long? {
val primitive = this as? JsonPrimitive ?: return null
return primitive.content.toLongOrNull()
}
private fun JsonElement?.asBooleanOrNull(): Boolean? {
val primitive = this as? JsonPrimitive ?: return null
val content = primitive.content.trim().lowercase()
return when (content) {
"true", "yes", "1" -> true
"false", "no", "0" -> false
else -> null
}
}
File diff suppressed because it is too large. Load diff
@@ -2,5 +2,5 @@
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@color/ic_launcher_background" />
<foreground android:drawable="@mipmap/ic_launcher_foreground" />
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
</adaptive-icon>
@@ -2,5 +2,5 @@
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@color/ic_launcher_background" />
<foreground android:drawable="@mipmap/ic_launcher_foreground" />
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
</adaptive-icon>
@@ -1,7 +1,8 @@
<resources>
<resources xmlns:tools="http://schemas.android.com/tools">
<style name="Theme.ClawdisNode" parent="Theme.Material3.DayNight.NoActionBar">
<item name="android:statusBarColor">@android:color/transparent</item>
<item name="android:navigationBarColor">@android:color/transparent</item>
<item name="android:windowLightStatusBar">false</item>
<item name="android:windowLightStatusBar" tools:targetApi="m">false</item>
</style>
</resources>
@@ -1,4 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<full-backup-content>
<include domain="file" path="." />
</full-backup-content>
@@ -1,9 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<data-extraction-rules>
<cloud-backup>
<include domain="file" path="." />
</cloud-backup>
<device-transfer>
<include domain="file" path="." />
</device-transfer>
</data-extraction-rules>
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<network-security-config xmlns:tools="http://schemas.android.com/tools">
<network-security-config>
<!-- This app is primarily used on a trusted tailnet; allow cleartext for IP-based endpoints too. -->
<base-config cleartextTrafficPermitted="true" tools:ignore="InsecureBaseConfiguration" />
<base-config cleartextTrafficPermitted="true" />
<!-- Allow HTTP for tailnet/local dev endpoints (e.g. canvas/background web). -->
<domain-config cleartextTrafficPermitted="true">
<domain includeSubdomains="true">clawdis.internal</domain>
@@ -1,14 +0,0 @@
package com.steipete.clawdis.node.bridge
import io.kotest.core.spec.style.StringSpec
import io.kotest.matchers.shouldBe
class BridgeEndpointKotestTest : StringSpec({
"manual endpoint builds stable id + name" {
val endpoint = BridgeEndpoint.manual("10.0.0.5", 18790)
endpoint.stableId shouldBe "manual|10.0.0.5|18790"
endpoint.name shouldBe "10.0.0.5:18790"
endpoint.host shouldBe "10.0.0.5"
endpoint.port shouldBe 18790
}
})
@@ -1,47 +0,0 @@
package com.steipete.clawdis.node.node
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
import kotlin.math.min
class JpegSizeLimiterTest {
@Test
fun compressesLargePayloadsUnderLimit() {
val maxBytes = 5 * 1024 * 1024
val result =
JpegSizeLimiter.compressToLimit(
initialWidth = 4000,
initialHeight = 3000,
startQuality = 95,
maxBytes = maxBytes,
encode = { width, height, quality ->
val estimated = (width.toLong() * height.toLong() * quality.toLong()) / 100
val size = min(maxBytes.toLong() * 2, estimated).toInt()
ByteArray(size)
},
)
assertTrue(result.bytes.size <= maxBytes)
assertTrue(result.width <= 4000)
assertTrue(result.height <= 3000)
assertTrue(result.quality <= 95)
}
@Test
fun keepsSmallPayloadsAsIs() {
val maxBytes = 5 * 1024 * 1024
val result =
JpegSizeLimiter.compressToLimit(
initialWidth = 800,
initialHeight = 600,
startQuality = 90,
maxBytes = maxBytes,
encode = { _, _, _ -> ByteArray(120_000) },
)
assertEquals(800, result.width)
assertEquals(600, result.height)
assertEquals(90, result.quality)
}
}
@@ -1,24 +1,9 @@
package com.steipete.clawdis.node.protocol
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonObject
import org.junit.Assert.assertEquals
import org.junit.Test
class ClawdisCanvasA2UIActionTest {
@Test
fun extractActionNameAcceptsNameOrAction() {
val nameObj = Json.parseToJsonElement("{\"name\":\"Hello\"}").jsonObject
assertEquals("Hello", ClawdisCanvasA2UIAction.extractActionName(nameObj))
val actionObj = Json.parseToJsonElement("{\"action\":\"Wave\"}").jsonObject
assertEquals("Wave", ClawdisCanvasA2UIAction.extractActionName(actionObj))
val fallbackObj =
Json.parseToJsonElement("{\"name\":\" \",\"action\":\"Fallback\"}").jsonObject
assertEquals("Fallback", ClawdisCanvasA2UIAction.extractActionName(fallbackObj))
}
@Test
fun formatAgentMessageMatchesSharedSpec() {
val msg =
@@ -47,3 +32,4 @@ class ClawdisCanvasA2UIActionTest {
)
}
}
@@ -1,55 +0,0 @@
package com.steipete.clawdis.node.voice
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
class TalkDirectiveParserTest {
@Test
fun parsesDirectiveAndStripsHeader() {
val input = """
{"voice":"voice-123","once":true}
Hello from talk mode.
""".trimIndent()
val result = TalkDirectiveParser.parse(input)
assertEquals("voice-123", result.directive?.voiceId)
assertEquals(true, result.directive?.once)
assertEquals("Hello from talk mode.", result.stripped.trim())
}
@Test
fun ignoresUnknownKeysButReportsThem() {
val input = """
{"voice":"abc","foo":1,"bar":"baz"}
Hi there.
""".trimIndent()
val result = TalkDirectiveParser.parse(input)
assertEquals("abc", result.directive?.voiceId)
assertTrue(result.unknownKeys.containsAll(listOf("bar", "foo")))
}
@Test
fun parsesAlternateKeys() {
val input = """
{"model_id":"eleven_v3","similarity_boost":0.4,"no_speaker_boost":true,"rate":200}
Speak.
""".trimIndent()
val result = TalkDirectiveParser.parse(input)
assertEquals("eleven_v3", result.directive?.modelId)
assertEquals(0.4, result.directive?.similarity)
assertEquals(false, result.directive?.speakerBoost)
assertEquals(200, result.directive?.rateWpm)
}
@Test
fun returnsNullWhenNoDirectivePresent() {
val input = """
{}
Hello.
""".trimIndent()
val result = TalkDirectiveParser.parse(input)
assertNull(result.directive)
assertEquals(input, result.stripped)
}
}
@@ -6,15 +6,6 @@ import Observation
import SwiftUI
import UIKit
protocol BridgePairingClient: Sendable {
func pairAndHello(
endpoint: NWEndpoint,
hello: BridgeHello,
onStatus: (@Sendable (String) -> Void)?) async throws -> String
}
extension BridgeClient: BridgePairingClient {}
@MainActor
@Observable
final class BridgeConnectionController {
@@ -25,16 +16,10 @@ final class BridgeConnectionController {
private let discovery = BridgeDiscoveryModel()
private weak var appModel: NodeAppModel?
private var didAutoConnect = false
private var seenStableIDs = Set<String>()
private let bridgeClientFactory: @Sendable () -> any BridgePairingClient
init(
appModel: NodeAppModel,
startDiscovery: Bool = true,
bridgeClientFactory: @escaping @Sendable () -> any BridgePairingClient = { BridgeClient() })
{
init(appModel: NodeAppModel, startDiscovery: Bool = true) {
self.appModel = appModel
self.bridgeClientFactory = bridgeClientFactory
BridgeSettingsStore.bootstrapPersistence()
let defaults = UserDefaults.standard
@@ -100,7 +85,7 @@ final class BridgeConnectionController {
let token = KeychainStore.loadString(
service: "com.steipete.clawdis.bridge",
account: self.keychainAccount(instanceId: instanceId))?
account: "bridge-token.\(instanceId)")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !token.isEmpty else { return }
@@ -114,40 +99,28 @@ final class BridgeConnectionController {
guard let port = NWEndpoint.Port(rawValue: UInt16(resolvedPort)) else { return }
self.didAutoConnect = true
let endpoint = NWEndpoint.hostPort(host: NWEndpoint.Host(manualHost), port: port)
self.startAutoConnect(endpoint: endpoint, token: token, instanceId: instanceId)
appModel.connectToBridge(
endpoint: .hostPort(host: NWEndpoint.Host(manualHost), port: port),
hello: self.makeHello(token: token))
return
}
let preferredStableID = defaults.string(forKey: "bridge.preferredStableID")?
let targetStableID = defaults.string(forKey: "bridge.lastDiscoveredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let lastDiscoveredStableID = defaults.string(forKey: "bridge.lastDiscoveredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let candidates = [preferredStableID, lastDiscoveredStableID].filter { !$0.isEmpty }
guard let targetStableID = candidates.first(where: { id in
self.bridges.contains(where: { $0.stableID == id })
}) else { return }
guard !targetStableID.isEmpty else { return }
guard let target = self.bridges.first(where: { $0.stableID == targetStableID }) else { return }
self.didAutoConnect = true
self.startAutoConnect(endpoint: target.endpoint, token: token, instanceId: instanceId)
appModel.connectToBridge(endpoint: target.endpoint, hello: self.makeHello(token: token))
}
private func updateLastDiscoveredBridge(from bridges: [BridgeDiscoveryModel.DiscoveredBridge]) {
let defaults = UserDefaults.standard
let preferred = defaults.string(forKey: "bridge.preferredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let existingLast = defaults.string(forKey: "bridge.lastDiscoveredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let newlyDiscovered = bridges.filter { self.seenStableIDs.insert($0.stableID).inserted }
guard let last = newlyDiscovered.last else { return }
// Avoid overriding user intent (preferred/lastDiscovered are also set on manual Connect).
guard preferred.isEmpty, existingLast.isEmpty else { return }
guard let first = bridges.first else { return }
defaults.set(first.stableID, forKey: "bridge.lastDiscoveredStableID")
BridgeSettingsStore.saveLastDiscoveredBridgeStableID(first.stableID)
UserDefaults.standard.set(last.stableID, forKey: "bridge.lastDiscoveredStableID")
BridgeSettingsStore.saveLastDiscoveredBridgeStableID(last.stableID)
}
private func makeHello(token: String) -> BridgeHello {
@@ -167,40 +140,6 @@ final class BridgeConnectionController {
commands: self.currentCommands())
}
private func keychainAccount(instanceId: String) -> String {
"bridge-token.\(instanceId)"
}
private func startAutoConnect(endpoint: NWEndpoint, token: String, instanceId: String) {
guard let appModel else { return }
Task { [weak self] in
guard let self else { return }
do {
let hello = self.makeHello(token: token)
let refreshed = try await self.bridgeClientFactory().pairAndHello(
endpoint: endpoint,
hello: hello,
onStatus: { status in
Task { @MainActor in
appModel.bridgeStatusText = status
}
})
let resolvedToken = refreshed.isEmpty ? token : refreshed
if !refreshed.isEmpty, refreshed != token {
_ = KeychainStore.saveString(
refreshed,
service: "com.steipete.clawdis.bridge",
account: self.keychainAccount(instanceId: instanceId))
}
appModel.connectToBridge(endpoint: endpoint, hello: self.makeHello(token: resolvedToken))
} catch {
await MainActor.run {
appModel.bridgeStatusText = "Bridge error: \(error.localizedDescription)"
}
}
}
}
private func resolvedDisplayName(defaults: UserDefaults) -> String {
let key = "node.displayName"
let existing = defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
@@ -292,47 +231,3 @@ final class BridgeConnectionController {
Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"
}
}
#if DEBUG
extension BridgeConnectionController {
func _test_makeHello(token: String) -> BridgeHello {
self.makeHello(token: token)
}
func _test_resolvedDisplayName(defaults: UserDefaults) -> String {
self.resolvedDisplayName(defaults: defaults)
}
func _test_currentCaps() -> [String] {
self.currentCaps()
}
func _test_currentCommands() -> [String] {
self.currentCommands()
}
func _test_platformString() -> String {
self.platformString()
}
func _test_deviceFamily() -> String {
self.deviceFamily()
}
func _test_modelIdentifier() -> String {
self.modelIdentifier()
}
func _test_appVersion() -> String {
self.appVersion()
}
func _test_setBridges(_ bridges: [BridgeDiscoveryModel.DiscoveredBridge]) {
self.bridges = bridges
}
func _test_triggerAutoConnect() {
self.maybeAutoConnect()
}
}
#endif
@@ -18,12 +18,6 @@ final class BridgeDiscoveryModel {
var endpoint: NWEndpoint
var stableID: String
var debugID: String
var lanHost: String?
var tailnetDns: String?
var gatewayPort: Int?
var bridgePort: Int?
var canvasPort: Int?
var cliPath: String?
}
var bridges: [DiscoveredBridge] = []
@@ -74,8 +68,7 @@ final class BridgeDiscoveryModel {
switch result.endpoint {
case let .service(name, _, _, _):
let decodedName = BonjourEscapes.decode(name)
let txt = result.endpoint.txtRecord?.dictionary ?? [:]
let advertisedName = txt["displayName"]
let advertisedName = result.endpoint.txtRecord?.dictionary["displayName"]
let prettyAdvertised = advertisedName
.map(Self.prettifyInstanceName)
.flatMap { $0.isEmpty ? nil : $0 }
@@ -84,13 +77,7 @@ final class BridgeDiscoveryModel {
name: prettyName,
endpoint: result.endpoint,
stableID: BridgeEndpointID.stableID(result.endpoint),
debugID: BridgeEndpointID.prettyDescription(result.endpoint),
lanHost: Self.txtValue(txt, key: "lanHost"),
tailnetDns: Self.txtValue(txt, key: "tailnetDns"),
gatewayPort: Self.txtIntValue(txt, key: "gatewayPort"),
bridgePort: Self.txtIntValue(txt, key: "bridgePort"),
canvasPort: Self.txtIntValue(txt, key: "canvasPort"),
cliPath: Self.txtValue(txt, key: "cliPath"))
debugID: BridgeEndpointID.prettyDescription(result.endpoint))
default:
return nil
}
@@ -204,14 +191,4 @@ final class BridgeDiscoveryModel {
.replacingOccurrences(of: #"\s+\(\d+\)$"#, with: "", options: .regularExpression)
return stripped.trimmingCharacters(in: .whitespacesAndNewlines)
}
private static func txtValue(_ dict: [String: String], key: String) -> String? {
let raw = dict[key]?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return raw.isEmpty ? nil : raw
}
private static func txtIntValue(_ dict: [String: String], key: String) -> Int? {
guard let raw = self.txtValue(dict, key: key) else { return nil }
return Int(raw)
}
}
@@ -84,14 +84,10 @@ actor CameraController {
}
withExtendedLifetime(delegate) {}
let maxPayloadBytes = 5 * 1024 * 1024
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
let maxEncodedBytes = (maxPayloadBytes / 4) * 3
let res = try JPEGTranscoder.transcodeToJPEG(
imageData: rawData,
maxWidthPx: maxWidth,
quality: quality,
maxBytes: maxEncodedBytes)
quality: quality)
return (
format: format.rawValue,
+2 -4
View File
@@ -4,20 +4,18 @@ import SwiftUI
struct ChatSheet: View {
@Environment(\.dismiss) private var dismiss
@State private var viewModel: ClawdisChatViewModel
private let userAccent: Color?
init(bridge: BridgeSession, sessionKey: String = "main", userAccent: Color? = nil) {
init(bridge: BridgeSession, sessionKey: String = "main") {
let transport = IOSBridgeChatTransport(bridge: bridge)
self._viewModel = State(
initialValue: ClawdisChatViewModel(
sessionKey: sessionKey,
transport: transport))
self.userAccent = userAccent
}
var body: some View {
NavigationStack {
ClawdisChatView(viewModel: self.viewModel, userAccent: self.userAccent)
ClawdisChatView(viewModel: self.viewModel)
.navigationTitle("Chat")
.navigationBarTitleDisplayMode(.inline)
.toolbar {
-19
View File
@@ -22,11 +22,6 @@
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>NSAppTransportSecurity</key>
<dict>
<key>NSAllowsArbitraryLoadsInWebContent</key>
<true/>
</dict>
<key>NSBonjourServices</key>
<array>
<string>_clawdis-bridge._tcp</string>
@@ -50,19 +45,5 @@
</array>
<key>UILaunchScreen</key>
<dict/>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
</dict>
</plist>
+9 -124
View File
@@ -22,28 +22,23 @@ final class NodeAppModel {
var bridgeServerName: String?
var bridgeRemoteAddress: String?
var connectedBridgeID: String?
var seamColorHex: String?
var mainSessionKey: String = "main"
private let bridge = BridgeSession()
private var bridgeTask: Task<Void, Never>?
private var voiceWakeSyncTask: Task<Void, Never>?
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
let voiceWake = VoiceWakeManager()
let talkMode = TalkModeManager()
private var lastAutoA2uiURL: String?
var bridgeSession: BridgeSession { self.bridge }
var cameraHUDText: String?
var cameraHUDKind: CameraHUDKind?
var cameraFlashNonce: Int = 0
var screenRecordActive: Bool = false
init() {
self.voiceWake.configure { [weak self] cmd in
guard let self else { return }
let sessionKey = await MainActor.run { self.mainSessionKey }
let sessionKey = "main"
do {
try await self.sendVoiceTranscript(text: cmd, sessionKey: sessionKey)
} catch {
@@ -53,9 +48,6 @@ final class NodeAppModel {
let enabled = UserDefaults.standard.bool(forKey: "voiceWake.enabled")
self.voiceWake.setEnabled(enabled)
self.talkMode.attachBridge(self.bridge)
let talkEnabled = UserDefaults.standard.bool(forKey: "talk.enabled")
self.talkMode.setEnabled(talkEnabled)
// Wire up deep links from canvas taps
self.screen.onDeepLink = { [weak self] url in
@@ -88,7 +80,7 @@ final class NodeAppModel {
}()
guard !userAction.isEmpty else { return }
guard let name = ClawdisCanvasA2UIAction.extractActionName(userAction) else { return }
guard let name = userAction["name"] as? String, !name.isEmpty else { return }
let actionId: String = {
let id = (userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return id.isEmpty ? UUID().uuidString : id
@@ -110,12 +102,14 @@ final class NodeAppModel {
let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"])
let sessionKey = "main"
let messageContext = ClawdisCanvasA2UIAction.AgentMessageContext(
let message = ClawdisCanvasA2UIAction.formatAgentMessage(
actionName: name,
session: .init(key: sessionKey, surfaceId: surfaceId),
component: .init(id: sourceComponentId, host: host, instanceId: instanceId),
sessionKey: sessionKey,
surfaceId: surfaceId,
sourceComponentId: sourceComponentId,
host: host,
instanceId: instanceId,
contextJSON: contextJSON)
let message = ClawdisCanvasA2UIAction.formatAgentMessage(messageContext)
let ok: Bool
var errorText: String?
@@ -152,21 +146,7 @@ final class NodeAppModel {
guard let raw = await self.bridge.currentCanvasHostUrl() else { return nil }
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
return base.appendingPathComponent("__clawdis__/a2ui/").absoluteString + "?platform=ios"
}
private func showA2UIOnConnectIfNeeded() async {
guard let a2uiUrl = await self.resolveA2UIHostURL() else { return }
let current = self.screen.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
if current.isEmpty || current == self.lastAutoA2uiURL {
self.screen.navigate(to: a2uiUrl)
self.lastAutoA2uiURL = a2uiUrl
}
}
private func showLocalCanvasOnDisconnect() {
self.lastAutoA2uiURL = nil
self.screen.showDefaultCanvas()
return base.appendingPathComponent("__clawdis__/a2ui/").absoluteString
}
func setScenePhase(_ phase: ScenePhase) {
@@ -184,10 +164,6 @@ final class NodeAppModel {
self.voiceWake.setEnabled(enabled)
}
func setTalkEnabled(_ enabled: Bool) {
self.talkMode.setEnabled(enabled)
}
func connectToBridge(
endpoint: NWEndpoint,
hello: BridgeHello)
@@ -227,9 +203,7 @@ final class NodeAppModel {
self.bridgeRemoteAddress = addr
}
}
await self.refreshBrandingFromGateway()
await self.startVoiceWakeSync()
await self.showA2UIOnConnectIfNeeded()
},
onInvoke: { [weak self] req in
guard let self else {
@@ -242,9 +216,6 @@ final class NodeAppModel {
})
if Task.isCancelled { break }
await MainActor.run {
self.showLocalCanvasOnDisconnect()
}
attempt += 1
let sleepSeconds = min(6.0, 0.35 * pow(1.7, Double(attempt)))
try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000))
@@ -255,7 +226,6 @@ final class NodeAppModel {
self.bridgeStatusText = "Bridge error: \(error.localizedDescription)"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.showLocalCanvasOnDisconnect()
}
let sleepSeconds = min(8.0, 0.5 * pow(1.7, Double(attempt)))
try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000))
@@ -267,9 +237,6 @@ final class NodeAppModel {
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeID = nil
self.seamColorHex = nil
self.mainSessionKey = "main"
self.showLocalCanvasOnDisconnect()
}
}
}
@@ -284,45 +251,6 @@ final class NodeAppModel {
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeID = nil
self.seamColorHex = nil
self.mainSessionKey = "main"
self.showLocalCanvasOnDisconnect()
}
var seamColor: Color {
Self.color(fromHex: self.seamColorHex) ?? Self.defaultSeamColor
}
private static let defaultSeamColor = Color(red: 79 / 255.0, green: 122 / 255.0, blue: 154 / 255.0)
private static func color(fromHex raw: String?) -> Color? {
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed
guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil }
let r = Double((value >> 16) & 0xFF) / 255.0
let g = Double((value >> 8) & 0xFF) / 255.0
let b = Double(value & 0xFF) / 255.0
return Color(red: r, green: g, blue: b)
}
private func refreshBrandingFromGateway() async {
do {
let res = try await self.bridge.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8)
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return }
guard let config = json["config"] as? [String: Any] else { return }
let ui = config["ui"] as? [String: Any]
let raw = (ui?["seamColor"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let session = config["session"] as? [String: Any]
let rawMainKey = (session?["mainKey"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let mainKey = rawMainKey.isEmpty ? "main" : rawMainKey
await MainActor.run {
self.seamColorHex = raw.isEmpty ? nil : raw
self.mainSessionKey = mainKey
}
} catch {
// ignore
}
}
func setGlobalWakeWords(_ words: [String]) async {
@@ -642,9 +570,6 @@ final class NodeAppModel {
NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4",
])
}
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
self.screenRecordActive = true
defer { self.screenRecordActive = false }
let path = try await self.screenRecorder.record(
screenIndex: params.screenIndex,
durationMs: params.durationMs,
@@ -735,43 +660,3 @@ final class NodeAppModel {
}
}
}
#if DEBUG
extension NodeAppModel {
func _test_handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
await self.handleInvoke(req)
}
static func _test_decodeParams<T: Decodable>(_ type: T.Type, from json: String?) throws -> T {
try self.decodeParams(type, from: json)
}
static func _test_encodePayload(_ obj: some Encodable) throws -> String {
try self.encodePayload(obj)
}
func _test_isCameraEnabled() -> Bool {
self.isCameraEnabled()
}
func _test_triggerCameraFlash() {
self.triggerCameraFlash()
}
func _test_showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) {
self.showCameraHUD(text: text, kind: kind, autoHideSeconds: autoHideSeconds)
}
func _test_handleCanvasA2UIAction(body: [String: Any]) async {
await self.handleCanvasA2UIAction(body: body)
}
func _test_resolveA2UIHostURL() async -> String? {
await self.resolveA2UIHostURL()
}
func _test_showLocalCanvasOnDisconnect() {
self.showLocalCanvasOnDisconnect()
}
}
#endif
+82 -118
View File
@@ -8,7 +8,6 @@ struct RootCanvas: View {
@Environment(\.scenePhase) private var scenePhase
@AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true
@AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false
@State private var presentedSheet: PresentedSheet?
@State private var voiceWakeToastText: String?
@State private var toastDismissTask: Task<Void, Never>?
@@ -51,20 +50,12 @@ struct RootCanvas: View {
case .settings:
SettingsTab()
case .chat:
ChatSheet(
bridge: self.appModel.bridgeSession,
sessionKey: self.appModel.mainSessionKey,
userAccent: self.appModel.seamColor)
ChatSheet(bridge: self.appModel.bridgeSession)
}
}
.onAppear { self.updateIdleTimer() }
.onChange(of: self.preventSleep) { _, _ in self.updateIdleTimer() }
.onChange(of: self.scenePhase) { _, _ in self.updateIdleTimer() }
.onAppear { self.updateCanvasDebugStatus() }
.onChange(of: self.canvasDebugStatusEnabled) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.bridgeStatusText) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.bridgeServerName) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.bridgeRemoteAddress) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in
guard let newValue else { return }
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -111,20 +102,9 @@ struct RootCanvas: View {
private func updateIdleTimer() {
UIApplication.shared.isIdleTimerDisabled = (self.scenePhase == .active && self.preventSleep)
}
private func updateCanvasDebugStatus() {
self.appModel.screen.setDebugStatusEnabled(self.canvasDebugStatusEnabled)
guard self.canvasDebugStatusEnabled else { return }
let title = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let subtitle = self.appModel.bridgeServerName ?? self.appModel.bridgeRemoteAddress
self.appModel.screen.updateDebugStatus(title: title, subtitle: subtitle)
}
}
private struct CanvasContent: View {
@Environment(NodeAppModel.self) private var appModel
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
@AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true
var systemColorScheme: ColorScheme
var bridgeStatus: StatusPill.BridgeState
var voiceWakeEnabled: Bool
@@ -146,21 +126,6 @@ private struct CanvasContent: View {
}
.accessibilityLabel("Chat")
if self.talkButtonEnabled {
// Talk mode lives on a side bubble so it doesn't get buried in settings.
OverlayButton(
systemImage: self.appModel.talkMode.isEnabled ? "waveform.circle.fill" : "waveform.circle",
brighten: self.brightenButtons,
tint: self.appModel.seamColor,
isActive: self.appModel.talkMode.isEnabled)
{
let next = !self.appModel.talkMode.isEnabled
self.talkEnabled = next
self.appModel.setTalkEnabled(next)
}
.accessibilityLabel("Talk Mode")
}
OverlayButton(systemImage: "gearshape.fill", brighten: self.brightenButtons) {
self.openSettings()
}
@@ -169,17 +134,10 @@ private struct CanvasContent: View {
.padding(.top, 10)
.padding(.trailing, 10)
}
.overlay(alignment: .center) {
if self.appModel.talkMode.isEnabled {
TalkOrbOverlay()
.transition(.opacity)
}
}
.overlay(alignment: .topLeading) {
StatusPill(
bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
activity: self.statusActivity,
brighten: self.brightenButtons,
onTap: {
self.openSettings()
@@ -197,78 +155,45 @@ private struct CanvasContent: View {
.transition(.move(edge: .top).combined(with: .opacity))
}
}
.overlay(alignment: .topLeading) {
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
CameraCaptureToast(
text: cameraHUDText,
kind: self.mapCameraKind(cameraHUDKind),
brighten: self.brightenButtons)
.padding(SwiftUI.Edge.Set.leading, 10)
.safeAreaPadding(SwiftUI.Edge.Set.top, 106)
.transition(
AnyTransition.move(edge: SwiftUI.Edge.top)
.combined(with: AnyTransition.opacity))
}
}
}
private var statusActivity: StatusPill.Activity? {
// Status pill owns transient activity state so it doesn't overlap the connection indicator.
if self.appModel.isBackgrounded {
return StatusPill.Activity(
title: "Foreground required",
systemImage: "exclamationmark.triangle.fill",
tint: .orange)
private func mapCameraKind(_ kind: NodeAppModel.CameraHUDKind) -> CameraCaptureToast.Kind {
switch kind {
case .photo:
.photo
case .recording:
.recording
case .success:
.success
case .error:
.error
}
let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let bridgeLower = bridgeStatus.lowercased()
if bridgeLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if bridgeLower.contains("approval") || bridgeLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
// Avoid duplicating the primary bridge status ("Connecting…") in the activity slot.
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
}
}
private struct OverlayButton: View {
let systemImage: String
let brighten: Bool
var tint: Color?
var isActive: Bool = false
let action: () -> Void
var body: some View {
Button(action: self.action) {
Image(systemName: self.systemImage)
.font(.system(size: 16, weight: .semibold))
.foregroundStyle(self.isActive ? (self.tint ?? .primary) : .primary)
.foregroundStyle(.primary)
.padding(10)
.background {
RoundedRectangle(cornerRadius: 12, style: .continuous)
@@ -286,26 +211,9 @@ private struct OverlayButton: View {
endPoint: .bottomTrailing))
.blendMode(.overlay)
}
.overlay {
if let tint {
RoundedRectangle(cornerRadius: 12, style: .continuous)
.fill(
LinearGradient(
colors: [
tint.opacity(self.isActive ? 0.22 : 0.14),
tint.opacity(self.isActive ? 0.10 : 0.06),
.clear,
],
startPoint: .topLeading,
endPoint: .bottomTrailing))
.blendMode(.overlay)
}
}
.overlay {
RoundedRectangle(cornerRadius: 12, style: .continuous)
.strokeBorder(
(self.tint ?? .white).opacity(self.isActive ? 0.34 : (self.brighten ? 0.24 : 0.18)),
lineWidth: self.isActive ? 0.7 : 0.5)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.35), radius: 12, y: 6)
}
@@ -339,3 +247,59 @@ private struct CameraFlashOverlay: View {
}
}
}
private struct CameraCaptureToast: View {
enum Kind {
case photo
case recording
case success
case error
}
var text: String
var kind: Kind
var brighten: Bool = false
var body: some View {
HStack(spacing: 10) {
self.icon
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
Text(self.text)
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
.lineLimit(1)
.truncationMode(.tail)
}
.padding(.vertical, 10)
.padding(.horizontal, 12)
.background {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.fill(.ultraThinMaterial)
.overlay {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
}
.accessibilityLabel("Camera")
.accessibilityValue(self.text)
}
@ViewBuilder
private var icon: some View {
switch self.kind {
case .photo:
Image(systemName: "camera.fill")
case .recording:
Image(systemName: "record.circle.fill")
.symbolRenderingMode(.palette)
.foregroundStyle(.red, .primary)
case .success:
Image(systemName: "checkmark.circle.fill")
case .error:
Image(systemName: "exclamationmark.triangle.fill")
}
}
}
-61
View File
@@ -26,7 +26,6 @@ struct RootTabs: View {
StatusPill(
bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
activity: self.statusActivity,
onTap: { self.selectedTab = 2 })
.padding(.leading, 10)
.safeAreaPadding(.top, 10)
@@ -80,64 +79,4 @@ struct RootTabs: View {
return .disconnected
}
private var statusActivity: StatusPill.Activity? {
// Keep the top pill consistent across tabs (camera + voice wake + pairing states).
if self.appModel.isBackgrounded {
return StatusPill.Activity(
title: "Foreground required",
systemImage: "exclamationmark.triangle.fill",
tint: .orange)
}
let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let bridgeLower = bridgeStatus.lowercased()
if bridgeLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if bridgeLower.contains("approval") || bridgeLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
// Avoid duplicating the primary bridge status ("Connecting…") in the activity slot.
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText = self.appModel.cameraHUDText,
let cameraHUDKind = self.appModel.cameraHUDKind,
!cameraHUDText.isEmpty
{
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
}
}
+5 -71
View File
@@ -19,18 +19,12 @@ final class ScreenController {
/// Callback invoked when the user clicks an A2UI action (e.g. button) inside the canvas web UI.
var onA2UIAction: (([String: Any]) -> Void)?
private var debugStatusEnabled: Bool = false
private var debugStatusTitle: String?
private var debugStatusSubtitle: String?
init() {
let config = WKWebViewConfiguration()
config.websiteDataStore = .nonPersistent()
let a2uiActionHandler = CanvasA2UIActionMessageHandler()
let userContentController = WKUserContentController()
for name in CanvasA2UIActionMessageHandler.handlerNames {
userContentController.add(a2uiActionHandler, name: name)
}
userContentController.add(a2uiActionHandler, name: CanvasA2UIActionMessageHandler.messageName)
config.userContentController = userContentController
self.navigationDelegate = ScreenNavigationDelegate()
self.a2uiActionHandler = a2uiActionHandler
@@ -43,7 +37,9 @@ final class ScreenController {
self.webView.scrollView.contentInset = .zero
self.webView.scrollView.scrollIndicatorInsets = .zero
self.webView.scrollView.automaticallyAdjustsScrollIndicatorInsets = false
self.applyScrollBehavior()
// Disable scroll to allow touch events to pass through to canvas
self.webView.scrollView.isScrollEnabled = false
self.webView.scrollView.bounces = false
self.webView.navigationDelegate = self.navigationDelegate
self.navigationDelegate.controller = self
a2uiActionHandler.controller = self
@@ -58,7 +54,6 @@ final class ScreenController {
func reload() {
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
self.applyScrollBehavior()
if trimmed.isEmpty {
guard let url = Self.canvasScaffoldURL else { return }
self.errorText = nil
@@ -83,39 +78,6 @@ final class ScreenController {
self.reload()
}
func setDebugStatusEnabled(_ enabled: Bool) {
self.debugStatusEnabled = enabled
self.applyDebugStatusIfNeeded()
}
func updateDebugStatus(title: String?, subtitle: String?) {
self.debugStatusTitle = title
self.debugStatusSubtitle = subtitle
self.applyDebugStatusIfNeeded()
}
fileprivate func applyDebugStatusIfNeeded() {
let enabled = self.debugStatusEnabled
let title = self.debugStatusTitle
let subtitle = self.debugStatusSubtitle
let js = """
(() => {
try {
const api = globalThis.__clawdis;
if (!api) return;
if (typeof api.setDebugStatusEnabled === 'function') {
api.setDebugStatusEnabled(\(enabled ? "true" : "false"));
}
if (!\(enabled ? "true" : "false")) return;
if (typeof api.setStatus === 'function') {
api.setStatus(\(Self.jsValue(title)), \(Self.jsValue(subtitle)));
}
} catch (_) {}
})()
"""
self.webView.evaluateJavaScript(js) { _, _ in }
}
func waitForA2UIReady(timeoutMs: Int) async -> Bool {
let clock = ContinuousClock()
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))
@@ -128,8 +90,7 @@ final class ScreenController {
} catch (_) { return false; }
})()
""")
let trimmed = res.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
if trimmed == "true" || trimmed == "1" { return true }
if res == "true" { return true }
} catch {
// ignore; page likely still loading
}
@@ -249,26 +210,6 @@ final class ScreenController {
return false
}
private func applyScrollBehavior() {
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
let allowScroll = !trimmed.isEmpty
let scrollView = self.webView.scrollView
// Default canvas needs raw touch events; external pages should scroll.
scrollView.isScrollEnabled = allowScroll
scrollView.bounces = allowScroll
}
private static func jsValue(_ value: String?) -> String {
guard let value else { return "null" }
if let data = try? JSONSerialization.data(withJSONObject: [value]),
let encoded = String(data: data, encoding: .utf8),
encoded.count >= 2
{
return String(encoded.dropFirst().dropLast())
}
return "null"
}
func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return false
@@ -375,11 +316,6 @@ private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
self.controller?.errorText = error.localizedDescription
}
func webView(_: WKWebView, didFinish _: WKNavigation?) {
self.controller?.errorText = nil
self.controller?.applyDebugStatusIfNeeded()
}
func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) {
self.controller?.errorText = error.localizedDescription
}
@@ -387,8 +323,6 @@ private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "clawdisCanvasA2UIAction"
static let legacyMessageNames = ["canvas", "a2ui", "userAction", "action"]
static let handlerNames = [messageName] + legacyMessageNames
weak var controller: ScreenController?
+105 -181
View File
@@ -1,28 +1,12 @@
import AVFoundation
import ReplayKit
final class ScreenRecordService: @unchecked Sendable {
@MainActor
final class ScreenRecordService {
private struct UncheckedSendableBox<T>: @unchecked Sendable {
let value: T
}
private final class CaptureState: @unchecked Sendable {
private let lock = NSLock()
var writer: AVAssetWriter?
var videoInput: AVAssetWriterInput?
var audioInput: AVAssetWriterInput?
var started = false
var sawVideo = false
var lastVideoTime: CMTime?
var handlerError: Error?
func withLock<T>(_ body: (CaptureState) -> T) -> T {
self.lock.lock()
defer { lock.unlock() }
return body(self)
}
}
enum ScreenRecordError: LocalizedError {
case invalidScreenIndex(Int)
case captureFailed(String)
@@ -67,158 +51,126 @@ final class ScreenRecordService: @unchecked Sendable {
}()
try? FileManager.default.removeItem(at: outURL)
let state = CaptureState()
let recordQueue = DispatchQueue(label: "com.steipete.clawdis.screenrecord")
let recorder = RPScreenRecorder.shared()
recorder.isMicrophoneEnabled = includeAudio
var writer: AVAssetWriter?
var videoInput: AVAssetWriterInput?
var audioInput: AVAssetWriterInput?
var started = false
var sawVideo = false
var lastVideoTime: CMTime?
var handlerError: Error?
let lock = NSLock()
func setHandlerError(_ error: Error) {
lock.lock()
defer { lock.unlock() }
if handlerError == nil { handlerError = error }
}
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
let handler: @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void = { sample, type, error in
// ReplayKit can call the capture handler on a background queue.
// Serialize writes to avoid queue asserts.
recordQueue.async {
if let error {
state.withLock { state in
if state.handlerError == nil { state.handlerError = error }
}
return
}
guard CMSampleBufferDataIsReady(sample) else { return }
switch type {
case .video:
let pts = CMSampleBufferGetPresentationTimeStamp(sample)
let shouldSkip = state.withLock { state in
if let lastVideoTime = state.lastVideoTime {
let delta = CMTimeSubtract(pts, lastVideoTime)
return delta.seconds < (1.0 / fpsValue)
}
return false
}
if shouldSkip { return }
if state.withLock({ $0.writer == nil }) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
state.withLock { state in
if state.handlerError == nil {
state.handlerError = ScreenRecordError.captureFailed("Missing image buffer")
}
}
return
}
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
do {
let w = try AVAssetWriter(outputURL: outURL, fileType: .mp4)
let settings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: width,
AVVideoHeightKey: height,
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
vInput.expectsMediaDataInRealTime = true
guard w.canAdd(vInput) else {
throw ScreenRecordError.writeFailed("Cannot add video input")
}
w.add(vInput)
if includeAudio {
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
aInput.expectsMediaDataInRealTime = true
if w.canAdd(aInput) {
w.add(aInput)
state.withLock { state in
state.audioInput = aInput
}
}
}
guard w.startWriting() else {
throw ScreenRecordError
.writeFailed(w.error?.localizedDescription ?? "Failed to start writer")
}
w.startSession(atSourceTime: pts)
state.withLock { state in
state.writer = w
state.videoInput = vInput
state.started = true
}
} catch {
state.withLock { state in
if state.handlerError == nil { state.handlerError = error }
}
return
}
}
let vInput = state.withLock { $0.videoInput }
let isStarted = state.withLock { $0.started }
guard let vInput, isStarted else { return }
if vInput.isReadyForMoreMediaData {
if vInput.append(sample) {
state.withLock { state in
state.sawVideo = true
state.lastVideoTime = pts
}
} else {
let err = state.withLock { $0.writer?.error }
if let err {
state.withLock { state in
if state.handlerError == nil {
state.handlerError = ScreenRecordError.writeFailed(err.localizedDescription)
}
}
}
}
}
case .audioApp, .audioMic:
let aInput = state.withLock { $0.audioInput }
let isStarted = state.withLock { $0.started }
guard includeAudio, let aInput, isStarted else { return }
if aInput.isReadyForMoreMediaData {
_ = aInput.append(sample)
}
@unknown default:
break
}
recorder.startCapture(handler: { sample, type, error in
if let error {
setHandlerError(error)
return
}
}
guard CMSampleBufferDataIsReady(sample) else { return }
let completion: @Sendable (Error?) -> Void = { error in
switch type {
case .video:
let pts = CMSampleBufferGetPresentationTimeStamp(sample)
if let lastVideoTime {
let delta = CMTimeSubtract(pts, lastVideoTime)
if delta.seconds < (1.0 / fpsValue) { return }
}
if writer == nil {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
setHandlerError(ScreenRecordError.captureFailed("Missing image buffer"))
return
}
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
do {
let w = try AVAssetWriter(outputURL: outURL, fileType: .mp4)
let settings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: width,
AVVideoHeightKey: height,
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
vInput.expectsMediaDataInRealTime = true
guard w.canAdd(vInput) else {
throw ScreenRecordError.writeFailed("Cannot add video input")
}
w.add(vInput)
if includeAudio {
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
aInput.expectsMediaDataInRealTime = true
if w.canAdd(aInput) {
w.add(aInput)
audioInput = aInput
}
}
guard w.startWriting() else {
throw ScreenRecordError
.writeFailed(w.error?.localizedDescription ?? "Failed to start writer")
}
w.startSession(atSourceTime: pts)
writer = w
videoInput = vInput
started = true
} catch {
setHandlerError(error)
return
}
}
guard let vInput = videoInput, started else { return }
if vInput.isReadyForMoreMediaData {
if vInput.append(sample) {
sawVideo = true
lastVideoTime = pts
} else {
if let err = writer?.error {
setHandlerError(ScreenRecordError.writeFailed(err.localizedDescription))
}
}
}
case .audioApp, .audioMic:
guard includeAudio, let aInput = audioInput, started else { return }
if aInput.isReadyForMoreMediaData {
_ = aInput.append(sample)
}
@unknown default:
break
}
}, completionHandler: { error in
if let error { cont.resume(throwing: error) } else { cont.resume() }
}
Task { @MainActor in
startReplayKitCapture(
includeAudio: includeAudio,
handler: handler,
completion: completion)
}
})
}
try await Task.sleep(nanoseconds: UInt64(durationMs) * 1_000_000)
let stopError = await withCheckedContinuation { cont in
Task { @MainActor in
stopReplayKitCapture { error in cont.resume(returning: error) }
}
recorder.stopCapture { error in cont.resume(returning: error) }
}
if let stopError { throw stopError }
let handlerErrorSnapshot = state.withLock { $0.handlerError }
if let handlerErrorSnapshot { throw handlerErrorSnapshot }
let writerSnapshot = state.withLock { $0.writer }
let videoInputSnapshot = state.withLock { $0.videoInput }
let audioInputSnapshot = state.withLock { $0.audioInput }
let sawVideoSnapshot = state.withLock { $0.sawVideo }
guard let writerSnapshot, let videoInputSnapshot, sawVideoSnapshot else {
if let handlerError { throw handlerError }
guard let writer, let videoInput, sawVideo else {
throw ScreenRecordError.captureFailed("No frames captured")
}
videoInputSnapshot.markAsFinished()
audioInputSnapshot?.markAsFinished()
videoInput.markAsFinished()
audioInput?.markAsFinished()
let writerBox = UncheckedSendableBox(value: writerSnapshot)
let writerBox = UncheckedSendableBox(value: writer)
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
writerBox.value.finishWriting {
let writer = writerBox.value
@@ -246,31 +198,3 @@ final class ScreenRecordService: @unchecked Sendable {
return min(30, max(1, v))
}
}
@MainActor
private func startReplayKitCapture(
includeAudio: Bool,
handler: @escaping @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void,
completion: @escaping @Sendable (Error?) -> Void)
{
let recorder = RPScreenRecorder.shared()
recorder.isMicrophoneEnabled = includeAudio
recorder.startCapture(handler: handler, completionHandler: completion)
}
@MainActor
private func stopReplayKitCapture(_ completion: @escaping @Sendable (Error?) -> Void) {
RPScreenRecorder.shared().stopCapture { error in completion(error) }
}
#if DEBUG
extension ScreenRecordService {
nonisolated static func _test_clampDurationMs(_ ms: Int?) -> Int {
self.clampDurationMs(ms)
}
nonisolated static func _test_clampFps(_ fps: Double?) -> Double {
self.clampFps(fps)
}
}
#endif
@@ -20,8 +20,6 @@ struct SettingsTab: View {
@AppStorage("node.displayName") private var displayName: String = "iOS Node"
@AppStorage("node.instanceId") private var instanceId: String = UUID().uuidString
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
@AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true
@AppStorage("camera.enabled") private var cameraEnabled: Bool = true
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true
@AppStorage("bridge.preferredStableID") private var preferredBridgeStableID: String = ""
@@ -30,7 +28,6 @@ struct SettingsTab: View {
@AppStorage("bridge.manual.host") private var manualBridgeHost: String = ""
@AppStorage("bridge.manual.port") private var manualBridgePort: Int = 18790
@AppStorage("bridge.discovery.debugLogs") private var discoveryDebugLogsEnabled: Bool = false
@AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false
@State private var connectStatus = ConnectStatusStore()
@State private var connectingBridgeID: String?
@State private var localIPAddress: String?
@@ -53,9 +50,6 @@ struct SettingsTab: View {
}
}
}
LabeledContent("Platform", value: self.platformString())
LabeledContent("Version", value: self.appVersion())
LabeledContent("Model", value: self.modelIdentifier())
}
Section("Bridge") {
@@ -148,8 +142,6 @@ struct SettingsTab: View {
NavigationLink("Discovery Logs") {
BridgeDiscoveryDebugLogView()
}
Toggle("Debug Canvas Status", isOn: self.$canvasDebugStatusEnabled)
}
}
@@ -158,12 +150,6 @@ struct SettingsTab: View {
.onChange(of: self.voiceWakeEnabled) { _, newValue in
self.appModel.setVoiceWakeEnabled(newValue)
}
Toggle("Talk Mode", isOn: self.$talkEnabled)
.onChange(of: self.talkEnabled) { _, newValue in
self.appModel.setTalkEnabled(newValue)
}
// Keep this separate so users can hide the side bubble without disabling Talk Mode.
Toggle("Show Talk Button", isOn: self.$talkButtonEnabled)
NavigationLink {
VoiceWakeWordsSettingsView()
@@ -238,12 +224,6 @@ struct SettingsTab: View {
HStack {
VStack(alignment: .leading, spacing: 2) {
Text(bridge.name)
let detailLines = self.bridgeDetailLines(bridge)
ForEach(detailLines, id: \.self) { line in
Text(line)
.font(.footnote)
.foregroundStyle(.secondary)
}
}
Spacer()
@@ -521,26 +501,4 @@ struct SettingsTab: View {
private static func httpURLString(host: String?, port: Int?, fallback: String) -> String {
SettingsNetworkingHelpers.httpURLString(host: host, port: port, fallback: fallback)
}
private func bridgeDetailLines(_ bridge: BridgeDiscoveryModel.DiscoveredBridge) -> [String] {
var lines: [String] = []
if let lanHost = bridge.lanHost { lines.append("LAN: \(lanHost)") }
if let tailnet = bridge.tailnetDns { lines.append("Tailnet: \(tailnet)") }
let gatewayPort = bridge.gatewayPort
let bridgePort = bridge.bridgePort
let canvasPort = bridge.canvasPort
if gatewayPort != nil || bridgePort != nil || canvasPort != nil {
let gw = gatewayPort.map(String.init) ?? "—"
let br = bridgePort.map(String.init) ?? "—"
let canvas = canvasPort.map(String.init) ?? "—"
lines.append("Ports: gw \(gw) · bridge \(br) · canvas \(canvas)")
}
if lines.isEmpty {
lines.append(bridge.debugID)
}
return lines
}
}
+9 -44
View File
@@ -1,8 +1,6 @@
import SwiftUI
struct StatusPill: View {
@Environment(\.scenePhase) private var scenePhase
enum BridgeState: Equatable {
case connected
case connecting
@@ -28,15 +26,8 @@ struct StatusPill: View {
}
}
struct Activity: Equatable {
var title: String
var systemImage: String
var tint: Color?
}
var bridge: BridgeState
var voiceWakeEnabled: Bool
var activity: Activity?
var brighten: Bool = false
var onTap: () -> Void
@@ -61,24 +52,10 @@ struct StatusPill: View {
.frame(height: 14)
.opacity(0.35)
if let activity {
HStack(spacing: 6) {
Image(systemName: activity.systemImage)
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(activity.tint ?? .primary)
Text(activity.title)
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(.primary)
.lineLimit(1)
}
.transition(.opacity.combined(with: .move(edge: .top)))
} else {
Image(systemName: self.voiceWakeEnabled ? "mic.fill" : "mic.slash")
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(self.voiceWakeEnabled ? .primary : .secondary)
.accessibilityLabel(self.voiceWakeEnabled ? "Voice Wake enabled" : "Voice Wake disabled")
.transition(.opacity.combined(with: .move(edge: .top)))
}
Image(systemName: self.voiceWakeEnabled ? "mic.fill" : "mic.slash")
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(self.voiceWakeEnabled ? .primary : .secondary)
.accessibilityLabel(self.voiceWakeEnabled ? "Voice Wake enabled" : "Voice Wake disabled")
}
.padding(.vertical, 8)
.padding(.horizontal, 12)
@@ -94,27 +71,15 @@ struct StatusPill: View {
}
.buttonStyle(.plain)
.accessibilityLabel("Status")
.accessibilityValue(self.accessibilityValue)
.onAppear { self.updatePulse(for: self.bridge, scenePhase: self.scenePhase) }
.onDisappear { self.pulse = false }
.accessibilityValue("\(self.bridge.title), Voice Wake \(self.voiceWakeEnabled ? "enabled" : "disabled")")
.onAppear { self.updatePulse(for: self.bridge) }
.onChange(of: self.bridge) { _, newValue in
self.updatePulse(for: newValue, scenePhase: self.scenePhase)
self.updatePulse(for: newValue)
}
.onChange(of: self.scenePhase) { _, newValue in
self.updatePulse(for: self.bridge, scenePhase: newValue)
}
.animation(.easeInOut(duration: 0.18), value: self.activity?.title)
}
private var accessibilityValue: String {
if let activity {
return "\(self.bridge.title), \(activity.title)"
}
return "\(self.bridge.title), Voice Wake \(self.voiceWakeEnabled ? "enabled" : "disabled")"
}
private func updatePulse(for bridge: BridgeState, scenePhase: ScenePhase) {
guard bridge == .connecting, scenePhase == .active else {
private func updatePulse(for bridge: BridgeState) {
guard bridge == .connecting else {
withAnimation(.easeOut(duration: 0.2)) { self.pulse = false }
return
}
@@ -1,713 +0,0 @@
import AVFAudio
import ClawdisKit
import Foundation
import Observation
import OSLog
import Speech
@MainActor
@Observable
final class TalkModeManager: NSObject {
private typealias SpeechRequest = SFSpeechAudioBufferRecognitionRequest
private static let defaultModelIdFallback = "eleven_v3"
var isEnabled: Bool = false
var isListening: Bool = false
var isSpeaking: Bool = false
var statusText: String = "Off"
private let audioEngine = AVAudioEngine()
private var speechRecognizer: SFSpeechRecognizer?
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
private var recognitionTask: SFSpeechRecognitionTask?
private var silenceTask: Task<Void, Never>?
private var lastHeard: Date?
private var lastTranscript: String = ""
private var lastSpokenText: String?
private var lastInterruptedAtSeconds: Double?
private var defaultVoiceId: String?
private var currentVoiceId: String?
private var defaultModelId: String?
private var currentModelId: String?
private var voiceOverrideActive = false
private var modelOverrideActive = false
private var defaultOutputFormat: String?
private var apiKey: String?
private var voiceAliases: [String: String] = [:]
private var interruptOnSpeech: Bool = true
private var mainSessionKey: String = "main"
private var fallbackVoiceId: String?
private var lastPlaybackWasPCM: Bool = false
var pcmPlayer: PCMStreamingAudioPlaying = PCMStreamingAudioPlayer.shared
var mp3Player: StreamingAudioPlaying = StreamingAudioPlayer.shared
private var bridge: BridgeSession?
private let silenceWindow: TimeInterval = 0.7
private var chatSubscribedSessionKeys = Set<String>()
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "TalkMode")
func attachBridge(_ bridge: BridgeSession) {
self.bridge = bridge
}
func setEnabled(_ enabled: Bool) {
self.isEnabled = enabled
if enabled {
self.logger.info("enabled")
Task { await self.start() }
} else {
self.logger.info("disabled")
self.stop()
}
}
func start() async {
guard self.isEnabled else { return }
if self.isListening { return }
self.logger.info("start")
self.statusText = "Requesting permissions…"
let micOk = await Self.requestMicrophonePermission()
guard micOk else {
self.logger.warning("start blocked: microphone permission denied")
self.statusText = "Microphone permission denied"
return
}
let speechOk = await Self.requestSpeechPermission()
guard speechOk else {
self.logger.warning("start blocked: speech permission denied")
self.statusText = "Speech recognition permission denied"
return
}
await self.reloadConfig()
do {
try Self.configureAudioSession()
try self.startRecognition()
self.isListening = true
self.statusText = "Listening"
self.startSilenceMonitor()
await self.subscribeChatIfNeeded(sessionKey: self.mainSessionKey)
self.logger.info("listening")
} catch {
self.isListening = false
self.statusText = "Start failed: \(error.localizedDescription)"
self.logger.error("start failed: \(error.localizedDescription, privacy: .public)")
}
}
func stop() {
self.isEnabled = false
self.isListening = false
self.statusText = "Off"
self.lastTranscript = ""
self.lastHeard = nil
self.silenceTask?.cancel()
self.silenceTask = nil
self.stopRecognition()
self.stopSpeaking()
self.lastInterruptedAtSeconds = nil
TalkSystemSpeechSynthesizer.shared.stop()
do {
try AVAudioSession.sharedInstance().setActive(false, options: [.notifyOthersOnDeactivation])
} catch {
self.logger.warning("audio session deactivate failed: \(error.localizedDescription, privacy: .public)")
}
Task { await self.unsubscribeAllChats() }
}
func userTappedOrb() {
self.stopSpeaking()
}
private func startRecognition() throws {
self.stopRecognition()
self.speechRecognizer = SFSpeechRecognizer()
guard let recognizer = self.speechRecognizer else {
throw NSError(domain: "TalkMode", code: 1, userInfo: [
NSLocalizedDescriptionKey: "Speech recognizer unavailable",
])
}
self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest()
self.recognitionRequest?.shouldReportPartialResults = true
guard let request = self.recognitionRequest else { return }
let input = self.audioEngine.inputNode
let format = input.outputFormat(forBus: 0)
input.removeTap(onBus: 0)
let tapBlock = Self.makeAudioTapAppendCallback(request: request)
input.installTap(onBus: 0, bufferSize: 2048, format: format, block: tapBlock)
self.audioEngine.prepare()
try self.audioEngine.start()
self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, error in
guard let self else { return }
if let error {
if !self.isSpeaking {
self.statusText = "Speech error: \(error.localizedDescription)"
}
self.logger.debug("speech recognition error: \(error.localizedDescription, privacy: .public)")
}
guard let result else { return }
let transcript = result.bestTranscription.formattedString
Task { @MainActor in
await self.handleTranscript(transcript: transcript, isFinal: result.isFinal)
}
}
}
private func stopRecognition() {
self.recognitionTask?.cancel()
self.recognitionTask = nil
self.recognitionRequest?.endAudio()
self.recognitionRequest = nil
self.audioEngine.inputNode.removeTap(onBus: 0)
self.audioEngine.stop()
self.speechRecognizer = nil
}
private nonisolated static func makeAudioTapAppendCallback(request: SpeechRequest) -> AVAudioNodeTapBlock {
{ buffer, _ in
request.append(buffer)
}
}
private func handleTranscript(transcript: String, isFinal: Bool) async {
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
if self.isSpeaking, self.interruptOnSpeech {
if self.shouldInterrupt(with: trimmed) {
self.stopSpeaking()
}
return
}
guard self.isListening else { return }
if !trimmed.isEmpty {
self.lastTranscript = trimmed
self.lastHeard = Date()
}
if isFinal {
self.lastTranscript = trimmed
}
}
private func startSilenceMonitor() {
self.silenceTask?.cancel()
self.silenceTask = Task { [weak self] in
guard let self else { return }
while self.isEnabled {
try? await Task.sleep(nanoseconds: 200_000_000)
await self.checkSilence()
}
}
}
private func checkSilence() async {
guard self.isListening, !self.isSpeaking else { return }
let transcript = self.lastTranscript.trimmingCharacters(in: .whitespacesAndNewlines)
guard !transcript.isEmpty else { return }
guard let lastHeard else { return }
if Date().timeIntervalSince(lastHeard) < self.silenceWindow { return }
await self.finalizeTranscript(transcript)
}
private func finalizeTranscript(_ transcript: String) async {
self.isListening = false
self.statusText = "Thinking…"
self.lastTranscript = ""
self.lastHeard = nil
self.stopRecognition()
await self.reloadConfig()
let prompt = self.buildPrompt(transcript: transcript)
guard let bridge else {
self.statusText = "Bridge not connected"
self.logger.warning("finalize: bridge not connected")
await self.start()
return
}
do {
let startedAt = Date().timeIntervalSince1970
let sessionKey = self.mainSessionKey
await self.subscribeChatIfNeeded(sessionKey: sessionKey)
self.logger.info(
"chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)")
let runId = try await self.sendChat(prompt, bridge: bridge)
self.logger.info("chat.send ok runId=\(runId, privacy: .public)")
let completion = await self.waitForChatCompletion(runId: runId, bridge: bridge, timeoutSeconds: 120)
if completion == .timeout {
self.logger.warning(
"chat completion timeout runId=\(runId, privacy: .public); attempting history fallback")
} else if completion == .aborted {
self.statusText = "Aborted"
self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)")
await self.start()
return
} else if completion == .error {
self.statusText = "Chat error"
self.logger.warning("chat completion error runId=\(runId, privacy: .public)")
await self.start()
return
}
guard let assistantText = try await self.waitForAssistantText(
bridge: bridge,
since: startedAt,
timeoutSeconds: completion == .final ? 12 : 25)
else {
self.statusText = "No reply"
self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)")
await self.start()
return
}
self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)")
await self.playAssistant(text: assistantText)
} catch {
self.statusText = "Talk failed: \(error.localizedDescription)"
self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)")
}
await self.start()
}
private func subscribeChatIfNeeded(sessionKey: String) async {
let key = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
guard !key.isEmpty else { return }
guard let bridge else { return }
guard !self.chatSubscribedSessionKeys.contains(key) else { return }
do {
let payload = "{\"sessionKey\":\"\(key)\"}"
try await bridge.sendEvent(event: "chat.subscribe", payloadJSON: payload)
self.chatSubscribedSessionKeys.insert(key)
self.logger.info("chat.subscribe ok sessionKey=\(key, privacy: .public)")
} catch {
self.logger
.warning(
"chat.subscribe failed sessionKey=\(key, privacy: .public) err=\(error.localizedDescription, privacy: .public)")
}
}
private func unsubscribeAllChats() async {
guard let bridge else { return }
let keys = self.chatSubscribedSessionKeys
self.chatSubscribedSessionKeys.removeAll()
for key in keys {
do {
let payload = "{\"sessionKey\":\"\(key)\"}"
try await bridge.sendEvent(event: "chat.unsubscribe", payloadJSON: payload)
} catch {
// ignore
}
}
}
private func buildPrompt(transcript: String) -> String {
let interrupted = self.lastInterruptedAtSeconds
self.lastInterruptedAtSeconds = nil
return TalkPromptBuilder.build(transcript: transcript, interruptedAtSeconds: interrupted)
}
private enum ChatCompletionState: CustomStringConvertible {
case final
case aborted
case error
case timeout
var description: String {
switch self {
case .final: "final"
case .aborted: "aborted"
case .error: "error"
case .timeout: "timeout"
}
}
}
private func sendChat(_ message: String, bridge: BridgeSession) async throws -> String {
struct SendResponse: Decodable { let runId: String }
let payload: [String: Any] = [
"sessionKey": self.mainSessionKey,
"message": message,
"thinking": "low",
"timeoutMs": 30000,
"idempotencyKey": UUID().uuidString,
]
let data = try JSONSerialization.data(withJSONObject: payload)
let json = String(decoding: data, as: UTF8.self)
let res = try await bridge.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 30)
let decoded = try JSONDecoder().decode(SendResponse.self, from: res)
return decoded.runId
}
private func waitForChatCompletion(
runId: String,
bridge: BridgeSession,
timeoutSeconds: Int = 120) async -> ChatCompletionState
{
let stream = await bridge.subscribeServerEvents(bufferingNewest: 200)
return await withTaskGroup(of: ChatCompletionState.self) { group in
group.addTask { [runId] in
for await evt in stream {
if Task.isCancelled { return .timeout }
guard evt.event == "chat", let payload = evt.payloadJSON else { continue }
guard let data = payload.data(using: .utf8) else { continue }
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { continue }
if (json["runId"] as? String) != runId { continue }
if let state = json["state"] as? String {
switch state {
case "final": return .final
case "aborted": return .aborted
case "error": return .error
default: break
}
}
}
return .timeout
}
group.addTask {
try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds) * 1_000_000_000)
return .timeout
}
let result = await group.next() ?? .timeout
group.cancelAll()
return result
}
}
private func waitForAssistantText(
bridge: BridgeSession,
since: Double,
timeoutSeconds: Int) async throws -> String?
{
let deadline = Date().addingTimeInterval(TimeInterval(timeoutSeconds))
while Date() < deadline {
if let text = try await self.fetchLatestAssistantText(bridge: bridge, since: since) {
return text
}
try? await Task.sleep(nanoseconds: 300_000_000)
}
return nil
}
private func fetchLatestAssistantText(bridge: BridgeSession, since: Double? = nil) async throws -> String? {
let res = try await bridge.request(
method: "chat.history",
paramsJSON: "{\"sessionKey\":\"\(self.mainSessionKey)\"}",
timeoutSeconds: 15)
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return nil }
guard let messages = json["messages"] as? [[String: Any]] else { return nil }
for msg in messages.reversed() {
guard (msg["role"] as? String) == "assistant" else { continue }
if let since, let timestamp = msg["timestamp"] as? Double,
TalkHistoryTimestamp.isAfter(timestamp, sinceSeconds: since) == false
{
continue
}
guard let content = msg["content"] as? [[String: Any]] else { continue }
let text = content.compactMap { $0["text"] as? String }.joined(separator: "\n")
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty { return trimmed }
}
return nil
}
private func playAssistant(text: String) async {
let parsed = TalkDirectiveParser.parse(text)
let directive = parsed.directive
let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines)
guard !cleaned.isEmpty else { return }
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
}
if let voice = resolvedVoice {
if directive?.once != true {
self.currentVoiceId = voice
self.voiceOverrideActive = true
}
}
if let model = directive?.modelId {
if directive?.once != true {
self.currentModelId = model
self.modelOverrideActive = true
}
}
self.statusText = "Generating voice…"
self.isSpeaking = true
self.lastSpokenText = cleaned
do {
let started = Date()
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
let resolvedKey =
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"]
let apiKey = resolvedKey?.trimmingCharacters(in: .whitespacesAndNewlines)
let preferredVoice = resolvedVoice ?? self.currentVoiceId ?? self.defaultVoiceId
let voiceId: String? = if let apiKey, !apiKey.isEmpty {
await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey)
} else {
nil
}
let canUseElevenLabs = (voiceId?.isEmpty == false) && (apiKey?.isEmpty == false)
if canUseElevenLabs, let voiceId, let apiKey {
let desiredOutputFormat = (directive?.outputFormat ?? self.defaultOutputFormat)?
.trimmingCharacters(in: .whitespacesAndNewlines)
let requestedOutputFormat = (desiredOutputFormat?.isEmpty == false) ? desiredOutputFormat : nil
let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(requestedOutputFormat ?? "pcm_44100")
if outputFormat == nil, let requestedOutputFormat {
self.logger.warning(
"talk output_format unsupported for local playback: \(requestedOutputFormat, privacy: .public)")
}
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest {
ElevenLabsTTSRequest(
text: cleaned,
modelId: modelId,
outputFormat: outputFormat,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
style: TalkTTSValidation.validatedUnit(directive?.style),
speakerBoost: directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
}
let request = makeRequest(outputFormat: outputFormat)
let client = ElevenLabsTTSClient(apiKey: apiKey)
let stream = client.streamSynthesize(voiceId: voiceId, request: request)
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.statusText = "Speaking…"
let sampleRate = TalkTTSValidation.pcmSampleRate(from: outputFormat)
let result: StreamingPlaybackResult
if let sampleRate {
self.lastPlaybackWasPCM = true
var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
if !playback.finished, playback.interruptedAt == nil {
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
self.logger.warning("pcm playback failed; retrying mp3")
self.lastPlaybackWasPCM = false
let mp3Stream = client.streamSynthesize(
voiceId: voiceId,
request: makeRequest(outputFormat: mp3Format))
playback = await self.mp3Player.play(stream: mp3Stream)
}
result = playback
} else {
self.lastPlaybackWasPCM = false
result = await self.mp3Player.play(stream: stream)
}
self.logger
.info(
"elevenlabs stream finished=\(result.finished, privacy: .public) dur=\(Date().timeIntervalSince(started), privacy: .public)s")
if !result.finished, let interruptedAt = result.interruptedAt {
self.lastInterruptedAtSeconds = interruptedAt
}
} else {
self.logger.warning("tts unavailable; falling back to system voice (missing key or voiceId)")
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.statusText = "Speaking (System)…"
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
}
} catch {
self.logger.error(
"tts failed: \(error.localizedDescription, privacy: .public); falling back to system voice")
do {
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.statusText = "Speaking (System)…"
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
} catch {
self.statusText = "Speak failed: \(error.localizedDescription)"
self.logger.error("system voice failed: \(error.localizedDescription, privacy: .public)")
}
}
self.stopRecognition()
self.isSpeaking = false
}
private func stopSpeaking(storeInterruption: Bool = true) {
guard self.isSpeaking else { return }
let interruptedAt = self.lastPlaybackWasPCM
? self.pcmPlayer.stop()
: self.mp3Player.stop()
if storeInterruption {
self.lastInterruptedAtSeconds = interruptedAt
}
_ = self.lastPlaybackWasPCM
? self.mp3Player.stop()
: self.pcmPlayer.stop()
TalkSystemSpeechSynthesizer.shared.stop()
self.isSpeaking = false
}
private func shouldInterrupt(with transcript: String) -> Bool {
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
guard trimmed.count >= 3 else { return false }
if let spoken = self.lastSpokenText?.lowercased(), spoken.contains(trimmed.lowercased()) {
return false
}
return true
}
private func resolveVoiceAlias(_ value: String?) -> String? {
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let normalized = trimmed.lowercased()
if let mapped = self.voiceAliases[normalized] { return mapped }
if self.voiceAliases.values.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) {
return trimmed
}
return Self.isLikelyVoiceId(trimmed) ? trimmed : nil
}
private func resolveVoiceId(preferred: String?, apiKey: String) async -> String? {
let trimmed = preferred?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if !trimmed.isEmpty {
if let resolved = self.resolveVoiceAlias(trimmed) { return resolved }
self.logger.warning("unknown voice alias \(trimmed, privacy: .public)")
}
if let fallbackVoiceId { return fallbackVoiceId }
do {
let voices = try await ElevenLabsTTSClient(apiKey: apiKey).listVoices()
guard let first = voices.first else {
self.logger.warning("elevenlabs voices list empty")
return nil
}
self.fallbackVoiceId = first.voiceId
if self.defaultVoiceId == nil {
self.defaultVoiceId = first.voiceId
}
if !self.voiceOverrideActive {
self.currentVoiceId = first.voiceId
}
let name = first.name ?? "unknown"
self.logger
.info("default voice selected \(name, privacy: .public) (\(first.voiceId, privacy: .public))")
return first.voiceId
} catch {
self.logger.error("elevenlabs list voices failed: \(error.localizedDescription, privacy: .public)")
return nil
}
}
private static func isLikelyVoiceId(_ value: String) -> Bool {
guard value.count >= 10 else { return false }
return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "_" }
}
private func reloadConfig() async {
guard let bridge else { return }
do {
let res = try await bridge.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8)
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return }
guard let config = json["config"] as? [String: Any] else { return }
let talk = config["talk"] as? [String: Any]
let session = config["session"] as? [String: Any]
let rawMainKey = (session?["mainKey"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
self.mainSessionKey = rawMainKey.isEmpty ? "main" : rawMainKey
self.defaultVoiceId = (talk?["voiceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
if let aliases = talk?["voiceAliases"] as? [String: Any] {
var resolved: [String: String] = [:]
for (key, value) in aliases {
guard let id = value as? String else { continue }
let normalizedKey = key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
let trimmedId = id.trimmingCharacters(in: .whitespacesAndNewlines)
guard !normalizedKey.isEmpty, !trimmedId.isEmpty else { continue }
resolved[normalizedKey] = trimmedId
}
self.voiceAliases = resolved
} else {
self.voiceAliases = [:]
}
if !self.voiceOverrideActive {
self.currentVoiceId = self.defaultVoiceId
}
let model = (talk?["modelId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
self.defaultModelId = (model?.isEmpty == false) ? model : Self.defaultModelIdFallback
if !self.modelOverrideActive {
self.currentModelId = self.defaultModelId
}
self.defaultOutputFormat = (talk?["outputFormat"] as? String)?
.trimmingCharacters(in: .whitespacesAndNewlines)
self.apiKey = (talk?["apiKey"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
if let interrupt = talk?["interruptOnSpeech"] as? Bool {
self.interruptOnSpeech = interrupt
}
} catch {
self.defaultModelId = Self.defaultModelIdFallback
if !self.modelOverrideActive {
self.currentModelId = self.defaultModelId
}
}
}
private static func configureAudioSession() throws {
let session = AVAudioSession.sharedInstance()
try session.setCategory(.playAndRecord, mode: .voiceChat, options: [
.duckOthers,
.mixWithOthers,
.allowBluetoothHFP,
.defaultToSpeaker,
])
try session.setActive(true, options: [])
}
private nonisolated static func requestMicrophonePermission() async -> Bool {
await withCheckedContinuation(isolation: nil) { cont in
AVAudioApplication.requestRecordPermission { ok in
cont.resume(returning: ok)
}
}
}
private nonisolated static func requestSpeechPermission() async -> Bool {
await withCheckedContinuation(isolation: nil) { cont in
SFSpeechRecognizer.requestAuthorization { status in
cont.resume(returning: status == .authorized)
}
}
}
}
@@ -1,70 +0,0 @@
import SwiftUI
struct TalkOrbOverlay: View {
@Environment(NodeAppModel.self) private var appModel
@State private var pulse: Bool = false
var body: some View {
let seam = self.appModel.seamColor
let status = self.appModel.talkMode.statusText.trimmingCharacters(in: .whitespacesAndNewlines)
VStack(spacing: 14) {
ZStack {
Circle()
.stroke(seam.opacity(0.26), lineWidth: 2)
.frame(width: 320, height: 320)
.scaleEffect(self.pulse ? 1.15 : 0.96)
.opacity(self.pulse ? 0.0 : 1.0)
.animation(.easeOut(duration: 1.3).repeatForever(autoreverses: false), value: self.pulse)
Circle()
.stroke(seam.opacity(0.18), lineWidth: 2)
.frame(width: 320, height: 320)
.scaleEffect(self.pulse ? 1.45 : 1.02)
.opacity(self.pulse ? 0.0 : 0.9)
.animation(.easeOut(duration: 1.9).repeatForever(autoreverses: false).delay(0.2), value: self.pulse)
Circle()
.fill(
RadialGradient(
colors: [
seam.opacity(0.95),
seam.opacity(0.40),
Color.black.opacity(0.55),
],
center: .center,
startRadius: 1,
endRadius: 112))
.frame(width: 190, height: 190)
.overlay(
Circle()
.stroke(seam.opacity(0.35), lineWidth: 1))
.shadow(color: seam.opacity(0.32), radius: 26, x: 0, y: 0)
.shadow(color: Color.black.opacity(0.50), radius: 22, x: 0, y: 10)
}
.contentShape(Circle())
.onTapGesture {
self.appModel.talkMode.userTappedOrb()
}
if !status.isEmpty, status != "Off" {
Text(status)
.font(.system(.footnote, design: .rounded).weight(.semibold))
.foregroundStyle(Color.white.opacity(0.92))
.padding(.horizontal, 12)
.padding(.vertical, 8)
.background(
Capsule()
.fill(Color.black.opacity(0.40))
.overlay(
Capsule().stroke(seam.opacity(0.22), lineWidth: 1)))
}
}
.padding(28)
.onAppear {
self.pulse = true
}
.accessibilityElement(children: .combine)
.accessibilityLabel("Talk Mode \(status)")
}
}
-5
View File
@@ -4,7 +4,6 @@ struct VoiceTab: View {
@Environment(NodeAppModel.self) private var appModel
@Environment(VoiceWakeManager.self) private var voiceWake
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
var body: some View {
NavigationStack {
@@ -15,7 +14,6 @@ struct VoiceTab: View {
Text(self.voiceWake.statusText)
.font(.footnote)
.foregroundStyle(.secondary)
LabeledContent("Talk Mode", value: self.talkEnabled ? "Enabled" : "Disabled")
}
Section("Notes") {
@@ -38,9 +36,6 @@ struct VoiceTab: View {
.onChange(of: self.voiceWakeEnabled) { _, newValue in
self.appModel.setVoiceWakeEnabled(newValue)
}
.onChange(of: self.talkEnabled) { _, newValue in
self.appModel.setTalkEnabled(newValue)
}
}
}
}
+25 -25
View File
@@ -2,7 +2,6 @@ import AVFAudio
import Foundation
import Observation
import Speech
import SwabbleKit
private func makeAudioTapEnqueueCallback(queue: AudioBufferQueue) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
{ buffer, _ in
@@ -290,18 +289,15 @@ final class VoiceWakeManager: NSObject {
private nonisolated func makeRecognitionResultHandler() -> @Sendable (SFSpeechRecognitionResult?, Error?) -> Void {
{ [weak self] result, error in
let transcript = result?.bestTranscription.formattedString
let segments = result.flatMap { result in
transcript.map { WakeWordSpeechSegments.from(transcription: result.bestTranscription, transcript: $0) }
} ?? []
let errorText = error?.localizedDescription
Task { @MainActor in
self?.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
self?.handleRecognitionCallback(transcript: transcript, errorText: errorText)
}
}
}
private func handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
private func handleRecognitionCallback(transcript: String?, errorText: String?) {
if let errorText {
self.statusText = "Recognizer error: \(errorText)"
self.isListening = false
@@ -317,7 +313,7 @@ final class VoiceWakeManager: NSObject {
}
guard let transcript else { return }
guard let cmd = self.extractCommand(from: transcript, segments: segments) else { return }
guard let cmd = self.extractCommand(from: transcript) else { return }
if cmd == self.lastDispatched { return }
self.lastDispatched = cmd
@@ -338,18 +334,30 @@ final class VoiceWakeManager: NSObject {
}
}
private func extractCommand(from transcript: String, segments: [WakeWordSegment]) -> String? {
Self.extractCommand(from: transcript, segments: segments, triggers: self.activeTriggerWords)
private func extractCommand(from transcript: String) -> String? {
Self.extractCommand(from: transcript, triggers: self.activeTriggerWords)
}
nonisolated static func extractCommand(
from transcript: String,
segments: [WakeWordSegment],
triggers: [String],
minPostTriggerGap: TimeInterval = 0.45) -> String?
{
let config = WakeWordGateConfig(triggers: triggers, minPostTriggerGap: minPostTriggerGap)
return WakeWordGate.match(transcript: transcript, segments: segments, config: config)?.command
nonisolated static func extractCommand(from transcript: String, triggers: [String]) -> String? {
var bestRange: Range<String.Index>?
for trigger in triggers {
let token = trigger.trimmingCharacters(in: .whitespacesAndNewlines)
guard !token.isEmpty else { continue }
guard let range = transcript.range(of: token, options: [.caseInsensitive, .backwards]) else { continue }
if let currentBest = bestRange {
if range.lowerBound > currentBest.lowerBound {
bestRange = range
}
} else {
bestRange = range
}
}
guard let bestRange else { return nil }
let after = transcript[bestRange.upperBound...]
let trimmed = after.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
return String(trimmed)
}
private static func configureAudioSession() throws {
@@ -379,11 +387,3 @@ final class VoiceWakeManager: NSObject {
}
}
}
#if DEBUG
extension VoiceWakeManager {
func _test_handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
self.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
}
}
#endif
-60
View File
@@ -1,60 +0,0 @@
Sources/Bridge/BridgeClient.swift
Sources/Bridge/BridgeConnectionController.swift
Sources/Bridge/BridgeDiscoveryDebugLogView.swift
Sources/Bridge/BridgeDiscoveryModel.swift
Sources/Bridge/BridgeEndpointID.swift
Sources/Bridge/BridgeSession.swift
Sources/Bridge/BridgeSettingsStore.swift
Sources/Bridge/KeychainStore.swift
Sources/Camera/CameraController.swift
Sources/Chat/ChatSheet.swift
Sources/Chat/IOSBridgeChatTransport.swift
Sources/ClawdisApp.swift
Sources/Model/NodeAppModel.swift
Sources/RootCanvas.swift
Sources/RootTabs.swift
Sources/Screen/ScreenController.swift
Sources/Screen/ScreenRecordService.swift
Sources/Screen/ScreenTab.swift
Sources/Screen/ScreenWebView.swift
Sources/Settings/SettingsNetworkingHelpers.swift
Sources/Settings/SettingsTab.swift
Sources/Settings/VoiceWakeWordsSettingsView.swift
Sources/Status/StatusPill.swift
Sources/Status/VoiceWakeToast.swift
Sources/Voice/VoiceTab.swift
Sources/Voice/VoiceWakeManager.swift
Sources/Voice/VoiceWakePreferences.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatComposer.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatMarkdownSplitter.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatMessageViews.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatModels.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatPayloadDecoding.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatSessions.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatSheets.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatTheme.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatTransport.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatView.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatViewModel.swift
../shared/ClawdisKit/Sources/ClawdisKit/AnyCodable.swift
../shared/ClawdisKit/Sources/ClawdisKit/BonjourEscapes.swift
../shared/ClawdisKit/Sources/ClawdisKit/BonjourTypes.swift
../shared/ClawdisKit/Sources/ClawdisKit/BridgeFrames.swift
../shared/ClawdisKit/Sources/ClawdisKit/CameraCommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasA2UIAction.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasA2UICommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasA2UIJSONL.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasCommandParams.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasCommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/Capabilities.swift
../shared/ClawdisKit/Sources/ClawdisKit/ClawdisKitResources.swift
../shared/ClawdisKit/Sources/ClawdisKit/DeepLinks.swift
../shared/ClawdisKit/Sources/ClawdisKit/JPEGTranscoder.swift
../shared/ClawdisKit/Sources/ClawdisKit/NodeError.swift
../shared/ClawdisKit/Sources/ClawdisKit/ScreenCommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/StoragePaths.swift
../shared/ClawdisKit/Sources/ClawdisKit/SystemCommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/TalkDirective.swift
../../Swabble/Sources/SwabbleKit/WakeWordGate.swift
Sources/Voice/TalkModeManager.swift
Sources/Voice/TalkOrbOverlay.swift
@@ -1,341 +0,0 @@
import ClawdisKit
import Foundation
import Network
import Testing
import UIKit
@testable import Clawdis
private struct KeychainEntry: Hashable {
let service: String
let account: String
}
private let bridgeService = "com.steipete.clawdis.bridge"
private let nodeService = "com.steipete.clawdis.node"
private let instanceIdEntry = KeychainEntry(service: nodeService, account: "instanceId")
private let preferredBridgeEntry = KeychainEntry(service: bridgeService, account: "preferredStableID")
private let lastBridgeEntry = KeychainEntry(service: bridgeService, account: "lastDiscoveredStableID")
private actor MockBridgePairingClient: BridgePairingClient {
private(set) var lastToken: String?
private let resultToken: String
init(resultToken: String) {
self.resultToken = resultToken
}
func pairAndHello(
endpoint: NWEndpoint,
hello: BridgeHello,
onStatus: (@Sendable (String) -> Void)?) async throws -> String
{
self.lastToken = hello.token
onStatus?("Testing…")
return self.resultToken
}
}
private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T {
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in updates.keys {
snapshot[key] = defaults.object(forKey: key)
}
for (key, value) in updates {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
defer {
for (key, value) in snapshot {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
return try body()
}
@MainActor
private func withUserDefaults<T>(
_ updates: [String: Any?],
_ body: () async throws -> T) async rethrows -> T
{
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in updates.keys {
snapshot[key] = defaults.object(forKey: key)
}
for (key, value) in updates {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
defer {
for (key, value) in snapshot {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
return try await body()
}
private func withKeychainValues<T>(_ updates: [KeychainEntry: String?], _ body: () throws -> T) rethrows -> T {
var snapshot: [KeychainEntry: String?] = [:]
for entry in updates.keys {
snapshot[entry] = KeychainStore.loadString(service: entry.service, account: entry.account)
}
for (entry, value) in updates {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
defer {
for (entry, value) in snapshot {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
}
return try body()
}
@MainActor
private func withKeychainValues<T>(
_ updates: [KeychainEntry: String?],
_ body: () async throws -> T) async rethrows -> T
{
var snapshot: [KeychainEntry: String?] = [:]
for entry in updates.keys {
snapshot[entry] = KeychainStore.loadString(service: entry.service, account: entry.account)
}
for (entry, value) in updates {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
defer {
for (entry, value) in snapshot {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
}
return try await body()
}
@Suite(.serialized) struct BridgeConnectionControllerTests {
@Test @MainActor func resolvedDisplayNameSetsDefaultWhenMissing() {
let defaults = UserDefaults.standard
let displayKey = "node.displayName"
withKeychainValues([instanceIdEntry: nil, preferredBridgeEntry: nil, lastBridgeEntry: nil]) {
withUserDefaults([displayKey: nil, "node.instanceId": "ios-test"]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(appModel: appModel, startDiscovery: false)
let resolved = controller._test_resolvedDisplayName(defaults: defaults)
#expect(!resolved.isEmpty)
#expect(defaults.string(forKey: displayKey) == resolved)
}
}
}
@Test @MainActor func resolvedDisplayNamePreservesCustomValue() {
let defaults = UserDefaults.standard
let displayKey = "node.displayName"
withKeychainValues([instanceIdEntry: nil, preferredBridgeEntry: nil, lastBridgeEntry: nil]) {
withUserDefaults([displayKey: "My iOS Node", "node.instanceId": "ios-test"]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(appModel: appModel, startDiscovery: false)
let resolved = controller._test_resolvedDisplayName(defaults: defaults)
#expect(resolved == "My iOS Node")
#expect(defaults.string(forKey: displayKey) == "My iOS Node")
}
}
}
@Test @MainActor func makeHelloBuildsCapsAndCommands() {
let defaults = UserDefaults.standard
let voiceWakeKey = VoiceWakePreferences.enabledKey
withKeychainValues([instanceIdEntry: nil, preferredBridgeEntry: nil, lastBridgeEntry: nil]) {
withUserDefaults([
"node.instanceId": "ios-test",
"node.displayName": "Test Node",
"camera.enabled": false,
voiceWakeKey: true,
]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(appModel: appModel, startDiscovery: false)
let hello = controller._test_makeHello(token: "token-123")
#expect(hello.nodeId == "ios-test")
#expect(hello.displayName == "Test Node")
#expect(hello.token == "token-123")
let caps = Set(hello.caps ?? [])
#expect(caps.contains(ClawdisCapability.canvas.rawValue))
#expect(caps.contains(ClawdisCapability.screen.rawValue))
#expect(caps.contains(ClawdisCapability.voiceWake.rawValue))
#expect(!caps.contains(ClawdisCapability.camera.rawValue))
let commands = Set(hello.commands ?? [])
#expect(commands.contains(ClawdisCanvasCommand.present.rawValue))
#expect(commands.contains(ClawdisScreenCommand.record.rawValue))
#expect(!commands.contains(ClawdisCameraCommand.snap.rawValue))
#expect(!(hello.platform ?? "").isEmpty)
#expect(!(hello.deviceFamily ?? "").isEmpty)
#expect(!(hello.modelIdentifier ?? "").isEmpty)
#expect(!(hello.version ?? "").isEmpty)
}
}
}
@Test @MainActor func makeHelloIncludesCameraCommandsWhenEnabled() {
withKeychainValues([instanceIdEntry: nil, preferredBridgeEntry: nil, lastBridgeEntry: nil]) {
withUserDefaults([
"node.instanceId": "ios-test",
"node.displayName": "Test Node",
"camera.enabled": true,
VoiceWakePreferences.enabledKey: false,
]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(appModel: appModel, startDiscovery: false)
let hello = controller._test_makeHello(token: "token-456")
let caps = Set(hello.caps ?? [])
#expect(caps.contains(ClawdisCapability.camera.rawValue))
let commands = Set(hello.commands ?? [])
#expect(commands.contains(ClawdisCameraCommand.snap.rawValue))
#expect(commands.contains(ClawdisCameraCommand.clip.rawValue))
}
}
}
@Test @MainActor func autoConnectRefreshesTokenOnUnauthorized() async {
let bridge = BridgeDiscoveryModel.DiscoveredBridge(
name: "Gateway",
endpoint: .hostPort(host: NWEndpoint.Host("127.0.0.1"), port: 18790),
stableID: "bridge-1",
debugID: "bridge-debug",
lanHost: "Mac.local",
tailnetDns: nil,
gatewayPort: 18789,
bridgePort: 18790,
canvasPort: 18793,
cliPath: nil)
let mock = MockBridgePairingClient(resultToken: "new-token")
let account = "bridge-token.ios-test"
await withKeychainValues([
instanceIdEntry: nil,
preferredBridgeEntry: nil,
lastBridgeEntry: nil,
KeychainEntry(service: bridgeService, account: account): "old-token",
]) {
await withUserDefaults([
"node.instanceId": "ios-test",
"bridge.lastDiscoveredStableID": "bridge-1",
"bridge.manual.enabled": false,
]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(
appModel: appModel,
startDiscovery: false,
bridgeClientFactory: { mock })
controller._test_setBridges([bridge])
controller._test_triggerAutoConnect()
for _ in 0..<20 {
if appModel.connectedBridgeID == bridge.stableID { break }
try? await Task.sleep(nanoseconds: 50_000_000)
}
#expect(appModel.connectedBridgeID == bridge.stableID)
let stored = KeychainStore.loadString(service: bridgeService, account: account)
#expect(stored == "new-token")
let lastToken = await mock.lastToken
#expect(lastToken == "old-token")
}
}
}
@Test @MainActor func autoConnectPrefersPreferredBridgeOverLastDiscovered() async {
let bridgeA = BridgeDiscoveryModel.DiscoveredBridge(
name: "Gateway A",
endpoint: .hostPort(host: NWEndpoint.Host("127.0.0.1"), port: 18790),
stableID: "bridge-1",
debugID: "bridge-a",
lanHost: "MacA.local",
tailnetDns: nil,
gatewayPort: 18789,
bridgePort: 18790,
canvasPort: 18793,
cliPath: nil)
let bridgeB = BridgeDiscoveryModel.DiscoveredBridge(
name: "Gateway B",
endpoint: .hostPort(host: NWEndpoint.Host("127.0.0.1"), port: 28790),
stableID: "bridge-2",
debugID: "bridge-b",
lanHost: "MacB.local",
tailnetDns: nil,
gatewayPort: 28789,
bridgePort: 28790,
canvasPort: 28793,
cliPath: nil)
let mock = MockBridgePairingClient(resultToken: "token-ok")
let account = "bridge-token.ios-test"
await withKeychainValues([
instanceIdEntry: nil,
preferredBridgeEntry: nil,
lastBridgeEntry: nil,
KeychainEntry(service: bridgeService, account: account): "old-token",
]) {
await withUserDefaults([
"node.instanceId": "ios-test",
"bridge.preferredStableID": "bridge-2",
"bridge.lastDiscoveredStableID": "bridge-1",
"bridge.manual.enabled": false,
]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(
appModel: appModel,
startDiscovery: false,
bridgeClientFactory: { mock })
controller._test_setBridges([bridgeA, bridgeB])
controller._test_triggerAutoConnect()
for _ in 0..<20 {
if appModel.connectedBridgeID == bridgeB.stableID { break }
try? await Task.sleep(nanoseconds: 50_000_000)
}
#expect(appModel.connectedBridgeID == bridgeB.stableID)
}
}
}
}
@@ -1,22 +0,0 @@
import Testing
@testable import Clawdis
@Suite(.serialized) struct BridgeDiscoveryModelTests {
@Test @MainActor func debugLoggingCapturesLifecycleAndResets() {
let model = BridgeDiscoveryModel()
#expect(model.debugLog.isEmpty)
#expect(model.statusText == "Idle")
model.setDebugLoggingEnabled(true)
#expect(model.debugLog.count >= 2)
model.stop()
#expect(model.statusText == "Stopped")
#expect(model.bridges.isEmpty)
#expect(model.debugLog.count >= 3)
model.setDebugLoggingEnabled(false)
#expect(model.debugLog.isEmpty)
}
}
@@ -1,127 +0,0 @@
import Foundation
import Testing
@testable import Clawdis
private struct KeychainEntry: Hashable {
let service: String
let account: String
}
private let bridgeService = "com.steipete.clawdis.bridge"
private let nodeService = "com.steipete.clawdis.node"
private let instanceIdEntry = KeychainEntry(service: nodeService, account: "instanceId")
private let preferredBridgeEntry = KeychainEntry(service: bridgeService, account: "preferredStableID")
private let lastBridgeEntry = KeychainEntry(service: bridgeService, account: "lastDiscoveredStableID")
private func snapshotDefaults(_ keys: [String]) -> [String: Any?] {
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in keys {
snapshot[key] = defaults.object(forKey: key)
}
return snapshot
}
private func applyDefaults(_ values: [String: Any?]) {
let defaults = UserDefaults.standard
for (key, value) in values {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
private func restoreDefaults(_ snapshot: [String: Any?]) {
applyDefaults(snapshot)
}
private func snapshotKeychain(_ entries: [KeychainEntry]) -> [KeychainEntry: String?] {
var snapshot: [KeychainEntry: String?] = [:]
for entry in entries {
snapshot[entry] = KeychainStore.loadString(service: entry.service, account: entry.account)
}
return snapshot
}
private func applyKeychain(_ values: [KeychainEntry: String?]) {
for (entry, value) in values {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
}
private func restoreKeychain(_ snapshot: [KeychainEntry: String?]) {
applyKeychain(snapshot)
}
@Suite(.serialized) struct BridgeSettingsStoreTests {
@Test func bootstrapCopiesDefaultsToKeychainWhenMissing() {
let defaultsKeys = [
"node.instanceId",
"bridge.preferredStableID",
"bridge.lastDiscoveredStableID",
]
let entries = [instanceIdEntry, preferredBridgeEntry, lastBridgeEntry]
let defaultsSnapshot = snapshotDefaults(defaultsKeys)
let keychainSnapshot = snapshotKeychain(entries)
defer {
restoreDefaults(defaultsSnapshot)
restoreKeychain(keychainSnapshot)
}
applyDefaults([
"node.instanceId": "node-test",
"bridge.preferredStableID": "preferred-test",
"bridge.lastDiscoveredStableID": "last-test",
])
applyKeychain([
instanceIdEntry: nil,
preferredBridgeEntry: nil,
lastBridgeEntry: nil,
])
BridgeSettingsStore.bootstrapPersistence()
#expect(KeychainStore.loadString(service: nodeService, account: "instanceId") == "node-test")
#expect(KeychainStore.loadString(service: bridgeService, account: "preferredStableID") == "preferred-test")
#expect(KeychainStore.loadString(service: bridgeService, account: "lastDiscoveredStableID") == "last-test")
}
@Test func bootstrapCopiesKeychainToDefaultsWhenMissing() {
let defaultsKeys = [
"node.instanceId",
"bridge.preferredStableID",
"bridge.lastDiscoveredStableID",
]
let entries = [instanceIdEntry, preferredBridgeEntry, lastBridgeEntry]
let defaultsSnapshot = snapshotDefaults(defaultsKeys)
let keychainSnapshot = snapshotKeychain(entries)
defer {
restoreDefaults(defaultsSnapshot)
restoreKeychain(keychainSnapshot)
}
applyDefaults([
"node.instanceId": nil,
"bridge.preferredStableID": nil,
"bridge.lastDiscoveredStableID": nil,
])
applyKeychain([
instanceIdEntry: "node-from-keychain",
preferredBridgeEntry: "preferred-from-keychain",
lastBridgeEntry: "last-from-keychain",
])
BridgeSettingsStore.bootstrapPersistence()
let defaults = UserDefaults.standard
#expect(defaults.string(forKey: "node.instanceId") == "node-from-keychain")
#expect(defaults.string(forKey: "bridge.preferredStableID") == "preferred-from-keychain")
#expect(defaults.string(forKey: "bridge.lastDiscoveredStableID") == "last-from-keychain")
}
}
@@ -1,13 +0,0 @@
import Testing
@testable import Clawdis
@Suite struct CameraControllerErrorTests {
@Test func errorDescriptionsAreStable() {
#expect(CameraController.CameraError.cameraUnavailable.errorDescription == "Camera unavailable")
#expect(CameraController.CameraError.microphoneUnavailable.errorDescription == "Microphone unavailable")
#expect(CameraController.CameraError.permissionDenied(kind: "Camera").errorDescription == "Camera permission denied")
#expect(CameraController.CameraError.invalidParams("bad").errorDescription == "bad")
#expect(CameraController.CameraError.captureFailed("nope").errorDescription == "nope")
#expect(CameraController.CameraError.exportFailed("export").errorDescription == "export")
}
}
@@ -1,194 +0,0 @@
import ClawdisKit
import Foundation
import Testing
import UIKit
@testable import Clawdis
private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T {
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in updates.keys {
snapshot[key] = defaults.object(forKey: key)
}
for (key, value) in updates {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
defer {
for (key, value) in snapshot {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
return try body()
}
@Suite(.serialized) struct NodeAppModelInvokeTests {
@Test @MainActor func decodeParamsFailsWithoutJSON() {
#expect(throws: Error.self) {
_ = try NodeAppModel._test_decodeParams(ClawdisCanvasNavigateParams.self, from: nil)
}
}
@Test @MainActor func encodePayloadEmitsJSON() throws {
struct Payload: Codable, Equatable {
var value: String
}
let json = try NodeAppModel._test_encodePayload(Payload(value: "ok"))
#expect(json.contains("\"value\""))
}
@Test @MainActor func handleInvokeRejectsBackgroundCommands() async {
let appModel = NodeAppModel()
appModel.setScenePhase(.background)
let req = BridgeInvokeRequest(id: "bg", command: ClawdisCanvasCommand.present.rawValue)
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.code == .backgroundUnavailable)
}
@Test @MainActor func handleInvokeRejectsCameraWhenDisabled() async {
let appModel = NodeAppModel()
let req = BridgeInvokeRequest(id: "cam", command: ClawdisCameraCommand.snap.rawValue)
let defaults = UserDefaults.standard
let key = "camera.enabled"
let previous = defaults.object(forKey: key)
defaults.set(false, forKey: key)
defer {
if let previous {
defaults.set(previous, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.code == .unavailable)
#expect(res.error?.message.contains("CAMERA_DISABLED") == true)
}
@Test @MainActor func handleInvokeRejectsInvalidScreenFormat() async {
let appModel = NodeAppModel()
let params = ClawdisScreenRecordParams(format: "gif")
let data = try? JSONEncoder().encode(params)
let json = data.flatMap { String(data: $0, encoding: .utf8) }
let req = BridgeInvokeRequest(
id: "screen",
command: ClawdisScreenCommand.record.rawValue,
paramsJSON: json)
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.message.contains("screen format must be mp4") == true)
}
@Test @MainActor func handleInvokeCanvasCommandsUpdateScreen() async throws {
let appModel = NodeAppModel()
appModel.screen.navigate(to: "http://example.com")
let present = BridgeInvokeRequest(id: "present", command: ClawdisCanvasCommand.present.rawValue)
let presentRes = await appModel._test_handleInvoke(present)
#expect(presentRes.ok == true)
#expect(appModel.screen.urlString.isEmpty)
let navigateParams = ClawdisCanvasNavigateParams(url: "http://localhost:18789/")
let navData = try JSONEncoder().encode(navigateParams)
let navJSON = String(decoding: navData, as: UTF8.self)
let navigate = BridgeInvokeRequest(
id: "nav",
command: ClawdisCanvasCommand.navigate.rawValue,
paramsJSON: navJSON)
let navRes = await appModel._test_handleInvoke(navigate)
#expect(navRes.ok == true)
#expect(appModel.screen.urlString == "http://localhost:18789/")
let evalParams = ClawdisCanvasEvalParams(javaScript: "1+1")
let evalData = try JSONEncoder().encode(evalParams)
let evalJSON = String(decoding: evalData, as: UTF8.self)
let eval = BridgeInvokeRequest(
id: "eval",
command: ClawdisCanvasCommand.evalJS.rawValue,
paramsJSON: evalJSON)
let evalRes = await appModel._test_handleInvoke(eval)
#expect(evalRes.ok == true)
let payloadData = try #require(evalRes.payloadJSON?.data(using: .utf8))
let payload = try JSONSerialization.jsonObject(with: payloadData) as? [String: Any]
#expect(payload?["result"] as? String == "2")
}
@Test @MainActor func handleInvokeA2UICommandsFailWhenHostMissing() async throws {
let appModel = NodeAppModel()
let reset = BridgeInvokeRequest(id: "reset", command: ClawdisCanvasA2UICommand.reset.rawValue)
let resetRes = await appModel._test_handleInvoke(reset)
#expect(resetRes.ok == false)
#expect(resetRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
let jsonl = "{\"beginRendering\":{}}"
let pushParams = ClawdisCanvasA2UIPushJSONLParams(jsonl: jsonl)
let pushData = try JSONEncoder().encode(pushParams)
let pushJSON = String(decoding: pushData, as: UTF8.self)
let push = BridgeInvokeRequest(
id: "push",
command: ClawdisCanvasA2UICommand.pushJSONL.rawValue,
paramsJSON: pushJSON)
let pushRes = await appModel._test_handleInvoke(push)
#expect(pushRes.ok == false)
#expect(pushRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
}
@Test @MainActor func handleInvokeUnknownCommandReturnsInvalidRequest() async {
let appModel = NodeAppModel()
let req = BridgeInvokeRequest(id: "unknown", command: "nope")
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.code == .invalidRequest)
}
@Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async {
let appModel = NodeAppModel()
let url = URL(string: "clawdis://agent?message=hello")!
await appModel.handleDeepLink(url: url)
#expect(appModel.screen.errorText?.contains("Bridge not connected") == true)
}
@Test @MainActor func handleDeepLinkRejectsOversizedMessage() async {
let appModel = NodeAppModel()
let msg = String(repeating: "a", count: 20001)
let url = URL(string: "clawdis://agent?message=\(msg)")!
await appModel.handleDeepLink(url: url)
#expect(appModel.screen.errorText?.contains("Deep link too large") == true)
}
@Test @MainActor func sendVoiceTranscriptThrowsWhenBridgeOffline() async {
let appModel = NodeAppModel()
await #expect(throws: Error.self) {
try await appModel.sendVoiceTranscript(text: "hello", sessionKey: "main")
}
}
@Test @MainActor func canvasA2UIActionDispatchesStatus() async {
let appModel = NodeAppModel()
let body: [String: Any] = [
"userAction": [
"name": "tap",
"id": "action-1",
"surfaceId": "main",
"sourceComponentId": "button-1",
"context": ["value": "ok"],
],
]
await appModel._test_handleCanvasA2UIAction(body: body)
#expect(appModel.screen.urlString.isEmpty)
}
}
@@ -16,15 +16,6 @@ import WebKit
#expect(scrollView.bounces == false)
}
@Test @MainActor func navigateEnablesScrollForWebPages() {
let screen = ScreenController()
screen.navigate(to: "https://example.com")
let scrollView = screen.webView.scrollView
#expect(scrollView.isScrollEnabled == true)
#expect(scrollView.bounces == true)
}
@Test @MainActor func navigateSlashShowsDefaultCanvas() {
let screen = ScreenController()
screen.navigate(to: "/")
@@ -1,32 +0,0 @@
import Testing
@testable import Clawdis
@Suite(.serialized) struct ScreenRecordServiceTests {
@Test func clampDefaultsAndBounds() {
#expect(ScreenRecordService._test_clampDurationMs(nil) == 10000)
#expect(ScreenRecordService._test_clampDurationMs(0) == 250)
#expect(ScreenRecordService._test_clampDurationMs(60001) == 60000)
#expect(ScreenRecordService._test_clampFps(nil) == 10)
#expect(ScreenRecordService._test_clampFps(0) == 1)
#expect(ScreenRecordService._test_clampFps(120) == 30)
#expect(ScreenRecordService._test_clampFps(.infinity) == 10)
}
@Test @MainActor func recordRejectsInvalidScreenIndex() async {
let recorder = ScreenRecordService()
do {
_ = try await recorder.record(
screenIndex: 1,
durationMs: 250,
fps: 5,
includeAudio: false,
outPath: nil)
Issue.record("Expected invalid screen index to throw")
} catch let error as ScreenRecordService.ScreenRecordError {
#expect(error.localizedDescription.contains("Invalid screen index") == true)
} catch {
Issue.record("Unexpected error type: \(error)")
}
}
}
@@ -1,90 +1,33 @@
import Foundation
import Testing
import SwabbleKit
@testable import Clawdis
@Suite struct VoiceWakeManagerExtractCommandTests {
@Test func extractCommandReturnsNilWhenNoTriggerFound() {
let transcript = "hello world"
let segments = makeSegments(
transcript: transcript,
words: [("hello", 0.0, 0.1), ("world", 0.2, 0.1)])
#expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["clawd"]) == nil)
#expect(VoiceWakeManager.extractCommand(from: "hello world", triggers: ["clawd"]) == nil)
}
@Test func extractCommandTrimsTokensAndResult() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.9, 0.1),
("thing", 1.1, 0.1),
])
let cmd = VoiceWakeManager.extractCommand(
from: transcript,
segments: segments,
triggers: [" clawd "],
minPostTriggerGap: 0.3)
let cmd = VoiceWakeManager.extractCommand(from: "hey clawd do thing ", triggers: [" clawd "])
#expect(cmd == "do thing")
}
@Test func extractCommandReturnsNilWhenGapTooShort() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.35, 0.1),
("thing", 0.5, 0.1),
])
let cmd = VoiceWakeManager.extractCommand(
from: transcript,
segments: segments,
triggers: ["clawd"],
minPostTriggerGap: 0.3)
#expect(cmd == nil)
@Test func extractCommandPicksLatestTriggerOccurrence() {
let transcript = "clawd first\nthen something\nclaude second"
let cmd = VoiceWakeManager.extractCommand(from: transcript, triggers: ["clawd", "claude"])
#expect(cmd == "second")
}
@Test func extractCommandIsCaseInsensitive() {
let cmd = VoiceWakeManager.extractCommand(from: "HELLO CLAWD run it", triggers: ["clawd"])
#expect(cmd == "run it")
}
@Test func extractCommandReturnsNilWhenNothingAfterTrigger() {
let transcript = "hey clawd"
let segments = makeSegments(
transcript: transcript,
words: [("hey", 0.0, 0.1), ("clawd", 0.2, 0.1)])
#expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["clawd"]) == nil)
#expect(VoiceWakeManager.extractCommand(from: "hey clawd \n", triggers: ["clawd"]) == nil)
}
@Test func extractCommandIgnoresEmptyTriggers() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.9, 0.1),
("thing", 1.1, 0.1),
])
let cmd = VoiceWakeManager.extractCommand(
from: transcript,
segments: segments,
triggers: ["", " ", "clawd"],
minPostTriggerGap: 0.3)
let cmd = VoiceWakeManager.extractCommand(from: "hey clawd do thing", triggers: ["", " ", "clawd"])
#expect(cmd == "do thing")
}
}
private func makeSegments(
transcript: String,
words: [(String, TimeInterval, TimeInterval)])
-> [WakeWordSegment] {
var searchStart = transcript.startIndex
var output: [WakeWordSegment] = []
for (word, start, duration) in words {
let range = transcript.range(of: word, range: searchStart..<transcript.endIndex)
output.append(WakeWordSegment(text: word, start: start, duration: duration, range: range))
if let range { searchStart = range.upperBound }
}
return output
}
@@ -1,65 +0,0 @@
import Foundation
import Testing
import SwabbleKit
@testable import Clawdis
@Suite(.serialized) struct VoiceWakeManagerStateTests {
@Test @MainActor func suspendAndResumeCycleUpdatesState() async {
let manager = VoiceWakeManager()
manager.isEnabled = true
manager.isListening = true
manager.statusText = "Listening"
let suspended = manager.suspendForExternalAudioCapture()
#expect(suspended == true)
#expect(manager.isListening == false)
#expect(manager.statusText == "Paused")
manager.resumeAfterExternalAudioCapture(wasSuspended: true)
try? await Task.sleep(nanoseconds: 900_000_000)
#expect(manager.statusText.contains("Voice Wake") == true)
}
@Test @MainActor func handleRecognitionCallbackRestartsOnError() async {
let manager = VoiceWakeManager()
manager.isEnabled = true
manager.isListening = true
manager._test_handleRecognitionCallback(transcript: nil, segments: [], errorText: "boom")
#expect(manager.statusText.contains("Recognizer error") == true)
#expect(manager.isListening == false)
try? await Task.sleep(nanoseconds: 900_000_000)
#expect(manager.statusText.contains("Voice Wake") == true)
}
@Test @MainActor func handleRecognitionCallbackDispatchesCommand() async {
let manager = VoiceWakeManager()
manager.triggerWords = ["clawd"]
manager.isEnabled = true
actor CaptureBox {
var value: String?
func set(_ next: String) { self.value = next }
}
let capture = CaptureBox()
manager.configure { cmd in
await capture.set(cmd)
}
let transcript = "clawd hello"
let clawdRange = transcript.range(of: "clawd")!
let helloRange = transcript.range(of: "hello")!
let segments = [
WakeWordSegment(text: "clawd", start: 0.0, duration: 0.2, range: clawdRange),
WakeWordSegment(text: "hello", start: 0.8, duration: 0.2, range: helloRange),
]
manager._test_handleRecognitionCallback(transcript: transcript, segments: segments, errorText: nil)
#expect(manager.lastTriggeredCommand == "hello")
#expect(manager.statusText == "Triggered")
try? await Task.sleep(nanoseconds: 300_000_000)
#expect(await capture.value == "hello")
}
}
-8
View File
@@ -1,5 +1,3 @@
require "shellwords"
default_platform(:ios)
def load_env_file(path)
@@ -63,12 +61,6 @@ platform :ios do
api_key = asc_api_key
team_id = ENV["IOS_DEVELOPMENT_TEAM"]
if team_id.nil? || team_id.strip.empty?
helper_path = File.expand_path("../../scripts/ios-team-id.sh", __dir__)
if File.exist?(helper_path)
team_id = sh("bash #{helper_path.shellescape}").strip
end
end
UI.user_error!("Missing IOS_DEVELOPMENT_TEAM (Apple Team ID). Add it to fastlane/.env or export it in your shell.") if team_id.nil? || team_id.strip.empty?
build_app(
+1 -2
View File
@@ -22,11 +22,10 @@ ASC_KEY_PATH=/absolute/path/to/AuthKey_XXXXXXXXXX.p8
IOS_DEVELOPMENT_TEAM=YOUR_TEAM_ID
```
Tip: run `scripts/ios-team-id.sh` from the repo root to print a Team ID to paste into `.env`. Fastlane falls back to this helper if `IOS_DEVELOPMENT_TEAM` is missing.
Run:
```bash
cd apps/ios
fastlane beta
```
+3 -21
View File
@@ -8,8 +8,6 @@ options:
packages:
ClawdisKit:
path: ../shared/ClawdisKit
Swabble:
path: ../../Swabble
schemes:
Clawdis:
@@ -31,42 +29,30 @@ targets:
- package: ClawdisKit
- package: ClawdisKit
product: ClawdisChatUI
- package: Swabble
product: SwabbleKit
- sdk: AppIntents.framework
preBuildScripts:
- name: SwiftFormat (lint)
basedOnDependencyAnalysis: false
inputFileLists:
- $(SRCROOT)/SwiftSources.input.xcfilelist
script: |
set -euo pipefail
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
if ! command -v swiftformat >/dev/null 2>&1; then
echo "error: swiftformat not found (brew install swiftformat)" >&2
exit 1
fi
swiftformat --lint --config "$SRCROOT/../../.swiftformat" \
--filelist "$SRCROOT/SwiftSources.input.xcfilelist"
"$SRCROOT/Sources" \
"$SRCROOT/../shared/ClawdisKit/Sources"
- name: SwiftLint
basedOnDependencyAnalysis: false
inputFileLists:
- $(SRCROOT)/SwiftSources.input.xcfilelist
script: |
set -euo pipefail
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
if ! command -v swiftlint >/dev/null 2>&1; then
echo "error: swiftlint not found (brew install swiftlint)" >&2
exit 1
fi
swiftlint lint --config "$SRCROOT/.swiftlint.yml" --use-script-input-file-lists
swiftlint lint --config "$SRCROOT/.swiftlint.yml"
settings:
base:
CODE_SIGN_IDENTITY: "Apple Development"
CODE_SIGN_STYLE: Manual
DEVELOPMENT_TEAM: Y5PE65HELJ
PRODUCT_BUNDLE_IDENTIFIER: com.steipete.clawdis.ios
PROVISIONING_PROFILE_SPECIFIER: "com.steipete.clawdis.ios Development"
SWIFT_VERSION: "6.0"
info:
path: Sources/Info.plist
@@ -79,8 +65,6 @@ targets:
UIBackgroundModes:
- audio
NSLocalNetworkUsageDescription: Clawdis discovers and connects to your Clawdis bridge on the local network.
NSAppTransportSecurity:
NSAllowsArbitraryLoadsInWebContent: true
NSBonjourServices:
- _clawdis-bridge._tcp
NSCameraUsageDescription: Clawdis can capture photos or short video clips when requested via the bridge.
@@ -94,8 +78,6 @@ targets:
- path: Tests
dependencies:
- target: Clawdis
- package: Swabble
product: SwabbleKit
settings:
base:
PRODUCT_BUNDLE_IDENTIFIER: com.steipete.clawdis.ios.tests
+1 -13
View File
@@ -15,10 +15,8 @@ let package = Package(
dependencies: [
.package(url: "https://github.com/orchetect/MenuBarExtraAccess", exact: "1.2.2"),
.package(url: "https://github.com/swiftlang/swift-subprocess.git", from: "0.1.0"),
.package(url: "https://github.com/apple/swift-log.git", from: "1.8.0"),
.package(url: "https://github.com/sparkle-project/Sparkle", from: "2.8.1"),
.package(path: "../shared/ClawdisKit"),
.package(path: "../../Swabble"),
.package(path: "../../Peekaboo/Core/PeekabooCore"),
.package(path: "../../Peekaboo/Core/PeekabooAutomationKit"),
],
@@ -43,17 +41,12 @@ let package = Package(
"ClawdisProtocol",
.product(name: "ClawdisKit", package: "ClawdisKit"),
.product(name: "ClawdisChatUI", package: "ClawdisKit"),
.product(name: "SwabbleKit", package: "swabble"),
.product(name: "MenuBarExtraAccess", package: "MenuBarExtraAccess"),
.product(name: "Subprocess", package: "swift-subprocess"),
.product(name: "Logging", package: "swift-log"),
.product(name: "Sparkle", package: "Sparkle"),
.product(name: "PeekabooBridge", package: "PeekabooCore"),
.product(name: "PeekabooAutomationKit", package: "PeekabooAutomationKit"),
],
exclude: [
"Resources/Info.plist",
],
resources: [
.copy("Resources/Clawdis.icns"),
.copy("Resources/DeviceModels"),
@@ -63,12 +56,7 @@ let package = Package(
]),
.testTarget(
name: "ClawdisIPCTests",
dependencies: [
"ClawdisIPC",
"Clawdis",
"ClawdisProtocol",
.product(name: "SwabbleKit", package: "swabble"),
],
dependencies: ["ClawdisIPC", "Clawdis", "ClawdisProtocol"],
swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"),
.enableExperimentalFeature("SwiftTesting"),
@@ -92,12 +92,10 @@ struct AboutSettings: View {
guard let updater, !self.didLoadUpdaterState else { return }
// Keep Sparkle’s auto-check setting in sync with the persisted toggle.
updater.automaticallyChecksForUpdates = self.autoCheckEnabled
updater.automaticallyDownloadsUpdates = self.autoCheckEnabled
self.didLoadUpdaterState = true
}
.onChange(of: self.autoCheckEnabled) { _, newValue in
self.updater?.automaticallyChecksForUpdates = newValue
self.updater?.automaticallyDownloadsUpdates = newValue
}
}
@@ -1,17 +0,0 @@
import Foundation
// Human-friendly age string (e.g., "2m ago").
func age(from date: Date, now: Date = .init()) -> String {
let seconds = max(0, Int(now.timeIntervalSince(date)))
let minutes = seconds / 60
let hours = minutes / 60
let days = hours / 24
if seconds < 60 { return "just now" }
if minutes == 1 { return "1 minute ago" }
if minutes < 60 { return "\(minutes)m ago" }
if hours == 1 { return "1 hour ago" }
if hours < 24 { return "\(hours)h ago" }
if days == 1 { return "yesterday" }
return "\(days)d ago"
}
@@ -0,0 +1,45 @@
import Foundation
struct AgentIdentity: Codable, Equatable {
var name: String
var theme: String
var emoji: String
var isEmpty: Bool {
self.name.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty &&
self.theme.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty &&
self.emoji.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
}
}
enum AgentIdentityEmoji {
static func suggest(theme: String) -> String {
let normalized = theme.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
if normalized.isEmpty { return "🦞" }
let table: [(needle: String, emoji: String)] = [
("lobster", "🦞"),
("sloth", "🦥"),
("octopus", "🐙"),
("crab", "🦀"),
("shark", "🦈"),
("cat", "🐈"),
("dog", "🐕"),
("owl", "🦉"),
("fox", "🦊"),
("otter", "🦦"),
("raccoon", "🦝"),
("badger", "🦡"),
("hedgehog", "🦔"),
("koala", "🐨"),
("penguin", "🐧"),
("frog", "🐸"),
("bear", "🐻"),
]
for entry in table where normalized.contains(entry.needle) {
return entry.emoji
}
return "🦞"
}
}
+45 -89
View File
@@ -9,18 +9,8 @@ enum AgentWorkspace {
static let userFilename = "USER.md"
static let bootstrapFilename = "BOOTSTRAP.md"
private static let templateDirname = "templates"
private static let ignoredEntries: Set<String> = [".DS_Store", ".git", ".gitignore"]
private static let templateEntries: Set<String> = [
AgentWorkspace.agentsFilename,
AgentWorkspace.soulFilename,
AgentWorkspace.identityFilename,
AgentWorkspace.userFilename,
AgentWorkspace.bootstrapFilename,
]
enum BootstrapSafety: Equatable {
case safe
case unsafe(reason: String)
}
static let identityStartMarker = "<!-- clawdis:identity:start -->"
static let identityEndMarker = "<!-- clawdis:identity:end -->"
static func displayPath(for url: URL) -> String {
let home = FileManager.default.homeDirectoryForCurrentUser.path
@@ -43,53 +33,7 @@ enum AgentWorkspace {
workspaceURL.appendingPathComponent(self.agentsFilename)
}
static func workspaceEntries(workspaceURL: URL) throws -> [String] {
let contents = try FileManager.default.contentsOfDirectory(atPath: workspaceURL.path)
return contents.filter { !self.ignoredEntries.contains($0) }
}
static func isWorkspaceEmpty(workspaceURL: URL) -> Bool {
let fm = FileManager.default
var isDir: ObjCBool = false
if !fm.fileExists(atPath: workspaceURL.path, isDirectory: &isDir) {
return true
}
guard isDir.boolValue else { return false }
guard let entries = try? self.workspaceEntries(workspaceURL: workspaceURL) else { return false }
return entries.isEmpty
}
static func isTemplateOnlyWorkspace(workspaceURL: URL) -> Bool {
guard let entries = try? self.workspaceEntries(workspaceURL: workspaceURL) else { return false }
guard !entries.isEmpty else { return true }
return Set(entries).isSubset(of: self.templateEntries)
}
static func bootstrapSafety(for workspaceURL: URL) -> BootstrapSafety {
let fm = FileManager.default
var isDir: ObjCBool = false
if !fm.fileExists(atPath: workspaceURL.path, isDirectory: &isDir) {
return .safe
}
if !isDir.boolValue {
return .unsafe(reason: "Workspace path points to a file.")
}
let agentsURL = self.agentsURL(workspaceURL: workspaceURL)
if fm.fileExists(atPath: agentsURL.path) {
return .safe
}
do {
let entries = try self.workspaceEntries(workspaceURL: workspaceURL)
return entries.isEmpty
? .safe
: .unsafe(reason: "Folder isn't empty. Choose a new folder or add AGENTS.md first.")
} catch {
return .unsafe(reason: "Couldn't inspect the workspace folder.")
}
}
static func bootstrap(workspaceURL: URL) throws -> URL {
let shouldSeedBootstrap = self.isWorkspaceEmpty(workspaceURL: workspaceURL)
try FileManager.default.createDirectory(at: workspaceURL, withIntermediateDirectories: true)
let agentsURL = self.agentsURL(workspaceURL: workspaceURL)
if !FileManager.default.fileExists(atPath: agentsURL.path) {
@@ -112,44 +56,36 @@ enum AgentWorkspace {
self.logger.info("Created USER.md at \(userURL.path, privacy: .public)")
}
let bootstrapURL = workspaceURL.appendingPathComponent(self.bootstrapFilename)
if shouldSeedBootstrap, !FileManager.default.fileExists(atPath: bootstrapURL.path) {
if !FileManager.default.fileExists(atPath: bootstrapURL.path) {
try self.defaultBootstrapTemplate().write(to: bootstrapURL, atomically: true, encoding: .utf8)
self.logger.info("Created BOOTSTRAP.md at \(bootstrapURL.path, privacy: .public)")
}
return agentsURL
}
static func needsBootstrap(workspaceURL: URL) -> Bool {
let fm = FileManager.default
var isDir: ObjCBool = false
if !fm.fileExists(atPath: workspaceURL.path, isDirectory: &isDir) {
return true
}
guard isDir.boolValue else { return true }
if self.hasIdentity(workspaceURL: workspaceURL) {
return false
}
let bootstrapURL = workspaceURL.appendingPathComponent(self.bootstrapFilename)
guard fm.fileExists(atPath: bootstrapURL.path) else { return false }
return self.isTemplateOnlyWorkspace(workspaceURL: workspaceURL)
}
static func upsertIdentity(workspaceURL: URL, identity: AgentIdentity) throws {
let agentsURL = try self.bootstrap(workspaceURL: workspaceURL)
var content = (try? String(contentsOf: agentsURL, encoding: .utf8)) ?? ""
let block = self.identityBlock(identity: identity)
static func hasIdentity(workspaceURL: URL) -> Bool {
let identityURL = workspaceURL.appendingPathComponent(self.identityFilename)
guard let contents = try? String(contentsOf: identityURL, encoding: .utf8) else { return false }
return self.identityLinesHaveValues(contents)
}
private static func identityLinesHaveValues(_ content: String) -> Bool {
for line in content.split(separator: "\n") {
let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines)
guard trimmed.hasPrefix("-"), let colon = trimmed.firstIndex(of: ":") else { continue }
let value = trimmed[trimmed.index(after: colon)...].trimmingCharacters(in: .whitespacesAndNewlines)
if !value.isEmpty {
return true
}
if let start = content.range(of: self.identityStartMarker),
let end = content.range(of: self.identityEndMarker),
start.lowerBound < end.upperBound
{
content.replaceSubrange(
start.lowerBound..<end.upperBound,
with: block.trimmingCharacters(in: .whitespacesAndNewlines))
} else if let insert = self.identityInsertRange(in: content) {
content.insert(contentsOf: "\n\n## Identity\n\(block)\n", at: insert.upperBound)
} else {
content = [content.trimmingCharacters(in: .whitespacesAndNewlines), "## Identity\n\(block)"]
.filter { !$0.isEmpty }
.joined(separator: "\n\n")
.appending("\n")
}
return false
try content.write(to: agentsURL, atomically: true, encoding: .utf8)
self.logger.info("Updated identity in \(agentsURL.path, privacy: .public)")
}
static func defaultTemplate() -> String {
@@ -336,5 +272,25 @@ enum AgentWorkspace {
return trimmed + "\n"
}
// Identity is written by the agent during the bootstrap ritual.
private static func identityBlock(identity: AgentIdentity) -> String {
let name = identity.name.trimmingCharacters(in: .whitespacesAndNewlines)
let theme = identity.theme.trimmingCharacters(in: .whitespacesAndNewlines)
let emoji = identity.emoji.trimmingCharacters(in: .whitespacesAndNewlines)
return """
\(self.identityStartMarker)
- Name: \(name)
- Theme: \(theme)
- Emoji: \(emoji)
\(self.identityEndMarker)
"""
}
private static func identityInsertRange(in content: String) -> Range<String.Index>? {
if let firstHeading = content.range(of: "\n") {
// Insert after the first line (usually "# AGENTS.md …")
return firstHeading
}
return nil
}
}
@@ -6,7 +6,7 @@ import SwiftUI
struct AnthropicAuthControls: View {
let connectionMode: AppState.ConnectionMode
@State private var oauthStatus: ClawdisOAuthStore.AnthropicOAuthStatus = ClawdisOAuthStore.anthropicOAuthStatus()
@State private var oauthStatus: PiOAuthStore.AnthropicOAuthStatus = PiOAuthStore.anthropicOAuthStatus()
@State private var pkce: AnthropicOAuth.PKCE?
@State private var code: String = ""
@State private var busy = false
@@ -15,19 +15,12 @@ struct AnthropicAuthControls: View {
@State private var autoConnectClipboard = true
@State private var lastPasteboardChangeCount = NSPasteboard.general.changeCount
private static let clipboardPoll: AnyPublisher<Date, Never> = {
if ProcessInfo.processInfo.isRunningTests {
return Empty(completeImmediately: false).eraseToAnyPublisher()
}
return Timer.publish(every: 0.4, on: .main, in: .common)
.autoconnect()
.eraseToAnyPublisher()
}()
private static let clipboardPoll = Timer.publish(every: 0.4, on: .main, in: .common).autoconnect()
var body: some View {
VStack(alignment: .leading, spacing: 10) {
if self.connectionMode != .local {
Text("Gateway isn’t running locally; OAuth must be created on the gateway host.")
Text("Gateway isn’t running locally; OAuth must be created on the gateway host where Pi runs.")
.font(.footnote)
.foregroundStyle(.secondary)
.fixedSize(horizontal: false, vertical: true)
@@ -42,10 +35,10 @@ struct AnthropicAuthControls: View {
.foregroundStyle(.secondary)
Spacer()
Button("Reveal") {
NSWorkspace.shared.activateFileViewerSelecting([ClawdisOAuthStore.oauthURL()])
NSWorkspace.shared.activateFileViewerSelecting([PiOAuthStore.oauthURL()])
}
.buttonStyle(.bordered)
.disabled(!FileManager.default.fileExists(atPath: ClawdisOAuthStore.oauthURL().path))
.disabled(!FileManager.default.fileExists(atPath: PiOAuthStore.oauthURL().path))
Button("Refresh") {
self.refresh()
@@ -53,7 +46,7 @@ struct AnthropicAuthControls: View {
.buttonStyle(.bordered)
}
Text(ClawdisOAuthStore.oauthURL().path)
Text(PiOAuthStore.oauthURL().path)
.font(.caption.monospaced())
.foregroundStyle(.secondary)
.lineLimit(1)
@@ -130,11 +123,7 @@ struct AnthropicAuthControls: View {
}
private func refresh() {
let imported = ClawdisOAuthStore.importLegacyAnthropicOAuthIfNeeded()
self.oauthStatus = ClawdisOAuthStore.anthropicOAuthStatus()
if imported != nil {
self.statusText = "Imported existing OAuth credentials."
}
self.oauthStatus = PiOAuthStore.anthropicOAuthStatus()
}
private func startOAuth() {
@@ -172,11 +161,11 @@ struct AnthropicAuthControls: View {
code: parsed.code,
state: parsed.state,
verifier: pkce.verifier)
try ClawdisOAuthStore.saveAnthropicOAuth(creds)
try PiOAuthStore.saveAnthropicOAuth(creds)
self.refresh()
self.pkce = nil
self.code = ""
self.statusText = "Connected. Clawdis can now use Claude via OAuth."
self.statusText = "Connected. Pi can now use Claude via OAuth."
} catch {
self.statusText = "OAuth failed: \(error.localizedDescription)"
}
@@ -207,28 +196,3 @@ struct AnthropicAuthControls: View {
Task { await self.finishOAuth() }
}
}
#if DEBUG
extension AnthropicAuthControls {
init(
connectionMode: AppState.ConnectionMode,
oauthStatus: ClawdisOAuthStore.AnthropicOAuthStatus,
pkce: AnthropicOAuth.PKCE? = nil,
code: String = "",
busy: Bool = false,
statusText: String? = nil,
autoDetectClipboard: Bool = true,
autoConnectClipboard: Bool = true)
{
self.connectionMode = connectionMode
self._oauthStatus = State(initialValue: oauthStatus)
self._pkce = State(initialValue: pkce)
self._code = State(initialValue: code)
self._busy = State(initialValue: busy)
self._statusText = State(initialValue: statusText)
self._autoDetectClipboard = State(initialValue: autoDetectClipboard)
self._autoConnectClipboard = State(initialValue: autoConnectClipboard)
self._lastPasteboardChangeCount = State(initialValue: NSPasteboard.general.changeCount)
}
}
#endif
+23 -67
View File
@@ -18,7 +18,7 @@ enum AnthropicAuthMode: Equatable {
var shortLabel: String {
switch self {
case .oauthFile: "OAuth (Clawdis token file)"
case .oauthFile: "OAuth (Pi token file)"
case .oauthEnv: "OAuth (env var)"
case .apiKeyEnv: "API key (env var)"
case .missing: "Missing credentials"
@@ -36,8 +36,7 @@ enum AnthropicAuthMode: Equatable {
enum AnthropicAuthResolver {
static func resolve(
environment: [String: String] = ProcessInfo.processInfo.environment,
oauthStatus: ClawdisOAuthStore.AnthropicOAuthStatus = ClawdisOAuthStore
.anthropicOAuthStatus()) -> AnthropicAuthMode
oauthStatus: PiOAuthStore.AnthropicOAuthStatus = PiOAuthStore.anthropicOAuthStatus()) -> AnthropicAuthMode
{
if oauthStatus.isConnected { return .oauthFile }
@@ -93,7 +92,7 @@ enum AnthropicOAuth {
URLQueryItem(name: "scope", value: self.scopes),
URLQueryItem(name: "code_challenge", value: pkce.challenge),
URLQueryItem(name: "code_challenge_method", value: "S256"),
// Match legacy flow: state is the verifier.
// Match Pi: state is the verifier.
URLQueryItem(name: "state", value: pkce.verifier),
]
return components.url!
@@ -141,7 +140,7 @@ enum AnthropicOAuth {
])
}
// Match legacy flow: expiresAt = now + expires_in - 5 minutes.
// Match Pi: expiresAt = now + expires_in - 5 minutes.
let expiresAtMs = Int64(Date().timeIntervalSince1970 * 1000)
+ Int64(expiresIn * 1000)
- Int64(5 * 60 * 1000)
@@ -151,11 +150,10 @@ enum AnthropicOAuth {
}
}
enum ClawdisOAuthStore {
enum PiOAuthStore {
static let oauthFilename = "oauth.json"
private static let providerKey = "anthropic"
private static let clawdisOAuthDirEnv = "CLAWDIS_OAUTH_DIR"
private static let legacyPiDirEnv = "PI_CODING_AGENT_DIR"
private static let piAgentDirEnv = "PI_CODING_AGENT_DIR"
enum AnthropicOAuthStatus: Equatable {
case missingFile
@@ -172,18 +170,18 @@ enum ClawdisOAuthStore {
var shortDescription: String {
switch self {
case .missingFile: "Clawdis OAuth token file not found"
case .unreadableFile: "Clawdis OAuth token file not readable"
case .invalidJSON: "Clawdis OAuth token file invalid"
case .missingProviderEntry: "No Anthropic entry in Clawdis OAuth token file"
case .missingFile: "Pi OAuth token file not found"
case .unreadableFile: "Pi OAuth token file not readable"
case .invalidJSON: "Pi OAuth token file invalid"
case .missingProviderEntry: "No Anthropic entry in Pi OAuth token file"
case .missingTokens: "Anthropic entry missing tokens"
case .connected: "Clawdis OAuth credentials found"
case .connected: "Pi OAuth credentials found"
}
}
}
static func oauthDir() -> URL {
if let override = ProcessInfo.processInfo.environment[self.clawdisOAuthDirEnv]?
if let override = ProcessInfo.processInfo.environment[self.piAgentDirEnv]?
.trimmingCharacters(in: .whitespacesAndNewlines),
!override.isEmpty
{
@@ -192,58 +190,14 @@ enum ClawdisOAuthStore {
}
return FileManager.default.homeDirectoryForCurrentUser
.appendingPathComponent(".clawdis", isDirectory: true)
.appendingPathComponent("credentials", isDirectory: true)
.appendingPathComponent(".pi", isDirectory: true)
.appendingPathComponent("agent", isDirectory: true)
}
static func oauthURL() -> URL {
self.oauthDir().appendingPathComponent(self.oauthFilename)
}
static func legacyOAuthURLs() -> [URL] {
var urls: [URL] = []
let env = ProcessInfo.processInfo.environment
if let override = env[self.legacyPiDirEnv]?.trimmingCharacters(in: .whitespacesAndNewlines),
!override.isEmpty
{
let expanded = NSString(string: override).expandingTildeInPath
urls.append(URL(fileURLWithPath: expanded, isDirectory: true).appendingPathComponent(self.oauthFilename))
}
let home = FileManager.default.homeDirectoryForCurrentUser
urls.append(home.appendingPathComponent(".pi/agent/\(self.oauthFilename)"))
urls.append(home.appendingPathComponent(".claude/\(self.oauthFilename)"))
urls.append(home.appendingPathComponent(".config/claude/\(self.oauthFilename)"))
urls.append(home.appendingPathComponent(".config/anthropic/\(self.oauthFilename)"))
var seen = Set<String>()
return urls.filter { url in
let path = url.standardizedFileURL.path
if seen.contains(path) { return false }
seen.insert(path)
return true
}
}
static func importLegacyAnthropicOAuthIfNeeded() -> URL? {
let dest = self.oauthURL()
guard !FileManager.default.fileExists(atPath: dest.path) else { return nil }
for url in self.legacyOAuthURLs() {
guard FileManager.default.fileExists(atPath: url.path) else { continue }
guard self.anthropicOAuthStatus(at: url).isConnected else { continue }
guard let storage = self.loadStorage(at: url) else { continue }
do {
try self.saveStorage(storage)
return url
} catch {
continue
}
}
return nil
}
static func anthropicOAuthStatus() -> AnthropicOAuthStatus {
self.anthropicOAuthStatus(at: self.oauthURL())
}
@@ -286,15 +240,17 @@ enum ClawdisOAuthStore {
return nil
}
private static func loadStorage(at url: URL) -> [String: Any]? {
guard let data = try? Data(contentsOf: url) else { return nil }
guard let json = try? JSONSerialization.jsonObject(with: data, options: []) else { return nil }
return json as? [String: Any]
}
static func saveAnthropicOAuth(_ creds: AnthropicOAuthCredentials) throws {
let url = self.oauthURL()
let existing: [String: Any] = self.loadStorage(at: url) ?? [:]
let existing: [String: Any] = if FileManager.default.fileExists(atPath: url.path),
let data = try? Data(contentsOf: url),
let json = try? JSONSerialization.jsonObject(with: data, options: []),
let dict = json as? [String: Any]
{
dict
} else {
[:]
}
var updated = existing
updated[self.providerKey] = [
@@ -1,22 +0,0 @@
import ClawdisProtocol
import Foundation
extension AnyCodable {
var stringValue: String? { self.value as? String }
var boolValue: Bool? { self.value as? Bool }
var intValue: Int? { self.value as? Int }
var doubleValue: Double? { self.value as? Double }
var dictionaryValue: [String: AnyCodable]? { self.value as? [String: AnyCodable] }
var arrayValue: [AnyCodable]? { self.value as? [AnyCodable] }
var foundationValue: Any {
switch self.value {
case let dict as [String: AnyCodable]:
dict.mapValues { $0.foundationValue }
case let array as [AnyCodable]:
array.map(\.foundationValue)
default:
self.value
}
}
}
-45
View File
@@ -38,7 +38,6 @@ final class AppState {
var debugPaneEnabled: Bool {
didSet {
self.ifNotPreview { UserDefaults.standard.set(self.debugPaneEnabled, forKey: "clawdis.debugPaneEnabled") }
CanvasManager.shared.refreshDebugStatus()
}
}
@@ -121,18 +120,6 @@ final class AppState {
forKey: voicePushToTalkEnabledKey) } }
}
var talkEnabled: Bool {
didSet {
self.ifNotPreview {
UserDefaults.standard.set(self.talkEnabled, forKey: talkEnabledKey)
Task { await TalkModeController.shared.setEnabled(self.talkEnabled) }
}
}
}
/// Gateway-provided UI accent color (hex). Optional; clients provide a default.
var seamColorHex: String?
var iconOverride: IconOverrideSelection {
didSet { self.ifNotPreview { UserDefaults.standard.set(self.iconOverride.rawValue, forKey: iconOverrideKey) } }
}
@@ -228,8 +215,6 @@ final class AppState {
.stringArray(forKey: voiceWakeAdditionalLocalesKey) ?? []
self.voicePushToTalkEnabled = UserDefaults.standard
.object(forKey: voicePushToTalkEnabledKey) as? Bool ?? false
self.talkEnabled = UserDefaults.standard.bool(forKey: talkEnabledKey)
self.seamColorHex = nil
if let storedHeartbeats = UserDefaults.standard.object(forKey: heartbeatsEnabledKey) as? Bool {
self.heartbeatsEnabled = storedHeartbeats
} else {
@@ -270,13 +255,9 @@ final class AppState {
if self.swabbleEnabled, !PermissionManager.voiceWakePermissionsGranted() {
self.swabbleEnabled = false
}
if self.talkEnabled, !PermissionManager.voiceWakePermissionsGranted() {
self.talkEnabled = false
}
if !self.isPreview {
Task { await VoiceWakeRuntime.shared.refresh(state: self) }
Task { await TalkModeController.shared.setEnabled(self.talkEnabled) }
}
}
@@ -330,31 +311,6 @@ final class AppState {
Task { await VoiceWakeRuntime.shared.refresh(state: self) }
}
func setTalkEnabled(_ enabled: Bool) async {
guard voiceWakeSupported else {
self.talkEnabled = false
await GatewayConnection.shared.talkMode(enabled: false, phase: "disabled")
return
}
self.talkEnabled = enabled
guard !self.isPreview else { return }
if !enabled {
await GatewayConnection.shared.talkMode(enabled: false, phase: "disabled")
return
}
if PermissionManager.voiceWakePermissionsGranted() {
await GatewayConnection.shared.talkMode(enabled: true, phase: "enabled")
return
}
let granted = await PermissionManager.ensureVoiceWakePermissions(interactive: true)
self.talkEnabled = granted
await GatewayConnection.shared.talkMode(enabled: granted, phase: granted ? "enabled" : "denied")
}
// MARK: - Global wake words sync (Gateway-owned)
func applyGlobalVoiceWakeTriggers(_ triggers: [String]) {
@@ -410,7 +366,6 @@ extension AppState {
state.voiceWakeLocaleID = Locale.current.identifier
state.voiceWakeAdditionalLocaleIDs = ["en-US", "de-DE"]
state.voicePushToTalkEnabled = false
state.talkEnabled = false
state.iconOverride = .system
state.heartbeatsEnabled = true
state.connectionMode = .local
@@ -1,22 +0,0 @@
import Foundation
enum AsyncTimeout {
static func withTimeout<T: Sendable>(
seconds: Double,
onTimeout: @escaping @Sendable () -> Error,
operation: @escaping @Sendable () async throws -> T) async throws -> T
{
let clamped = max(0, seconds)
return try await withThrowingTaskGroup(of: T.self) { group in
group.addTask { try await operation() }
group.addTask {
try await Task.sleep(nanoseconds: UInt64(clamped * 1_000_000_000))
throw onTimeout()
}
let result = try await group.next()
group.cancelAll()
if let result { return result }
throw onTimeout()
}
}
}
@@ -503,40 +503,3 @@ enum BridgePairingApprover {
}
}
}
#if DEBUG
extension BridgeServer {
func exerciseForTesting() async {
let conn = NWConnection(to: .hostPort(host: "127.0.0.1", port: 22), using: .tcp)
let handler = BridgeConnectionHandler(connection: conn, logger: self.logger)
self.connections["node-1"] = handler
self.nodeInfoById["node-1"] = BridgeNodeInfo(
nodeId: "node-1",
displayName: "Node One",
platform: "macOS",
version: "1.0.0",
deviceFamily: "Mac",
modelIdentifier: "MacBookPro18,1",
remoteAddress: "127.0.0.1",
caps: ["chat", "voice"])
_ = self.connectedNodeIds()
_ = self.connectedNodes()
self.handleListenerState(.ready)
self.handleListenerState(.failed(NWError.posix(.ECONNREFUSED)))
self.handleListenerState(.waiting(NWError.posix(.ETIMEDOUT)))
self.handleListenerState(.cancelled)
self.handleListenerState(.setup)
let subscribe = BridgeEventFrame(event: "chat.subscribe", payloadJSON: "{\"sessionKey\":\"main\"}")
await self.handleEvent(nodeId: "node-1", evt: subscribe)
let unsubscribe = BridgeEventFrame(event: "chat.unsubscribe", payloadJSON: "{\"sessionKey\":\"main\"}")
await self.handleEvent(nodeId: "node-1", evt: unsubscribe)
let invalid = BridgeRPCRequest(id: "req-1", method: "invalid.method", paramsJSON: nil)
_ = await self.handleRequest(nodeId: "node-1", req: invalid)
}
}
#endif
@@ -1,102 +0,0 @@
import Foundation
@MainActor
enum CLIInstaller {
private static func embeddedHelperURL() -> URL {
Bundle.main.bundleURL.appendingPathComponent("Contents/Resources/Relay/clawdis")
}
static func installedLocation() -> String? {
self.installedLocation(
searchPaths: cliHelperSearchPaths,
embeddedHelper: self.embeddedHelperURL(),
fileManager: .default)
}
static func installedLocation(
searchPaths: [String],
embeddedHelper: URL,
fileManager: FileManager) -> String?
{
let embedded = embeddedHelper.resolvingSymlinksInPath()
for basePath in searchPaths {
let candidate = URL(fileURLWithPath: basePath).appendingPathComponent("clawdis").path
var isDirectory: ObjCBool = false
guard fileManager.fileExists(atPath: candidate, isDirectory: &isDirectory),
!isDirectory.boolValue
else {
continue
}
guard fileManager.isExecutableFile(atPath: candidate) else { continue }
let resolved = URL(fileURLWithPath: candidate).resolvingSymlinksInPath()
if resolved == embedded {
return candidate
}
}
return nil
}
static func isInstalled() -> Bool {
self.installedLocation() != nil
}
static func install(statusHandler: @escaping @Sendable (String) async -> Void) async {
let helper = self.embeddedHelperURL()
guard FileManager.default.isExecutableFile(atPath: helper.path) else {
await statusHandler(
"Embedded CLI missing in bundle; repackage via scripts/package-mac-app.sh " +
"(or restart-mac.sh without SKIP_GATEWAY_PACKAGE=1).")
return
}
let targets = cliHelperSearchPaths.map { "\($0)/clawdis" }
let result = await self.privilegedSymlink(source: helper.path, targets: targets)
await statusHandler(result)
}
private static func privilegedSymlink(source: String, targets: [String]) async -> String {
let escapedSource = self.shellEscape(source)
let targetList = targets.map(self.shellEscape).joined(separator: " ")
let cmds = [
"mkdir -p /usr/local/bin /opt/homebrew/bin",
targets.map { "ln -sf \(escapedSource) \($0)" }.joined(separator: "; "),
].joined(separator: "; ")
let script = """
do shell script "\(cmds)" with administrator privileges
"""
let proc = Process()
proc.executableURL = URL(fileURLWithPath: "/usr/bin/osascript")
proc.arguments = ["-e", script]
let pipe = Pipe()
proc.standardOutput = pipe
proc.standardError = pipe
do {
try proc.run()
proc.waitUntilExit()
let data = pipe.fileHandleForReading.readToEndSafely()
let output = String(data: data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if proc.terminationStatus == 0 {
return output.isEmpty ? "CLI helper linked into \(targetList)" : output
}
if output.lowercased().contains("user canceled") {
return "Install canceled"
}
return "Failed to install CLI helper: \(output)"
} catch {
return "Failed to run installer: \(error.localizedDescription)"
}
}
private static func shellEscape(_ path: String) -> String {
"'" + path.replacingOccurrences(of: "'", with: "'\"'\"'") + "'"
}
}
@@ -79,14 +79,7 @@ actor CameraCaptureService {
}
withExtendedLifetime(delegate) {}
let maxPayloadBytes = 5 * 1024 * 1024
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
let maxEncodedBytes = (maxPayloadBytes / 4) * 3
let res = try JPEGTranscoder.transcodeToJPEG(
imageData: rawData,
maxWidthPx: maxWidth,
quality: quality,
maxBytes: maxEncodedBytes)
let res = try JPEGTranscoder.transcodeToJPEG(imageData: rawData, maxWidthPx: maxWidth, quality: quality)
return (data: res.data, size: CGSize(width: res.widthPx, height: res.heightPx))
}
@@ -1,147 +0,0 @@
import AppKit
import ClawdisIPC
import ClawdisKit
import Foundation
import WebKit
final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "clawdisCanvasA2UIAction"
private let sessionKey: String
init(sessionKey: String) {
self.sessionKey = sessionKey
super.init()
}
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
guard message.name == Self.messageName else { return }
// Only accept actions from local Canvas content (not arbitrary web pages).
guard let webView = message.webView, let url = webView.url else { return }
if url.scheme == CanvasScheme.scheme {
// ok
} else if Self.isLocalNetworkCanvasURL(url) {
// ok
} else {
return
}
let body: [String: Any] = {
if let dict = message.body as? [String: Any] { return dict }
if let dict = message.body as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !body.isEmpty else { return }
let userActionAny = body["userAction"] ?? body
let userAction: [String: Any] = {
if let dict = userActionAny as? [String: Any] { return dict }
if let dict = userActionAny as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !userAction.isEmpty else { return }
guard let name = ClawdisCanvasA2UIAction.extractActionName(userAction) else { return }
let actionId =
(userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
?? UUID().uuidString
canvasWindowLogger.info("A2UI action \(name, privacy: .public) session=\(self.sessionKey, privacy: .public)")
let surfaceId = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
.nonEmpty ?? "main"
let sourceComponentId = (userAction["sourceComponentId"] as? String)?
.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "-"
let instanceId = InstanceIdentity.instanceId.lowercased()
let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"])
// Token-efficient and unambiguous. The agent should treat this as a UI event and (by default) update Canvas.
let messageContext = ClawdisCanvasA2UIAction.AgentMessageContext(
actionName: name,
session: .init(key: self.sessionKey, surfaceId: surfaceId),
component: .init(id: sourceComponentId, host: InstanceIdentity.displayName, instanceId: instanceId),
contextJSON: contextJSON)
let text = ClawdisCanvasA2UIAction.formatAgentMessage(messageContext)
Task { [weak webView] in
if AppStateStore.shared.connectionMode == .local {
GatewayProcessManager.shared.setActive(true)
}
let result = await GatewayConnection.shared.sendAgent(GatewayAgentInvocation(
message: text,
sessionKey: self.sessionKey,
thinking: "low",
deliver: false,
to: nil,
channel: .last,
idempotencyKey: actionId))
await MainActor.run {
guard let webView else { return }
let js = ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(
actionId: actionId,
ok: result.ok,
error: result.error)
webView.evaluateJavaScript(js) { _, _ in }
}
if !result.ok {
canvasWindowLogger.error(
"""
A2UI action send failed name=\(name, privacy: .public) \
error=\(result.error ?? "unknown", privacy: .public)
""")
}
}
}
static func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return false
}
guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else {
return false
}
if host == "localhost" { return true }
if host.hasSuffix(".local") { return true }
if host.hasSuffix(".ts.net") { return true }
if host.hasSuffix(".tailscale.net") { return true }
if !host.contains("."), !host.contains(":") { return true }
if let ipv4 = Self.parseIPv4(host) {
return Self.isLocalNetworkIPv4(ipv4)
}
return false
}
static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? {
let parts = host.split(separator: ".", omittingEmptySubsequences: false)
guard parts.count == 4 else { return nil }
let bytes: [UInt8] = parts.compactMap { UInt8($0) }
guard bytes.count == 4 else { return nil }
return (bytes[0], bytes[1], bytes[2], bytes[3])
}
static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool {
let (a, b, _, _) = ip
if a == 10 { return true }
if a == 172, (16...31).contains(Int(b)) { return true }
if a == 192, b == 168 { return true }
if a == 127 { return true }
if a == 169, b == 254 { return true }
if a == 100, (64...127).contains(Int(b)) { return true }
return false
}
// Formatting helpers live in ClawdisKit (`ClawdisCanvasA2UIAction`).
}
@@ -1,225 +0,0 @@
import AppKit
import QuartzCore
final class HoverChromeContainerView: NSView {
private let content: NSView
private let chrome: CanvasChromeOverlayView
private var tracking: NSTrackingArea?
var onClose: (() -> Void)?
init(containing content: NSView) {
self.content = content
self.chrome = CanvasChromeOverlayView(frame: .zero)
super.init(frame: .zero)
self.wantsLayer = true
self.layer?.cornerRadius = 12
self.layer?.masksToBounds = true
self.layer?.backgroundColor = NSColor.windowBackgroundColor.cgColor
self.content.translatesAutoresizingMaskIntoConstraints = false
self.addSubview(self.content)
self.chrome.translatesAutoresizingMaskIntoConstraints = false
self.chrome.alphaValue = 0
self.chrome.onClose = { [weak self] in self?.onClose?() }
self.addSubview(self.chrome)
NSLayoutConstraint.activate([
self.content.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.content.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.content.topAnchor.constraint(equalTo: self.topAnchor),
self.content.bottomAnchor.constraint(equalTo: self.bottomAnchor),
self.chrome.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.chrome.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.chrome.topAnchor.constraint(equalTo: self.topAnchor),
self.chrome.bottomAnchor.constraint(equalTo: self.bottomAnchor),
])
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
override func updateTrackingAreas() {
super.updateTrackingAreas()
if let tracking {
self.removeTrackingArea(tracking)
}
let area = NSTrackingArea(
rect: self.bounds,
options: [.activeAlways, .mouseEnteredAndExited, .inVisibleRect],
owner: self,
userInfo: nil)
self.addTrackingArea(area)
self.tracking = area
}
private final class CanvasDragHandleView: NSView {
override func mouseDown(with event: NSEvent) {
self.window?.performDrag(with: event)
}
override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true }
}
private final class CanvasResizeHandleView: NSView {
private var startPoint: NSPoint = .zero
private var startFrame: NSRect = .zero
override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true }
override func mouseDown(with event: NSEvent) {
guard let window else { return }
_ = window.makeFirstResponder(self)
self.startPoint = NSEvent.mouseLocation
self.startFrame = window.frame
super.mouseDown(with: event)
}
override func mouseDragged(with _: NSEvent) {
guard let window else { return }
let current = NSEvent.mouseLocation
let dx = current.x - self.startPoint.x
let dy = current.y - self.startPoint.y
var frame = self.startFrame
frame.size.width = max(CanvasLayout.minPanelSize.width, frame.size.width + dx)
frame.origin.y += dy
frame.size.height = max(CanvasLayout.minPanelSize.height, frame.size.height - dy)
if let screen = window.screen {
frame = CanvasWindowController.constrainFrame(frame, toVisibleFrame: screen.visibleFrame)
}
window.setFrame(frame, display: true)
}
}
private final class CanvasChromeOverlayView: NSView {
var onClose: (() -> Void)?
private let dragHandle = CanvasDragHandleView(frame: .zero)
private let resizeHandle = CanvasResizeHandleView(frame: .zero)
private final class PassthroughVisualEffectView: NSVisualEffectView {
override func hitTest(_: NSPoint) -> NSView? { nil }
}
private let closeBackground: NSVisualEffectView = {
let v = PassthroughVisualEffectView(frame: .zero)
v.material = .hudWindow
v.blendingMode = .withinWindow
v.state = .active
v.appearance = NSAppearance(named: .vibrantDark)
v.wantsLayer = true
v.layer?.cornerRadius = 10
v.layer?.masksToBounds = true
v.layer?.borderWidth = 1
v.layer?.borderColor = NSColor.white.withAlphaComponent(0.22).cgColor
v.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.28).cgColor
v.layer?.shadowColor = NSColor.black.withAlphaComponent(0.35).cgColor
v.layer?.shadowOpacity = 0.35
v.layer?.shadowRadius = 8
v.layer?.shadowOffset = .zero
return v
}()
private let closeButton: NSButton = {
let cfg = NSImage.SymbolConfiguration(pointSize: 8, weight: .semibold)
let img = NSImage(systemSymbolName: "xmark", accessibilityDescription: "Close")?
.withSymbolConfiguration(cfg)
?? NSImage(size: NSSize(width: 18, height: 18))
let btn = NSButton(image: img, target: nil, action: nil)
btn.isBordered = false
btn.bezelStyle = .regularSquare
btn.imageScaling = .scaleProportionallyDown
btn.contentTintColor = NSColor.white.withAlphaComponent(0.92)
btn.toolTip = "Close"
return btn
}()
override init(frame frameRect: NSRect) {
super.init(frame: frameRect)
self.wantsLayer = true
self.layer?.cornerRadius = 12
self.layer?.masksToBounds = true
self.layer?.borderWidth = 1
self.layer?.borderColor = NSColor.black.withAlphaComponent(0.18).cgColor
self.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.02).cgColor
self.dragHandle.translatesAutoresizingMaskIntoConstraints = false
self.dragHandle.wantsLayer = true
self.dragHandle.layer?.backgroundColor = NSColor.clear.cgColor
self.addSubview(self.dragHandle)
self.resizeHandle.translatesAutoresizingMaskIntoConstraints = false
self.resizeHandle.wantsLayer = true
self.resizeHandle.layer?.backgroundColor = NSColor.clear.cgColor
self.addSubview(self.resizeHandle)
self.closeBackground.translatesAutoresizingMaskIntoConstraints = false
self.addSubview(self.closeBackground)
self.closeButton.translatesAutoresizingMaskIntoConstraints = false
self.closeButton.target = self
self.closeButton.action = #selector(self.handleClose)
self.addSubview(self.closeButton)
NSLayoutConstraint.activate([
self.dragHandle.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.dragHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.dragHandle.topAnchor.constraint(equalTo: self.topAnchor),
self.dragHandle.heightAnchor.constraint(equalToConstant: 30),
self.closeBackground.centerXAnchor.constraint(equalTo: self.closeButton.centerXAnchor),
self.closeBackground.centerYAnchor.constraint(equalTo: self.closeButton.centerYAnchor),
self.closeBackground.widthAnchor.constraint(equalToConstant: 20),
self.closeBackground.heightAnchor.constraint(equalToConstant: 20),
self.closeButton.trailingAnchor.constraint(equalTo: self.trailingAnchor, constant: -8),
self.closeButton.topAnchor.constraint(equalTo: self.topAnchor, constant: 8),
self.closeButton.widthAnchor.constraint(equalToConstant: 16),
self.closeButton.heightAnchor.constraint(equalToConstant: 16),
self.resizeHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.resizeHandle.bottomAnchor.constraint(equalTo: self.bottomAnchor),
self.resizeHandle.widthAnchor.constraint(equalToConstant: 18),
self.resizeHandle.heightAnchor.constraint(equalToConstant: 18),
])
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
override func hitTest(_ point: NSPoint) -> NSView? {
// When the chrome is hidden, do not intercept any mouse events (let the WKWebView receive them).
guard self.alphaValue > 0.02 else { return nil }
if self.closeButton.frame.contains(point) { return self.closeButton }
if self.dragHandle.frame.contains(point) { return self.dragHandle }
if self.resizeHandle.frame.contains(point) { return self.resizeHandle }
return nil
}
@objc private func handleClose() {
self.onClose?()
}
}
override func mouseEntered(with _: NSEvent) {
NSAnimationContext.runAnimationGroup { ctx in
ctx.duration = 0.12
ctx.timingFunction = CAMediaTimingFunction(name: .easeOut)
self.chrome.animator().alphaValue = 1
}
}
override func mouseExited(with _: NSEvent) {
NSAnimationContext.runAnimationGroup { ctx in
ctx.duration = 0.16
ctx.timingFunction = CAMediaTimingFunction(name: .easeOut)
self.chrome.animator().alphaValue = 0
}
}
}
@@ -11,12 +11,6 @@ final class CanvasManager {
private var panelController: CanvasWindowController?
private var panelSessionKey: String?
private var lastAutoA2UIUrl: String?
private var gatewayWatchTask: Task<Void, Never>?
private init() {
self.startGatewayObserver()
}
var onPanelVisibilityChanged: ((Bool) -> Void)?
@@ -56,7 +50,6 @@ final class CanvasManager {
}
controller.presentAnchoredPanel(anchorProvider: anchorProvider)
controller.applyPreferredPlacement(placement)
self.refreshDebugStatus()
// Existing session: only navigate when an explicit target was provided.
if let normalizedTarget {
@@ -67,7 +60,6 @@ final class CanvasManager {
effectiveTarget: normalizedTarget)
}
self.maybeAutoNavigateToA2UIAsync(controller: controller)
return CanvasShowResult(
directory: controller.directoryPath,
target: target,
@@ -101,10 +93,6 @@ final class CanvasManager {
Self.logger.debug("showDetailed showCanvas effectiveTarget=\(effectiveTarget, privacy: .public)")
controller.showCanvas(path: effectiveTarget)
Self.logger.debug("showDetailed showCanvas done")
if normalizedTarget == nil {
self.maybeAutoNavigateToA2UIAsync(controller: controller)
}
self.refreshDebugStatus()
return self.makeShowResult(
directory: controller.directoryPath,
@@ -136,63 +124,6 @@ final class CanvasManager {
return try await controller.snapshot(to: outPath)
}
// MARK: - Gateway A2UI auto-nav
private func startGatewayObserver() {
self.gatewayWatchTask?.cancel()
self.gatewayWatchTask = Task { [weak self] in
guard let self else { return }
let stream = await GatewayConnection.shared.subscribe(bufferingNewest: 1)
for await push in stream {
self.handleGatewayPush(push)
}
}
}
private func handleGatewayPush(_ push: GatewayPush) {
guard case let .snapshot(snapshot) = push else { return }
let a2uiUrl = Self.resolveA2UIHostUrl(from: snapshot.canvashosturl)
guard let controller = self.panelController else { return }
self.maybeAutoNavigateToA2UI(controller: controller, a2uiUrl: a2uiUrl)
}
private func maybeAutoNavigateToA2UIAsync(controller: CanvasWindowController) {
Task { [weak self] in
guard let self else { return }
let a2uiUrl = await self.resolveA2UIHostUrl()
await MainActor.run {
guard self.panelController === controller else { return }
self.maybeAutoNavigateToA2UI(controller: controller, a2uiUrl: a2uiUrl)
}
}
}
private func maybeAutoNavigateToA2UI(controller: CanvasWindowController, a2uiUrl: String?) {
guard let a2uiUrl else { return }
guard controller.shouldAutoNavigateToA2UI(lastAutoTarget: self.lastAutoA2UIUrl) else { return }
controller.load(target: a2uiUrl)
self.lastAutoA2UIUrl = a2uiUrl
}
private func resolveA2UIHostUrl() async -> String? {
let raw = await GatewayConnection.shared.canvasHostUrl()
return Self.resolveA2UIHostUrl(from: raw)
}
func refreshDebugStatus() {
guard let controller = self.panelController else { return }
let enabled = AppStateStore.shared.debugPaneEnabled
let title = GatewayProcessManager.shared.status.label
let subtitle = AppStateStore.shared.connectionMode.rawValue
controller.updateDebugStatus(enabled: enabled, title: title, subtitle: subtitle)
}
private static func resolveA2UIHostUrl(from raw: String?) -> String? {
let trimmed = raw?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
return base.appendingPathComponent("__clawdis__/a2ui/").absoluteString + "?platform=macos"
}
// MARK: - Anchoring
private static func mouseAnchorProvider() -> NSRect? {
@@ -240,20 +240,3 @@ final class CanvasSchemeHandler: NSObject, WKURLSchemeHandler {
}
}
}
#if DEBUG
extension CanvasSchemeHandler {
func _testResponse(for url: URL) -> (mime: String, data: Data) {
let response = self.response(for: url)
return (response.mime, response.data)
}
func _testResolveFileURL(sessionRoot: URL, requestPath: String) -> URL? {
self.resolveFileURL(sessionRoot: sessionRoot, requestPath: requestPath)
}
func _testTextEncodingName(for mimeType: String) -> String? {
self.textEncodingName(forMimeType: mimeType)
}
}
#endif
+937 -2
View File
@@ -1,8 +1,14 @@
import AppKit
import ClawdisIPC
import ClawdisKit
import Foundation
import OSLog
import QuartzCore
import WebKit
let canvasWindowLogger = Logger(subsystem: "com.steipete.clawdis", category: "Canvas")
private let canvasWindowLogger = Logger(subsystem: "com.steipete.clawdis", category: "Canvas")
enum CanvasLayout {
private enum CanvasLayout {
static let panelSize = NSSize(width: 520, height: 680)
static let windowSize = NSSize(width: 1120, height: 840)
static let anchorPadding: CGFloat = 8
@@ -24,3 +30,932 @@ enum CanvasPresentation {
return false
}
}
@MainActor
final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NSWindowDelegate {
private let sessionKey: String
private let root: URL
private let sessionDir: URL
private let schemeHandler: CanvasSchemeHandler
private let webView: WKWebView
private var a2uiActionMessageHandler: CanvasA2UIActionMessageHandler?
private let watcher: CanvasFileWatcher
private let container: HoverChromeContainerView
let presentation: CanvasPresentation
private var preferredPlacement: CanvasPlacement?
var onVisibilityChanged: ((Bool) -> Void)?
init(sessionKey: String, root: URL, presentation: CanvasPresentation) throws {
self.sessionKey = sessionKey
self.root = root
self.presentation = presentation
canvasWindowLogger.debug("CanvasWindowController init start session=\(sessionKey, privacy: .public)")
let safeSessionKey = CanvasWindowController.sanitizeSessionKey(sessionKey)
canvasWindowLogger.debug("CanvasWindowController init sanitized session=\(safeSessionKey, privacy: .public)")
self.sessionDir = root.appendingPathComponent(safeSessionKey, isDirectory: true)
try FileManager.default.createDirectory(at: self.sessionDir, withIntermediateDirectories: true)
canvasWindowLogger.debug("CanvasWindowController init session dir ready")
self.schemeHandler = CanvasSchemeHandler(root: root)
canvasWindowLogger.debug("CanvasWindowController init scheme handler ready")
let config = WKWebViewConfiguration()
config.userContentController = WKUserContentController()
config.preferences.isElementFullscreenEnabled = true
config.preferences.setValue(true, forKey: "developerExtrasEnabled")
canvasWindowLogger.debug("CanvasWindowController init config ready")
config.setURLSchemeHandler(self.schemeHandler, forURLScheme: CanvasScheme.scheme)
canvasWindowLogger.debug("CanvasWindowController init scheme handler installed")
// Bridge A2UI "a2uiaction" DOM events back into the native agent loop.
//
// Prefer WKScriptMessageHandler when WebKit exposes it, otherwise fall back to an unattended deep link
// (includes the app-generated key so it won't prompt).
canvasWindowLogger.debug("CanvasWindowController init building A2UI bridge script")
let deepLinkKey = DeepLinkHandler.currentCanvasKey()
let injectedSessionKey = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "main"
let bridgeScript = """
(() => {
try {
if (location.protocol !== '\(CanvasScheme.scheme):') return;
if (globalThis.__clawdisA2UIBridgeInstalled) return;
globalThis.__clawdisA2UIBridgeInstalled = true;
const deepLinkKey = \(Self.jsStringLiteral(deepLinkKey));
const sessionKey = \(Self.jsStringLiteral(injectedSessionKey));
const machineName = \(Self.jsStringLiteral(InstanceIdentity.displayName));
const instanceId = \(Self.jsStringLiteral(InstanceIdentity.instanceId));
globalThis.addEventListener('a2uiaction', (evt) => {
try {
const payload = evt?.detail ?? evt?.payload ?? null;
if (!payload || payload.eventType !== 'a2ui.action') return;
const action = payload.action ?? null;
const name = action?.name ?? '';
if (!name) return;
const context = Array.isArray(action?.context) ? action.context : [];
const userAction = {
id: (globalThis.crypto?.randomUUID?.() ?? String(Date.now())),
name,
surfaceId: payload.surfaceId ?? 'main',
sourceComponentId: payload.sourceComponentId ?? '',
dataContextPath: payload.dataContextPath ?? '',
timestamp: new Date().toISOString(),
...(context.length ? { context } : {}),
};
const handler = globalThis.webkit?.messageHandlers?.clawdisCanvasA2UIAction;
// If the bundled A2UI shell is present, let it forward actions so we keep its richer
// context resolution (data model path lookups, surface detection, etc.).
const hasBundledA2UIHost = !!globalThis.clawdisA2UI || !!document.querySelector('clawdis-a2ui-host');
if (hasBundledA2UIHost && handler?.postMessage) return;
// Otherwise, forward directly when possible.
if (!hasBundledA2UIHost && handler?.postMessage) {
handler.postMessage({ userAction });
return;
}
const ctx = userAction.context ? (' ctx=' + JSON.stringify(userAction.context)) : '';
const message =
'CANVAS_A2UI action=' + userAction.name +
' session=' + sessionKey +
' surface=' + userAction.surfaceId +
' component=' + (userAction.sourceComponentId || '-') +
' host=' + machineName.replace(/\\s+/g, '_') +
' instance=' + instanceId +
ctx +
' default=update_canvas';
const params = new URLSearchParams();
params.set('message', message);
params.set('sessionKey', sessionKey);
params.set('thinking', 'low');
params.set('deliver', 'false');
params.set('channel', 'last');
params.set('key', deepLinkKey);
location.href = 'clawdis://agent?' + params.toString();
} catch {}
}, true);
} catch {}
})();
"""
config.userContentController.addUserScript(
WKUserScript(source: bridgeScript, injectionTime: .atDocumentStart, forMainFrameOnly: true))
canvasWindowLogger.debug("CanvasWindowController init A2UI bridge installed")
canvasWindowLogger.debug("CanvasWindowController init creating WKWebView")
self.webView = WKWebView(frame: .zero, configuration: config)
// Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays.
self.webView.setValue(true, forKey: "drawsBackground")
let sessionDir = self.sessionDir
let webView = self.webView
self.watcher = CanvasFileWatcher(url: sessionDir) { [weak webView] in
Task { @MainActor in
guard let webView else { return }
// Only auto-reload when we are showing local canvas content.
guard webView.url?.scheme == CanvasScheme.scheme else { return }
let path = webView.url?.path ?? ""
if path == "/" || path.isEmpty {
let indexA = sessionDir.appendingPathComponent("index.html", isDirectory: false)
let indexB = sessionDir.appendingPathComponent("index.htm", isDirectory: false)
if !FileManager.default.fileExists(atPath: indexA.path),
!FileManager.default.fileExists(atPath: indexB.path)
{
return
}
}
webView.reload()
}
}
self.container = HoverChromeContainerView(containing: self.webView)
let window = Self.makeWindow(for: presentation, contentView: self.container)
canvasWindowLogger.debug("CanvasWindowController init makeWindow done")
super.init(window: window)
let handler = CanvasA2UIActionMessageHandler(sessionKey: sessionKey)
self.a2uiActionMessageHandler = handler
self.webView.configuration.userContentController.add(handler, name: CanvasA2UIActionMessageHandler.messageName)
self.webView.navigationDelegate = self
self.window?.delegate = self
self.container.onClose = { [weak self] in
self?.hideCanvas()
}
self.watcher.start()
canvasWindowLogger.debug("CanvasWindowController init done")
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
@MainActor deinit {
self.webView.configuration.userContentController
.removeScriptMessageHandler(forName: CanvasA2UIActionMessageHandler.messageName)
self.watcher.stop()
}
func applyPreferredPlacement(_ placement: CanvasPlacement?) {
self.preferredPlacement = placement
}
func showCanvas(path: String? = nil) {
if case let .panel(anchorProvider) = self.presentation {
self.presentAnchoredPanel(anchorProvider: anchorProvider)
if let path {
self.load(target: path)
}
return
}
self.showWindow(nil)
self.window?.makeKeyAndOrderFront(nil)
NSApp.activate(ignoringOtherApps: true)
if let path {
self.load(target: path)
}
self.onVisibilityChanged?(true)
}
func hideCanvas() {
if case .panel = self.presentation {
self.persistFrameIfPanel()
}
self.window?.orderOut(nil)
self.onVisibilityChanged?(false)
}
func load(target: String) {
let trimmed = target.trimmingCharacters(in: .whitespacesAndNewlines)
if let url = URL(string: trimmed), let scheme = url.scheme?.lowercased() {
if scheme == "https" || scheme == "http" {
canvasWindowLogger.debug("canvas load url \(url.absoluteString, privacy: .public)")
self.webView.load(URLRequest(url: url))
return
}
if scheme == "file" {
canvasWindowLogger.debug("canvas load file \(url.absoluteString, privacy: .public)")
self.loadFile(url)
return
}
}
// Convenience: absolute file paths resolve as local files when they exist.
// (Avoid treating Canvas routes like "/" as filesystem paths.)
if trimmed.hasPrefix("/") {
var isDir: ObjCBool = false
if FileManager.default.fileExists(atPath: trimmed, isDirectory: &isDir), !isDir.boolValue {
let url = URL(fileURLWithPath: trimmed)
canvasWindowLogger.debug("canvas load file \(url.absoluteString, privacy: .public)")
self.loadFile(url)
return
}
}
guard let url = CanvasScheme.makeURL(
session: CanvasWindowController.sanitizeSessionKey(self.sessionKey),
path: trimmed)
else {
canvasWindowLogger
.error(
"invalid canvas url session=\(self.sessionKey, privacy: .public) path=\(trimmed, privacy: .public)")
return
}
canvasWindowLogger.debug("canvas load canvas \(url.absoluteString, privacy: .public)")
self.webView.load(URLRequest(url: url))
}
private func loadFile(_ url: URL) {
let fileURL = url.isFileURL ? url : URL(fileURLWithPath: url.path)
let accessDir = fileURL.deletingLastPathComponent()
self.webView.loadFileURL(fileURL, allowingReadAccessTo: accessDir)
}
func eval(javaScript: String) async throws -> String {
try await withCheckedThrowingContinuation { cont in
self.webView.evaluateJavaScript(javaScript) { result, error in
if let error {
cont.resume(throwing: error)
return
}
if let result {
cont.resume(returning: String(describing: result))
} else {
cont.resume(returning: "")
}
}
}
}
func snapshot(to outPath: String?) async throws -> String {
let image: NSImage = try await withCheckedThrowingContinuation { cont in
self.webView.takeSnapshot(with: nil) { image, error in
if let error {
cont.resume(throwing: error)
return
}
guard let image else {
cont.resume(throwing: NSError(domain: "Canvas", code: 11, userInfo: [
NSLocalizedDescriptionKey: "snapshot returned nil image",
]))
return
}
cont.resume(returning: image)
}
}
guard let tiff = image.tiffRepresentation,
let rep = NSBitmapImageRep(data: tiff),
let png = rep.representation(using: .png, properties: [:])
else {
throw NSError(domain: "Canvas", code: 12, userInfo: [
NSLocalizedDescriptionKey: "failed to encode png",
])
}
let path: String
if let outPath, !outPath.isEmpty {
path = outPath
} else {
let ts = Int(Date().timeIntervalSince1970)
path = "/tmp/clawdis-canvas-\(CanvasWindowController.sanitizeSessionKey(self.sessionKey))-\(ts).png"
}
try png.write(to: URL(fileURLWithPath: path), options: [.atomic])
return path
}
var directoryPath: String {
self.sessionDir.path
}
// MARK: - Window
private static func makeWindow(for presentation: CanvasPresentation, contentView: NSView) -> NSWindow {
switch presentation {
case .window:
let window = NSWindow(
contentRect: NSRect(origin: .zero, size: CanvasLayout.windowSize),
styleMask: [.titled, .closable, .resizable, .miniaturizable],
backing: .buffered,
defer: false)
window.title = "Clawdis Canvas"
window.isReleasedWhenClosed = false
window.contentView = contentView
window.center()
window.minSize = NSSize(width: 880, height: 680)
return window
case .panel:
let panel = CanvasPanel(
contentRect: NSRect(origin: .zero, size: CanvasLayout.panelSize),
styleMask: [.borderless, .resizable],
backing: .buffered,
defer: false)
// Keep Canvas below the Voice Wake overlay panel.
panel.level = NSWindow.Level(rawValue: NSWindow.Level.statusBar.rawValue - 1)
panel.hasShadow = true
panel.isMovable = false
panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary]
panel.titleVisibility = .hidden
panel.titlebarAppearsTransparent = true
panel.backgroundColor = .clear
panel.isOpaque = false
panel.contentView = contentView
panel.becomesKeyOnlyIfNeeded = true
panel.hidesOnDeactivate = false
panel.minSize = CanvasLayout.minPanelSize
return panel
}
}
func presentAnchoredPanel(anchorProvider: @escaping () -> NSRect?) {
guard case .panel = self.presentation, let window else { return }
self.repositionPanel(using: anchorProvider)
window.makeKeyAndOrderFront(nil)
NSApp.activate(ignoringOtherApps: true)
window.makeFirstResponder(self.webView)
VoiceWakeOverlayController.shared.bringToFrontIfVisible()
self.onVisibilityChanged?(true)
}
private func repositionPanel(using anchorProvider: () -> NSRect?) {
guard let panel = self.window else { return }
let anchor = anchorProvider()
let targetScreen = Self.screen(forAnchor: anchor)
?? Self.screenContainingMouseCursor()
?? panel.screen
?? NSScreen.main
?? NSScreen.screens.first
let restored = Self.loadRestoredFrame(sessionKey: self.sessionKey)
let restoredIsValid = if let restored, let targetScreen {
Self.isFrameMeaningfullyVisible(restored, on: targetScreen)
} else {
restored != nil
}
var frame = if let restored, restoredIsValid {
restored
} else {
Self.defaultTopRightFrame(panel: panel, screen: targetScreen)
}
// Apply agent placement as partial overrides:
// - If agent provides x/y, override origin.
// - If agent provides width/height, override size.
// - If agent provides only size, keep the remembered origin.
if let placement = self.preferredPlacement {
if let x = placement.x { frame.origin.x = x }
if let y = placement.y { frame.origin.y = y }
if let w = placement.width { frame.size.width = max(CanvasLayout.minPanelSize.width, CGFloat(w)) }
if let h = placement.height { frame.size.height = max(CanvasLayout.minPanelSize.height, CGFloat(h)) }
}
self.setPanelFrame(frame, on: targetScreen)
}
private static func defaultTopRightFrame(panel: NSWindow, screen: NSScreen?) -> NSRect {
let w = max(CanvasLayout.minPanelSize.width, panel.frame.width)
let h = max(CanvasLayout.minPanelSize.height, panel.frame.height)
return WindowPlacement.topRightFrame(
size: NSSize(width: w, height: h),
padding: CanvasLayout.defaultPadding,
on: screen)
}
private func setPanelFrame(_ frame: NSRect, on screen: NSScreen?) {
guard let panel = self.window else { return }
guard let s = screen ?? panel.screen ?? NSScreen.main ?? NSScreen.screens.first else {
panel.setFrame(frame, display: false)
self.persistFrameIfPanel()
return
}
let constrained = Self.constrainFrame(frame, toVisibleFrame: s.visibleFrame)
panel.setFrame(constrained, display: false)
self.persistFrameIfPanel()
}
private static func screen(forAnchor anchor: NSRect?) -> NSScreen? {
guard let anchor else { return nil }
let center = NSPoint(x: anchor.midX, y: anchor.midY)
return NSScreen.screens.first { screen in
screen.frame.contains(anchor.origin) || screen.frame.contains(center)
}
}
private static func screenContainingMouseCursor() -> NSScreen? {
let point = NSEvent.mouseLocation
return NSScreen.screens.first { $0.frame.contains(point) }
}
private static func isFrameMeaningfullyVisible(_ frame: NSRect, on screen: NSScreen) -> Bool {
frame.intersects(screen.visibleFrame.insetBy(dx: 12, dy: 12))
}
fileprivate static func constrainFrame(_ frame: NSRect, toVisibleFrame bounds: NSRect) -> NSRect {
if bounds == .zero { return frame }
var next = frame
next.size.width = min(max(CanvasLayout.minPanelSize.width, next.size.width), bounds.width)
next.size.height = min(max(CanvasLayout.minPanelSize.height, next.size.height), bounds.height)
let maxX = bounds.maxX - next.size.width
let maxY = bounds.maxY - next.size.height
next.origin.x = maxX >= bounds.minX ? min(max(next.origin.x, bounds.minX), maxX) : bounds.minX
next.origin.y = maxY >= bounds.minY ? min(max(next.origin.y, bounds.minY), maxY) : bounds.minY
next.origin.x = round(next.origin.x)
next.origin.y = round(next.origin.y)
return next
}
// MARK: - WKNavigationDelegate
@MainActor
func webView(
_: WKWebView,
decidePolicyFor navigationAction: WKNavigationAction,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
{
guard let url = navigationAction.request.url else {
decisionHandler(.cancel)
return
}
let scheme = url.scheme?.lowercased()
// Deep links: allow local Canvas content to invoke the agent without bouncing through NSWorkspace.
if scheme == "clawdis" {
if self.webView.url?.scheme == CanvasScheme.scheme {
Task { await DeepLinkHandler.shared.handle(url: url) }
} else {
canvasWindowLogger
.debug("ignoring deep link from non-canvas page \(url.absoluteString, privacy: .public)")
}
decisionHandler(.cancel)
return
}
// Keep web content inside the panel when reasonable.
// `about:blank` and friends are common internal navigations for WKWebView; never send them to NSWorkspace.
if scheme == CanvasScheme.scheme
|| scheme == "https"
|| scheme == "http"
|| scheme == "about"
|| scheme == "blob"
|| scheme == "data"
|| scheme == "javascript"
{
decisionHandler(.allow)
return
}
// Only open external URLs when there is a registered handler, otherwise macOS will show a confusing
// "There is no application set to open the URL ..." alert (e.g. for about:blank).
if let appURL = NSWorkspace.shared.urlForApplication(toOpen: url) {
NSWorkspace.shared.open(
[url],
withApplicationAt: appURL,
configuration: NSWorkspace.OpenConfiguration(),
completionHandler: nil)
} else {
canvasWindowLogger.debug("no application to open url \(url.absoluteString, privacy: .public)")
}
decisionHandler(.cancel)
}
// MARK: - NSWindowDelegate
func windowWillClose(_: Notification) {
self.onVisibilityChanged?(false)
}
func windowDidMove(_: Notification) {
self.persistFrameIfPanel()
}
func windowDidEndLiveResize(_: Notification) {
self.persistFrameIfPanel()
}
private func persistFrameIfPanel() {
guard case .panel = self.presentation, let window else { return }
Self.storeRestoredFrame(window.frame, sessionKey: self.sessionKey)
}
// MARK: - Helpers
private static func sanitizeSessionKey(_ key: String) -> String {
let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty { return "main" }
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+")
let scalars = trimmed.unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" }
return String(scalars)
}
private static func jsStringLiteral(_ value: String) -> String {
let data = try? JSONEncoder().encode(value)
return data.flatMap { String(data: $0, encoding: .utf8) } ?? "\"\""
}
private static func storedFrameDefaultsKey(sessionKey: String) -> String {
"clawdis.canvas.frame.\(self.sanitizeSessionKey(sessionKey))"
}
private static func loadRestoredFrame(sessionKey: String) -> NSRect? {
let key = self.storedFrameDefaultsKey(sessionKey: sessionKey)
guard let arr = UserDefaults.standard.array(forKey: key) as? [Double], arr.count == 4 else { return nil }
let rect = NSRect(x: arr[0], y: arr[1], width: arr[2], height: arr[3])
if rect.width < CanvasLayout.minPanelSize.width || rect.height < CanvasLayout.minPanelSize.height { return nil }
return rect
}
private static func storeRestoredFrame(_ frame: NSRect, sessionKey: String) {
let key = self.storedFrameDefaultsKey(sessionKey: sessionKey)
UserDefaults.standard.set(
[Double(frame.origin.x), Double(frame.origin.y), Double(frame.size.width), Double(frame.size.height)],
forKey: key)
}
}
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "clawdisCanvasA2UIAction"
private let sessionKey: String
init(sessionKey: String) {
self.sessionKey = sessionKey
super.init()
}
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
guard message.name == Self.messageName else { return }
// Only accept actions from local Canvas content (not arbitrary web pages).
guard let webView = message.webView, let url = webView.url else { return }
if url.scheme == CanvasScheme.scheme {
// ok
} else if Self.isLocalNetworkCanvasURL(url) {
// ok
} else {
return
}
let body: [String: Any] = {
if let dict = message.body as? [String: Any] { return dict }
if let dict = message.body as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !body.isEmpty else { return }
let userActionAny = body["userAction"] ?? body
let userAction: [String: Any] = {
if let dict = userActionAny as? [String: Any] { return dict }
if let dict = userActionAny as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !userAction.isEmpty else { return }
guard let name = userAction["name"] as? String, !name.isEmpty else { return }
let actionId =
(userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
?? UUID().uuidString
canvasWindowLogger.info("A2UI action \(name, privacy: .public) session=\(self.sessionKey, privacy: .public)")
let surfaceId = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
.nonEmpty ?? "main"
let sourceComponentId = (userAction["sourceComponentId"] as? String)?
.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "-"
let instanceId = InstanceIdentity.instanceId.lowercased()
let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"])
// Token-efficient and unambiguous. The agent should treat this as a UI event and (by default) update Canvas.
let text = ClawdisCanvasA2UIAction.formatAgentMessage(
actionName: name,
sessionKey: self.sessionKey,
surfaceId: surfaceId,
sourceComponentId: sourceComponentId,
host: InstanceIdentity.displayName,
instanceId: instanceId,
contextJSON: contextJSON)
Task { [weak webView] in
if AppStateStore.shared.connectionMode == .local {
GatewayProcessManager.shared.setActive(true)
}
let result = await GatewayConnection.shared.sendAgent(GatewayAgentInvocation(
message: text,
sessionKey: self.sessionKey,
thinking: "low",
deliver: false,
to: nil,
channel: .last,
idempotencyKey: actionId))
await MainActor.run {
guard let webView else { return }
let js = ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(
actionId: actionId,
ok: result.ok,
error: result.error)
webView.evaluateJavaScript(js) { _, _ in }
}
if !result.ok {
canvasWindowLogger.error(
"""
A2UI action send failed name=\(name, privacy: .public) \
error=\(result.error ?? "unknown", privacy: .public)
""")
}
}
}
private static func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return false
}
guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else {
return false
}
if host == "localhost" { return true }
if host.hasSuffix(".local") { return true }
if host.hasSuffix(".ts.net") { return true }
if host.hasSuffix(".tailscale.net") { return true }
if !host.contains("."), !host.contains(":") { return true }
if let ipv4 = Self.parseIPv4(host) {
return Self.isLocalNetworkIPv4(ipv4)
}
return false
}
private static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? {
let parts = host.split(separator: ".", omittingEmptySubsequences: false)
guard parts.count == 4 else { return nil }
let bytes: [UInt8] = parts.compactMap { UInt8($0) }
guard bytes.count == 4 else { return nil }
return (bytes[0], bytes[1], bytes[2], bytes[3])
}
private static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool {
let (a, b, _, _) = ip
if a == 10 { return true }
if a == 172, (16...31).contains(Int(b)) { return true }
if a == 192, b == 168 { return true }
if a == 127 { return true }
if a == 169, b == 254 { return true }
if a == 100, (64...127).contains(Int(b)) { return true }
return false
}
// Formatting helpers live in ClawdisKit (`ClawdisCanvasA2UIAction`).
}
// MARK: - Hover chrome container
private final class HoverChromeContainerView: NSView {
private let content: NSView
private let chrome: CanvasChromeOverlayView
private var tracking: NSTrackingArea?
var onClose: (() -> Void)?
init(containing content: NSView) {
self.content = content
self.chrome = CanvasChromeOverlayView(frame: .zero)
super.init(frame: .zero)
self.wantsLayer = true
self.layer?.cornerRadius = 12
self.layer?.masksToBounds = true
self.layer?.backgroundColor = NSColor.windowBackgroundColor.cgColor
self.content.translatesAutoresizingMaskIntoConstraints = false
self.addSubview(self.content)
self.chrome.translatesAutoresizingMaskIntoConstraints = false
self.chrome.alphaValue = 0
self.chrome.onClose = { [weak self] in self?.onClose?() }
self.addSubview(self.chrome)
NSLayoutConstraint.activate([
self.content.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.content.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.content.topAnchor.constraint(equalTo: self.topAnchor),
self.content.bottomAnchor.constraint(equalTo: self.bottomAnchor),
self.chrome.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.chrome.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.chrome.topAnchor.constraint(equalTo: self.topAnchor),
self.chrome.bottomAnchor.constraint(equalTo: self.bottomAnchor),
])
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
override func updateTrackingAreas() {
super.updateTrackingAreas()
if let tracking {
self.removeTrackingArea(tracking)
}
let area = NSTrackingArea(
rect: self.bounds,
options: [.activeAlways, .mouseEnteredAndExited, .inVisibleRect],
owner: self,
userInfo: nil)
self.addTrackingArea(area)
self.tracking = area
}
private final class CanvasDragHandleView: NSView {
override func mouseDown(with event: NSEvent) {
self.window?.performDrag(with: event)
}
override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true }
}
private final class CanvasResizeHandleView: NSView {
private var startPoint: NSPoint = .zero
private var startFrame: NSRect = .zero
override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true }
override func mouseDown(with event: NSEvent) {
guard let window else { return }
_ = window.makeFirstResponder(self)
self.startPoint = NSEvent.mouseLocation
self.startFrame = window.frame
super.mouseDown(with: event)
}
override func mouseDragged(with _: NSEvent) {
guard let window else { return }
let current = NSEvent.mouseLocation
let dx = current.x - self.startPoint.x
let dy = current.y - self.startPoint.y
var frame = self.startFrame
frame.size.width = max(CanvasLayout.minPanelSize.width, frame.size.width + dx)
frame.origin.y += dy
frame.size.height = max(CanvasLayout.minPanelSize.height, frame.size.height - dy)
if let screen = window.screen {
frame = CanvasWindowController.constrainFrame(frame, toVisibleFrame: screen.visibleFrame)
}
window.setFrame(frame, display: true)
}
}
private final class CanvasChromeOverlayView: NSView {
var onClose: (() -> Void)?
private let dragHandle = CanvasDragHandleView(frame: .zero)
private let resizeHandle = CanvasResizeHandleView(frame: .zero)
private final class PassthroughVisualEffectView: NSVisualEffectView {
override func hitTest(_: NSPoint) -> NSView? { nil }
}
private let closeBackground: NSVisualEffectView = {
let v = PassthroughVisualEffectView(frame: .zero)
v.material = .hudWindow
v.blendingMode = .withinWindow
v.state = .active
v.appearance = NSAppearance(named: .vibrantDark)
v.wantsLayer = true
v.layer?.cornerRadius = 10
v.layer?.masksToBounds = true
v.layer?.borderWidth = 1
v.layer?.borderColor = NSColor.white.withAlphaComponent(0.22).cgColor
v.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.28).cgColor
v.layer?.shadowColor = NSColor.black.withAlphaComponent(0.35).cgColor
v.layer?.shadowOpacity = 0.35
v.layer?.shadowRadius = 8
v.layer?.shadowOffset = .zero
return v
}()
private let closeButton: NSButton = {
let cfg = NSImage.SymbolConfiguration(pointSize: 8, weight: .semibold)
let img = NSImage(systemSymbolName: "xmark", accessibilityDescription: "Close")?
.withSymbolConfiguration(cfg)
?? NSImage(size: NSSize(width: 18, height: 18))
let btn = NSButton(image: img, target: nil, action: nil)
btn.isBordered = false
btn.bezelStyle = .regularSquare
btn.imageScaling = .scaleProportionallyDown
btn.contentTintColor = NSColor.white.withAlphaComponent(0.92)
btn.toolTip = "Close"
return btn
}()
override init(frame frameRect: NSRect) {
super.init(frame: frameRect)
self.wantsLayer = true
self.layer?.cornerRadius = 12
self.layer?.masksToBounds = true
self.layer?.borderWidth = 1
self.layer?.borderColor = NSColor.black.withAlphaComponent(0.18).cgColor
self.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.02).cgColor
self.dragHandle.translatesAutoresizingMaskIntoConstraints = false
self.dragHandle.wantsLayer = true
self.dragHandle.layer?.backgroundColor = NSColor.clear.cgColor
self.addSubview(self.dragHandle)
self.resizeHandle.translatesAutoresizingMaskIntoConstraints = false
self.resizeHandle.wantsLayer = true
self.resizeHandle.layer?.backgroundColor = NSColor.clear.cgColor
self.addSubview(self.resizeHandle)
self.closeBackground.translatesAutoresizingMaskIntoConstraints = false
self.addSubview(self.closeBackground)
self.closeButton.translatesAutoresizingMaskIntoConstraints = false
self.closeButton.target = self
self.closeButton.action = #selector(self.handleClose)
self.addSubview(self.closeButton)
NSLayoutConstraint.activate([
self.dragHandle.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.dragHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.dragHandle.topAnchor.constraint(equalTo: self.topAnchor),
self.dragHandle.heightAnchor.constraint(equalToConstant: 30),
self.closeBackground.centerXAnchor.constraint(equalTo: self.closeButton.centerXAnchor),
self.closeBackground.centerYAnchor.constraint(equalTo: self.closeButton.centerYAnchor),
self.closeBackground.widthAnchor.constraint(equalToConstant: 20),
self.closeBackground.heightAnchor.constraint(equalToConstant: 20),
self.closeButton.trailingAnchor.constraint(equalTo: self.trailingAnchor, constant: -8),
self.closeButton.topAnchor.constraint(equalTo: self.topAnchor, constant: 8),
self.closeButton.widthAnchor.constraint(equalToConstant: 16),
self.closeButton.heightAnchor.constraint(equalToConstant: 16),
self.resizeHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.resizeHandle.bottomAnchor.constraint(equalTo: self.bottomAnchor),
self.resizeHandle.widthAnchor.constraint(equalToConstant: 18),
self.resizeHandle.heightAnchor.constraint(equalToConstant: 18),
])
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
override func hitTest(_ point: NSPoint) -> NSView? {
// When the chrome is hidden, do not intercept any mouse events (let the WKWebView receive them).
guard self.alphaValue > 0.02 else { return nil }
if self.closeButton.frame.contains(point) { return self.closeButton }
if self.dragHandle.frame.contains(point) { return self.dragHandle }
if self.resizeHandle.frame.contains(point) { return self.resizeHandle }
return nil
}
@objc private func handleClose() {
self.onClose?()
}
}
override func mouseEntered(with _: NSEvent) {
NSAnimationContext.runAnimationGroup { ctx in
ctx.duration = 0.12
ctx.timingFunction = CAMediaTimingFunction(name: .easeOut)
self.chrome.animator().alphaValue = 1
}
}
override func mouseExited(with _: NSEvent) {
NSAnimationContext.runAnimationGroup { ctx in
ctx.duration = 0.16
ctx.timingFunction = CAMediaTimingFunction(name: .easeOut)
self.chrome.animator().alphaValue = 0
}
}
}
@@ -1,43 +0,0 @@
import AppKit
import Foundation
extension CanvasWindowController {
// MARK: - Helpers
static func sanitizeSessionKey(_ key: String) -> String {
let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty { return "main" }
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+")
let scalars = trimmed.unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" }
return String(scalars)
}
static func jsStringLiteral(_ value: String) -> String {
let data = try? JSONEncoder().encode(value)
return data.flatMap { String(data: $0, encoding: .utf8) } ?? "\"\""
}
static func jsOptionalStringLiteral(_ value: String?) -> String {
guard let value else { return "null" }
return Self.jsStringLiteral(value)
}
static func storedFrameDefaultsKey(sessionKey: String) -> String {
"clawdis.canvas.frame.\(self.sanitizeSessionKey(sessionKey))"
}
static func loadRestoredFrame(sessionKey: String) -> NSRect? {
let key = self.storedFrameDefaultsKey(sessionKey: sessionKey)
guard let arr = UserDefaults.standard.array(forKey: key) as? [Double], arr.count == 4 else { return nil }
let rect = NSRect(x: arr[0], y: arr[1], width: arr[2], height: arr[3])
if rect.width < CanvasLayout.minPanelSize.width || rect.height < CanvasLayout.minPanelSize.height { return nil }
return rect
}
static func storeRestoredFrame(_ frame: NSRect, sessionKey: String) {
let key = self.storedFrameDefaultsKey(sessionKey: sessionKey)
UserDefaults.standard.set(
[Double(frame.origin.x), Double(frame.origin.y), Double(frame.size.width), Double(frame.size.height)],
forKey: key)
}
}
@@ -1,62 +0,0 @@
import AppKit
import WebKit
extension CanvasWindowController {
// MARK: - WKNavigationDelegate
@MainActor
func webView(
_: WKWebView,
decidePolicyFor navigationAction: WKNavigationAction,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
{
guard let url = navigationAction.request.url else {
decisionHandler(.cancel)
return
}
let scheme = url.scheme?.lowercased()
// Deep links: allow local Canvas content to invoke the agent without bouncing through NSWorkspace.
if scheme == "clawdis" {
if self.webView.url?.scheme == CanvasScheme.scheme {
Task { await DeepLinkHandler.shared.handle(url: url) }
} else {
canvasWindowLogger
.debug("ignoring deep link from non-canvas page \(url.absoluteString, privacy: .public)")
}
decisionHandler(.cancel)
return
}
// Keep web content inside the panel when reasonable.
// `about:blank` and friends are common internal navigations for WKWebView; never send them to NSWorkspace.
if scheme == CanvasScheme.scheme
|| scheme == "https"
|| scheme == "http"
|| scheme == "about"
|| scheme == "blob"
|| scheme == "data"
|| scheme == "javascript"
{
decisionHandler(.allow)
return
}
// Only open external URLs when there is a registered handler, otherwise macOS will show a confusing
// "There is no application set to open the URL ..." alert (e.g. for about:blank).
if let appURL = NSWorkspace.shared.urlForApplication(toOpen: url) {
NSWorkspace.shared.open(
[url],
withApplicationAt: appURL,
configuration: NSWorkspace.OpenConfiguration(),
completionHandler: nil)
} else {
canvasWindowLogger.debug("no application to open url \(url.absoluteString, privacy: .public)")
}
decisionHandler(.cancel)
}
func webView(_: WKWebView, didFinish _: WKNavigation?) {
self.applyDebugStatusIfNeeded()
}
}
Loaded 100 of 575 files, more files were not shown because too many files have changed in this diff. Show more