Compare commits

..
Author SHA1 Message Date
Peter Steinberger 2401abe17e docs: update changelog for codesign fix 2026-01-01 17:30:22 +01:00
Peter Steinberger 56ea6b6e43 fix: align tool schemas and health snapshot 2026-01-01 17:30:19 +01:00
Peter Steinberger 04691ed598 chore: apply biome formatting 2026-01-01 17:30:15 +01:00
Petter Blomberg fe5e58af91 scripts: fix ad-hoc signing crashes and bash unbound variable error 2026-01-01 15:29:01 +01:00
Peter Steinberger 1a539b9830 fix(macos): restore swift test build 2026-01-01 11:05:14 +01:00
Peter Steinberger 3addd3420b fix: tidy web chat composer layout 2026-01-01 11:05:14 +01:00
Peter Steinberger 6ea10dd153 fix: allow direct file input uploads 2026-01-01 09:44:29 +00:00
Peter Steinberger bf0bee58b3 fix: improve browser upload triggering 2026-01-01 09:35:20 +00:00
Peter Steinberger fbcbc60e85 feat: unify skills config 2026-01-01 10:07:31 +01:00
Peter Steinberger 0a9f06d60f docs: annotate nix path resolution 2026-01-01 09:30:12 +01:00
Peter Steinberger f6956320f9 feat: centralize config paths and expose in snapshot 2026-01-01 09:22:37 +01:00
Peter Steinberger 20bc323963 docs: note nix support 2026-01-01 09:17:24 +01:00
Peter Steinberger bcead5f0f4 fix: honor nix config overrides in mac app 2026-01-01 09:17:21 +01:00
Peter Steinberger cf3049ae34 Merge pull request #40 from joshp123/upstream-preview-nix-2025-12-20
Nix mode support + macOS Info.plist template
2026-01-01 09:15:41 +01:00
Peter Steinberger ad9a9d8d35 Merge remote-tracking branch 'origin/main' into upstream-preview-nix-2025-12-20 2026-01-01 09:15:28 +01:00
Peter Steinberger 14e9077584 chore: add bench-model script 2026-01-01 08:59:31 +01:00
Peter Steinberger 43cf526b5f docs: thank contributor for PR #64 2026-01-01 08:59:24 +01:00
Peter Steinberger 2d5c401d11 fix: prefer module bundle for device models 2026-01-01 08:58:54 +01:00
Peter Steinberger 78cf68549f Merge pull request #64 from mbelinky/fix-instances-crash
Fix Instances crash by bundling device model resources
2026-01-01 08:58:35 +01:00
Peter Steinberger dececccd8e docs: thank contributor for PR #65 2026-01-01 08:55:51 +01:00
Mariano Belinky 941ad27551 Bundle Control UI in Mac app 2026-01-01 08:55:09 +01:00
Peter Steinberger 24e95ab38e docs: update changelog for PR #66 2026-01-01 08:37:49 +01:00
Mariano Belinky c4de0b8255 Use user home for pnpm path 2026-01-01 08:35:54 +01:00
Peter Steinberger 7baaca4a76 docs: add model latency bench notes 2025-12-31 22:39:42 +01:00
Mariano Belinky ea248f6743 Fix device model resources for Instances 2025-12-31 16:45:35 +01:00
Peter Steinberger f03605d8ae test: add minimax live test 2025-12-31 16:31:23 +01:00
Peter Steinberger 0babf08926 chore: add mac app logging coverage 2025-12-31 16:28:51 +01:00
Peter Steinberger 6517b05abe feat: add swift-log app logging controls 2025-12-31 16:03:18 +01:00
Peter Steinberger fa91b5fd03 docs: update changelog for Android chat bubble 2025-12-31 12:50:34 +01:00
Manuel Jiménez Torres f831ccfc63 fix(android): wrong text color in user chat bubbles 2025-12-31 12:48:59 +01:00
Peter Steinberger 12084fc4f9 test: extend Z.AI live test timeout 2025-12-31 12:43:34 +01:00
Peter Steinberger 21237dae98 feat: add Z.AI env support and live test 2025-12-31 11:36:57 +01:00
Peter Steinberger 4bdc25d072 docs: link Anthropic OAuth setup 2025-12-31 11:35:42 +01:00
Peter Steinberger 2f55abace2 fix: add brew installer for ordercli skill 2025-12-31 04:52:40 +01:00
Peter Steinberger 3213e5df2d feat: add gifgrep skill 2025-12-31 04:52:37 +01:00
Peter Steinberger 7e40147aa3 fix: gate web chat/talk on mobile nodes 2025-12-30 22:05:17 +01:00
Peter Steinberger a2a26b26fb fix: satisfy swiftformat in chat view 2025-12-30 20:41:12 +01:00
Peter Steinberger b3cf07d6cb feat: add ui theme toggle 2025-12-30 20:25:58 +01:00
Peter Steinberger ed76cd7574 fix: restore talk orb hit testing 2025-12-30 20:25:52 +01:00
Peter Steinberger 01b8a71ee6 docs: clarify browser wait guidance 2025-12-30 19:22:38 +00:00
Peter Steinberger cc86bbf27d feat: add food-order skill 2025-12-30 15:43:13 +01:00
Peter Steinberger 42cbb11de8 build: update a2ui bundle 2025-12-30 14:43:34 +01:00
Peter Steinberger 52303e8eda docs: update changelog for status pill 2025-12-30 14:39:33 +01:00
Peter Steinberger cf903be4a7 fix: avoid duplicate gateway reconnecting pill 2025-12-30 14:37:59 +01:00
Peter Steinberger 6306786645 fix: allow mp3 fallback result 2025-12-30 14:35:53 +01:00
Peter Steinberger d7b267843e fix: fallback mp3 when pcm blocked 2025-12-30 14:32:47 +01:00
Peter Steinberger 3aefe375c1 chore: update deps and add control ui routing tests 2025-12-30 14:30:46 +01:00
Peter Steinberger 3d6cc435ef fix: hop audio to main actor 2025-12-30 14:22:03 +01:00
Peter Steinberger 973bd3a427 fix: improve talk overlay input + drag 2025-12-30 14:18:51 +01:00
Peter Steinberger 7d1ec51df5 fix: modernize chat scroll position 2025-12-30 13:52:12 +01:00
Peter Steinberger 9fb74399c8 refactor: inject audio players 2025-12-30 13:46:14 +01:00
Peter Steinberger bc0a6fffd1 fix: tighten macOS menu device rows 2025-12-30 13:31:11 +01:00
Peter Steinberger fa85dd6527 docs: note macOS menu layout 2025-12-30 12:57:10 +01:00
Peter Steinberger 73d595eecc chore: sync local changes 2025-12-30 12:53:17 +01:00
Peter Steinberger 3bf8b9ccf4 fix: default android talk pcm_24000 2025-12-30 12:52:56 +01:00
Peter Steinberger 83262a67b1 refactor: extract elevenlabs kit 2025-12-30 12:48:09 +01:00
Peter Steinberger 66952a682d test: add pcm streaming smoke 2025-12-30 12:27:06 +01:00
Peter Steinberger 9df22c0093 fix: address talk streaming build 2025-12-30 12:20:32 +01:00
Peter Steinberger 27adfb76fa fix: stream elevenlabs tts playback 2025-12-30 12:17:40 +01:00
Peter Steinberger 9c532eac07 feat(talk): pause + drag overlay orb 2025-12-30 11:37:52 +01:00
Peter Steinberger 2814815312 feat: add talk voice alias map 2025-12-30 11:35:29 +01:00
Peter Steinberger ab27586674 test: cover external chat completion 2025-12-30 11:23:45 +01:00
Peter Steinberger 2749c5cac3 fix: clear external streaming bubbles 2025-12-30 11:21:57 +01:00
Peter Steinberger 715cf311df fix(ui): move mac talk orb to corner 2025-12-30 11:20:14 +01:00
Peter Steinberger 312443235d fix(ios): unblock device builds 2025-12-30 11:16:15 +01:00
Peter Steinberger 0d95d63258 fix(macos): await-safe session key selection 2025-12-30 11:07:34 +01:00
Peter Steinberger f86772f26c fix(talk): harden TTS + add system fallback 2025-12-30 07:40:02 +01:00
Peter Steinberger a7617e4d79 fix(ui): refine talk overlays 2025-12-30 06:47:35 +01:00
Peter Steinberger 7612a83fa2 fix(talk): align sessions and chat UI 2025-12-30 06:47:19 +01:00
Peter Steinberger afbd18e8df fix(talk): harden playback, interrupts, and timeouts 2025-12-30 06:05:43 +01:00
Peter Steinberger be2bc61d38 fix(talk): hard-timeout ElevenLabs synthesis 2025-12-30 05:46:47 +01:00
Peter Steinberger dcee8beb99 style: biome format gateway server tests 2025-12-30 05:34:53 +01:00
Peter Steinberger fb8f72d5a9 feat(ui): add centered talk orb 2025-12-30 05:27:29 +01:00
Peter Steinberger b3f2416a09 test: reduce flaky timeouts 2025-12-30 05:27:18 +01:00
Peter Steinberger b5ae2ccc3c fix(voice): sync talk mode chat events 2025-12-30 05:27:11 +01:00
Peter Steinberger 05efc3eace fix: avoid iOS talk mode audio tap crash 2025-12-30 04:52:57 +01:00
Peter Steinberger 24f8ff7548 chore(protocol): regenerate Swift gateway models 2025-12-30 04:42:08 +01:00
Peter Steinberger c0c6782a17 fix(android): stabilize BridgeSession shutdown 2025-12-30 04:42:02 +01:00
Peter Steinberger d2ac672f47 feat: add ui.seamColor accent 2025-12-30 04:14:36 +01:00
Peter Steinberger e3d8d5f300 fix(macos): prevent Talk Mode audio hang 2025-12-30 04:14:16 +01:00
Peter Steinberger c5d5c9fcb5 fix: make android canvas background visible 2025-12-30 04:02:52 +01:00
Peter Steinberger 2e040ee07a fix: brighten android canvas 2025-12-30 03:58:18 +01:00
Peter Steinberger 9846c46434 fix: tag A2UI platform and boost Android canvas 2025-12-30 03:49:24 +01:00
Peter Steinberger 5c7c1af44e fix: android talk timestamp parsing 2025-12-30 02:05:14 +01:00
Peter Steinberger e119a82334 feat: talk mode key distribution and tts polling 2025-12-30 01:57:58 +01:00
Peter Steinberger 02db68aa67 fix(macos): hide Restart Gateway when remote 2025-12-30 01:57:58 +01:00
Peter Steinberger 10e1e7fd44 chore: apply biome formatting 2025-12-30 00:16:07 +00:00
Peter Steinberger 7aabe73521 chore: sync pending changes 2025-12-30 00:59:30 +01:00
Peter Steinberger 37f85bb2d1 fix: expand talk overlay bounds 2025-12-30 00:58:58 +01:00
Peter Steinberger 39fccc3699 fix: talk overlay + elevenlabs defaults 2025-12-30 00:51:17 +01:00
Peter Steinberger 53eccc1c1e fix: wire talk menu + mac build 2025-12-30 00:17:10 +01:00
Peter Steinberger c56292a6ec feat: move talk mode to overlay button 2025-12-30 00:01:21 +01:00
Peter Steinberger 857cd6a28a fix: align ios lint and android build 2025-12-29 23:45:58 +01:00
Peter Steinberger 303954ae8c feat: extend status activity indicators 2025-12-29 23:42:22 +01:00
Peter Steinberger 3c338d1858 fix: adjust android talk parser for kotlin json 2025-12-29 23:26:38 +01:00
Peter Steinberger 20d7882033 feat: add talk mode across nodes 2025-12-29 23:21:05 +01:00
Peter Steinberger 6927b0fb8d fix: align camera payload caps 2025-12-29 23:20:55 +01:00
Peter Steinberger 6e83f95c83 fix: clamp tool images to 5MB 2025-12-29 22:13:39 +00:00
Peter Steinberger 8f0c8a6561 fix: cap camera snap payload size 2025-12-29 23:12:20 +01:00
Peter Steinberger a61b7056d5 feat: surface camera activity in status pill 2025-12-29 23:12:03 +01:00
Peter Steinberger f41ade9417 feat(skills): add obsidian skill 2025-12-29 22:51:42 +01:00
Peter Steinberger b0396e196f fix: refresh bridge tokens and enrich node settings 2025-12-29 22:11:12 +01:00
Peter Steinberger cf42fabfd8 test: add ios swift testing + android kotest 2025-12-29 21:10:44 +01:00
Peter Steinberger 52263bd5a3 fix: avoid cli gateway close race 2025-12-29 20:45:50 +01:00
Peter Steinberger 24151a2028 fix: mark screen recorder sendable 2025-12-29 20:28:06 +01:00
Peter Steinberger c11e2d9e5e fix: avoid self capture in ReplayKit start 2025-12-29 20:26:49 +01:00
Peter Steinberger a8c9b2810b fix: align ReplayKit stopCapture call 2025-12-29 20:25:44 +01:00
Peter Steinberger 7a849ab7d1 fix: isolate ReplayKit capture state 2025-12-29 20:24:34 +01:00
Peter Steinberger c14d738d37 fix: avoid screen recorder data races 2025-12-29 20:22:26 +01:00
Peter Steinberger 65478a6ff3 fix: avoid main-actor stopCapture error 2025-12-29 20:20:14 +01:00
Peter Steinberger 41be9232fe fix: prevent iOS screen capture crash 2025-12-29 20:10:36 +01:00
Peter Steinberger 653932e50d fix: show connected nodes only 2025-12-29 18:35:52 +01:00
Peter Steinberger 09ef991e1a chore: harden restart script 2025-12-29 18:09:27 +01:00
Josh Palmer 0f7029583c macOS: load device models from bundle resources 2025-12-29 17:49:13 +01:00
Josh Palmer 10eced9971 fix: use telegram token file for sends and guard console EPIPE 2025-12-29 17:49:13 +01:00
Josh Palmer 1d8b47785c feat(macos): add current TeamID to Peekaboo allowlist
Problem: The bridge only accepts the upstream TeamID, so packaged builds signed locally (Nix/CI) can’t use the bridge even though they are the same app.

Fix: Include the running app’s TeamID (from its code signature) in the allowlist.

Safety: TeamID gating remains; this just adds the app’s own TeamID to preserve permissions/automation in reproducible installs.
2025-12-29 17:49:13 +01:00
Josh Palmer ced271bec1 chore(macos): harden mktemp templates in codesign 2025-12-29 17:49:13 +01:00
Josh Palmer 5d19afd422 feat: improve health checks (telegram tokenFile + hints) 2025-12-29 17:49:13 +01:00
Josh Palmer b7363f7c18 feat: Nix mode config, UX, onboarding, SwiftPM plist, docs 2025-12-29 17:49:13 +01:00
Peter Steinberger aa2700ffa7 chore: set ios signing team for device builds 2025-12-29 17:38:21 +01:00
Peter Steinberger 510e2a1d17 fix: menu devices list 2025-12-29 17:31:23 +01:00
Peter Steinberger ebfe55f909 fix: enable canvas webview scrolling on mobile nodes 2025-12-29 17:13:31 +01:00
Peter Steinberger 26fa9dea97 chore: bump version to 2.0.0-beta5 2025-12-28 14:38:48 +00:00
Peter Steinberger 3bb4c0c237 fix: report macos product version in presence 2025-12-28 14:34:07 +00:00
Peter Steinberger 255a875a2a chore: refresh a2ui bundle hash 2025-12-28 12:06:48 +00:00
Peter Steinberger 2b5f3f1361 docs: clarify watchdog reconnect note 2025-12-28 12:05:06 +00:00
Peter Steinberger eb158545fc fix: force web reconnect on stalled close 2025-12-28 12:04:20 +00:00
Peter Steinberger cade7b1132 docs: clarify gateway readiness in changelog 2025-12-28 10:30:40 +00:00
Peter Steinberger d529736597 fix(macos): fully stop Voice Wake runtime when disabled 2025-12-28 10:17:30 +00:00
Peter Steinberger 8dfc031c4d fix: start gateway before control channel 2025-12-28 09:24:43 +00:00
Peter Steinberger 91c9859000 fix: harden heartbeat acks + gateway reconnect 2025-12-27 20:02:27 +00:00
Peter Steinberger 3a485a14a4 fix: skip whatsapp heartbeat when provider inactive 2025-12-27 19:34:10 +00:00
Peter Steinberger a61c27c4d0 fix: correct beta3 appcast URL 2025-12-27 20:00:08 +01:00
Peter Steinberger e5cae2a2e4 chore: release 2.0.0-beta4 2025-12-27 19:43:43 +01:00
Peter Steinberger 7f961237f9 chore: harden release checks 2025-12-27 19:35:39 +01:00
Peter Steinberger 69a6538567 docs: note notarytool profile 2025-12-27 19:24:24 +01:00
Peter Steinberger 5b3c18ab84 chore: release 2.0.0-beta3 2025-12-27 19:02:35 +01:00
Peter Steinberger 907371453d fix(macos): soften light mode usage bar track 2025-12-27 14:05:36 +01:00
Peter Steinberger 81abffd145 fix(macos): boost light mode usage bar contrast 2025-12-27 14:03:45 +01:00
Peter Steinberger 44ef8fe5c8 fix(macos): refresh sessions on menu open 2025-12-27 13:49:03 +01:00
Peter Steinberger cae78b3f91 fix: treat /model status as model list 2025-12-27 12:10:44 +00:00
Peter Steinberger c0fb814658 fix: normalize imports for lint 2025-12-27 04:02:13 +01:00
Peter Steinberger 7ce0140c81 docs: update changelog 2025-12-27 03:21:25 +01:00
Peter Steinberger 12b3034921 chore(canvas): update a2ui bundle hash 2025-12-27 03:21:20 +01:00
Peter Steinberger ec482ac867 fix(macos): tighten chat window chrome 2025-12-27 03:21:14 +01:00
Peter Steinberger ae52fb7a01 fix(macos): relax chat window min size 2025-12-27 02:55:24 +01:00
Peter Steinberger e8ff08e121 fix(macos): round chat window chrome 2025-12-27 02:51:59 +01:00
Peter Steinberger cc8e104cd6 fix(macos): enforce chat window default size 2025-12-27 02:43:50 +01:00
Peter Steinberger 5919a277bb fix(macos): stabilize menu width tracking 2025-12-27 02:43:50 +01:00
Peter Steinberger 96911d7790 fix: enqueue system event on model switch 2025-12-27 01:17:12 +00:00
Peter Steinberger acd3f7dba7 fix(macos): lock menu width on hover 2025-12-27 01:50:25 +01:00
Peter Steinberger 8aff3979db docs: add local lmstudio setup 2025-12-27 00:48:19 +00:00
Peter Steinberger eafcd862be chore: update protocol models 2025-12-27 01:45:58 +01:00
Peter Steinberger 8826170635 fix: resolve CI lint and android build 2025-12-27 01:41:43 +01:00
Peter Steinberger c54e4d0900 refactor: node tools and canvas host url 2025-12-27 01:36:29 +01:00
Peter Steinberger 52ca5c4aa2 fix: drop identity emoji response prefix 2025-12-27 00:36:04 +00:00
Peter Steinberger 95f8f80e74 fix: allow empty responsePrefix 2025-12-27 00:33:04 +00:00
Peter Steinberger 7e380bb6f8 fix: enable lmstudio responses and drop think tags 2025-12-27 00:28:52 +00:00
Peter Steinberger 2477ffd860 chore: fix lint/test gating 2025-12-26 23:54:30 +00:00
Peter Steinberger a3dc46bf9d fix(a2ui): center status overlay 2025-12-27 00:28:38 +01:00
Peter Steinberger 5c8e1b6eef feat: add model aliases + minimax shortlist 2025-12-26 23:26:14 +00:00
Peter Steinberger ae9a8ce34c fix(a2ui): center status overlay 2025-12-27 00:23:27 +01:00
Peter Steinberger 67b9a675f5 fix(macos): allow http loads in canvas webview 2025-12-27 00:20:58 +01:00
Peter Steinberger fae11e5a55 fix(gateway): advertise reachable canvas host 2025-12-27 00:07:19 +01:00
Peter Steinberger 4daf75a469 fix(macos): enforce node bridge timeouts 2025-12-27 00:02:41 +01:00
Peter Steinberger d0293649cd fix(macos): refresh menu sessions without resizing 2025-12-26 22:48:58 +01:00
Peter Steinberger 353366ac54 fix(macos): expand highlighted menu rows to full width 2025-12-26 22:41:29 +01:00
Peter Steinberger 1a8ffebb00 fix(macos): stabilize menu row width 2025-12-26 22:34:18 +01:00
Peter Steinberger 5ffbddcc57 feat(mac): add allow camera toggle 2025-12-26 21:33:22 +00:00
Peter Steinberger 5fbcbe7e52 feat(mac): add discord connections UI 2025-12-26 21:33:22 +00:00
Peter Steinberger 7daa93cf5a fix(macos): expand menu hover highlight width 2025-12-26 22:30:29 +01:00
Peter Steinberger 9e32f29d19 test: organize heartbeat test imports 2025-12-26 21:29:49 +00:00
Peter Steinberger 1f25e38c2d fix(macos): keep menu width stable while open 2025-12-26 22:27:24 +01:00
Peter Steinberger c10a386d17 fix(macos): detect and reset stale SSH tunnels 2025-12-26 22:12:33 +01:00
Peter Steinberger a13db82d28 fix(nodes): improve version reporting 2025-12-26 21:45:00 +01:00
Peter Steinberger ec392dc870 feat(mac): add node ssh and compact versions 2025-12-26 20:42:49 +00:00
Peter Steinberger 90d00fb095 fix(mac): reorder menu toggles 2025-12-26 20:42:45 +00:00
Peter Steinberger e336b7f27e fix: use final heartbeat payload 2025-12-26 20:39:20 +00:00
Peter Steinberger 7f4c992dd7 fix(mac): move action group below toggles 2025-12-26 20:31:37 +00:00
Peter Steinberger ba1626a5b9 fix(ios): accept truthy A2UI ready check 2025-12-26 21:17:37 +01:00
Peter Steinberger ab73c40bfe fix(mac): refine node submenu copy behavior 2025-12-26 20:05:23 +00:00
Peter Steinberger 4016bc2416 fix(a2ui): center empty canvas text 2025-12-26 20:43:45 +01:00
Peter Steinberger 9302daadc1 fix(mac): align node details 2025-12-26 19:32:48 +00:00
Peter Steinberger de7429e148 fix(mac): show node versions in menu 2025-12-26 19:25:28 +00:00
Peter Steinberger 5892bd45d8 fix(mac): tweak menu icons 2025-12-26 19:23:53 +00:00
Peter Steinberger 9317eccfc8 fix(mac): regroup menubar sections 2025-12-26 19:18:12 +00:00
Peter Steinberger 1236c4dafb refactor: make browser actions ref-only 2025-12-26 19:02:27 +00:00
Peter Steinberger f50f18f65a feat(mac): refine menubar nodes layout 2025-12-26 19:02:27 +00:00
Peter Steinberger 747cc4daa5 fix: gate libsignal session logs behind verbose 2025-12-26 19:02:27 +00:00
Peter Steinberger 51b6a785e6 fix(canvas): center debug status overlay 2025-12-26 20:01:23 +01:00
Peter Steinberger f4d41ef254 chore(ios): auto team id fallback 2025-12-26 18:19:48 +01:00
Peter Steinberger b9d80aa535 chore(ios): add team id helper 2025-12-26 18:16:13 +01:00
Peter Steinberger 2f8213ca9a fix(a2ui): skip bundle when inputs unchanged 2025-12-26 18:11:00 +01:00
Peter Steinberger 541b8cbb6c fix(ios): silence device build warnings 2025-12-26 18:09:44 +01:00
Peter Steinberger ed2e738ea4 fix: provider startup order and enable flags 2025-12-26 16:54:53 +00:00
Peter Steinberger 17d9ba256b fix(discord): ignore destroy promise 2025-12-26 17:21:32 +01:00
Peter Steinberger 15dbac8193 docs: update beta3 changelog 2025-12-26 17:21:29 +01:00
Peter Steinberger 2119854246 build: skip a2ui bundling in build 2025-12-26 16:00:35 +01:00
Peter Steinberger 034c93fd65 fix: align discord types 2025-12-26 14:47:15 +01:00
Peter Steinberger ce91aba4de fix: apply biome formatting 2025-12-26 14:38:37 +01:00
Peter Steinberger e33c09f8d4 fix(tests): align discord + queue changes 2025-12-26 14:32:57 +01:00
Peter Steinberger a678c3f53e refactor(queue): remove drop mode 2025-12-26 14:29:28 +01:00
Peter Steinberger 3e4fc7ff7f feat(queue): add reset/default directive 2025-12-26 14:24:53 +01:00
Peter Steinberger 8dda07a1e9 feat(queue): add queue modes and discord gating 2025-12-26 13:35:44 +01:00
Peter Steinberger e9f1851c5d chore: ignore bun build artifacts 2025-12-26 13:20:30 +01:00
Shadow ac659ff5a7 feat(discord): Discord transport 2025-12-26 13:20:30 +01:00
Peter Steinberger 557f8e5a04 fix: restore build after deps update 2025-12-26 12:17:36 +00:00
Peter Steinberger 54de5ad3fa test: isolate vitest home 2025-12-26 11:45:16 +00:00
Peter Steinberger 0709586e3a fix: support mocked model registry in catalog 2025-12-26 11:53:55 +01:00
Peter Steinberger 82ced33747 fix: align pi model discovery with auth storage 2025-12-26 11:49:13 +01:00
Peter Steinberger d31c5d7a2c style: format web inbound 2025-12-26 11:39:48 +01:00
Peter Steinberger 2045487d5e fix: extract quoted WhatsApp reply text 2025-12-26 10:51:08 +01:00
Peter Steinberger 4611e799b7 docs: note inbox listener cleanup 2025-12-26 09:37:38 +00:00
Peter Steinberger ffe9a2435b fix: clean up web inbox listeners on close 2025-12-26 09:27:06 +00:00
Peter Steinberger f5d8876384 test: expand compaction retry coverage 2025-12-26 10:22:04 +01:00
Peter Steinberger d28265cfbe fix: handle embedded agent overflow 2025-12-26 10:20:21 +01:00
Peter Steinberger 8059e83c49 chore: bump pi-mono deps 2025-12-26 10:20:21 +01:00
Peter Steinberger d6f07c9f91 chore: fix lint after logging tweaks 2025-12-26 09:08:37 +00:00
Peter Steinberger 917cb8fa67 fix: brighten gateway model console log 2025-12-26 08:45:15 +00:00
Peter Steinberger 461db9e469 fix: split whatsapp listen hint from subsystem log 2025-12-26 08:41:58 +00:00
Peter Steinberger 112908886c fix: log heartbeat failure reasons 2025-12-26 08:34:42 +00:00
Peter Steinberger f734801da1 fix: correct heartbeat log formatting 2025-12-26 08:17:29 +00:00
meaningfool ea6dc7c710 fix: correctly define pnpm workspace and clean up vite build scripts
This change adds the missing 'packages' definition to pnpm-workspace.yaml, allowing pnpm to correctly install dependencies for the 'ui' sub-package. This resolves the 'vite: command not found' error during 'ui:build'. It also reverts the temporary 'pnpm dlx' workarounds in ui/package.json.
2025-12-26 09:13:17 +01:00
Peter Steinberger cd81348ca5 chore: fix env spread lint 2025-12-26 02:02:49 +00:00
Peter Steinberger ad91a09b07 ci: avoid macos runner queue 2025-12-26 02:02:49 +00:00
Peter Steinberger 040f73a3f4 docs: clarify heartbeat defaults 2025-12-26 03:02:11 +01:00
Peter Steinberger 0d8e0ddc4f feat: unify gateway heartbeat 2025-12-26 02:35:40 +01:00
Peter Steinberger 8f9d7405ed style: fix biome formatting 2025-12-26 00:50:46 +00:00
Peter Steinberger 72267e97ca docs: note hour durations 2025-12-26 01:36:08 +01:00
Peter Steinberger 19f87f0a89 feat: allow hour durations 2025-12-26 01:34:46 +01:00
Peter Steinberger 9f7b1f0942 feat: move heartbeat config to agent.heartbeat 2025-12-26 01:13:42 +01:00
Peter Steinberger 1ef888ca23 refactor(config): drop agent.provider 2025-12-26 01:13:42 +01:00
Peter Steinberger 8b815bce94 feat(config): allow provider/model shorthand 2025-12-26 01:13:42 +01:00
Peter Steinberger 97539db36d ci: skip ios job 2025-12-26 00:04:46 +00:00
Peter Steinberger 655fa5b8e0 style: fix pi embedded runner lint 2025-12-25 23:58:37 +00:00
Peter Steinberger 9fbd3cc16f ci: ignore ios failures 2025-12-25 23:55:55 +00:00
Rolf FredheimandClaude Opus 4.5 2295cbb815 feat(agent): add maxConcurrent config for parallel message handling
Adds `agent.maxConcurrent` config option to control how many agent runs
can execute in parallel across all conversations. Default remains 1
(sequential) for backwards compatibility.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-25 23:55:41 +01:00
Peter Steinberger 198f8ea700 fix(agent): serialize runs per session 2025-12-25 23:50:52 +01:00
Peter Steinberger 9fa9199747 docs: note multi-agent session rule 2025-12-25 23:50:46 +01:00
Peter Steinberger 1cd167a59a ci: run on node 24 2025-12-25 23:05:09 +01:00
Peter Steinberger 2868dc975c chore: require node >=22.12 and fix swiftformat lint 2025-12-25 23:02:31 +01:00
meaningfool 214ab16eb2 fix: correctly define pnpm workspace and clean up vite build scripts
This change adds the missing 'packages' definition to pnpm-workspace.yaml, allowing pnpm to correctly install dependencies for the 'ui' sub-package. This resolves the 'vite: command not found' error during 'ui:build'. It also reverts the temporary 'pnpm dlx' workarounds in ui/package.json.
2025-12-25 22:52:22 +01:00
Peter Steinberger 1c88d9575e fix(webchat): refresh bubbles on theme change 2025-12-25 22:35:46 +01:00
Peter Steinberger 1e4e02ddd3 docs: update beta3 changelog 2025-12-25 21:15:45 +00:00
Peter Steinberger f6fcddbe0b fix: relax tool typing for bash tools 2025-12-25 20:27:05 +00:00
Peter Steinberger 474180c112 style: fix bash tools lint 2025-12-25 20:20:38 +00:00
Peter Steinberger c860573f13 style: fix biome formatting 2025-12-25 20:13:48 +00:00
Peter Steinberger c9c7354009 chore: add gateway:watch 2025-12-25 18:44:23 +00:00
Peter Steinberger 42eb7640f9 feat: add gateway restart tool 2025-12-25 18:05:37 +00:00
Peter Steinberger aafcd569b1 feat: line-based process logs 2025-12-25 18:03:57 +00:00
Peter Steinberger b549307ccf docs: add Sparkle HTML release notes 2025-12-25 04:27:20 +01:00
Peter Steinberger 57090d4f8d fix: align chat scroll anchor 2025-12-25 04:10:47 +01:00
Peter Steinberger 764f7586de fix: adjust tool casts for build 2025-12-25 03:36:04 +01:00
Peter Steinberger d96f2abc4e fix: resolve agent tool typing 2025-12-25 03:33:09 +01:00
Peter Steinberger 92f467e81c fix: clean agent bash lint 2025-12-25 03:29:36 +01:00
Peter Steinberger 2442186a31 fix: silence view warnings 2025-12-25 03:23:31 +01:00
Peter Steinberger 9fb74cb58a test: assert bridge does not add loopback listener 2025-12-25 01:41:09 +00:00
Peter Steinberger 81e11c1d91 fix: bridge tailnet bind also listens on loopback 2025-12-25 01:37:47 +00:00
Peter Steinberger dc93350e0a docs: add background bash changelog 2025-12-25 00:54:08 +00:00
Peter Steinberger 3c6432da1f feat: add background bash sessions 2025-12-25 00:25:11 +00:00
Peter Steinberger 4eecb6841a docs: add gmail hook quickstart 2025-12-24 22:59:09 +00:00
Peter Steinberger 3b83d3ff3a fix: preserve tool action enums 2025-12-24 22:50:40 +00:00
Peter Steinberger 88b92a9605 style: format gmail hooks and tools 2025-12-24 23:11:14 +01:00
Peter Steinberger 3bb5baa6d2 fix: default tailscale serve in settings 2025-12-24 22:09:23 +00:00
Peter Steinberger 59443d7ec6 style: format reply changes 2025-12-24 23:06:20 +01:00
Peter Steinberger c1d170e13d docs: note tailscale gmail path behavior 2025-12-24 21:56:21 +00:00
Peter Steinberger cffac6e11a fix: auto gmail serve path for tailscale 2025-12-24 21:56:17 +00:00
Peter Steinberger 79870472e1 fix: expose union tool parameters 2025-12-24 21:48:22 +00:00
Peter Steinberger 1b69c94f76 docs: clarify reply threading change 2025-12-24 22:37:32 +01:00
Peter Steinberger cf8d1cf0e7 fix: avoid threaded replies for agent output 2025-12-24 22:36:42 +01:00
Peter Steinberger 009fbeb543 chore: add gmail hook setup notes 2025-12-24 21:20:20 +00:00
Peter Steinberger 9ceb8731d3 chore: clarify gmail serve path 2025-12-24 21:20:20 +00:00
Peter Steinberger 8f934bf817 docs: update file size guidance 2025-12-24 22:19:10 +01:00
Peter Steinberger 88be2701f4 refactor: split utilities 2025-12-24 22:16:06 +01:00
Peter Steinberger 8ee62f0ac8 style: format locator selector 2025-12-24 21:49:31 +01:00
Peter Steinberger 4d4308af78 fix: resolve coverage profile symbol at runtime 2025-12-24 21:43:46 +01:00
Peter Steinberger f7c5eff35e docs: link webhook docs 2025-12-24 20:07:24 +00:00
Peter Steinberger 3bc1644f34 refactor: split canvas window 2025-12-24 21:04:52 +01:00
Peter Steinberger 27025b71db feat: add selector-based browser actions 2025-12-24 19:52:28 +00:00
Peter Steinberger 523d9ec3c2 feat: add gmail hooks wizard 2025-12-24 19:48:35 +00:00
Peter Steinberger aeb5455555 feat: add webhook hook mappings
# Conflicts:
#	src/gateway/server.ts
2025-12-24 19:48:05 +00:00
Peter Steinberger 337390b590 fix: allow overlay present access 2025-12-24 20:24:37 +01:00
Peter Steinberger 836d950e05 fix: restore voice wake overlay build 2025-12-24 20:17:01 +01:00
Peter Steinberger ad096f77fc refactor: split voice wake overlay 2025-12-24 20:09:56 +01:00
Peter Steinberger 3774494f7e test: add ios coverage tests 2025-12-24 20:00:51 +01:00
Peter Steinberger 14fae5af9e test: add ios coverage hooks 2025-12-24 20:00:45 +01:00
Peter Steinberger 65b48561a9 refactor: split critter status label 2025-12-24 19:56:24 +01:00
Peter Steinberger 842dc14c18 style: format port guardian 2025-12-24 19:41:32 +01:00
Peter Steinberger af1afa7ba6 style: format cron settings 2025-12-24 19:40:11 +01:00
Peter Steinberger 8c4c5e524b refactor: split cron settings 2025-12-24 19:36:10 +01:00
Peter Steinberger 204bd7d2c4 test: add mac coverage helpers 2025-12-24 19:29:44 +01:00
Peter Steinberger f44014ff00 refactor: split onboarding view 2025-12-24 19:29:27 +01:00
Peter Steinberger 01719b02e2 test: cover bridge settings discovery 2025-12-24 18:07:41 +01:00
Peter Steinberger 4ba86bbe00 test: cover bridge hello defaults 2025-12-24 18:07:38 +01:00
Peter Steinberger b85503b3b2 fix: guard hook payload strings 2025-12-24 17:49:52 +01:00
Peter Steinberger 131a9aa1ac style: format macos sources 2025-12-24 17:47:35 +01:00
Peter Steinberger bd223606b1 style: format gateway server 2025-12-24 17:45:39 +01:00
Peter Steinberger f4fb80e523 test: expand overlay coverage 2025-12-24 17:43:30 +01:00
Peter Steinberger 49e466dd40 test: expand menu and node coverage 2025-12-24 17:43:30 +01:00
Peter Steinberger deec315f6a test: expand settings coverage 2025-12-24 17:43:30 +01:00
Peter Steinberger 7fafe54e16 test: expand onboarding coverage 2025-12-24 17:43:30 +01:00
Peter Steinberger bdcbc829a0 test: add coverage flush helper 2025-12-24 17:43:30 +01:00
Peter Steinberger 4a64e86ecb chore: update changelog 2025-12-24 14:39:26 +00:00
Peter Steinberger 1e2946ebc6 test: extend webhook coverage 2025-12-24 14:39:21 +00:00
Peter Steinberger 1ed5ca3fde feat: add gateway webhooks 2025-12-24 14:33:05 +00:00
Peter Steinberger aa62ac4042 fix: use recognition update segments 2025-12-24 15:27:06 +01:00
Peter Steinberger e8f24910bd style: swiftformat chat ui 2025-12-24 15:10:31 +01:00
Peter Steinberger 8d34e54dc5 fix: address swiftlint warnings 2025-12-24 15:10:22 +01:00
Peter Steinberger c5ede3f167 build: align Commander dependency 2025-12-24 14:44:56 +01:00
Peter Steinberger 1cd108e891 fix: clear wake word match warning 2025-12-24 14:44:50 +01:00
Peter Steinberger 8878fd3028 ui: merge tool call results 2025-12-24 14:38:43 +01:00
Peter Steinberger a22d4e7962 fix: import AnyCodable for tool cards 2025-12-24 14:35:06 +01:00
Peter Steinberger 25d2d7389f ui: render tool call cards 2025-12-24 14:29:40 +01:00
Peter Steinberger 816b784399 ui: constrain typing indicator width 2025-12-24 14:10:32 +01:00
Peter Steinberger c250f092bb test: cover overlay level throttling 2025-12-24 13:54:03 +01:00
Peter Steinberger b9c2bdf641 docs: update changelog 2025-12-24 13:52:41 +01:00
Peter Steinberger 5ba90db049 perf: throttle voice overlay updates 2025-12-24 13:51:41 +01:00
Peter Steinberger 88d20c5419 perf: gate idle pulse animations 2025-12-24 13:51:40 +01:00
Peter Steinberger e158bee95f perf: reduce chat animation churn 2025-12-24 13:51:40 +01:00
Peter Steinberger 0139a77e94 fix: resolve ts build errors 2025-12-24 00:57:11 +00:00
Peter Steinberger e76d1b899b fix: clean telegram parse error logging 2025-12-24 00:53:27 +00:00
Peter Steinberger 3fcdd6c9d7 feat: enforce final tag parsing for embedded PI 2025-12-24 00:52:33 +00:00
Peter Steinberger bc916dbf35 feat: require final tag format in system prompt 2025-12-24 00:52:30 +00:00
Peter Steinberger 96da2efb13 style: swiftformat gateway process manager 2025-12-24 00:33:40 +00:00
Peter Steinberger 267cdf20e1 style: fix biome lint 2025-12-24 00:33:35 +00:00
Peter Steinberger 20c7df35c4 docs: note config refactor 2025-12-24 00:24:05 +00:00
Peter Steinberger 0f06e9926b docs: update routing/messages/session config 2025-12-24 00:22:57 +00:00
Peter Steinberger 93af424ce5 refactor: move inbound config 2025-12-24 00:22:52 +00:00
Peter Steinberger 5e07400cd1 refactor: update macOS config paths 2025-12-23 23:45:27 +00:00
Peter Steinberger 364a6a9444 feat: add per-session model selection 2025-12-23 23:45:20 +00:00
Peter Steinberger b6bfd8e34f fix: anchor typing loop to run 2025-12-23 15:03:05 +00:00
Peter Steinberger b05981ef27 fix: add reasoning tag hint for local providers 2025-12-23 14:34:56 +00:00
Peter Steinberger 42f1a56832 test: cover system prompt owner numbers 2025-12-23 14:20:09 +00:00
Peter Steinberger f667d56701 fix: tag owner numbers in system prompt 2025-12-23 14:19:41 +00:00
Peter Steinberger df5284beaf fix: suppress thinking stream + typing 2025-12-23 14:17:18 +00:00
Peter Steinberger 6d551b0d6e fix: normalize tool schemas for lm studio 2025-12-23 14:09:07 +00:00
Peter Steinberger 25e6339e2e chore: bump pi-mono deps 2025-12-23 14:07:54 +00:00
Peter Steinberger f70fd30cd3 chore: include runtime info in system prompt 2025-12-23 14:05:43 +00:00
Peter Steinberger 863d26558a fix: delay typing until reply payload 2025-12-23 13:55:01 +00:00
Peter Steinberger cba12a1abd fix: inject group activation in system prompt 2025-12-23 13:32:07 +00:00
Peter Steinberger 96d57a18ee chore: demote reply chunk logs 2025-12-23 13:25:56 +00:00
Peter Steinberger e54ed10bc1 fix: honor /new resets with mentions in groups 2025-12-23 13:20:11 +00:00
Peter Steinberger c8c807adcc refactor: drop PAM auth and require password for funnel 2025-12-23 13:13:09 +00:00
Peter Steinberger cd6ed79433 fix: honor group requireMention default 2025-12-23 12:53:30 +00:00
Peter Steinberger ea4b3b74bb chore: log whatsapp identity on start 2025-12-23 12:45:18 +00:00
Peter Steinberger facfd64787 fix: avoid spawning duplicate gateway when external listener exists 2025-12-23 12:43:51 +00:00
Peter Steinberger 760a83d256 docs: add offline memory system proposal 2025-12-23 13:36:59 +01:00
Peter Steinberger bbff19698b chore: flatten provider console subsystems 2025-12-23 11:27:14 +00:00
Peter Steinberger 6f38cb162c chore: bump internal version to beta3 2025-12-23 04:28:09 +01:00
Peter Steinberger af82224f82 fix: relax Sparkle delegate isolation 2025-12-23 03:36:56 +01:00
Peter Steinberger a938e9473b fix: isolate Sparkle delegate conformance 2025-12-23 03:28:39 +01:00
Peter Steinberger 3e88553d52 fix: isolate updater factory on main actor 2025-12-23 03:16:47 +01:00
Peter Steinberger 56245d5646 fix: strip repeated heartbeat ok tails 2025-12-23 03:12:24 +01:00
Peter Steinberger 4af08b1606 fix: preserve whatsapp group JIDs 2025-12-23 03:05:59 +01:00
Peter Steinberger fc4a395c88 chore: update gateway protocol models 2025-12-23 03:05:04 +01:00
Peter Steinberger de1813ab32 docs: add beta3 changelog 2025-12-23 03:02:30 +01:00
Peter Steinberger 89ace66972 style: format macOS sources 2025-12-23 03:02:09 +01:00
Peter Steinberger 63f1857bda docs: add WhatsApp integration guide 2025-12-23 03:00:27 +01:00
Peter Steinberger 279500cba4 fix: resolve build errors 2025-12-23 03:00:04 +01:00
Peter Steinberger 183270b443 fix: correct models config schema 2025-12-23 02:50:26 +01:00
Peter Steinberger a5f4332f21 style: apply biome formatting 2025-12-23 02:49:49 +01:00
Peter Steinberger 6fad79f581 docs: document custom model providers 2025-12-23 02:48:57 +01:00
Peter Steinberger dff6274a93 test: cover models config merge 2025-12-23 02:48:54 +01:00
Peter Steinberger 082c872469 feat: support custom model providers 2025-12-23 02:48:48 +01:00
Peter Steinberger 67a3dda53a fix: inject reply context into body 2025-12-23 02:44:38 +01:00
Peter Steinberger 950432eac0 test: update whatsapp reply quote assertions 2025-12-23 02:30:21 +01:00
Peter Steinberger 6550e7d562 fix: add whatsapp reply context 2025-12-23 02:30:21 +01:00
Peter Steinberger ffe75f3e20 🤖 codex: add telegram reply context
# Conflicts:
#	src/telegram/bot.ts
2025-12-23 02:30:21 +01:00
Tu Nombre Real 8431874b15 fix(macOS): remove redundant kickstart -k causing gateway restart loop
The launchd bootstrap already starts the gateway job. The subsequent
kickstart -k was killing it immediately after startup, and combined
with KeepAlive=true, this caused a port-conflict restart loop where
launchd would try to restart while the old instance was still
shutting down.

Symptoms: 'Bootstrap failed: 5: Input/output error' and repeated
'Gateway failed to start: another gateway instance is already
listening' messages in the log.
2025-12-23 01:57:54 +01:00
Peter Steinberger 54d2ccda99 feat(mac): surface update-ready state 2025-12-23 01:42:33 +01:00
Peter Steinberger 926b6d9464 chore: format wake gate + chat theme 2025-12-23 01:41:13 +01:00
Peter Steinberger abfb6832c3 fix(mac): default session menu checks 2025-12-23 01:36:01 +01:00
Peter Steinberger ceeea359fc chore: remove shared build artifacts 2025-12-23 01:32:02 +01:00
Peter Steinberger ef35868bef feat: share wake gate via SwabbleKit 2025-12-23 01:31:59 +01:00
Peter Steinberger cf48d297dd docs: explain tool exposure in pi-mono 2025-12-23 00:29:38 +00:00
Peter Steinberger 2b20e3d2b0 chore: resolve docs list from cwd 2025-12-23 00:28:55 +00:00
Peter Steinberger 918cbdcf03 refactor: lint cleanups and helpers 2025-12-23 00:28:55 +00:00
Peter Steinberger f5837dff9c chore: add oxlint type-aware lint 2025-12-23 00:28:55 +00:00
Peter Steinberger ce04308c17 refactor: remove session syncing metadata 2025-12-23 00:50:51 +01:00
Peter Steinberger c0c20ebf3e feat: replace clawdis skills with tools 2025-12-22 23:40:57 +00:00
Peter Steinberger 823195a122 style(mac): increase session row padding 2025-12-23 00:10:38 +01:00
Peter Steinberger 581583abb4 fix(mac): drop syncing menu + show state checks 2025-12-23 00:10:38 +01:00
Peter Steinberger 882fd48408 style: add visual effect host for chat 2025-12-23 00:10:38 +01:00
Peter Steinberger 91238df13f chore: alias console subsystem names 2025-12-22 23:06:15 +00:00
Peter Steinberger ca806897c2 Template: Add smart heartbeat logic for baby agents
- Added heartbeat section with proactive check guidelines
- Includes email, calendar, weather, mentions rotation
- Track checks in heartbeat-state.json
- Know when to reach out vs stay quiet
- Proactive work suggestions (memory, git, docs)

Goal: Baby agents should check in 2-4x daily, not just HEARTBEAT_OK
2025-12-22 22:55:27 +00:00
Peter Steinberger 9118884e92 fix(web): restore creds before auth check 2025-12-22 22:55:27 +00:00
Peter Steinberger e403f8b620 style(pi): sort imports 2025-12-22 22:55:27 +00:00
Peter Steinberger 6205b955da style(mac): adjust session row padding and menu options 2025-12-22 23:30:25 +01:00
Peter Steinberger d265a04b19 style(mac): pad session rows + thicken bars 2025-12-22 23:22:36 +01:00
Peter Steinberger afc09744b4 fix(mac): size highlighted session rows 2025-12-22 22:59:59 +01:00
Peter Steinberger 1e1d76d600 fix(mac): restore sessions bars with injected submenus 2025-12-22 22:49:37 +01:00
Peter Steinberger 0b70aa0c56 fix(mac): hide sessions header when disconnected 2025-12-22 22:09:26 +01:00
Peter Steinberger 4ca6591045 refactor: move OAuth storage and drop legacy sessions 2025-12-22 21:02:48 +00:00
Peter Steinberger 9717f2d374 fix: bump pi deps and fix lint 2025-12-22 20:45:38 +00:00
Peter Steinberger 469c8a1a4b fix(mac): show disconnected sessions + sleeping eyes 2025-12-22 21:13:33 +01:00
Peter Steinberger 9d47b15575 fix(mac): sessions error UI + sleeping icon 2025-12-22 21:02:45 +01:00
Peter Steinberger a11a204b8e chore(submodules): bump Peekaboo 2025-12-22 19:44:48 +00:00
Peter Steinberger e3c3d108fe refactor(logging): shorten subsystem prefixes 2025-12-22 19:42:22 +00:00
Peter Steinberger 8cadb5cf18 docs: update group chat commands 2025-12-22 20:36:34 +01:00
Peter Steinberger f10c8f2b4c feat: add group activation command 2025-12-22 20:36:29 +01:00
Peter Steinberger 5d2d701e1e docs: note mac studio session log location 2025-12-22 20:26:23 +01:00
Peter Steinberger f24d8473b1 fix(mac): restore session usage bar 2025-12-22 20:14:54 +01:00
Peter Steinberger 3412ff7003 style: add macos chat glass background 2025-12-22 19:55:17 +01:00
Peter Steinberger 15e468f5dd feat: add group chat activation mode 2025-12-22 19:32:12 +01:00
Peter Steinberger a0dd504991 feat(mac): sessions submenus 2025-12-22 19:29:24 +01:00
Peter Steinberger 19b847b23b style: tighten macos chat composer 2025-12-22 19:08:23 +01:00
Peter Steinberger 3b134c8fef style: tighten chat compose spacing 2025-12-22 19:01:58 +01:00
Peter Steinberger c872f37aae fix: remove redundant await in CanvasManager 2025-12-22 18:53:14 +01:00
Peter Steinberger 3ce5b9b0d9 test: extend gateway sigterm timeouts 2025-12-22 18:52:35 +01:00
Peter Steinberger 2d7c5f8c53 refactor: migrate embedded pi to sdk 2025-12-22 18:05:44 +01:00
Peter Steinberger 79c0fd27a0 fix: center debug status overlay 2025-12-21 20:43:06 +01:00
Peter Steinberger b06d1ed072 docs(logging): clarify console color behavior 2025-12-21 17:36:30 +00:00
Peter Steinberger 52e7a4456a refactor(logging): streamline whatsapp console output 2025-12-21 17:36:24 +00:00
Peter Steinberger f1202ff152 chore: fix lint + build 2025-12-21 15:58:37 +01:00
Peter Steinberger e4db7cbd2b chore: bump Peekaboo submodule 2025-12-21 15:57:09 +01:00
Peter Steinberger ff63204d17 fix(web): harden WhatsApp creds persistence 2025-12-21 13:58:31 +00:00
Peter Steinberger 4f3a3e93a9 style: biome formatting 2025-12-21 13:58:27 +00:00
Peter Steinberger b56d4b90ce fix(logging): repair chalk/tslog typing 2025-12-21 13:58:22 +00:00
Peter Steinberger 6c2f9b3150 chore: update Peekaboo submodule 2025-12-21 14:50:28 +01:00
Peter Steinberger a808cdce13 fix(android): drop duplicate scaffold asset 2025-12-21 14:50:28 +01:00
Peter Steinberger a8629e1855 fix(logging): simplify tty color detection 2025-12-21 13:34:13 +00:00
Peter Steinberger 0146784e18 feat(logging): add console color modes 2025-12-21 13:26:50 +00:00
Peter Steinberger 249b85af1e refactor(gateway): switch logs to subsystem logger 2025-12-21 13:24:15 +00:00
Peter Steinberger efc12ab28d refactor(browser): use subsystem logger 2025-12-21 13:24:15 +00:00
Peter Steinberger 5b2e7d4464 refactor(logging): add subsystem console formatting 2025-12-21 13:24:15 +00:00
Peter Steinberger bcd3c13e2c feat(macos): surface canvas debug status 2025-12-21 14:21:06 +01:00
Peter Steinberger 7932e966db feat(android): toggle debug canvas status 2025-12-21 14:21:06 +01:00
Peter Steinberger 30d84643db feat(ios): toggle debug canvas status 2025-12-21 14:21:06 +01:00
Peter Steinberger 264c91e620 feat(canvas): gate debug status overlay 2025-12-21 14:21:06 +01:00
Peter Steinberger db89be4106 chore: update peekaboo submodule 2025-12-21 13:10:20 +00:00
Peter Steinberger 85816a5ee2 fix(cli): hint peekaboo unauthorized 2025-12-21 13:09:48 +00:00
Peter Steinberger 5449e44381 chore: bump Peekaboo submodule 2025-12-21 14:01:28 +01:00
Peter Steinberger 20630b8744 chore: bump Peekaboo + menu cleanup 2025-12-21 13:59:41 +01:00
Peter Steinberger 3b63d1cb77 fix: auto-restart WhatsApp QR login 2025-12-21 13:36:26 +01:00
Peter Steinberger 5703b9e737 docs: clarify restart semantics 2025-12-21 12:47:18 +01:00
Peter Steinberger 02787b5674 build(mac): add notarize flow for release artifacts 2025-12-21 12:33:45 +01:00
Peter Steinberger 4021da524c fix(chat-ui): avoid animated initial scroll 2025-12-21 12:33:41 +01:00
Peter Steinberger 5adec0eae0 fix: align canvas defaults and A2UI auto-nav 2025-12-21 12:32:36 +01:00
Peter Steinberger 3f44f0b753 ui: simplify dashboard health status 2025-12-21 12:31:56 +01:00
Peter Steinberger 2a975f751b refactor(macos): regroup menu sections 2025-12-21 12:29:29 +01:00
Peter Steinberger 03bd049291 docs: refine header ctas for github pages 2025-12-21 12:29:29 +01:00
Peter Steinberger 6ddd36666e feat(ui): make chat the landing view 2025-12-21 11:24:39 +00:00
Peter Steinberger 3791db006e docs: add github/download buttons to pages header 2025-12-21 12:19:08 +01:00
Peter Steinberger 6bf8c0c17a docs: note npm release pitfalls 2025-12-21 04:10:20 +01:00
Peter Steinberger 80e1934f4e style: fix tailscale swiftformat 2025-12-21 03:52:28 +01:00
Peter Steinberger 7415fdb79b chore: whitelist npm files 2025-12-21 03:48:23 +01:00
Peter Steinberger b850b0dacf ci: install swiftlint and swiftformat for ios 2025-12-21 03:44:18 +01:00
Peter Steinberger 04e3d0c2fe style: swiftformat cleanup 2025-12-21 03:44:12 +01:00
Peter Steinberger 3810519671 chore: update appcast for 2.0.0-beta2 2025-12-21 03:29:03 +01:00
Peter Steinberger a08c8ef1fa chore: bump version to 2.0.0-beta2 2025-12-21 03:21:49 +01:00
Peter Steinberger 6496a288b8 fix: add A2UI inset vars 2025-12-21 03:21:49 +01:00
Peter Steinberger 9f72eb3374 docs: add canvas gutter guidance 2025-12-21 03:21:48 +01:00
Peter Steinberger e71c71c6c2 fix: add canvas gutter vars for A2UI 2025-12-21 03:21:48 +01:00
Peter Steinberger 0197fb35fe fix: clear canvas error banner on load 2025-12-21 03:21:48 +01:00
Peter Steinberger bcc5891e03 fix(mac): allow tailscale localapi http 2025-12-21 02:17:55 +00:00
Peter Steinberger f90ab3c4c2 fix(mac): trim onboarding checklist 2025-12-21 01:57:18 +00:00
Peter Steinberger 79280f3d93 fix(mac): tighten onboarding layout 2025-12-21 01:57:18 +00:00
Peter Steinberger ce79d0b9a4 docs: add Peter tailnet/gateway notes 2025-12-21 02:55:32 +01:00
Peter Steinberger a5b4a01594 fix(mac): shrink onboarding + respect existing workspace 2025-12-21 01:51:48 +00:00
Peter Steinberger 5b25eeb449 refactor(macos): remove manual identity onboarding 2025-12-21 01:39:50 +00:00
Peter Steinberger fb259e8a50 fix(mac): shrink onboarding height 2025-12-21 01:35:27 +00:00
Peter Steinberger b82dfe08a2 fix: prefer header mime for media extensions 2025-12-21 02:34:19 +01:00
Peter Steinberger 4671c9e672 fix: align A2UI canvas background 2025-12-21 02:34:19 +01:00
Peter Steinberger 00cdcd4d28 fix(mac): guard onboarding workspace bootstrap 2025-12-21 01:31:31 +00:00
Peter Steinberger 4e1fe88195 Give workspace templates actual personality
- SOUL.md: Philosophy over bullet points, genuine vs performative help
- IDENTITY.md: Invites creativity, frames identity as discovery
- USER.md: Learning about a person, not building a dossier
- BOOTSTRAP.md: Conversational first-run, not robotic steps
- AGENTS.md: 'This folder is home' - clear, direct, practical
- TOOLS.md: Explains why separate from skills, real examples

New agents should boot with spark, not corporate drone energy. 🦞
2025-12-21 01:24:13 +00:00
Peter Steinberger 28ad475ab4 feat(mac): add tailscale settings 2025-12-21 01:16:49 +00:00
Peter Steinberger 104e265633 docs: clarify wacli usage 2025-12-21 02:14:52 +01:00
Peter Steinberger 382d237a60 build: silence mac packaging warnings 2025-12-21 02:06:12 +01:00
Peter Steinberger de2fd659ab fix(mac): shrink onboarding height 2025-12-21 00:57:11 +00:00
Peter Steinberger d2fda411f3 docs: add 2.0.0-beta2 changelog 2025-12-21 01:54:27 +01:00
Peter Steinberger e02944c323 docs: fix npmjs header image 2025-12-21 01:54:27 +01:00
Peter Steinberger a01f4998c5 ci: split ios workflow 2025-12-21 00:49:20 +00:00
Peter Steinberger aa198594fd fix(mac): avoid buttonStyle ternary 2025-12-21 00:49:07 +00:00
Peter Steinberger 406a94bf76 fix: use A2UI message context 2025-12-21 01:48:21 +01:00
Peter Steinberger fef1841fee build: update iOS lint scripts 2025-12-21 01:48:21 +01:00
Peter Steinberger 1cb85fdea8 fix(mac): disambiguate skills install ForEach 2025-12-21 00:47:49 +00:00
Peter Steinberger 78263e81f1 fix(mac): restore skills install ForEach 2025-12-21 00:46:38 +00:00
Peter Steinberger 053c8d5731 feat(gateway): add tailscale auth + pam 2025-12-21 00:44:39 +00:00
Peter Steinberger d69064f364 fix(gateway): avoid crash in handshake auth 2025-12-21 00:41:06 +00:00
Peter Steinberger fedb24caf1 fix(ui): stabilize skills action column 2025-12-21 00:37:29 +00:00
Peter Steinberger 6ff8371254 feat(ui): expand control dashboard 2025-12-21 00:34:39 +00:00
Peter Steinberger 7b6eaa819e chore: ignore ClawdisKit .swiftpm 2025-12-21 01:10:06 +01:00
Peter Steinberger e94aa296e2 feat: refine skills install actions 2025-12-21 01:07:35 +01:00
Peter Steinberger 98891103d0 fix: streamline WhatsApp login flow 2025-12-21 01:07:35 +01:00
Peter Steinberger 383097a03a fix: emit delta-only node system events 2025-12-21 01:07:35 +01:00
Peter Steinberger 2b2f13ca79 fix: restore canvas action bridge 2025-12-21 01:07:35 +01:00
Peter Steinberger 78159a9435 fix(onboarding): nudge bottom padding 2025-12-20 23:52:45 +00:00
Peter Steinberger b4af7b919e fix(macos): simplify skills view and resize onboarding 2025-12-20 23:45:50 +00:00
Peter Steinberger 65056915d3 fix(onboarding): lift bottom bar 2025-12-20 23:36:24 +00:00
Peter Steinberger bc3f744e45 chore(canvas): refresh a2ui bundle 2025-12-21 00:25:56 +01:00
Peter Steinberger fb8da15b01 chore(canvas): rebuild a2ui bundle 2025-12-21 00:25:56 +01:00
Peter Steinberger 62f624b66b fix(mac): re-ensure remote gateway tunnel 2025-12-21 00:25:56 +01:00
Peter Steinberger ef20053e72 style(tests): format gateway server test 2025-12-21 00:25:56 +01:00
Peter Steinberger aae68e4f82 style(chatui): fix SwiftFormat warnings 2025-12-21 00:25:56 +01:00
Peter Steinberger 1d715d7b1b chore(ios): link AppIntents framework 2025-12-21 00:24:24 +01:00
Peter Steinberger 1d7110ea8f fix(onboarding): fit chat card 2025-12-20 23:15:35 +00:00
Peter Steinberger 80f70a58e3 fix(chat): refine onboarding bubbles 2025-12-20 23:15:29 +00:00
Peter Steinberger f7aabeba04 chore(deps): update lockfile 2025-12-20 23:00:31 +00:00
Peter Steinberger 02f6cac9d6 style(chat): use integrated bubble tail 2025-12-20 23:00:21 +00:00
Peter Steinberger df54fc6098 test(gateway): cover provider status/logout RPCs 2025-12-20 23:51:36 +01:00
Peter Steinberger fe0fb8d296 chore(canvas): rebuild a2ui bundle 2025-12-20 22:45:15 +00:00
Peter Steinberger 591120a7f7 chore(deps): update dependencies 2025-12-20 22:45:15 +00:00
Peter Steinberger 878f074494 chore(android): update kotlin compiler settings 2025-12-20 23:43:28 +01:00
Peter Steinberger c1050da852 chore(android): update icons and platform config 2025-12-20 23:43:28 +01:00
Peter Steinberger 873daf079c feat(web): emit provider status updates 2025-12-20 23:43:27 +01:00
Peter Steinberger df9e4bdd63 chore(macos): tidy discovery and runtime 2025-12-20 23:43:27 +01:00
Peter Steinberger 43ba1671f1 feat(macos): add connections settings
# Conflicts:
#	apps/macos/Sources/Clawdis/SettingsRootView.swift
2025-12-20 23:43:27 +01:00
Peter Steinberger ce4b68d5fb fix: pre-size menu context card 2025-12-20 23:43:27 +01:00
Peter Steinberger 8c18dd40a3 feat(macos): load models from gateway 2025-12-20 23:43:27 +01:00
Peter Steinberger e3015bbfb7 test(gateway): cover models.list 2025-12-20 23:43:27 +01:00
Peter Steinberger 817abd8b5f feat(gateway): add models.list 2025-12-20 23:43:27 +01:00
Peter Steinberger dbc9b00de5 docs: improve oracle skill guidance 2025-12-20 23:41:07 +01:00
Peter Steinberger b635e83651 chore(pi): bump deps, drop steerable transport 2025-12-20 22:38:12 +00:00
Peter Steinberger 7aeacdcc6c style(settings): widen window 2025-12-20 22:23:15 +00:00
Peter Steinberger 16e4a0c4bd style(onboarding): refine bubble tails 2025-12-20 22:23:06 +00:00
Peter Steinberger d613800516 fix(onboarding): anchor bottom bar and reduce height 2025-12-20 22:16:13 +00:00
Peter Steinberger 94b89216f7 style(onboarding): add speech bubble tails 2025-12-20 22:08:01 +00:00
Peter Steinberger 153e09120a style(onboarding): lower bottom row 2025-12-20 22:07:51 +00:00
Peter Steinberger 238c0c1b86 fix(onboarding): clearer bubbles and tighter composer 2025-12-20 22:03:24 +00:00
Peter Steinberger 98ff213708 style(onboarding): lower bottom controls 2025-12-20 22:03:13 +00:00
Peter Steinberger 8a2a07eddb fix(macos): always show CLI installer 2025-12-20 22:00:51 +00:00
Peter Steinberger 9076d543f3 fix(onboarding): restore bubbles and spacing 2025-12-20 21:56:03 +00:00
Peter Steinberger cd77dc9563 fix(onboarding): restore chat bubble styling 2025-12-20 21:47:43 +00:00
Peter Steinberger 9ccf80848d style(onboarding): reduce window height 2025-12-20 21:33:56 +00:00
575 changed files with 50504 additions and 13909 deletions

No files matched your search

+41 -2
View File
@@ -34,7 +34,7 @@ jobs:
if: matrix.runtime == 'node' if: matrix.runtime == 'node'
uses: actions/setup-node@v4 uses: actions/setup-node@v4
with: with:
node-version: 22 node-version: 24
check-latest: true check-latest: true
- name: Setup Bun - name: Setup Bun
@@ -49,7 +49,7 @@ jobs:
if: matrix.runtime == 'bun' if: matrix.runtime == 'bun'
uses: actions/setup-node@v4 uses: actions/setup-node@v4
with: with:
node-version: 22 node-version: 24
check-latest: true check-latest: true
- name: Runtime versions - name: Runtime versions
@@ -106,6 +106,7 @@ jobs:
run: bunx tsc -p tsconfig.json run: bunx tsc -p tsconfig.json
macos-app: macos-app:
if: github.event_name == 'pull_request'
runs-on: macos-latest runs-on: macos-latest
steps: steps:
- name: Checkout - name: Checkout
@@ -170,6 +171,44 @@ jobs:
sleep $((attempt * 20)) sleep $((attempt * 20))
done done
exit 1 exit 1
ios:
if: false # ignore iOS in CI for now
runs-on: macos-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: false
- name: Checkout submodules (retry)
run: |
set -euo pipefail
git submodule sync --recursive
for attempt in 1 2 3 4 5; do
if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then
exit 0
fi
echo "Submodule update failed (attempt $attempt/5). Retrying…"
sleep $((attempt * 10))
done
exit 1
- name: Select Xcode 26.1
run: |
sudo xcode-select -s /Applications/Xcode_26.1.app
xcodebuild -version
- name: Install XcodeGen
run: brew install xcodegen
- name: Install SwiftLint / SwiftFormat
run: brew install swiftlint swiftformat
- name: Show toolchain
run: |
sw_vers
xcodebuild -version
swift --version
- name: Generate iOS project - name: Generate iOS project
run: | run: |
+5
View File
@@ -1,25 +1,30 @@
node_modules node_modules
.env .env
dist dist
*.bun-build
pnpm-lock.yaml pnpm-lock.yaml
coverage coverage
.pnpm-store .pnpm-store
.worktrees/ .worktrees/
.DS_Store .DS_Store
**/.DS_Store **/.DS_Store
ui/src/ui/__screenshots__/
# Bun build artifacts # Bun build artifacts
*.bun-build *.bun-build
apps/macos/.build/ apps/macos/.build/
apps/shared/ClawdisKit/.build/ apps/shared/ClawdisKit/.build/
bin/
bin/clawdis-mac bin/clawdis-mac
bin/docs-list bin/docs-list
apps/macos/.build-local/ apps/macos/.build-local/
apps/macos/.swiftpm/ apps/macos/.swiftpm/
apps/shared/ClawdisKit/.swiftpm/
Core/ Core/
apps/ios/*.xcodeproj/ apps/ios/*.xcodeproj/
apps/ios/*.xcworkspace/ apps/ios/*.xcworkspace/
apps/ios/.swiftpm/ apps/ios/.swiftpm/
vendor/
# Vendor build artifacts # Vendor build artifacts
vendor/a2ui/renderers/lit/dist/ vendor/a2ui/renderers/lit/dist/
+10 -2
View File
@@ -16,13 +16,14 @@
- Language: TypeScript (ESM). Prefer strict typing; avoid `any`. - Language: TypeScript (ESM). Prefer strict typing; avoid `any`.
- Formatting/linting via Biome; run `pnpm lint` before commits. - Formatting/linting via Biome; run `pnpm lint` before commits.
- Keep files concise; extract helpers instead of “V2” copies. Use existing patterns for CLI options and dependency injection via `createDefaultDeps`. - Keep files concise; extract helpers instead of “V2” copies. Use existing patterns for CLI options and dependency injection via `createDefaultDeps`.
- Keep every file ≤ 500 LOC; refactor or split before exceeding and check frequently. - Aim to keep files under ~700 LOC; guideline only (not a hard guardrail). Split/refactor when it improves clarity or testability.
## Testing Guidelines ## Testing Guidelines
- Framework: Vitest with V8 coverage thresholds (70% lines/branches/functions/statements). - Framework: Vitest with V8 coverage thresholds (70% lines/branches/functions/statements).
- Naming: match source names with `*.test.ts`; e2e in `*.e2e.test.ts`. - Naming: match source names with `*.test.ts`; e2e in `*.e2e.test.ts`.
- Run `pnpm test` (or `pnpm test:coverage`) before pushing when you touch logic. - Run `pnpm test` (or `pnpm test:coverage`) before pushing when you touch logic.
- Pure test additions/fixes generally do **not** need a changelog entry unless they alter user-facing behavior or the user asks for one. - Pure test additions/fixes generally do **not** need a changelog entry unless they alter user-facing behavior or the user asks for one.
- Mobile: before using a simulator, check for connected real devices (iOS + Android) and prefer them when available.
## Commit & Pull Request Guidelines ## Commit & Pull Request Guidelines
- Create commits with `scripts/committer "<msg>" <file...>`; avoid manual `git add`/`git commit` so staging stays scoped. - Create commits with `scripts/committer "<msg>" <file...>`; avoid manual `git add`/`git commit` so staging stays scoped.
@@ -40,10 +41,17 @@
- macOS logs: use `./scripts/clawlog.sh` (aka `vtlog`) to query unified logs for subsystem `com.steipete.clawdis`; it supports follow/tail/category filters and expects passwordless sudo for `/usr/bin/log`. - macOS logs: use `./scripts/clawlog.sh` (aka `vtlog`) to query unified logs for subsystem `com.steipete.clawdis`; it supports follow/tail/category filters and expects passwordless sudo for `/usr/bin/log`.
- Also read the shared guardrails at `~/Projects/oracle/AGENTS.md` and `~/Projects/agent-scripts/AGENTS.MD` before making changes; align with any cross-repo rules noted there. - Also read the shared guardrails at `~/Projects/oracle/AGENTS.md` and `~/Projects/agent-scripts/AGENTS.MD` before making changes; align with any cross-repo rules noted there.
- SwiftUI state management (iOS/macOS): prefer the `Observation` framework (`@Observable`, `@Bindable`) over `ObservableObject`/`@StateObject`; don’t introduce new `ObservableObject` unless required for compatibility, and migrate existing usages when touching related code. - SwiftUI state management (iOS/macOS): prefer the `Observation` framework (`@Observable`, `@Bindable`) over `ObservableObject`/`@StateObject`; don’t introduce new `ObservableObject` unless required for compatibility, and migrate existing usages when touching related code.
- **Restart apps:** “restart iOS/Android apps” means rebuild (recompile/install) and relaunch, not just kill/launch.
- **Device checks:** before testing, verify connected real devices (iOS/Android) before reaching for simulators/emulators.
- iOS Team ID lookup: `security find-identity -p codesigning -v` → use Apple Development (…) TEAMID. Fallback: `defaults read com.apple.dt.Xcode IDEProvisioningTeamIdentifiers`.
- A2UI bundle hash: `src/canvas-host/a2ui/.bundle.hash` is auto-generated; regenerate via `pnpm canvas:a2ui:bundle` (or `scripts/bundle-a2ui.sh`) instead of manual conflict resolution.
- Notary key file lives at `~/Library/CloudStorage/Dropbox/Backup/AppStore/AuthKey_NJF3NFGTS3.p8` (Sparkle keys live under `~/Library/CloudStorage/Dropbox/Backup/Sparkle`).
- **Multi-agent safety:** do **not** create/apply/drop `git stash` entries unless Peter explicitly asks (this includes `git pull --rebase --autostash`). Assume other agents may be working; keep unrelated WIP untouched and avoid cross-cutting state changes. - **Multi-agent safety:** do **not** create/apply/drop `git stash` entries unless Peter explicitly asks (this includes `git pull --rebase --autostash`). Assume other agents may be working; keep unrelated WIP untouched and avoid cross-cutting state changes.
- **Multi-agent safety:** when Peter says "push", you may `git pull --rebase` to integrate latest changes (never discard other agents' work). When Peter says "commit", scope to your changes only. When Peter says "commit all", commit everything in grouped chunks.
- **Multi-agent safety:** do **not** create/remove/modify `git worktree` checkouts (or edit `.worktrees/*`) unless Peter explicitly asks. - **Multi-agent safety:** do **not** create/remove/modify `git worktree` checkouts (or edit `.worktrees/*`) unless Peter explicitly asks.
- **Multi-agent safety:** do **not** switch branches / check out a different branch unless Peter explicitly asks. - **Multi-agent safety:** do **not** switch branches / check out a different branch unless Peter explicitly asks.
- When asked to open a “session” file, open the Pi session logs under `~/.clawdis/sessions/*.jsonl` (newest unless a specific ID is given), not the default `sessions.json`. - **Multi-agent safety:** running multiple agents is OK as long as each agent has its own session.
- When asked to open a “session” file, open the Pi session logs under `~/.clawdis/sessions/*.jsonl` (newest unless a specific ID is given), not the default `sessions.json`. If logs are needed from Mac Studio, SSH via Tailscale and read the same path there.
- Menubar dimming + restart flow mirrors Trimmy: use `scripts/restart-mac.sh` (kills all Clawdis variants, runs `swift build`, packages, relaunches). Icon dimming depends on MenuBarExtraAccess wiring in AppMain; keep `appearsDisabled` updates intact when touching the status item. - Menubar dimming + restart flow mirrors Trimmy: use `scripts/restart-mac.sh` (kills all Clawdis variants, runs `swift build`, packages, relaunches). Icon dimming depends on MenuBarExtraAccess wiring in AppMain; keep `appearsDisabled` updates intact when touching the status item.
- Never send streaming/partial replies to external messaging surfaces (WhatsApp, Telegram); only final replies should be delivered there. Streaming/tool events may still go to internal UIs/control channel. - Never send streaming/partial replies to external messaging surfaces (WhatsApp, Telegram); only final replies should be delivered there. Streaming/tool events may still go to internal UIs/control channel.
- Voice wake forwarding tips: - Voice wake forwarding tips:
+229 -7
View File
@@ -1,5 +1,227 @@
# Changelog # Changelog
## 2.0.0-beta5 — Unreleased
### Breaking
- Skills config schema moved under `skills.*`:
- `skillsLoad.extraDirs` → `skills.load.extraDirs`
- `skillsInstall.*` → `skills.install.*`
- per-skill config map moved to `skills.entries` (e.g. `skills.peekaboo.enabled` → `skills.entries.peekaboo.enabled`)
- new optional bundled allowlist: `skills.allowBundled` (only affects bundled skills)
### Features
- Talk mode: continuous speech conversations (macOS/iOS/Android) with ElevenLabs TTS, reply directives, and optional interrupt-on-speech.
- UI: add optional `ui.seamColor` accent to tint the Talk Mode side bubble (macOS/iOS/Android).
- Nix mode: opt-in declarative config + read-only settings UI when `CLAWDIS_NIX_MODE=1` (thanks @joshp123 for the persistence — earned my trust; I'll merge these going forward).
- Agent runtime: accept legacy `Z_AI_API_KEY` for Z.AI provider auth (maps to `ZAI_API_KEY`).
- Tests: add a Z.AI live test gate for smoke validation when keys are present.
- macOS Debug: add app log verbosity and rolling file log toggle for swift-log-backed app logs.
### Fixes
- macOS codesign: skip hardened runtime for ad-hoc signing and avoid empty options args (#70) — thanks @petter-b
- Docs/agent tools: clarify that browser `wait` should be avoided by default and used only in exceptional cases.
- Browser tools: `upload` supports auto-click refs, direct `inputRef`/`element` file inputs, and emits input/change after `setFiles` so JS-heavy sites pick up attachments.
- macOS: Voice Wake now fully tears down the Speech pipeline when disabled (cancel pending restarts, drop stale callbacks) to avoid high CPU in the background.
- macOS menu: add a Talk Mode action alongside the Open Dashboard/Chat/Canvas entries.
- macOS Debug: hide “Restart Gateway” when the app won’t start a local gateway (remote mode / attach-only).
- macOS Talk Mode: orb overlay refresh, ElevenLabs request logging, API key status in settings, and auto-select first voice when none is configured.
- macOS Talk Mode: add hard timeout around ElevenLabs TTS synthesis to avoid getting stuck “speaking” forever on hung requests.
- macOS Talk Mode: avoid stuck playback when the audio player never starts (fail-fast + watchdog).
- macOS Talk Mode: fix audio stop ordering so disabling Talk Mode always stops in-flight playback.
- macOS Talk Mode: throttle audio-level updates (avoid per-buffer task creation) to reduce CPU/task churn.
- macOS Talk Mode: increase overlay window size so wave rings don’t clip; close button is hover-only and closer to the orb.
- Talk Mode: fall back to system TTS when ElevenLabs is unavailable, returns non-audio, or playback fails (macOS/iOS/Android).
- Talk Mode: stream PCM on macOS/iOS for lower latency (incremental playback); Android continues MP3 streaming.
- Talk Mode: validate ElevenLabs v3 stability and latency tier directives before sending requests.
- iOS/Android Talk Mode: auto-select the first ElevenLabs voice when none is configured.
- ElevenLabs: add retry/backoff for 429/5xx and include content-type in errors for debugging.
- Talk Mode: align to the gateway’s main session key and fall back to history polling when chat events drop (prevents stuck “thinking” / missing messages).
- Talk Mode: treat history timestamps as seconds or milliseconds to avoid stale assistant picks (macOS/iOS/Android).
- Chat UI: clear streaming/tool bubbles when external runs finish, preventing duplicate assistant bubbles.
- Chat UI: user bubbles use `ui.seamColor` (fallback to a calmer default blue).
- Android Chat UI: use `onPrimary` for user bubble text to preserve contrast (thanks @Syhids).
- Control UI: sync sidebar navigation with the URL for deep-linking, and auto-scroll chat to the latest message.
- Control UI: disable Web Chat + Talk when no iOS/Android node is connected; refreshed Web Chat styling and keyboard send.
- macOS Web Chat: fix composer layout so the connection pill and send button stay inside the input field.
- macOS: bundle Control UI assets into the app relay so the packaged app can serve them (thanks @mbelinky).
- Talk Mode: wait for chat history to surface the assistant reply before starting TTS (macOS/iOS/Android).
- iOS Talk Mode: fix chat completion wait to time out even if no events arrive (prevents “Thinking…” hangs).
- iOS Talk Mode: keep recognition running during playback to support interrupt-on-speech.
- iOS Talk Mode: preserve directive voice/model overrides across config reloads and add ElevenLabs request timeouts.
- iOS/Android Talk Mode: explicitly `chat.subscribe` when Talk Mode is active, so completion events arrive even if the Chat UI isn’t open.
- Chat UI: refresh history when another client finishes a run in the same session, so Talk Mode + Voice Wake transcripts appear consistently.
- Gateway: `voice.transcript` now also maps agent bus output to `chat` events, ensuring chat UIs refresh for voice-triggered runs.
- iOS/Android: show a centered Talk Mode orb overlay while Talk Mode is enabled.
- Gateway config: inject `talk.apiKey` from `ELEVENLABS_API_KEY`/shell profile so nodes can fetch it on demand.
- Canvas A2UI: tag requests with `platform=android|ios|macos` and boost Android canvas background contrast.
- iOS/Android nodes: enable scrolling for loaded web pages in the Canvas WebView (default scaffold stays touch-first).
- macOS menu: device list now uses `node.list` (devices only; no agent/tool presence entries).
- macOS menu: device list now shows connected nodes only.
- macOS menu: device rows now pack platform/version on the first line, and command lists wrap in submenus.
- macOS menu: split device platform/version across first and second rows for better fit.
- iOS node: fix ReplayKit screen recording crash caused by queue isolation assertions during capture.
- iOS Talk Mode: avoid audio tap queue assertions when starting recognition.
- macOS: use $HOME/Library/pnpm for SSH PATH exports (thanks @mbelinky).
- iOS/Android nodes: bridge auto-connect refreshes stale tokens and settings now show richer bridge/device details.
- macOS: bundle device model resources to prevent Instances crashes (thanks @mbelinky).
- iOS/Android nodes: status pill now surfaces camera activity instead of overlay toasts.
- iOS/Android/macOS nodes: camera snaps recompress to keep base64 payloads under 5 MB.
- iOS/Android nodes: status pill now surfaces pairing, screen recording, voice wake, and foreground-required states.
- iOS/Android nodes: avoid duplicating “Gateway reconnecting…” when the bridge is already connecting.
- iOS/Android nodes: Talk Mode now lives on a side bubble (with an iOS toggle to hide it), and Android settings no longer show the Talk Mode switch.
- macOS menu: top status line now shows pending node pairing approvals (incl. repairs).
- CLI: avoid spurious gateway close errors after successful request/response cycles.
- Agent runtime: clamp tool-result images to the 5MB Anthropic limit to avoid hard request rejections.
- Tests: add Swift Testing coverage for camera errors and Kotest coverage for Android bridge endpoints.
## 2.0.0-beta4 — 2025-12-27
### Fixes
- Package contents: include Discord/hooks build outputs in the npm tarball to avoid missing module errors.
- Heartbeat replies now drop any output containing `HEARTBEAT_OK`, preventing stray emoji/text from being delivered.
- macOS menu now refreshes the control channel after the gateway starts and shows “Connecting to gateway…” while the gateway is coming up.
- macOS local mode now waits for the gateway to be ready before configuring the control channel, avoiding false “no connection” flashes.
- WhatsApp watchdog now forces a reconnect even if the socket close event stalls (force-close to unblock reconnect loop).
- Gateway presence now reports macOS product version (via `sw_vers`) instead of Darwin kernel version.
## 2.0.0-beta3 — 2025-12-27
### Highlights
- First-class Clawdis tools (browser, canvas, nodes, cron) replace the old `clawdis-*` skills; tool schemas are now injected directly into the agent runtime.
- Per-session model selection + custom model providers: `models.providers` merges into `~/.clawdis/agent/models.json` (merge/replace modes) for LiteLLM, local OpenAI-compatible servers, Anthropic proxies, etc.
- Group chat activation modes: per-group `/activation mention|always` command with status visibility.
- Discord bot transport for DMs and guild text channels, with allowlists + mention gating.
- Gateway webhooks: external `wake` and isolated `agent` hooks with dedicated token auth.
- Hook mappings + Gmail Pub/Sub helper (`clawdis hooks gmail setup/run`) with auto-renew + Tailscale Funnel support.
- Command queue modes + per-session overrides (`/queue ...`) and new `agent.maxConcurrent` cap for safe parallelism across sessions.
- Background bash tasks: `bash` auto-yields after 20s (or on demand) with a `process` tool to list/poll/log/write/kill sessions.
- Gateway in-process restart: `clawdis_gateway` tool action triggers a SIGUSR1 restart without needing a supervisor.
### Breaking
- Config refactor: `inbound.*` removed; use top-level `routing` (allowlists + group rules + transcription), `messages` (prefixes/timestamps), and `session` (scoping/store/mainKey). No legacy keys read.
- Heartbeat config moved to `agent.heartbeat`: set `every: "30m"` (duration string) and optional `model`. `agent.heartbeatMinutes` is removed, and heartbeats are disabled unless `agent.heartbeat.every` is set.
- Heartbeats now run via the gateway runner (main session) and deliver to the last used channel by default. WhatsApp reply-heartbeat behavior is removed; use `agent.heartbeat.target`/`to` (or `target: "none"`) to control delivery.
- Browser `act` no longer accepts CSS `selector`; use `snapshot` refs (default `ai`) or `evaluate` as an escape hatch.
### Fixes
- Heartbeat replies now strip repeated `HEARTBEAT_OK` tails to avoid accidental “OK OK” spam.
- Heartbeat delivery now uses the last non-empty payload, preventing tool preambles from swallowing the final reply.
- Heartbeats now skip WhatsApp delivery when the web provider is inactive or unlinked (instead of logging “no active gateway listener”).
- Heartbeat failure logs now include the error reason instead of `[object Object]`.
- Duration strings now accept `h` (hours) where durations are parsed (e.g., heartbeat intervals).
- WhatsApp inbound now normalizes more wrapper types so quoted reply bodies are extracted reliably.
- WhatsApp send now preserves existing JIDs (including group `@g.us`) instead of coercing to `@s.whatsapp.net`. (Thanks @arun-8687.)
- Telegram/WhatsApp: reply context stays in `Body`/`ReplyTo*`, but outbound replies no longer thread to the original message. (Thanks @joshp123 for the PR and follow-up question.)
- Suppressed libsignal session cleanup spam from console logs unless verbose mode is enabled.
- WhatsApp web creds persistence hardened; credentials are restored before auth checks and QR login auto-restarts if it stalls.
- Group chats now honor `routing.groupChat.requireMention=false` as the default activation when no per-group override exists.
- Gateway auth no longer supports PAM/system mode; use token or shared password.
- Tailscale Funnel now requires password auth (no token-only public exposure).
- Group `/new` resets now work with @mentions so activation guidance appears on fresh sessions.
- Group chat activation context is now injected into the system prompt at session start (and after activation changes), including /new greetings.
- Typing indicators now start only once a reply payload is produced (no "thinking" typing for silent runs).
- WhatsApp group typing now starts immediately only when the bot is mentioned; otherwise it waits until real output exists.
- Streamed `<think>` segments are stripped before partial replies are emitted.
- System prompt now tags allowlisted owner numbers as the user identity to avoid mistaken “friend” assumptions.
- LM Studio/Ollama replies now require <final> tags; streaming ignores content until <final> begins.
- LM Studio responses API: tools payloads no longer include `strict: null`, and LM Studio no longer gets forced `<think>/<final>` tags.
- Identity emoji no longer auto-prefixes replies (set `messages.responsePrefix` explicitly if desired).
- Model switches now enqueue a system event so the next run knows the active model.
- `/model status` now lists available models (same as `/model`).
- `process log` pagination is now line-based (omit `offset` to grab the last N lines).
- macOS WebChat: assistant bubbles now update correctly when toggling light/dark mode.
- macOS: avoid spawning a duplicate gateway process when an external listener already exists.
- Node bridge: when binding to a non-loopback host (e.g. Tailnet IP), also listens on `127.0.0.1` for local connections (without creating duplicate loopback listeners for `0.0.0.0`/`127.0.0.1` binds).
- UI perf: pause repeat animations when scenes are inactive (typing dots, onboarding glow, iOS status pulse), throttle voice overlay level updates, and reduce overlay focus churn.
- Canvas defaults/A2UI auto-nav aligned; debug status overlay centered; redundant await removed in `CanvasManager`.
- Gateway launchd loop fixed by removing redundant `kickstart -k`.
- CLI now hints when Peekaboo is unauthorized.
- WhatsApp web inbox listeners now clean up on close to avoid duplicate handlers.
- Gateway startup now brings up browser control before external providers; WhatsApp/Telegram/Discord auto-start can be disabled with `web.enabled`, `telegram.enabled`, or `discord.enabled`.
### Providers & Routing
- New Discord provider for DMs + guild text channels with allowlists and mention-gated replies by default.
- `routing.queue` now controls queue vs interrupt behavior globally + per surface (defaults: WhatsApp/Telegram interrupt, Discord/WebChat queue).
- `/queue <mode>` supports one-shot or per-session overrides; `/queue reset|default` clears overrides.
- `agent.maxConcurrent` caps global parallel runs while keeping per-session serialization.
### macOS app
- Update-ready state surfaced in the menu; menu sections regrouped with session submenus.
- Menu bar now shows a dedicated Nodes section under Context with inline rows, overflow submenu, and iconized actions.
- Nodes now expose consistent inline details with per-node submenus for quick copy of key fields.
- Node rows now show compact app versions (build numbers moved to submenus) and offer SSH launch from Bonjour when available.
- Menu actions are grouped below toggles; Open Canvas hides when disabled and Voice Wake now anchors the mic picker.
- Connections now include Discord provider status + configuration UI.
- Menu bar gains an Allow Camera toggle alongside Canvas.
- Session list polish: sleeping/disconnected/error states, usage bar restored, padding + bar sizing tuned, syncing menu removed, header hidden when disconnected.
- Chat UI polish: tool call cards + merged tool results, glass background, tighter composer spacing, visual effect host tweaks.
- OAuth storage moved; legacy session syncing metadata removed.
- Remote SSH tunnels now get health checks; Debug → Ports highlights unhealthy tunnels and offers Reset SSH tunnel.
- Menu bar session/node sections no longer reflow while open, keeping hover highlights aligned.
- Menu hover highlights now span the full width (including submenu arrows).
- Menu session rows now refresh while open without width changes (no more stuck “Loading sessions…”).
- Menu width no longer grows on hover when moving the mouse across rows.
- Context usage bars now have higher contrast in light mode.
- macOS node timeouts now share a single async timeout helper for consistent behavior.
- WebChat window defaults tightened (narrower width, edge-to-edge layout) and the SwiftUI tag removed from the title.
### Nodes & Canvas
- Debug status overlay gated and toggleable on macOS/iOS/Android nodes.
- Gateway now derives the canvas host URL via a shared helper for bridge + WS handshakes (avoids loopback pitfalls).
- `canvas a2ui push` validates JSONL with line errors, rejects v0.9 payloads, and supports `--text` quick renders.
- `nodes rename` lets you override paired node display names without editing JSON.
- Android scaffold asset cleanup; iOS canvas/voice wake adjustments.
### Logging & Observability
- New subsystem console formatter with color modes, shortened prefixes, and TTY detection; browser/gateway logs route through the subsystem logger.
- WhatsApp console output streamlined; chalk/tslog typing fixes.
### Web UI
- Chat is now the dashboard landing view; health status simplified; initial scroll animation removed.
### Build, Dev, Docs
- Notarization flow added for macOS release artifacts; packaging scripts updated.
- macOS signing auto-selects Developer ID → Apple Distribution → Apple Development; no ad-hoc fallback.
- Added type-aware oxlint; docs list resolves from cwd; formatting/lint cleanup and dependency bumps (Peekaboo).
- Docs refreshed for tools, custom model providers, Discord, queue/routing, group activation commands, logging, restart semantics, release notes, GitHub pages CTAs, and npm pitfalls.
- `pnpm build` now skips A2UI bundling for faster builds (run `pnpm canvas:a2ui:bundle` when needed).
### Tests
- Coverage added for models config merging, WhatsApp reply context, QR login flows, auto-reply behavior, and gateway SIGTERM timeouts.
- Added gateway webhook coverage (auth, validation, and summary posting).
- Vitest now isolates HOME/XDG config roots so tests never touch a real `~/.clawdis` install.
## 2.0.0-beta2 — 2025-12-21
Second beta focused on bundled gateway packaging, skills management, onboarding polish, and provider reliability.
### Highlights
- Bundled gateway packaging: bun-compiled embedded gateway, new `gateway-daemon` command, launchd support, DMG packaging (zip+DMG).
- Skills platform: managed/bundled skills, install metadata + installers (uv), skill search + website, media/transcription helpers.
- macOS app: new Connections settings w/ provider status + QR login, skills settings redesign w/ install targets, models list loaded from the Gateway, clearer local/remote gateway choices.
- Web/agent UX: tool summary streaming + runtime toggle, WhatsApp QR login tool, agent steering queue, voice wake routes to main session, workspace bootstrap ritual.
### Gateway & providers
- Gateway: `models.list`, provider status events + RPC coverage, tailscale auth + PAM, bind-mode config, enriched agent WS logs, safer upgrade socket handling, fixed handshake auth crash.
- WhatsApp Web: QR login flow improvements (logged-out clearing, wait flow), self-chat mode handling, removed batching delay, web inbox made non-blocking.
- Telegram: normalized chat IDs with clearer error reporting.
### Canvas & browser control
- Canvas host served on Gateway port; removed standalone canvasHost port config; restored action bridge; refreshed A2UI bundle + message context; bridge canvas host for nodes.
- A2UI full-screen gutters + status clearance after successful load to avoid overlay collisions.
- Browser control API simplified; added MCP tool dispatch + native actions; control server can start without Playwright; hook timeouts extended.
### macOS UI polish
- Onboarding chat UI: kickoff flow, bubble tails, spacing + bottom bar refinements, window sizing tweaks, show Dock icon during onboarding.
- Skills UI: stabilized action column, fixed install target access, refined list layout and sizing, always show CLI installer.
- Remote/local gateway: auto-enable local gateway, clearer labels, re-ensure remote tunnel, hide local bridge discovery in remote mode.
### Build, CI, deps
- Bundled playwright-core + chromium-bidi/long; bun gateway bytecode builds; swiftformat/biome CI fixes; iOS lint script updates; Android icon/compiler updates; ignored new ClawdisKit `.swiftpm` path.
### Docs
- README architecture refresh + npm header image fix; onboarding/bootstrap steps; skills install guidance + new skills; browser/canvas control docs; bundled gateway + DMG packaging notes.
## 2.0.0-beta1 — 2025-12-19 ## 2.0.0-beta1 — 2025-12-19
First Clawdis release post rebrand. This is a semver-major because we dropped legacy providers/agents and moved defaults to new paths while adding a full macOS companion app, a WebSocket Gateway, and an iOS node. First Clawdis release post rebrand. This is a semver-major because we dropped legacy providers/agents and moved defaults to new paths while adding a full macOS companion app, a WebSocket Gateway, and an iOS node.
@@ -9,7 +231,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
### Breaking ### Breaking
- Renamed to **Clawdis**: defaults now live under `~/.clawdis` (sessions in `~/.clawdis/sessions/`, IPC at `~/.clawdis/clawdis.sock`, logs in `/tmp/clawdis`). Launchd labels and config filenames follow the new name; legacy stores are copied forward on first run. - Renamed to **Clawdis**: defaults now live under `~/.clawdis` (sessions in `~/.clawdis/sessions/`, IPC at `~/.clawdis/clawdis.sock`, logs in `/tmp/clawdis`). Launchd labels and config filenames follow the new name; legacy stores are copied forward on first run.
- Pi only: `inbound.reply.agent.kind` accepts only `"pi"`, and the agent CLI/CLI flags for Claude/Codex/Gemini were removed. The Pi CLI runs in RPC mode with a persistent worker. - Pi only: only the embedded Pi runtime remains, and the agent CLI/CLI flags for Claude/Codex/Gemini were removed. The Pi CLI runs in RPC mode with a persistent worker.
- WhatsApp Web is the only transport; Twilio support and related CLI flags/tests were removed. - WhatsApp Web is the only transport; Twilio support and related CLI flags/tests were removed.
- Direct chats now collapse into a single `main` session by default (no config needed); groups stay isolated as `group:<jid>`. - Direct chats now collapse into a single `main` session by default (no config needed); groups stay isolated as `group:<jid>`.
- Gateway is now a loopback-only WebSocket daemon (`ws://127.0.0.1:18789`) that owns all providers/state; clients (CLI, WebChat, macOS app, nodes) connect to it. Start it explicitly (`clawdis gateway …`) or via Clawdis.app; helper subcommands no longer auto-spawn a gateway. - Gateway is now a loopback-only WebSocket daemon (`ws://127.0.0.1:18789`) that owns all providers/state; clients (CLI, WebChat, macOS app, nodes) connect to it. Start it explicitly (`clawdis gateway …`) or via Clawdis.app; helper subcommands no longer auto-spawn a gateway.
@@ -57,7 +279,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.5.0 — 2025-12-05 ## 1.5.0 — 2025-12-05
### Breaking ### Breaking
- Dropped all non-Pi agents (Claude, Codex, Gemini, Opencode); `inbound.reply.agent.kind` now only accepts `"pi"` and related CLI helpers have been removed. - Dropped all non-Pi agents (Claude, Codex, Gemini, Opencode); only the embedded Pi runtime remains and related CLI helpers have been removed.
- Removed Twilio support and all related commands/options (webhook/up/provider flags/wait-poll); CLAWDIS is Baileys Web-only. - Removed Twilio support and all related commands/options (webhook/up/provider flags/wait-poll); CLAWDIS is Baileys Web-only.
### Changes ### Changes
@@ -84,7 +306,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.4.0 — 2025-12-03 ## 1.4.0 — 2025-12-03
### Highlights ### Highlights
- **Thinking directives & state:** `/t|/think|/thinking <level>` (aliases off|minimal|low|medium|high|max/highest). Inline applies to that message; directive-only message pins the level for the session; `/think:off` clears. Resolution: inline > session override > `inbound.reply.thinkingDefault` > off. Pi gets `--thinking <level>` (except off); other agents append cue words (`think` → `think hard` → `think harder` → `ultrathink`). Heartbeat probe uses `HEARTBEAT /think:high`. - **Thinking directives & state:** `/t|/think|/thinking <level>` (aliases off|minimal|low|medium|high|max/highest). Inline applies to that message; directive-only message pins the level for the session; `/think:off` clears. Resolution: inline > session override > `agent.thinkingDefault` > off. Pi gets `--thinking <level>` (except off); other agents append cue words (`think` → `think hard` → `think harder` → `ultrathink`). Heartbeat probe uses `HEARTBEAT /think:high`.
- **Group chats (web provider):** Clawdis now fully supports WhatsApp groups: mention-gated triggers (including image-only @ mentions), recent group history injection, per-group sessions, sender attribution, and a first-turn primer with group subject/member roster; heartbeats are skipped for groups. - **Group chats (web provider):** Clawdis now fully supports WhatsApp groups: mention-gated triggers (including image-only @ mentions), recent group history injection, per-group sessions, sender attribution, and a first-turn primer with group subject/member roster; heartbeats are skipped for groups.
- **Group session primer:** The first turn of a group session now tells the agent it is in a WhatsApp group and lists known members/subject so it can address the right speaker. - **Group session primer:** The first turn of a group session now tells the agent it is in a WhatsApp group and lists known members/subject so it can address the right speaker.
- **Media failures are surfaced:** When a web auto-reply media fetch/send fails (e.g., HTTP 404), we now append a warning to the fallback text so you know the attachment was skipped. - **Media failures are surfaced:** When a web auto-reply media fetch/send fails (e.g., HTTP 404), we now append a warning to the fallback text so you know the attachment was skipped.
@@ -126,7 +348,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.3.0 — 2025-12-02 ## 1.3.0 — 2025-12-02
### Highlights ### Highlights
- **Pluggable agents (Claude, Pi, Codex, Opencode):** `inbound.reply.agent` selects CLI/parser; per-agent argv builders and NDJSON parsers enable swapping without template changes. - **Pluggable agents (Claude, Pi, Codex, Opencode):** agent selection via config/CLI plus per-agent argv builders and NDJSON parsers enable swapping without template changes.
- **Safety stop words:** `stop|esc|abort|wait|exit` immediately reply “Agent was aborted.” and mark the session so the next prompt is prefixed with an abort reminder. - **Safety stop words:** `stop|esc|abort|wait|exit` immediately reply “Agent was aborted.” and mark the session so the next prompt is prefixed with an abort reminder.
- **Agent session reliability:** Only Claude returns a stable `session_id`; others may reset between runs. - **Agent session reliability:** Only Claude returns a stable `session_id`; others may reset between runs.
@@ -143,7 +365,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
- Batched inbound messages with timestamps; typing indicator after sends. - Batched inbound messages with timestamps; typing indicator after sends.
- Watchdog restarts WhatsApp after long inactivity; heartbeat logging includes minutes since last message. - Watchdog restarts WhatsApp after long inactivity; heartbeat logging includes minutes since last message.
- Early `allowFrom` filtering before decryption. - Early `allowFrom` filtering before decryption.
- Same-phone mode with echo detection and optional `inbound.samePhoneMarker`. - Same-phone mode with echo detection and optional message prefix marker.
## 1.2.2 — 2025-11-28 ## 1.2.2 — 2025-11-28
@@ -173,10 +395,10 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
## 1.1.0 — 2025-11-26 ## 1.1.0 — 2025-11-26
### Changes ### Changes
- Web auto-replies resize/recompress media and honor `inbound.reply.mediaMaxMb`. - Web auto-replies resize/recompress media and honor `agent.mediaMaxMb`.
- Detect media kind, enforce provider caps (images ≤6MB, audio/video ≤16MB, docs ≤100MB). - Detect media kind, enforce provider caps (images ≤6MB, audio/video ≤16MB, docs ≤100MB).
- `session.sendSystemOnce` and optional `sessionIntro`. - `session.sendSystemOnce` and optional `sessionIntro`.
- Typing indicator refresh during commands; configurable via `inbound.reply.typingIntervalSeconds`. - Typing indicator refresh during commands; configurable via `agent.typingIntervalSeconds`.
- Optional audio transcription via external CLI. - Optional audio transcription via external CLI.
- Command replies return structured payload/meta; respect `mediaMaxMb`; log Claude metadata; include `cwd` in timeout messages. - Command replies return structured payload/meta; respect `mediaMaxMb`; log Claude metadata; include `cwd` in timeout messages.
- Web provider refactor; logout command; web-only gateway start helper. - Web provider refactor; logout command; web-only gateway start helper.
Submodule Peekaboo updated: be03d6e3df...9db365b73c.
+49 -7
View File
@@ -1,7 +1,7 @@
# 🦞 CLAWDIS — Personal AI Assistant # 🦞 CLAWDIS — Personal AI Assistant
<p align="center"> <p align="center">
<img src="docs/whatsapp-clawd.jpg" alt="CLAWDIS" width="400"> <img src="https://raw.githubusercontent.com/steipete/clawdis/main/docs/whatsapp-clawd.jpg" alt="CLAWDIS" width="400">
</p> </p>
<p align="center"> <p align="center">
@@ -15,10 +15,12 @@
</p> </p>
**Clawdis** is a *personal AI assistant* you run on your own devices. **Clawdis** is a *personal AI assistant* you run on your own devices.
It answers you on the surfaces you already use (WhatsApp, Telegram, WebChat), can speak and listen on macOS/iOS, and can render a live Canvas you control. The Gateway is just the control plane — the product is the assistant. It answers you on the surfaces you already use (WhatsApp, Telegram, Discord, WebChat), can speak and listen on macOS/iOS, and can render a live Canvas you control. The Gateway is just the control plane — the product is the assistant.
If you want a private, single-user assistant that feels local, fast, and always-on, this is it. If you want a private, single-user assistant that feels local, fast, and always-on, this is it.
Using Claude Pro/Max subscription? See `docs/onboarding.md` for the Anthropic OAuth setup.
``` ```
Your surfaces Your surfaces
│ │
@@ -38,11 +40,13 @@ Your surfaces
## What Clawdis does ## What Clawdis does
- **Personal assistant** — one user, one identity, one memory surface. - **Personal assistant** — one user, one identity, one memory surface.
- **Multi-surface inbox** — WhatsApp, Telegram, WebChat, macOS, iOS. - **Multi-surface inbox** — WhatsApp, Telegram, Discord, WebChat, macOS, iOS.
- **Voice wake + push-to-talk** — local speech recognition on macOS/iOS. - **Voice wake + push-to-talk** — local speech recognition on macOS/iOS.
- **Canvas** — a live visual workspace you can drive from the agent. - **Canvas** — a live visual workspace you can drive from the agent.
- **Automation-ready** — browser control, media handling, and tool streaming. - **Automation-ready** — browser control, media handling, and tool streaming.
- **Local-first control plane** — the Gateway owns state, everything else connects. - **Local-first control plane** — the Gateway owns state, everything else connects.
- **Group chats** — mention-based by default, `/activation always|mention` per group (owner-only).
- **Nix mode** — opt-in declarative config + read-only UI when `CLAWDIS_NIX_MODE=1`.
## How it works (short) ## How it works (short)
@@ -66,15 +70,29 @@ pnpm clawdis login
# Start the gateway # Start the gateway
pnpm clawdis gateway --port 18789 --verbose pnpm clawdis gateway --port 18789 --verbose
# Dev loop (auto-reload on TS changes)
pnpm gateway:watch
# Send a message # Send a message
pnpm clawdis send --to +1234567890 --message "Hello from Clawdis" pnpm clawdis send --to +1234567890 --message "Hello from Clawdis"
# Talk to the assistant (optionally deliver back to WhatsApp/Telegram) # Talk to the assistant (optionally deliver back to WhatsApp/Telegram/Discord)
pnpm clawdis agent --message "Ship checklist" --thinking high pnpm clawdis agent --message "Ship checklist" --thinking high
``` ```
If you run from source, prefer `pnpm clawdis …` (not global `clawdis`). If you run from source, prefer `pnpm clawdis …` (not global `clawdis`).
## Chat commands
Send these in WhatsApp/Telegram/WebChat (group commands are owner-only):
- `/status` — health + session info (group shows activation mode)
- `/new` or `/reset` — reset the session
- `/think <level>` — off|minimal|low|medium|high
- `/verbose on|off`
- `/restart` — restart the gateway (owner-only in groups)
- `/activation mention|always` — group activation toggle (groups only)
## Architecture ## Architecture
### TypeScript Gateway (src/gateway/server.ts) ### TypeScript Gateway (src/gateway/server.ts)
@@ -121,7 +139,7 @@ Runbook: `docs/ios/connect.md`.
## Agent workspace + skills ## Agent workspace + skills
- Workspace root: `~/clawd` (configurable via `inbound.workspace`). - Workspace root: `~/clawd` (configurable via `agent.workspace`).
- Injected prompt files: `AGENTS.md`, `SOUL.md`, `TOOLS.md`. - Injected prompt files: `AGENTS.md`, `SOUL.md`, `TOOLS.md`.
- Skills: `~/clawd/skills/<skill>/SKILL.md`. - Skills: `~/clawd/skills/<skill>/SKILL.md`.
@@ -131,7 +149,7 @@ Minimal `~/.clawdis/clawdis.json`:
```json5 ```json5
{ {
inbound: { routing: {
allowFrom: ["+1234567890"] allowFrom: ["+1234567890"]
} }
} }
@@ -140,7 +158,7 @@ Minimal `~/.clawdis/clawdis.json`:
### WhatsApp ### WhatsApp
- Link the device: `pnpm clawdis login` (stores creds in `~/.clawdis/credentials`). - Link the device: `pnpm clawdis login` (stores creds in `~/.clawdis/credentials`).
- Allowlist who can talk to the assistant via `inbound.allowFrom`. - Allowlist who can talk to the assistant via `routing.allowFrom`.
### Telegram ### Telegram
@@ -155,6 +173,19 @@ Minimal `~/.clawdis/clawdis.json`:
} }
``` ```
### Discord
- Set `DISCORD_BOT_TOKEN` or `discord.token` (env wins).
- Optional: set `discord.requireMention`, `discord.allowFrom`, or `discord.mediaMaxMb` as needed.
```json5
{
discord: {
token: "1234abcd"
}
}
```
Browser control (optional): Browser control (optional):
```json5 ```json5
@@ -171,10 +202,21 @@ Browser control (optional):
- [`docs/index.md`](docs/index.md) (overview) - [`docs/index.md`](docs/index.md) (overview)
- [`docs/configuration.md`](docs/configuration.md) - [`docs/configuration.md`](docs/configuration.md)
- [`docs/group-messages.md`](docs/group-messages.md)
- [`docs/gateway.md`](docs/gateway.md) - [`docs/gateway.md`](docs/gateway.md)
- [`docs/web.md`](docs/web.md) - [`docs/web.md`](docs/web.md)
- [`docs/discovery.md`](docs/discovery.md) - [`docs/discovery.md`](docs/discovery.md)
- [`docs/agent.md`](docs/agent.md) - [`docs/agent.md`](docs/agent.md)
- [`docs/discord.md`](docs/discord.md)
- Webhooks + external triggers: [`docs/webhook.md`](docs/webhook.md)
- Gmail hooks (email → wake): [`docs/gmail-pubsub.md`](docs/gmail-pubsub.md)
## Email hooks (Gmail)
```bash
clawdis hooks gmail setup --account you@gmail.com
clawdis hooks gmail run
```
- [`docs/security.md`](docs/security.md) - [`docs/security.md`](docs/security.md)
- [`docs/troubleshooting.md`](docs/troubleshooting.md) - [`docs/troubleshooting.md`](docs/troubleshooting.md)
- [`docs/ios/connect.md`](docs/ios/connect.md) - [`docs/ios/connect.md`](docs/ios/connect.md)
+11
View File
@@ -0,0 +1,11 @@
# Changelog
## 0.2.0 — 2025-12-23
### Highlights
- Added `SwabbleKit` (multi-platform wake-word gate utilities with segment-aware gap detection).
- Swabble package now supports iOS + macOS consumers; CLI remains macOS 26-only.
### Changes
- CLI wake-word matching/stripping routed through `SwabbleKit` helpers.
- Speech pipeline types now explicitly gated to macOS 26 / iOS 26 availability.
+1 -10
View File
@@ -1,15 +1,6 @@
{ {
"originHash" : "3018b2c8c183d55b57ad0c4526b2380ac3b957d13a3a86e1b2845e81323c443a", "originHash" : "2012d083159d375d07febbc184c592c569d7ab48247045e35a762e3269d4cadc",
"pins" : [ "pins" : [
{
"identity" : "commander",
"kind" : "remoteSourceControl",
"location" : "https://github.com/steipete/Commander.git",
"state" : {
"revision" : "8b8cb4f34315ce9e5307b3a2bcd77ff73f586a02",
"version" : "0.2.0"
}
},
{ {
"identity" : "swift-syntax", "identity" : "swift-syntax",
"kind" : "remoteSourceControl", "kind" : "remoteSourceControl",
+21 -2
View File
@@ -4,14 +4,16 @@ import PackageDescription
let package = Package( let package = Package(
name: "swabble", name: "swabble",
platforms: [ platforms: [
.macOS(.v26), .macOS(.v15),
.iOS(.v17),
], ],
products: [ products: [
.library(name: "Swabble", targets: ["Swabble"]), .library(name: "Swabble", targets: ["Swabble"]),
.library(name: "SwabbleKit", targets: ["SwabbleKit"]),
.executable(name: "swabble", targets: ["SwabbleCLI"]), .executable(name: "swabble", targets: ["SwabbleCLI"]),
], ],
dependencies: [ dependencies: [
.package(url: "https://github.com/steipete/Commander.git", from: "0.2.0"), .package(path: "../Peekaboo/Commander"),
.package(url: "https://github.com/apple/swift-testing", from: "0.99.0"), .package(url: "https://github.com/apple/swift-testing", from: "0.99.0"),
], ],
targets: [ targets: [
@@ -19,13 +21,30 @@ let package = Package(
name: "Swabble", name: "Swabble",
path: "Sources/SwabbleCore", path: "Sources/SwabbleCore",
swiftSettings: []), swiftSettings: []),
.target(
name: "SwabbleKit",
path: "Sources/SwabbleKit",
swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"),
]),
.executableTarget( .executableTarget(
name: "SwabbleCLI", name: "SwabbleCLI",
dependencies: [ dependencies: [
"Swabble", "Swabble",
"SwabbleKit",
.product(name: "Commander", package: "Commander"), .product(name: "Commander", package: "Commander"),
], ],
path: "Sources/swabble"), path: "Sources/swabble"),
.testTarget(
name: "SwabbleKitTests",
dependencies: [
"SwabbleKit",
.product(name: "Testing", package: "swift-testing"),
],
swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"),
.enableExperimentalFeature("SwiftTesting"),
]),
.testTarget( .testTarget(
name: "swabbleTests", name: "swabbleTests",
dependencies: [ dependencies: [
+8 -4
View File
@@ -1,9 +1,10 @@
# 🎙️ swabble — Speech.framework wake-word hook daemon (macOS 26) # 🎙️ swabble — Speech.framework wake-word hook daemon (macOS 26)
swabble is a Swift 6.2, macOS 26-only rewrite of the brabble voice daemon. It listens on your mic, gates on a wake word, transcribes locally using Apple's new SpeechAnalyzer + SpeechTranscriber, then fires a shell hook with the transcript. No cloud calls, no Whisper binaries. swabble is a Swift 6.2 wake-word hook daemon. The CLI targets macOS 26 (SpeechAnalyzer + SpeechTranscriber). The shared `SwabbleKit` target is multi-platform and exposes wake-word gating utilities for iOS/macOS apps.
- **Local-only**: Speech.framework on-device models; zero network usage. - **Local-only**: Speech.framework on-device models; zero network usage.
- **Wake word**: Default `clawd` (aliases `claude`), optional `--no-wake` bypass. - **Wake word**: Default `clawd` (aliases `claude`), optional `--no-wake` bypass.
- **SwabbleKit**: Shared wake gate utilities (gap-based gating when you provide speech segments).
- **Hooks**: Run any command with prefix/env, cooldown, min_chars, timeout. - **Hooks**: Run any command with prefix/env, cooldown, min_chars, timeout.
- **Services**: launchd helper stubs for start/stop/install. - **Services**: launchd helper stubs for start/stop/install.
- **File transcribe**: TXT or SRT with time ranges (using AttributedString splits). - **File transcribe**: TXT or SRT with time ranges (using AttributedString splits).
@@ -30,7 +31,7 @@ swift run swabble transcribe /path/to/audio.m4a --format srt --output out.srt
``` ```
## Use as a library ## Use as a library
Add swabble as a SwiftPM dependency and import the `Swabble` product to reuse the Speech pipeline, config loader, hook executor, and transcript store in your own app: Add swabble as a SwiftPM dependency and import the `Swabble` or `SwabbleKit` product:
```swift ```swift
// Package.swift // Package.swift
@@ -38,7 +39,10 @@ dependencies: [
.package(url: "https://github.com/steipete/swabble.git", branch: "main"), .package(url: "https://github.com/steipete/swabble.git", branch: "main"),
], ],
targets: [ targets: [
.target(name: "MyApp", dependencies: [.product(name: "Swabble", package: "swabble")]), .target(name: "MyApp", dependencies: [
.product(name: "Swabble", package: "swabble"), // Speech pipeline (macOS 26+ / iOS 26+)
.product(name: "SwabbleKit", package: "swabble"), // Wake-word gate utilities (iOS 17+ / macOS 15+)
]),
] ]
``` ```
@@ -93,7 +97,7 @@ Environment variables:
## Speech pipeline ## Speech pipeline
- `AVAudioEngine` tap → `BufferConverter` → `AnalyzerInput` → `SpeechAnalyzer` with a `SpeechTranscriber` module. - `AVAudioEngine` tap → `BufferConverter` → `AnalyzerInput` → `SpeechAnalyzer` with a `SpeechTranscriber` module.
- Requests volatile + final results; wake gating is string match on partial/final. - Requests volatile + final results; the CLI uses text-only wake gating today.
- Authorization requested at first start; requires macOS 26 + new Speech.framework APIs. - Authorization requested at first start; requires macOS 26 + new Speech.framework APIs.
## Development ## Development
@@ -2,11 +2,13 @@ import AVFoundation
import Foundation import Foundation
import Speech import Speech
@available(macOS 26.0, iOS 26.0, *)
public struct SpeechSegment: Sendable { public struct SpeechSegment: Sendable {
public let text: String public let text: String
public let isFinal: Bool public let isFinal: Bool
} }
@available(macOS 26.0, iOS 26.0, *)
public enum SpeechPipelineError: Error { public enum SpeechPipelineError: Error {
case authorizationDenied case authorizationDenied
case analyzerFormatUnavailable case analyzerFormatUnavailable
@@ -14,6 +16,7 @@ public enum SpeechPipelineError: Error {
} }
/// Live microphone → SpeechAnalyzer → SpeechTranscriber pipeline. /// Live microphone → SpeechAnalyzer → SpeechTranscriber pipeline.
@available(macOS 26.0, iOS 26.0, *)
public actor SpeechPipeline { public actor SpeechPipeline {
private struct UnsafeBuffer: @unchecked Sendable { let buffer: AVAudioPCMBuffer } private struct UnsafeBuffer: @unchecked Sendable { let buffer: AVAudioPCMBuffer }
@@ -0,0 +1,192 @@
import Foundation
public struct WakeWordSegment: Sendable, Equatable {
public let text: String
public let start: TimeInterval
public let duration: TimeInterval
public let range: Range<String.Index>?
public init(text: String, start: TimeInterval, duration: TimeInterval, range: Range<String.Index>? = nil) {
self.text = text
self.start = start
self.duration = duration
self.range = range
}
public var end: TimeInterval { self.start + self.duration }
}
public struct WakeWordGateConfig: Sendable, Equatable {
public var triggers: [String]
public var minPostTriggerGap: TimeInterval
public var minCommandLength: Int
public init(
triggers: [String],
minPostTriggerGap: TimeInterval = 0.45,
minCommandLength: Int = 1)
{
self.triggers = triggers
self.minPostTriggerGap = minPostTriggerGap
self.minCommandLength = minCommandLength
}
}
public struct WakeWordGateMatch: Sendable, Equatable {
public let triggerEndTime: TimeInterval
public let postGap: TimeInterval
public let command: String
public init(triggerEndTime: TimeInterval, postGap: TimeInterval, command: String) {
self.triggerEndTime = triggerEndTime
self.postGap = postGap
self.command = command
}
}
public enum WakeWordGate {
private struct Token {
let normalized: String
let start: TimeInterval
let end: TimeInterval
let range: Range<String.Index>?
let text: String
}
private struct TriggerTokens {
let tokens: [String]
}
public static func match(
transcript: String,
segments: [WakeWordSegment],
config: WakeWordGateConfig)
-> WakeWordGateMatch? {
let triggerTokens = self.normalizeTriggers(config.triggers)
guard !triggerTokens.isEmpty else { return nil }
let tokens = self.normalizeSegments(segments)
guard !tokens.isEmpty else { return nil }
var best: (index: Int, triggerEnd: TimeInterval, gap: TimeInterval)?
for trigger in triggerTokens {
let count = trigger.tokens.count
guard count > 0, tokens.count > count else { continue }
for i in 0...(tokens.count - count - 1) {
let matched = (0..<count).allSatisfy { tokens[i + $0].normalized == trigger.tokens[$0] }
if !matched { continue }
let triggerEnd = tokens[i + count - 1].end
let nextToken = tokens[i + count]
let gap = nextToken.start - triggerEnd
if gap < config.minPostTriggerGap { continue }
if let best, i <= best.index { continue }
best = (i, triggerEnd, gap)
}
}
guard let best else { return nil }
let command = self.commandText(transcript: transcript, segments: segments, triggerEndTime: best.triggerEnd)
.trimmingCharacters(in: Self.whitespaceAndPunctuation)
guard command.count >= config.minCommandLength else { return nil }
return WakeWordGateMatch(triggerEndTime: best.triggerEnd, postGap: best.gap, command: command)
}
public static func commandText(
transcript: String,
segments: [WakeWordSegment],
triggerEndTime: TimeInterval)
-> String {
let threshold = triggerEndTime + 0.001
for segment in segments where segment.start >= threshold {
if normalizeToken(segment.text).isEmpty { continue }
if let range = segment.range {
let slice = transcript[range.lowerBound...]
return String(slice).trimmingCharacters(in: Self.whitespaceAndPunctuation)
}
break
}
let text = segments
.filter { $0.start >= threshold && !self.normalizeToken($0.text).isEmpty }
.map(\.text)
.joined(separator: " ")
return text.trimmingCharacters(in: Self.whitespaceAndPunctuation)
}
public static func matchesTextOnly(text: String, triggers: [String]) -> Bool {
guard !text.isEmpty else { return false }
let normalized = text.lowercased()
for trigger in triggers {
let token = trigger.trimmingCharacters(in: self.whitespaceAndPunctuation).lowercased()
if token.isEmpty { continue }
if normalized.contains(token) { return true }
}
return false
}
public static func stripWake(text: String, triggers: [String]) -> String {
var out = text
for trigger in triggers {
let token = trigger.trimmingCharacters(in: self.whitespaceAndPunctuation)
guard !token.isEmpty else { continue }
out = out.replacingOccurrences(of: token, with: "", options: [.caseInsensitive])
}
return out.trimmingCharacters(in: self.whitespaceAndPunctuation)
}
private static func normalizeTriggers(_ triggers: [String]) -> [TriggerTokens] {
var output: [TriggerTokens] = []
for trigger in triggers {
let tokens = trigger
.split(whereSeparator: { $0.isWhitespace })
.map { self.normalizeToken(String($0)) }
.filter { !$0.isEmpty }
if tokens.isEmpty { continue }
output.append(TriggerTokens(tokens: tokens))
}
return output
}
private static func normalizeSegments(_ segments: [WakeWordSegment]) -> [Token] {
segments.compactMap { segment in
let normalized = self.normalizeToken(segment.text)
guard !normalized.isEmpty else { return nil }
return Token(
normalized: normalized,
start: segment.start,
end: segment.end,
range: segment.range,
text: segment.text)
}
}
private static func normalizeToken(_ token: String) -> String {
token
.trimmingCharacters(in: self.whitespaceAndPunctuation)
.lowercased()
}
private static let whitespaceAndPunctuation = CharacterSet.whitespacesAndNewlines
.union(.punctuationCharacters)
}
#if canImport(Speech)
import Speech
public enum WakeWordSpeechSegments {
public static func from(transcription: SFTranscription, transcript: String) -> [WakeWordSegment] {
transcription.segments.map { segment in
let range = Range(segment.substringRange, in: transcript)
return WakeWordSegment(
text: segment.substring,
start: segment.timestamp,
duration: segment.duration,
range: range)
}
}
}
#endif
@@ -1,6 +1,7 @@
import Commander import Commander
import Foundation import Foundation
@available(macOS 26.0, *)
@MainActor @MainActor
enum CLIRegistry { enum CLIRegistry {
static var descriptors: [CommandDescriptor] { static var descriptors: [CommandDescriptor] {
@@ -1,7 +1,9 @@
import Commander import Commander
import Foundation import Foundation
import Swabble import Swabble
import SwabbleKit
@available(macOS 26.0, *)
@MainActor @MainActor
struct ServeCommand: ParsableCommand { struct ServeCommand: ParsableCommand {
@Option(name: .long("config"), help: "Path to config JSON") var configPath: String? @Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
@@ -68,17 +70,12 @@ struct ServeCommand: ParsableCommand {
} }
private static func matchesWake(text: String, cfg: SwabbleConfig) -> Bool { private static func matchesWake(text: String, cfg: SwabbleConfig) -> Bool {
let lowered = text.lowercased() let triggers = [cfg.wake.word] + cfg.wake.aliases
if lowered.contains(cfg.wake.word.lowercased()) { return true } return WakeWordGate.matchesTextOnly(text: text, triggers: triggers)
return cfg.wake.aliases.contains(where: { lowered.contains($0.lowercased()) })
} }
private static func stripWake(text: String, cfg: SwabbleConfig) -> String { private static func stripWake(text: String, cfg: SwabbleConfig) -> String {
var out = text let triggers = [cfg.wake.word] + cfg.wake.aliases
out = out.replacingOccurrences(of: cfg.wake.word, with: "", options: [.caseInsensitive]) return WakeWordGate.stripWake(text: text, triggers: triggers)
for alias in cfg.wake.aliases {
out = out.replacingOccurrences(of: alias, with: "", options: [.caseInsensitive])
}
return out.trimmingCharacters(in: .whitespacesAndNewlines)
} }
} }
+9 -2
View File
@@ -1,6 +1,7 @@
import Commander import Commander
import Foundation import Foundation
@available(macOS 26.0, *)
@MainActor @MainActor
private func runCLI() async -> Int32 { private func runCLI() async -> Int32 {
do { do {
@@ -15,6 +16,7 @@ private func runCLI() async -> Int32 {
} }
} }
@available(macOS 26.0, *)
@MainActor @MainActor
private func dispatch(invocation: CommandInvocation) async throws { private func dispatch(invocation: CommandInvocation) async throws {
let parsed = invocation.parsedValues let parsed = invocation.parsedValues
@@ -95,5 +97,10 @@ private func dispatch(invocation: CommandInvocation) async throws {
} }
} }
let exitCode = await runCLI() if #available(macOS 26.0, *) {
exit(exitCode) let exitCode = await runCLI()
exit(exitCode)
} else {
fputs("error: swabble requires macOS 26 or newer\n", stderr)
exit(1)
}
@@ -0,0 +1,63 @@
import Foundation
import Testing
import SwabbleKit
@Suite struct WakeWordGateTests {
@Test func matchRequiresGapAfterTrigger() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.35, 0.1),
("thing", 0.5, 0.1),
])
let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
#expect(WakeWordGate.match(transcript: transcript, segments: segments, config: config) == nil)
}
@Test func matchAllowsGapAndExtractsCommand() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.9, 0.1),
("thing", 1.1, 0.1),
])
let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
#expect(match?.command == "do thing")
}
@Test func matchHandlesMultiWordTriggers() {
let transcript = "hey clawd do it"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.8, 0.1),
("it", 1.0, 0.1),
])
let config = WakeWordGateConfig(triggers: ["hey clawd"], minPostTriggerGap: 0.3)
let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
#expect(match?.command == "do it")
}
}
private func makeSegments(
transcript: String,
words: [(String, TimeInterval, TimeInterval)])
-> [WakeWordSegment] {
var searchStart = transcript.startIndex
var output: [WakeWordSegment] = []
for (word, start, duration) in words {
let range = transcript.range(of: word, range: searchStart..<transcript.endIndex)
output.append(WakeWordSegment(text: word, start: start, duration: duration, range: range))
if let range { searchStart = range.upperBound }
}
return output
}
+4 -3
View File
@@ -1,11 +1,12 @@
# swabble — macOS 26 speech hook daemon (Swift 6.2) # swabble — macOS 26 speech hook daemon (Swift 6.2)
Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framework (SpeechAnalyzer + SpeechTranscriber) instead of whisper.cpp. Local-only, wake word gated, dispatches a shell hook with the transcript. Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framework (SpeechAnalyzer + SpeechTranscriber) instead of whisper.cpp. Local-only, wake word gated, dispatches a shell hook with the transcript. Shared wake-gate utilities live in `SwabbleKit` for reuse by other apps (iOS/macOS).
## Requirements ## Requirements
- macOS 26+, Swift 6.2, Speech.framework with on-device assets. - macOS 26+, Swift 6.2, Speech.framework with on-device assets.
- Local only; no network calls during transcription. - Local only; no network calls during transcription.
- Wake word gating (default "clawd" plus aliases) with bypass flag `--no-wake`. - Wake word gating (default "clawd" plus aliases) with bypass flag `--no-wake`.
- `SwabbleKit` target (multi-platform) providing wake-word gating helpers that can use speech segment timing to require a post-trigger gap.
- Hook execution with cooldown, min_chars, timeout, prefix, env vars. - Hook execution with cooldown, min_chars, timeout, prefix, env vars.
- Simple config at `~/.config/swabble/config.json` (JSON, Codable) — no TOML. - Simple config at `~/.config/swabble/config.json` (JSON, Codable) — no TOML.
- CLI implemented with Commander (SwiftPM package `steipete/Commander`); core types are available via the SwiftPM library product `Swabble` for embedding. - CLI implemented with Commander (SwiftPM package `steipete/Commander`); core types are available via the SwiftPM library product `Swabble` for embedding.
@@ -17,7 +18,7 @@ Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framewo
- **CLI layer (Commander)**: Root command `swabble` with subcommands `serve`, `transcribe`, `test-hook`, `mic list|set`, `doctor`, `health`, `tail-log`. Runtime flags from Commander (`-v/--verbose`, `--json-output`, `--log-level`). Custom `--config` path applies everywhere. - **CLI layer (Commander)**: Root command `swabble` with subcommands `serve`, `transcribe`, `test-hook`, `mic list|set`, `doctor`, `health`, `tail-log`. Runtime flags from Commander (`-v/--verbose`, `--json-output`, `--log-level`). Custom `--config` path applies everywhere.
- **Config**: `SwabbleConfig` Codable. Fields: audio device name/index, wake (enabled/word/aliases/sensitivity placeholder), hook (command/args/prefix/cooldown/min_chars/timeout/env), logging (level, format), transcripts (enabled, max kept), speech (locale, enableEtiquetteReplacements flag). Stored JSON; default written by `setup`. - **Config**: `SwabbleConfig` Codable. Fields: audio device name/index, wake (enabled/word/aliases/sensitivity placeholder), hook (command/args/prefix/cooldown/min_chars/timeout/env), logging (level, format), transcripts (enabled, max kept), speech (locale, enableEtiquetteReplacements flag). Stored JSON; default written by `setup`.
- **Audio + Speech pipeline**: `SpeechPipeline` wraps `AVAudioEngine` input → `SpeechAnalyzer` with `SpeechTranscriber` module. Emits partial/final transcripts via async stream. Requests `.audioTimeRange` when transcripts enabled. Handles Speech permission and asset download prompts ahead of capture. - **Audio + Speech pipeline**: `SpeechPipeline` wraps `AVAudioEngine` input → `SpeechAnalyzer` with `SpeechTranscriber` module. Emits partial/final transcripts via async stream. Requests `.audioTimeRange` when transcripts enabled. Handles Speech permission and asset download prompts ahead of capture.
- **Wake gate**: text-based keyword match against latest partial/final; strips wake term before hook dispatch. `--no-wake` disables. - **Wake gate**: CLI currently uses text-only keyword match; shared `SwabbleKit` gate can enforce a minimum pause between the wake word and the next token when speech segments are available. `--no-wake` disables gating.
- **Hook executor**: async `HookExecutor` spawns `Process` with configured args, prefix substitution `${hostname}`. Enforces cooldown + timeout; injects env `SWABBLE_TEXT`, `SWABBLE_PREFIX` plus user env map. - **Hook executor**: async `HookExecutor` spawns `Process` with configured args, prefix substitution `${hostname}`. Enforces cooldown + timeout; injects env `SWABBLE_TEXT`, `SWABBLE_PREFIX` plus user env map.
- **Transcripts store**: in-memory ring buffer; optional persisted JSON lines under `~/Library/Application Support/swabble/transcripts.log`. - **Transcripts store**: in-memory ring buffer; optional persisted JSON lines under `~/Library/Application Support/swabble/transcripts.log`.
- **Logging**: simple structured logger to stderr; respects log level. - **Logging**: simple structured logger to stderr; respects log level.
@@ -25,7 +26,7 @@ Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framewo
## Out of scope (initial cut) ## Out of scope (initial cut)
- Model management (Speech handles assets). - Model management (Speech handles assets).
- Launchd helper (planned follow-up). - Launchd helper (planned follow-up).
- Advanced wake-word detector (text match only for now). - Advanced wake-word detector (segment-aware gate now lives in `SwabbleKit`; CLI still text-only until segment timing is plumbed through).
## Open decisions ## Open decisions
- Whether to expose a UNIX control socket for `status`/`health` (currently planned as stdin/out direct calls). - Whether to expose a UNIX control socket for `status`/`health` (currently planned as stdin/out direct calls).
+30 -20
View File
@@ -1,21 +1,31 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" standalone="yes"?>
<rss version="2.0" <rss xmlns:sparkle="http://www.andymatuschak.org/xml-namespaces/sparkle" version="2.0">
xmlns:sparkle="http://www.andymatuschak.org/xml-namespaces/sparkle" <channel>
xmlns:dc="http://purl.org/dc/elements/1.1/"> <title>Clawdis</title>
<channel> <item>
<title>Clawdis Updates</title> <title>2.0.0-beta3</title>
<link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link> <pubDate>Sat, 27 Dec 2025 19:02:02 +0100</pubDate>
<description>Signed update feed for the Clawdis macOS companion app.</description> <link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link>
<item> <sparkle:version>2.0.0-beta3</sparkle:version>
<title>Clawdis 2.0.0-beta1</title> <sparkle:shortVersionString>2.0.0-beta3</sparkle:shortVersionString>
<sparkle:releaseNotesLink>https://github.com/steipete/clawdis/releases/tag/v2.0.0-beta1</sparkle:releaseNotesLink> <sparkle:minimumSystemVersion>15.0</sparkle:minimumSystemVersion>
<pubDate>Fri, 19 Dec 2025 17:19:50 +0000</pubDate> <enclosure url="https://github.com/steipete/clawdis/releases/download/v2.0.0-beta3/Clawdis-2.0.0-beta3.zip" length="70407960" type="application/octet-stream" sparkle:edSignature="A8ySMmbLRrpIkqkrmc9QrC+6om8Iyqray/6x/YNiJxDoJeXjp2T5t8XT0CKJeNBUlDkzIj/fwiK53v0qQ59cDQ=="/>
<enclosure url="https://github.com/steipete/clawdis/releases/download/v2.0.0-beta1/Clawdis-2.0.0-beta1.zip" </item>
sparkle:edSignature="oEpGD46U4ZyBBSY9/piUIFDJU+KlFB751JIWOW2yS0sRNHKszyG5khDHg9o9bV9Zo8DOCNF/HOi88jmtHJAaCQ==" <item>
sparkle:version="2.0.0-beta1" <title>2.0.0-beta4</title>
sparkle:shortVersionString="2.0.0-beta1" <pubDate>Sat, 27 Dec 2025 19:43:22 +0100</pubDate>
length="72410016" <link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link>
type="application/octet-stream" /> <sparkle:version>2.0.0-beta4</sparkle:version>
</item> <sparkle:shortVersionString>2.0.0-beta4</sparkle:shortVersionString>
</channel> <sparkle:minimumSystemVersion>15.0</sparkle:minimumSystemVersion>
<description><![CDATA[<h2>Clawdis 2.0.0-beta4</h2>
<h3>Fixes</h3>
<ul>
<li>Package contents: include Discord/hooks build outputs in the npm tarball to avoid missing module errors.</li>
</ul>
<p><a href="https://github.com/steipete/clawdis/blob/main/CHANGELOG.md">View full changelog</a></p>
]]></description>
<enclosure url="https://github.com/steipete/clawdis/releases/download/v2.0.0-beta4/Clawdis-2.0.0-beta4.zip" length="70407894" type="application/octet-stream" sparkle:edSignature="HmuiC7TnUn80ZApnKfb6w+JGSrjc3uUOndMrtbTp42bkBSVifbttNVazqvJueGBo4MgoJV8CP+zQNzVmtVihAQ=="/>
</item>
</channel>
</rss> </rss>
+21 -6
View File
@@ -20,7 +20,7 @@ android {
minSdk = 31 minSdk = 31
targetSdk = 36 targetSdk = 36
versionCode = 1 versionCode = 1
versionName = "0.1" versionName = "2.0.0-beta3"
} }
buildTypes { buildTypes {
@@ -31,6 +31,7 @@ android {
buildFeatures { buildFeatures {
compose = true compose = true
buildConfig = true
} }
compileOptions { compileOptions {
@@ -38,15 +39,21 @@ android {
targetCompatibility = JavaVersion.VERSION_17 targetCompatibility = JavaVersion.VERSION_17
} }
kotlinOptions {
jvmTarget = "17"
}
packaging { packaging {
resources { resources {
excludes += "/META-INF/{AL2.0,LGPL2.1}" excludes += "/META-INF/{AL2.0,LGPL2.1}"
} }
} }
lint {
disable += setOf("IconLauncherShape")
}
}
kotlin {
compilerOptions {
jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17)
}
} }
dependencies { dependencies {
@@ -56,7 +63,8 @@ dependencies {
implementation("androidx.core:core-ktx:1.17.0") implementation("androidx.core:core-ktx:1.17.0")
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.10.0") implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.10.0")
implementation("androidx.activity:activity-compose:1.12.1") implementation("androidx.activity:activity-compose:1.12.2")
implementation("androidx.webkit:webkit:1.14.0")
implementation("androidx.compose.ui:ui") implementation("androidx.compose.ui:ui")
implementation("androidx.compose.ui:ui-tooling-preview") implementation("androidx.compose.ui:ui-tooling-preview")
@@ -86,4 +94,11 @@ dependencies {
testImplementation("junit:junit:4.13.2") testImplementation("junit:junit:4.13.2")
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
testImplementation("io.kotest:kotest-runner-junit5-jvm:6.0.7")
testImplementation("io.kotest:kotest-assertions-core-jvm:6.0.7")
testRuntimeOnly("org.junit.vintage:junit-vintage-engine:5.13.3")
}
tasks.withType<Test>().configureEach {
useJUnitPlatform()
} }
@@ -12,12 +12,20 @@
<uses-permission <uses-permission
android:name="android.permission.ACCESS_FINE_LOCATION" android:name="android.permission.ACCESS_FINE_LOCATION"
android:maxSdkVersion="32" /> android:maxSdkVersion="32" />
<uses-permission
android:name="android.permission.ACCESS_COARSE_LOCATION"
android:maxSdkVersion="32" />
<uses-permission android:name="android.permission.CAMERA" /> <uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" /> <uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-feature
android:name="android.hardware.camera"
android:required="false" />
<application <application
android:name=".NodeApp" android:name=".NodeApp"
android:allowBackup="true" android:allowBackup="true"
android:dataExtractionRules="@xml/data_extraction_rules"
android:fullBackupContent="@xml/backup_rules"
android:icon="@mipmap/ic_launcher" android:icon="@mipmap/ic_launcher"
android:roundIcon="@mipmap/ic_launcher_round" android:roundIcon="@mipmap/ic_launcher_round"
android:label="@string/app_name" android:label="@string/app_name"
@@ -23,9 +23,12 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val statusText: StateFlow<String> = runtime.statusText val statusText: StateFlow<String> = runtime.statusText
val serverName: StateFlow<String?> = runtime.serverName val serverName: StateFlow<String?> = runtime.serverName
val remoteAddress: StateFlow<String?> = runtime.remoteAddress val remoteAddress: StateFlow<String?> = runtime.remoteAddress
val isForeground: StateFlow<Boolean> = runtime.isForeground
val seamColorArgb: StateFlow<Long> = runtime.seamColorArgb
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
val screenRecordActive: StateFlow<Boolean> = runtime.screenRecordActive
val instanceId: StateFlow<String> = runtime.instanceId val instanceId: StateFlow<String> = runtime.instanceId
val displayName: StateFlow<String> = runtime.displayName val displayName: StateFlow<String> = runtime.displayName
@@ -35,9 +38,14 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode
val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening
val talkEnabled: StateFlow<Boolean> = runtime.talkEnabled
val talkStatusText: StateFlow<String> = runtime.talkStatusText
val talkIsListening: StateFlow<Boolean> = runtime.talkIsListening
val talkIsSpeaking: StateFlow<Boolean> = runtime.talkIsSpeaking
val manualEnabled: StateFlow<Boolean> = runtime.manualEnabled val manualEnabled: StateFlow<Boolean> = runtime.manualEnabled
val manualHost: StateFlow<String> = runtime.manualHost val manualHost: StateFlow<String> = runtime.manualHost
val manualPort: StateFlow<Int> = runtime.manualPort val manualPort: StateFlow<Int> = runtime.manualPort
val canvasDebugStatusEnabled: StateFlow<Boolean> = runtime.canvasDebugStatusEnabled
val chatSessionKey: StateFlow<String> = runtime.chatSessionKey val chatSessionKey: StateFlow<String> = runtime.chatSessionKey
val chatSessionId: StateFlow<String?> = runtime.chatSessionId val chatSessionId: StateFlow<String?> = runtime.chatSessionId
@@ -78,6 +86,10 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.setManualPort(value) runtime.setManualPort(value)
} }
fun setCanvasDebugStatusEnabled(value: Boolean) {
runtime.setCanvasDebugStatusEnabled(value)
}
fun setWakeWords(words: List<String>) { fun setWakeWords(words: List<String>) {
runtime.setWakeWords(words) runtime.setWakeWords(words)
} }
@@ -90,6 +102,10 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.setVoiceWakeMode(mode) runtime.setVoiceWakeMode(mode)
} }
fun setTalkEnabled(enabled: Boolean) {
runtime.setTalkEnabled(enabled)
}
fun connect(endpoint: BridgeEndpoint) { fun connect(endpoint: BridgeEndpoint) {
runtime.connect(endpoint) runtime.connect(endpoint)
} }
@@ -10,7 +10,6 @@ import android.content.Context
import android.content.Intent import android.content.Intent
import android.content.pm.PackageManager import android.content.pm.PackageManager
import android.content.pm.ServiceInfo import android.content.pm.ServiceInfo
import android.os.Build
import androidx.core.app.NotificationCompat import androidx.core.app.NotificationCompat
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
@@ -85,7 +84,6 @@ class NodeForegroundService : Service() {
override fun onBind(intent: Intent?) = null override fun onBind(intent: Intent?) = null
private fun ensureChannel() { private fun ensureChannel() {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return
val mgr = getSystemService(NotificationManager::class.java) val mgr = getSystemService(NotificationManager::class.java)
val channel = val channel =
NotificationChannel( NotificationChannel(
@@ -101,12 +99,7 @@ class NodeForegroundService : Service() {
private fun buildNotification(title: String, text: String): Notification { private fun buildNotification(title: String, text: String): Notification {
val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP) val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP)
val flags = val flags = PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
} else {
PendingIntent.FLAG_UPDATE_CURRENT
}
val stopPending = PendingIntent.getService(this, 2, stopIntent, flags) val stopPending = PendingIntent.getService(this, 2, stopIntent, flags)
return NotificationCompat.Builder(this, CHANNEL_ID) return NotificationCompat.Builder(this, CHANNEL_ID)
@@ -126,11 +119,6 @@ class NodeForegroundService : Service() {
} }
private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) { private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) {
if (Build.VERSION.SDK_INT < 29) {
startForeground(NOTIFICATION_ID, notification)
return
}
if (didStartForeground && requiresMic == lastRequiresMic) { if (didStartForeground && requiresMic == lastRequiresMic) {
updateNotification(notification) updateNotification(notification)
return return
@@ -162,11 +150,7 @@ class NodeForegroundService : Service() {
fun start(context: Context) { fun start(context: Context) {
val intent = Intent(context, NodeForegroundService::class.java) val intent = Intent(context, NodeForegroundService::class.java)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { context.startForegroundService(intent)
context.startForegroundService(intent)
} else {
context.startService(intent)
}
} }
fun stop(context: Context) { fun stop(context: Context) {
@@ -16,6 +16,7 @@ import com.steipete.clawdis.node.bridge.BridgeEndpoint
import com.steipete.clawdis.node.bridge.BridgePairingClient import com.steipete.clawdis.node.bridge.BridgePairingClient
import com.steipete.clawdis.node.bridge.BridgeSession import com.steipete.clawdis.node.bridge.BridgeSession
import com.steipete.clawdis.node.node.CameraCaptureManager import com.steipete.clawdis.node.node.CameraCaptureManager
import com.steipete.clawdis.node.BuildConfig
import com.steipete.clawdis.node.node.CanvasController import com.steipete.clawdis.node.node.CanvasController
import com.steipete.clawdis.node.node.ScreenRecordManager import com.steipete.clawdis.node.node.ScreenRecordManager
import com.steipete.clawdis.node.protocol.ClawdisCapability import com.steipete.clawdis.node.protocol.ClawdisCapability
@@ -24,6 +25,7 @@ import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UIAction
import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UICommand import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UICommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasCommand import com.steipete.clawdis.node.protocol.ClawdisCanvasCommand
import com.steipete.clawdis.node.protocol.ClawdisScreenCommand import com.steipete.clawdis.node.protocol.ClawdisScreenCommand
import com.steipete.clawdis.node.voice.TalkModeManager
import com.steipete.clawdis.node.voice.VoiceWakeManager import com.steipete.clawdis.node.voice.VoiceWakeManager
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
@@ -68,7 +70,7 @@ class NodeRuntime(context: Context) {
payloadJson = payloadJson =
buildJsonObject { buildJsonObject {
put("message", JsonPrimitive(command)) put("message", JsonPrimitive(command))
put("sessionKey", JsonPrimitive("main")) put("sessionKey", JsonPrimitive(mainSessionKey.value))
put("thinking", JsonPrimitive(chatThinkingLevel.value)) put("thinking", JsonPrimitive(chatThinkingLevel.value))
put("deliver", JsonPrimitive(false)) put("deliver", JsonPrimitive(false))
}.toString(), }.toString(),
@@ -83,6 +85,15 @@ class NodeRuntime(context: Context) {
val voiceWakeStatusText: StateFlow<String> val voiceWakeStatusText: StateFlow<String>
get() = voiceWake.statusText get() = voiceWake.statusText
val talkStatusText: StateFlow<String>
get() = talkMode.statusText
val talkIsListening: StateFlow<Boolean>
get() = talkMode.isListening
val talkIsSpeaking: StateFlow<Boolean>
get() = talkMode.isSpeaking
private val discovery = BridgeDiscovery(appContext, scope = scope) private val discovery = BridgeDiscovery(appContext, scope = scope)
val bridges: StateFlow<List<BridgeEndpoint>> = discovery.bridges val bridges: StateFlow<List<BridgeEndpoint>> = discovery.bridges
val discoveryStatusText: StateFlow<String> = discovery.statusText val discoveryStatusText: StateFlow<String> = discovery.statusText
@@ -93,6 +104,9 @@ class NodeRuntime(context: Context) {
private val _statusText = MutableStateFlow("Offline") private val _statusText = MutableStateFlow("Offline")
val statusText: StateFlow<String> = _statusText.asStateFlow() val statusText: StateFlow<String> = _statusText.asStateFlow()
private val _mainSessionKey = MutableStateFlow("main")
val mainSessionKey: StateFlow<String> = _mainSessionKey.asStateFlow()
private val cameraHudSeq = AtomicLong(0) private val cameraHudSeq = AtomicLong(0)
private val _cameraHud = MutableStateFlow<CameraHudState?>(null) private val _cameraHud = MutableStateFlow<CameraHudState?>(null)
val cameraHud: StateFlow<CameraHudState?> = _cameraHud.asStateFlow() val cameraHud: StateFlow<CameraHudState?> = _cameraHud.asStateFlow()
@@ -100,15 +114,23 @@ class NodeRuntime(context: Context) {
private val _cameraFlashToken = MutableStateFlow(0L) private val _cameraFlashToken = MutableStateFlow(0L)
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow() val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
private val _screenRecordActive = MutableStateFlow(false)
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
private val _serverName = MutableStateFlow<String?>(null) private val _serverName = MutableStateFlow<String?>(null)
val serverName: StateFlow<String?> = _serverName.asStateFlow() val serverName: StateFlow<String?> = _serverName.asStateFlow()
private val _remoteAddress = MutableStateFlow<String?>(null) private val _remoteAddress = MutableStateFlow<String?>(null)
val remoteAddress: StateFlow<String?> = _remoteAddress.asStateFlow() val remoteAddress: StateFlow<String?> = _remoteAddress.asStateFlow()
private val _seamColorArgb = MutableStateFlow(DEFAULT_SEAM_COLOR_ARGB)
val seamColorArgb: StateFlow<Long> = _seamColorArgb.asStateFlow()
private val _isForeground = MutableStateFlow(true) private val _isForeground = MutableStateFlow(true)
val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow() val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow()
private var lastAutoA2uiUrl: String? = null
private val session = private val session =
BridgeSession( BridgeSession(
scope = scope, scope = scope,
@@ -117,7 +139,10 @@ class NodeRuntime(context: Context) {
_serverName.value = name _serverName.value = name
_remoteAddress.value = remote _remoteAddress.value = remote
_isConnected.value = true _isConnected.value = true
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
scope.launch { refreshBrandingFromGateway() }
scope.launch { refreshWakeWordsFromGateway() } scope.launch { refreshWakeWordsFromGateway() }
maybeNavigateToA2uiOnConnect()
}, },
onDisconnected = { message -> handleSessionDisconnected(message) }, onDisconnected = { message -> handleSessionDisconnected(message) },
onEvent = { event, payloadJson -> onEvent = { event, payloadJson ->
@@ -129,13 +154,33 @@ class NodeRuntime(context: Context) {
) )
private val chat = ChatController(scope = scope, session = session, json = json) private val chat = ChatController(scope = scope, session = session, json = json)
private val talkMode: TalkModeManager by lazy {
TalkModeManager(context = appContext, scope = scope).also { it.attachSession(session) }
}
private fun handleSessionDisconnected(message: String) { private fun handleSessionDisconnected(message: String) {
_statusText.value = message _statusText.value = message
_serverName.value = null _serverName.value = null
_remoteAddress.value = null _remoteAddress.value = null
_isConnected.value = false _isConnected.value = false
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
_mainSessionKey.value = "main"
chat.onDisconnected(message) chat.onDisconnected(message)
showLocalCanvasOnDisconnect()
}
private fun maybeNavigateToA2uiOnConnect() {
val a2uiUrl = resolveA2uiHostUrl() ?: return
val current = canvas.currentUrl()?.trim().orEmpty()
if (current.isEmpty() || current == lastAutoA2uiUrl) {
lastAutoA2uiUrl = a2uiUrl
canvas.navigate(a2uiUrl)
}
}
private fun showLocalCanvasOnDisconnect() {
lastAutoA2uiUrl = null
canvas.navigate("")
} }
val instanceId: StateFlow<String> = prefs.instanceId val instanceId: StateFlow<String> = prefs.instanceId
@@ -144,10 +189,12 @@ class NodeRuntime(context: Context) {
val preventSleep: StateFlow<Boolean> = prefs.preventSleep val preventSleep: StateFlow<Boolean> = prefs.preventSleep
val wakeWords: StateFlow<List<String>> = prefs.wakeWords val wakeWords: StateFlow<List<String>> = prefs.wakeWords
val voiceWakeMode: StateFlow<VoiceWakeMode> = prefs.voiceWakeMode val voiceWakeMode: StateFlow<VoiceWakeMode> = prefs.voiceWakeMode
val talkEnabled: StateFlow<Boolean> = prefs.talkEnabled
val manualEnabled: StateFlow<Boolean> = prefs.manualEnabled val manualEnabled: StateFlow<Boolean> = prefs.manualEnabled
val manualHost: StateFlow<String> = prefs.manualHost val manualHost: StateFlow<String> = prefs.manualHost
val manualPort: StateFlow<Int> = prefs.manualPort val manualPort: StateFlow<Int> = prefs.manualPort
val lastDiscoveredStableId: StateFlow<String> = prefs.lastDiscoveredStableId val lastDiscoveredStableId: StateFlow<String> = prefs.lastDiscoveredStableId
val canvasDebugStatusEnabled: StateFlow<Boolean> = prefs.canvasDebugStatusEnabled
private var didAutoConnect = false private var didAutoConnect = false
private var suppressWakeWordsSync = false private var suppressWakeWordsSync = false
@@ -198,6 +245,13 @@ class NodeRuntime(context: Context) {
} }
} }
scope.launch {
talkEnabled.collect { enabled ->
talkMode.setEnabled(enabled)
externalAudioCaptureActive.value = enabled
}
}
scope.launch(Dispatchers.Default) { scope.launch(Dispatchers.Default) {
bridges.collect { list -> bridges.collect { list ->
if (list.isNotEmpty()) { if (list.isNotEmpty()) {
@@ -228,6 +282,22 @@ class NodeRuntime(context: Context) {
connect(target) connect(target)
} }
} }
scope.launch {
combine(
canvasDebugStatusEnabled,
statusText,
serverName,
remoteAddress,
) { debugEnabled, status, server, remote ->
Quad(debugEnabled, status, server, remote)
}.distinctUntilChanged()
.collect { (debugEnabled, status, server, remote) ->
canvas.setDebugStatusEnabled(debugEnabled)
if (!debugEnabled) return@collect
canvas.setDebugStatus(status, server ?: remote)
}
}
} }
fun setForeground(value: Boolean) { fun setForeground(value: Boolean) {
@@ -258,6 +328,10 @@ class NodeRuntime(context: Context) {
prefs.setManualPort(value) prefs.setManualPort(value)
} }
fun setCanvasDebugStatusEnabled(value: Boolean) {
prefs.setCanvasDebugStatusEnabled(value)
}
fun setWakeWords(words: List<String>) { fun setWakeWords(words: List<String>) {
prefs.setWakeWords(words) prefs.setWakeWords(words)
scheduleWakeWordsSyncIfNeeded() scheduleWakeWordsSyncIfNeeded()
@@ -271,6 +345,10 @@ class NodeRuntime(context: Context) {
prefs.setVoiceWakeMode(mode) prefs.setVoiceWakeMode(mode)
} }
fun setTalkEnabled(value: Boolean) {
prefs.setTalkEnabled(value)
}
fun connect(endpoint: BridgeEndpoint) { fun connect(endpoint: BridgeEndpoint) {
scope.launch { scope.launch {
_statusText.value = "Connecting…" _statusText.value = "Connecting…"
@@ -307,6 +385,13 @@ class NodeRuntime(context: Context) {
add(ClawdisCapability.VoiceWake.rawValue) add(ClawdisCapability.VoiceWake.rawValue)
} }
} }
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
val advertisedVersion =
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
BridgePairingClient().pairAndHello( BridgePairingClient().pairAndHello(
endpoint = endpoint, endpoint = endpoint,
hello = hello =
@@ -315,7 +400,7 @@ class NodeRuntime(context: Context) {
displayName = displayName.value, displayName = displayName.value,
token = null, token = null,
platform = "Android", platform = "Android",
version = "dev", version = advertisedVersion,
deviceFamily = "Android", deviceFamily = "Android",
modelIdentifier = modelIdentifier, modelIdentifier = modelIdentifier,
caps = caps, caps = caps,
@@ -333,6 +418,13 @@ class NodeRuntime(context: Context) {
val authToken = requireNotNull(resolved.token).trim() val authToken = requireNotNull(resolved.token).trim()
prefs.saveBridgeToken(authToken) prefs.saveBridgeToken(authToken)
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
val advertisedVersion =
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
session.connect( session.connect(
endpoint = endpoint, endpoint = endpoint,
hello = hello =
@@ -341,7 +433,7 @@ class NodeRuntime(context: Context) {
displayName = displayName.value, displayName = displayName.value,
token = authToken, token = authToken,
platform = "Android", platform = "Android",
version = "dev", version = advertisedVersion,
deviceFamily = "Android", deviceFamily = "Android",
modelIdentifier = modelIdentifier, modelIdentifier = modelIdentifier,
caps = caps =
@@ -396,8 +488,7 @@ class NodeRuntime(context: Context) {
val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty {
java.util.UUID.randomUUID().toString() java.util.UUID.randomUUID().toString()
} }
val name = (userActionObj["name"] as? JsonPrimitive)?.content?.trim().orEmpty() val name = ClawdisCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch
if (name.isEmpty()) return@launch
val surfaceId = val surfaceId =
(userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" } (userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" }
@@ -495,6 +586,7 @@ class NodeRuntime(context: Context) {
return return
} }
talkMode.handleBridgeEvent(event, payloadJson)
chat.handleBridgeEvent(event, payloadJson) chat.handleBridgeEvent(event, payloadJson)
} }
@@ -536,6 +628,25 @@ class NodeRuntime(context: Context) {
} }
} }
private suspend fun refreshBrandingFromGateway() {
if (!_isConnected.value) return
try {
val res = session.request("config.get", "{}")
val root = json.parseToJsonElement(res).asObjectOrNull()
val config = root?.get("config").asObjectOrNull()
val ui = config?.get("ui").asObjectOrNull()
val raw = ui?.get("seamColor").asStringOrNull()?.trim()
val sessionCfg = config?.get("session").asObjectOrNull()
val rawMainKey = sessionCfg?.get("mainKey").asStringOrNull()?.trim()
_mainSessionKey.value = rawMainKey?.takeIf { it.isNotEmpty() } ?: "main"
val parsed = parseHexColorArgb(raw)
_seamColorArgb.value = parsed ?: DEFAULT_SEAM_COLOR_ARGB
} catch (_: Throwable) {
// ignore
}
}
private suspend fun handleInvoke(command: String, paramsJson: String?): BridgeSession.InvokeResult { private suspend fun handleInvoke(command: String, paramsJson: String?): BridgeSession.InvokeResult {
if ( if (
command.startsWith(ClawdisCanvasCommand.NamespacePrefix) || command.startsWith(ClawdisCanvasCommand.NamespacePrefix) ||
@@ -677,14 +788,20 @@ class NodeRuntime(context: Context) {
} }
} }
ClawdisScreenCommand.Record.rawValue -> { ClawdisScreenCommand.Record.rawValue -> {
val res = // Status pill mirrors screen recording state so it stays visible without overlay stacking.
try { _screenRecordActive.value = true
screenRecorder.record(paramsJson) try {
} catch (err: Throwable) { val res =
val (code, message) = invokeErrorFromThrowable(err) try {
return BridgeSession.InvokeResult.error(code = code, message = message) screenRecorder.record(paramsJson)
} } catch (err: Throwable) {
BridgeSession.InvokeResult.ok(res.payloadJson) val (code, message) = invokeErrorFromThrowable(err)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
BridgeSession.InvokeResult.ok(res.payloadJson)
} finally {
_screenRecordActive.value = false
}
} }
else -> else ->
BridgeSession.InvokeResult.error( BridgeSession.InvokeResult.error(
@@ -727,7 +844,7 @@ class NodeRuntime(context: Context) {
val raw = session.currentCanvasHostUrl()?.trim().orEmpty() val raw = session.currentCanvasHostUrl()?.trim().orEmpty()
if (raw.isBlank()) return null if (raw.isBlank()) return null
val base = raw.trimEnd('/') val base = raw.trimEnd('/')
return "${base}/__clawdis__/a2ui/" return "${base}/__clawdis__/a2ui/?platform=android"
} }
private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean { private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
@@ -813,6 +930,8 @@ class NodeRuntime(context: Context) {
private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D) private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
private const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A
private const val a2uiReadyCheckJS: String = private const val a2uiReadyCheckJS: String =
""" """
(() => { (() => {
@@ -867,3 +986,12 @@ private fun JsonElement?.asStringOrNull(): String? =
is JsonPrimitive -> content is JsonPrimitive -> content
else -> null else -> null
} }
private fun parseHexColorArgb(raw: String?): Long? {
val trimmed = raw?.trim().orEmpty()
if (trimmed.isEmpty()) return null
val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed
if (hex.length != 6) return null
val rgb = hex.toLongOrNull(16) ?: return null
return 0xFF000000L or rgb
}
@@ -3,6 +3,7 @@
package com.steipete.clawdis.node package com.steipete.clawdis.node
import android.content.Context import android.content.Context
import androidx.core.content.edit
import androidx.security.crypto.EncryptedSharedPreferences import androidx.security.crypto.EncryptedSharedPreferences
import androidx.security.crypto.MasterKey import androidx.security.crypto.MasterKey
import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.MutableStateFlow
@@ -62,50 +63,62 @@ class SecurePrefs(context: Context) {
MutableStateFlow(prefs.getString("bridge.lastDiscoveredStableId", "")!!) MutableStateFlow(prefs.getString("bridge.lastDiscoveredStableId", "")!!)
val lastDiscoveredStableId: StateFlow<String> = _lastDiscoveredStableId val lastDiscoveredStableId: StateFlow<String> = _lastDiscoveredStableId
private val _canvasDebugStatusEnabled =
MutableStateFlow(prefs.getBoolean("canvas.debugStatusEnabled", false))
val canvasDebugStatusEnabled: StateFlow<Boolean> = _canvasDebugStatusEnabled
private val _wakeWords = MutableStateFlow(loadWakeWords()) private val _wakeWords = MutableStateFlow(loadWakeWords())
val wakeWords: StateFlow<List<String>> = _wakeWords val wakeWords: StateFlow<List<String>> = _wakeWords
private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode()) private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode())
val voiceWakeMode: StateFlow<VoiceWakeMode> = _voiceWakeMode val voiceWakeMode: StateFlow<VoiceWakeMode> = _voiceWakeMode
private val _talkEnabled = MutableStateFlow(prefs.getBoolean("talk.enabled", false))
val talkEnabled: StateFlow<Boolean> = _talkEnabled
fun setLastDiscoveredStableId(value: String) { fun setLastDiscoveredStableId(value: String) {
val trimmed = value.trim() val trimmed = value.trim()
prefs.edit().putString("bridge.lastDiscoveredStableId", trimmed).apply() prefs.edit { putString("bridge.lastDiscoveredStableId", trimmed) }
_lastDiscoveredStableId.value = trimmed _lastDiscoveredStableId.value = trimmed
} }
fun setDisplayName(value: String) { fun setDisplayName(value: String) {
val trimmed = value.trim() val trimmed = value.trim()
prefs.edit().putString(displayNameKey, trimmed).apply() prefs.edit { putString(displayNameKey, trimmed) }
_displayName.value = trimmed _displayName.value = trimmed
} }
fun setCameraEnabled(value: Boolean) { fun setCameraEnabled(value: Boolean) {
prefs.edit().putBoolean("camera.enabled", value).apply() prefs.edit { putBoolean("camera.enabled", value) }
_cameraEnabled.value = value _cameraEnabled.value = value
} }
fun setPreventSleep(value: Boolean) { fun setPreventSleep(value: Boolean) {
prefs.edit().putBoolean("screen.preventSleep", value).apply() prefs.edit { putBoolean("screen.preventSleep", value) }
_preventSleep.value = value _preventSleep.value = value
} }
fun setManualEnabled(value: Boolean) { fun setManualEnabled(value: Boolean) {
prefs.edit().putBoolean("bridge.manual.enabled", value).apply() prefs.edit { putBoolean("bridge.manual.enabled", value) }
_manualEnabled.value = value _manualEnabled.value = value
} }
fun setManualHost(value: String) { fun setManualHost(value: String) {
val trimmed = value.trim() val trimmed = value.trim()
prefs.edit().putString("bridge.manual.host", trimmed).apply() prefs.edit { putString("bridge.manual.host", trimmed) }
_manualHost.value = trimmed _manualHost.value = trimmed
} }
fun setManualPort(value: Int) { fun setManualPort(value: Int) {
prefs.edit().putInt("bridge.manual.port", value).apply() prefs.edit { putInt("bridge.manual.port", value) }
_manualPort.value = value _manualPort.value = value
} }
fun setCanvasDebugStatusEnabled(value: Boolean) {
prefs.edit { putBoolean("canvas.debugStatusEnabled", value) }
_canvasDebugStatusEnabled.value = value
}
fun loadBridgeToken(): String? { fun loadBridgeToken(): String? {
val key = "bridge.token.${_instanceId.value}" val key = "bridge.token.${_instanceId.value}"
return prefs.getString(key, null) return prefs.getString(key, null)
@@ -113,14 +126,14 @@ class SecurePrefs(context: Context) {
fun saveBridgeToken(token: String) { fun saveBridgeToken(token: String) {
val key = "bridge.token.${_instanceId.value}" val key = "bridge.token.${_instanceId.value}"
prefs.edit().putString(key, token.trim()).apply() prefs.edit { putString(key, token.trim()) }
} }
private fun loadOrCreateInstanceId(): String { private fun loadOrCreateInstanceId(): String {
val existing = prefs.getString("node.instanceId", null)?.trim() val existing = prefs.getString("node.instanceId", null)?.trim()
if (!existing.isNullOrBlank()) return existing if (!existing.isNullOrBlank()) return existing
val fresh = UUID.randomUUID().toString() val fresh = UUID.randomUUID().toString()
prefs.edit().putString("node.instanceId", fresh).apply() prefs.edit { putString("node.instanceId", fresh) }
return fresh return fresh
} }
@@ -131,7 +144,7 @@ class SecurePrefs(context: Context) {
val candidate = DeviceNames.bestDefaultNodeName(context).trim() val candidate = DeviceNames.bestDefaultNodeName(context).trim()
val resolved = candidate.ifEmpty { "Android Node" } val resolved = candidate.ifEmpty { "Android Node" }
prefs.edit().putString(displayNameKey, resolved).apply() prefs.edit { putString(displayNameKey, resolved) }
return resolved return resolved
} }
@@ -139,22 +152,27 @@ class SecurePrefs(context: Context) {
val sanitized = WakeWords.sanitize(words, defaultWakeWords) val sanitized = WakeWords.sanitize(words, defaultWakeWords)
val encoded = val encoded =
JsonArray(sanitized.map { JsonPrimitive(it) }).toString() JsonArray(sanitized.map { JsonPrimitive(it) }).toString()
prefs.edit().putString("voiceWake.triggerWords", encoded).apply() prefs.edit { putString("voiceWake.triggerWords", encoded) }
_wakeWords.value = sanitized _wakeWords.value = sanitized
} }
fun setVoiceWakeMode(mode: VoiceWakeMode) { fun setVoiceWakeMode(mode: VoiceWakeMode) {
prefs.edit().putString(voiceWakeModeKey, mode.rawValue).apply() prefs.edit { putString(voiceWakeModeKey, mode.rawValue) }
_voiceWakeMode.value = mode _voiceWakeMode.value = mode
} }
fun setTalkEnabled(value: Boolean) {
prefs.edit { putBoolean("talk.enabled", value) }
_talkEnabled.value = value
}
private fun loadVoiceWakeMode(): VoiceWakeMode { private fun loadVoiceWakeMode(): VoiceWakeMode {
val raw = prefs.getString(voiceWakeModeKey, null) val raw = prefs.getString(voiceWakeModeKey, null)
val resolved = VoiceWakeMode.fromRawValue(raw) val resolved = VoiceWakeMode.fromRawValue(raw)
// Default ON (foreground) when unset. // Default ON (foreground) when unset.
if (raw.isNullOrBlank()) { if (raw.isNullOrBlank()) {
prefs.edit().putString(voiceWakeModeKey, resolved.rawValue).apply() prefs.edit { putString(voiceWakeModeKey, resolved.rawValue) }
} }
return resolved return resolved
@@ -6,7 +6,6 @@ import android.net.DnsResolver
import android.net.NetworkCapabilities import android.net.NetworkCapabilities
import android.net.nsd.NsdManager import android.net.nsd.NsdManager
import android.net.nsd.NsdServiceInfo import android.net.nsd.NsdServiceInfo
import android.os.Build
import android.os.CancellationSignal import android.os.CancellationSignal
import android.util.Log import android.util.Log
import java.io.IOException import java.io.IOException
@@ -44,6 +43,7 @@ import org.xbill.DNS.Type
import kotlin.coroutines.resume import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException import kotlin.coroutines.resumeWithException
@Suppress("DEPRECATION")
class BridgeDiscovery( class BridgeDiscovery(
context: Context, context: Context,
private val scope: CoroutineScope, private val scope: CoroutineScope,
@@ -130,20 +130,36 @@ class BridgeDiscovery(
object : NsdManager.ResolveListener { object : NsdManager.ResolveListener {
override fun onResolveFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {} override fun onResolveFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {}
override fun onServiceResolved(resolved: NsdServiceInfo) { override fun onServiceResolved(resolved: NsdServiceInfo) {
val host = resolved.host?.hostAddress ?: return val host = resolved.host?.hostAddress ?: return
val port = resolved.port val port = resolved.port
if (port <= 0) return if (port <= 0) return
val rawServiceName = resolved.serviceName val rawServiceName = resolved.serviceName
val serviceName = BonjourEscapes.decode(rawServiceName) val serviceName = BonjourEscapes.decode(rawServiceName)
val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName) val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName)
val id = stableId(serviceName, "local.") val lanHost = txt(resolved, "lanHost")
localById[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port) val tailnetDns = txt(resolved, "tailnetDns")
publish() val gatewayPort = txtInt(resolved, "gatewayPort")
} val bridgePort = txtInt(resolved, "bridgePort")
}, val canvasPort = txtInt(resolved, "canvasPort")
) val id = stableId(serviceName, "local.")
localById[id] =
BridgeEndpoint(
stableId = id,
name = displayName,
host = host,
port = port,
lanHost = lanHost,
tailnetDns = tailnetDns,
gatewayPort = gatewayPort,
bridgePort = bridgePort,
canvasPort = canvasPort,
)
publish()
}
},
)
} }
private fun publish() { private fun publish() {
@@ -181,7 +197,6 @@ class BridgeDiscovery(
} }
private fun txt(info: NsdServiceInfo, key: String): String? { private fun txt(info: NsdServiceInfo, key: String): String? {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) return null
val bytes = info.attributes[key] ?: return null val bytes = info.attributes[key] ?: return null
return try { return try {
String(bytes, Charsets.UTF_8).trim().ifEmpty { null } String(bytes, Charsets.UTF_8).trim().ifEmpty { null }
@@ -190,6 +205,10 @@ class BridgeDiscovery(
} }
} }
private fun txtInt(info: NsdServiceInfo, key: String): Int? {
return txt(info, key)?.toIntOrNull()
}
private suspend fun refreshUnicast(domain: String) { private suspend fun refreshUnicast(domain: String) {
val ptrName = "${serviceType}${domain}" val ptrName = "${serviceType}${domain}"
val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return
@@ -228,8 +247,24 @@ class BridgeDiscovery(
} }
val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain)) val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain))
val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName) val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName)
val lanHost = txtValue(txt, "lanHost")
val tailnetDns = txtValue(txt, "tailnetDns")
val gatewayPort = txtIntValue(txt, "gatewayPort")
val bridgePort = txtIntValue(txt, "bridgePort")
val canvasPort = txtIntValue(txt, "canvasPort")
val id = stableId(instanceName, domain) val id = stableId(instanceName, domain)
next[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port) next[id] =
BridgeEndpoint(
stableId = id,
name = displayName,
host = host,
port = port,
lanHost = lanHost,
tailnetDns = tailnetDns,
gatewayPort = gatewayPort,
bridgePort = bridgePort,
canvasPort = canvasPort,
)
} }
unicastById.clear() unicastById.clear()
@@ -401,7 +436,7 @@ class BridgeDiscovery(
dns.rawQuery( dns.rawQuery(
network, network,
wireQuery, wireQuery,
0, DnsResolver.FLAG_EMPTY,
dnsExecutor, dnsExecutor,
signal, signal,
object : DnsResolver.Callback<ByteArray> { object : DnsResolver.Callback<ByteArray> {
@@ -435,6 +470,10 @@ class BridgeDiscovery(
return null return null
} }
private fun txtIntValue(records: List<TXTRecord>, key: String): Int? {
return txtValue(records, key)?.toIntOrNull()
}
private fun decodeDnsTxtString(raw: String): String { private fun decodeDnsTxtString(raw: String): String {
// dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes. // dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes.
// Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible. // Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible.
@@ -5,6 +5,11 @@ data class BridgeEndpoint(
val name: String, val name: String,
val host: String, val host: String,
val port: Int, val port: Int,
val lanHost: String? = null,
val tailnetDns: String? = null,
val gatewayPort: Int? = null,
val bridgePort: Int? = null,
val canvasPort: Int? = null,
) { ) {
companion object { companion object {
fun manual(host: String, port: Int): BridgeEndpoint = fun manual(host: String, port: Int): BridgeEndpoint =
@@ -16,4 +21,3 @@ data class BridgeEndpoint(
) )
} }
} }
@@ -11,6 +11,7 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import com.steipete.clawdis.node.BuildConfig
import kotlinx.serialization.json.Json import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonObject
@@ -23,6 +24,7 @@ import java.io.BufferedWriter
import java.io.InputStreamReader import java.io.InputStreamReader
import java.io.OutputStreamWriter import java.io.OutputStreamWriter
import java.net.InetSocketAddress import java.net.InetSocketAddress
import java.net.URI
import java.net.Socket import java.net.Socket
import java.util.UUID import java.util.UUID
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
@@ -75,6 +77,8 @@ class BridgeSession(
fun disconnect() { fun disconnect() {
desired = null desired = null
// Unblock connectOnce() read loop. Coroutine cancellation alone won't interrupt BufferedReader.readLine().
currentConnection?.closeQuietly()
scope.launch(Dispatchers.IO) { scope.launch(Dispatchers.IO) {
job?.cancelAndJoin() job?.cancelAndJoin()
job = null job = null
@@ -213,7 +217,17 @@ class BridgeSession(
when (first["type"].asStringOrNull()) { when (first["type"].asStringOrNull()) {
"hello-ok" -> { "hello-ok" -> {
val name = first["serverName"].asStringOrNull() ?: "Bridge" val name = first["serverName"].asStringOrNull() ?: "Bridge"
canvasHostUrl = first["canvasHostUrl"].asStringOrNull()?.trim()?.ifEmpty { null } val rawCanvasUrl = first["canvasHostUrl"].asStringOrNull()?.trim()?.ifEmpty { null }
canvasHostUrl = normalizeCanvasHostUrl(rawCanvasUrl, endpoint)
if (BuildConfig.DEBUG) {
// Local JVM unit tests use android.jar stubs; Log.d can throw "not mocked".
runCatching {
android.util.Log.d(
"ClawdisBridge",
"canvasHostUrl resolved=${canvasHostUrl ?: "none"} (raw=${rawCanvasUrl ?: "none"})",
)
}
}
onConnected(name, conn.remoteAddress) onConnected(name, conn.remoteAddress)
} }
"error" -> { "error" -> {
@@ -292,6 +306,37 @@ class BridgeSession(
conn.closeQuietly() conn.closeQuietly()
} }
} }
private fun normalizeCanvasHostUrl(raw: String?, endpoint: BridgeEndpoint): String? {
val trimmed = raw?.trim().orEmpty()
val parsed = trimmed.takeIf { it.isNotBlank() }?.let { runCatching { URI(it) }.getOrNull() }
val host = parsed?.host?.trim().orEmpty()
val port = parsed?.port ?: -1
val scheme = parsed?.scheme?.trim().orEmpty().ifBlank { "http" }
if (trimmed.isNotBlank() && !isLoopbackHost(host)) {
return trimmed
}
val fallbackHost =
endpoint.tailnetDns?.trim().takeIf { !it.isNullOrEmpty() }
?: endpoint.lanHost?.trim().takeIf { !it.isNullOrEmpty() }
?: endpoint.host.trim()
if (fallbackHost.isEmpty()) return trimmed.ifBlank { null }
val fallbackPort = endpoint.canvasPort ?: if (port > 0) port else 18793
val formattedHost = if (fallbackHost.contains(":")) "[${fallbackHost}]" else fallbackHost
return "$scheme://$formattedHost:$fallbackPort"
}
private fun isLoopbackHost(raw: String?): Boolean {
val host = raw?.trim()?.lowercase().orEmpty()
if (host.isEmpty()) return false
if (host == "localhost") return true
if (host == "::1") return true
if (host == "0.0.0.0" || host == "::") return true
return host.startsWith("127.")
}
} }
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
@@ -2,6 +2,7 @@ package com.steipete.clawdis.node.node
import android.Manifest import android.Manifest
import android.content.Context import android.content.Context
import android.annotation.SuppressLint
import android.graphics.Bitmap import android.graphics.Bitmap
import android.graphics.BitmapFactory import android.graphics.BitmapFactory
import android.util.Base64 import android.util.Base64
@@ -18,6 +19,7 @@ import androidx.camera.video.VideoCapture
import androidx.camera.video.VideoRecordEvent import androidx.camera.video.VideoRecordEvent
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
import androidx.core.content.ContextCompat.checkSelfPermission import androidx.core.content.ContextCompat.checkSelfPermission
import androidx.core.graphics.scale
import com.steipete.clawdis.node.PermissionRequester import com.steipete.clawdis.node.PermissionRequester
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.suspendCancellableCoroutine
@@ -26,6 +28,7 @@ import kotlinx.coroutines.withContext
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.io.File import java.io.File
import java.util.concurrent.Executor import java.util.concurrent.Executor
import kotlin.math.roundToInt
import kotlin.coroutines.resume import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException import kotlin.coroutines.resumeWithException
@@ -92,22 +95,45 @@ class CameraCaptureManager(private val context: Context) {
(decoded.height.toDouble() * (maxWidth.toDouble() / decoded.width.toDouble())) (decoded.height.toDouble() * (maxWidth.toDouble() / decoded.width.toDouble()))
.toInt() .toInt()
.coerceAtLeast(1) .coerceAtLeast(1)
Bitmap.createScaledBitmap(decoded, maxWidth, h, true) decoded.scale(maxWidth, h)
} else { } else {
decoded decoded
} }
val out = ByteArrayOutputStream() val maxPayloadBytes = 5 * 1024 * 1024
val jpegQuality = (quality * 100.0).toInt().coerceIn(10, 100) // Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
if (!scaled.compress(Bitmap.CompressFormat.JPEG, jpegQuality, out)) { val maxEncodedBytes = (maxPayloadBytes / 4) * 3
throw IllegalStateException("UNAVAILABLE: failed to encode JPEG") val result =
} JpegSizeLimiter.compressToLimit(
val base64 = Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) initialWidth = scaled.width,
initialHeight = scaled.height,
startQuality = (quality * 100.0).roundToInt().coerceIn(10, 100),
maxBytes = maxEncodedBytes,
encode = { width, height, q ->
val bitmap =
if (width == scaled.width && height == scaled.height) {
scaled
} else {
scaled.scale(width, height)
}
val out = ByteArrayOutputStream()
if (!bitmap.compress(Bitmap.CompressFormat.JPEG, q, out)) {
if (bitmap !== scaled) bitmap.recycle()
throw IllegalStateException("UNAVAILABLE: failed to encode JPEG")
}
if (bitmap !== scaled) {
bitmap.recycle()
}
out.toByteArray()
},
)
val base64 = Base64.encodeToString(result.bytes, Base64.NO_WRAP)
Payload( Payload(
"""{"format":"jpg","base64":"$base64","width":${scaled.width},"height":${scaled.height}}""", """{"format":"jpg","base64":"$base64","width":${result.width},"height":${result.height}}""",
) )
} }
@SuppressLint("MissingPermission")
suspend fun clip(paramsJson: String?): Payload = suspend fun clip(paramsJson: String?): Payload =
withContext(Dispatchers.Main) { withContext(Dispatchers.Main) {
ensureCameraPermission() ensureCameraPermission()
@@ -1,19 +1,23 @@
package com.steipete.clawdis.node.node package com.steipete.clawdis.node.node
import android.graphics.Bitmap import android.graphics.Bitmap
import android.os.Build
import android.graphics.Canvas import android.graphics.Canvas
import android.os.Looper import android.os.Looper
import android.util.Log
import android.webkit.WebView import android.webkit.WebView
import androidx.core.graphics.createBitmap
import androidx.core.graphics.scale
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import android.util.Base64 import android.util.Base64
import org.json.JSONObject
import kotlinx.serialization.json.Json import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.JsonPrimitive
import com.steipete.clawdis.node.BuildConfig
import kotlin.coroutines.resume import kotlin.coroutines.resume
class CanvasController { class CanvasController {
@@ -24,6 +28,9 @@ class CanvasController {
@Volatile private var webView: WebView? = null @Volatile private var webView: WebView? = null
@Volatile private var url: String? = null @Volatile private var url: String? = null
@Volatile private var debugStatusEnabled: Boolean = false
@Volatile private var debugStatusTitle: String? = null
@Volatile private var debugStatusSubtitle: String? = null
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html" private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
@@ -35,6 +42,7 @@ class CanvasController {
fun attach(webView: WebView) { fun attach(webView: WebView) {
this.webView = webView this.webView = webView
reload() reload()
applyDebugStatus()
} }
fun navigate(url: String) { fun navigate(url: String) {
@@ -43,6 +51,25 @@ class CanvasController {
reload() reload()
} }
fun currentUrl(): String? = url
fun isDefaultCanvas(): Boolean = url == null
fun setDebugStatusEnabled(enabled: Boolean) {
debugStatusEnabled = enabled
applyDebugStatus()
}
fun setDebugStatus(title: String?, subtitle: String?) {
debugStatusTitle = title
debugStatusSubtitle = subtitle
applyDebugStatus()
}
fun onPageFinished() {
applyDebugStatus()
}
private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) { private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) {
val wv = webView ?: return val wv = webView ?: return
if (Looper.myLooper() == Looper.getMainLooper()) { if (Looper.myLooper() == Looper.getMainLooper()) {
@@ -56,13 +83,45 @@ class CanvasController {
val currentUrl = url val currentUrl = url
withWebViewOnMain { wv -> withWebViewOnMain { wv ->
if (currentUrl == null) { if (currentUrl == null) {
if (BuildConfig.DEBUG) {
Log.d("ClawdisCanvas", "load scaffold: $scaffoldAssetUrl")
}
wv.loadUrl(scaffoldAssetUrl) wv.loadUrl(scaffoldAssetUrl)
} else { } else {
if (BuildConfig.DEBUG) {
Log.d("ClawdisCanvas", "load url: $currentUrl")
}
wv.loadUrl(currentUrl) wv.loadUrl(currentUrl)
} }
} }
} }
private fun applyDebugStatus() {
val enabled = debugStatusEnabled
val title = debugStatusTitle
val subtitle = debugStatusSubtitle
withWebViewOnMain { wv ->
val titleJs = title?.let { JSONObject.quote(it) } ?: "null"
val subtitleJs = subtitle?.let { JSONObject.quote(it) } ?: "null"
val js = """
(() => {
try {
const api = globalThis.__clawdis;
if (!api) return;
if (typeof api.setDebugStatusEnabled === 'function') {
api.setDebugStatusEnabled(${if (enabled) "true" else "false"});
}
if (!${if (enabled) "true" else "false"}) return;
if (typeof api.setStatus === 'function') {
api.setStatus($titleJs, $subtitleJs);
}
} catch (_) {}
})();
""".trimIndent()
wv.evaluateJavascript(js, null)
}
}
suspend fun eval(javaScript: String): String = suspend fun eval(javaScript: String): String =
withContext(Dispatchers.Main) { withContext(Dispatchers.Main) {
val wv = webView ?: throw IllegalStateException("no webview") val wv = webView ?: throw IllegalStateException("no webview")
@@ -80,7 +139,7 @@ class CanvasController {
val scaled = val scaled =
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
Bitmap.createScaledBitmap(bmp, maxWidth, h, true) bmp.scale(maxWidth, h)
} else { } else {
bmp bmp
} }
@@ -97,7 +156,7 @@ class CanvasController {
val scaled = val scaled =
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
Bitmap.createScaledBitmap(bmp, maxWidth, h, true) bmp.scale(maxWidth, h)
} else { } else {
bmp bmp
} }
@@ -116,7 +175,7 @@ class CanvasController {
suspendCancellableCoroutine { cont -> suspendCancellableCoroutine { cont ->
val width = width.coerceAtLeast(1) val width = width.coerceAtLeast(1)
val height = height.coerceAtLeast(1) val height = height.coerceAtLeast(1)
val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) val bitmap = createBitmap(width, height, Bitmap.Config.ARGB_8888)
// WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable // WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable
// cross-version snapshot for this lightweight "canvas" use-case. // cross-version snapshot for this lightweight "canvas" use-case.
@@ -0,0 +1,61 @@
package com.steipete.clawdis.node.node
import kotlin.math.max
import kotlin.math.min
import kotlin.math.roundToInt
internal data class JpegSizeLimiterResult(
val bytes: ByteArray,
val width: Int,
val height: Int,
val quality: Int,
)
internal object JpegSizeLimiter {
fun compressToLimit(
initialWidth: Int,
initialHeight: Int,
startQuality: Int,
maxBytes: Int,
minQuality: Int = 20,
minSize: Int = 256,
scaleStep: Double = 0.85,
maxScaleAttempts: Int = 6,
maxQualityAttempts: Int = 6,
encode: (width: Int, height: Int, quality: Int) -> ByteArray,
): JpegSizeLimiterResult {
require(initialWidth > 0 && initialHeight > 0) { "Invalid image size" }
require(maxBytes > 0) { "Invalid maxBytes" }
var width = initialWidth
var height = initialHeight
val clampedStartQuality = startQuality.coerceIn(minQuality, 100)
var best = JpegSizeLimiterResult(bytes = encode(width, height, clampedStartQuality), width = width, height = height, quality = clampedStartQuality)
if (best.bytes.size <= maxBytes) return best
repeat(maxScaleAttempts) {
var quality = clampedStartQuality
repeat(maxQualityAttempts) {
val bytes = encode(width, height, quality)
best = JpegSizeLimiterResult(bytes = bytes, width = width, height = height, quality = quality)
if (bytes.size <= maxBytes) return best
if (quality <= minQuality) return@repeat
quality = max(minQuality, (quality * 0.75).roundToInt())
}
val minScale = (minSize.toDouble() / min(width, height).toDouble()).coerceAtMost(1.0)
val nextScale = max(scaleStep, minScale)
val nextWidth = max(minSize, (width * nextScale).roundToInt())
val nextHeight = max(minSize, (height * nextScale).roundToInt())
if (nextWidth == width && nextHeight == height) return@repeat
width = min(nextWidth, width)
height = min(nextHeight, height)
}
if (best.bytes.size > maxBytes) {
throw IllegalStateException("CAMERA_TOO_LARGE: ${best.bytes.size} bytes > $maxBytes bytes")
}
return best
}
}
@@ -1,6 +1,24 @@
package com.steipete.clawdis.node.protocol package com.steipete.clawdis.node.protocol
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
object ClawdisCanvasA2UIAction { object ClawdisCanvasA2UIAction {
fun extractActionName(userAction: JsonObject): String? {
val name =
(userAction["name"] as? JsonPrimitive)
?.content
?.trim()
.orEmpty()
if (name.isNotEmpty()) return name
val action =
(userAction["action"] as? JsonPrimitive)
?.content
?.trim()
.orEmpty()
return action.ifEmpty { null }
}
fun sanitizeTagValue(value: String): String { fun sanitizeTagValue(value: String): String {
val trimmed = value.trim().ifEmpty { "-" } val trimmed = value.trim().ifEmpty { "-" }
val normalized = trimmed.replace(" ", "_") val normalized = trimmed.replace(" ", "_")
@@ -46,4 +64,3 @@ object ClawdisCanvasA2UIAction {
return "window.dispatchEvent(new CustomEvent('clawdis:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));" return "window.dispatchEvent(new CustomEvent('clawdis:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));"
} }
} }
@@ -1,64 +1,26 @@
package com.steipete.clawdis.node.ui package com.steipete.clawdis.node.ui
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
import androidx.compose.animation.slideInVertically
import androidx.compose.animation.slideOutVertically
import androidx.compose.foundation.background import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.statusBarsPadding
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.CheckCircle
import androidx.compose.material.icons.filled.Error
import androidx.compose.material.icons.filled.FiberManualRecord
import androidx.compose.material.icons.filled.PhotoCamera
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableFloatStateOf import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.remember import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.CameraHudKind
import com.steipete.clawdis.node.CameraHudState
import kotlinx.coroutines.delay import kotlinx.coroutines.delay
@Composable @Composable
fun CameraHudOverlay( fun CameraFlashOverlay(
hud: CameraHudState?, token: Long,
flashToken: Long,
modifier: Modifier = Modifier, modifier: Modifier = Modifier,
) { ) {
Box(modifier = modifier.fillMaxSize()) { Box(modifier = modifier.fillMaxSize()) {
CameraFlash(token = flashToken) CameraFlash(token = token)
AnimatedVisibility(
visible = hud != null,
enter = slideInVertically(initialOffsetY = { -it / 2 }) + fadeIn(),
exit = slideOutVertically(targetOffsetY = { -it / 2 }) + fadeOut(),
modifier = Modifier.align(Alignment.TopStart).statusBarsPadding().padding(start = 12.dp, top = 58.dp),
) {
if (hud != null) {
Toast(hud = hud)
}
}
} }
} }
@@ -80,44 +42,3 @@ private fun CameraFlash(token: Long) {
.background(Color.White), .background(Color.White),
) )
} }
@Composable
private fun Toast(hud: CameraHudState) {
Surface(
shape = RoundedCornerShape(14.dp),
color = MaterialTheme.colorScheme.surface.copy(alpha = 0.85f),
tonalElevation = 2.dp,
shadowElevation = 8.dp,
) {
Row(
modifier = Modifier.padding(vertical = 10.dp, horizontal = 12.dp),
verticalAlignment = Alignment.CenterVertically,
) {
when (hud.kind) {
CameraHudKind.Photo -> {
Icon(Icons.Default.PhotoCamera, contentDescription = null)
Spacer(Modifier.size(10.dp))
CircularProgressIndicator(modifier = Modifier.size(14.dp), strokeWidth = 2.dp)
}
CameraHudKind.Recording -> {
Icon(Icons.Default.FiberManualRecord, contentDescription = null, tint = Color.Red)
}
CameraHudKind.Success -> {
Icon(Icons.Default.CheckCircle, contentDescription = null)
}
CameraHudKind.Error -> {
Icon(Icons.Default.Error, contentDescription = null)
}
}
Spacer(Modifier.size(10.dp))
Text(
text = hud.message,
style = MaterialTheme.typography.bodyMedium,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
}
}
@@ -7,12 +7,18 @@ import android.graphics.Color
import android.util.Log import android.util.Log
import android.view.View import android.view.View
import android.webkit.JavascriptInterface import android.webkit.JavascriptInterface
import android.webkit.ConsoleMessage
import android.webkit.WebChromeClient
import android.webkit.WebView import android.webkit.WebView
import android.webkit.WebSettings import android.webkit.WebSettings
import android.webkit.WebResourceError import android.webkit.WebResourceError
import android.webkit.WebResourceRequest import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse import android.webkit.WebResourceResponse
import android.webkit.WebViewClient import android.webkit.WebViewClient
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.webkit.WebSettingsCompat
import androidx.webkit.WebViewFeature
import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Column
@@ -28,10 +34,20 @@ import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.FilledTonalIconButton import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.Icon import androidx.compose.material3.Icon
import androidx.compose.material3.IconButtonDefaults import androidx.compose.material3.IconButtonDefaults
import androidx.compose.material3.LocalContentColor
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.ModalBottomSheet import androidx.compose.material3.ModalBottomSheet
import androidx.compose.material3.rememberModalBottomSheetState import androidx.compose.material3.rememberModalBottomSheetState
import androidx.compose.material.icons.Icons import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ChatBubble import androidx.compose.material.icons.filled.ChatBubble
import androidx.compose.material.icons.filled.CheckCircle
import androidx.compose.material.icons.filled.Error
import androidx.compose.material.icons.filled.FiberManualRecord
import androidx.compose.material.icons.filled.PhotoCamera
import androidx.compose.material.icons.filled.RecordVoiceOver
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Report
import androidx.compose.material.icons.filled.ScreenShare
import androidx.compose.material.icons.filled.Settings import androidx.compose.material.icons.filled.Settings
import androidx.compose.runtime.Composable import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState import androidx.compose.runtime.collectAsState
@@ -41,12 +57,15 @@ import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color as ComposeColor
import androidx.compose.ui.graphics.lerp
import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView import androidx.compose.ui.viewinterop.AndroidView
import androidx.compose.ui.window.Popup import androidx.compose.ui.window.Popup
import androidx.compose.ui.window.PopupProperties import androidx.compose.ui.window.PopupProperties
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
import com.steipete.clawdis.node.CameraHudKind
import com.steipete.clawdis.node.MainViewModel import com.steipete.clawdis.node.MainViewModel
@OptIn(ExperimentalMaterial3Api::class) @OptIn(ExperimentalMaterial3Api::class)
@@ -60,6 +79,105 @@ fun RootScreen(viewModel: MainViewModel) {
val statusText by viewModel.statusText.collectAsState() val statusText by viewModel.statusText.collectAsState()
val cameraHud by viewModel.cameraHud.collectAsState() val cameraHud by viewModel.cameraHud.collectAsState()
val cameraFlashToken by viewModel.cameraFlashToken.collectAsState() val cameraFlashToken by viewModel.cameraFlashToken.collectAsState()
val screenRecordActive by viewModel.screenRecordActive.collectAsState()
val isForeground by viewModel.isForeground.collectAsState()
val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState()
val talkEnabled by viewModel.talkEnabled.collectAsState()
val talkStatusText by viewModel.talkStatusText.collectAsState()
val talkIsListening by viewModel.talkIsListening.collectAsState()
val talkIsSpeaking by viewModel.talkIsSpeaking.collectAsState()
val seamColorArgb by viewModel.seamColorArgb.collectAsState()
val seamColor = remember(seamColorArgb) { ComposeColor(seamColorArgb) }
val audioPermissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
if (granted) viewModel.setTalkEnabled(true)
}
val activity =
remember(cameraHud, screenRecordActive, isForeground, statusText, voiceWakeStatusText) {
// Status pill owns transient activity state so it doesn't overlap the connection indicator.
if (!isForeground) {
return@remember StatusActivity(
title = "Foreground required",
icon = Icons.Default.Report,
contentDescription = "Foreground required",
)
}
val lowerStatus = statusText.lowercase()
if (lowerStatus.contains("repair")) {
return@remember StatusActivity(
title = "Repairing…",
icon = Icons.Default.Refresh,
contentDescription = "Repairing",
)
}
if (lowerStatus.contains("pairing") || lowerStatus.contains("approval")) {
return@remember StatusActivity(
title = "Approval pending",
icon = Icons.Default.RecordVoiceOver,
contentDescription = "Approval pending",
)
}
// Avoid duplicating the primary bridge status ("Connecting…") in the activity slot.
if (screenRecordActive) {
return@remember StatusActivity(
title = "Recording screen…",
icon = Icons.Default.ScreenShare,
contentDescription = "Recording screen",
tint = androidx.compose.ui.graphics.Color.Red,
)
}
cameraHud?.let { hud ->
return@remember when (hud.kind) {
CameraHudKind.Photo ->
StatusActivity(
title = hud.message,
icon = Icons.Default.PhotoCamera,
contentDescription = "Taking photo",
)
CameraHudKind.Recording ->
StatusActivity(
title = hud.message,
icon = Icons.Default.FiberManualRecord,
contentDescription = "Recording",
tint = androidx.compose.ui.graphics.Color.Red,
)
CameraHudKind.Success ->
StatusActivity(
title = hud.message,
icon = Icons.Default.CheckCircle,
contentDescription = "Capture finished",
)
CameraHudKind.Error ->
StatusActivity(
title = hud.message,
icon = Icons.Default.Error,
contentDescription = "Capture failed",
tint = androidx.compose.ui.graphics.Color.Red,
)
}
}
if (voiceWakeStatusText.contains("Microphone permission", ignoreCase = true)) {
return@remember StatusActivity(
title = "Mic permission",
icon = Icons.Default.Error,
contentDescription = "Mic permission required",
)
}
if (voiceWakeStatusText == "Paused") {
val suffix = if (!isForeground) " (background)" else ""
return@remember StatusActivity(
title = "Voice Wake paused$suffix",
icon = Icons.Default.RecordVoiceOver,
contentDescription = "Voice Wake paused",
)
}
null
}
val bridgeState = val bridgeState =
remember(serverName, statusText) { remember(serverName, statusText) {
@@ -80,9 +198,9 @@ fun RootScreen(viewModel: MainViewModel) {
CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize()) CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize())
} }
// Camera HUD (flash + toast) must be in a Popup to render above the WebView. // Camera flash must be in a Popup to render above the WebView.
Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) { Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) {
CameraHudOverlay(hud = cameraHud, flashToken = cameraFlashToken, modifier = Modifier.fillMaxSize()) CameraFlashOverlay(token = cameraFlashToken, modifier = Modifier.fillMaxSize())
} }
// Keep the overlay buttons above the WebView canvas (AndroidView), otherwise they may not receive touches. // Keep the overlay buttons above the WebView canvas (AndroidView), otherwise they may not receive touches.
@@ -90,6 +208,7 @@ fun RootScreen(viewModel: MainViewModel) {
StatusPill( StatusPill(
bridge = bridgeState, bridge = bridgeState,
voiceEnabled = voiceEnabled, voiceEnabled = voiceEnabled,
activity = activity,
onClick = { sheet = Sheet.Settings }, onClick = { sheet = Sheet.Settings },
modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(start = 12.dp, top = 12.dp), modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(start = 12.dp, top = 12.dp),
) )
@@ -106,6 +225,38 @@ fun RootScreen(viewModel: MainViewModel) {
icon = { Icon(Icons.Default.ChatBubble, contentDescription = "Chat") }, icon = { Icon(Icons.Default.ChatBubble, contentDescription = "Chat") },
) )
// Talk mode gets a dedicated side bubble instead of burying it in settings.
val baseOverlay = overlayContainerColor()
val talkContainer =
lerp(
baseOverlay,
seamColor.copy(alpha = baseOverlay.alpha),
if (talkEnabled) 0.35f else 0.22f,
)
val talkContent = if (talkEnabled) seamColor else overlayIconColor()
OverlayIconButton(
onClick = {
val next = !talkEnabled
if (next) {
val micOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
viewModel.setTalkEnabled(true)
} else {
viewModel.setTalkEnabled(false)
}
},
containerColor = talkContainer,
contentColor = talkContent,
icon = {
Icon(
Icons.Default.RecordVoiceOver,
contentDescription = "Talk Mode",
)
},
)
OverlayIconButton( OverlayIconButton(
onClick = { sheet = Sheet.Settings }, onClick = { sheet = Sheet.Settings },
icon = { Icon(Icons.Default.Settings, contentDescription = "Settings") }, icon = { Icon(Icons.Default.Settings, contentDescription = "Settings") },
@@ -113,6 +264,17 @@ fun RootScreen(viewModel: MainViewModel) {
} }
} }
if (talkEnabled) {
Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) {
TalkOrbOverlay(
seamColor = seamColor,
statusText = talkStatusText,
isListening = talkIsListening,
isSpeaking = talkIsSpeaking,
)
}
}
val currentSheet = sheet val currentSheet = sheet
if (currentSheet != null) { if (currentSheet != null) {
ModalBottomSheet( ModalBottomSheet(
@@ -136,14 +298,16 @@ private enum class Sheet {
private fun OverlayIconButton( private fun OverlayIconButton(
onClick: () -> Unit, onClick: () -> Unit,
icon: @Composable () -> Unit, icon: @Composable () -> Unit,
containerColor: ComposeColor? = null,
contentColor: ComposeColor? = null,
) { ) {
FilledTonalIconButton( FilledTonalIconButton(
onClick = onClick, onClick = onClick,
modifier = Modifier.size(44.dp), modifier = Modifier.size(44.dp),
colors = colors =
IconButtonDefaults.filledTonalIconButtonColors( IconButtonDefaults.filledTonalIconButtonColors(
containerColor = overlayContainerColor(), containerColor = containerColor ?: overlayContainerColor(),
contentColor = overlayIconColor(), contentColor = contentColor ?: overlayIconColor(),
), ),
) { ) {
icon() icon()
@@ -163,6 +327,19 @@ private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier)
// Some embedded web UIs (incl. the "background website") use localStorage/sessionStorage. // Some embedded web UIs (incl. the "background website") use localStorage/sessionStorage.
settings.domStorageEnabled = true settings.domStorageEnabled = true
settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE
if (WebViewFeature.isFeatureSupported(WebViewFeature.FORCE_DARK)) {
WebSettingsCompat.setForceDark(settings, WebSettingsCompat.FORCE_DARK_OFF)
}
if (WebViewFeature.isFeatureSupported(WebViewFeature.ALGORITHMIC_DARKENING)) {
WebSettingsCompat.setAlgorithmicDarkeningAllowed(settings, false)
}
if (isDebuggable) {
Log.d("ClawdisWebView", "userAgent: ${settings.userAgentString}")
}
isScrollContainer = true
overScrollMode = View.OVER_SCROLL_IF_CONTENT_SCROLLS
isVerticalScrollBarEnabled = true
isHorizontalScrollBarEnabled = true
webViewClient = webViewClient =
object : WebViewClient() { object : WebViewClient() {
override fun onReceivedError( override fun onReceivedError(
@@ -187,15 +364,49 @@ private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier)
"onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}", "onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}",
) )
} }
}
setBackgroundColor(Color.BLACK)
setLayerType(View.LAYER_TYPE_HARDWARE, null)
addJavascriptInterface( override fun onPageFinished(view: WebView, url: String?) {
if (isDebuggable) {
Log.d("ClawdisWebView", "onPageFinished: $url")
}
viewModel.canvas.onPageFinished()
}
override fun onRenderProcessGone(
view: WebView,
detail: android.webkit.RenderProcessGoneDetail,
): Boolean {
if (isDebuggable) {
Log.e(
"ClawdisWebView",
"onRenderProcessGone didCrash=${detail.didCrash()} priorityAtExit=${detail.rendererPriorityAtExit()}",
)
}
return true
}
}
webChromeClient =
object : WebChromeClient() {
override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean {
if (!isDebuggable) return false
val msg = consoleMessage ?: return false
Log.d(
"ClawdisWebView",
"console ${msg.messageLevel()} @ ${msg.sourceId()}:${msg.lineNumber()} ${msg.message()}",
)
return false
}
}
// Use default layer/background; avoid forcing a black fill over WebView content.
val a2uiBridge =
CanvasA2UIActionBridge { payload -> CanvasA2UIActionBridge { payload ->
viewModel.handleCanvasA2UIActionFromWebView(payload) viewModel.handleCanvasA2UIActionFromWebView(payload)
}, }
CanvasA2UIActionBridge.interfaceName, addJavascriptInterface(a2uiBridge, CanvasA2UIActionBridge.interfaceName)
addJavascriptInterface(
CanvasA2UIActionLegacyBridge(a2uiBridge),
CanvasA2UIActionLegacyBridge.interfaceName,
) )
viewModel.canvas.attach(this) viewModel.canvas.attach(this)
} }
@@ -215,3 +426,19 @@ private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) {
const val interfaceName: String = "clawdisCanvasA2UIAction" const val interfaceName: String = "clawdisCanvasA2UIAction"
} }
} }
private class CanvasA2UIActionLegacyBridge(private val bridge: CanvasA2UIActionBridge) {
@JavascriptInterface
fun canvasAction(payload: String?) {
bridge.postMessage(payload)
}
@JavascriptInterface
fun postMessage(payload: String?) {
bridge.postMessage(payload)
}
companion object {
const val interfaceName: String = "Android"
}
}
@@ -2,6 +2,7 @@ package com.steipete.clawdis.node.ui
import android.Manifest import android.Manifest
import android.content.pm.PackageManager import android.content.pm.PackageManager
import android.os.Build
import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.animation.AnimatedVisibility import androidx.compose.animation.AnimatedVisibility
@@ -46,6 +47,7 @@ import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
import com.steipete.clawdis.node.BuildConfig
import com.steipete.clawdis.node.MainViewModel import com.steipete.clawdis.node.MainViewModel
import com.steipete.clawdis.node.NodeForegroundService import com.steipete.clawdis.node.NodeForegroundService
import com.steipete.clawdis.node.VoiceWakeMode import com.steipete.clawdis.node.VoiceWakeMode
@@ -64,6 +66,7 @@ fun SettingsSheet(viewModel: MainViewModel) {
val manualEnabled by viewModel.manualEnabled.collectAsState() val manualEnabled by viewModel.manualEnabled.collectAsState()
val manualHost by viewModel.manualHost.collectAsState() val manualHost by viewModel.manualHost.collectAsState()
val manualPort by viewModel.manualPort.collectAsState() val manualPort by viewModel.manualPort.collectAsState()
val canvasDebugStatusEnabled by viewModel.canvasDebugStatusEnabled.collectAsState()
val statusText by viewModel.statusText.collectAsState() val statusText by viewModel.statusText.collectAsState()
val serverName by viewModel.serverName.collectAsState() val serverName by viewModel.serverName.collectAsState()
val remoteAddress by viewModel.remoteAddress.collectAsState() val remoteAddress by viewModel.remoteAddress.collectAsState()
@@ -73,6 +76,22 @@ fun SettingsSheet(viewModel: MainViewModel) {
val listState = rememberLazyListState() val listState = rememberLazyListState()
val (wakeWordsText, setWakeWordsText) = remember { mutableStateOf("") } val (wakeWordsText, setWakeWordsText) = remember { mutableStateOf("") }
val (advancedExpanded, setAdvancedExpanded) = remember { mutableStateOf(false) } val (advancedExpanded, setAdvancedExpanded) = remember { mutableStateOf(false) }
val deviceModel =
remember {
listOfNotNull(Build.MANUFACTURER, Build.MODEL)
.joinToString(" ")
.trim()
.ifEmpty { "Android" }
}
val appVersion =
remember {
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
}
LaunchedEffect(wakeWords) { setWakeWordsText(wakeWords.joinToString(", ")) } LaunchedEffect(wakeWords) { setWakeWordsText(wakeWords.joinToString(", ")) }
@@ -141,6 +160,8 @@ fun SettingsSheet(viewModel: MainViewModel) {
) )
} }
item { Text("Instance ID: $instanceId", color = MaterialTheme.colorScheme.onSurfaceVariant) } item { Text("Instance ID: $instanceId", color = MaterialTheme.colorScheme.onSurfaceVariant) }
item { Text("Device: $deviceModel", color = MaterialTheme.colorScheme.onSurfaceVariant) }
item { Text("Version: $appVersion", color = MaterialTheme.colorScheme.onSurfaceVariant) }
item { HorizontalDivider() } item { HorizontalDivider() }
@@ -180,9 +201,27 @@ fun SettingsSheet(viewModel: MainViewModel) {
item { Text("No bridges found yet.", color = MaterialTheme.colorScheme.onSurfaceVariant) } item { Text("No bridges found yet.", color = MaterialTheme.colorScheme.onSurfaceVariant) }
} else { } else {
items(items = visibleBridges, key = { it.stableId }) { bridge -> items(items = visibleBridges, key = { it.stableId }) { bridge ->
val detailLines =
buildList {
add("IP: ${bridge.host}:${bridge.port}")
bridge.lanHost?.let { add("LAN: $it") }
bridge.tailnetDns?.let { add("Tailnet: $it") }
if (bridge.gatewayPort != null || bridge.bridgePort != null || bridge.canvasPort != null) {
val gw = bridge.gatewayPort?.toString() ?: "—"
val br = (bridge.bridgePort ?: bridge.port).toString()
val canvas = bridge.canvasPort?.toString() ?: "—"
add("Ports: gw $gw · bridge $br · canvas $canvas")
}
}
ListItem( ListItem(
headlineContent = { Text(bridge.name) }, headlineContent = { Text(bridge.name) },
supportingContent = { Text("${bridge.host}:${bridge.port}") }, supportingContent = {
Column(verticalArrangement = Arrangement.spacedBy(2.dp)) {
detailLines.forEach { line ->
Text(line, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
},
trailingContent = { trailingContent = {
Button( Button(
onClick = { onClick = {
@@ -394,6 +433,23 @@ fun SettingsSheet(viewModel: MainViewModel) {
) )
} }
item { HorizontalDivider() }
// Debug
item { Text("Debug", style = MaterialTheme.typography.titleSmall) }
item {
ListItem(
headlineContent = { Text("Debug Canvas Status") },
supportingContent = { Text("Show status text in the canvas when debug is enabled.") },
trailingContent = {
Switch(
checked = canvasDebugStatusEnabled,
onCheckedChange = viewModel::setCanvasDebugStatusEnabled,
)
},
)
}
item { Spacer(modifier = Modifier.height(20.dp)) } item { Spacer(modifier = Modifier.height(20.dp)) }
} }
} }
@@ -28,6 +28,7 @@ import androidx.compose.ui.unit.dp
fun StatusPill( fun StatusPill(
bridge: BridgeState, bridge: BridgeState,
voiceEnabled: Boolean, voiceEnabled: Boolean,
activity: StatusActivity? = null,
onClick: () -> Unit, onClick: () -> Unit,
modifier: Modifier = Modifier, modifier: Modifier = Modifier,
) { ) {
@@ -62,23 +63,49 @@ fun StatusPill(
color = MaterialTheme.colorScheme.onSurfaceVariant, color = MaterialTheme.colorScheme.onSurfaceVariant,
) )
Icon( if (activity != null) {
imageVector = if (voiceEnabled) Icons.Default.Mic else Icons.Default.MicOff, Row(
contentDescription = if (voiceEnabled) "Voice enabled" else "Voice disabled", horizontalArrangement = Arrangement.spacedBy(6.dp),
tint = verticalAlignment = Alignment.CenterVertically,
if (voiceEnabled) { ) {
overlayIconColor() Icon(
} else { imageVector = activity.icon,
MaterialTheme.colorScheme.onSurfaceVariant contentDescription = activity.contentDescription,
}, tint = activity.tint ?: overlayIconColor(),
modifier = Modifier.size(18.dp), modifier = Modifier.size(18.dp),
) )
Text(
text = activity.title,
style = MaterialTheme.typography.labelLarge,
maxLines = 1,
)
}
} else {
Icon(
imageVector = if (voiceEnabled) Icons.Default.Mic else Icons.Default.MicOff,
contentDescription = if (voiceEnabled) "Voice enabled" else "Voice disabled",
tint =
if (voiceEnabled) {
overlayIconColor()
} else {
MaterialTheme.colorScheme.onSurfaceVariant
},
modifier = Modifier.size(18.dp),
)
}
Spacer(modifier = Modifier.width(2.dp)) Spacer(modifier = Modifier.width(2.dp))
} }
} }
} }
data class StatusActivity(
val title: String,
val icon: androidx.compose.ui.graphics.vector.ImageVector,
val contentDescription: String,
val tint: Color? = null,
)
enum class BridgeState(val title: String, val color: Color) { enum class BridgeState(val title: String, val color: Color) {
Connected("Connected", Color(0xFF2ECC71)), Connected("Connected", Color(0xFF2ECC71)),
Connecting("Connecting…", Color(0xFFF1C40F)), Connecting("Connecting…", Color(0xFFF1C40F)),
@@ -0,0 +1,134 @@
package com.steipete.clawdis.node.ui
import androidx.compose.animation.core.LinearEasing
import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
@Composable
fun TalkOrbOverlay(
seamColor: Color,
statusText: String,
isListening: Boolean,
isSpeaking: Boolean,
modifier: Modifier = Modifier,
) {
val transition = rememberInfiniteTransition(label = "talk-orb")
val t by
transition.animateFloat(
initialValue = 0f,
targetValue = 1f,
animationSpec =
infiniteRepeatable(
animation = tween(durationMillis = 1500, easing = LinearEasing),
repeatMode = RepeatMode.Restart,
),
label = "pulse",
)
val trimmed = statusText.trim()
val showStatus = trimmed.isNotEmpty() && trimmed != "Off"
val phase =
when {
isSpeaking -> "Speaking"
isListening -> "Listening"
else -> "Thinking"
}
Column(
modifier = modifier.padding(24.dp),
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.spacedBy(12.dp),
) {
Box(contentAlignment = Alignment.Center) {
Canvas(modifier = Modifier.size(360.dp)) {
val center = this.center
val baseRadius = size.minDimension * 0.30f
val ring1 = 1.05f + (t * 0.25f)
val ring2 = 1.20f + (t * 0.55f)
val ringAlpha1 = (1f - t) * 0.34f
val ringAlpha2 = (1f - t) * 0.22f
drawCircle(
color = seamColor.copy(alpha = ringAlpha1),
radius = baseRadius * ring1,
center = center,
style = Stroke(width = 3.dp.toPx()),
)
drawCircle(
color = seamColor.copy(alpha = ringAlpha2),
radius = baseRadius * ring2,
center = center,
style = Stroke(width = 3.dp.toPx()),
)
drawCircle(
brush =
Brush.radialGradient(
colors =
listOf(
seamColor.copy(alpha = 0.92f),
seamColor.copy(alpha = 0.40f),
Color.Black.copy(alpha = 0.56f),
),
center = center,
radius = baseRadius * 1.35f,
),
radius = baseRadius,
center = center,
)
drawCircle(
color = seamColor.copy(alpha = 0.34f),
radius = baseRadius,
center = center,
style = Stroke(width = 1.dp.toPx()),
)
}
}
if (showStatus) {
Surface(
color = Color.Black.copy(alpha = 0.40f),
shape = CircleShape,
) {
Text(
text = trimmed,
modifier = Modifier.padding(horizontal = 14.dp, vertical = 8.dp),
color = Color.White.copy(alpha = 0.92f),
style = MaterialTheme.typography.labelLarge,
fontWeight = FontWeight.SemiBold,
)
}
} else {
Text(
text = phase,
color = Color.White.copy(alpha = 0.80f),
style = MaterialTheme.typography.labelLarge,
fontWeight = FontWeight.SemiBold,
)
}
}
}
@@ -17,6 +17,7 @@ import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.AnnotatedString import androidx.compose.ui.text.AnnotatedString
@@ -31,7 +32,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
@Composable @Composable
fun ChatMarkdown(text: String) { fun ChatMarkdown(text: String, textColor: Color) {
val blocks = remember(text) { splitMarkdown(text) } val blocks = remember(text) { splitMarkdown(text) }
val inlineCodeBg = MaterialTheme.colorScheme.surfaceContainerLow val inlineCodeBg = MaterialTheme.colorScheme.surfaceContainerLow
@@ -44,7 +45,7 @@ fun ChatMarkdown(text: String) {
Text( Text(
text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg), text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg),
style = MaterialTheme.typography.bodyMedium, style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurface, color = textColor,
) )
} }
is ChatMarkdownBlock.Code -> { is ChatMarkdownBlock.Code -> {
@@ -7,11 +7,9 @@ import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme import androidx.compose.material3.MaterialTheme
@@ -60,20 +58,21 @@ fun ChatMessageBubble(message: ChatMessage) {
.background(bubbleBackground(isUser)) .background(bubbleBackground(isUser))
.padding(horizontal = 12.dp, vertical = 10.dp), .padding(horizontal = 12.dp, vertical = 10.dp),
) { ) {
ChatMessageBody(content = message.content) val textColor = textColorOverBubble(isUser)
ChatMessageBody(content = message.content, textColor = textColor)
} }
} }
} }
} }
@Composable @Composable
private fun ChatMessageBody(content: List<ChatMessageContent>) { private fun ChatMessageBody(content: List<ChatMessageContent>, textColor: Color) {
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
for (part in content) { for (part in content) {
when (part.type) { when (part.type) {
"text" -> { "text" -> {
val text = part.text ?: continue val text = part.text ?: continue
ChatMarkdown(text = text) ChatMarkdown(text = text, textColor = textColor)
} }
else -> { else -> {
val b64 = part.base64 ?: continue val b64 = part.base64 ?: continue
@@ -131,7 +130,7 @@ fun ChatStreamingAssistantBubble(text: String) {
color = MaterialTheme.colorScheme.surfaceContainer, color = MaterialTheme.colorScheme.surfaceContainer,
) { ) {
Box(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp)) { Box(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp)) {
ChatMarkdown(text = text) ChatMarkdown(text = text, textColor = MaterialTheme.colorScheme.onSurface)
} }
} }
} }
@@ -150,6 +149,15 @@ private fun bubbleBackground(isUser: Boolean): Brush {
} }
} }
@Composable
private fun textColorOverBubble(isUser: Boolean): Color {
return if (isUser) {
MaterialTheme.colorScheme.onPrimary
} else {
MaterialTheme.colorScheme.onSurface
}
}
@Composable @Composable
private fun ChatBase64Image(base64: String, mimeType: String?) { private fun ChatBase64Image(base64: String, mimeType: String?) {
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) } var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
@@ -0,0 +1,98 @@
package com.steipete.clawdis.node.voice
import android.media.MediaDataSource
import kotlin.math.min
internal class StreamingMediaDataSource : MediaDataSource() {
private data class Chunk(val start: Long, val data: ByteArray)
private val lock = Object()
private val chunks = ArrayList<Chunk>()
private var totalSize: Long = 0
private var closed = false
private var finished = false
private var lastReadIndex = 0
fun append(data: ByteArray) {
if (data.isEmpty()) return
synchronized(lock) {
if (closed || finished) return
val chunk = Chunk(totalSize, data)
chunks.add(chunk)
totalSize += data.size.toLong()
lock.notifyAll()
}
}
fun finish() {
synchronized(lock) {
if (closed) return
finished = true
lock.notifyAll()
}
}
fun fail() {
synchronized(lock) {
closed = true
lock.notifyAll()
}
}
override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int {
if (position < 0) return -1
synchronized(lock) {
while (!closed && !finished && position >= totalSize) {
lock.wait()
}
if (closed) return -1
if (position >= totalSize && finished) return -1
val available = (totalSize - position).toInt()
val toRead = min(size, available)
var remaining = toRead
var destOffset = offset
var pos = position
var index = findChunkIndex(pos)
while (remaining > 0 && index < chunks.size) {
val chunk = chunks[index]
val inChunkOffset = (pos - chunk.start).toInt()
if (inChunkOffset >= chunk.data.size) {
index++
continue
}
val copyLen = min(remaining, chunk.data.size - inChunkOffset)
System.arraycopy(chunk.data, inChunkOffset, buffer, destOffset, copyLen)
remaining -= copyLen
destOffset += copyLen
pos += copyLen
if (inChunkOffset + copyLen >= chunk.data.size) {
index++
}
}
return toRead - remaining
}
}
override fun getSize(): Long = -1
override fun close() {
synchronized(lock) {
closed = true
lock.notifyAll()
}
}
private fun findChunkIndex(position: Long): Int {
var index = lastReadIndex
while (index < chunks.size) {
val chunk = chunks[index]
if (position < chunk.start + chunk.data.size) break
index++
}
lastReadIndex = index
return index
}
}
@@ -0,0 +1,191 @@
package com.steipete.clawdis.node.voice
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
private val directiveJson = Json { ignoreUnknownKeys = true }
data class TalkDirective(
val voiceId: String? = null,
val modelId: String? = null,
val speed: Double? = null,
val rateWpm: Int? = null,
val stability: Double? = null,
val similarity: Double? = null,
val style: Double? = null,
val speakerBoost: Boolean? = null,
val seed: Long? = null,
val normalize: String? = null,
val language: String? = null,
val outputFormat: String? = null,
val latencyTier: Int? = null,
val once: Boolean? = null,
)
data class TalkDirectiveParseResult(
val directive: TalkDirective?,
val stripped: String,
val unknownKeys: List<String>,
)
object TalkDirectiveParser {
fun parse(text: String): TalkDirectiveParseResult {
val normalized = text.replace("\r\n", "\n")
val lines = normalized.split("\n").toMutableList()
if (lines.isEmpty()) return TalkDirectiveParseResult(null, text, emptyList())
val firstNonEmpty = lines.indexOfFirst { it.trim().isNotEmpty() }
if (firstNonEmpty == -1) return TalkDirectiveParseResult(null, text, emptyList())
val head = lines[firstNonEmpty].trim()
if (!head.startsWith("{") || !head.endsWith("}")) {
return TalkDirectiveParseResult(null, text, emptyList())
}
val obj = parseJsonObject(head) ?: return TalkDirectiveParseResult(null, text, emptyList())
val speakerBoost =
boolValue(obj, listOf("speaker_boost", "speakerBoost"))
?: boolValue(obj, listOf("no_speaker_boost", "noSpeakerBoost"))?.not()
val directive = TalkDirective(
voiceId = stringValue(obj, listOf("voice", "voice_id", "voiceId")),
modelId = stringValue(obj, listOf("model", "model_id", "modelId")),
speed = doubleValue(obj, listOf("speed")),
rateWpm = intValue(obj, listOf("rate", "wpm")),
stability = doubleValue(obj, listOf("stability")),
similarity = doubleValue(obj, listOf("similarity", "similarity_boost", "similarityBoost")),
style = doubleValue(obj, listOf("style")),
speakerBoost = speakerBoost,
seed = longValue(obj, listOf("seed")),
normalize = stringValue(obj, listOf("normalize", "apply_text_normalization")),
language = stringValue(obj, listOf("lang", "language_code", "language")),
outputFormat = stringValue(obj, listOf("output_format", "format")),
latencyTier = intValue(obj, listOf("latency", "latency_tier", "latencyTier")),
once = boolValue(obj, listOf("once")),
)
val hasDirective = listOf(
directive.voiceId,
directive.modelId,
directive.speed,
directive.rateWpm,
directive.stability,
directive.similarity,
directive.style,
directive.speakerBoost,
directive.seed,
directive.normalize,
directive.language,
directive.outputFormat,
directive.latencyTier,
directive.once,
).any { it != null }
if (!hasDirective) return TalkDirectiveParseResult(null, text, emptyList())
val knownKeys = setOf(
"voice", "voice_id", "voiceid",
"model", "model_id", "modelid",
"speed", "rate", "wpm",
"stability", "similarity", "similarity_boost", "similarityboost",
"style",
"speaker_boost", "speakerboost",
"no_speaker_boost", "nospeakerboost",
"seed",
"normalize", "apply_text_normalization",
"lang", "language_code", "language",
"output_format", "format",
"latency", "latency_tier", "latencytier",
"once",
)
val unknownKeys = obj.keys.filter { !knownKeys.contains(it.lowercase()) }.sorted()
lines.removeAt(firstNonEmpty)
if (firstNonEmpty < lines.size) {
if (lines[firstNonEmpty].trim().isEmpty()) {
lines.removeAt(firstNonEmpty)
}
}
return TalkDirectiveParseResult(directive, lines.joinToString("\n"), unknownKeys)
}
private fun parseJsonObject(line: String): JsonObject? {
return try {
directiveJson.parseToJsonElement(line) as? JsonObject
} catch (_: Throwable) {
null
}
}
private fun stringValue(obj: JsonObject, keys: List<String>): String? {
for (key in keys) {
val value = obj[key].asStringOrNull()?.trim()
if (!value.isNullOrEmpty()) return value
}
return null
}
private fun doubleValue(obj: JsonObject, keys: List<String>): Double? {
for (key in keys) {
val value = obj[key].asDoubleOrNull()
if (value != null) return value
}
return null
}
private fun intValue(obj: JsonObject, keys: List<String>): Int? {
for (key in keys) {
val value = obj[key].asIntOrNull()
if (value != null) return value
}
return null
}
private fun longValue(obj: JsonObject, keys: List<String>): Long? {
for (key in keys) {
val value = obj[key].asLongOrNull()
if (value != null) return value
}
return null
}
private fun boolValue(obj: JsonObject, keys: List<String>): Boolean? {
for (key in keys) {
val value = obj[key].asBooleanOrNull()
if (value != null) return value
}
return null
}
}
private fun JsonElement?.asStringOrNull(): String? =
(this as? JsonPrimitive)?.takeIf { it.isString }?.content
private fun JsonElement?.asDoubleOrNull(): Double? {
val primitive = this as? JsonPrimitive ?: return null
return primitive.content.toDoubleOrNull()
}
private fun JsonElement?.asIntOrNull(): Int? {
val primitive = this as? JsonPrimitive ?: return null
return primitive.content.toIntOrNull()
}
private fun JsonElement?.asLongOrNull(): Long? {
val primitive = this as? JsonPrimitive ?: return null
return primitive.content.toLongOrNull()
}
private fun JsonElement?.asBooleanOrNull(): Boolean? {
val primitive = this as? JsonPrimitive ?: return null
val content = primitive.content.trim().lowercase()
return when (content) {
"true", "yes", "1" -> true
"false", "no", "0" -> false
else -> null
}
}
File diff suppressed because it is too large. Load diff
@@ -2,5 +2,5 @@
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android"> <adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@color/ic_launcher_background" /> <background android:drawable="@color/ic_launcher_background" />
<foreground android:drawable="@mipmap/ic_launcher_foreground" /> <foreground android:drawable="@mipmap/ic_launcher_foreground" />
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
</adaptive-icon> </adaptive-icon>
@@ -2,5 +2,5 @@
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android"> <adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@color/ic_launcher_background" /> <background android:drawable="@color/ic_launcher_background" />
<foreground android:drawable="@mipmap/ic_launcher_foreground" /> <foreground android:drawable="@mipmap/ic_launcher_foreground" />
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
</adaptive-icon> </adaptive-icon>
@@ -1,8 +1,7 @@
<resources xmlns:tools="http://schemas.android.com/tools"> <resources>
<style name="Theme.ClawdisNode" parent="Theme.Material3.DayNight.NoActionBar"> <style name="Theme.ClawdisNode" parent="Theme.Material3.DayNight.NoActionBar">
<item name="android:statusBarColor">@android:color/transparent</item> <item name="android:statusBarColor">@android:color/transparent</item>
<item name="android:navigationBarColor">@android:color/transparent</item> <item name="android:navigationBarColor">@android:color/transparent</item>
<item name="android:windowLightStatusBar" tools:targetApi="m">false</item> <item name="android:windowLightStatusBar">false</item>
</style> </style>
</resources> </resources>
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<full-backup-content>
<include domain="file" path="." />
</full-backup-content>
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<data-extraction-rules>
<cloud-backup>
<include domain="file" path="." />
</cloud-backup>
<device-transfer>
<include domain="file" path="." />
</device-transfer>
</data-extraction-rules>
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<network-security-config> <network-security-config xmlns:tools="http://schemas.android.com/tools">
<!-- This app is primarily used on a trusted tailnet; allow cleartext for IP-based endpoints too. --> <!-- This app is primarily used on a trusted tailnet; allow cleartext for IP-based endpoints too. -->
<base-config cleartextTrafficPermitted="true" /> <base-config cleartextTrafficPermitted="true" tools:ignore="InsecureBaseConfiguration" />
<!-- Allow HTTP for tailnet/local dev endpoints (e.g. canvas/background web). --> <!-- Allow HTTP for tailnet/local dev endpoints (e.g. canvas/background web). -->
<domain-config cleartextTrafficPermitted="true"> <domain-config cleartextTrafficPermitted="true">
<domain includeSubdomains="true">clawdis.internal</domain> <domain includeSubdomains="true">clawdis.internal</domain>
@@ -0,0 +1,14 @@
package com.steipete.clawdis.node.bridge
import io.kotest.core.spec.style.StringSpec
import io.kotest.matchers.shouldBe
class BridgeEndpointKotestTest : StringSpec({
"manual endpoint builds stable id + name" {
val endpoint = BridgeEndpoint.manual("10.0.0.5", 18790)
endpoint.stableId shouldBe "manual|10.0.0.5|18790"
endpoint.name shouldBe "10.0.0.5:18790"
endpoint.host shouldBe "10.0.0.5"
endpoint.port shouldBe 18790
}
})
@@ -0,0 +1,47 @@
package com.steipete.clawdis.node.node
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
import kotlin.math.min
class JpegSizeLimiterTest {
@Test
fun compressesLargePayloadsUnderLimit() {
val maxBytes = 5 * 1024 * 1024
val result =
JpegSizeLimiter.compressToLimit(
initialWidth = 4000,
initialHeight = 3000,
startQuality = 95,
maxBytes = maxBytes,
encode = { width, height, quality ->
val estimated = (width.toLong() * height.toLong() * quality.toLong()) / 100
val size = min(maxBytes.toLong() * 2, estimated).toInt()
ByteArray(size)
},
)
assertTrue(result.bytes.size <= maxBytes)
assertTrue(result.width <= 4000)
assertTrue(result.height <= 3000)
assertTrue(result.quality <= 95)
}
@Test
fun keepsSmallPayloadsAsIs() {
val maxBytes = 5 * 1024 * 1024
val result =
JpegSizeLimiter.compressToLimit(
initialWidth = 800,
initialHeight = 600,
startQuality = 90,
maxBytes = maxBytes,
encode = { _, _, _ -> ByteArray(120_000) },
)
assertEquals(800, result.width)
assertEquals(600, result.height)
assertEquals(90, result.quality)
}
}
@@ -1,9 +1,24 @@
package com.steipete.clawdis.node.protocol package com.steipete.clawdis.node.protocol
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonObject
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
import org.junit.Test import org.junit.Test
class ClawdisCanvasA2UIActionTest { class ClawdisCanvasA2UIActionTest {
@Test
fun extractActionNameAcceptsNameOrAction() {
val nameObj = Json.parseToJsonElement("{\"name\":\"Hello\"}").jsonObject
assertEquals("Hello", ClawdisCanvasA2UIAction.extractActionName(nameObj))
val actionObj = Json.parseToJsonElement("{\"action\":\"Wave\"}").jsonObject
assertEquals("Wave", ClawdisCanvasA2UIAction.extractActionName(actionObj))
val fallbackObj =
Json.parseToJsonElement("{\"name\":\" \",\"action\":\"Fallback\"}").jsonObject
assertEquals("Fallback", ClawdisCanvasA2UIAction.extractActionName(fallbackObj))
}
@Test @Test
fun formatAgentMessageMatchesSharedSpec() { fun formatAgentMessageMatchesSharedSpec() {
val msg = val msg =
@@ -32,4 +47,3 @@ class ClawdisCanvasA2UIActionTest {
) )
} }
} }
@@ -0,0 +1,55 @@
package com.steipete.clawdis.node.voice
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
class TalkDirectiveParserTest {
@Test
fun parsesDirectiveAndStripsHeader() {
val input = """
{"voice":"voice-123","once":true}
Hello from talk mode.
""".trimIndent()
val result = TalkDirectiveParser.parse(input)
assertEquals("voice-123", result.directive?.voiceId)
assertEquals(true, result.directive?.once)
assertEquals("Hello from talk mode.", result.stripped.trim())
}
@Test
fun ignoresUnknownKeysButReportsThem() {
val input = """
{"voice":"abc","foo":1,"bar":"baz"}
Hi there.
""".trimIndent()
val result = TalkDirectiveParser.parse(input)
assertEquals("abc", result.directive?.voiceId)
assertTrue(result.unknownKeys.containsAll(listOf("bar", "foo")))
}
@Test
fun parsesAlternateKeys() {
val input = """
{"model_id":"eleven_v3","similarity_boost":0.4,"no_speaker_boost":true,"rate":200}
Speak.
""".trimIndent()
val result = TalkDirectiveParser.parse(input)
assertEquals("eleven_v3", result.directive?.modelId)
assertEquals(0.4, result.directive?.similarity)
assertEquals(false, result.directive?.speakerBoost)
assertEquals(200, result.directive?.rateWpm)
}
@Test
fun returnsNullWhenNoDirectivePresent() {
val input = """
{}
Hello.
""".trimIndent()
val result = TalkDirectiveParser.parse(input)
assertNull(result.directive)
assertEquals(input, result.stripped)
}
}
@@ -6,6 +6,15 @@ import Observation
import SwiftUI import SwiftUI
import UIKit import UIKit
protocol BridgePairingClient: Sendable {
func pairAndHello(
endpoint: NWEndpoint,
hello: BridgeHello,
onStatus: (@Sendable (String) -> Void)?) async throws -> String
}
extension BridgeClient: BridgePairingClient {}
@MainActor @MainActor
@Observable @Observable
final class BridgeConnectionController { final class BridgeConnectionController {
@@ -16,10 +25,16 @@ final class BridgeConnectionController {
private let discovery = BridgeDiscoveryModel() private let discovery = BridgeDiscoveryModel()
private weak var appModel: NodeAppModel? private weak var appModel: NodeAppModel?
private var didAutoConnect = false private var didAutoConnect = false
private var seenStableIDs = Set<String>()
init(appModel: NodeAppModel, startDiscovery: Bool = true) { private let bridgeClientFactory: @Sendable () -> any BridgePairingClient
init(
appModel: NodeAppModel,
startDiscovery: Bool = true,
bridgeClientFactory: @escaping @Sendable () -> any BridgePairingClient = { BridgeClient() })
{
self.appModel = appModel self.appModel = appModel
self.bridgeClientFactory = bridgeClientFactory
BridgeSettingsStore.bootstrapPersistence() BridgeSettingsStore.bootstrapPersistence()
let defaults = UserDefaults.standard let defaults = UserDefaults.standard
@@ -85,7 +100,7 @@ final class BridgeConnectionController {
let token = KeychainStore.loadString( let token = KeychainStore.loadString(
service: "com.steipete.clawdis.bridge", service: "com.steipete.clawdis.bridge",
account: "bridge-token.\(instanceId)")? account: self.keychainAccount(instanceId: instanceId))?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" .trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !token.isEmpty else { return } guard !token.isEmpty else { return }
@@ -99,28 +114,40 @@ final class BridgeConnectionController {
guard let port = NWEndpoint.Port(rawValue: UInt16(resolvedPort)) else { return } guard let port = NWEndpoint.Port(rawValue: UInt16(resolvedPort)) else { return }
self.didAutoConnect = true self.didAutoConnect = true
appModel.connectToBridge( let endpoint = NWEndpoint.hostPort(host: NWEndpoint.Host(manualHost), port: port)
endpoint: .hostPort(host: NWEndpoint.Host(manualHost), port: port), self.startAutoConnect(endpoint: endpoint, token: token, instanceId: instanceId)
hello: self.makeHello(token: token))
return return
} }
let targetStableID = defaults.string(forKey: "bridge.lastDiscoveredStableID")? let preferredStableID = defaults.string(forKey: "bridge.preferredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" .trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !targetStableID.isEmpty else { return } let lastDiscoveredStableID = defaults.string(forKey: "bridge.lastDiscoveredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let candidates = [preferredStableID, lastDiscoveredStableID].filter { !$0.isEmpty }
guard let targetStableID = candidates.first(where: { id in
self.bridges.contains(where: { $0.stableID == id })
}) else { return }
guard let target = self.bridges.first(where: { $0.stableID == targetStableID }) else { return } guard let target = self.bridges.first(where: { $0.stableID == targetStableID }) else { return }
self.didAutoConnect = true self.didAutoConnect = true
appModel.connectToBridge(endpoint: target.endpoint, hello: self.makeHello(token: token)) self.startAutoConnect(endpoint: target.endpoint, token: token, instanceId: instanceId)
} }
private func updateLastDiscoveredBridge(from bridges: [BridgeDiscoveryModel.DiscoveredBridge]) { private func updateLastDiscoveredBridge(from bridges: [BridgeDiscoveryModel.DiscoveredBridge]) {
let newlyDiscovered = bridges.filter { self.seenStableIDs.insert($0.stableID).inserted } let defaults = UserDefaults.standard
guard let last = newlyDiscovered.last else { return } let preferred = defaults.string(forKey: "bridge.preferredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let existingLast = defaults.string(forKey: "bridge.lastDiscoveredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
UserDefaults.standard.set(last.stableID, forKey: "bridge.lastDiscoveredStableID") // Avoid overriding user intent (preferred/lastDiscovered are also set on manual Connect).
BridgeSettingsStore.saveLastDiscoveredBridgeStableID(last.stableID) guard preferred.isEmpty, existingLast.isEmpty else { return }
guard let first = bridges.first else { return }
defaults.set(first.stableID, forKey: "bridge.lastDiscoveredStableID")
BridgeSettingsStore.saveLastDiscoveredBridgeStableID(first.stableID)
} }
private func makeHello(token: String) -> BridgeHello { private func makeHello(token: String) -> BridgeHello {
@@ -140,6 +167,40 @@ final class BridgeConnectionController {
commands: self.currentCommands()) commands: self.currentCommands())
} }
private func keychainAccount(instanceId: String) -> String {
"bridge-token.\(instanceId)"
}
private func startAutoConnect(endpoint: NWEndpoint, token: String, instanceId: String) {
guard let appModel else { return }
Task { [weak self] in
guard let self else { return }
do {
let hello = self.makeHello(token: token)
let refreshed = try await self.bridgeClientFactory().pairAndHello(
endpoint: endpoint,
hello: hello,
onStatus: { status in
Task { @MainActor in
appModel.bridgeStatusText = status
}
})
let resolvedToken = refreshed.isEmpty ? token : refreshed
if !refreshed.isEmpty, refreshed != token {
_ = KeychainStore.saveString(
refreshed,
service: "com.steipete.clawdis.bridge",
account: self.keychainAccount(instanceId: instanceId))
}
appModel.connectToBridge(endpoint: endpoint, hello: self.makeHello(token: resolvedToken))
} catch {
await MainActor.run {
appModel.bridgeStatusText = "Bridge error: \(error.localizedDescription)"
}
}
}
}
private func resolvedDisplayName(defaults: UserDefaults) -> String { private func resolvedDisplayName(defaults: UserDefaults) -> String {
let key = "node.displayName" let key = "node.displayName"
let existing = defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" let existing = defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
@@ -231,3 +292,47 @@ final class BridgeConnectionController {
Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev" Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"
} }
} }
#if DEBUG
extension BridgeConnectionController {
func _test_makeHello(token: String) -> BridgeHello {
self.makeHello(token: token)
}
func _test_resolvedDisplayName(defaults: UserDefaults) -> String {
self.resolvedDisplayName(defaults: defaults)
}
func _test_currentCaps() -> [String] {
self.currentCaps()
}
func _test_currentCommands() -> [String] {
self.currentCommands()
}
func _test_platformString() -> String {
self.platformString()
}
func _test_deviceFamily() -> String {
self.deviceFamily()
}
func _test_modelIdentifier() -> String {
self.modelIdentifier()
}
func _test_appVersion() -> String {
self.appVersion()
}
func _test_setBridges(_ bridges: [BridgeDiscoveryModel.DiscoveredBridge]) {
self.bridges = bridges
}
func _test_triggerAutoConnect() {
self.maybeAutoConnect()
}
}
#endif
@@ -18,6 +18,12 @@ final class BridgeDiscoveryModel {
var endpoint: NWEndpoint var endpoint: NWEndpoint
var stableID: String var stableID: String
var debugID: String var debugID: String
var lanHost: String?
var tailnetDns: String?
var gatewayPort: Int?
var bridgePort: Int?
var canvasPort: Int?
var cliPath: String?
} }
var bridges: [DiscoveredBridge] = [] var bridges: [DiscoveredBridge] = []
@@ -68,7 +74,8 @@ final class BridgeDiscoveryModel {
switch result.endpoint { switch result.endpoint {
case let .service(name, _, _, _): case let .service(name, _, _, _):
let decodedName = BonjourEscapes.decode(name) let decodedName = BonjourEscapes.decode(name)
let advertisedName = result.endpoint.txtRecord?.dictionary["displayName"] let txt = result.endpoint.txtRecord?.dictionary ?? [:]
let advertisedName = txt["displayName"]
let prettyAdvertised = advertisedName let prettyAdvertised = advertisedName
.map(Self.prettifyInstanceName) .map(Self.prettifyInstanceName)
.flatMap { $0.isEmpty ? nil : $0 } .flatMap { $0.isEmpty ? nil : $0 }
@@ -77,7 +84,13 @@ final class BridgeDiscoveryModel {
name: prettyName, name: prettyName,
endpoint: result.endpoint, endpoint: result.endpoint,
stableID: BridgeEndpointID.stableID(result.endpoint), stableID: BridgeEndpointID.stableID(result.endpoint),
debugID: BridgeEndpointID.prettyDescription(result.endpoint)) debugID: BridgeEndpointID.prettyDescription(result.endpoint),
lanHost: Self.txtValue(txt, key: "lanHost"),
tailnetDns: Self.txtValue(txt, key: "tailnetDns"),
gatewayPort: Self.txtIntValue(txt, key: "gatewayPort"),
bridgePort: Self.txtIntValue(txt, key: "bridgePort"),
canvasPort: Self.txtIntValue(txt, key: "canvasPort"),
cliPath: Self.txtValue(txt, key: "cliPath"))
default: default:
return nil return nil
} }
@@ -191,4 +204,14 @@ final class BridgeDiscoveryModel {
.replacingOccurrences(of: #"\s+\(\d+\)$"#, with: "", options: .regularExpression) .replacingOccurrences(of: #"\s+\(\d+\)$"#, with: "", options: .regularExpression)
return stripped.trimmingCharacters(in: .whitespacesAndNewlines) return stripped.trimmingCharacters(in: .whitespacesAndNewlines)
} }
private static func txtValue(_ dict: [String: String], key: String) -> String? {
let raw = dict[key]?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return raw.isEmpty ? nil : raw
}
private static func txtIntValue(_ dict: [String: String], key: String) -> Int? {
guard let raw = self.txtValue(dict, key: key) else { return nil }
return Int(raw)
}
} }
@@ -84,10 +84,14 @@ actor CameraController {
} }
withExtendedLifetime(delegate) {} withExtendedLifetime(delegate) {}
let maxPayloadBytes = 5 * 1024 * 1024
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
let maxEncodedBytes = (maxPayloadBytes / 4) * 3
let res = try JPEGTranscoder.transcodeToJPEG( let res = try JPEGTranscoder.transcodeToJPEG(
imageData: rawData, imageData: rawData,
maxWidthPx: maxWidth, maxWidthPx: maxWidth,
quality: quality) quality: quality,
maxBytes: maxEncodedBytes)
return ( return (
format: format.rawValue, format: format.rawValue,
+4 -2
View File
@@ -4,18 +4,20 @@ import SwiftUI
struct ChatSheet: View { struct ChatSheet: View {
@Environment(\.dismiss) private var dismiss @Environment(\.dismiss) private var dismiss
@State private var viewModel: ClawdisChatViewModel @State private var viewModel: ClawdisChatViewModel
private let userAccent: Color?
init(bridge: BridgeSession, sessionKey: String = "main") { init(bridge: BridgeSession, sessionKey: String = "main", userAccent: Color? = nil) {
let transport = IOSBridgeChatTransport(bridge: bridge) let transport = IOSBridgeChatTransport(bridge: bridge)
self._viewModel = State( self._viewModel = State(
initialValue: ClawdisChatViewModel( initialValue: ClawdisChatViewModel(
sessionKey: sessionKey, sessionKey: sessionKey,
transport: transport)) transport: transport))
self.userAccent = userAccent
} }
var body: some View { var body: some View {
NavigationStack { NavigationStack {
ClawdisChatView(viewModel: self.viewModel) ClawdisChatView(viewModel: self.viewModel, userAccent: self.userAccent)
.navigationTitle("Chat") .navigationTitle("Chat")
.navigationBarTitleDisplayMode(.inline) .navigationBarTitleDisplayMode(.inline)
.toolbar { .toolbar {
+19
View File
@@ -22,6 +22,11 @@
<string>1.0</string> <string>1.0</string>
<key>CFBundleVersion</key> <key>CFBundleVersion</key>
<string>1</string> <string>1</string>
<key>NSAppTransportSecurity</key>
<dict>
<key>NSAllowsArbitraryLoadsInWebContent</key>
<true/>
</dict>
<key>NSBonjourServices</key> <key>NSBonjourServices</key>
<array> <array>
<string>_clawdis-bridge._tcp</string> <string>_clawdis-bridge._tcp</string>
@@ -45,5 +50,19 @@
</array> </array>
<key>UILaunchScreen</key> <key>UILaunchScreen</key>
<dict/> <dict/>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
</dict> </dict>
</plist> </plist>
+124 -9
View File
@@ -22,23 +22,28 @@ final class NodeAppModel {
var bridgeServerName: String? var bridgeServerName: String?
var bridgeRemoteAddress: String? var bridgeRemoteAddress: String?
var connectedBridgeID: String? var connectedBridgeID: String?
var seamColorHex: String?
var mainSessionKey: String = "main"
private let bridge = BridgeSession() private let bridge = BridgeSession()
private var bridgeTask: Task<Void, Never>? private var bridgeTask: Task<Void, Never>?
private var voiceWakeSyncTask: Task<Void, Never>? private var voiceWakeSyncTask: Task<Void, Never>?
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>? @ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
let voiceWake = VoiceWakeManager() let voiceWake = VoiceWakeManager()
let talkMode = TalkModeManager()
private var lastAutoA2uiURL: String?
var bridgeSession: BridgeSession { self.bridge } var bridgeSession: BridgeSession { self.bridge }
var cameraHUDText: String? var cameraHUDText: String?
var cameraHUDKind: CameraHUDKind? var cameraHUDKind: CameraHUDKind?
var cameraFlashNonce: Int = 0 var cameraFlashNonce: Int = 0
var screenRecordActive: Bool = false
init() { init() {
self.voiceWake.configure { [weak self] cmd in self.voiceWake.configure { [weak self] cmd in
guard let self else { return } guard let self else { return }
let sessionKey = "main" let sessionKey = await MainActor.run { self.mainSessionKey }
do { do {
try await self.sendVoiceTranscript(text: cmd, sessionKey: sessionKey) try await self.sendVoiceTranscript(text: cmd, sessionKey: sessionKey)
} catch { } catch {
@@ -48,6 +53,9 @@ final class NodeAppModel {
let enabled = UserDefaults.standard.bool(forKey: "voiceWake.enabled") let enabled = UserDefaults.standard.bool(forKey: "voiceWake.enabled")
self.voiceWake.setEnabled(enabled) self.voiceWake.setEnabled(enabled)
self.talkMode.attachBridge(self.bridge)
let talkEnabled = UserDefaults.standard.bool(forKey: "talk.enabled")
self.talkMode.setEnabled(talkEnabled)
// Wire up deep links from canvas taps // Wire up deep links from canvas taps
self.screen.onDeepLink = { [weak self] url in self.screen.onDeepLink = { [weak self] url in
@@ -80,7 +88,7 @@ final class NodeAppModel {
}() }()
guard !userAction.isEmpty else { return } guard !userAction.isEmpty else { return }
guard let name = userAction["name"] as? String, !name.isEmpty else { return } guard let name = ClawdisCanvasA2UIAction.extractActionName(userAction) else { return }
let actionId: String = { let actionId: String = {
let id = (userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" let id = (userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return id.isEmpty ? UUID().uuidString : id return id.isEmpty ? UUID().uuidString : id
@@ -102,14 +110,12 @@ final class NodeAppModel {
let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"]) let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"])
let sessionKey = "main" let sessionKey = "main"
let message = ClawdisCanvasA2UIAction.formatAgentMessage( let messageContext = ClawdisCanvasA2UIAction.AgentMessageContext(
actionName: name, actionName: name,
sessionKey: sessionKey, session: .init(key: sessionKey, surfaceId: surfaceId),
surfaceId: surfaceId, component: .init(id: sourceComponentId, host: host, instanceId: instanceId),
sourceComponentId: sourceComponentId,
host: host,
instanceId: instanceId,
contextJSON: contextJSON) contextJSON: contextJSON)
let message = ClawdisCanvasA2UIAction.formatAgentMessage(messageContext)
let ok: Bool let ok: Bool
var errorText: String? var errorText: String?
@@ -146,7 +152,21 @@ final class NodeAppModel {
guard let raw = await self.bridge.currentCanvasHostUrl() else { return nil } guard let raw = await self.bridge.currentCanvasHostUrl() else { return nil }
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil } guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
return base.appendingPathComponent("__clawdis__/a2ui/").absoluteString return base.appendingPathComponent("__clawdis__/a2ui/").absoluteString + "?platform=ios"
}
private func showA2UIOnConnectIfNeeded() async {
guard let a2uiUrl = await self.resolveA2UIHostURL() else { return }
let current = self.screen.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
if current.isEmpty || current == self.lastAutoA2uiURL {
self.screen.navigate(to: a2uiUrl)
self.lastAutoA2uiURL = a2uiUrl
}
}
private func showLocalCanvasOnDisconnect() {
self.lastAutoA2uiURL = nil
self.screen.showDefaultCanvas()
} }
func setScenePhase(_ phase: ScenePhase) { func setScenePhase(_ phase: ScenePhase) {
@@ -164,6 +184,10 @@ final class NodeAppModel {
self.voiceWake.setEnabled(enabled) self.voiceWake.setEnabled(enabled)
} }
func setTalkEnabled(_ enabled: Bool) {
self.talkMode.setEnabled(enabled)
}
func connectToBridge( func connectToBridge(
endpoint: NWEndpoint, endpoint: NWEndpoint,
hello: BridgeHello) hello: BridgeHello)
@@ -203,7 +227,9 @@ final class NodeAppModel {
self.bridgeRemoteAddress = addr self.bridgeRemoteAddress = addr
} }
} }
await self.refreshBrandingFromGateway()
await self.startVoiceWakeSync() await self.startVoiceWakeSync()
await self.showA2UIOnConnectIfNeeded()
}, },
onInvoke: { [weak self] req in onInvoke: { [weak self] req in
guard let self else { guard let self else {
@@ -216,6 +242,9 @@ final class NodeAppModel {
}) })
if Task.isCancelled { break } if Task.isCancelled { break }
await MainActor.run {
self.showLocalCanvasOnDisconnect()
}
attempt += 1 attempt += 1
let sleepSeconds = min(6.0, 0.35 * pow(1.7, Double(attempt))) let sleepSeconds = min(6.0, 0.35 * pow(1.7, Double(attempt)))
try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000)) try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000))
@@ -226,6 +255,7 @@ final class NodeAppModel {
self.bridgeStatusText = "Bridge error: \(error.localizedDescription)" self.bridgeStatusText = "Bridge error: \(error.localizedDescription)"
self.bridgeServerName = nil self.bridgeServerName = nil
self.bridgeRemoteAddress = nil self.bridgeRemoteAddress = nil
self.showLocalCanvasOnDisconnect()
} }
let sleepSeconds = min(8.0, 0.5 * pow(1.7, Double(attempt))) let sleepSeconds = min(8.0, 0.5 * pow(1.7, Double(attempt)))
try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000)) try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000))
@@ -237,6 +267,9 @@ final class NodeAppModel {
self.bridgeServerName = nil self.bridgeServerName = nil
self.bridgeRemoteAddress = nil self.bridgeRemoteAddress = nil
self.connectedBridgeID = nil self.connectedBridgeID = nil
self.seamColorHex = nil
self.mainSessionKey = "main"
self.showLocalCanvasOnDisconnect()
} }
} }
} }
@@ -251,6 +284,45 @@ final class NodeAppModel {
self.bridgeServerName = nil self.bridgeServerName = nil
self.bridgeRemoteAddress = nil self.bridgeRemoteAddress = nil
self.connectedBridgeID = nil self.connectedBridgeID = nil
self.seamColorHex = nil
self.mainSessionKey = "main"
self.showLocalCanvasOnDisconnect()
}
var seamColor: Color {
Self.color(fromHex: self.seamColorHex) ?? Self.defaultSeamColor
}
private static let defaultSeamColor = Color(red: 79 / 255.0, green: 122 / 255.0, blue: 154 / 255.0)
private static func color(fromHex raw: String?) -> Color? {
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed
guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil }
let r = Double((value >> 16) & 0xFF) / 255.0
let g = Double((value >> 8) & 0xFF) / 255.0
let b = Double(value & 0xFF) / 255.0
return Color(red: r, green: g, blue: b)
}
private func refreshBrandingFromGateway() async {
do {
let res = try await self.bridge.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8)
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return }
guard let config = json["config"] as? [String: Any] else { return }
let ui = config["ui"] as? [String: Any]
let raw = (ui?["seamColor"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let session = config["session"] as? [String: Any]
let rawMainKey = (session?["mainKey"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let mainKey = rawMainKey.isEmpty ? "main" : rawMainKey
await MainActor.run {
self.seamColorHex = raw.isEmpty ? nil : raw
self.mainSessionKey = mainKey
}
} catch {
// ignore
}
} }
func setGlobalWakeWords(_ words: [String]) async { func setGlobalWakeWords(_ words: [String]) async {
@@ -570,6 +642,9 @@ final class NodeAppModel {
NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4", NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4",
]) ])
} }
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
self.screenRecordActive = true
defer { self.screenRecordActive = false }
let path = try await self.screenRecorder.record( let path = try await self.screenRecorder.record(
screenIndex: params.screenIndex, screenIndex: params.screenIndex,
durationMs: params.durationMs, durationMs: params.durationMs,
@@ -660,3 +735,43 @@ final class NodeAppModel {
} }
} }
} }
#if DEBUG
extension NodeAppModel {
func _test_handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
await self.handleInvoke(req)
}
static func _test_decodeParams<T: Decodable>(_ type: T.Type, from json: String?) throws -> T {
try self.decodeParams(type, from: json)
}
static func _test_encodePayload(_ obj: some Encodable) throws -> String {
try self.encodePayload(obj)
}
func _test_isCameraEnabled() -> Bool {
self.isCameraEnabled()
}
func _test_triggerCameraFlash() {
self.triggerCameraFlash()
}
func _test_showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) {
self.showCameraHUD(text: text, kind: kind, autoHideSeconds: autoHideSeconds)
}
func _test_handleCanvasA2UIAction(body: [String: Any]) async {
await self.handleCanvasA2UIAction(body: body)
}
func _test_resolveA2UIHostURL() async -> String? {
await self.resolveA2UIHostURL()
}
func _test_showLocalCanvasOnDisconnect() {
self.showLocalCanvasOnDisconnect()
}
}
#endif
+118 -82
View File
@@ -8,6 +8,7 @@ struct RootCanvas: View {
@Environment(\.scenePhase) private var scenePhase @Environment(\.scenePhase) private var scenePhase
@AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false @AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true @AppStorage("screen.preventSleep") private var preventSleep: Bool = true
@AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false
@State private var presentedSheet: PresentedSheet? @State private var presentedSheet: PresentedSheet?
@State private var voiceWakeToastText: String? @State private var voiceWakeToastText: String?
@State private var toastDismissTask: Task<Void, Never>? @State private var toastDismissTask: Task<Void, Never>?
@@ -50,12 +51,20 @@ struct RootCanvas: View {
case .settings: case .settings:
SettingsTab() SettingsTab()
case .chat: case .chat:
ChatSheet(bridge: self.appModel.bridgeSession) ChatSheet(
bridge: self.appModel.bridgeSession,
sessionKey: self.appModel.mainSessionKey,
userAccent: self.appModel.seamColor)
} }
} }
.onAppear { self.updateIdleTimer() } .onAppear { self.updateIdleTimer() }
.onChange(of: self.preventSleep) { _, _ in self.updateIdleTimer() } .onChange(of: self.preventSleep) { _, _ in self.updateIdleTimer() }
.onChange(of: self.scenePhase) { _, _ in self.updateIdleTimer() } .onChange(of: self.scenePhase) { _, _ in self.updateIdleTimer() }
.onAppear { self.updateCanvasDebugStatus() }
.onChange(of: self.canvasDebugStatusEnabled) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.bridgeStatusText) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.bridgeServerName) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.bridgeRemoteAddress) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in .onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in
guard let newValue else { return } guard let newValue else { return }
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -102,9 +111,20 @@ struct RootCanvas: View {
private func updateIdleTimer() { private func updateIdleTimer() {
UIApplication.shared.isIdleTimerDisabled = (self.scenePhase == .active && self.preventSleep) UIApplication.shared.isIdleTimerDisabled = (self.scenePhase == .active && self.preventSleep)
} }
private func updateCanvasDebugStatus() {
self.appModel.screen.setDebugStatusEnabled(self.canvasDebugStatusEnabled)
guard self.canvasDebugStatusEnabled else { return }
let title = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let subtitle = self.appModel.bridgeServerName ?? self.appModel.bridgeRemoteAddress
self.appModel.screen.updateDebugStatus(title: title, subtitle: subtitle)
}
} }
private struct CanvasContent: View { private struct CanvasContent: View {
@Environment(NodeAppModel.self) private var appModel
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
@AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true
var systemColorScheme: ColorScheme var systemColorScheme: ColorScheme
var bridgeStatus: StatusPill.BridgeState var bridgeStatus: StatusPill.BridgeState
var voiceWakeEnabled: Bool var voiceWakeEnabled: Bool
@@ -126,6 +146,21 @@ private struct CanvasContent: View {
} }
.accessibilityLabel("Chat") .accessibilityLabel("Chat")
if self.talkButtonEnabled {
// Talk mode lives on a side bubble so it doesn't get buried in settings.
OverlayButton(
systemImage: self.appModel.talkMode.isEnabled ? "waveform.circle.fill" : "waveform.circle",
brighten: self.brightenButtons,
tint: self.appModel.seamColor,
isActive: self.appModel.talkMode.isEnabled)
{
let next = !self.appModel.talkMode.isEnabled
self.talkEnabled = next
self.appModel.setTalkEnabled(next)
}
.accessibilityLabel("Talk Mode")
}
OverlayButton(systemImage: "gearshape.fill", brighten: self.brightenButtons) { OverlayButton(systemImage: "gearshape.fill", brighten: self.brightenButtons) {
self.openSettings() self.openSettings()
} }
@@ -134,10 +169,17 @@ private struct CanvasContent: View {
.padding(.top, 10) .padding(.top, 10)
.padding(.trailing, 10) .padding(.trailing, 10)
} }
.overlay(alignment: .center) {
if self.appModel.talkMode.isEnabled {
TalkOrbOverlay()
.transition(.opacity)
}
}
.overlay(alignment: .topLeading) { .overlay(alignment: .topLeading) {
StatusPill( StatusPill(
bridge: self.bridgeStatus, bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled, voiceWakeEnabled: self.voiceWakeEnabled,
activity: self.statusActivity,
brighten: self.brightenButtons, brighten: self.brightenButtons,
onTap: { onTap: {
self.openSettings() self.openSettings()
@@ -155,45 +197,78 @@ private struct CanvasContent: View {
.transition(.move(edge: .top).combined(with: .opacity)) .transition(.move(edge: .top).combined(with: .opacity))
} }
} }
.overlay(alignment: .topLeading) {
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
CameraCaptureToast(
text: cameraHUDText,
kind: self.mapCameraKind(cameraHUDKind),
brighten: self.brightenButtons)
.padding(SwiftUI.Edge.Set.leading, 10)
.safeAreaPadding(SwiftUI.Edge.Set.top, 106)
.transition(
AnyTransition.move(edge: SwiftUI.Edge.top)
.combined(with: AnyTransition.opacity))
}
}
} }
private func mapCameraKind(_ kind: NodeAppModel.CameraHUDKind) -> CameraCaptureToast.Kind { private var statusActivity: StatusPill.Activity? {
switch kind { // Status pill owns transient activity state so it doesn't overlap the connection indicator.
case .photo: if self.appModel.isBackgrounded {
.photo return StatusPill.Activity(
case .recording: title: "Foreground required",
.recording systemImage: "exclamationmark.triangle.fill",
case .success: tint: .orange)
.success
case .error:
.error
} }
let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let bridgeLower = bridgeStatus.lowercased()
if bridgeLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if bridgeLower.contains("approval") || bridgeLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
// Avoid duplicating the primary bridge status ("Connecting…") in the activity slot.
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
} }
} }
private struct OverlayButton: View { private struct OverlayButton: View {
let systemImage: String let systemImage: String
let brighten: Bool let brighten: Bool
var tint: Color?
var isActive: Bool = false
let action: () -> Void let action: () -> Void
var body: some View { var body: some View {
Button(action: self.action) { Button(action: self.action) {
Image(systemName: self.systemImage) Image(systemName: self.systemImage)
.font(.system(size: 16, weight: .semibold)) .font(.system(size: 16, weight: .semibold))
.foregroundStyle(.primary) .foregroundStyle(self.isActive ? (self.tint ?? .primary) : .primary)
.padding(10) .padding(10)
.background { .background {
RoundedRectangle(cornerRadius: 12, style: .continuous) RoundedRectangle(cornerRadius: 12, style: .continuous)
@@ -211,9 +286,26 @@ private struct OverlayButton: View {
endPoint: .bottomTrailing)) endPoint: .bottomTrailing))
.blendMode(.overlay) .blendMode(.overlay)
} }
.overlay {
if let tint {
RoundedRectangle(cornerRadius: 12, style: .continuous)
.fill(
LinearGradient(
colors: [
tint.opacity(self.isActive ? 0.22 : 0.14),
tint.opacity(self.isActive ? 0.10 : 0.06),
.clear,
],
startPoint: .topLeading,
endPoint: .bottomTrailing))
.blendMode(.overlay)
}
}
.overlay { .overlay {
RoundedRectangle(cornerRadius: 12, style: .continuous) RoundedRectangle(cornerRadius: 12, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5) .strokeBorder(
(self.tint ?? .white).opacity(self.isActive ? 0.34 : (self.brighten ? 0.24 : 0.18)),
lineWidth: self.isActive ? 0.7 : 0.5)
} }
.shadow(color: .black.opacity(0.35), radius: 12, y: 6) .shadow(color: .black.opacity(0.35), radius: 12, y: 6)
} }
@@ -247,59 +339,3 @@ private struct CameraFlashOverlay: View {
} }
} }
} }
private struct CameraCaptureToast: View {
enum Kind {
case photo
case recording
case success
case error
}
var text: String
var kind: Kind
var brighten: Bool = false
var body: some View {
HStack(spacing: 10) {
self.icon
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
Text(self.text)
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
.lineLimit(1)
.truncationMode(.tail)
}
.padding(.vertical, 10)
.padding(.horizontal, 12)
.background {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.fill(.ultraThinMaterial)
.overlay {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
}
.accessibilityLabel("Camera")
.accessibilityValue(self.text)
}
@ViewBuilder
private var icon: some View {
switch self.kind {
case .photo:
Image(systemName: "camera.fill")
case .recording:
Image(systemName: "record.circle.fill")
.symbolRenderingMode(.palette)
.foregroundStyle(.red, .primary)
case .success:
Image(systemName: "checkmark.circle.fill")
case .error:
Image(systemName: "exclamationmark.triangle.fill")
}
}
}
+61
View File
@@ -26,6 +26,7 @@ struct RootTabs: View {
StatusPill( StatusPill(
bridge: self.bridgeStatus, bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled, voiceWakeEnabled: self.voiceWakeEnabled,
activity: self.statusActivity,
onTap: { self.selectedTab = 2 }) onTap: { self.selectedTab = 2 })
.padding(.leading, 10) .padding(.leading, 10)
.safeAreaPadding(.top, 10) .safeAreaPadding(.top, 10)
@@ -79,4 +80,64 @@ struct RootTabs: View {
return .disconnected return .disconnected
} }
private var statusActivity: StatusPill.Activity? {
// Keep the top pill consistent across tabs (camera + voice wake + pairing states).
if self.appModel.isBackgrounded {
return StatusPill.Activity(
title: "Foreground required",
systemImage: "exclamationmark.triangle.fill",
tint: .orange)
}
let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let bridgeLower = bridgeStatus.lowercased()
if bridgeLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if bridgeLower.contains("approval") || bridgeLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
// Avoid duplicating the primary bridge status ("Connecting…") in the activity slot.
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText = self.appModel.cameraHUDText,
let cameraHUDKind = self.appModel.cameraHUDKind,
!cameraHUDText.isEmpty
{
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
}
} }
+71 -5
View File
@@ -19,12 +19,18 @@ final class ScreenController {
/// Callback invoked when the user clicks an A2UI action (e.g. button) inside the canvas web UI. /// Callback invoked when the user clicks an A2UI action (e.g. button) inside the canvas web UI.
var onA2UIAction: (([String: Any]) -> Void)? var onA2UIAction: (([String: Any]) -> Void)?
private var debugStatusEnabled: Bool = false
private var debugStatusTitle: String?
private var debugStatusSubtitle: String?
init() { init() {
let config = WKWebViewConfiguration() let config = WKWebViewConfiguration()
config.websiteDataStore = .nonPersistent() config.websiteDataStore = .nonPersistent()
let a2uiActionHandler = CanvasA2UIActionMessageHandler() let a2uiActionHandler = CanvasA2UIActionMessageHandler()
let userContentController = WKUserContentController() let userContentController = WKUserContentController()
userContentController.add(a2uiActionHandler, name: CanvasA2UIActionMessageHandler.messageName) for name in CanvasA2UIActionMessageHandler.handlerNames {
userContentController.add(a2uiActionHandler, name: name)
}
config.userContentController = userContentController config.userContentController = userContentController
self.navigationDelegate = ScreenNavigationDelegate() self.navigationDelegate = ScreenNavigationDelegate()
self.a2uiActionHandler = a2uiActionHandler self.a2uiActionHandler = a2uiActionHandler
@@ -37,9 +43,7 @@ final class ScreenController {
self.webView.scrollView.contentInset = .zero self.webView.scrollView.contentInset = .zero
self.webView.scrollView.scrollIndicatorInsets = .zero self.webView.scrollView.scrollIndicatorInsets = .zero
self.webView.scrollView.automaticallyAdjustsScrollIndicatorInsets = false self.webView.scrollView.automaticallyAdjustsScrollIndicatorInsets = false
// Disable scroll to allow touch events to pass through to canvas self.applyScrollBehavior()
self.webView.scrollView.isScrollEnabled = false
self.webView.scrollView.bounces = false
self.webView.navigationDelegate = self.navigationDelegate self.webView.navigationDelegate = self.navigationDelegate
self.navigationDelegate.controller = self self.navigationDelegate.controller = self
a2uiActionHandler.controller = self a2uiActionHandler.controller = self
@@ -54,6 +58,7 @@ final class ScreenController {
func reload() { func reload() {
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines) let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
self.applyScrollBehavior()
if trimmed.isEmpty { if trimmed.isEmpty {
guard let url = Self.canvasScaffoldURL else { return } guard let url = Self.canvasScaffoldURL else { return }
self.errorText = nil self.errorText = nil
@@ -78,6 +83,39 @@ final class ScreenController {
self.reload() self.reload()
} }
func setDebugStatusEnabled(_ enabled: Bool) {
self.debugStatusEnabled = enabled
self.applyDebugStatusIfNeeded()
}
func updateDebugStatus(title: String?, subtitle: String?) {
self.debugStatusTitle = title
self.debugStatusSubtitle = subtitle
self.applyDebugStatusIfNeeded()
}
fileprivate func applyDebugStatusIfNeeded() {
let enabled = self.debugStatusEnabled
let title = self.debugStatusTitle
let subtitle = self.debugStatusSubtitle
let js = """
(() => {
try {
const api = globalThis.__clawdis;
if (!api) return;
if (typeof api.setDebugStatusEnabled === 'function') {
api.setDebugStatusEnabled(\(enabled ? "true" : "false"));
}
if (!\(enabled ? "true" : "false")) return;
if (typeof api.setStatus === 'function') {
api.setStatus(\(Self.jsValue(title)), \(Self.jsValue(subtitle)));
}
} catch (_) {}
})()
"""
self.webView.evaluateJavaScript(js) { _, _ in }
}
func waitForA2UIReady(timeoutMs: Int) async -> Bool { func waitForA2UIReady(timeoutMs: Int) async -> Bool {
let clock = ContinuousClock() let clock = ContinuousClock()
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs)) let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))
@@ -90,7 +128,8 @@ final class ScreenController {
} catch (_) { return false; } } catch (_) { return false; }
})() })()
""") """)
if res == "true" { return true } let trimmed = res.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
if trimmed == "true" || trimmed == "1" { return true }
} catch { } catch {
// ignore; page likely still loading // ignore; page likely still loading
} }
@@ -210,6 +249,26 @@ final class ScreenController {
return false return false
} }
private func applyScrollBehavior() {
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
let allowScroll = !trimmed.isEmpty
let scrollView = self.webView.scrollView
// Default canvas needs raw touch events; external pages should scroll.
scrollView.isScrollEnabled = allowScroll
scrollView.bounces = allowScroll
}
private static func jsValue(_ value: String?) -> String {
guard let value else { return "null" }
if let data = try? JSONSerialization.data(withJSONObject: [value]),
let encoded = String(data: data, encoding: .utf8),
encoded.count >= 2
{
return String(encoded.dropFirst().dropLast())
}
return "null"
}
func isLocalNetworkCanvasURL(_ url: URL) -> Bool { func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else { guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return false return false
@@ -316,6 +375,11 @@ private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
self.controller?.errorText = error.localizedDescription self.controller?.errorText = error.localizedDescription
} }
func webView(_: WKWebView, didFinish _: WKNavigation?) {
self.controller?.errorText = nil
self.controller?.applyDebugStatusIfNeeded()
}
func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) { func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) {
self.controller?.errorText = error.localizedDescription self.controller?.errorText = error.localizedDescription
} }
@@ -323,6 +387,8 @@ private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler { private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "clawdisCanvasA2UIAction" static let messageName = "clawdisCanvasA2UIAction"
static let legacyMessageNames = ["canvas", "a2ui", "userAction", "action"]
static let handlerNames = [messageName] + legacyMessageNames
weak var controller: ScreenController? weak var controller: ScreenController?
+173 -97
View File
@@ -1,12 +1,28 @@
import AVFoundation import AVFoundation
import ReplayKit import ReplayKit
@MainActor final class ScreenRecordService: @unchecked Sendable {
final class ScreenRecordService {
private struct UncheckedSendableBox<T>: @unchecked Sendable { private struct UncheckedSendableBox<T>: @unchecked Sendable {
let value: T let value: T
} }
private final class CaptureState: @unchecked Sendable {
private let lock = NSLock()
var writer: AVAssetWriter?
var videoInput: AVAssetWriterInput?
var audioInput: AVAssetWriterInput?
var started = false
var sawVideo = false
var lastVideoTime: CMTime?
var handlerError: Error?
func withLock<T>(_ body: (CaptureState) -> T) -> T {
self.lock.lock()
defer { lock.unlock() }
return body(self)
}
}
enum ScreenRecordError: LocalizedError { enum ScreenRecordError: LocalizedError {
case invalidScreenIndex(Int) case invalidScreenIndex(Int)
case captureFailed(String) case captureFailed(String)
@@ -51,126 +67,158 @@ final class ScreenRecordService {
}() }()
try? FileManager.default.removeItem(at: outURL) try? FileManager.default.removeItem(at: outURL)
let recorder = RPScreenRecorder.shared() let state = CaptureState()
recorder.isMicrophoneEnabled = includeAudio let recordQueue = DispatchQueue(label: "com.steipete.clawdis.screenrecord")
var writer: AVAssetWriter?
var videoInput: AVAssetWriterInput?
var audioInput: AVAssetWriterInput?
var started = false
var sawVideo = false
var lastVideoTime: CMTime?
var handlerError: Error?
let lock = NSLock()
func setHandlerError(_ error: Error) {
lock.lock()
defer { lock.unlock() }
if handlerError == nil { handlerError = error }
}
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
recorder.startCapture(handler: { sample, type, error in let handler: @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void = { sample, type, error in
if let error { // ReplayKit can call the capture handler on a background queue.
setHandlerError(error) // Serialize writes to avoid queue asserts.
return recordQueue.async {
} if let error {
guard CMSampleBufferDataIsReady(sample) else { return } state.withLock { state in
if state.handlerError == nil { state.handlerError = error }
switch type {
case .video:
let pts = CMSampleBufferGetPresentationTimeStamp(sample)
if let lastVideoTime {
let delta = CMTimeSubtract(pts, lastVideoTime)
if delta.seconds < (1.0 / fpsValue) { return }
}
if writer == nil {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
setHandlerError(ScreenRecordError.captureFailed("Missing image buffer"))
return
} }
let width = CVPixelBufferGetWidth(imageBuffer) return
let height = CVPixelBufferGetHeight(imageBuffer) }
do { guard CMSampleBufferDataIsReady(sample) else { return }
let w = try AVAssetWriter(outputURL: outURL, fileType: .mp4)
let settings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: width,
AVVideoHeightKey: height,
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
vInput.expectsMediaDataInRealTime = true
guard w.canAdd(vInput) else {
throw ScreenRecordError.writeFailed("Cannot add video input")
}
w.add(vInput)
if includeAudio { switch type {
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil) case .video:
aInput.expectsMediaDataInRealTime = true let pts = CMSampleBufferGetPresentationTimeStamp(sample)
if w.canAdd(aInput) { let shouldSkip = state.withLock { state in
w.add(aInput) if let lastVideoTime = state.lastVideoTime {
audioInput = aInput let delta = CMTimeSubtract(pts, lastVideoTime)
return delta.seconds < (1.0 / fpsValue)
}
return false
}
if shouldSkip { return }
if state.withLock({ $0.writer == nil }) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
state.withLock { state in
if state.handlerError == nil {
state.handlerError = ScreenRecordError.captureFailed("Missing image buffer")
}
}
return
}
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
do {
let w = try AVAssetWriter(outputURL: outURL, fileType: .mp4)
let settings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: width,
AVVideoHeightKey: height,
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
vInput.expectsMediaDataInRealTime = true
guard w.canAdd(vInput) else {
throw ScreenRecordError.writeFailed("Cannot add video input")
}
w.add(vInput)
if includeAudio {
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
aInput.expectsMediaDataInRealTime = true
if w.canAdd(aInput) {
w.add(aInput)
state.withLock { state in
state.audioInput = aInput
}
}
}
guard w.startWriting() else {
throw ScreenRecordError
.writeFailed(w.error?.localizedDescription ?? "Failed to start writer")
}
w.startSession(atSourceTime: pts)
state.withLock { state in
state.writer = w
state.videoInput = vInput
state.started = true
}
} catch {
state.withLock { state in
if state.handlerError == nil { state.handlerError = error }
}
return
}
}
let vInput = state.withLock { $0.videoInput }
let isStarted = state.withLock { $0.started }
guard let vInput, isStarted else { return }
if vInput.isReadyForMoreMediaData {
if vInput.append(sample) {
state.withLock { state in
state.sawVideo = true
state.lastVideoTime = pts
}
} else {
let err = state.withLock { $0.writer?.error }
if let err {
state.withLock { state in
if state.handlerError == nil {
state.handlerError = ScreenRecordError.writeFailed(err.localizedDescription)
}
}
} }
} }
guard w.startWriting() else {
throw ScreenRecordError
.writeFailed(w.error?.localizedDescription ?? "Failed to start writer")
}
w.startSession(atSourceTime: pts)
writer = w
videoInput = vInput
started = true
} catch {
setHandlerError(error)
return
} }
}
guard let vInput = videoInput, started else { return } case .audioApp, .audioMic:
if vInput.isReadyForMoreMediaData { let aInput = state.withLock { $0.audioInput }
if vInput.append(sample) { let isStarted = state.withLock { $0.started }
sawVideo = true guard includeAudio, let aInput, isStarted else { return }
lastVideoTime = pts if aInput.isReadyForMoreMediaData {
} else { _ = aInput.append(sample)
if let err = writer?.error {
setHandlerError(ScreenRecordError.writeFailed(err.localizedDescription))
}
} }
}
case .audioApp, .audioMic: @unknown default:
guard includeAudio, let aInput = audioInput, started else { return } break
if aInput.isReadyForMoreMediaData {
_ = aInput.append(sample)
} }
@unknown default:
break
} }
}, completionHandler: { error in }
let completion: @Sendable (Error?) -> Void = { error in
if let error { cont.resume(throwing: error) } else { cont.resume() } if let error { cont.resume(throwing: error) } else { cont.resume() }
}) }
Task { @MainActor in
startReplayKitCapture(
includeAudio: includeAudio,
handler: handler,
completion: completion)
}
} }
try await Task.sleep(nanoseconds: UInt64(durationMs) * 1_000_000) try await Task.sleep(nanoseconds: UInt64(durationMs) * 1_000_000)
let stopError = await withCheckedContinuation { cont in let stopError = await withCheckedContinuation { cont in
recorder.stopCapture { error in cont.resume(returning: error) } Task { @MainActor in
stopReplayKitCapture { error in cont.resume(returning: error) }
}
} }
if let stopError { throw stopError } if let stopError { throw stopError }
if let handlerError { throw handlerError } let handlerErrorSnapshot = state.withLock { $0.handlerError }
guard let writer, let videoInput, sawVideo else { if let handlerErrorSnapshot { throw handlerErrorSnapshot }
let writerSnapshot = state.withLock { $0.writer }
let videoInputSnapshot = state.withLock { $0.videoInput }
let audioInputSnapshot = state.withLock { $0.audioInput }
let sawVideoSnapshot = state.withLock { $0.sawVideo }
guard let writerSnapshot, let videoInputSnapshot, sawVideoSnapshot else {
throw ScreenRecordError.captureFailed("No frames captured") throw ScreenRecordError.captureFailed("No frames captured")
} }
videoInput.markAsFinished() videoInputSnapshot.markAsFinished()
audioInput?.markAsFinished() audioInputSnapshot?.markAsFinished()
let writerBox = UncheckedSendableBox(value: writer) let writerBox = UncheckedSendableBox(value: writerSnapshot)
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
writerBox.value.finishWriting { writerBox.value.finishWriting {
let writer = writerBox.value let writer = writerBox.value
@@ -198,3 +246,31 @@ final class ScreenRecordService {
return min(30, max(1, v)) return min(30, max(1, v))
} }
} }
@MainActor
private func startReplayKitCapture(
includeAudio: Bool,
handler: @escaping @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void,
completion: @escaping @Sendable (Error?) -> Void)
{
let recorder = RPScreenRecorder.shared()
recorder.isMicrophoneEnabled = includeAudio
recorder.startCapture(handler: handler, completionHandler: completion)
}
@MainActor
private func stopReplayKitCapture(_ completion: @escaping @Sendable (Error?) -> Void) {
RPScreenRecorder.shared().stopCapture { error in completion(error) }
}
#if DEBUG
extension ScreenRecordService {
nonisolated static func _test_clampDurationMs(_ ms: Int?) -> Int {
self.clampDurationMs(ms)
}
nonisolated static func _test_clampFps(_ fps: Double?) -> Double {
self.clampFps(fps)
}
}
#endif
@@ -20,6 +20,8 @@ struct SettingsTab: View {
@AppStorage("node.displayName") private var displayName: String = "iOS Node" @AppStorage("node.displayName") private var displayName: String = "iOS Node"
@AppStorage("node.instanceId") private var instanceId: String = UUID().uuidString @AppStorage("node.instanceId") private var instanceId: String = UUID().uuidString
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false @AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
@AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true
@AppStorage("camera.enabled") private var cameraEnabled: Bool = true @AppStorage("camera.enabled") private var cameraEnabled: Bool = true
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true @AppStorage("screen.preventSleep") private var preventSleep: Bool = true
@AppStorage("bridge.preferredStableID") private var preferredBridgeStableID: String = "" @AppStorage("bridge.preferredStableID") private var preferredBridgeStableID: String = ""
@@ -28,6 +30,7 @@ struct SettingsTab: View {
@AppStorage("bridge.manual.host") private var manualBridgeHost: String = "" @AppStorage("bridge.manual.host") private var manualBridgeHost: String = ""
@AppStorage("bridge.manual.port") private var manualBridgePort: Int = 18790 @AppStorage("bridge.manual.port") private var manualBridgePort: Int = 18790
@AppStorage("bridge.discovery.debugLogs") private var discoveryDebugLogsEnabled: Bool = false @AppStorage("bridge.discovery.debugLogs") private var discoveryDebugLogsEnabled: Bool = false
@AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false
@State private var connectStatus = ConnectStatusStore() @State private var connectStatus = ConnectStatusStore()
@State private var connectingBridgeID: String? @State private var connectingBridgeID: String?
@State private var localIPAddress: String? @State private var localIPAddress: String?
@@ -50,6 +53,9 @@ struct SettingsTab: View {
} }
} }
} }
LabeledContent("Platform", value: self.platformString())
LabeledContent("Version", value: self.appVersion())
LabeledContent("Model", value: self.modelIdentifier())
} }
Section("Bridge") { Section("Bridge") {
@@ -142,6 +148,8 @@ struct SettingsTab: View {
NavigationLink("Discovery Logs") { NavigationLink("Discovery Logs") {
BridgeDiscoveryDebugLogView() BridgeDiscoveryDebugLogView()
} }
Toggle("Debug Canvas Status", isOn: self.$canvasDebugStatusEnabled)
} }
} }
@@ -150,6 +158,12 @@ struct SettingsTab: View {
.onChange(of: self.voiceWakeEnabled) { _, newValue in .onChange(of: self.voiceWakeEnabled) { _, newValue in
self.appModel.setVoiceWakeEnabled(newValue) self.appModel.setVoiceWakeEnabled(newValue)
} }
Toggle("Talk Mode", isOn: self.$talkEnabled)
.onChange(of: self.talkEnabled) { _, newValue in
self.appModel.setTalkEnabled(newValue)
}
// Keep this separate so users can hide the side bubble without disabling Talk Mode.
Toggle("Show Talk Button", isOn: self.$talkButtonEnabled)
NavigationLink { NavigationLink {
VoiceWakeWordsSettingsView() VoiceWakeWordsSettingsView()
@@ -224,6 +238,12 @@ struct SettingsTab: View {
HStack { HStack {
VStack(alignment: .leading, spacing: 2) { VStack(alignment: .leading, spacing: 2) {
Text(bridge.name) Text(bridge.name)
let detailLines = self.bridgeDetailLines(bridge)
ForEach(detailLines, id: \.self) { line in
Text(line)
.font(.footnote)
.foregroundStyle(.secondary)
}
} }
Spacer() Spacer()
@@ -501,4 +521,26 @@ struct SettingsTab: View {
private static func httpURLString(host: String?, port: Int?, fallback: String) -> String { private static func httpURLString(host: String?, port: Int?, fallback: String) -> String {
SettingsNetworkingHelpers.httpURLString(host: host, port: port, fallback: fallback) SettingsNetworkingHelpers.httpURLString(host: host, port: port, fallback: fallback)
} }
private func bridgeDetailLines(_ bridge: BridgeDiscoveryModel.DiscoveredBridge) -> [String] {
var lines: [String] = []
if let lanHost = bridge.lanHost { lines.append("LAN: \(lanHost)") }
if let tailnet = bridge.tailnetDns { lines.append("Tailnet: \(tailnet)") }
let gatewayPort = bridge.gatewayPort
let bridgePort = bridge.bridgePort
let canvasPort = bridge.canvasPort
if gatewayPort != nil || bridgePort != nil || canvasPort != nil {
let gw = gatewayPort.map(String.init) ?? "—"
let br = bridgePort.map(String.init) ?? "—"
let canvas = canvasPort.map(String.init) ?? "—"
lines.append("Ports: gw \(gw) · bridge \(br) · canvas \(canvas)")
}
if lines.isEmpty {
lines.append(bridge.debugID)
}
return lines
}
} }
+44 -9
View File
@@ -1,6 +1,8 @@
import SwiftUI import SwiftUI
struct StatusPill: View { struct StatusPill: View {
@Environment(\.scenePhase) private var scenePhase
enum BridgeState: Equatable { enum BridgeState: Equatable {
case connected case connected
case connecting case connecting
@@ -26,8 +28,15 @@ struct StatusPill: View {
} }
} }
struct Activity: Equatable {
var title: String
var systemImage: String
var tint: Color?
}
var bridge: BridgeState var bridge: BridgeState
var voiceWakeEnabled: Bool var voiceWakeEnabled: Bool
var activity: Activity?
var brighten: Bool = false var brighten: Bool = false
var onTap: () -> Void var onTap: () -> Void
@@ -52,10 +61,24 @@ struct StatusPill: View {
.frame(height: 14) .frame(height: 14)
.opacity(0.35) .opacity(0.35)
Image(systemName: self.voiceWakeEnabled ? "mic.fill" : "mic.slash") if let activity {
.font(.system(size: 13, weight: .semibold)) HStack(spacing: 6) {
.foregroundStyle(self.voiceWakeEnabled ? .primary : .secondary) Image(systemName: activity.systemImage)
.accessibilityLabel(self.voiceWakeEnabled ? "Voice Wake enabled" : "Voice Wake disabled") .font(.system(size: 13, weight: .semibold))
.foregroundStyle(activity.tint ?? .primary)
Text(activity.title)
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(.primary)
.lineLimit(1)
}
.transition(.opacity.combined(with: .move(edge: .top)))
} else {
Image(systemName: self.voiceWakeEnabled ? "mic.fill" : "mic.slash")
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(self.voiceWakeEnabled ? .primary : .secondary)
.accessibilityLabel(self.voiceWakeEnabled ? "Voice Wake enabled" : "Voice Wake disabled")
.transition(.opacity.combined(with: .move(edge: .top)))
}
} }
.padding(.vertical, 8) .padding(.vertical, 8)
.padding(.horizontal, 12) .padding(.horizontal, 12)
@@ -71,15 +94,27 @@ struct StatusPill: View {
} }
.buttonStyle(.plain) .buttonStyle(.plain)
.accessibilityLabel("Status") .accessibilityLabel("Status")
.accessibilityValue("\(self.bridge.title), Voice Wake \(self.voiceWakeEnabled ? "enabled" : "disabled")") .accessibilityValue(self.accessibilityValue)
.onAppear { self.updatePulse(for: self.bridge) } .onAppear { self.updatePulse(for: self.bridge, scenePhase: self.scenePhase) }
.onDisappear { self.pulse = false }
.onChange(of: self.bridge) { _, newValue in .onChange(of: self.bridge) { _, newValue in
self.updatePulse(for: newValue) self.updatePulse(for: newValue, scenePhase: self.scenePhase)
} }
.onChange(of: self.scenePhase) { _, newValue in
self.updatePulse(for: self.bridge, scenePhase: newValue)
}
.animation(.easeInOut(duration: 0.18), value: self.activity?.title)
} }
private func updatePulse(for bridge: BridgeState) { private var accessibilityValue: String {
guard bridge == .connecting else { if let activity {
return "\(self.bridge.title), \(activity.title)"
}
return "\(self.bridge.title), Voice Wake \(self.voiceWakeEnabled ? "enabled" : "disabled")"
}
private func updatePulse(for bridge: BridgeState, scenePhase: ScenePhase) {
guard bridge == .connecting, scenePhase == .active else {
withAnimation(.easeOut(duration: 0.2)) { self.pulse = false } withAnimation(.easeOut(duration: 0.2)) { self.pulse = false }
return return
} }
@@ -0,0 +1,713 @@
import AVFAudio
import ClawdisKit
import Foundation
import Observation
import OSLog
import Speech
@MainActor
@Observable
final class TalkModeManager: NSObject {
private typealias SpeechRequest = SFSpeechAudioBufferRecognitionRequest
private static let defaultModelIdFallback = "eleven_v3"
var isEnabled: Bool = false
var isListening: Bool = false
var isSpeaking: Bool = false
var statusText: String = "Off"
private let audioEngine = AVAudioEngine()
private var speechRecognizer: SFSpeechRecognizer?
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
private var recognitionTask: SFSpeechRecognitionTask?
private var silenceTask: Task<Void, Never>?
private var lastHeard: Date?
private var lastTranscript: String = ""
private var lastSpokenText: String?
private var lastInterruptedAtSeconds: Double?
private var defaultVoiceId: String?
private var currentVoiceId: String?
private var defaultModelId: String?
private var currentModelId: String?
private var voiceOverrideActive = false
private var modelOverrideActive = false
private var defaultOutputFormat: String?
private var apiKey: String?
private var voiceAliases: [String: String] = [:]
private var interruptOnSpeech: Bool = true
private var mainSessionKey: String = "main"
private var fallbackVoiceId: String?
private var lastPlaybackWasPCM: Bool = false
var pcmPlayer: PCMStreamingAudioPlaying = PCMStreamingAudioPlayer.shared
var mp3Player: StreamingAudioPlaying = StreamingAudioPlayer.shared
private var bridge: BridgeSession?
private let silenceWindow: TimeInterval = 0.7
private var chatSubscribedSessionKeys = Set<String>()
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "TalkMode")
func attachBridge(_ bridge: BridgeSession) {
self.bridge = bridge
}
func setEnabled(_ enabled: Bool) {
self.isEnabled = enabled
if enabled {
self.logger.info("enabled")
Task { await self.start() }
} else {
self.logger.info("disabled")
self.stop()
}
}
func start() async {
guard self.isEnabled else { return }
if self.isListening { return }
self.logger.info("start")
self.statusText = "Requesting permissions…"
let micOk = await Self.requestMicrophonePermission()
guard micOk else {
self.logger.warning("start blocked: microphone permission denied")
self.statusText = "Microphone permission denied"
return
}
let speechOk = await Self.requestSpeechPermission()
guard speechOk else {
self.logger.warning("start blocked: speech permission denied")
self.statusText = "Speech recognition permission denied"
return
}
await self.reloadConfig()
do {
try Self.configureAudioSession()
try self.startRecognition()
self.isListening = true
self.statusText = "Listening"
self.startSilenceMonitor()
await self.subscribeChatIfNeeded(sessionKey: self.mainSessionKey)
self.logger.info("listening")
} catch {
self.isListening = false
self.statusText = "Start failed: \(error.localizedDescription)"
self.logger.error("start failed: \(error.localizedDescription, privacy: .public)")
}
}
func stop() {
self.isEnabled = false
self.isListening = false
self.statusText = "Off"
self.lastTranscript = ""
self.lastHeard = nil
self.silenceTask?.cancel()
self.silenceTask = nil
self.stopRecognition()
self.stopSpeaking()
self.lastInterruptedAtSeconds = nil
TalkSystemSpeechSynthesizer.shared.stop()
do {
try AVAudioSession.sharedInstance().setActive(false, options: [.notifyOthersOnDeactivation])
} catch {
self.logger.warning("audio session deactivate failed: \(error.localizedDescription, privacy: .public)")
}
Task { await self.unsubscribeAllChats() }
}
func userTappedOrb() {
self.stopSpeaking()
}
private func startRecognition() throws {
self.stopRecognition()
self.speechRecognizer = SFSpeechRecognizer()
guard let recognizer = self.speechRecognizer else {
throw NSError(domain: "TalkMode", code: 1, userInfo: [
NSLocalizedDescriptionKey: "Speech recognizer unavailable",
])
}
self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest()
self.recognitionRequest?.shouldReportPartialResults = true
guard let request = self.recognitionRequest else { return }
let input = self.audioEngine.inputNode
let format = input.outputFormat(forBus: 0)
input.removeTap(onBus: 0)
let tapBlock = Self.makeAudioTapAppendCallback(request: request)
input.installTap(onBus: 0, bufferSize: 2048, format: format, block: tapBlock)
self.audioEngine.prepare()
try self.audioEngine.start()
self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, error in
guard let self else { return }
if let error {
if !self.isSpeaking {
self.statusText = "Speech error: \(error.localizedDescription)"
}
self.logger.debug("speech recognition error: \(error.localizedDescription, privacy: .public)")
}
guard let result else { return }
let transcript = result.bestTranscription.formattedString
Task { @MainActor in
await self.handleTranscript(transcript: transcript, isFinal: result.isFinal)
}
}
}
private func stopRecognition() {
self.recognitionTask?.cancel()
self.recognitionTask = nil
self.recognitionRequest?.endAudio()
self.recognitionRequest = nil
self.audioEngine.inputNode.removeTap(onBus: 0)
self.audioEngine.stop()
self.speechRecognizer = nil
}
private nonisolated static func makeAudioTapAppendCallback(request: SpeechRequest) -> AVAudioNodeTapBlock {
{ buffer, _ in
request.append(buffer)
}
}
private func handleTranscript(transcript: String, isFinal: Bool) async {
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
if self.isSpeaking, self.interruptOnSpeech {
if self.shouldInterrupt(with: trimmed) {
self.stopSpeaking()
}
return
}
guard self.isListening else { return }
if !trimmed.isEmpty {
self.lastTranscript = trimmed
self.lastHeard = Date()
}
if isFinal {
self.lastTranscript = trimmed
}
}
private func startSilenceMonitor() {
self.silenceTask?.cancel()
self.silenceTask = Task { [weak self] in
guard let self else { return }
while self.isEnabled {
try? await Task.sleep(nanoseconds: 200_000_000)
await self.checkSilence()
}
}
}
private func checkSilence() async {
guard self.isListening, !self.isSpeaking else { return }
let transcript = self.lastTranscript.trimmingCharacters(in: .whitespacesAndNewlines)
guard !transcript.isEmpty else { return }
guard let lastHeard else { return }
if Date().timeIntervalSince(lastHeard) < self.silenceWindow { return }
await self.finalizeTranscript(transcript)
}
private func finalizeTranscript(_ transcript: String) async {
self.isListening = false
self.statusText = "Thinking…"
self.lastTranscript = ""
self.lastHeard = nil
self.stopRecognition()
await self.reloadConfig()
let prompt = self.buildPrompt(transcript: transcript)
guard let bridge else {
self.statusText = "Bridge not connected"
self.logger.warning("finalize: bridge not connected")
await self.start()
return
}
do {
let startedAt = Date().timeIntervalSince1970
let sessionKey = self.mainSessionKey
await self.subscribeChatIfNeeded(sessionKey: sessionKey)
self.logger.info(
"chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)")
let runId = try await self.sendChat(prompt, bridge: bridge)
self.logger.info("chat.send ok runId=\(runId, privacy: .public)")
let completion = await self.waitForChatCompletion(runId: runId, bridge: bridge, timeoutSeconds: 120)
if completion == .timeout {
self.logger.warning(
"chat completion timeout runId=\(runId, privacy: .public); attempting history fallback")
} else if completion == .aborted {
self.statusText = "Aborted"
self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)")
await self.start()
return
} else if completion == .error {
self.statusText = "Chat error"
self.logger.warning("chat completion error runId=\(runId, privacy: .public)")
await self.start()
return
}
guard let assistantText = try await self.waitForAssistantText(
bridge: bridge,
since: startedAt,
timeoutSeconds: completion == .final ? 12 : 25)
else {
self.statusText = "No reply"
self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)")
await self.start()
return
}
self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)")
await self.playAssistant(text: assistantText)
} catch {
self.statusText = "Talk failed: \(error.localizedDescription)"
self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)")
}
await self.start()
}
private func subscribeChatIfNeeded(sessionKey: String) async {
let key = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
guard !key.isEmpty else { return }
guard let bridge else { return }
guard !self.chatSubscribedSessionKeys.contains(key) else { return }
do {
let payload = "{\"sessionKey\":\"\(key)\"}"
try await bridge.sendEvent(event: "chat.subscribe", payloadJSON: payload)
self.chatSubscribedSessionKeys.insert(key)
self.logger.info("chat.subscribe ok sessionKey=\(key, privacy: .public)")
} catch {
self.logger
.warning(
"chat.subscribe failed sessionKey=\(key, privacy: .public) err=\(error.localizedDescription, privacy: .public)")
}
}
private func unsubscribeAllChats() async {
guard let bridge else { return }
let keys = self.chatSubscribedSessionKeys
self.chatSubscribedSessionKeys.removeAll()
for key in keys {
do {
let payload = "{\"sessionKey\":\"\(key)\"}"
try await bridge.sendEvent(event: "chat.unsubscribe", payloadJSON: payload)
} catch {
// ignore
}
}
}
private func buildPrompt(transcript: String) -> String {
let interrupted = self.lastInterruptedAtSeconds
self.lastInterruptedAtSeconds = nil
return TalkPromptBuilder.build(transcript: transcript, interruptedAtSeconds: interrupted)
}
private enum ChatCompletionState: CustomStringConvertible {
case final
case aborted
case error
case timeout
var description: String {
switch self {
case .final: "final"
case .aborted: "aborted"
case .error: "error"
case .timeout: "timeout"
}
}
}
private func sendChat(_ message: String, bridge: BridgeSession) async throws -> String {
struct SendResponse: Decodable { let runId: String }
let payload: [String: Any] = [
"sessionKey": self.mainSessionKey,
"message": message,
"thinking": "low",
"timeoutMs": 30000,
"idempotencyKey": UUID().uuidString,
]
let data = try JSONSerialization.data(withJSONObject: payload)
let json = String(decoding: data, as: UTF8.self)
let res = try await bridge.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 30)
let decoded = try JSONDecoder().decode(SendResponse.self, from: res)
return decoded.runId
}
private func waitForChatCompletion(
runId: String,
bridge: BridgeSession,
timeoutSeconds: Int = 120) async -> ChatCompletionState
{
let stream = await bridge.subscribeServerEvents(bufferingNewest: 200)
return await withTaskGroup(of: ChatCompletionState.self) { group in
group.addTask { [runId] in
for await evt in stream {
if Task.isCancelled { return .timeout }
guard evt.event == "chat", let payload = evt.payloadJSON else { continue }
guard let data = payload.data(using: .utf8) else { continue }
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { continue }
if (json["runId"] as? String) != runId { continue }
if let state = json["state"] as? String {
switch state {
case "final": return .final
case "aborted": return .aborted
case "error": return .error
default: break
}
}
}
return .timeout
}
group.addTask {
try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds) * 1_000_000_000)
return .timeout
}
let result = await group.next() ?? .timeout
group.cancelAll()
return result
}
}
private func waitForAssistantText(
bridge: BridgeSession,
since: Double,
timeoutSeconds: Int) async throws -> String?
{
let deadline = Date().addingTimeInterval(TimeInterval(timeoutSeconds))
while Date() < deadline {
if let text = try await self.fetchLatestAssistantText(bridge: bridge, since: since) {
return text
}
try? await Task.sleep(nanoseconds: 300_000_000)
}
return nil
}
private func fetchLatestAssistantText(bridge: BridgeSession, since: Double? = nil) async throws -> String? {
let res = try await bridge.request(
method: "chat.history",
paramsJSON: "{\"sessionKey\":\"\(self.mainSessionKey)\"}",
timeoutSeconds: 15)
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return nil }
guard let messages = json["messages"] as? [[String: Any]] else { return nil }
for msg in messages.reversed() {
guard (msg["role"] as? String) == "assistant" else { continue }
if let since, let timestamp = msg["timestamp"] as? Double,
TalkHistoryTimestamp.isAfter(timestamp, sinceSeconds: since) == false
{
continue
}
guard let content = msg["content"] as? [[String: Any]] else { continue }
let text = content.compactMap { $0["text"] as? String }.joined(separator: "\n")
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty { return trimmed }
}
return nil
}
private func playAssistant(text: String) async {
let parsed = TalkDirectiveParser.parse(text)
let directive = parsed.directive
let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines)
guard !cleaned.isEmpty else { return }
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
}
if let voice = resolvedVoice {
if directive?.once != true {
self.currentVoiceId = voice
self.voiceOverrideActive = true
}
}
if let model = directive?.modelId {
if directive?.once != true {
self.currentModelId = model
self.modelOverrideActive = true
}
}
self.statusText = "Generating voice…"
self.isSpeaking = true
self.lastSpokenText = cleaned
do {
let started = Date()
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
let resolvedKey =
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"]
let apiKey = resolvedKey?.trimmingCharacters(in: .whitespacesAndNewlines)
let preferredVoice = resolvedVoice ?? self.currentVoiceId ?? self.defaultVoiceId
let voiceId: String? = if let apiKey, !apiKey.isEmpty {
await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey)
} else {
nil
}
let canUseElevenLabs = (voiceId?.isEmpty == false) && (apiKey?.isEmpty == false)
if canUseElevenLabs, let voiceId, let apiKey {
let desiredOutputFormat = (directive?.outputFormat ?? self.defaultOutputFormat)?
.trimmingCharacters(in: .whitespacesAndNewlines)
let requestedOutputFormat = (desiredOutputFormat?.isEmpty == false) ? desiredOutputFormat : nil
let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(requestedOutputFormat ?? "pcm_44100")
if outputFormat == nil, let requestedOutputFormat {
self.logger.warning(
"talk output_format unsupported for local playback: \(requestedOutputFormat, privacy: .public)")
}
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest {
ElevenLabsTTSRequest(
text: cleaned,
modelId: modelId,
outputFormat: outputFormat,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
style: TalkTTSValidation.validatedUnit(directive?.style),
speakerBoost: directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
}
let request = makeRequest(outputFormat: outputFormat)
let client = ElevenLabsTTSClient(apiKey: apiKey)
let stream = client.streamSynthesize(voiceId: voiceId, request: request)
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.statusText = "Speaking…"
let sampleRate = TalkTTSValidation.pcmSampleRate(from: outputFormat)
let result: StreamingPlaybackResult
if let sampleRate {
self.lastPlaybackWasPCM = true
var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
if !playback.finished, playback.interruptedAt == nil {
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
self.logger.warning("pcm playback failed; retrying mp3")
self.lastPlaybackWasPCM = false
let mp3Stream = client.streamSynthesize(
voiceId: voiceId,
request: makeRequest(outputFormat: mp3Format))
playback = await self.mp3Player.play(stream: mp3Stream)
}
result = playback
} else {
self.lastPlaybackWasPCM = false
result = await self.mp3Player.play(stream: stream)
}
self.logger
.info(
"elevenlabs stream finished=\(result.finished, privacy: .public) dur=\(Date().timeIntervalSince(started), privacy: .public)s")
if !result.finished, let interruptedAt = result.interruptedAt {
self.lastInterruptedAtSeconds = interruptedAt
}
} else {
self.logger.warning("tts unavailable; falling back to system voice (missing key or voiceId)")
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.statusText = "Speaking (System)…"
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
}
} catch {
self.logger.error(
"tts failed: \(error.localizedDescription, privacy: .public); falling back to system voice")
do {
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.statusText = "Speaking (System)…"
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
} catch {
self.statusText = "Speak failed: \(error.localizedDescription)"
self.logger.error("system voice failed: \(error.localizedDescription, privacy: .public)")
}
}
self.stopRecognition()
self.isSpeaking = false
}
private func stopSpeaking(storeInterruption: Bool = true) {
guard self.isSpeaking else { return }
let interruptedAt = self.lastPlaybackWasPCM
? self.pcmPlayer.stop()
: self.mp3Player.stop()
if storeInterruption {
self.lastInterruptedAtSeconds = interruptedAt
}
_ = self.lastPlaybackWasPCM
? self.mp3Player.stop()
: self.pcmPlayer.stop()
TalkSystemSpeechSynthesizer.shared.stop()
self.isSpeaking = false
}
private func shouldInterrupt(with transcript: String) -> Bool {
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
guard trimmed.count >= 3 else { return false }
if let spoken = self.lastSpokenText?.lowercased(), spoken.contains(trimmed.lowercased()) {
return false
}
return true
}
private func resolveVoiceAlias(_ value: String?) -> String? {
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let normalized = trimmed.lowercased()
if let mapped = self.voiceAliases[normalized] { return mapped }
if self.voiceAliases.values.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) {
return trimmed
}
return Self.isLikelyVoiceId(trimmed) ? trimmed : nil
}
private func resolveVoiceId(preferred: String?, apiKey: String) async -> String? {
let trimmed = preferred?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if !trimmed.isEmpty {
if let resolved = self.resolveVoiceAlias(trimmed) { return resolved }
self.logger.warning("unknown voice alias \(trimmed, privacy: .public)")
}
if let fallbackVoiceId { return fallbackVoiceId }
do {
let voices = try await ElevenLabsTTSClient(apiKey: apiKey).listVoices()
guard let first = voices.first else {
self.logger.warning("elevenlabs voices list empty")
return nil
}
self.fallbackVoiceId = first.voiceId
if self.defaultVoiceId == nil {
self.defaultVoiceId = first.voiceId
}
if !self.voiceOverrideActive {
self.currentVoiceId = first.voiceId
}
let name = first.name ?? "unknown"
self.logger
.info("default voice selected \(name, privacy: .public) (\(first.voiceId, privacy: .public))")
return first.voiceId
} catch {
self.logger.error("elevenlabs list voices failed: \(error.localizedDescription, privacy: .public)")
return nil
}
}
private static func isLikelyVoiceId(_ value: String) -> Bool {
guard value.count >= 10 else { return false }
return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "_" }
}
private func reloadConfig() async {
guard let bridge else { return }
do {
let res = try await bridge.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8)
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return }
guard let config = json["config"] as? [String: Any] else { return }
let talk = config["talk"] as? [String: Any]
let session = config["session"] as? [String: Any]
let rawMainKey = (session?["mainKey"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
self.mainSessionKey = rawMainKey.isEmpty ? "main" : rawMainKey
self.defaultVoiceId = (talk?["voiceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
if let aliases = talk?["voiceAliases"] as? [String: Any] {
var resolved: [String: String] = [:]
for (key, value) in aliases {
guard let id = value as? String else { continue }
let normalizedKey = key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
let trimmedId = id.trimmingCharacters(in: .whitespacesAndNewlines)
guard !normalizedKey.isEmpty, !trimmedId.isEmpty else { continue }
resolved[normalizedKey] = trimmedId
}
self.voiceAliases = resolved
} else {
self.voiceAliases = [:]
}
if !self.voiceOverrideActive {
self.currentVoiceId = self.defaultVoiceId
}
let model = (talk?["modelId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
self.defaultModelId = (model?.isEmpty == false) ? model : Self.defaultModelIdFallback
if !self.modelOverrideActive {
self.currentModelId = self.defaultModelId
}
self.defaultOutputFormat = (talk?["outputFormat"] as? String)?
.trimmingCharacters(in: .whitespacesAndNewlines)
self.apiKey = (talk?["apiKey"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
if let interrupt = talk?["interruptOnSpeech"] as? Bool {
self.interruptOnSpeech = interrupt
}
} catch {
self.defaultModelId = Self.defaultModelIdFallback
if !self.modelOverrideActive {
self.currentModelId = self.defaultModelId
}
}
}
private static func configureAudioSession() throws {
let session = AVAudioSession.sharedInstance()
try session.setCategory(.playAndRecord, mode: .voiceChat, options: [
.duckOthers,
.mixWithOthers,
.allowBluetoothHFP,
.defaultToSpeaker,
])
try session.setActive(true, options: [])
}
private nonisolated static func requestMicrophonePermission() async -> Bool {
await withCheckedContinuation(isolation: nil) { cont in
AVAudioApplication.requestRecordPermission { ok in
cont.resume(returning: ok)
}
}
}
private nonisolated static func requestSpeechPermission() async -> Bool {
await withCheckedContinuation(isolation: nil) { cont in
SFSpeechRecognizer.requestAuthorization { status in
cont.resume(returning: status == .authorized)
}
}
}
}
@@ -0,0 +1,70 @@
import SwiftUI
struct TalkOrbOverlay: View {
@Environment(NodeAppModel.self) private var appModel
@State private var pulse: Bool = false
var body: some View {
let seam = self.appModel.seamColor
let status = self.appModel.talkMode.statusText.trimmingCharacters(in: .whitespacesAndNewlines)
VStack(spacing: 14) {
ZStack {
Circle()
.stroke(seam.opacity(0.26), lineWidth: 2)
.frame(width: 320, height: 320)
.scaleEffect(self.pulse ? 1.15 : 0.96)
.opacity(self.pulse ? 0.0 : 1.0)
.animation(.easeOut(duration: 1.3).repeatForever(autoreverses: false), value: self.pulse)
Circle()
.stroke(seam.opacity(0.18), lineWidth: 2)
.frame(width: 320, height: 320)
.scaleEffect(self.pulse ? 1.45 : 1.02)
.opacity(self.pulse ? 0.0 : 0.9)
.animation(.easeOut(duration: 1.9).repeatForever(autoreverses: false).delay(0.2), value: self.pulse)
Circle()
.fill(
RadialGradient(
colors: [
seam.opacity(0.95),
seam.opacity(0.40),
Color.black.opacity(0.55),
],
center: .center,
startRadius: 1,
endRadius: 112))
.frame(width: 190, height: 190)
.overlay(
Circle()
.stroke(seam.opacity(0.35), lineWidth: 1))
.shadow(color: seam.opacity(0.32), radius: 26, x: 0, y: 0)
.shadow(color: Color.black.opacity(0.50), radius: 22, x: 0, y: 10)
}
.contentShape(Circle())
.onTapGesture {
self.appModel.talkMode.userTappedOrb()
}
if !status.isEmpty, status != "Off" {
Text(status)
.font(.system(.footnote, design: .rounded).weight(.semibold))
.foregroundStyle(Color.white.opacity(0.92))
.padding(.horizontal, 12)
.padding(.vertical, 8)
.background(
Capsule()
.fill(Color.black.opacity(0.40))
.overlay(
Capsule().stroke(seam.opacity(0.22), lineWidth: 1)))
}
}
.padding(28)
.onAppear {
self.pulse = true
}
.accessibilityElement(children: .combine)
.accessibilityLabel("Talk Mode \(status)")
}
}
+5
View File
@@ -4,6 +4,7 @@ struct VoiceTab: View {
@Environment(NodeAppModel.self) private var appModel @Environment(NodeAppModel.self) private var appModel
@Environment(VoiceWakeManager.self) private var voiceWake @Environment(VoiceWakeManager.self) private var voiceWake
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false @AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
var body: some View { var body: some View {
NavigationStack { NavigationStack {
@@ -14,6 +15,7 @@ struct VoiceTab: View {
Text(self.voiceWake.statusText) Text(self.voiceWake.statusText)
.font(.footnote) .font(.footnote)
.foregroundStyle(.secondary) .foregroundStyle(.secondary)
LabeledContent("Talk Mode", value: self.talkEnabled ? "Enabled" : "Disabled")
} }
Section("Notes") { Section("Notes") {
@@ -36,6 +38,9 @@ struct VoiceTab: View {
.onChange(of: self.voiceWakeEnabled) { _, newValue in .onChange(of: self.voiceWakeEnabled) { _, newValue in
self.appModel.setVoiceWakeEnabled(newValue) self.appModel.setVoiceWakeEnabled(newValue)
} }
.onChange(of: self.talkEnabled) { _, newValue in
self.appModel.setTalkEnabled(newValue)
}
} }
} }
} }
+25 -25
View File
@@ -2,6 +2,7 @@ import AVFAudio
import Foundation import Foundation
import Observation import Observation
import Speech import Speech
import SwabbleKit
private func makeAudioTapEnqueueCallback(queue: AudioBufferQueue) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void { private func makeAudioTapEnqueueCallback(queue: AudioBufferQueue) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
{ buffer, _ in { buffer, _ in
@@ -289,15 +290,18 @@ final class VoiceWakeManager: NSObject {
private nonisolated func makeRecognitionResultHandler() -> @Sendable (SFSpeechRecognitionResult?, Error?) -> Void { private nonisolated func makeRecognitionResultHandler() -> @Sendable (SFSpeechRecognitionResult?, Error?) -> Void {
{ [weak self] result, error in { [weak self] result, error in
let transcript = result?.bestTranscription.formattedString let transcript = result?.bestTranscription.formattedString
let segments = result.flatMap { result in
transcript.map { WakeWordSpeechSegments.from(transcription: result.bestTranscription, transcript: $0) }
} ?? []
let errorText = error?.localizedDescription let errorText = error?.localizedDescription
Task { @MainActor in Task { @MainActor in
self?.handleRecognitionCallback(transcript: transcript, errorText: errorText) self?.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
} }
} }
} }
private func handleRecognitionCallback(transcript: String?, errorText: String?) { private func handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
if let errorText { if let errorText {
self.statusText = "Recognizer error: \(errorText)" self.statusText = "Recognizer error: \(errorText)"
self.isListening = false self.isListening = false
@@ -313,7 +317,7 @@ final class VoiceWakeManager: NSObject {
} }
guard let transcript else { return } guard let transcript else { return }
guard let cmd = self.extractCommand(from: transcript) else { return } guard let cmd = self.extractCommand(from: transcript, segments: segments) else { return }
if cmd == self.lastDispatched { return } if cmd == self.lastDispatched { return }
self.lastDispatched = cmd self.lastDispatched = cmd
@@ -334,30 +338,18 @@ final class VoiceWakeManager: NSObject {
} }
} }
private func extractCommand(from transcript: String) -> String? { private func extractCommand(from transcript: String, segments: [WakeWordSegment]) -> String? {
Self.extractCommand(from: transcript, triggers: self.activeTriggerWords) Self.extractCommand(from: transcript, segments: segments, triggers: self.activeTriggerWords)
} }
nonisolated static func extractCommand(from transcript: String, triggers: [String]) -> String? { nonisolated static func extractCommand(
var bestRange: Range<String.Index>? from transcript: String,
for trigger in triggers { segments: [WakeWordSegment],
let token = trigger.trimmingCharacters(in: .whitespacesAndNewlines) triggers: [String],
guard !token.isEmpty else { continue } minPostTriggerGap: TimeInterval = 0.45) -> String?
guard let range = transcript.range(of: token, options: [.caseInsensitive, .backwards]) else { continue } {
if let currentBest = bestRange { let config = WakeWordGateConfig(triggers: triggers, minPostTriggerGap: minPostTriggerGap)
if range.lowerBound > currentBest.lowerBound { return WakeWordGate.match(transcript: transcript, segments: segments, config: config)?.command
bestRange = range
}
} else {
bestRange = range
}
}
guard let bestRange else { return nil }
let after = transcript[bestRange.upperBound...]
let trimmed = after.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
return String(trimmed)
} }
private static func configureAudioSession() throws { private static func configureAudioSession() throws {
@@ -387,3 +379,11 @@ final class VoiceWakeManager: NSObject {
} }
} }
} }
#if DEBUG
extension VoiceWakeManager {
func _test_handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
self.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
}
}
#endif
+60
View File
@@ -0,0 +1,60 @@
Sources/Bridge/BridgeClient.swift
Sources/Bridge/BridgeConnectionController.swift
Sources/Bridge/BridgeDiscoveryDebugLogView.swift
Sources/Bridge/BridgeDiscoveryModel.swift
Sources/Bridge/BridgeEndpointID.swift
Sources/Bridge/BridgeSession.swift
Sources/Bridge/BridgeSettingsStore.swift
Sources/Bridge/KeychainStore.swift
Sources/Camera/CameraController.swift
Sources/Chat/ChatSheet.swift
Sources/Chat/IOSBridgeChatTransport.swift
Sources/ClawdisApp.swift
Sources/Model/NodeAppModel.swift
Sources/RootCanvas.swift
Sources/RootTabs.swift
Sources/Screen/ScreenController.swift
Sources/Screen/ScreenRecordService.swift
Sources/Screen/ScreenTab.swift
Sources/Screen/ScreenWebView.swift
Sources/Settings/SettingsNetworkingHelpers.swift
Sources/Settings/SettingsTab.swift
Sources/Settings/VoiceWakeWordsSettingsView.swift
Sources/Status/StatusPill.swift
Sources/Status/VoiceWakeToast.swift
Sources/Voice/VoiceTab.swift
Sources/Voice/VoiceWakeManager.swift
Sources/Voice/VoiceWakePreferences.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatComposer.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatMarkdownSplitter.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatMessageViews.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatModels.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatPayloadDecoding.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatSessions.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatSheets.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatTheme.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatTransport.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatView.swift
../shared/ClawdisKit/Sources/ClawdisChatUI/ChatViewModel.swift
../shared/ClawdisKit/Sources/ClawdisKit/AnyCodable.swift
../shared/ClawdisKit/Sources/ClawdisKit/BonjourEscapes.swift
../shared/ClawdisKit/Sources/ClawdisKit/BonjourTypes.swift
../shared/ClawdisKit/Sources/ClawdisKit/BridgeFrames.swift
../shared/ClawdisKit/Sources/ClawdisKit/CameraCommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasA2UIAction.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasA2UICommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasA2UIJSONL.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasCommandParams.swift
../shared/ClawdisKit/Sources/ClawdisKit/CanvasCommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/Capabilities.swift
../shared/ClawdisKit/Sources/ClawdisKit/ClawdisKitResources.swift
../shared/ClawdisKit/Sources/ClawdisKit/DeepLinks.swift
../shared/ClawdisKit/Sources/ClawdisKit/JPEGTranscoder.swift
../shared/ClawdisKit/Sources/ClawdisKit/NodeError.swift
../shared/ClawdisKit/Sources/ClawdisKit/ScreenCommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/StoragePaths.swift
../shared/ClawdisKit/Sources/ClawdisKit/SystemCommands.swift
../shared/ClawdisKit/Sources/ClawdisKit/TalkDirective.swift
../../Swabble/Sources/SwabbleKit/WakeWordGate.swift
Sources/Voice/TalkModeManager.swift
Sources/Voice/TalkOrbOverlay.swift
@@ -0,0 +1,341 @@
import ClawdisKit
import Foundation
import Network
import Testing
import UIKit
@testable import Clawdis
private struct KeychainEntry: Hashable {
let service: String
let account: String
}
private let bridgeService = "com.steipete.clawdis.bridge"
private let nodeService = "com.steipete.clawdis.node"
private let instanceIdEntry = KeychainEntry(service: nodeService, account: "instanceId")
private let preferredBridgeEntry = KeychainEntry(service: bridgeService, account: "preferredStableID")
private let lastBridgeEntry = KeychainEntry(service: bridgeService, account: "lastDiscoveredStableID")
private actor MockBridgePairingClient: BridgePairingClient {
private(set) var lastToken: String?
private let resultToken: String
init(resultToken: String) {
self.resultToken = resultToken
}
func pairAndHello(
endpoint: NWEndpoint,
hello: BridgeHello,
onStatus: (@Sendable (String) -> Void)?) async throws -> String
{
self.lastToken = hello.token
onStatus?("Testing…")
return self.resultToken
}
}
private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T {
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in updates.keys {
snapshot[key] = defaults.object(forKey: key)
}
for (key, value) in updates {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
defer {
for (key, value) in snapshot {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
return try body()
}
@MainActor
private func withUserDefaults<T>(
_ updates: [String: Any?],
_ body: () async throws -> T) async rethrows -> T
{
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in updates.keys {
snapshot[key] = defaults.object(forKey: key)
}
for (key, value) in updates {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
defer {
for (key, value) in snapshot {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
return try await body()
}
private func withKeychainValues<T>(_ updates: [KeychainEntry: String?], _ body: () throws -> T) rethrows -> T {
var snapshot: [KeychainEntry: String?] = [:]
for entry in updates.keys {
snapshot[entry] = KeychainStore.loadString(service: entry.service, account: entry.account)
}
for (entry, value) in updates {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
defer {
for (entry, value) in snapshot {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
}
return try body()
}
@MainActor
private func withKeychainValues<T>(
_ updates: [KeychainEntry: String?],
_ body: () async throws -> T) async rethrows -> T
{
var snapshot: [KeychainEntry: String?] = [:]
for entry in updates.keys {
snapshot[entry] = KeychainStore.loadString(service: entry.service, account: entry.account)
}
for (entry, value) in updates {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
defer {
for (entry, value) in snapshot {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
}
return try await body()
}
@Suite(.serialized) struct BridgeConnectionControllerTests {
@Test @MainActor func resolvedDisplayNameSetsDefaultWhenMissing() {
let defaults = UserDefaults.standard
let displayKey = "node.displayName"
withKeychainValues([instanceIdEntry: nil, preferredBridgeEntry: nil, lastBridgeEntry: nil]) {
withUserDefaults([displayKey: nil, "node.instanceId": "ios-test"]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(appModel: appModel, startDiscovery: false)
let resolved = controller._test_resolvedDisplayName(defaults: defaults)
#expect(!resolved.isEmpty)
#expect(defaults.string(forKey: displayKey) == resolved)
}
}
}
@Test @MainActor func resolvedDisplayNamePreservesCustomValue() {
let defaults = UserDefaults.standard
let displayKey = "node.displayName"
withKeychainValues([instanceIdEntry: nil, preferredBridgeEntry: nil, lastBridgeEntry: nil]) {
withUserDefaults([displayKey: "My iOS Node", "node.instanceId": "ios-test"]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(appModel: appModel, startDiscovery: false)
let resolved = controller._test_resolvedDisplayName(defaults: defaults)
#expect(resolved == "My iOS Node")
#expect(defaults.string(forKey: displayKey) == "My iOS Node")
}
}
}
@Test @MainActor func makeHelloBuildsCapsAndCommands() {
let defaults = UserDefaults.standard
let voiceWakeKey = VoiceWakePreferences.enabledKey
withKeychainValues([instanceIdEntry: nil, preferredBridgeEntry: nil, lastBridgeEntry: nil]) {
withUserDefaults([
"node.instanceId": "ios-test",
"node.displayName": "Test Node",
"camera.enabled": false,
voiceWakeKey: true,
]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(appModel: appModel, startDiscovery: false)
let hello = controller._test_makeHello(token: "token-123")
#expect(hello.nodeId == "ios-test")
#expect(hello.displayName == "Test Node")
#expect(hello.token == "token-123")
let caps = Set(hello.caps ?? [])
#expect(caps.contains(ClawdisCapability.canvas.rawValue))
#expect(caps.contains(ClawdisCapability.screen.rawValue))
#expect(caps.contains(ClawdisCapability.voiceWake.rawValue))
#expect(!caps.contains(ClawdisCapability.camera.rawValue))
let commands = Set(hello.commands ?? [])
#expect(commands.contains(ClawdisCanvasCommand.present.rawValue))
#expect(commands.contains(ClawdisScreenCommand.record.rawValue))
#expect(!commands.contains(ClawdisCameraCommand.snap.rawValue))
#expect(!(hello.platform ?? "").isEmpty)
#expect(!(hello.deviceFamily ?? "").isEmpty)
#expect(!(hello.modelIdentifier ?? "").isEmpty)
#expect(!(hello.version ?? "").isEmpty)
}
}
}
@Test @MainActor func makeHelloIncludesCameraCommandsWhenEnabled() {
withKeychainValues([instanceIdEntry: nil, preferredBridgeEntry: nil, lastBridgeEntry: nil]) {
withUserDefaults([
"node.instanceId": "ios-test",
"node.displayName": "Test Node",
"camera.enabled": true,
VoiceWakePreferences.enabledKey: false,
]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(appModel: appModel, startDiscovery: false)
let hello = controller._test_makeHello(token: "token-456")
let caps = Set(hello.caps ?? [])
#expect(caps.contains(ClawdisCapability.camera.rawValue))
let commands = Set(hello.commands ?? [])
#expect(commands.contains(ClawdisCameraCommand.snap.rawValue))
#expect(commands.contains(ClawdisCameraCommand.clip.rawValue))
}
}
}
@Test @MainActor func autoConnectRefreshesTokenOnUnauthorized() async {
let bridge = BridgeDiscoveryModel.DiscoveredBridge(
name: "Gateway",
endpoint: .hostPort(host: NWEndpoint.Host("127.0.0.1"), port: 18790),
stableID: "bridge-1",
debugID: "bridge-debug",
lanHost: "Mac.local",
tailnetDns: nil,
gatewayPort: 18789,
bridgePort: 18790,
canvasPort: 18793,
cliPath: nil)
let mock = MockBridgePairingClient(resultToken: "new-token")
let account = "bridge-token.ios-test"
await withKeychainValues([
instanceIdEntry: nil,
preferredBridgeEntry: nil,
lastBridgeEntry: nil,
KeychainEntry(service: bridgeService, account: account): "old-token",
]) {
await withUserDefaults([
"node.instanceId": "ios-test",
"bridge.lastDiscoveredStableID": "bridge-1",
"bridge.manual.enabled": false,
]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(
appModel: appModel,
startDiscovery: false,
bridgeClientFactory: { mock })
controller._test_setBridges([bridge])
controller._test_triggerAutoConnect()
for _ in 0..<20 {
if appModel.connectedBridgeID == bridge.stableID { break }
try? await Task.sleep(nanoseconds: 50_000_000)
}
#expect(appModel.connectedBridgeID == bridge.stableID)
let stored = KeychainStore.loadString(service: bridgeService, account: account)
#expect(stored == "new-token")
let lastToken = await mock.lastToken
#expect(lastToken == "old-token")
}
}
}
@Test @MainActor func autoConnectPrefersPreferredBridgeOverLastDiscovered() async {
let bridgeA = BridgeDiscoveryModel.DiscoveredBridge(
name: "Gateway A",
endpoint: .hostPort(host: NWEndpoint.Host("127.0.0.1"), port: 18790),
stableID: "bridge-1",
debugID: "bridge-a",
lanHost: "MacA.local",
tailnetDns: nil,
gatewayPort: 18789,
bridgePort: 18790,
canvasPort: 18793,
cliPath: nil)
let bridgeB = BridgeDiscoveryModel.DiscoveredBridge(
name: "Gateway B",
endpoint: .hostPort(host: NWEndpoint.Host("127.0.0.1"), port: 28790),
stableID: "bridge-2",
debugID: "bridge-b",
lanHost: "MacB.local",
tailnetDns: nil,
gatewayPort: 28789,
bridgePort: 28790,
canvasPort: 28793,
cliPath: nil)
let mock = MockBridgePairingClient(resultToken: "token-ok")
let account = "bridge-token.ios-test"
await withKeychainValues([
instanceIdEntry: nil,
preferredBridgeEntry: nil,
lastBridgeEntry: nil,
KeychainEntry(service: bridgeService, account: account): "old-token",
]) {
await withUserDefaults([
"node.instanceId": "ios-test",
"bridge.preferredStableID": "bridge-2",
"bridge.lastDiscoveredStableID": "bridge-1",
"bridge.manual.enabled": false,
]) {
let appModel = NodeAppModel()
let controller = BridgeConnectionController(
appModel: appModel,
startDiscovery: false,
bridgeClientFactory: { mock })
controller._test_setBridges([bridgeA, bridgeB])
controller._test_triggerAutoConnect()
for _ in 0..<20 {
if appModel.connectedBridgeID == bridgeB.stableID { break }
try? await Task.sleep(nanoseconds: 50_000_000)
}
#expect(appModel.connectedBridgeID == bridgeB.stableID)
}
}
}
}
@@ -0,0 +1,22 @@
import Testing
@testable import Clawdis
@Suite(.serialized) struct BridgeDiscoveryModelTests {
@Test @MainActor func debugLoggingCapturesLifecycleAndResets() {
let model = BridgeDiscoveryModel()
#expect(model.debugLog.isEmpty)
#expect(model.statusText == "Idle")
model.setDebugLoggingEnabled(true)
#expect(model.debugLog.count >= 2)
model.stop()
#expect(model.statusText == "Stopped")
#expect(model.bridges.isEmpty)
#expect(model.debugLog.count >= 3)
model.setDebugLoggingEnabled(false)
#expect(model.debugLog.isEmpty)
}
}
@@ -0,0 +1,127 @@
import Foundation
import Testing
@testable import Clawdis
private struct KeychainEntry: Hashable {
let service: String
let account: String
}
private let bridgeService = "com.steipete.clawdis.bridge"
private let nodeService = "com.steipete.clawdis.node"
private let instanceIdEntry = KeychainEntry(service: nodeService, account: "instanceId")
private let preferredBridgeEntry = KeychainEntry(service: bridgeService, account: "preferredStableID")
private let lastBridgeEntry = KeychainEntry(service: bridgeService, account: "lastDiscoveredStableID")
private func snapshotDefaults(_ keys: [String]) -> [String: Any?] {
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in keys {
snapshot[key] = defaults.object(forKey: key)
}
return snapshot
}
private func applyDefaults(_ values: [String: Any?]) {
let defaults = UserDefaults.standard
for (key, value) in values {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
private func restoreDefaults(_ snapshot: [String: Any?]) {
applyDefaults(snapshot)
}
private func snapshotKeychain(_ entries: [KeychainEntry]) -> [KeychainEntry: String?] {
var snapshot: [KeychainEntry: String?] = [:]
for entry in entries {
snapshot[entry] = KeychainStore.loadString(service: entry.service, account: entry.account)
}
return snapshot
}
private func applyKeychain(_ values: [KeychainEntry: String?]) {
for (entry, value) in values {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
}
private func restoreKeychain(_ snapshot: [KeychainEntry: String?]) {
applyKeychain(snapshot)
}
@Suite(.serialized) struct BridgeSettingsStoreTests {
@Test func bootstrapCopiesDefaultsToKeychainWhenMissing() {
let defaultsKeys = [
"node.instanceId",
"bridge.preferredStableID",
"bridge.lastDiscoveredStableID",
]
let entries = [instanceIdEntry, preferredBridgeEntry, lastBridgeEntry]
let defaultsSnapshot = snapshotDefaults(defaultsKeys)
let keychainSnapshot = snapshotKeychain(entries)
defer {
restoreDefaults(defaultsSnapshot)
restoreKeychain(keychainSnapshot)
}
applyDefaults([
"node.instanceId": "node-test",
"bridge.preferredStableID": "preferred-test",
"bridge.lastDiscoveredStableID": "last-test",
])
applyKeychain([
instanceIdEntry: nil,
preferredBridgeEntry: nil,
lastBridgeEntry: nil,
])
BridgeSettingsStore.bootstrapPersistence()
#expect(KeychainStore.loadString(service: nodeService, account: "instanceId") == "node-test")
#expect(KeychainStore.loadString(service: bridgeService, account: "preferredStableID") == "preferred-test")
#expect(KeychainStore.loadString(service: bridgeService, account: "lastDiscoveredStableID") == "last-test")
}
@Test func bootstrapCopiesKeychainToDefaultsWhenMissing() {
let defaultsKeys = [
"node.instanceId",
"bridge.preferredStableID",
"bridge.lastDiscoveredStableID",
]
let entries = [instanceIdEntry, preferredBridgeEntry, lastBridgeEntry]
let defaultsSnapshot = snapshotDefaults(defaultsKeys)
let keychainSnapshot = snapshotKeychain(entries)
defer {
restoreDefaults(defaultsSnapshot)
restoreKeychain(keychainSnapshot)
}
applyDefaults([
"node.instanceId": nil,
"bridge.preferredStableID": nil,
"bridge.lastDiscoveredStableID": nil,
])
applyKeychain([
instanceIdEntry: "node-from-keychain",
preferredBridgeEntry: "preferred-from-keychain",
lastBridgeEntry: "last-from-keychain",
])
BridgeSettingsStore.bootstrapPersistence()
let defaults = UserDefaults.standard
#expect(defaults.string(forKey: "node.instanceId") == "node-from-keychain")
#expect(defaults.string(forKey: "bridge.preferredStableID") == "preferred-from-keychain")
#expect(defaults.string(forKey: "bridge.lastDiscoveredStableID") == "last-from-keychain")
}
}
@@ -0,0 +1,13 @@
import Testing
@testable import Clawdis
@Suite struct CameraControllerErrorTests {
@Test func errorDescriptionsAreStable() {
#expect(CameraController.CameraError.cameraUnavailable.errorDescription == "Camera unavailable")
#expect(CameraController.CameraError.microphoneUnavailable.errorDescription == "Microphone unavailable")
#expect(CameraController.CameraError.permissionDenied(kind: "Camera").errorDescription == "Camera permission denied")
#expect(CameraController.CameraError.invalidParams("bad").errorDescription == "bad")
#expect(CameraController.CameraError.captureFailed("nope").errorDescription == "nope")
#expect(CameraController.CameraError.exportFailed("export").errorDescription == "export")
}
}
@@ -0,0 +1,194 @@
import ClawdisKit
import Foundation
import Testing
import UIKit
@testable import Clawdis
private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T {
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in updates.keys {
snapshot[key] = defaults.object(forKey: key)
}
for (key, value) in updates {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
defer {
for (key, value) in snapshot {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
return try body()
}
@Suite(.serialized) struct NodeAppModelInvokeTests {
@Test @MainActor func decodeParamsFailsWithoutJSON() {
#expect(throws: Error.self) {
_ = try NodeAppModel._test_decodeParams(ClawdisCanvasNavigateParams.self, from: nil)
}
}
@Test @MainActor func encodePayloadEmitsJSON() throws {
struct Payload: Codable, Equatable {
var value: String
}
let json = try NodeAppModel._test_encodePayload(Payload(value: "ok"))
#expect(json.contains("\"value\""))
}
@Test @MainActor func handleInvokeRejectsBackgroundCommands() async {
let appModel = NodeAppModel()
appModel.setScenePhase(.background)
let req = BridgeInvokeRequest(id: "bg", command: ClawdisCanvasCommand.present.rawValue)
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.code == .backgroundUnavailable)
}
@Test @MainActor func handleInvokeRejectsCameraWhenDisabled() async {
let appModel = NodeAppModel()
let req = BridgeInvokeRequest(id: "cam", command: ClawdisCameraCommand.snap.rawValue)
let defaults = UserDefaults.standard
let key = "camera.enabled"
let previous = defaults.object(forKey: key)
defaults.set(false, forKey: key)
defer {
if let previous {
defaults.set(previous, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.code == .unavailable)
#expect(res.error?.message.contains("CAMERA_DISABLED") == true)
}
@Test @MainActor func handleInvokeRejectsInvalidScreenFormat() async {
let appModel = NodeAppModel()
let params = ClawdisScreenRecordParams(format: "gif")
let data = try? JSONEncoder().encode(params)
let json = data.flatMap { String(data: $0, encoding: .utf8) }
let req = BridgeInvokeRequest(
id: "screen",
command: ClawdisScreenCommand.record.rawValue,
paramsJSON: json)
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.message.contains("screen format must be mp4") == true)
}
@Test @MainActor func handleInvokeCanvasCommandsUpdateScreen() async throws {
let appModel = NodeAppModel()
appModel.screen.navigate(to: "http://example.com")
let present = BridgeInvokeRequest(id: "present", command: ClawdisCanvasCommand.present.rawValue)
let presentRes = await appModel._test_handleInvoke(present)
#expect(presentRes.ok == true)
#expect(appModel.screen.urlString.isEmpty)
let navigateParams = ClawdisCanvasNavigateParams(url: "http://localhost:18789/")
let navData = try JSONEncoder().encode(navigateParams)
let navJSON = String(decoding: navData, as: UTF8.self)
let navigate = BridgeInvokeRequest(
id: "nav",
command: ClawdisCanvasCommand.navigate.rawValue,
paramsJSON: navJSON)
let navRes = await appModel._test_handleInvoke(navigate)
#expect(navRes.ok == true)
#expect(appModel.screen.urlString == "http://localhost:18789/")
let evalParams = ClawdisCanvasEvalParams(javaScript: "1+1")
let evalData = try JSONEncoder().encode(evalParams)
let evalJSON = String(decoding: evalData, as: UTF8.self)
let eval = BridgeInvokeRequest(
id: "eval",
command: ClawdisCanvasCommand.evalJS.rawValue,
paramsJSON: evalJSON)
let evalRes = await appModel._test_handleInvoke(eval)
#expect(evalRes.ok == true)
let payloadData = try #require(evalRes.payloadJSON?.data(using: .utf8))
let payload = try JSONSerialization.jsonObject(with: payloadData) as? [String: Any]
#expect(payload?["result"] as? String == "2")
}
@Test @MainActor func handleInvokeA2UICommandsFailWhenHostMissing() async throws {
let appModel = NodeAppModel()
let reset = BridgeInvokeRequest(id: "reset", command: ClawdisCanvasA2UICommand.reset.rawValue)
let resetRes = await appModel._test_handleInvoke(reset)
#expect(resetRes.ok == false)
#expect(resetRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
let jsonl = "{\"beginRendering\":{}}"
let pushParams = ClawdisCanvasA2UIPushJSONLParams(jsonl: jsonl)
let pushData = try JSONEncoder().encode(pushParams)
let pushJSON = String(decoding: pushData, as: UTF8.self)
let push = BridgeInvokeRequest(
id: "push",
command: ClawdisCanvasA2UICommand.pushJSONL.rawValue,
paramsJSON: pushJSON)
let pushRes = await appModel._test_handleInvoke(push)
#expect(pushRes.ok == false)
#expect(pushRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
}
@Test @MainActor func handleInvokeUnknownCommandReturnsInvalidRequest() async {
let appModel = NodeAppModel()
let req = BridgeInvokeRequest(id: "unknown", command: "nope")
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.code == .invalidRequest)
}
@Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async {
let appModel = NodeAppModel()
let url = URL(string: "clawdis://agent?message=hello")!
await appModel.handleDeepLink(url: url)
#expect(appModel.screen.errorText?.contains("Bridge not connected") == true)
}
@Test @MainActor func handleDeepLinkRejectsOversizedMessage() async {
let appModel = NodeAppModel()
let msg = String(repeating: "a", count: 20001)
let url = URL(string: "clawdis://agent?message=\(msg)")!
await appModel.handleDeepLink(url: url)
#expect(appModel.screen.errorText?.contains("Deep link too large") == true)
}
@Test @MainActor func sendVoiceTranscriptThrowsWhenBridgeOffline() async {
let appModel = NodeAppModel()
await #expect(throws: Error.self) {
try await appModel.sendVoiceTranscript(text: "hello", sessionKey: "main")
}
}
@Test @MainActor func canvasA2UIActionDispatchesStatus() async {
let appModel = NodeAppModel()
let body: [String: Any] = [
"userAction": [
"name": "tap",
"id": "action-1",
"surfaceId": "main",
"sourceComponentId": "button-1",
"context": ["value": "ok"],
],
]
await appModel._test_handleCanvasA2UIAction(body: body)
#expect(appModel.screen.urlString.isEmpty)
}
}
@@ -16,6 +16,15 @@ import WebKit
#expect(scrollView.bounces == false) #expect(scrollView.bounces == false)
} }
@Test @MainActor func navigateEnablesScrollForWebPages() {
let screen = ScreenController()
screen.navigate(to: "https://example.com")
let scrollView = screen.webView.scrollView
#expect(scrollView.isScrollEnabled == true)
#expect(scrollView.bounces == true)
}
@Test @MainActor func navigateSlashShowsDefaultCanvas() { @Test @MainActor func navigateSlashShowsDefaultCanvas() {
let screen = ScreenController() let screen = ScreenController()
screen.navigate(to: "/") screen.navigate(to: "/")
@@ -0,0 +1,32 @@
import Testing
@testable import Clawdis
@Suite(.serialized) struct ScreenRecordServiceTests {
@Test func clampDefaultsAndBounds() {
#expect(ScreenRecordService._test_clampDurationMs(nil) == 10000)
#expect(ScreenRecordService._test_clampDurationMs(0) == 250)
#expect(ScreenRecordService._test_clampDurationMs(60001) == 60000)
#expect(ScreenRecordService._test_clampFps(nil) == 10)
#expect(ScreenRecordService._test_clampFps(0) == 1)
#expect(ScreenRecordService._test_clampFps(120) == 30)
#expect(ScreenRecordService._test_clampFps(.infinity) == 10)
}
@Test @MainActor func recordRejectsInvalidScreenIndex() async {
let recorder = ScreenRecordService()
do {
_ = try await recorder.record(
screenIndex: 1,
durationMs: 250,
fps: 5,
includeAudio: false,
outPath: nil)
Issue.record("Expected invalid screen index to throw")
} catch let error as ScreenRecordService.ScreenRecordError {
#expect(error.localizedDescription.contains("Invalid screen index") == true)
} catch {
Issue.record("Unexpected error type: \(error)")
}
}
}
@@ -1,33 +1,90 @@
import Foundation
import Testing import Testing
import SwabbleKit
@testable import Clawdis @testable import Clawdis
@Suite struct VoiceWakeManagerExtractCommandTests { @Suite struct VoiceWakeManagerExtractCommandTests {
@Test func extractCommandReturnsNilWhenNoTriggerFound() { @Test func extractCommandReturnsNilWhenNoTriggerFound() {
#expect(VoiceWakeManager.extractCommand(from: "hello world", triggers: ["clawd"]) == nil) let transcript = "hello world"
let segments = makeSegments(
transcript: transcript,
words: [("hello", 0.0, 0.1), ("world", 0.2, 0.1)])
#expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["clawd"]) == nil)
} }
@Test func extractCommandTrimsTokensAndResult() { @Test func extractCommandTrimsTokensAndResult() {
let cmd = VoiceWakeManager.extractCommand(from: "hey clawd do thing ", triggers: [" clawd "]) let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.9, 0.1),
("thing", 1.1, 0.1),
])
let cmd = VoiceWakeManager.extractCommand(
from: transcript,
segments: segments,
triggers: [" clawd "],
minPostTriggerGap: 0.3)
#expect(cmd == "do thing") #expect(cmd == "do thing")
} }
@Test func extractCommandPicksLatestTriggerOccurrence() { @Test func extractCommandReturnsNilWhenGapTooShort() {
let transcript = "clawd first\nthen something\nclaude second" let transcript = "hey clawd do thing"
let cmd = VoiceWakeManager.extractCommand(from: transcript, triggers: ["clawd", "claude"]) let segments = makeSegments(
#expect(cmd == "second") transcript: transcript,
} words: [
("hey", 0.0, 0.1),
@Test func extractCommandIsCaseInsensitive() { ("clawd", 0.2, 0.1),
let cmd = VoiceWakeManager.extractCommand(from: "HELLO CLAWD run it", triggers: ["clawd"]) ("do", 0.35, 0.1),
#expect(cmd == "run it") ("thing", 0.5, 0.1),
])
let cmd = VoiceWakeManager.extractCommand(
from: transcript,
segments: segments,
triggers: ["clawd"],
minPostTriggerGap: 0.3)
#expect(cmd == nil)
} }
@Test func extractCommandReturnsNilWhenNothingAfterTrigger() { @Test func extractCommandReturnsNilWhenNothingAfterTrigger() {
#expect(VoiceWakeManager.extractCommand(from: "hey clawd \n", triggers: ["clawd"]) == nil) let transcript = "hey clawd"
let segments = makeSegments(
transcript: transcript,
words: [("hey", 0.0, 0.1), ("clawd", 0.2, 0.1)])
#expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["clawd"]) == nil)
} }
@Test func extractCommandIgnoresEmptyTriggers() { @Test func extractCommandIgnoresEmptyTriggers() {
let cmd = VoiceWakeManager.extractCommand(from: "hey clawd do thing", triggers: ["", " ", "clawd"]) let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.9, 0.1),
("thing", 1.1, 0.1),
])
let cmd = VoiceWakeManager.extractCommand(
from: transcript,
segments: segments,
triggers: ["", " ", "clawd"],
minPostTriggerGap: 0.3)
#expect(cmd == "do thing") #expect(cmd == "do thing")
} }
} }
private func makeSegments(
transcript: String,
words: [(String, TimeInterval, TimeInterval)])
-> [WakeWordSegment] {
var searchStart = transcript.startIndex
var output: [WakeWordSegment] = []
for (word, start, duration) in words {
let range = transcript.range(of: word, range: searchStart..<transcript.endIndex)
output.append(WakeWordSegment(text: word, start: start, duration: duration, range: range))
if let range { searchStart = range.upperBound }
}
return output
}
@@ -0,0 +1,65 @@
import Foundation
import Testing
import SwabbleKit
@testable import Clawdis
@Suite(.serialized) struct VoiceWakeManagerStateTests {
@Test @MainActor func suspendAndResumeCycleUpdatesState() async {
let manager = VoiceWakeManager()
manager.isEnabled = true
manager.isListening = true
manager.statusText = "Listening"
let suspended = manager.suspendForExternalAudioCapture()
#expect(suspended == true)
#expect(manager.isListening == false)
#expect(manager.statusText == "Paused")
manager.resumeAfterExternalAudioCapture(wasSuspended: true)
try? await Task.sleep(nanoseconds: 900_000_000)
#expect(manager.statusText.contains("Voice Wake") == true)
}
@Test @MainActor func handleRecognitionCallbackRestartsOnError() async {
let manager = VoiceWakeManager()
manager.isEnabled = true
manager.isListening = true
manager._test_handleRecognitionCallback(transcript: nil, segments: [], errorText: "boom")
#expect(manager.statusText.contains("Recognizer error") == true)
#expect(manager.isListening == false)
try? await Task.sleep(nanoseconds: 900_000_000)
#expect(manager.statusText.contains("Voice Wake") == true)
}
@Test @MainActor func handleRecognitionCallbackDispatchesCommand() async {
let manager = VoiceWakeManager()
manager.triggerWords = ["clawd"]
manager.isEnabled = true
actor CaptureBox {
var value: String?
func set(_ next: String) { self.value = next }
}
let capture = CaptureBox()
manager.configure { cmd in
await capture.set(cmd)
}
let transcript = "clawd hello"
let clawdRange = transcript.range(of: "clawd")!
let helloRange = transcript.range(of: "hello")!
let segments = [
WakeWordSegment(text: "clawd", start: 0.0, duration: 0.2, range: clawdRange),
WakeWordSegment(text: "hello", start: 0.8, duration: 0.2, range: helloRange),
]
manager._test_handleRecognitionCallback(transcript: transcript, segments: segments, errorText: nil)
#expect(manager.lastTriggeredCommand == "hello")
#expect(manager.statusText == "Triggered")
try? await Task.sleep(nanoseconds: 300_000_000)
#expect(await capture.value == "hello")
}
}
+8
View File
@@ -1,3 +1,5 @@
require "shellwords"
default_platform(:ios) default_platform(:ios)
def load_env_file(path) def load_env_file(path)
@@ -61,6 +63,12 @@ platform :ios do
api_key = asc_api_key api_key = asc_api_key
team_id = ENV["IOS_DEVELOPMENT_TEAM"] team_id = ENV["IOS_DEVELOPMENT_TEAM"]
if team_id.nil? || team_id.strip.empty?
helper_path = File.expand_path("../../scripts/ios-team-id.sh", __dir__)
if File.exist?(helper_path)
team_id = sh("bash #{helper_path.shellescape}").strip
end
end
UI.user_error!("Missing IOS_DEVELOPMENT_TEAM (Apple Team ID). Add it to fastlane/.env or export it in your shell.") if team_id.nil? || team_id.strip.empty? UI.user_error!("Missing IOS_DEVELOPMENT_TEAM (Apple Team ID). Add it to fastlane/.env or export it in your shell.") if team_id.nil? || team_id.strip.empty?
build_app( build_app(
+2 -1
View File
@@ -22,10 +22,11 @@ ASC_KEY_PATH=/absolute/path/to/AuthKey_XXXXXXXXXX.p8
IOS_DEVELOPMENT_TEAM=YOUR_TEAM_ID IOS_DEVELOPMENT_TEAM=YOUR_TEAM_ID
``` ```
Tip: run `scripts/ios-team-id.sh` from the repo root to print a Team ID to paste into `.env`. Fastlane falls back to this helper if `IOS_DEVELOPMENT_TEAM` is missing.
Run: Run:
```bash ```bash
cd apps/ios cd apps/ios
fastlane beta fastlane beta
``` ```
+21 -3
View File
@@ -8,6 +8,8 @@ options:
packages: packages:
ClawdisKit: ClawdisKit:
path: ../shared/ClawdisKit path: ../shared/ClawdisKit
Swabble:
path: ../../Swabble
schemes: schemes:
Clawdis: Clawdis:
@@ -29,30 +31,42 @@ targets:
- package: ClawdisKit - package: ClawdisKit
- package: ClawdisKit - package: ClawdisKit
product: ClawdisChatUI product: ClawdisChatUI
- package: Swabble
product: SwabbleKit
- sdk: AppIntents.framework
preBuildScripts: preBuildScripts:
- name: SwiftFormat (lint) - name: SwiftFormat (lint)
basedOnDependencyAnalysis: false basedOnDependencyAnalysis: false
inputFileLists:
- $(SRCROOT)/SwiftSources.input.xcfilelist
script: | script: |
set -euo pipefail set -euo pipefail
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
if ! command -v swiftformat >/dev/null 2>&1; then if ! command -v swiftformat >/dev/null 2>&1; then
echo "error: swiftformat not found (brew install swiftformat)" >&2 echo "error: swiftformat not found (brew install swiftformat)" >&2
exit 1 exit 1
fi fi
swiftformat --lint --config "$SRCROOT/../../.swiftformat" \ swiftformat --lint --config "$SRCROOT/../../.swiftformat" \
"$SRCROOT/Sources" \ --filelist "$SRCROOT/SwiftSources.input.xcfilelist"
"$SRCROOT/../shared/ClawdisKit/Sources"
- name: SwiftLint - name: SwiftLint
basedOnDependencyAnalysis: false basedOnDependencyAnalysis: false
inputFileLists:
- $(SRCROOT)/SwiftSources.input.xcfilelist
script: | script: |
set -euo pipefail set -euo pipefail
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
if ! command -v swiftlint >/dev/null 2>&1; then if ! command -v swiftlint >/dev/null 2>&1; then
echo "error: swiftlint not found (brew install swiftlint)" >&2 echo "error: swiftlint not found (brew install swiftlint)" >&2
exit 1 exit 1
fi fi
swiftlint lint --config "$SRCROOT/.swiftlint.yml" swiftlint lint --config "$SRCROOT/.swiftlint.yml" --use-script-input-file-lists
settings: settings:
base: base:
CODE_SIGN_IDENTITY: "Apple Development"
CODE_SIGN_STYLE: Manual
DEVELOPMENT_TEAM: Y5PE65HELJ
PRODUCT_BUNDLE_IDENTIFIER: com.steipete.clawdis.ios PRODUCT_BUNDLE_IDENTIFIER: com.steipete.clawdis.ios
PROVISIONING_PROFILE_SPECIFIER: "com.steipete.clawdis.ios Development"
SWIFT_VERSION: "6.0" SWIFT_VERSION: "6.0"
info: info:
path: Sources/Info.plist path: Sources/Info.plist
@@ -65,6 +79,8 @@ targets:
UIBackgroundModes: UIBackgroundModes:
- audio - audio
NSLocalNetworkUsageDescription: Clawdis discovers and connects to your Clawdis bridge on the local network. NSLocalNetworkUsageDescription: Clawdis discovers and connects to your Clawdis bridge on the local network.
NSAppTransportSecurity:
NSAllowsArbitraryLoadsInWebContent: true
NSBonjourServices: NSBonjourServices:
- _clawdis-bridge._tcp - _clawdis-bridge._tcp
NSCameraUsageDescription: Clawdis can capture photos or short video clips when requested via the bridge. NSCameraUsageDescription: Clawdis can capture photos or short video clips when requested via the bridge.
@@ -78,6 +94,8 @@ targets:
- path: Tests - path: Tests
dependencies: dependencies:
- target: Clawdis - target: Clawdis
- package: Swabble
product: SwabbleKit
settings: settings:
base: base:
PRODUCT_BUNDLE_IDENTIFIER: com.steipete.clawdis.ios.tests PRODUCT_BUNDLE_IDENTIFIER: com.steipete.clawdis.ios.tests
+13 -1
View File
@@ -15,8 +15,10 @@ let package = Package(
dependencies: [ dependencies: [
.package(url: "https://github.com/orchetect/MenuBarExtraAccess", exact: "1.2.2"), .package(url: "https://github.com/orchetect/MenuBarExtraAccess", exact: "1.2.2"),
.package(url: "https://github.com/swiftlang/swift-subprocess.git", from: "0.1.0"), .package(url: "https://github.com/swiftlang/swift-subprocess.git", from: "0.1.0"),
.package(url: "https://github.com/apple/swift-log.git", from: "1.8.0"),
.package(url: "https://github.com/sparkle-project/Sparkle", from: "2.8.1"), .package(url: "https://github.com/sparkle-project/Sparkle", from: "2.8.1"),
.package(path: "../shared/ClawdisKit"), .package(path: "../shared/ClawdisKit"),
.package(path: "../../Swabble"),
.package(path: "../../Peekaboo/Core/PeekabooCore"), .package(path: "../../Peekaboo/Core/PeekabooCore"),
.package(path: "../../Peekaboo/Core/PeekabooAutomationKit"), .package(path: "../../Peekaboo/Core/PeekabooAutomationKit"),
], ],
@@ -41,12 +43,17 @@ let package = Package(
"ClawdisProtocol", "ClawdisProtocol",
.product(name: "ClawdisKit", package: "ClawdisKit"), .product(name: "ClawdisKit", package: "ClawdisKit"),
.product(name: "ClawdisChatUI", package: "ClawdisKit"), .product(name: "ClawdisChatUI", package: "ClawdisKit"),
.product(name: "SwabbleKit", package: "swabble"),
.product(name: "MenuBarExtraAccess", package: "MenuBarExtraAccess"), .product(name: "MenuBarExtraAccess", package: "MenuBarExtraAccess"),
.product(name: "Subprocess", package: "swift-subprocess"), .product(name: "Subprocess", package: "swift-subprocess"),
.product(name: "Logging", package: "swift-log"),
.product(name: "Sparkle", package: "Sparkle"), .product(name: "Sparkle", package: "Sparkle"),
.product(name: "PeekabooBridge", package: "PeekabooCore"), .product(name: "PeekabooBridge", package: "PeekabooCore"),
.product(name: "PeekabooAutomationKit", package: "PeekabooAutomationKit"), .product(name: "PeekabooAutomationKit", package: "PeekabooAutomationKit"),
], ],
exclude: [
"Resources/Info.plist",
],
resources: [ resources: [
.copy("Resources/Clawdis.icns"), .copy("Resources/Clawdis.icns"),
.copy("Resources/DeviceModels"), .copy("Resources/DeviceModels"),
@@ -56,7 +63,12 @@ let package = Package(
]), ]),
.testTarget( .testTarget(
name: "ClawdisIPCTests", name: "ClawdisIPCTests",
dependencies: ["ClawdisIPC", "Clawdis", "ClawdisProtocol"], dependencies: [
"ClawdisIPC",
"Clawdis",
"ClawdisProtocol",
.product(name: "SwabbleKit", package: "swabble"),
],
swiftSettings: [ swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"), .enableUpcomingFeature("StrictConcurrency"),
.enableExperimentalFeature("SwiftTesting"), .enableExperimentalFeature("SwiftTesting"),
@@ -92,10 +92,12 @@ struct AboutSettings: View {
guard let updater, !self.didLoadUpdaterState else { return } guard let updater, !self.didLoadUpdaterState else { return }
// Keep Sparkle’s auto-check setting in sync with the persisted toggle. // Keep Sparkle’s auto-check setting in sync with the persisted toggle.
updater.automaticallyChecksForUpdates = self.autoCheckEnabled updater.automaticallyChecksForUpdates = self.autoCheckEnabled
updater.automaticallyDownloadsUpdates = self.autoCheckEnabled
self.didLoadUpdaterState = true self.didLoadUpdaterState = true
} }
.onChange(of: self.autoCheckEnabled) { _, newValue in .onChange(of: self.autoCheckEnabled) { _, newValue in
self.updater?.automaticallyChecksForUpdates = newValue self.updater?.automaticallyChecksForUpdates = newValue
self.updater?.automaticallyDownloadsUpdates = newValue
} }
} }
@@ -0,0 +1,17 @@
import Foundation
// Human-friendly age string (e.g., "2m ago").
func age(from date: Date, now: Date = .init()) -> String {
let seconds = max(0, Int(now.timeIntervalSince(date)))
let minutes = seconds / 60
let hours = minutes / 60
let days = hours / 24
if seconds < 60 { return "just now" }
if minutes == 1 { return "1 minute ago" }
if minutes < 60 { return "\(minutes)m ago" }
if hours == 1 { return "1 hour ago" }
if hours < 24 { return "\(hours)h ago" }
if days == 1 { return "yesterday" }
return "\(days)d ago"
}
@@ -1,45 +0,0 @@
import Foundation
struct AgentIdentity: Codable, Equatable {
var name: String
var theme: String
var emoji: String
var isEmpty: Bool {
self.name.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty &&
self.theme.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty &&
self.emoji.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
}
}
enum AgentIdentityEmoji {
static func suggest(theme: String) -> String {
let normalized = theme.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
if normalized.isEmpty { return "🦞" }
let table: [(needle: String, emoji: String)] = [
("lobster", "🦞"),
("sloth", "🦥"),
("octopus", "🐙"),
("crab", "🦀"),
("shark", "🦈"),
("cat", "🐈"),
("dog", "🐕"),
("owl", "🦉"),
("fox", "🦊"),
("otter", "🦦"),
("raccoon", "🦝"),
("badger", "🦡"),
("hedgehog", "🦔"),
("koala", "🐨"),
("penguin", "🐧"),
("frog", "🐸"),
("bear", "🐻"),
]
for entry in table where normalized.contains(entry.needle) {
return entry.emoji
}
return "🦞"
}
}
+89 -45
View File
@@ -9,8 +9,18 @@ enum AgentWorkspace {
static let userFilename = "USER.md" static let userFilename = "USER.md"
static let bootstrapFilename = "BOOTSTRAP.md" static let bootstrapFilename = "BOOTSTRAP.md"
private static let templateDirname = "templates" private static let templateDirname = "templates"
static let identityStartMarker = "<!-- clawdis:identity:start -->" private static let ignoredEntries: Set<String> = [".DS_Store", ".git", ".gitignore"]
static let identityEndMarker = "<!-- clawdis:identity:end -->" private static let templateEntries: Set<String> = [
AgentWorkspace.agentsFilename,
AgentWorkspace.soulFilename,
AgentWorkspace.identityFilename,
AgentWorkspace.userFilename,
AgentWorkspace.bootstrapFilename,
]
enum BootstrapSafety: Equatable {
case safe
case unsafe(reason: String)
}
static func displayPath(for url: URL) -> String { static func displayPath(for url: URL) -> String {
let home = FileManager.default.homeDirectoryForCurrentUser.path let home = FileManager.default.homeDirectoryForCurrentUser.path
@@ -33,7 +43,53 @@ enum AgentWorkspace {
workspaceURL.appendingPathComponent(self.agentsFilename) workspaceURL.appendingPathComponent(self.agentsFilename)
} }
static func workspaceEntries(workspaceURL: URL) throws -> [String] {
let contents = try FileManager.default.contentsOfDirectory(atPath: workspaceURL.path)
return contents.filter { !self.ignoredEntries.contains($0) }
}
static func isWorkspaceEmpty(workspaceURL: URL) -> Bool {
let fm = FileManager.default
var isDir: ObjCBool = false
if !fm.fileExists(atPath: workspaceURL.path, isDirectory: &isDir) {
return true
}
guard isDir.boolValue else { return false }
guard let entries = try? self.workspaceEntries(workspaceURL: workspaceURL) else { return false }
return entries.isEmpty
}
static func isTemplateOnlyWorkspace(workspaceURL: URL) -> Bool {
guard let entries = try? self.workspaceEntries(workspaceURL: workspaceURL) else { return false }
guard !entries.isEmpty else { return true }
return Set(entries).isSubset(of: self.templateEntries)
}
static func bootstrapSafety(for workspaceURL: URL) -> BootstrapSafety {
let fm = FileManager.default
var isDir: ObjCBool = false
if !fm.fileExists(atPath: workspaceURL.path, isDirectory: &isDir) {
return .safe
}
if !isDir.boolValue {
return .unsafe(reason: "Workspace path points to a file.")
}
let agentsURL = self.agentsURL(workspaceURL: workspaceURL)
if fm.fileExists(atPath: agentsURL.path) {
return .safe
}
do {
let entries = try self.workspaceEntries(workspaceURL: workspaceURL)
return entries.isEmpty
? .safe
: .unsafe(reason: "Folder isn't empty. Choose a new folder or add AGENTS.md first.")
} catch {
return .unsafe(reason: "Couldn't inspect the workspace folder.")
}
}
static func bootstrap(workspaceURL: URL) throws -> URL { static func bootstrap(workspaceURL: URL) throws -> URL {
let shouldSeedBootstrap = self.isWorkspaceEmpty(workspaceURL: workspaceURL)
try FileManager.default.createDirectory(at: workspaceURL, withIntermediateDirectories: true) try FileManager.default.createDirectory(at: workspaceURL, withIntermediateDirectories: true)
let agentsURL = self.agentsURL(workspaceURL: workspaceURL) let agentsURL = self.agentsURL(workspaceURL: workspaceURL)
if !FileManager.default.fileExists(atPath: agentsURL.path) { if !FileManager.default.fileExists(atPath: agentsURL.path) {
@@ -56,36 +112,44 @@ enum AgentWorkspace {
self.logger.info("Created USER.md at \(userURL.path, privacy: .public)") self.logger.info("Created USER.md at \(userURL.path, privacy: .public)")
} }
let bootstrapURL = workspaceURL.appendingPathComponent(self.bootstrapFilename) let bootstrapURL = workspaceURL.appendingPathComponent(self.bootstrapFilename)
if !FileManager.default.fileExists(atPath: bootstrapURL.path) { if shouldSeedBootstrap, !FileManager.default.fileExists(atPath: bootstrapURL.path) {
try self.defaultBootstrapTemplate().write(to: bootstrapURL, atomically: true, encoding: .utf8) try self.defaultBootstrapTemplate().write(to: bootstrapURL, atomically: true, encoding: .utf8)
self.logger.info("Created BOOTSTRAP.md at \(bootstrapURL.path, privacy: .public)") self.logger.info("Created BOOTSTRAP.md at \(bootstrapURL.path, privacy: .public)")
} }
return agentsURL return agentsURL
} }
static func upsertIdentity(workspaceURL: URL, identity: AgentIdentity) throws { static func needsBootstrap(workspaceURL: URL) -> Bool {
let agentsURL = try self.bootstrap(workspaceURL: workspaceURL) let fm = FileManager.default
var content = (try? String(contentsOf: agentsURL, encoding: .utf8)) ?? "" var isDir: ObjCBool = false
let block = self.identityBlock(identity: identity) if !fm.fileExists(atPath: workspaceURL.path, isDirectory: &isDir) {
return true
if let start = content.range(of: self.identityStartMarker),
let end = content.range(of: self.identityEndMarker),
start.lowerBound < end.upperBound
{
content.replaceSubrange(
start.lowerBound..<end.upperBound,
with: block.trimmingCharacters(in: .whitespacesAndNewlines))
} else if let insert = self.identityInsertRange(in: content) {
content.insert(contentsOf: "\n\n## Identity\n\(block)\n", at: insert.upperBound)
} else {
content = [content.trimmingCharacters(in: .whitespacesAndNewlines), "## Identity\n\(block)"]
.filter { !$0.isEmpty }
.joined(separator: "\n\n")
.appending("\n")
} }
guard isDir.boolValue else { return true }
if self.hasIdentity(workspaceURL: workspaceURL) {
return false
}
let bootstrapURL = workspaceURL.appendingPathComponent(self.bootstrapFilename)
guard fm.fileExists(atPath: bootstrapURL.path) else { return false }
return self.isTemplateOnlyWorkspace(workspaceURL: workspaceURL)
}
try content.write(to: agentsURL, atomically: true, encoding: .utf8) static func hasIdentity(workspaceURL: URL) -> Bool {
self.logger.info("Updated identity in \(agentsURL.path, privacy: .public)") let identityURL = workspaceURL.appendingPathComponent(self.identityFilename)
guard let contents = try? String(contentsOf: identityURL, encoding: .utf8) else { return false }
return self.identityLinesHaveValues(contents)
}
private static func identityLinesHaveValues(_ content: String) -> Bool {
for line in content.split(separator: "\n") {
let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines)
guard trimmed.hasPrefix("-"), let colon = trimmed.firstIndex(of: ":") else { continue }
let value = trimmed[trimmed.index(after: colon)...].trimmingCharacters(in: .whitespacesAndNewlines)
if !value.isEmpty {
return true
}
}
return false
} }
static func defaultTemplate() -> String { static func defaultTemplate() -> String {
@@ -272,25 +336,5 @@ enum AgentWorkspace {
return trimmed + "\n" return trimmed + "\n"
} }
private static func identityBlock(identity: AgentIdentity) -> String { // Identity is written by the agent during the bootstrap ritual.
let name = identity.name.trimmingCharacters(in: .whitespacesAndNewlines)
let theme = identity.theme.trimmingCharacters(in: .whitespacesAndNewlines)
let emoji = identity.emoji.trimmingCharacters(in: .whitespacesAndNewlines)
return """
\(self.identityStartMarker)
- Name: \(name)
- Theme: \(theme)
- Emoji: \(emoji)
\(self.identityEndMarker)
"""
}
private static func identityInsertRange(in content: String) -> Range<String.Index>? {
if let firstHeading = content.range(of: "\n") {
// Insert after the first line (usually "# AGENTS.md …")
return firstHeading
}
return nil
}
} }
@@ -6,7 +6,7 @@ import SwiftUI
struct AnthropicAuthControls: View { struct AnthropicAuthControls: View {
let connectionMode: AppState.ConnectionMode let connectionMode: AppState.ConnectionMode
@State private var oauthStatus: PiOAuthStore.AnthropicOAuthStatus = PiOAuthStore.anthropicOAuthStatus() @State private var oauthStatus: ClawdisOAuthStore.AnthropicOAuthStatus = ClawdisOAuthStore.anthropicOAuthStatus()
@State private var pkce: AnthropicOAuth.PKCE? @State private var pkce: AnthropicOAuth.PKCE?
@State private var code: String = "" @State private var code: String = ""
@State private var busy = false @State private var busy = false
@@ -15,12 +15,19 @@ struct AnthropicAuthControls: View {
@State private var autoConnectClipboard = true @State private var autoConnectClipboard = true
@State private var lastPasteboardChangeCount = NSPasteboard.general.changeCount @State private var lastPasteboardChangeCount = NSPasteboard.general.changeCount
private static let clipboardPoll = Timer.publish(every: 0.4, on: .main, in: .common).autoconnect() private static let clipboardPoll: AnyPublisher<Date, Never> = {
if ProcessInfo.processInfo.isRunningTests {
return Empty(completeImmediately: false).eraseToAnyPublisher()
}
return Timer.publish(every: 0.4, on: .main, in: .common)
.autoconnect()
.eraseToAnyPublisher()
}()
var body: some View { var body: some View {
VStack(alignment: .leading, spacing: 10) { VStack(alignment: .leading, spacing: 10) {
if self.connectionMode != .local { if self.connectionMode != .local {
Text("Gateway isn’t running locally; OAuth must be created on the gateway host where Pi runs.") Text("Gateway isn’t running locally; OAuth must be created on the gateway host.")
.font(.footnote) .font(.footnote)
.foregroundStyle(.secondary) .foregroundStyle(.secondary)
.fixedSize(horizontal: false, vertical: true) .fixedSize(horizontal: false, vertical: true)
@@ -35,10 +42,10 @@ struct AnthropicAuthControls: View {
.foregroundStyle(.secondary) .foregroundStyle(.secondary)
Spacer() Spacer()
Button("Reveal") { Button("Reveal") {
NSWorkspace.shared.activateFileViewerSelecting([PiOAuthStore.oauthURL()]) NSWorkspace.shared.activateFileViewerSelecting([ClawdisOAuthStore.oauthURL()])
} }
.buttonStyle(.bordered) .buttonStyle(.bordered)
.disabled(!FileManager.default.fileExists(atPath: PiOAuthStore.oauthURL().path)) .disabled(!FileManager.default.fileExists(atPath: ClawdisOAuthStore.oauthURL().path))
Button("Refresh") { Button("Refresh") {
self.refresh() self.refresh()
@@ -46,7 +53,7 @@ struct AnthropicAuthControls: View {
.buttonStyle(.bordered) .buttonStyle(.bordered)
} }
Text(PiOAuthStore.oauthURL().path) Text(ClawdisOAuthStore.oauthURL().path)
.font(.caption.monospaced()) .font(.caption.monospaced())
.foregroundStyle(.secondary) .foregroundStyle(.secondary)
.lineLimit(1) .lineLimit(1)
@@ -123,7 +130,11 @@ struct AnthropicAuthControls: View {
} }
private func refresh() { private func refresh() {
self.oauthStatus = PiOAuthStore.anthropicOAuthStatus() let imported = ClawdisOAuthStore.importLegacyAnthropicOAuthIfNeeded()
self.oauthStatus = ClawdisOAuthStore.anthropicOAuthStatus()
if imported != nil {
self.statusText = "Imported existing OAuth credentials."
}
} }
private func startOAuth() { private func startOAuth() {
@@ -161,11 +172,11 @@ struct AnthropicAuthControls: View {
code: parsed.code, code: parsed.code,
state: parsed.state, state: parsed.state,
verifier: pkce.verifier) verifier: pkce.verifier)
try PiOAuthStore.saveAnthropicOAuth(creds) try ClawdisOAuthStore.saveAnthropicOAuth(creds)
self.refresh() self.refresh()
self.pkce = nil self.pkce = nil
self.code = "" self.code = ""
self.statusText = "Connected. Pi can now use Claude via OAuth." self.statusText = "Connected. Clawdis can now use Claude via OAuth."
} catch { } catch {
self.statusText = "OAuth failed: \(error.localizedDescription)" self.statusText = "OAuth failed: \(error.localizedDescription)"
} }
@@ -196,3 +207,28 @@ struct AnthropicAuthControls: View {
Task { await self.finishOAuth() } Task { await self.finishOAuth() }
} }
} }
#if DEBUG
extension AnthropicAuthControls {
init(
connectionMode: AppState.ConnectionMode,
oauthStatus: ClawdisOAuthStore.AnthropicOAuthStatus,
pkce: AnthropicOAuth.PKCE? = nil,
code: String = "",
busy: Bool = false,
statusText: String? = nil,
autoDetectClipboard: Bool = true,
autoConnectClipboard: Bool = true)
{
self.connectionMode = connectionMode
self._oauthStatus = State(initialValue: oauthStatus)
self._pkce = State(initialValue: pkce)
self._code = State(initialValue: code)
self._busy = State(initialValue: busy)
self._statusText = State(initialValue: statusText)
self._autoDetectClipboard = State(initialValue: autoDetectClipboard)
self._autoConnectClipboard = State(initialValue: autoConnectClipboard)
self._lastPasteboardChangeCount = State(initialValue: NSPasteboard.general.changeCount)
}
}
#endif
+67 -23
View File
@@ -18,7 +18,7 @@ enum AnthropicAuthMode: Equatable {
var shortLabel: String { var shortLabel: String {
switch self { switch self {
case .oauthFile: "OAuth (Pi token file)" case .oauthFile: "OAuth (Clawdis token file)"
case .oauthEnv: "OAuth (env var)" case .oauthEnv: "OAuth (env var)"
case .apiKeyEnv: "API key (env var)" case .apiKeyEnv: "API key (env var)"
case .missing: "Missing credentials" case .missing: "Missing credentials"
@@ -36,7 +36,8 @@ enum AnthropicAuthMode: Equatable {
enum AnthropicAuthResolver { enum AnthropicAuthResolver {
static func resolve( static func resolve(
environment: [String: String] = ProcessInfo.processInfo.environment, environment: [String: String] = ProcessInfo.processInfo.environment,
oauthStatus: PiOAuthStore.AnthropicOAuthStatus = PiOAuthStore.anthropicOAuthStatus()) -> AnthropicAuthMode oauthStatus: ClawdisOAuthStore.AnthropicOAuthStatus = ClawdisOAuthStore
.anthropicOAuthStatus()) -> AnthropicAuthMode
{ {
if oauthStatus.isConnected { return .oauthFile } if oauthStatus.isConnected { return .oauthFile }
@@ -92,7 +93,7 @@ enum AnthropicOAuth {
URLQueryItem(name: "scope", value: self.scopes), URLQueryItem(name: "scope", value: self.scopes),
URLQueryItem(name: "code_challenge", value: pkce.challenge), URLQueryItem(name: "code_challenge", value: pkce.challenge),
URLQueryItem(name: "code_challenge_method", value: "S256"), URLQueryItem(name: "code_challenge_method", value: "S256"),
// Match Pi: state is the verifier. // Match legacy flow: state is the verifier.
URLQueryItem(name: "state", value: pkce.verifier), URLQueryItem(name: "state", value: pkce.verifier),
] ]
return components.url! return components.url!
@@ -140,7 +141,7 @@ enum AnthropicOAuth {
]) ])
} }
// Match Pi: expiresAt = now + expires_in - 5 minutes. // Match legacy flow: expiresAt = now + expires_in - 5 minutes.
let expiresAtMs = Int64(Date().timeIntervalSince1970 * 1000) let expiresAtMs = Int64(Date().timeIntervalSince1970 * 1000)
+ Int64(expiresIn * 1000) + Int64(expiresIn * 1000)
- Int64(5 * 60 * 1000) - Int64(5 * 60 * 1000)
@@ -150,10 +151,11 @@ enum AnthropicOAuth {
} }
} }
enum PiOAuthStore { enum ClawdisOAuthStore {
static let oauthFilename = "oauth.json" static let oauthFilename = "oauth.json"
private static let providerKey = "anthropic" private static let providerKey = "anthropic"
private static let piAgentDirEnv = "PI_CODING_AGENT_DIR" private static let clawdisOAuthDirEnv = "CLAWDIS_OAUTH_DIR"
private static let legacyPiDirEnv = "PI_CODING_AGENT_DIR"
enum AnthropicOAuthStatus: Equatable { enum AnthropicOAuthStatus: Equatable {
case missingFile case missingFile
@@ -170,18 +172,18 @@ enum PiOAuthStore {
var shortDescription: String { var shortDescription: String {
switch self { switch self {
case .missingFile: "Pi OAuth token file not found" case .missingFile: "Clawdis OAuth token file not found"
case .unreadableFile: "Pi OAuth token file not readable" case .unreadableFile: "Clawdis OAuth token file not readable"
case .invalidJSON: "Pi OAuth token file invalid" case .invalidJSON: "Clawdis OAuth token file invalid"
case .missingProviderEntry: "No Anthropic entry in Pi OAuth token file" case .missingProviderEntry: "No Anthropic entry in Clawdis OAuth token file"
case .missingTokens: "Anthropic entry missing tokens" case .missingTokens: "Anthropic entry missing tokens"
case .connected: "Pi OAuth credentials found" case .connected: "Clawdis OAuth credentials found"
} }
} }
} }
static func oauthDir() -> URL { static func oauthDir() -> URL {
if let override = ProcessInfo.processInfo.environment[self.piAgentDirEnv]? if let override = ProcessInfo.processInfo.environment[self.clawdisOAuthDirEnv]?
.trimmingCharacters(in: .whitespacesAndNewlines), .trimmingCharacters(in: .whitespacesAndNewlines),
!override.isEmpty !override.isEmpty
{ {
@@ -190,14 +192,58 @@ enum PiOAuthStore {
} }
return FileManager.default.homeDirectoryForCurrentUser return FileManager.default.homeDirectoryForCurrentUser
.appendingPathComponent(".pi", isDirectory: true) .appendingPathComponent(".clawdis", isDirectory: true)
.appendingPathComponent("agent", isDirectory: true) .appendingPathComponent("credentials", isDirectory: true)
} }
static func oauthURL() -> URL { static func oauthURL() -> URL {
self.oauthDir().appendingPathComponent(self.oauthFilename) self.oauthDir().appendingPathComponent(self.oauthFilename)
} }
static func legacyOAuthURLs() -> [URL] {
var urls: [URL] = []
let env = ProcessInfo.processInfo.environment
if let override = env[self.legacyPiDirEnv]?.trimmingCharacters(in: .whitespacesAndNewlines),
!override.isEmpty
{
let expanded = NSString(string: override).expandingTildeInPath
urls.append(URL(fileURLWithPath: expanded, isDirectory: true).appendingPathComponent(self.oauthFilename))
}
let home = FileManager.default.homeDirectoryForCurrentUser
urls.append(home.appendingPathComponent(".pi/agent/\(self.oauthFilename)"))
urls.append(home.appendingPathComponent(".claude/\(self.oauthFilename)"))
urls.append(home.appendingPathComponent(".config/claude/\(self.oauthFilename)"))
urls.append(home.appendingPathComponent(".config/anthropic/\(self.oauthFilename)"))
var seen = Set<String>()
return urls.filter { url in
let path = url.standardizedFileURL.path
if seen.contains(path) { return false }
seen.insert(path)
return true
}
}
static func importLegacyAnthropicOAuthIfNeeded() -> URL? {
let dest = self.oauthURL()
guard !FileManager.default.fileExists(atPath: dest.path) else { return nil }
for url in self.legacyOAuthURLs() {
guard FileManager.default.fileExists(atPath: url.path) else { continue }
guard self.anthropicOAuthStatus(at: url).isConnected else { continue }
guard let storage = self.loadStorage(at: url) else { continue }
do {
try self.saveStorage(storage)
return url
} catch {
continue
}
}
return nil
}
static func anthropicOAuthStatus() -> AnthropicOAuthStatus { static func anthropicOAuthStatus() -> AnthropicOAuthStatus {
self.anthropicOAuthStatus(at: self.oauthURL()) self.anthropicOAuthStatus(at: self.oauthURL())
} }
@@ -240,17 +286,15 @@ enum PiOAuthStore {
return nil return nil
} }
private static func loadStorage(at url: URL) -> [String: Any]? {
guard let data = try? Data(contentsOf: url) else { return nil }
guard let json = try? JSONSerialization.jsonObject(with: data, options: []) else { return nil }
return json as? [String: Any]
}
static func saveAnthropicOAuth(_ creds: AnthropicOAuthCredentials) throws { static func saveAnthropicOAuth(_ creds: AnthropicOAuthCredentials) throws {
let url = self.oauthURL() let url = self.oauthURL()
let existing: [String: Any] = if FileManager.default.fileExists(atPath: url.path), let existing: [String: Any] = self.loadStorage(at: url) ?? [:]
let data = try? Data(contentsOf: url),
let json = try? JSONSerialization.jsonObject(with: data, options: []),
let dict = json as? [String: Any]
{
dict
} else {
[:]
}
var updated = existing var updated = existing
updated[self.providerKey] = [ updated[self.providerKey] = [
@@ -0,0 +1,22 @@
import ClawdisProtocol
import Foundation
extension AnyCodable {
var stringValue: String? { self.value as? String }
var boolValue: Bool? { self.value as? Bool }
var intValue: Int? { self.value as? Int }
var doubleValue: Double? { self.value as? Double }
var dictionaryValue: [String: AnyCodable]? { self.value as? [String: AnyCodable] }
var arrayValue: [AnyCodable]? { self.value as? [AnyCodable] }
var foundationValue: Any {
switch self.value {
case let dict as [String: AnyCodable]:
dict.mapValues { $0.foundationValue }
case let array as [AnyCodable]:
array.map(\.foundationValue)
default:
self.value
}
}
}
+45
View File
@@ -38,6 +38,7 @@ final class AppState {
var debugPaneEnabled: Bool { var debugPaneEnabled: Bool {
didSet { didSet {
self.ifNotPreview { UserDefaults.standard.set(self.debugPaneEnabled, forKey: "clawdis.debugPaneEnabled") } self.ifNotPreview { UserDefaults.standard.set(self.debugPaneEnabled, forKey: "clawdis.debugPaneEnabled") }
CanvasManager.shared.refreshDebugStatus()
} }
} }
@@ -120,6 +121,18 @@ final class AppState {
forKey: voicePushToTalkEnabledKey) } } forKey: voicePushToTalkEnabledKey) } }
} }
var talkEnabled: Bool {
didSet {
self.ifNotPreview {
UserDefaults.standard.set(self.talkEnabled, forKey: talkEnabledKey)
Task { await TalkModeController.shared.setEnabled(self.talkEnabled) }
}
}
}
/// Gateway-provided UI accent color (hex). Optional; clients provide a default.
var seamColorHex: String?
var iconOverride: IconOverrideSelection { var iconOverride: IconOverrideSelection {
didSet { self.ifNotPreview { UserDefaults.standard.set(self.iconOverride.rawValue, forKey: iconOverrideKey) } } didSet { self.ifNotPreview { UserDefaults.standard.set(self.iconOverride.rawValue, forKey: iconOverrideKey) } }
} }
@@ -215,6 +228,8 @@ final class AppState {
.stringArray(forKey: voiceWakeAdditionalLocalesKey) ?? [] .stringArray(forKey: voiceWakeAdditionalLocalesKey) ?? []
self.voicePushToTalkEnabled = UserDefaults.standard self.voicePushToTalkEnabled = UserDefaults.standard
.object(forKey: voicePushToTalkEnabledKey) as? Bool ?? false .object(forKey: voicePushToTalkEnabledKey) as? Bool ?? false
self.talkEnabled = UserDefaults.standard.bool(forKey: talkEnabledKey)
self.seamColorHex = nil
if let storedHeartbeats = UserDefaults.standard.object(forKey: heartbeatsEnabledKey) as? Bool { if let storedHeartbeats = UserDefaults.standard.object(forKey: heartbeatsEnabledKey) as? Bool {
self.heartbeatsEnabled = storedHeartbeats self.heartbeatsEnabled = storedHeartbeats
} else { } else {
@@ -255,9 +270,13 @@ final class AppState {
if self.swabbleEnabled, !PermissionManager.voiceWakePermissionsGranted() { if self.swabbleEnabled, !PermissionManager.voiceWakePermissionsGranted() {
self.swabbleEnabled = false self.swabbleEnabled = false
} }
if self.talkEnabled, !PermissionManager.voiceWakePermissionsGranted() {
self.talkEnabled = false
}
if !self.isPreview { if !self.isPreview {
Task { await VoiceWakeRuntime.shared.refresh(state: self) } Task { await VoiceWakeRuntime.shared.refresh(state: self) }
Task { await TalkModeController.shared.setEnabled(self.talkEnabled) }
} }
} }
@@ -311,6 +330,31 @@ final class AppState {
Task { await VoiceWakeRuntime.shared.refresh(state: self) } Task { await VoiceWakeRuntime.shared.refresh(state: self) }
} }
func setTalkEnabled(_ enabled: Bool) async {
guard voiceWakeSupported else {
self.talkEnabled = false
await GatewayConnection.shared.talkMode(enabled: false, phase: "disabled")
return
}
self.talkEnabled = enabled
guard !self.isPreview else { return }
if !enabled {
await GatewayConnection.shared.talkMode(enabled: false, phase: "disabled")
return
}
if PermissionManager.voiceWakePermissionsGranted() {
await GatewayConnection.shared.talkMode(enabled: true, phase: "enabled")
return
}
let granted = await PermissionManager.ensureVoiceWakePermissions(interactive: true)
self.talkEnabled = granted
await GatewayConnection.shared.talkMode(enabled: granted, phase: granted ? "enabled" : "denied")
}
// MARK: - Global wake words sync (Gateway-owned) // MARK: - Global wake words sync (Gateway-owned)
func applyGlobalVoiceWakeTriggers(_ triggers: [String]) { func applyGlobalVoiceWakeTriggers(_ triggers: [String]) {
@@ -366,6 +410,7 @@ extension AppState {
state.voiceWakeLocaleID = Locale.current.identifier state.voiceWakeLocaleID = Locale.current.identifier
state.voiceWakeAdditionalLocaleIDs = ["en-US", "de-DE"] state.voiceWakeAdditionalLocaleIDs = ["en-US", "de-DE"]
state.voicePushToTalkEnabled = false state.voicePushToTalkEnabled = false
state.talkEnabled = false
state.iconOverride = .system state.iconOverride = .system
state.heartbeatsEnabled = true state.heartbeatsEnabled = true
state.connectionMode = .local state.connectionMode = .local
@@ -0,0 +1,22 @@
import Foundation
enum AsyncTimeout {
static func withTimeout<T: Sendable>(
seconds: Double,
onTimeout: @escaping @Sendable () -> Error,
operation: @escaping @Sendable () async throws -> T) async throws -> T
{
let clamped = max(0, seconds)
return try await withThrowingTaskGroup(of: T.self) { group in
group.addTask { try await operation() }
group.addTask {
try await Task.sleep(nanoseconds: UInt64(clamped * 1_000_000_000))
throw onTimeout()
}
let result = try await group.next()
group.cancelAll()
if let result { return result }
throw onTimeout()
}
}
}
@@ -503,3 +503,40 @@ enum BridgePairingApprover {
} }
} }
} }
#if DEBUG
extension BridgeServer {
func exerciseForTesting() async {
let conn = NWConnection(to: .hostPort(host: "127.0.0.1", port: 22), using: .tcp)
let handler = BridgeConnectionHandler(connection: conn, logger: self.logger)
self.connections["node-1"] = handler
self.nodeInfoById["node-1"] = BridgeNodeInfo(
nodeId: "node-1",
displayName: "Node One",
platform: "macOS",
version: "1.0.0",
deviceFamily: "Mac",
modelIdentifier: "MacBookPro18,1",
remoteAddress: "127.0.0.1",
caps: ["chat", "voice"])
_ = self.connectedNodeIds()
_ = self.connectedNodes()
self.handleListenerState(.ready)
self.handleListenerState(.failed(NWError.posix(.ECONNREFUSED)))
self.handleListenerState(.waiting(NWError.posix(.ETIMEDOUT)))
self.handleListenerState(.cancelled)
self.handleListenerState(.setup)
let subscribe = BridgeEventFrame(event: "chat.subscribe", payloadJSON: "{\"sessionKey\":\"main\"}")
await self.handleEvent(nodeId: "node-1", evt: subscribe)
let unsubscribe = BridgeEventFrame(event: "chat.unsubscribe", payloadJSON: "{\"sessionKey\":\"main\"}")
await self.handleEvent(nodeId: "node-1", evt: unsubscribe)
let invalid = BridgeRPCRequest(id: "req-1", method: "invalid.method", paramsJSON: nil)
_ = await self.handleRequest(nodeId: "node-1", req: invalid)
}
}
#endif
@@ -0,0 +1,102 @@
import Foundation
@MainActor
enum CLIInstaller {
private static func embeddedHelperURL() -> URL {
Bundle.main.bundleURL.appendingPathComponent("Contents/Resources/Relay/clawdis")
}
static func installedLocation() -> String? {
self.installedLocation(
searchPaths: cliHelperSearchPaths,
embeddedHelper: self.embeddedHelperURL(),
fileManager: .default)
}
static func installedLocation(
searchPaths: [String],
embeddedHelper: URL,
fileManager: FileManager) -> String?
{
let embedded = embeddedHelper.resolvingSymlinksInPath()
for basePath in searchPaths {
let candidate = URL(fileURLWithPath: basePath).appendingPathComponent("clawdis").path
var isDirectory: ObjCBool = false
guard fileManager.fileExists(atPath: candidate, isDirectory: &isDirectory),
!isDirectory.boolValue
else {
continue
}
guard fileManager.isExecutableFile(atPath: candidate) else { continue }
let resolved = URL(fileURLWithPath: candidate).resolvingSymlinksInPath()
if resolved == embedded {
return candidate
}
}
return nil
}
static func isInstalled() -> Bool {
self.installedLocation() != nil
}
static func install(statusHandler: @escaping @Sendable (String) async -> Void) async {
let helper = self.embeddedHelperURL()
guard FileManager.default.isExecutableFile(atPath: helper.path) else {
await statusHandler(
"Embedded CLI missing in bundle; repackage via scripts/package-mac-app.sh " +
"(or restart-mac.sh without SKIP_GATEWAY_PACKAGE=1).")
return
}
let targets = cliHelperSearchPaths.map { "\($0)/clawdis" }
let result = await self.privilegedSymlink(source: helper.path, targets: targets)
await statusHandler(result)
}
private static func privilegedSymlink(source: String, targets: [String]) async -> String {
let escapedSource = self.shellEscape(source)
let targetList = targets.map(self.shellEscape).joined(separator: " ")
let cmds = [
"mkdir -p /usr/local/bin /opt/homebrew/bin",
targets.map { "ln -sf \(escapedSource) \($0)" }.joined(separator: "; "),
].joined(separator: "; ")
let script = """
do shell script "\(cmds)" with administrator privileges
"""
let proc = Process()
proc.executableURL = URL(fileURLWithPath: "/usr/bin/osascript")
proc.arguments = ["-e", script]
let pipe = Pipe()
proc.standardOutput = pipe
proc.standardError = pipe
do {
try proc.run()
proc.waitUntilExit()
let data = pipe.fileHandleForReading.readToEndSafely()
let output = String(data: data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if proc.terminationStatus == 0 {
return output.isEmpty ? "CLI helper linked into \(targetList)" : output
}
if output.lowercased().contains("user canceled") {
return "Install canceled"
}
return "Failed to install CLI helper: \(output)"
} catch {
return "Failed to run installer: \(error.localizedDescription)"
}
}
private static func shellEscape(_ path: String) -> String {
"'" + path.replacingOccurrences(of: "'", with: "'\"'\"'") + "'"
}
}
@@ -79,7 +79,14 @@ actor CameraCaptureService {
} }
withExtendedLifetime(delegate) {} withExtendedLifetime(delegate) {}
let res = try JPEGTranscoder.transcodeToJPEG(imageData: rawData, maxWidthPx: maxWidth, quality: quality) let maxPayloadBytes = 5 * 1024 * 1024
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
let maxEncodedBytes = (maxPayloadBytes / 4) * 3
let res = try JPEGTranscoder.transcodeToJPEG(
imageData: rawData,
maxWidthPx: maxWidth,
quality: quality,
maxBytes: maxEncodedBytes)
return (data: res.data, size: CGSize(width: res.widthPx, height: res.heightPx)) return (data: res.data, size: CGSize(width: res.widthPx, height: res.heightPx))
} }
@@ -0,0 +1,147 @@
import AppKit
import ClawdisIPC
import ClawdisKit
import Foundation
import WebKit
final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "clawdisCanvasA2UIAction"
private let sessionKey: String
init(sessionKey: String) {
self.sessionKey = sessionKey
super.init()
}
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
guard message.name == Self.messageName else { return }
// Only accept actions from local Canvas content (not arbitrary web pages).
guard let webView = message.webView, let url = webView.url else { return }
if url.scheme == CanvasScheme.scheme {
// ok
} else if Self.isLocalNetworkCanvasURL(url) {
// ok
} else {
return
}
let body: [String: Any] = {
if let dict = message.body as? [String: Any] { return dict }
if let dict = message.body as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !body.isEmpty else { return }
let userActionAny = body["userAction"] ?? body
let userAction: [String: Any] = {
if let dict = userActionAny as? [String: Any] { return dict }
if let dict = userActionAny as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !userAction.isEmpty else { return }
guard let name = ClawdisCanvasA2UIAction.extractActionName(userAction) else { return }
let actionId =
(userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
?? UUID().uuidString
canvasWindowLogger.info("A2UI action \(name, privacy: .public) session=\(self.sessionKey, privacy: .public)")
let surfaceId = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
.nonEmpty ?? "main"
let sourceComponentId = (userAction["sourceComponentId"] as? String)?
.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "-"
let instanceId = InstanceIdentity.instanceId.lowercased()
let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"])
// Token-efficient and unambiguous. The agent should treat this as a UI event and (by default) update Canvas.
let messageContext = ClawdisCanvasA2UIAction.AgentMessageContext(
actionName: name,
session: .init(key: self.sessionKey, surfaceId: surfaceId),
component: .init(id: sourceComponentId, host: InstanceIdentity.displayName, instanceId: instanceId),
contextJSON: contextJSON)
let text = ClawdisCanvasA2UIAction.formatAgentMessage(messageContext)
Task { [weak webView] in
if AppStateStore.shared.connectionMode == .local {
GatewayProcessManager.shared.setActive(true)
}
let result = await GatewayConnection.shared.sendAgent(GatewayAgentInvocation(
message: text,
sessionKey: self.sessionKey,
thinking: "low",
deliver: false,
to: nil,
channel: .last,
idempotencyKey: actionId))
await MainActor.run {
guard let webView else { return }
let js = ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(
actionId: actionId,
ok: result.ok,
error: result.error)
webView.evaluateJavaScript(js) { _, _ in }
}
if !result.ok {
canvasWindowLogger.error(
"""
A2UI action send failed name=\(name, privacy: .public) \
error=\(result.error ?? "unknown", privacy: .public)
""")
}
}
}
static func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return false
}
guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else {
return false
}
if host == "localhost" { return true }
if host.hasSuffix(".local") { return true }
if host.hasSuffix(".ts.net") { return true }
if host.hasSuffix(".tailscale.net") { return true }
if !host.contains("."), !host.contains(":") { return true }
if let ipv4 = Self.parseIPv4(host) {
return Self.isLocalNetworkIPv4(ipv4)
}
return false
}
static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? {
let parts = host.split(separator: ".", omittingEmptySubsequences: false)
guard parts.count == 4 else { return nil }
let bytes: [UInt8] = parts.compactMap { UInt8($0) }
guard bytes.count == 4 else { return nil }
return (bytes[0], bytes[1], bytes[2], bytes[3])
}
static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool {
let (a, b, _, _) = ip
if a == 10 { return true }
if a == 172, (16...31).contains(Int(b)) { return true }
if a == 192, b == 168 { return true }
if a == 127 { return true }
if a == 169, b == 254 { return true }
if a == 100, (64...127).contains(Int(b)) { return true }
return false
}
// Formatting helpers live in ClawdisKit (`ClawdisCanvasA2UIAction`).
}
@@ -0,0 +1,225 @@
import AppKit
import QuartzCore
final class HoverChromeContainerView: NSView {
private let content: NSView
private let chrome: CanvasChromeOverlayView
private var tracking: NSTrackingArea?
var onClose: (() -> Void)?
init(containing content: NSView) {
self.content = content
self.chrome = CanvasChromeOverlayView(frame: .zero)
super.init(frame: .zero)
self.wantsLayer = true
self.layer?.cornerRadius = 12
self.layer?.masksToBounds = true
self.layer?.backgroundColor = NSColor.windowBackgroundColor.cgColor
self.content.translatesAutoresizingMaskIntoConstraints = false
self.addSubview(self.content)
self.chrome.translatesAutoresizingMaskIntoConstraints = false
self.chrome.alphaValue = 0
self.chrome.onClose = { [weak self] in self?.onClose?() }
self.addSubview(self.chrome)
NSLayoutConstraint.activate([
self.content.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.content.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.content.topAnchor.constraint(equalTo: self.topAnchor),
self.content.bottomAnchor.constraint(equalTo: self.bottomAnchor),
self.chrome.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.chrome.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.chrome.topAnchor.constraint(equalTo: self.topAnchor),
self.chrome.bottomAnchor.constraint(equalTo: self.bottomAnchor),
])
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
override func updateTrackingAreas() {
super.updateTrackingAreas()
if let tracking {
self.removeTrackingArea(tracking)
}
let area = NSTrackingArea(
rect: self.bounds,
options: [.activeAlways, .mouseEnteredAndExited, .inVisibleRect],
owner: self,
userInfo: nil)
self.addTrackingArea(area)
self.tracking = area
}
private final class CanvasDragHandleView: NSView {
override func mouseDown(with event: NSEvent) {
self.window?.performDrag(with: event)
}
override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true }
}
private final class CanvasResizeHandleView: NSView {
private var startPoint: NSPoint = .zero
private var startFrame: NSRect = .zero
override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true }
override func mouseDown(with event: NSEvent) {
guard let window else { return }
_ = window.makeFirstResponder(self)
self.startPoint = NSEvent.mouseLocation
self.startFrame = window.frame
super.mouseDown(with: event)
}
override func mouseDragged(with _: NSEvent) {
guard let window else { return }
let current = NSEvent.mouseLocation
let dx = current.x - self.startPoint.x
let dy = current.y - self.startPoint.y
var frame = self.startFrame
frame.size.width = max(CanvasLayout.minPanelSize.width, frame.size.width + dx)
frame.origin.y += dy
frame.size.height = max(CanvasLayout.minPanelSize.height, frame.size.height - dy)
if let screen = window.screen {
frame = CanvasWindowController.constrainFrame(frame, toVisibleFrame: screen.visibleFrame)
}
window.setFrame(frame, display: true)
}
}
private final class CanvasChromeOverlayView: NSView {
var onClose: (() -> Void)?
private let dragHandle = CanvasDragHandleView(frame: .zero)
private let resizeHandle = CanvasResizeHandleView(frame: .zero)
private final class PassthroughVisualEffectView: NSVisualEffectView {
override func hitTest(_: NSPoint) -> NSView? { nil }
}
private let closeBackground: NSVisualEffectView = {
let v = PassthroughVisualEffectView(frame: .zero)
v.material = .hudWindow
v.blendingMode = .withinWindow
v.state = .active
v.appearance = NSAppearance(named: .vibrantDark)
v.wantsLayer = true
v.layer?.cornerRadius = 10
v.layer?.masksToBounds = true
v.layer?.borderWidth = 1
v.layer?.borderColor = NSColor.white.withAlphaComponent(0.22).cgColor
v.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.28).cgColor
v.layer?.shadowColor = NSColor.black.withAlphaComponent(0.35).cgColor
v.layer?.shadowOpacity = 0.35
v.layer?.shadowRadius = 8
v.layer?.shadowOffset = .zero
return v
}()
private let closeButton: NSButton = {
let cfg = NSImage.SymbolConfiguration(pointSize: 8, weight: .semibold)
let img = NSImage(systemSymbolName: "xmark", accessibilityDescription: "Close")?
.withSymbolConfiguration(cfg)
?? NSImage(size: NSSize(width: 18, height: 18))
let btn = NSButton(image: img, target: nil, action: nil)
btn.isBordered = false
btn.bezelStyle = .regularSquare
btn.imageScaling = .scaleProportionallyDown
btn.contentTintColor = NSColor.white.withAlphaComponent(0.92)
btn.toolTip = "Close"
return btn
}()
override init(frame frameRect: NSRect) {
super.init(frame: frameRect)
self.wantsLayer = true
self.layer?.cornerRadius = 12
self.layer?.masksToBounds = true
self.layer?.borderWidth = 1
self.layer?.borderColor = NSColor.black.withAlphaComponent(0.18).cgColor
self.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.02).cgColor
self.dragHandle.translatesAutoresizingMaskIntoConstraints = false
self.dragHandle.wantsLayer = true
self.dragHandle.layer?.backgroundColor = NSColor.clear.cgColor
self.addSubview(self.dragHandle)
self.resizeHandle.translatesAutoresizingMaskIntoConstraints = false
self.resizeHandle.wantsLayer = true
self.resizeHandle.layer?.backgroundColor = NSColor.clear.cgColor
self.addSubview(self.resizeHandle)
self.closeBackground.translatesAutoresizingMaskIntoConstraints = false
self.addSubview(self.closeBackground)
self.closeButton.translatesAutoresizingMaskIntoConstraints = false
self.closeButton.target = self
self.closeButton.action = #selector(self.handleClose)
self.addSubview(self.closeButton)
NSLayoutConstraint.activate([
self.dragHandle.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.dragHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.dragHandle.topAnchor.constraint(equalTo: self.topAnchor),
self.dragHandle.heightAnchor.constraint(equalToConstant: 30),
self.closeBackground.centerXAnchor.constraint(equalTo: self.closeButton.centerXAnchor),
self.closeBackground.centerYAnchor.constraint(equalTo: self.closeButton.centerYAnchor),
self.closeBackground.widthAnchor.constraint(equalToConstant: 20),
self.closeBackground.heightAnchor.constraint(equalToConstant: 20),
self.closeButton.trailingAnchor.constraint(equalTo: self.trailingAnchor, constant: -8),
self.closeButton.topAnchor.constraint(equalTo: self.topAnchor, constant: 8),
self.closeButton.widthAnchor.constraint(equalToConstant: 16),
self.closeButton.heightAnchor.constraint(equalToConstant: 16),
self.resizeHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.resizeHandle.bottomAnchor.constraint(equalTo: self.bottomAnchor),
self.resizeHandle.widthAnchor.constraint(equalToConstant: 18),
self.resizeHandle.heightAnchor.constraint(equalToConstant: 18),
])
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
override func hitTest(_ point: NSPoint) -> NSView? {
// When the chrome is hidden, do not intercept any mouse events (let the WKWebView receive them).
guard self.alphaValue > 0.02 else { return nil }
if self.closeButton.frame.contains(point) { return self.closeButton }
if self.dragHandle.frame.contains(point) { return self.dragHandle }
if self.resizeHandle.frame.contains(point) { return self.resizeHandle }
return nil
}
@objc private func handleClose() {
self.onClose?()
}
}
override func mouseEntered(with _: NSEvent) {
NSAnimationContext.runAnimationGroup { ctx in
ctx.duration = 0.12
ctx.timingFunction = CAMediaTimingFunction(name: .easeOut)
self.chrome.animator().alphaValue = 1
}
}
override func mouseExited(with _: NSEvent) {
NSAnimationContext.runAnimationGroup { ctx in
ctx.duration = 0.16
ctx.timingFunction = CAMediaTimingFunction(name: .easeOut)
self.chrome.animator().alphaValue = 0
}
}
}
@@ -11,6 +11,12 @@ final class CanvasManager {
private var panelController: CanvasWindowController? private var panelController: CanvasWindowController?
private var panelSessionKey: String? private var panelSessionKey: String?
private var lastAutoA2UIUrl: String?
private var gatewayWatchTask: Task<Void, Never>?
private init() {
self.startGatewayObserver()
}
var onPanelVisibilityChanged: ((Bool) -> Void)? var onPanelVisibilityChanged: ((Bool) -> Void)?
@@ -50,6 +56,7 @@ final class CanvasManager {
} }
controller.presentAnchoredPanel(anchorProvider: anchorProvider) controller.presentAnchoredPanel(anchorProvider: anchorProvider)
controller.applyPreferredPlacement(placement) controller.applyPreferredPlacement(placement)
self.refreshDebugStatus()
// Existing session: only navigate when an explicit target was provided. // Existing session: only navigate when an explicit target was provided.
if let normalizedTarget { if let normalizedTarget {
@@ -60,6 +67,7 @@ final class CanvasManager {
effectiveTarget: normalizedTarget) effectiveTarget: normalizedTarget)
} }
self.maybeAutoNavigateToA2UIAsync(controller: controller)
return CanvasShowResult( return CanvasShowResult(
directory: controller.directoryPath, directory: controller.directoryPath,
target: target, target: target,
@@ -93,6 +101,10 @@ final class CanvasManager {
Self.logger.debug("showDetailed showCanvas effectiveTarget=\(effectiveTarget, privacy: .public)") Self.logger.debug("showDetailed showCanvas effectiveTarget=\(effectiveTarget, privacy: .public)")
controller.showCanvas(path: effectiveTarget) controller.showCanvas(path: effectiveTarget)
Self.logger.debug("showDetailed showCanvas done") Self.logger.debug("showDetailed showCanvas done")
if normalizedTarget == nil {
self.maybeAutoNavigateToA2UIAsync(controller: controller)
}
self.refreshDebugStatus()
return self.makeShowResult( return self.makeShowResult(
directory: controller.directoryPath, directory: controller.directoryPath,
@@ -124,6 +136,63 @@ final class CanvasManager {
return try await controller.snapshot(to: outPath) return try await controller.snapshot(to: outPath)
} }
// MARK: - Gateway A2UI auto-nav
private func startGatewayObserver() {
self.gatewayWatchTask?.cancel()
self.gatewayWatchTask = Task { [weak self] in
guard let self else { return }
let stream = await GatewayConnection.shared.subscribe(bufferingNewest: 1)
for await push in stream {
self.handleGatewayPush(push)
}
}
}
private func handleGatewayPush(_ push: GatewayPush) {
guard case let .snapshot(snapshot) = push else { return }
let a2uiUrl = Self.resolveA2UIHostUrl(from: snapshot.canvashosturl)
guard let controller = self.panelController else { return }
self.maybeAutoNavigateToA2UI(controller: controller, a2uiUrl: a2uiUrl)
}
private func maybeAutoNavigateToA2UIAsync(controller: CanvasWindowController) {
Task { [weak self] in
guard let self else { return }
let a2uiUrl = await self.resolveA2UIHostUrl()
await MainActor.run {
guard self.panelController === controller else { return }
self.maybeAutoNavigateToA2UI(controller: controller, a2uiUrl: a2uiUrl)
}
}
}
private func maybeAutoNavigateToA2UI(controller: CanvasWindowController, a2uiUrl: String?) {
guard let a2uiUrl else { return }
guard controller.shouldAutoNavigateToA2UI(lastAutoTarget: self.lastAutoA2UIUrl) else { return }
controller.load(target: a2uiUrl)
self.lastAutoA2UIUrl = a2uiUrl
}
private func resolveA2UIHostUrl() async -> String? {
let raw = await GatewayConnection.shared.canvasHostUrl()
return Self.resolveA2UIHostUrl(from: raw)
}
func refreshDebugStatus() {
guard let controller = self.panelController else { return }
let enabled = AppStateStore.shared.debugPaneEnabled
let title = GatewayProcessManager.shared.status.label
let subtitle = AppStateStore.shared.connectionMode.rawValue
controller.updateDebugStatus(enabled: enabled, title: title, subtitle: subtitle)
}
private static func resolveA2UIHostUrl(from raw: String?) -> String? {
let trimmed = raw?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
return base.appendingPathComponent("__clawdis__/a2ui/").absoluteString + "?platform=macos"
}
// MARK: - Anchoring // MARK: - Anchoring
private static func mouseAnchorProvider() -> NSRect? { private static func mouseAnchorProvider() -> NSRect? {
@@ -240,3 +240,20 @@ final class CanvasSchemeHandler: NSObject, WKURLSchemeHandler {
} }
} }
} }
#if DEBUG
extension CanvasSchemeHandler {
func _testResponse(for url: URL) -> (mime: String, data: Data) {
let response = self.response(for: url)
return (response.mime, response.data)
}
func _testResolveFileURL(sessionRoot: URL, requestPath: String) -> URL? {
self.resolveFileURL(sessionRoot: sessionRoot, requestPath: requestPath)
}
func _testTextEncodingName(for mimeType: String) -> String? {
self.textEncodingName(forMimeType: mimeType)
}
}
#endif
+2 -937
View File
@@ -1,14 +1,8 @@
import AppKit import AppKit
import ClawdisIPC
import ClawdisKit
import Foundation
import OSLog
import QuartzCore
import WebKit
private let canvasWindowLogger = Logger(subsystem: "com.steipete.clawdis", category: "Canvas") let canvasWindowLogger = Logger(subsystem: "com.steipete.clawdis", category: "Canvas")
private enum CanvasLayout { enum CanvasLayout {
static let panelSize = NSSize(width: 520, height: 680) static let panelSize = NSSize(width: 520, height: 680)
static let windowSize = NSSize(width: 1120, height: 840) static let windowSize = NSSize(width: 1120, height: 840)
static let anchorPadding: CGFloat = 8 static let anchorPadding: CGFloat = 8
@@ -30,932 +24,3 @@ enum CanvasPresentation {
return false return false
} }
} }
@MainActor
final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NSWindowDelegate {
private let sessionKey: String
private let root: URL
private let sessionDir: URL
private let schemeHandler: CanvasSchemeHandler
private let webView: WKWebView
private var a2uiActionMessageHandler: CanvasA2UIActionMessageHandler?
private let watcher: CanvasFileWatcher
private let container: HoverChromeContainerView
let presentation: CanvasPresentation
private var preferredPlacement: CanvasPlacement?
var onVisibilityChanged: ((Bool) -> Void)?
init(sessionKey: String, root: URL, presentation: CanvasPresentation) throws {
self.sessionKey = sessionKey
self.root = root
self.presentation = presentation
canvasWindowLogger.debug("CanvasWindowController init start session=\(sessionKey, privacy: .public)")
let safeSessionKey = CanvasWindowController.sanitizeSessionKey(sessionKey)
canvasWindowLogger.debug("CanvasWindowController init sanitized session=\(safeSessionKey, privacy: .public)")
self.sessionDir = root.appendingPathComponent(safeSessionKey, isDirectory: true)
try FileManager.default.createDirectory(at: self.sessionDir, withIntermediateDirectories: true)
canvasWindowLogger.debug("CanvasWindowController init session dir ready")
self.schemeHandler = CanvasSchemeHandler(root: root)
canvasWindowLogger.debug("CanvasWindowController init scheme handler ready")
let config = WKWebViewConfiguration()
config.userContentController = WKUserContentController()
config.preferences.isElementFullscreenEnabled = true
config.preferences.setValue(true, forKey: "developerExtrasEnabled")
canvasWindowLogger.debug("CanvasWindowController init config ready")
config.setURLSchemeHandler(self.schemeHandler, forURLScheme: CanvasScheme.scheme)
canvasWindowLogger.debug("CanvasWindowController init scheme handler installed")
// Bridge A2UI "a2uiaction" DOM events back into the native agent loop.
//
// Prefer WKScriptMessageHandler when WebKit exposes it, otherwise fall back to an unattended deep link
// (includes the app-generated key so it won't prompt).
canvasWindowLogger.debug("CanvasWindowController init building A2UI bridge script")
let deepLinkKey = DeepLinkHandler.currentCanvasKey()
let injectedSessionKey = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "main"
let bridgeScript = """
(() => {
try {
if (location.protocol !== '\(CanvasScheme.scheme):') return;
if (globalThis.__clawdisA2UIBridgeInstalled) return;
globalThis.__clawdisA2UIBridgeInstalled = true;
const deepLinkKey = \(Self.jsStringLiteral(deepLinkKey));
const sessionKey = \(Self.jsStringLiteral(injectedSessionKey));
const machineName = \(Self.jsStringLiteral(InstanceIdentity.displayName));
const instanceId = \(Self.jsStringLiteral(InstanceIdentity.instanceId));
globalThis.addEventListener('a2uiaction', (evt) => {
try {
const payload = evt?.detail ?? evt?.payload ?? null;
if (!payload || payload.eventType !== 'a2ui.action') return;
const action = payload.action ?? null;
const name = action?.name ?? '';
if (!name) return;
const context = Array.isArray(action?.context) ? action.context : [];
const userAction = {
id: (globalThis.crypto?.randomUUID?.() ?? String(Date.now())),
name,
surfaceId: payload.surfaceId ?? 'main',
sourceComponentId: payload.sourceComponentId ?? '',
dataContextPath: payload.dataContextPath ?? '',
timestamp: new Date().toISOString(),
...(context.length ? { context } : {}),
};
const handler = globalThis.webkit?.messageHandlers?.clawdisCanvasA2UIAction;
// If the bundled A2UI shell is present, let it forward actions so we keep its richer
// context resolution (data model path lookups, surface detection, etc.).
const hasBundledA2UIHost = !!globalThis.clawdisA2UI || !!document.querySelector('clawdis-a2ui-host');
if (hasBundledA2UIHost && handler?.postMessage) return;
// Otherwise, forward directly when possible.
if (!hasBundledA2UIHost && handler?.postMessage) {
handler.postMessage({ userAction });
return;
}
const ctx = userAction.context ? (' ctx=' + JSON.stringify(userAction.context)) : '';
const message =
'CANVAS_A2UI action=' + userAction.name +
' session=' + sessionKey +
' surface=' + userAction.surfaceId +
' component=' + (userAction.sourceComponentId || '-') +
' host=' + machineName.replace(/\\s+/g, '_') +
' instance=' + instanceId +
ctx +
' default=update_canvas';
const params = new URLSearchParams();
params.set('message', message);
params.set('sessionKey', sessionKey);
params.set('thinking', 'low');
params.set('deliver', 'false');
params.set('channel', 'last');
params.set('key', deepLinkKey);
location.href = 'clawdis://agent?' + params.toString();
} catch {}
}, true);
} catch {}
})();
"""
config.userContentController.addUserScript(
WKUserScript(source: bridgeScript, injectionTime: .atDocumentStart, forMainFrameOnly: true))
canvasWindowLogger.debug("CanvasWindowController init A2UI bridge installed")
canvasWindowLogger.debug("CanvasWindowController init creating WKWebView")
self.webView = WKWebView(frame: .zero, configuration: config)
// Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays.
self.webView.setValue(true, forKey: "drawsBackground")
let sessionDir = self.sessionDir
let webView = self.webView
self.watcher = CanvasFileWatcher(url: sessionDir) { [weak webView] in
Task { @MainActor in
guard let webView else { return }
// Only auto-reload when we are showing local canvas content.
guard webView.url?.scheme == CanvasScheme.scheme else { return }
let path = webView.url?.path ?? ""
if path == "/" || path.isEmpty {
let indexA = sessionDir.appendingPathComponent("index.html", isDirectory: false)
let indexB = sessionDir.appendingPathComponent("index.htm", isDirectory: false)
if !FileManager.default.fileExists(atPath: indexA.path),
!FileManager.default.fileExists(atPath: indexB.path)
{
return
}
}
webView.reload()
}
}
self.container = HoverChromeContainerView(containing: self.webView)
let window = Self.makeWindow(for: presentation, contentView: self.container)
canvasWindowLogger.debug("CanvasWindowController init makeWindow done")
super.init(window: window)
let handler = CanvasA2UIActionMessageHandler(sessionKey: sessionKey)
self.a2uiActionMessageHandler = handler
self.webView.configuration.userContentController.add(handler, name: CanvasA2UIActionMessageHandler.messageName)
self.webView.navigationDelegate = self
self.window?.delegate = self
self.container.onClose = { [weak self] in
self?.hideCanvas()
}
self.watcher.start()
canvasWindowLogger.debug("CanvasWindowController init done")
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
@MainActor deinit {
self.webView.configuration.userContentController
.removeScriptMessageHandler(forName: CanvasA2UIActionMessageHandler.messageName)
self.watcher.stop()
}
func applyPreferredPlacement(_ placement: CanvasPlacement?) {
self.preferredPlacement = placement
}
func showCanvas(path: String? = nil) {
if case let .panel(anchorProvider) = self.presentation {
self.presentAnchoredPanel(anchorProvider: anchorProvider)
if let path {
self.load(target: path)
}
return
}
self.showWindow(nil)
self.window?.makeKeyAndOrderFront(nil)
NSApp.activate(ignoringOtherApps: true)
if let path {
self.load(target: path)
}
self.onVisibilityChanged?(true)
}
func hideCanvas() {
if case .panel = self.presentation {
self.persistFrameIfPanel()
}
self.window?.orderOut(nil)
self.onVisibilityChanged?(false)
}
func load(target: String) {
let trimmed = target.trimmingCharacters(in: .whitespacesAndNewlines)
if let url = URL(string: trimmed), let scheme = url.scheme?.lowercased() {
if scheme == "https" || scheme == "http" {
canvasWindowLogger.debug("canvas load url \(url.absoluteString, privacy: .public)")
self.webView.load(URLRequest(url: url))
return
}
if scheme == "file" {
canvasWindowLogger.debug("canvas load file \(url.absoluteString, privacy: .public)")
self.loadFile(url)
return
}
}
// Convenience: absolute file paths resolve as local files when they exist.
// (Avoid treating Canvas routes like "/" as filesystem paths.)
if trimmed.hasPrefix("/") {
var isDir: ObjCBool = false
if FileManager.default.fileExists(atPath: trimmed, isDirectory: &isDir), !isDir.boolValue {
let url = URL(fileURLWithPath: trimmed)
canvasWindowLogger.debug("canvas load file \(url.absoluteString, privacy: .public)")
self.loadFile(url)
return
}
}
guard let url = CanvasScheme.makeURL(
session: CanvasWindowController.sanitizeSessionKey(self.sessionKey),
path: trimmed)
else {
canvasWindowLogger
.error(
"invalid canvas url session=\(self.sessionKey, privacy: .public) path=\(trimmed, privacy: .public)")
return
}
canvasWindowLogger.debug("canvas load canvas \(url.absoluteString, privacy: .public)")
self.webView.load(URLRequest(url: url))
}
private func loadFile(_ url: URL) {
let fileURL = url.isFileURL ? url : URL(fileURLWithPath: url.path)
let accessDir = fileURL.deletingLastPathComponent()
self.webView.loadFileURL(fileURL, allowingReadAccessTo: accessDir)
}
func eval(javaScript: String) async throws -> String {
try await withCheckedThrowingContinuation { cont in
self.webView.evaluateJavaScript(javaScript) { result, error in
if let error {
cont.resume(throwing: error)
return
}
if let result {
cont.resume(returning: String(describing: result))
} else {
cont.resume(returning: "")
}
}
}
}
func snapshot(to outPath: String?) async throws -> String {
let image: NSImage = try await withCheckedThrowingContinuation { cont in
self.webView.takeSnapshot(with: nil) { image, error in
if let error {
cont.resume(throwing: error)
return
}
guard let image else {
cont.resume(throwing: NSError(domain: "Canvas", code: 11, userInfo: [
NSLocalizedDescriptionKey: "snapshot returned nil image",
]))
return
}
cont.resume(returning: image)
}
}
guard let tiff = image.tiffRepresentation,
let rep = NSBitmapImageRep(data: tiff),
let png = rep.representation(using: .png, properties: [:])
else {
throw NSError(domain: "Canvas", code: 12, userInfo: [
NSLocalizedDescriptionKey: "failed to encode png",
])
}
let path: String
if let outPath, !outPath.isEmpty {
path = outPath
} else {
let ts = Int(Date().timeIntervalSince1970)
path = "/tmp/clawdis-canvas-\(CanvasWindowController.sanitizeSessionKey(self.sessionKey))-\(ts).png"
}
try png.write(to: URL(fileURLWithPath: path), options: [.atomic])
return path
}
var directoryPath: String {
self.sessionDir.path
}
// MARK: - Window
private static func makeWindow(for presentation: CanvasPresentation, contentView: NSView) -> NSWindow {
switch presentation {
case .window:
let window = NSWindow(
contentRect: NSRect(origin: .zero, size: CanvasLayout.windowSize),
styleMask: [.titled, .closable, .resizable, .miniaturizable],
backing: .buffered,
defer: false)
window.title = "Clawdis Canvas"
window.isReleasedWhenClosed = false
window.contentView = contentView
window.center()
window.minSize = NSSize(width: 880, height: 680)
return window
case .panel:
let panel = CanvasPanel(
contentRect: NSRect(origin: .zero, size: CanvasLayout.panelSize),
styleMask: [.borderless, .resizable],
backing: .buffered,
defer: false)
// Keep Canvas below the Voice Wake overlay panel.
panel.level = NSWindow.Level(rawValue: NSWindow.Level.statusBar.rawValue - 1)
panel.hasShadow = true
panel.isMovable = false
panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary]
panel.titleVisibility = .hidden
panel.titlebarAppearsTransparent = true
panel.backgroundColor = .clear
panel.isOpaque = false
panel.contentView = contentView
panel.becomesKeyOnlyIfNeeded = true
panel.hidesOnDeactivate = false
panel.minSize = CanvasLayout.minPanelSize
return panel
}
}
func presentAnchoredPanel(anchorProvider: @escaping () -> NSRect?) {
guard case .panel = self.presentation, let window else { return }
self.repositionPanel(using: anchorProvider)
window.makeKeyAndOrderFront(nil)
NSApp.activate(ignoringOtherApps: true)
window.makeFirstResponder(self.webView)
VoiceWakeOverlayController.shared.bringToFrontIfVisible()
self.onVisibilityChanged?(true)
}
private func repositionPanel(using anchorProvider: () -> NSRect?) {
guard let panel = self.window else { return }
let anchor = anchorProvider()
let targetScreen = Self.screen(forAnchor: anchor)
?? Self.screenContainingMouseCursor()
?? panel.screen
?? NSScreen.main
?? NSScreen.screens.first
let restored = Self.loadRestoredFrame(sessionKey: self.sessionKey)
let restoredIsValid = if let restored, let targetScreen {
Self.isFrameMeaningfullyVisible(restored, on: targetScreen)
} else {
restored != nil
}
var frame = if let restored, restoredIsValid {
restored
} else {
Self.defaultTopRightFrame(panel: panel, screen: targetScreen)
}
// Apply agent placement as partial overrides:
// - If agent provides x/y, override origin.
// - If agent provides width/height, override size.
// - If agent provides only size, keep the remembered origin.
if let placement = self.preferredPlacement {
if let x = placement.x { frame.origin.x = x }
if let y = placement.y { frame.origin.y = y }
if let w = placement.width { frame.size.width = max(CanvasLayout.minPanelSize.width, CGFloat(w)) }
if let h = placement.height { frame.size.height = max(CanvasLayout.minPanelSize.height, CGFloat(h)) }
}
self.setPanelFrame(frame, on: targetScreen)
}
private static func defaultTopRightFrame(panel: NSWindow, screen: NSScreen?) -> NSRect {
let w = max(CanvasLayout.minPanelSize.width, panel.frame.width)
let h = max(CanvasLayout.minPanelSize.height, panel.frame.height)
return WindowPlacement.topRightFrame(
size: NSSize(width: w, height: h),
padding: CanvasLayout.defaultPadding,
on: screen)
}
private func setPanelFrame(_ frame: NSRect, on screen: NSScreen?) {
guard let panel = self.window else { return }
guard let s = screen ?? panel.screen ?? NSScreen.main ?? NSScreen.screens.first else {
panel.setFrame(frame, display: false)
self.persistFrameIfPanel()
return
}
let constrained = Self.constrainFrame(frame, toVisibleFrame: s.visibleFrame)
panel.setFrame(constrained, display: false)
self.persistFrameIfPanel()
}
private static func screen(forAnchor anchor: NSRect?) -> NSScreen? {
guard let anchor else { return nil }
let center = NSPoint(x: anchor.midX, y: anchor.midY)
return NSScreen.screens.first { screen in
screen.frame.contains(anchor.origin) || screen.frame.contains(center)
}
}
private static func screenContainingMouseCursor() -> NSScreen? {
let point = NSEvent.mouseLocation
return NSScreen.screens.first { $0.frame.contains(point) }
}
private static func isFrameMeaningfullyVisible(_ frame: NSRect, on screen: NSScreen) -> Bool {
frame.intersects(screen.visibleFrame.insetBy(dx: 12, dy: 12))
}
fileprivate static func constrainFrame(_ frame: NSRect, toVisibleFrame bounds: NSRect) -> NSRect {
if bounds == .zero { return frame }
var next = frame
next.size.width = min(max(CanvasLayout.minPanelSize.width, next.size.width), bounds.width)
next.size.height = min(max(CanvasLayout.minPanelSize.height, next.size.height), bounds.height)
let maxX = bounds.maxX - next.size.width
let maxY = bounds.maxY - next.size.height
next.origin.x = maxX >= bounds.minX ? min(max(next.origin.x, bounds.minX), maxX) : bounds.minX
next.origin.y = maxY >= bounds.minY ? min(max(next.origin.y, bounds.minY), maxY) : bounds.minY
next.origin.x = round(next.origin.x)
next.origin.y = round(next.origin.y)
return next
}
// MARK: - WKNavigationDelegate
@MainActor
func webView(
_: WKWebView,
decidePolicyFor navigationAction: WKNavigationAction,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
{
guard let url = navigationAction.request.url else {
decisionHandler(.cancel)
return
}
let scheme = url.scheme?.lowercased()
// Deep links: allow local Canvas content to invoke the agent without bouncing through NSWorkspace.
if scheme == "clawdis" {
if self.webView.url?.scheme == CanvasScheme.scheme {
Task { await DeepLinkHandler.shared.handle(url: url) }
} else {
canvasWindowLogger
.debug("ignoring deep link from non-canvas page \(url.absoluteString, privacy: .public)")
}
decisionHandler(.cancel)
return
}
// Keep web content inside the panel when reasonable.
// `about:blank` and friends are common internal navigations for WKWebView; never send them to NSWorkspace.
if scheme == CanvasScheme.scheme
|| scheme == "https"
|| scheme == "http"
|| scheme == "about"
|| scheme == "blob"
|| scheme == "data"
|| scheme == "javascript"
{
decisionHandler(.allow)
return
}
// Only open external URLs when there is a registered handler, otherwise macOS will show a confusing
// "There is no application set to open the URL ..." alert (e.g. for about:blank).
if let appURL = NSWorkspace.shared.urlForApplication(toOpen: url) {
NSWorkspace.shared.open(
[url],
withApplicationAt: appURL,
configuration: NSWorkspace.OpenConfiguration(),
completionHandler: nil)
} else {
canvasWindowLogger.debug("no application to open url \(url.absoluteString, privacy: .public)")
}
decisionHandler(.cancel)
}
// MARK: - NSWindowDelegate
func windowWillClose(_: Notification) {
self.onVisibilityChanged?(false)
}
func windowDidMove(_: Notification) {
self.persistFrameIfPanel()
}
func windowDidEndLiveResize(_: Notification) {
self.persistFrameIfPanel()
}
private func persistFrameIfPanel() {
guard case .panel = self.presentation, let window else { return }
Self.storeRestoredFrame(window.frame, sessionKey: self.sessionKey)
}
// MARK: - Helpers
private static func sanitizeSessionKey(_ key: String) -> String {
let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty { return "main" }
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+")
let scalars = trimmed.unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" }
return String(scalars)
}
private static func jsStringLiteral(_ value: String) -> String {
let data = try? JSONEncoder().encode(value)
return data.flatMap { String(data: $0, encoding: .utf8) } ?? "\"\""
}
private static func storedFrameDefaultsKey(sessionKey: String) -> String {
"clawdis.canvas.frame.\(self.sanitizeSessionKey(sessionKey))"
}
private static func loadRestoredFrame(sessionKey: String) -> NSRect? {
let key = self.storedFrameDefaultsKey(sessionKey: sessionKey)
guard let arr = UserDefaults.standard.array(forKey: key) as? [Double], arr.count == 4 else { return nil }
let rect = NSRect(x: arr[0], y: arr[1], width: arr[2], height: arr[3])
if rect.width < CanvasLayout.minPanelSize.width || rect.height < CanvasLayout.minPanelSize.height { return nil }
return rect
}
private static func storeRestoredFrame(_ frame: NSRect, sessionKey: String) {
let key = self.storedFrameDefaultsKey(sessionKey: sessionKey)
UserDefaults.standard.set(
[Double(frame.origin.x), Double(frame.origin.y), Double(frame.size.width), Double(frame.size.height)],
forKey: key)
}
}
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "clawdisCanvasA2UIAction"
private let sessionKey: String
init(sessionKey: String) {
self.sessionKey = sessionKey
super.init()
}
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
guard message.name == Self.messageName else { return }
// Only accept actions from local Canvas content (not arbitrary web pages).
guard let webView = message.webView, let url = webView.url else { return }
if url.scheme == CanvasScheme.scheme {
// ok
} else if Self.isLocalNetworkCanvasURL(url) {
// ok
} else {
return
}
let body: [String: Any] = {
if let dict = message.body as? [String: Any] { return dict }
if let dict = message.body as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !body.isEmpty else { return }
let userActionAny = body["userAction"] ?? body
let userAction: [String: Any] = {
if let dict = userActionAny as? [String: Any] { return dict }
if let dict = userActionAny as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !userAction.isEmpty else { return }
guard let name = userAction["name"] as? String, !name.isEmpty else { return }
let actionId =
(userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
?? UUID().uuidString
canvasWindowLogger.info("A2UI action \(name, privacy: .public) session=\(self.sessionKey, privacy: .public)")
let surfaceId = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
.nonEmpty ?? "main"
let sourceComponentId = (userAction["sourceComponentId"] as? String)?
.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "-"
let instanceId = InstanceIdentity.instanceId.lowercased()
let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"])
// Token-efficient and unambiguous. The agent should treat this as a UI event and (by default) update Canvas.
let text = ClawdisCanvasA2UIAction.formatAgentMessage(
actionName: name,
sessionKey: self.sessionKey,
surfaceId: surfaceId,
sourceComponentId: sourceComponentId,
host: InstanceIdentity.displayName,
instanceId: instanceId,
contextJSON: contextJSON)
Task { [weak webView] in
if AppStateStore.shared.connectionMode == .local {
GatewayProcessManager.shared.setActive(true)
}
let result = await GatewayConnection.shared.sendAgent(GatewayAgentInvocation(
message: text,
sessionKey: self.sessionKey,
thinking: "low",
deliver: false,
to: nil,
channel: .last,
idempotencyKey: actionId))
await MainActor.run {
guard let webView else { return }
let js = ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(
actionId: actionId,
ok: result.ok,
error: result.error)
webView.evaluateJavaScript(js) { _, _ in }
}
if !result.ok {
canvasWindowLogger.error(
"""
A2UI action send failed name=\(name, privacy: .public) \
error=\(result.error ?? "unknown", privacy: .public)
""")
}
}
}
private static func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return false
}
guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else {
return false
}
if host == "localhost" { return true }
if host.hasSuffix(".local") { return true }
if host.hasSuffix(".ts.net") { return true }
if host.hasSuffix(".tailscale.net") { return true }
if !host.contains("."), !host.contains(":") { return true }
if let ipv4 = Self.parseIPv4(host) {
return Self.isLocalNetworkIPv4(ipv4)
}
return false
}
private static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? {
let parts = host.split(separator: ".", omittingEmptySubsequences: false)
guard parts.count == 4 else { return nil }
let bytes: [UInt8] = parts.compactMap { UInt8($0) }
guard bytes.count == 4 else { return nil }
return (bytes[0], bytes[1], bytes[2], bytes[3])
}
private static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool {
let (a, b, _, _) = ip
if a == 10 { return true }
if a == 172, (16...31).contains(Int(b)) { return true }
if a == 192, b == 168 { return true }
if a == 127 { return true }
if a == 169, b == 254 { return true }
if a == 100, (64...127).contains(Int(b)) { return true }
return false
}
// Formatting helpers live in ClawdisKit (`ClawdisCanvasA2UIAction`).
}
// MARK: - Hover chrome container
private final class HoverChromeContainerView: NSView {
private let content: NSView
private let chrome: CanvasChromeOverlayView
private var tracking: NSTrackingArea?
var onClose: (() -> Void)?
init(containing content: NSView) {
self.content = content
self.chrome = CanvasChromeOverlayView(frame: .zero)
super.init(frame: .zero)
self.wantsLayer = true
self.layer?.cornerRadius = 12
self.layer?.masksToBounds = true
self.layer?.backgroundColor = NSColor.windowBackgroundColor.cgColor
self.content.translatesAutoresizingMaskIntoConstraints = false
self.addSubview(self.content)
self.chrome.translatesAutoresizingMaskIntoConstraints = false
self.chrome.alphaValue = 0
self.chrome.onClose = { [weak self] in self?.onClose?() }
self.addSubview(self.chrome)
NSLayoutConstraint.activate([
self.content.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.content.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.content.topAnchor.constraint(equalTo: self.topAnchor),
self.content.bottomAnchor.constraint(equalTo: self.bottomAnchor),
self.chrome.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.chrome.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.chrome.topAnchor.constraint(equalTo: self.topAnchor),
self.chrome.bottomAnchor.constraint(equalTo: self.bottomAnchor),
])
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
override func updateTrackingAreas() {
super.updateTrackingAreas()
if let tracking {
self.removeTrackingArea(tracking)
}
let area = NSTrackingArea(
rect: self.bounds,
options: [.activeAlways, .mouseEnteredAndExited, .inVisibleRect],
owner: self,
userInfo: nil)
self.addTrackingArea(area)
self.tracking = area
}
private final class CanvasDragHandleView: NSView {
override func mouseDown(with event: NSEvent) {
self.window?.performDrag(with: event)
}
override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true }
}
private final class CanvasResizeHandleView: NSView {
private var startPoint: NSPoint = .zero
private var startFrame: NSRect = .zero
override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true }
override func mouseDown(with event: NSEvent) {
guard let window else { return }
_ = window.makeFirstResponder(self)
self.startPoint = NSEvent.mouseLocation
self.startFrame = window.frame
super.mouseDown(with: event)
}
override func mouseDragged(with _: NSEvent) {
guard let window else { return }
let current = NSEvent.mouseLocation
let dx = current.x - self.startPoint.x
let dy = current.y - self.startPoint.y
var frame = self.startFrame
frame.size.width = max(CanvasLayout.minPanelSize.width, frame.size.width + dx)
frame.origin.y += dy
frame.size.height = max(CanvasLayout.minPanelSize.height, frame.size.height - dy)
if let screen = window.screen {
frame = CanvasWindowController.constrainFrame(frame, toVisibleFrame: screen.visibleFrame)
}
window.setFrame(frame, display: true)
}
}
private final class CanvasChromeOverlayView: NSView {
var onClose: (() -> Void)?
private let dragHandle = CanvasDragHandleView(frame: .zero)
private let resizeHandle = CanvasResizeHandleView(frame: .zero)
private final class PassthroughVisualEffectView: NSVisualEffectView {
override func hitTest(_: NSPoint) -> NSView? { nil }
}
private let closeBackground: NSVisualEffectView = {
let v = PassthroughVisualEffectView(frame: .zero)
v.material = .hudWindow
v.blendingMode = .withinWindow
v.state = .active
v.appearance = NSAppearance(named: .vibrantDark)
v.wantsLayer = true
v.layer?.cornerRadius = 10
v.layer?.masksToBounds = true
v.layer?.borderWidth = 1
v.layer?.borderColor = NSColor.white.withAlphaComponent(0.22).cgColor
v.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.28).cgColor
v.layer?.shadowColor = NSColor.black.withAlphaComponent(0.35).cgColor
v.layer?.shadowOpacity = 0.35
v.layer?.shadowRadius = 8
v.layer?.shadowOffset = .zero
return v
}()
private let closeButton: NSButton = {
let cfg = NSImage.SymbolConfiguration(pointSize: 8, weight: .semibold)
let img = NSImage(systemSymbolName: "xmark", accessibilityDescription: "Close")?
.withSymbolConfiguration(cfg)
?? NSImage(size: NSSize(width: 18, height: 18))
let btn = NSButton(image: img, target: nil, action: nil)
btn.isBordered = false
btn.bezelStyle = .regularSquare
btn.imageScaling = .scaleProportionallyDown
btn.contentTintColor = NSColor.white.withAlphaComponent(0.92)
btn.toolTip = "Close"
return btn
}()
override init(frame frameRect: NSRect) {
super.init(frame: frameRect)
self.wantsLayer = true
self.layer?.cornerRadius = 12
self.layer?.masksToBounds = true
self.layer?.borderWidth = 1
self.layer?.borderColor = NSColor.black.withAlphaComponent(0.18).cgColor
self.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.02).cgColor
self.dragHandle.translatesAutoresizingMaskIntoConstraints = false
self.dragHandle.wantsLayer = true
self.dragHandle.layer?.backgroundColor = NSColor.clear.cgColor
self.addSubview(self.dragHandle)
self.resizeHandle.translatesAutoresizingMaskIntoConstraints = false
self.resizeHandle.wantsLayer = true
self.resizeHandle.layer?.backgroundColor = NSColor.clear.cgColor
self.addSubview(self.resizeHandle)
self.closeBackground.translatesAutoresizingMaskIntoConstraints = false
self.addSubview(self.closeBackground)
self.closeButton.translatesAutoresizingMaskIntoConstraints = false
self.closeButton.target = self
self.closeButton.action = #selector(self.handleClose)
self.addSubview(self.closeButton)
NSLayoutConstraint.activate([
self.dragHandle.leadingAnchor.constraint(equalTo: self.leadingAnchor),
self.dragHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.dragHandle.topAnchor.constraint(equalTo: self.topAnchor),
self.dragHandle.heightAnchor.constraint(equalToConstant: 30),
self.closeBackground.centerXAnchor.constraint(equalTo: self.closeButton.centerXAnchor),
self.closeBackground.centerYAnchor.constraint(equalTo: self.closeButton.centerYAnchor),
self.closeBackground.widthAnchor.constraint(equalToConstant: 20),
self.closeBackground.heightAnchor.constraint(equalToConstant: 20),
self.closeButton.trailingAnchor.constraint(equalTo: self.trailingAnchor, constant: -8),
self.closeButton.topAnchor.constraint(equalTo: self.topAnchor, constant: 8),
self.closeButton.widthAnchor.constraint(equalToConstant: 16),
self.closeButton.heightAnchor.constraint(equalToConstant: 16),
self.resizeHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.resizeHandle.bottomAnchor.constraint(equalTo: self.bottomAnchor),
self.resizeHandle.widthAnchor.constraint(equalToConstant: 18),
self.resizeHandle.heightAnchor.constraint(equalToConstant: 18),
])
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
override func hitTest(_ point: NSPoint) -> NSView? {
// When the chrome is hidden, do not intercept any mouse events (let the WKWebView receive them).
guard self.alphaValue > 0.02 else { return nil }
if self.closeButton.frame.contains(point) { return self.closeButton }
if self.dragHandle.frame.contains(point) { return self.dragHandle }
if self.resizeHandle.frame.contains(point) { return self.resizeHandle }
return nil
}
@objc private func handleClose() {
self.onClose?()
}
}
override func mouseEntered(with _: NSEvent) {
NSAnimationContext.runAnimationGroup { ctx in
ctx.duration = 0.12
ctx.timingFunction = CAMediaTimingFunction(name: .easeOut)
self.chrome.animator().alphaValue = 1
}
}
override func mouseExited(with _: NSEvent) {
NSAnimationContext.runAnimationGroup { ctx in
ctx.duration = 0.16
ctx.timingFunction = CAMediaTimingFunction(name: .easeOut)
self.chrome.animator().alphaValue = 0
}
}
}
@@ -0,0 +1,43 @@
import AppKit
import Foundation
extension CanvasWindowController {
// MARK: - Helpers
static func sanitizeSessionKey(_ key: String) -> String {
let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty { return "main" }
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+")
let scalars = trimmed.unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" }
return String(scalars)
}
static func jsStringLiteral(_ value: String) -> String {
let data = try? JSONEncoder().encode(value)
return data.flatMap { String(data: $0, encoding: .utf8) } ?? "\"\""
}
static func jsOptionalStringLiteral(_ value: String?) -> String {
guard let value else { return "null" }
return Self.jsStringLiteral(value)
}
static func storedFrameDefaultsKey(sessionKey: String) -> String {
"clawdis.canvas.frame.\(self.sanitizeSessionKey(sessionKey))"
}
static func loadRestoredFrame(sessionKey: String) -> NSRect? {
let key = self.storedFrameDefaultsKey(sessionKey: sessionKey)
guard let arr = UserDefaults.standard.array(forKey: key) as? [Double], arr.count == 4 else { return nil }
let rect = NSRect(x: arr[0], y: arr[1], width: arr[2], height: arr[3])
if rect.width < CanvasLayout.minPanelSize.width || rect.height < CanvasLayout.minPanelSize.height { return nil }
return rect
}
static func storeRestoredFrame(_ frame: NSRect, sessionKey: String) {
let key = self.storedFrameDefaultsKey(sessionKey: sessionKey)
UserDefaults.standard.set(
[Double(frame.origin.x), Double(frame.origin.y), Double(frame.size.width), Double(frame.size.height)],
forKey: key)
}
}
@@ -0,0 +1,62 @@
import AppKit
import WebKit
extension CanvasWindowController {
// MARK: - WKNavigationDelegate
@MainActor
func webView(
_: WKWebView,
decidePolicyFor navigationAction: WKNavigationAction,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
{
guard let url = navigationAction.request.url else {
decisionHandler(.cancel)
return
}
let scheme = url.scheme?.lowercased()
// Deep links: allow local Canvas content to invoke the agent without bouncing through NSWorkspace.
if scheme == "clawdis" {
if self.webView.url?.scheme == CanvasScheme.scheme {
Task { await DeepLinkHandler.shared.handle(url: url) }
} else {
canvasWindowLogger
.debug("ignoring deep link from non-canvas page \(url.absoluteString, privacy: .public)")
}
decisionHandler(.cancel)
return
}
// Keep web content inside the panel when reasonable.
// `about:blank` and friends are common internal navigations for WKWebView; never send them to NSWorkspace.
if scheme == CanvasScheme.scheme
|| scheme == "https"
|| scheme == "http"
|| scheme == "about"
|| scheme == "blob"
|| scheme == "data"
|| scheme == "javascript"
{
decisionHandler(.allow)
return
}
// Only open external URLs when there is a registered handler, otherwise macOS will show a confusing
// "There is no application set to open the URL ..." alert (e.g. for about:blank).
if let appURL = NSWorkspace.shared.urlForApplication(toOpen: url) {
NSWorkspace.shared.open(
[url],
withApplicationAt: appURL,
configuration: NSWorkspace.OpenConfiguration(),
completionHandler: nil)
} else {
canvasWindowLogger.debug("no application to open url \(url.absoluteString, privacy: .public)")
}
decisionHandler(.cancel)
}
func webView(_: WKWebView, didFinish _: WKNavigation?) {
self.applyDebugStatusIfNeeded()
}
}
Loaded 100 of 575 files, more files were not shown because too many files have changed in this diff. Show more