Compare commits

..
Author SHA1 Message Date
Peter Steinberger e9502768af Presence: add device identity fields 2025-12-17 21:17:51 +01:00
4240 changed files with 790719 additions and 21838 deletions

No files matched your search

+7 -64
View File
@@ -15,20 +15,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: false
- name: Checkout submodules (retry)
run: |
set -euo pipefail
git submodule sync --recursive
for attempt in 1 2 3 4 5; do
if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then
exit 0
fi
echo "Submodule update failed (attempt $attempt/5). Retrying…"
sleep $((attempt * 10))
done
exit 1
submodules: recursive
- name: Setup Node.js
if: matrix.runtime == 'node'
@@ -111,20 +98,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: false
- name: Checkout submodules (retry)
run: |
set -euo pipefail
git submodule sync --recursive
for attempt in 1 2 3 4 5; do
if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then
exit 0
fi
echo "Submodule update failed (attempt $attempt/5). Retrying…"
sleep $((attempt * 10))
done
exit 1
submodules: recursive
- name: Select Xcode 26.1
run: |
@@ -148,28 +122,10 @@ jobs:
run: swiftformat --lint apps/macos/Sources --config .swiftformat
- name: Swift build (release)
run: |
set -euo pipefail
for attempt in 1 2 3; do
if swift build --package-path apps/macos --configuration release; then
exit 0
fi
echo "swift build failed (attempt $attempt/3). Retrying…"
sleep $((attempt * 20))
done
exit 1
run: swift build --package-path apps/macos --configuration release
- name: Swift tests (coverage)
run: |
set -euo pipefail
for attempt in 1 2 3; do
if swift test --package-path apps/macos --parallel --enable-code-coverage --show-codecov-path; then
exit 0
fi
echo "swift test failed (attempt $attempt/3). Retrying…"
sleep $((attempt * 20))
done
exit 1
run: swift test --package-path apps/macos --parallel --enable-code-coverage --show-codecov-path
- name: Generate iOS project
run: |
@@ -270,7 +226,7 @@ jobs:
RESULT_BUNDLE_PATH="$RUNNER_TEMP/Clawdis-iOS.xcresult"
xcrun xccov view --report --only-targets "$RESULT_BUNDLE_PATH"
- name: iOS coverage gate (43%)
- name: iOS coverage gate (50%)
run: |
set -euo pipefail
RESULT_BUNDLE_PATH="$RUNNER_TEMP/Clawdis-iOS.xcresult"
@@ -281,7 +237,7 @@ jobs:
import sys
target_name = "Clawdis.app"
minimum = 0.43
minimum = 0.50
report = json.loads(
subprocess.check_output(
@@ -311,20 +267,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: false
- name: Checkout submodules (retry)
run: |
set -euo pipefail
git submodule sync --recursive
for attempt in 1 2 3 4 5; do
if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then
exit 0
fi
echo "Submodule update failed (attempt $attempt/5). Retrying…"
sleep $((attempt * 10))
done
exit 1
submodules: recursive
- name: Setup Java
uses: actions/setup-java@v4
-4
View File
@@ -10,7 +10,6 @@ coverage
apps/macos/.build/
apps/shared/ClawdisKit/.build/
bin/clawdis-mac
bin/docs-list
apps/macos/.build-local/
apps/macos/.swiftpm/
Core/
@@ -18,9 +17,6 @@ apps/ios/*.xcodeproj/
apps/ios/*.xcworkspace/
apps/ios/.swiftpm/
# Vendor build artifacts
vendor/a2ui/renderers/lit/dist/
# fastlane (iOS)
apps/ios/fastlane/README.md
apps/ios/fastlane/report.xml
-1
View File
@@ -1,4 +1,3 @@
[submodule "Peekaboo"]
path = Peekaboo
url = https://github.com/steipete/Peekaboo.git
branch = main
+1 -3
View File
@@ -40,9 +40,7 @@
- macOS logs: use `./scripts/clawlog.sh` (aka `vtlog`) to query unified logs for subsystem `com.steipete.clawdis`; it supports follow/tail/category filters and expects passwordless sudo for `/usr/bin/log`.
- Also read the shared guardrails at `~/Projects/oracle/AGENTS.md` and `~/Projects/agent-scripts/AGENTS.MD` before making changes; align with any cross-repo rules noted there.
- SwiftUI state management (iOS/macOS): prefer the `Observation` framework (`@Observable`, `@Bindable`) over `ObservableObject`/`@StateObject`; don’t introduce new `ObservableObject` unless required for compatibility, and migrate existing usages when touching related code.
- **Multi-agent safety:** do **not** create/apply/drop `git stash` entries unless Peter explicitly asks (this includes `git pull --rebase --autostash`). Assume other agents may be working; keep unrelated WIP untouched and avoid cross-cutting state changes.
- **Multi-agent safety:** do **not** create/remove/modify `git worktree` checkouts (or edit `.worktrees/*`) unless Peter explicitly asks.
- **Multi-agent safety:** do **not** switch branches / check out a different branch unless Peter explicitly asks.
- **Multi-agent safety:** do **not** create/apply/drop `git stash` entries unless Peter explicitly asks. Assume other agents may be working; keep unrelated WIP untouched and avoid cross-cutting state changes.
- When asked to open a “session” file, open the Pi session logs under `~/.clawdis/sessions/*.jsonl` (newest unless a specific ID is given), not the default `sessions.json`.
- Menubar dimming + restart flow mirrors Trimmy: use `scripts/restart-mac.sh` (kills all Clawdis variants, runs `swift build`, packages, relaunches). Icon dimming depends on MenuBarExtraAccess wiring in AppMain; keep `appearsDisabled` updates intact when touching the status item.
- Never send streaming/partial replies to external messaging surfaces (WhatsApp, Telegram); only final replies should be delivered there. Streaming/tool events may still go to internal UIs/control channel.
+9 -7
View File
@@ -1,12 +1,14 @@
# Changelog
## 2.0.0-beta1 — 2025-12-19
First Clawdis release post rebrand. This is a semver-major because we dropped legacy providers/agents and moved defaults to new paths while adding a full macOS companion app, a WebSocket Gateway, and an iOS node.
## 2.0.0 — Unreleased
### Bug Fixes
- macOS: Voice Wake / push-to-talk no longer initialize `AVAudioEngine` at app launch, preventing Bluetooth headphones from switching into headset profile when voice features are unused. (Thanks @Nachx639)
## 2.0.0-beta1 — 2025-12-14
First Clawdis release post rebrand. This is a semver-major because we dropped legacy providers/agents and moved defaults to new paths while adding a full macOS companion app, a WebSocket Gateway, and an iOS node (Iris).
### Breaking
- Renamed to **Clawdis**: defaults now live under `~/.clawdis` (sessions in `~/.clawdis/sessions/`, IPC at `~/.clawdis/clawdis.sock`, logs in `/tmp/clawdis`). Launchd labels and config filenames follow the new name; legacy stores are copied forward on first run.
- Pi only: `inbound.reply.agent.kind` accepts only `"pi"`, and the agent CLI/CLI flags for Claude/Codex/Gemini were removed. The Pi CLI runs in RPC mode with a persistent worker.
@@ -17,7 +19,7 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
### Gateway, nodes, and automation
- New typed Gateway WS protocol (JSON schema validated) with `clawdis gateway {health,status,send,agent,call}` helpers and structured presence/instance updates for all clients.
- Optional LAN-facing bridge (`tcp://0.0.0.0:18790`) keeps the Gateway loopback-only while enabling direct Bonjour-discovered connections for paired nodes.
- Node pairing + management via `clawdis nodes {pending,approve,reject,invoke}` (used by the iOS node and future remote nodes).
- Node pairing + management via `clawdis nodes {pending,approve,reject,invoke}` (used by the iOS node “Iris” and future remote nodes).
- Cron jobs are Gateway-owned (`clawdis cron …`) with run history stored as JSONL and support for “isolated summary” posting into the main session.
### macOS companion app
@@ -27,10 +29,10 @@ First Clawdis release post rebrand. This is a semver-major because we dropped le
- **Browser control**: manage clawd’s dedicated Chrome/Chromium with tab listing/open/focus/close, screenshots, DOM query/dump, and “AI snapshots” (aria/domSnapshot/ai) via `clawdis browser …` and UI controls.
- **Remote gateway control**: Bonjour discovery for local masters plus SSH-tunnel fallback for remote control when multicast is unavailable.
### iOS node
### iOS node (Iris)
- New iOS companion app that pairs to the Gateway bridge, reports presence as a node, and exposes a WKWebView “Canvas” for agent-driven UI.
- `clawdis nodes invoke` supports `canvas.eval` and `canvas.snapshot` to drive and verify the iOS Canvas (fails fast when the iOS node is backgrounded).
- Voice wake words are configurable in-app; the iOS node reconnects to the last bridge when credentials are still present in Keychain.
- `clawdis nodes invoke` supports `screen.eval` and `screen.snapshot` to drive and verify the iOS Canvas (fails fast when Iris is backgrounded).
- Voice wake words are configurable in-app; Iris reconnects to the last bridge when credentials are still present in Keychain.
### WhatsApp & agent experience
- Group chats fully supported: mention-gated triggers (including media-only captions), sender attribution, session primer with subject/member roster, allowlist bypass when you’re @‑mentioned, and safer handling of view-once/ephemeral media.
Submodule Peekaboo updated: be03d6e3df...4ff4d3db24.
+14 -29
View File
@@ -22,16 +22,15 @@ WhatsApp / Telegram
│
▼
┌──────────────────────────┐
│ Gateway │ ws://127.0.0.1:18789 (default: loopback)
│ (control UI) │ http://127.0.0.1:18789/ui/
│ Gateway │ ws://127.0.0.1:18789 (loopback-only)
│ (single source) │ tcp://0.0.0.0:18790 (optional Bridge)
└───────────┬───────────────┘
│
├─ Pi agent (RPC)
├─ CLI (clawdis …)
├─ Control UI (browser)
├─ WebChat (loopback UI)
├─ macOS app (Clawdis.app)
└─ iOS node via Bridge + pairing
└─ iOS node (Iris) via Bridge + pairing
```
## Why "CLAWDIS"?
@@ -54,16 +53,14 @@ Because every space lobster needs a time-and-space machine. The Doctor has a TAR
- 🎤 **Voice & transcription hooks** — Voice Wake (macOS/iOS) + optional transcription pipeline
- 🔧 **Tool Streaming** — Real-time display (💻📄✍️📝)
- 🖥️ **macOS Companion (Clawdis.app)** — Menu bar controls, Voice Wake, WebChat, onboarding, remote gateway control
- 📱 **iOS node** — Pairs as a node, exposes a Canvas surface, forwards voice wake transcripts
- 📱 **iOS Node (Iris)** — Pairs as a node, exposes a Canvas surface, forwards voice wake transcripts
Only the Pi CLI is supported now; legacy Claude/Codex/Gemini paths have been removed.
## Network model (the “new reality”)
- **One Gateway per host**. The Gateway is the only process allowed to own the WhatsApp Web session.
- **Loopback-first**: the Gateway WebSocket listens on `ws://127.0.0.1:18789` by default.
- To expose it on your tailnet, set `gateway.bind: "tailnet"` (or run `clawdis gateway --bind tailnet`) and set `CLAWDIS_GATEWAY_TOKEN` (required for non-loopback binds).
- The browser Control UI is served from the Gateway at `http://<host>:18789/ui/` when assets are built.
- **Loopback-first**: the Gateway WebSocket listens on `ws://127.0.0.1:18789` and is not exposed on the LAN.
- **Bridge for nodes**: when enabled, the Gateway also exposes a bridge on `tcp://0.0.0.0:18790` for paired nodes (Bonjour-discoverable). For tailnet-only setups, set `bridge.bind: "tailnet"` in `~/.clawdis/clawdis.json`.
- **Remote control**: use a VPN/tailnet or an SSH tunnel (`ssh -N -L 18789:127.0.0.1:18789 user@host`). The macOS app can drive this flow.
- **Wide-Area Bonjour (optional)**: for auto-discovery across networks (Vienna ⇄ London) over Tailscale, use unicast DNS-SD on `clawdis.internal.`; see `docs/bonjour.md`.
@@ -72,7 +69,7 @@ Only the Pi CLI is supported now; legacy Claude/Codex/Gemini paths have been rem
- **TypeScript (ESM)**: CLI + Gateway live in `src/` and run on Node ≥ 22.
- **macOS app (Swift)**: menu bar companion lives in `apps/macos/`.
- **iOS app (Swift)**: iOS node prototype lives in `apps/ios/`.
- **iOS app (Swift)**: Iris node prototype lives in `apps/ios/`.
## Quick Start
@@ -82,7 +79,6 @@ Runtime requirement: **Node ≥22.0.0** (not bundled). The macOS app and CLI bot
# From source (recommended while the npm package is still settling)
pnpm install
pnpm build
pnpm ui:build
# Link your WhatsApp (stores creds under ~/.clawdis/credentials)
pnpm clawdis login
@@ -90,9 +86,6 @@ pnpm clawdis login
# Start the gateway (WebSocket control plane)
pnpm clawdis gateway --port 18789 --verbose
# Open the browser Control UI (after ui:build)
# http://127.0.0.1:18789/ui/
# Send a WhatsApp message (WhatsApp sends go through the Gateway)
pnpm clawdis send --to +1234567890 --message "Hello from the CLAWDIS!"
@@ -103,19 +96,11 @@ pnpm clawdis agent --message "Ship checklist" --thinking high
pnpm clawdis gateway --force
```
### Agent workspace + skills
Clawdis runs the embedded agent with its working directory set to the agent workspace (default: `~/clawd`, configurable via `inbound.workspace`).
- Workspace files injected into the system prompt: `AGENTS.md`, `SOUL.md`, `TOOLS.md`
- Custom skills: `<workspace>/skills/<skill-name>/SKILL.md` (default: `~/clawd/skills/<skill-name>/SKILL.md`; only this location is scanned)
## Companion Apps
### macOS Companion (Clawdis.app)
- A menu bar app that can start/stop the Gateway, show health/presence, and provide a local ops UI.
- Instances UI shows friendly hardware model names (from the vendored MIT dataset under `apps/macos/Sources/Clawdis/Resources/DeviceModels/`).
- **Voice Wake** (on-device speech recognition) and Push-to-talk overlay.
- **WebChat** embed + debug tooling (logs, status, heartbeats, sessions).
- Hosts **PeekabooBridge** for UI automation brokering (for clawd workflows).
@@ -132,13 +117,13 @@ If delivery fails (e.g. WhatsApp disconnected / Telegram token missing), Clawdis
Build/run the mac app with `./scripts/restart-mac.sh` (packages, installs, and launches), or `swift build --package-path apps/macos && open dist/Clawdis.app`.
### iOS node (internal)
### iOS Node (Iris) (internal)
The iOS node app is an internal/prototype app that connects as a **remote node**:
Iris is an internal/prototype iOS app that connects as a **remote node**:
- **Voice trigger:** forwards transcripts into the Gateway (agent runs + wakeups).
- **Canvas screen:** a WKWebView + `<canvas>` surface the agent can control (via `canvas.eval` / `canvas.snapshot` over `node.invoke`).
- **Discovery + pairing:** finds the bridge via Bonjour (`_clawdis-bridge._tcp`) and uses Gateway-owned pairing (`clawdis nodes pending|approve`); `clawdis nodes status` shows paired nodes + capabilities.
- **Canvas screen:** a WKWebView + `<canvas>` surface the agent can control (via `screen.eval` / `screen.snapshot` over `node.invoke`).
- **Discovery + pairing:** finds the bridge via Bonjour (`_clawdis-bridge._tcp`) and uses Gateway-owned pairing (`clawdis nodes pending|approve`).
Runbook: `docs/ios/connect.md`
@@ -170,7 +155,6 @@ Optional: enable/configure clawd’s dedicated browser control (defaults are alr
- [Configuration Guide](./docs/configuration.md)
- [Gateway runbook](./docs/gateway.md)
- [Web surfaces (Control UI)](./docs/web.md)
- [Discovery + transports](./docs/discovery.md)
- [Bonjour / mDNS + Wide-Area Bonjour](./docs/bonjour.md)
- [Agent Runtime](./docs/agent.md)
@@ -179,7 +163,7 @@ Optional: enable/configure clawd’s dedicated browser control (defaults are alr
- [Troubleshooting](./docs/troubleshooting.md)
- [The Lore](./docs/lore.md) 🦞
- [Telegram (Bot API)](./docs/telegram.md)
- [iOS node runbook](./docs/ios/connect.md)
- [iOS node runbook (Iris)](./docs/ios/connect.md)
- [macOS app spec](./docs/clawdis-mac.md)
## Clawd
@@ -212,13 +196,14 @@ Bot-mode support (grammY only) shares the same `main` session as WhatsApp/WebCha
| `clawdis send` | Send a message (WhatsApp default; `--provider telegram` for bot mode). WhatsApp sends go via the Gateway WS; Telegram sends are direct. |
| `clawdis agent` | Talk directly to the agent (no WhatsApp send) |
| `clawdis browser ...` | Manage clawd’s dedicated browser (status/tabs/open/screenshot). |
| `clawdis gateway` | Start the Gateway server (WS control plane). Params: `--port`, `--bind`, `--token`, `--force`, `--verbose`. |
| `clawdis gateway` | Start the Gateway server (WS control plane). Params: `--port`, `--token`, `--force`, `--verbose`. |
| `clawdis gateway health|status|send|agent|call` | Gateway WS clients; assume a running gateway. |
| `clawdis wake` | Enqueue a system event and optionally trigger a heartbeat via the Gateway. |
| `clawdis cron ...` | Manage scheduled jobs (via Gateway). |
| `clawdis nodes ...` | Manage nodes (pairing + status) via the Gateway. |
| `clawdis nodes ...` | Manage Gateway-owned node pairing. |
| `clawdis status` | Web session health + session store summary |
| `clawdis health` | Reports cached provider state from the running gateway. |
| `clawdis webchat` | Start the loopback-only WebChat HTTP server |
#### Gateway client params (WS only)
- `--url` (default `ws://127.0.0.1:18789`)
-11
View File
@@ -6,16 +6,5 @@
<title>Clawdis Updates</title>
<link>https://raw.githubusercontent.com/steipete/clawdis/main/appcast.xml</link>
<description>Signed update feed for the Clawdis macOS companion app.</description>
<item>
<title>Clawdis 2.0.0-beta1</title>
<sparkle:releaseNotesLink>https://github.com/steipete/clawdis/releases/tag/v2.0.0-beta1</sparkle:releaseNotesLink>
<pubDate>Fri, 19 Dec 2025 17:19:50 +0000</pubDate>
<enclosure url="https://github.com/steipete/clawdis/releases/download/v2.0.0-beta1/Clawdis-2.0.0-beta1.zip"
sparkle:edSignature="oEpGD46U4ZyBBSY9/piUIFDJU+KlFB751JIWOW2yS0sRNHKszyG5khDHg9o9bV9Zo8DOCNF/HOi88jmtHJAaCQ=="
sparkle:version="2.0.0-beta1"
sparkle:shortVersionString="2.0.0-beta1"
length="72410016"
type="application/octet-stream" />
</item>
</channel>
</rss>
+1 -1
View File
@@ -1,6 +1,6 @@
## Clawdis Node (Android) (internal)
Modern Android node app: connects to the **Gateway-owned bridge** (`_clawdis-bridge._tcp`) over TCP and exposes **Canvas + Chat + Camera**.
Modern Android “node” app (Iris parity): connects to the **Gateway-owned bridge** (`_clawdis-bridge._tcp`) over TCP and exposes **Canvas + Chat + Camera**.
Notes:
- The node keeps the connection alive via a **foreground service** (persistent notification with a Disconnect action).
-7
View File
@@ -9,12 +9,6 @@ android {
namespace = "com.steipete.clawdis.node"
compileSdk = 36
sourceSets {
getByName("main") {
assets.srcDir(file("../../shared/ClawdisKit/Sources/ClawdisKit/Resources"))
}
}
defaultConfig {
applicationId = "com.steipete.clawdis.node"
minSdk = 31
@@ -61,7 +55,6 @@ dependencies {
implementation("androidx.compose.ui:ui")
implementation("androidx.compose.ui:ui-tooling-preview")
implementation("androidx.compose.material3:material3")
implementation("androidx.compose.material:material-icons-extended")
implementation("androidx.navigation:navigation-compose:2.9.6")
debugImplementation("androidx.compose.ui:ui-tooling")
@@ -3,8 +3,6 @@
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<uses-permission
android:name="android.permission.NEARBY_WIFI_DEVICES"
@@ -22,12 +20,11 @@
android:roundIcon="@mipmap/ic_launcher_round"
android:label="@string/app_name"
android:supportsRtl="true"
android:networkSecurityConfig="@xml/network_security_config"
android:theme="@style/Theme.ClawdisNode">
<service
android:name=".NodeForegroundService"
android:exported="false"
android:foregroundServiceType="dataSync|microphone|mediaProjection" />
android:foregroundServiceType="dataSync" />
<activity
android:name=".MainActivity"
android:exported="true">
@@ -1,15 +0,0 @@
package com.steipete.clawdis.node
enum class CameraHudKind {
Photo,
Recording,
Success,
Error,
}
data class CameraHudState(
val token: Long,
val kind: CameraHudKind,
val message: String,
)
@@ -1,26 +0,0 @@
package com.steipete.clawdis.node
import android.content.Context
import android.os.Build
import android.provider.Settings
object DeviceNames {
fun bestDefaultNodeName(context: Context): String {
val deviceName =
runCatching {
Settings.Global.getString(context.contentResolver, "device_name")
}
.getOrNull()
?.trim()
.orEmpty()
if (deviceName.isNotEmpty()) return deviceName
val model =
listOfNotNull(Build.MANUFACTURER?.takeIf { it.isNotBlank() }, Build.MODEL?.takeIf { it.isNotBlank() })
.joinToString(" ")
.trim()
return model.ifEmpty { "Android Node" }
}
}
@@ -1,61 +1,28 @@
package com.steipete.clawdis.node
import android.Manifest
import android.content.pm.ApplicationInfo
import android.os.Bundle
import android.os.Build
import android.view.WindowManager
import android.webkit.WebView
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.activity.viewModels
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.ui.Modifier
import androidx.core.content.ContextCompat
import androidx.core.view.WindowCompat
import androidx.core.view.WindowInsetsCompat
import androidx.core.view.WindowInsetsControllerCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle
import com.steipete.clawdis.node.ui.RootScreen
import com.steipete.clawdis.node.ui.ClawdisTheme
import kotlinx.coroutines.launch
class MainActivity : ComponentActivity() {
private val viewModel: MainViewModel by viewModels()
private lateinit var permissionRequester: PermissionRequester
private lateinit var screenCaptureRequester: ScreenCaptureRequester
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
val isDebuggable = (applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE) != 0
WebView.setWebContentsDebuggingEnabled(isDebuggable)
applyImmersiveMode()
requestDiscoveryPermissionsIfNeeded()
requestNotificationPermissionIfNeeded()
NodeForegroundService.start(this)
permissionRequester = PermissionRequester(this)
screenCaptureRequester = ScreenCaptureRequester(this)
viewModel.camera.attachLifecycleOwner(this)
viewModel.camera.attachPermissionRequester(permissionRequester)
viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester)
viewModel.screenRecorder.attachPermissionRequester(permissionRequester)
lifecycleScope.launch {
repeatOnLifecycle(Lifecycle.State.STARTED) {
viewModel.preventSleep.collect { enabled ->
if (enabled) {
window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
} else {
window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
}
}
}
}
setContent {
ClawdisTheme {
MaterialTheme {
Surface(modifier = Modifier) {
RootScreen(viewModel = viewModel)
}
@@ -63,18 +30,6 @@ class MainActivity : ComponentActivity() {
}
}
override fun onResume() {
super.onResume()
applyImmersiveMode()
}
override fun onWindowFocusChanged(hasFocus: Boolean) {
super.onWindowFocusChanged(hasFocus)
if (hasFocus) {
applyImmersiveMode()
}
}
override fun onStart() {
super.onStart()
viewModel.setForeground(true)
@@ -85,14 +40,6 @@ class MainActivity : ComponentActivity() {
super.onStop()
}
private fun applyImmersiveMode() {
WindowCompat.setDecorFitsSystemWindows(window, false)
val controller = WindowInsetsControllerCompat(window, window.decorView)
controller.systemBarsBehavior =
WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE
controller.hide(WindowInsetsCompat.Type.systemBars())
}
private fun requestDiscoveryPermissionsIfNeeded() {
if (Build.VERSION.SDK_INT >= 33) {
val ok =
@@ -3,10 +3,8 @@ package com.steipete.clawdis.node
import android.app.Application
import androidx.lifecycle.AndroidViewModel
import com.steipete.clawdis.node.bridge.BridgeEndpoint
import com.steipete.clawdis.node.chat.OutgoingAttachment
import com.steipete.clawdis.node.node.CameraCaptureManager
import com.steipete.clawdis.node.node.CanvasController
import com.steipete.clawdis.node.node.ScreenRecordManager
import kotlinx.coroutines.flow.StateFlow
class MainViewModel(app: Application) : AndroidViewModel(app) {
@@ -14,40 +12,24 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val canvas: CanvasController = runtime.canvas
val camera: CameraCaptureManager = runtime.camera
val screenRecorder: ScreenRecordManager = runtime.screenRecorder
val bridges: StateFlow<List<BridgeEndpoint>> = runtime.bridges
val discoveryStatusText: StateFlow<String> = runtime.discoveryStatusText
val isConnected: StateFlow<Boolean> = runtime.isConnected
val statusText: StateFlow<String> = runtime.statusText
val serverName: StateFlow<String?> = runtime.serverName
val remoteAddress: StateFlow<String?> = runtime.remoteAddress
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
val instanceId: StateFlow<String> = runtime.instanceId
val displayName: StateFlow<String> = runtime.displayName
val cameraEnabled: StateFlow<Boolean> = runtime.cameraEnabled
val preventSleep: StateFlow<Boolean> = runtime.preventSleep
val wakeWords: StateFlow<List<String>> = runtime.wakeWords
val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode
val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening
val manualEnabled: StateFlow<Boolean> = runtime.manualEnabled
val manualHost: StateFlow<String> = runtime.manualHost
val manualPort: StateFlow<Int> = runtime.manualPort
val chatSessionKey: StateFlow<String> = runtime.chatSessionKey
val chatSessionId: StateFlow<String?> = runtime.chatSessionId
val chatMessages = runtime.chatMessages
val chatMessages: StateFlow<List<NodeRuntime.ChatMessage>> = runtime.chatMessages
val chatError: StateFlow<String?> = runtime.chatError
val chatHealthOk: StateFlow<Boolean> = runtime.chatHealthOk
val chatThinkingLevel: StateFlow<String> = runtime.chatThinkingLevel
val chatStreamingAssistantText: StateFlow<String?> = runtime.chatStreamingAssistantText
val chatPendingToolCalls = runtime.chatPendingToolCalls
val chatSessions = runtime.chatSessions
val pendingRunCount: StateFlow<Int> = runtime.pendingRunCount
fun setForeground(value: Boolean) {
@@ -62,10 +44,6 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.setCameraEnabled(value)
}
fun setPreventSleep(value: Boolean) {
runtime.setPreventSleep(value)
}
fun setManualEnabled(value: Boolean) {
runtime.setManualEnabled(value)
}
@@ -86,10 +64,6 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.resetWakeWordsDefaults()
}
fun setVoiceWakeMode(mode: VoiceWakeMode) {
runtime.setVoiceWakeMode(mode)
}
fun connect(endpoint: BridgeEndpoint) {
runtime.connect(endpoint)
}
@@ -102,35 +76,11 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.disconnect()
}
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
runtime.handleCanvasA2UIActionFromWebView(payloadJson)
}
fun loadChat(sessionKey: String = "main") {
runtime.loadChat(sessionKey)
}
fun refreshChat() {
runtime.refreshChat()
}
fun refreshChatSessions(limit: Int? = null) {
runtime.refreshChatSessions(limit = limit)
}
fun setChatThinkingLevel(level: String) {
runtime.setChatThinkingLevel(level)
}
fun switchChatSession(sessionKey: String) {
runtime.switchChatSession(sessionKey)
}
fun abortChat() {
runtime.abortChat()
}
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
runtime.sendChat(message = message, thinking = thinking, attachments = attachments)
fun sendChat(sessionKey: String = "main", message: String) {
runtime.sendChat(sessionKey, message)
}
}
@@ -5,14 +5,11 @@ import android.app.NotificationChannel
import android.app.NotificationManager
import android.app.Service
import android.app.PendingIntent
import android.Manifest
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.content.pm.ServiceInfo
import android.os.Build
import androidx.core.app.NotificationCompat
import androidx.core.content.ContextCompat
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
@@ -24,42 +21,26 @@ import kotlinx.coroutines.launch
class NodeForegroundService : Service() {
private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
private var notificationJob: Job? = null
private var lastRequiresMic = false
private var didStartForeground = false
override fun onCreate() {
super.onCreate()
ensureChannel()
val initial = buildNotification(title = "Clawdis Node", text = "Starting…")
startForegroundWithTypes(notification = initial, requiresMic = false)
if (Build.VERSION.SDK_INT >= 29) {
startForeground(NOTIFICATION_ID, initial, ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
} else {
startForeground(NOTIFICATION_ID, initial)
}
val runtime = (application as NodeApp).runtime
notificationJob =
scope.launch {
combine(
runtime.statusText,
runtime.serverName,
runtime.isConnected,
runtime.voiceWakeMode,
runtime.voiceWakeIsListening,
) { status, server, connected, voiceMode, voiceListening ->
Quint(status, server, connected, voiceMode, voiceListening)
}.collect { (status, server, connected, voiceMode, voiceListening) ->
combine(runtime.statusText, runtime.serverName, runtime.isConnected) { status, server, connected ->
Triple(status, server, connected)
}.collect { (status, server, connected) ->
val title = if (connected) "Clawdis Node · Connected" else "Clawdis Node"
val voiceSuffix =
if (voiceMode == VoiceWakeMode.Always) {
if (voiceListening) " · Voice Wake: Listening" else " · Voice Wake: Paused"
} else {
""
}
val text = (server?.let { "$status · $it" } ?: status) + voiceSuffix
val requiresMic =
voiceMode == VoiceWakeMode.Always && hasRecordAudioPermission()
startForegroundWithTypes(
notification = buildNotification(title = title, text = text),
requiresMic = requiresMic,
)
val text = server?.let { "$status · $it" } ?: status
updateNotification(buildNotification(title = title, text = text))
}
}
}
@@ -125,35 +106,6 @@ class NodeForegroundService : Service() {
mgr.notify(NOTIFICATION_ID, notification)
}
private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) {
if (Build.VERSION.SDK_INT < 29) {
startForeground(NOTIFICATION_ID, notification)
return
}
if (didStartForeground && requiresMic == lastRequiresMic) {
updateNotification(notification)
return
}
lastRequiresMic = requiresMic
val types =
if (requiresMic) {
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
} else {
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
}
startForeground(NOTIFICATION_ID, notification, types)
didStartForeground = true
}
private fun hasRecordAudioPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
)
}
companion object {
private const val CHANNEL_ID = "connection"
private const val NOTIFICATION_ID = 1
@@ -175,5 +127,3 @@ class NodeForegroundService : Service() {
}
}
}
private data class Quint<A, B, C, D, E>(val first: A, val second: B, val third: C, val fourth: D, val fifth: E)
@@ -1,30 +1,13 @@
package com.steipete.clawdis.node
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.os.Build
import android.os.SystemClock
import androidx.core.content.ContextCompat
import com.steipete.clawdis.node.chat.ChatController
import com.steipete.clawdis.node.chat.ChatMessage
import com.steipete.clawdis.node.chat.ChatPendingToolCall
import com.steipete.clawdis.node.chat.ChatSessionEntry
import com.steipete.clawdis.node.chat.OutgoingAttachment
import com.steipete.clawdis.node.bridge.BridgeDiscovery
import com.steipete.clawdis.node.bridge.BridgeEndpoint
import com.steipete.clawdis.node.bridge.BridgePairingClient
import com.steipete.clawdis.node.bridge.BridgeSession
import com.steipete.clawdis.node.node.CameraCaptureManager
import com.steipete.clawdis.node.node.CanvasController
import com.steipete.clawdis.node.node.ScreenRecordManager
import com.steipete.clawdis.node.protocol.ClawdisCapability
import com.steipete.clawdis.node.protocol.ClawdisCameraCommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UIAction
import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UICommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasCommand
import com.steipete.clawdis.node.protocol.ClawdisScreenCommand
import com.steipete.clawdis.node.voice.VoiceWakeManager
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
@@ -33,9 +16,7 @@ import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
@@ -43,8 +24,6 @@ import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import java.util.concurrent.atomic.AtomicLong
class NodeRuntime(context: Context) {
private val appContext = context.applicationContext
@@ -53,53 +32,17 @@ class NodeRuntime(context: Context) {
val prefs = SecurePrefs(appContext)
val canvas = CanvasController()
val camera = CameraCaptureManager(appContext)
val screenRecorder = ScreenRecordManager(appContext)
private val json = Json { ignoreUnknownKeys = true }
private val externalAudioCaptureActive = MutableStateFlow(false)
private val voiceWake: VoiceWakeManager by lazy {
VoiceWakeManager(
context = appContext,
scope = scope,
onCommand = { command ->
session.sendEvent(
event = "agent.request",
payloadJson =
buildJsonObject {
put("message", JsonPrimitive(command))
put("sessionKey", JsonPrimitive("main"))
put("thinking", JsonPrimitive(chatThinkingLevel.value))
put("deliver", JsonPrimitive(false))
}.toString(),
)
},
)
}
val voiceWakeIsListening: StateFlow<Boolean>
get() = voiceWake.isListening
val voiceWakeStatusText: StateFlow<String>
get() = voiceWake.statusText
private val discovery = BridgeDiscovery(appContext, scope = scope)
val bridges: StateFlow<List<BridgeEndpoint>> = discovery.bridges
val discoveryStatusText: StateFlow<String> = discovery.statusText
private val _isConnected = MutableStateFlow(false)
val isConnected: StateFlow<Boolean> = _isConnected.asStateFlow()
private val _statusText = MutableStateFlow("Offline")
private val _statusText = MutableStateFlow("Not connected")
val statusText: StateFlow<String> = _statusText.asStateFlow()
private val cameraHudSeq = AtomicLong(0)
private val _cameraHud = MutableStateFlow<CameraHudState?>(null)
val cameraHud: StateFlow<CameraHudState?> = _cameraHud.asStateFlow()
private val _cameraFlashToken = MutableStateFlow(0L)
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
private val _serverName = MutableStateFlow<String?>(null)
val serverName: StateFlow<String?> = _serverName.asStateFlow()
@@ -119,7 +62,12 @@ class NodeRuntime(context: Context) {
_isConnected.value = true
scope.launch { refreshWakeWordsFromGateway() }
},
onDisconnected = { message -> handleSessionDisconnected(message) },
onDisconnected = { message ->
_statusText.value = message
_serverName.value = null
_remoteAddress.value = null
_isConnected.value = false
},
onEvent = { event, payloadJson ->
handleBridgeEvent(event, payloadJson)
},
@@ -128,22 +76,10 @@ class NodeRuntime(context: Context) {
},
)
private val chat = ChatController(scope = scope, session = session, json = json)
private fun handleSessionDisconnected(message: String) {
_statusText.value = message
_serverName.value = null
_remoteAddress.value = null
_isConnected.value = false
chat.onDisconnected(message)
}
val instanceId: StateFlow<String> = prefs.instanceId
val displayName: StateFlow<String> = prefs.displayName
val cameraEnabled: StateFlow<Boolean> = prefs.cameraEnabled
val preventSleep: StateFlow<Boolean> = prefs.preventSleep
val wakeWords: StateFlow<List<String>> = prefs.wakeWords
val voiceWakeMode: StateFlow<VoiceWakeMode> = prefs.voiceWakeMode
val manualEnabled: StateFlow<Boolean> = prefs.manualEnabled
val manualHost: StateFlow<String> = prefs.manualHost
val manualPort: StateFlow<Int> = prefs.manualPort
@@ -153,51 +89,19 @@ class NodeRuntime(context: Context) {
private var suppressWakeWordsSync = false
private var wakeWordsSyncJob: Job? = null
val chatSessionKey: StateFlow<String> = chat.sessionKey
val chatSessionId: StateFlow<String?> = chat.sessionId
val chatMessages: StateFlow<List<ChatMessage>> = chat.messages
val chatError: StateFlow<String?> = chat.errorText
val chatHealthOk: StateFlow<Boolean> = chat.healthOk
val chatThinkingLevel: StateFlow<String> = chat.thinkingLevel
val chatStreamingAssistantText: StateFlow<String?> = chat.streamingAssistantText
val chatPendingToolCalls: StateFlow<List<ChatPendingToolCall>> = chat.pendingToolCalls
val chatSessions: StateFlow<List<ChatSessionEntry>> = chat.sessions
val pendingRunCount: StateFlow<Int> = chat.pendingRunCount
data class ChatMessage(val id: String, val role: String, val text: String, val timestampMs: Long?)
private val _chatMessages = MutableStateFlow<List<ChatMessage>>(emptyList())
val chatMessages: StateFlow<List<ChatMessage>> = _chatMessages.asStateFlow()
private val _chatError = MutableStateFlow<String?>(null)
val chatError: StateFlow<String?> = _chatError.asStateFlow()
private val pendingRuns = mutableSetOf<String>()
private val _pendingRunCount = MutableStateFlow(0)
val pendingRunCount: StateFlow<Int> = _pendingRunCount.asStateFlow()
init {
scope.launch {
combine(
voiceWakeMode,
isForeground,
externalAudioCaptureActive,
wakeWords,
) { mode, foreground, externalAudio, words ->
Quad(mode, foreground, externalAudio, words)
}.distinctUntilChanged()
.collect { (mode, foreground, externalAudio, words) ->
voiceWake.setTriggerWords(words)
val shouldListen =
when (mode) {
VoiceWakeMode.Off -> false
VoiceWakeMode.Foreground -> foreground
VoiceWakeMode.Always -> true
} && !externalAudio
if (!shouldListen) {
voiceWake.stop(statusText = if (mode == VoiceWakeMode.Off) "Off" else "Paused")
return@collect
}
if (!hasRecordAudioPermission()) {
voiceWake.stop(statusText = "Microphone permission required")
return@collect
}
voiceWake.start()
}
}
scope.launch(Dispatchers.Default) {
bridges.collect { list ->
if (list.isNotEmpty()) {
@@ -242,10 +146,6 @@ class NodeRuntime(context: Context) {
prefs.setCameraEnabled(value)
}
fun setPreventSleep(value: Boolean) {
prefs.setPreventSleep(value)
}
fun setManualEnabled(value: Boolean) {
prefs.setManualEnabled(value)
}
@@ -267,46 +167,13 @@ class NodeRuntime(context: Context) {
setWakeWords(SecurePrefs.defaultWakeWords)
}
fun setVoiceWakeMode(mode: VoiceWakeMode) {
prefs.setVoiceWakeMode(mode)
}
fun connect(endpoint: BridgeEndpoint) {
scope.launch {
_statusText.value = "Connecting…"
val storedToken = prefs.loadBridgeToken()
val modelIdentifier = listOfNotNull(Build.MANUFACTURER, Build.MODEL)
.joinToString(" ")
.trim()
.ifEmpty { null }
val invokeCommands =
buildList {
add(ClawdisCanvasCommand.Present.rawValue)
add(ClawdisCanvasCommand.Hide.rawValue)
add(ClawdisCanvasCommand.Navigate.rawValue)
add(ClawdisCanvasCommand.Eval.rawValue)
add(ClawdisCanvasCommand.Snapshot.rawValue)
add(ClawdisCanvasA2UICommand.Push.rawValue)
add(ClawdisCanvasA2UICommand.PushJSONL.rawValue)
add(ClawdisCanvasA2UICommand.Reset.rawValue)
add(ClawdisScreenCommand.Record.rawValue)
if (cameraEnabled.value) {
add(ClawdisCameraCommand.Snap.rawValue)
add(ClawdisCameraCommand.Clip.rawValue)
}
}
val resolved =
if (storedToken.isNullOrBlank()) {
_statusText.value = "Pairing…"
val caps = buildList {
add(ClawdisCapability.Canvas.rawValue)
add(ClawdisCapability.Screen.rawValue)
if (cameraEnabled.value) add(ClawdisCapability.Camera.rawValue)
if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) {
add(ClawdisCapability.VoiceWake.rawValue)
}
}
BridgePairingClient().pairAndHello(
endpoint = endpoint,
hello =
@@ -317,9 +184,7 @@ class NodeRuntime(context: Context) {
platform = "Android",
version = "dev",
deviceFamily = "Android",
modelIdentifier = modelIdentifier,
caps = caps,
commands = invokeCommands,
modelIdentifier = Build.MODEL,
),
)
} else {
@@ -343,29 +208,12 @@ class NodeRuntime(context: Context) {
platform = "Android",
version = "dev",
deviceFamily = "Android",
modelIdentifier = modelIdentifier,
caps =
buildList {
add(ClawdisCapability.Canvas.rawValue)
add(ClawdisCapability.Screen.rawValue)
if (cameraEnabled.value) add(ClawdisCapability.Camera.rawValue)
if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) {
add(ClawdisCapability.VoiceWake.rawValue)
}
},
commands = invokeCommands,
modelIdentifier = Build.MODEL,
),
)
}
}
private fun hasRecordAudioPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
)
}
fun connectManual() {
val host = manualHost.value.trim()
val port = manualPort.value
@@ -380,110 +228,58 @@ class NodeRuntime(context: Context) {
session.disconnect()
}
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
fun loadChat(sessionKey: String = "main") {
scope.launch {
val trimmed = payloadJson.trim()
if (trimmed.isEmpty()) return@launch
val root =
try {
json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch
} catch (_: Throwable) {
return@launch
}
val userActionObj = (root["userAction"] as? JsonObject) ?: root
val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty {
java.util.UUID.randomUUID().toString()
}
val name = (userActionObj["name"] as? JsonPrimitive)?.content?.trim().orEmpty()
if (name.isEmpty()) return@launch
val surfaceId =
(userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" }
val sourceComponentId =
(userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" }
val contextJson = (userActionObj["context"] as? JsonObject)?.toString()
val sessionKey = "main"
val message =
ClawdisCanvasA2UIAction.formatAgentMessage(
actionName = name,
sessionKey = sessionKey,
surfaceId = surfaceId,
sourceComponentId = sourceComponentId,
host = displayName.value,
instanceId = instanceId.value.lowercase(),
contextJson = contextJson,
)
val connected = isConnected.value
var error: String? = null
if (connected) {
try {
session.sendEvent(
event = "agent.request",
payloadJson =
buildJsonObject {
put("message", JsonPrimitive(message))
put("sessionKey", JsonPrimitive(sessionKey))
put("thinking", JsonPrimitive("low"))
put("deliver", JsonPrimitive(false))
put("key", JsonPrimitive(actionId))
}.toString(),
)
} catch (e: Throwable) {
error = e.message ?: "send failed"
}
} else {
error = "bridge not connected"
_chatError.value = null
try {
// Best-effort; push events are optional, but improve latency.
session.sendEvent("chat.subscribe", """{"sessionKey":"$sessionKey"}""")
} catch (_: Throwable) {
// ignore
}
try {
canvas.eval(
ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(
actionId = actionId,
ok = connected && error == null,
error = error,
),
)
} catch (_: Throwable) {
// ignore
val res = session.request("chat.history", """{"sessionKey":"$sessionKey"}""")
_chatMessages.value = parseHistory(res)
} catch (e: Exception) {
_chatError.value = e.message
}
}
}
fun loadChat(sessionKey: String = "main") {
chat.load(sessionKey)
}
fun sendChat(sessionKey: String = "main", message: String, thinking: String = "off") {
val trimmed = message.trim()
if (trimmed.isEmpty()) return
scope.launch {
_chatError.value = null
val idem = java.util.UUID.randomUUID().toString()
fun refreshChat() {
chat.refresh()
}
_chatMessages.value =
_chatMessages.value +
ChatMessage(
id = java.util.UUID.randomUUID().toString(),
role = "user",
text = trimmed,
timestampMs = System.currentTimeMillis(),
)
fun refreshChatSessions(limit: Int? = null) {
chat.refreshSessions(limit = limit)
}
fun setChatThinkingLevel(level: String) {
chat.setThinkingLevel(level)
}
fun switchChatSession(sessionKey: String) {
chat.switchSession(sessionKey)
}
fun abortChat() {
chat.abort()
}
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
chat.sendMessage(message = message, thinkingLevel = thinking, attachments = attachments)
try {
val params =
"""{"sessionKey":"$sessionKey","message":${trimmed.toJsonString()},"thinking":"$thinking","timeoutMs":30000,"idempotencyKey":"$idem"}"""
val res = session.request("chat.send", params)
val runId = parseRunId(res) ?: idem
pendingRuns.add(runId)
_pendingRunCount.value = pendingRuns.size
} catch (e: Exception) {
_chatError.value = e.message
}
}
}
private fun handleBridgeEvent(event: String, payloadJson: String?) {
if (payloadJson.isNullOrBlank()) return
if (event == "voicewake.changed") {
if (payloadJson.isNullOrBlank()) return
try {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val array = payload["triggers"] as? JsonArray ?: return
@@ -495,7 +291,40 @@ class NodeRuntime(context: Context) {
return
}
chat.handleBridgeEvent(event, payloadJson)
if (event != "chat") return
try {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val state = payload["state"].asStringOrNull()
val runId = payload["runId"].asStringOrNull()
if (!runId.isNullOrBlank()) {
pendingRuns.remove(runId)
_pendingRunCount.value = pendingRuns.size
}
when (state) {
"final" -> {
val msgObj = payload["message"].asObjectOrNull()
val role = msgObj?.get("role").asStringOrNull() ?: "assistant"
val text = extractTextFromMessage(msgObj)
if (!text.isNullOrBlank()) {
_chatMessages.value =
_chatMessages.value +
ChatMessage(
id = java.util.UUID.randomUUID().toString(),
role = role,
text = text,
timestampMs = System.currentTimeMillis(),
)
}
}
"error" -> {
_chatError.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed"
}
}
} catch (_: Throwable) {
// ignore
}
}
private fun applyWakeWordsFromGateway(words: List<String>) {
@@ -536,21 +365,56 @@ class NodeRuntime(context: Context) {
}
}
private fun parseHistory(historyJson: String): List<ChatMessage> {
val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return emptyList()
val raw = root["messages"] ?: return emptyList()
val array = raw as? JsonArray ?: return emptyList()
return array.mapNotNull { item ->
val obj = item as? JsonObject ?: return@mapNotNull null
val role = obj["role"].asStringOrNull() ?: return@mapNotNull null
val text = extractTextFromMessage(obj) ?: return@mapNotNull null
ChatMessage(
id = java.util.UUID.randomUUID().toString(),
role = role,
text = text,
timestampMs = null,
)
}
}
private fun extractTextFromMessage(msgObj: JsonObject?): String? {
if (msgObj == null) return null
val content = msgObj["content"] ?: return null
return when (content) {
is JsonPrimitive -> content.asStringOrNull()
else -> {
val arr = (content as? JsonArray) ?: return null
arr.mapNotNull { part ->
val p = part as? JsonObject ?: return@mapNotNull null
p["text"].asStringOrNull()
}.joinToString("\n").trim().ifBlank { null }
}
}
}
private fun parseRunId(resJson: String): String? {
return try {
json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull()
} catch (_: Throwable) {
null
}
}
private suspend fun handleInvoke(command: String, paramsJson: String?): BridgeSession.InvokeResult {
if (
command.startsWith(ClawdisCanvasCommand.NamespacePrefix) ||
command.startsWith(ClawdisCanvasA2UICommand.NamespacePrefix) ||
command.startsWith(ClawdisCameraCommand.NamespacePrefix) ||
command.startsWith(ClawdisScreenCommand.NamespacePrefix)
) {
if (command.startsWith("screen.") || command.startsWith("camera.")) {
if (!isForeground.value) {
return BridgeSession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground",
message = "NODE_BACKGROUND_UNAVAILABLE: screen/camera commands require foreground",
)
}
}
if (command.startsWith(ClawdisCameraCommand.NamespacePrefix) && !cameraEnabled.value) {
if (command.startsWith("camera.") && !cameraEnabled.value) {
return BridgeSession.InvokeResult.error(
code = "CAMERA_DISABLED",
message = "CAMERA_DISABLED: enable Camera in Settings",
@@ -558,18 +422,19 @@ class NodeRuntime(context: Context) {
}
return when (command) {
ClawdisCanvasCommand.Present.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
"screen.show" -> BridgeSession.InvokeResult.ok(null)
"screen.hide" -> BridgeSession.InvokeResult.ok(null)
"screen.setMode" -> {
val mode = CanvasController.parseMode(paramsJson)
canvas.setMode(mode)
BridgeSession.InvokeResult.ok(null)
}
ClawdisCanvasCommand.Hide.rawValue -> BridgeSession.InvokeResult.ok(null)
ClawdisCanvasCommand.Navigate.rawValue -> {
"screen.navigate" -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
if (url != null) canvas.navigate(url)
BridgeSession.InvokeResult.ok(null)
}
ClawdisCanvasCommand.Eval.rawValue -> {
"screen.eval" -> {
val js =
CanvasController.parseEvalJs(paramsJson)
?: return BridgeSession.InvokeResult.error(
@@ -587,87 +452,25 @@ class NodeRuntime(context: Context) {
}
BridgeSession.InvokeResult.ok("""{"result":${result.toJsonString()}}""")
}
ClawdisCanvasCommand.Snapshot.rawValue -> {
val snapshotParams = CanvasController.parseSnapshotParams(paramsJson)
"screen.snapshot" -> {
val maxWidth = CanvasController.parseSnapshotMaxWidth(paramsJson)
val base64 =
try {
canvas.snapshotBase64(
format = snapshotParams.format,
quality = snapshotParams.quality,
maxWidth = snapshotParams.maxWidth,
)
canvas.snapshotPngBase64(maxWidth = maxWidth)
} catch (err: Throwable) {
return BridgeSession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
)
}
BridgeSession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""")
BridgeSession.InvokeResult.ok("""{"format":"png","base64":"$base64"}""")
}
ClawdisCanvasA2UICommand.Reset.rawValue -> {
val ready = ensureA2uiReady()
if (!ready) {
return BridgeSession.InvokeResult.error(code = "A2UI_NOT_READY", message = "A2UI not ready")
}
val res = canvas.eval(a2uiResetJS)
BridgeSession.InvokeResult.ok(res)
}
ClawdisCanvasA2UICommand.Push.rawValue, ClawdisCanvasA2UICommand.PushJSONL.rawValue -> {
val messages =
try {
decodeA2uiMessages(command, paramsJson)
} catch (err: Throwable) {
return BridgeSession.InvokeResult.error(code = "INVALID_REQUEST", message = err.message ?: "invalid A2UI payload")
}
val ready = ensureA2uiReady()
if (!ready) {
return BridgeSession.InvokeResult.error(code = "A2UI_NOT_READY", message = "A2UI not ready")
}
val js = a2uiApplyMessagesJS(messages)
val res = canvas.eval(js)
BridgeSession.InvokeResult.ok(res)
}
ClawdisCameraCommand.Snap.rawValue -> {
showCameraHud(message = "Taking photo…", kind = CameraHudKind.Photo)
triggerCameraFlash()
val res =
try {
camera.snap(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2200)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
showCameraHud(message = "Photo captured", kind = CameraHudKind.Success, autoHideMs = 1600)
"camera.snap" -> {
val res = camera.snap(paramsJson)
BridgeSession.InvokeResult.ok(res.payloadJson)
}
ClawdisCameraCommand.Clip.rawValue -> {
val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false
if (includeAudio) externalAudioCaptureActive.value = true
try {
showCameraHud(message = "Recording…", kind = CameraHudKind.Recording)
val res =
try {
camera.clip(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2400)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
showCameraHud(message = "Clip captured", kind = CameraHudKind.Success, autoHideMs = 1800)
BridgeSession.InvokeResult.ok(res.payloadJson)
} finally {
if (includeAudio) externalAudioCaptureActive.value = false
}
}
ClawdisScreenCommand.Record.rawValue -> {
val res =
try {
screenRecorder.record(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
"camera.clip" -> {
val res = camera.clip(paramsJson)
BridgeSession.InvokeResult.ok(res.payloadJson)
}
else ->
@@ -677,155 +480,6 @@ class NodeRuntime(context: Context) {
)
}
}
private fun triggerCameraFlash() {
// Token is used as a pulse trigger; value doesn't matter as long as it changes.
_cameraFlashToken.value = SystemClock.elapsedRealtimeNanos()
}
private fun showCameraHud(message: String, kind: CameraHudKind, autoHideMs: Long? = null) {
val token = cameraHudSeq.incrementAndGet()
_cameraHud.value = CameraHudState(token = token, kind = kind, message = message)
if (autoHideMs != null && autoHideMs > 0) {
scope.launch {
delay(autoHideMs)
if (_cameraHud.value?.token == token) _cameraHud.value = null
}
}
}
private fun invokeErrorFromThrowable(err: Throwable): Pair<String, String> {
val raw = (err.message ?: "").trim()
if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: camera error"
val idx = raw.indexOf(':')
if (idx <= 0) return "UNAVAILABLE" to raw
val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" }
val message = raw.substring(idx + 1).trim().ifEmpty { raw }
// Preserve full string for callers/logging, but keep the returned message human-friendly.
return code to "$code: $message"
}
private suspend fun ensureA2uiReady(): Boolean {
try {
val already = canvas.eval(a2uiReadyCheckJS)
if (already == "true") return true
} catch (_: Throwable) {
// ignore
}
// Ensure the default canvas scaffold is loaded; A2UI is now hosted there.
canvas.navigate("")
repeat(50) {
try {
val ready = canvas.eval(a2uiReadyCheckJS)
if (ready == "true") return true
} catch (_: Throwable) {
// ignore
}
delay(120)
}
return false
}
private fun decodeA2uiMessages(command: String, paramsJson: String?): String {
val raw = paramsJson?.trim().orEmpty()
if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
val hasMessagesArray = obj["messages"] is JsonArray
if (command == ClawdisCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) {
val jsonl = jsonlField
if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
val messages =
jsonl
.lineSequence()
.map { it.trim() }
.filter { it.isNotBlank() }
.mapIndexed { idx, line ->
val el = json.parseToJsonElement(line)
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
.toList()
return JsonArray(messages).toString()
}
val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
val out =
arr.mapIndexed { idx, el ->
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
return JsonArray(out).toString()
}
private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) {
if (msg.containsKey("createSurface")) {
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.",
)
}
val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface")
val matched = msg.keys.filter { allowed.contains(it) }
if (matched.size != 1) {
val found = msg.keys.sorted().joinToString(", ")
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found",
)
}
}
}
private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
private const val a2uiReadyCheckJS: String =
"""
(() => {
try {
return !!globalThis.clawdisA2UI && typeof globalThis.clawdisA2UI.applyMessages === 'function';
} catch (_) {
return false;
}
})()
"""
private const val a2uiResetJS: String =
"""
(() => {
try {
if (!globalThis.clawdisA2UI) return { ok: false, error: "missing clawdisA2UI" };
return globalThis.clawdisA2UI.reset();
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
"""
private fun a2uiApplyMessagesJS(messagesJson: String): String {
return """
(() => {
try {
if (!globalThis.clawdisA2UI) return { ok: false, error: "missing clawdisA2UI" };
const messages = $messagesJson;
return globalThis.clawdisA2UI.applyMessages(messages);
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
""".trimIndent()
}
private fun String.toJsonString(): String {
@@ -1,132 +0,0 @@
package com.steipete.clawdis.node
import android.content.pm.PackageManager
import android.content.Intent
import android.Manifest
import android.net.Uri
import android.provider.Settings
import androidx.appcompat.app.AlertDialog
import androidx.activity.ComponentActivity
import androidx.activity.result.ActivityResultLauncher
import androidx.activity.result.contract.ActivityResultContracts
import androidx.core.content.ContextCompat
import androidx.core.app.ActivityCompat
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlin.coroutines.resume
class PermissionRequester(private val activity: ComponentActivity) {
private val mutex = Mutex()
private var pending: CompletableDeferred<Map<String, Boolean>>? = null
private val launcher: ActivityResultLauncher<Array<String>> =
activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result ->
val p = pending
pending = null
p?.complete(result)
}
suspend fun requestIfMissing(
permissions: List<String>,
timeoutMs: Long = 20_000,
): Map<String, Boolean> =
mutex.withLock {
val missing =
permissions.filter { perm ->
ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED
}
if (missing.isEmpty()) {
return permissions.associateWith { true }
}
val needsRationale =
missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) }
if (needsRationale) {
val proceed = showRationaleDialog(missing)
if (!proceed) {
return permissions.associateWith { perm ->
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
}
}
}
val deferred = CompletableDeferred<Map<String, Boolean>>()
pending = deferred
withContext(Dispatchers.Main) {
launcher.launch(missing.toTypedArray())
}
val result =
withContext(Dispatchers.Default) {
kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() }
}
// Merge: if something was already granted, treat it as granted even if launcher omitted it.
val merged =
permissions.associateWith { perm ->
val nowGranted =
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
result[perm] == true || nowGranted
}
val denied =
merged.filterValues { !it }.keys.filter {
!ActivityCompat.shouldShowRequestPermissionRationale(activity, it)
}
if (denied.isNotEmpty()) {
showSettingsDialog(denied)
}
return merged
}
private suspend fun showRationaleDialog(permissions: List<String>): Boolean =
withContext(Dispatchers.Main) {
suspendCancellableCoroutine { cont ->
AlertDialog.Builder(activity)
.setTitle("Permission required")
.setMessage(buildRationaleMessage(permissions))
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
.setOnCancelListener { cont.resume(false) }
.show()
}
}
private fun showSettingsDialog(permissions: List<String>) {
AlertDialog.Builder(activity)
.setTitle("Enable permission in Settings")
.setMessage(buildSettingsMessage(permissions))
.setPositiveButton("Open Settings") { _, _ ->
val intent =
Intent(
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
Uri.fromParts("package", activity.packageName, null),
)
activity.startActivity(intent)
}
.setNegativeButton("Cancel", null)
.show()
}
private fun buildRationaleMessage(permissions: List<String>): String {
val labels = permissions.map { permissionLabel(it) }
return "Clawdis needs ${labels.joinToString(", ")} to capture camera media."
}
private fun buildSettingsMessage(permissions: List<String>): String {
val labels = permissions.map { permissionLabel(it) }
return "Please enable ${labels.joinToString(", ")} in Android Settings to continue."
}
private fun permissionLabel(permission: String): String =
when (permission) {
Manifest.permission.CAMERA -> "Camera"
Manifest.permission.RECORD_AUDIO -> "Microphone"
else -> permission
}
}
@@ -1,65 +0,0 @@
package com.steipete.clawdis.node
import android.app.Activity
import android.content.Context
import android.content.Intent
import android.media.projection.MediaProjectionManager
import androidx.activity.ComponentActivity
import androidx.activity.result.ActivityResultLauncher
import androidx.activity.result.contract.ActivityResultContracts
import androidx.appcompat.app.AlertDialog
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeout
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlin.coroutines.resume
class ScreenCaptureRequester(private val activity: ComponentActivity) {
data class CaptureResult(val resultCode: Int, val data: Intent)
private val mutex = Mutex()
private var pending: CompletableDeferred<CaptureResult?>? = null
private val launcher: ActivityResultLauncher<Intent> =
activity.registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
val p = pending
pending = null
val data = result.data
if (result.resultCode == Activity.RESULT_OK && data != null) {
p?.complete(CaptureResult(result.resultCode, data))
} else {
p?.complete(null)
}
}
suspend fun requestCapture(timeoutMs: Long = 20_000): CaptureResult? =
mutex.withLock {
val proceed = showRationaleDialog()
if (!proceed) return null
val mgr = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
val intent = mgr.createScreenCaptureIntent()
val deferred = CompletableDeferred<CaptureResult?>()
pending = deferred
withContext(Dispatchers.Main) { launcher.launch(intent) }
withContext(Dispatchers.Default) { withTimeout(timeoutMs) { deferred.await() } }
}
private suspend fun showRationaleDialog(): Boolean =
withContext(Dispatchers.Main) {
suspendCancellableCoroutine { cont ->
AlertDialog.Builder(activity)
.setTitle("Screen recording required")
.setMessage("Clawdis needs to record the screen for this command.")
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
.setOnCancelListener { cont.resume(false) }
.show()
}
}
}
@@ -1,5 +1,3 @@
@file:Suppress("DEPRECATION")
package com.steipete.clawdis.node
import android.content.Context
@@ -16,8 +14,6 @@ import java.util.UUID
class SecurePrefs(context: Context) {
companion object {
val defaultWakeWords: List<String> = listOf("clawd", "claude")
private const val displayNameKey = "node.displayName"
private const val voiceWakeModeKey = "voiceWake.mode"
}
private val json = Json { ignoreUnknownKeys = true }
@@ -39,16 +35,12 @@ class SecurePrefs(context: Context) {
private val _instanceId = MutableStateFlow(loadOrCreateInstanceId())
val instanceId: StateFlow<String> = _instanceId
private val _displayName =
MutableStateFlow(loadOrMigrateDisplayName(context = context))
private val _displayName = MutableStateFlow(prefs.getString("node.displayName", "Android Node")!!)
val displayName: StateFlow<String> = _displayName
private val _cameraEnabled = MutableStateFlow(prefs.getBoolean("camera.enabled", true))
val cameraEnabled: StateFlow<Boolean> = _cameraEnabled
private val _preventSleep = MutableStateFlow(prefs.getBoolean("screen.preventSleep", true))
val preventSleep: StateFlow<Boolean> = _preventSleep
private val _manualEnabled = MutableStateFlow(prefs.getBoolean("bridge.manual.enabled", false))
val manualEnabled: StateFlow<Boolean> = _manualEnabled
@@ -65,9 +57,6 @@ class SecurePrefs(context: Context) {
private val _wakeWords = MutableStateFlow(loadWakeWords())
val wakeWords: StateFlow<List<String>> = _wakeWords
private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode())
val voiceWakeMode: StateFlow<VoiceWakeMode> = _voiceWakeMode
fun setLastDiscoveredStableId(value: String) {
val trimmed = value.trim()
prefs.edit().putString("bridge.lastDiscoveredStableId", trimmed).apply()
@@ -76,7 +65,7 @@ class SecurePrefs(context: Context) {
fun setDisplayName(value: String) {
val trimmed = value.trim()
prefs.edit().putString(displayNameKey, trimmed).apply()
prefs.edit().putString("node.displayName", trimmed).apply()
_displayName.value = trimmed
}
@@ -85,11 +74,6 @@ class SecurePrefs(context: Context) {
_cameraEnabled.value = value
}
fun setPreventSleep(value: Boolean) {
prefs.edit().putBoolean("screen.preventSleep", value).apply()
_preventSleep.value = value
}
fun setManualEnabled(value: Boolean) {
prefs.edit().putBoolean("bridge.manual.enabled", value).apply()
_manualEnabled.value = value
@@ -124,17 +108,6 @@ class SecurePrefs(context: Context) {
return fresh
}
private fun loadOrMigrateDisplayName(context: Context): String {
val existing = prefs.getString(displayNameKey, null)?.trim().orEmpty()
if (existing.isNotEmpty() && existing != "Android Node") return existing
val candidate = DeviceNames.bestDefaultNodeName(context).trim()
val resolved = candidate.ifEmpty { "Android Node" }
prefs.edit().putString(displayNameKey, resolved).apply()
return resolved
}
fun setWakeWords(words: List<String>) {
val sanitized = WakeWords.sanitize(words, defaultWakeWords)
val encoded =
@@ -143,23 +116,6 @@ class SecurePrefs(context: Context) {
_wakeWords.value = sanitized
}
fun setVoiceWakeMode(mode: VoiceWakeMode) {
prefs.edit().putString(voiceWakeModeKey, mode.rawValue).apply()
_voiceWakeMode.value = mode
}
private fun loadVoiceWakeMode(): VoiceWakeMode {
val raw = prefs.getString(voiceWakeModeKey, null)
val resolved = VoiceWakeMode.fromRawValue(raw)
// Default ON (foreground) when unset.
if (raw.isNullOrBlank()) {
prefs.edit().putString(voiceWakeModeKey, resolved.rawValue).apply()
}
return resolved
}
private fun loadWakeWords(): List<String> {
val raw = prefs.getString("voiceWake.triggerWords", null)?.trim()
if (raw.isNullOrEmpty()) return defaultWakeWords
@@ -1,15 +0,0 @@
package com.steipete.clawdis.node
enum class VoiceWakeMode(val rawValue: String) {
Off("off"),
Foreground("foreground"),
Always("always"),
;
companion object {
fun fromRawValue(raw: String?): VoiceWakeMode {
return entries.firstOrNull { it.rawValue == raw?.trim()?.lowercase() } ?: Foreground
}
}
}
@@ -1,30 +0,0 @@
package com.steipete.clawdis.node.bridge
object BonjourEscapes {
fun decode(input: String): String {
if (input.isEmpty()) return input
val out = StringBuilder(input.length)
var i = 0
while (i < input.length) {
if (input[i] == '\\' && i + 3 < input.length) {
val d0 = input[i + 1]
val d1 = input[i + 2]
val d2 = input[i + 3]
if (d0.isDigit() && d1.isDigit() && d2.isDigit()) {
val value =
((d0.code - '0'.code) * 100) + ((d1.code - '0'.code) * 10) + (d2.code - '0'.code)
if (value in 0..0x10FFFF) {
out.appendCodePoint(value)
i += 4
continue
}
}
}
out.append(input[i])
i += 1
}
return out.toString()
}
}
@@ -1,21 +1,11 @@
package com.steipete.clawdis.node.bridge
import android.content.Context
import android.net.ConnectivityManager
import android.net.DnsResolver
import android.net.NetworkCapabilities
import android.net.nsd.NsdManager
import android.net.nsd.NsdServiceInfo
import android.os.Build
import android.os.CancellationSignal
import android.util.Log
import java.io.IOException
import java.net.InetSocketAddress
import java.nio.ByteBuffer
import java.nio.charset.CodingErrorAction
import java.net.InetAddress
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Executor
import java.util.concurrent.Executors
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
@@ -24,50 +14,26 @@ import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import org.xbill.DNS.AAAARecord
import org.xbill.DNS.ARecord
import org.xbill.DNS.DClass
import org.xbill.DNS.ExtendedResolver
import org.xbill.DNS.Message
import org.xbill.DNS.Name
import org.xbill.DNS.Lookup
import org.xbill.DNS.PTRRecord
import org.xbill.DNS.Record
import org.xbill.DNS.Rcode
import org.xbill.DNS.Resolver
import org.xbill.DNS.SRVRecord
import org.xbill.DNS.Section
import org.xbill.DNS.SimpleResolver
import org.xbill.DNS.TextParseException
import org.xbill.DNS.TXTRecord
import org.xbill.DNS.Type
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
class BridgeDiscovery(
context: Context,
private val scope: CoroutineScope,
) {
private val nsd = context.getSystemService(NsdManager::class.java)
private val connectivity = context.getSystemService(ConnectivityManager::class.java)
private val dns = DnsResolver.getInstance()
private val serviceType = "_clawdis-bridge._tcp."
private val wideAreaDomain = "clawdis.internal."
private val logTag = "Clawdis/BridgeDiscovery"
private val localById = ConcurrentHashMap<String, BridgeEndpoint>()
private val unicastById = ConcurrentHashMap<String, BridgeEndpoint>()
private val _bridges = MutableStateFlow<List<BridgeEndpoint>>(emptyList())
val bridges: StateFlow<List<BridgeEndpoint>> = _bridges.asStateFlow()
private val _statusText = MutableStateFlow("Searching…")
val statusText: StateFlow<String> = _statusText.asStateFlow()
private var unicastJob: Job? = null
private val dnsExecutor: Executor = Executors.newCachedThreadPool()
@Volatile private var lastWideAreaRcode: Int? = null
@Volatile private var lastWideAreaCount: Int = 0
private val discoveryListener =
object : NsdManager.DiscoveryListener {
@@ -82,8 +48,7 @@ class BridgeDiscovery(
}
override fun onServiceLost(serviceInfo: NsdServiceInfo) {
val serviceName = BonjourEscapes.decode(serviceInfo.serviceName)
val id = stableId(serviceName, "local.")
val id = stableId(serviceInfo.serviceName, "local.")
localById.remove(id)
publish()
}
@@ -135,10 +100,8 @@ class BridgeDiscovery(
val port = resolved.port
if (port <= 0) return
val rawServiceName = resolved.serviceName
val serviceName = BonjourEscapes.decode(rawServiceName)
val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName)
val id = stableId(serviceName, "local.")
val displayName = txt(resolved, "displayName") ?: resolved.serviceName
val id = stableId(resolved.serviceName, "local.")
localById[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port)
publish()
}
@@ -149,27 +112,6 @@ class BridgeDiscovery(
private fun publish() {
_bridges.value =
(localById.values + unicastById.values).sortedBy { it.name.lowercase() }
_statusText.value = buildStatusText()
}
private fun buildStatusText(): String {
val localCount = localById.size
val wideRcode = lastWideAreaRcode
val wideCount = lastWideAreaCount
val wide =
when (wideRcode) {
null -> "Wide: ?"
Rcode.NOERROR -> "Wide: $wideCount"
Rcode.NXDOMAIN -> "Wide: NXDOMAIN"
else -> "Wide: ${Rcode.string(wideRcode)}"
}
return when {
localCount == 0 && wideRcode == null -> "Searching for bridges…"
localCount == 0 -> "$wide"
else -> "Local: $localCount • $wide"
}
}
private fun stableId(serviceName: String, domain: String): String {
@@ -192,58 +134,34 @@ class BridgeDiscovery(
private suspend fun refreshUnicast(domain: String) {
val ptrName = "${serviceType}${domain}"
val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return
val ptrRecords = records(ptrMsg, Section.ANSWER).mapNotNull { it as? PTRRecord }
val ptrRecords = lookup(ptrName, Type.PTR).mapNotNull { it as? PTRRecord }
val next = LinkedHashMap<String, BridgeEndpoint>()
for (ptr in ptrRecords) {
val instanceFqdn = ptr.target.toString()
val srv =
recordByName(ptrMsg, instanceFqdn, Type.SRV) as? SRVRecord
?: run {
val msg = lookupUnicastMessage(instanceFqdn, Type.SRV) ?: return@run null
recordByName(msg, instanceFqdn, Type.SRV) as? SRVRecord
}
?: continue
val srv = lookup(instanceFqdn, Type.SRV).firstOrNull { it is SRVRecord } as? SRVRecord ?: continue
val port = srv.port
if (port <= 0) continue
val targetFqdn = srv.target.toString()
val targetName = stripTrailingDot(srv.target.toString())
val host =
resolveHostFromMessage(ptrMsg, targetFqdn)
?: resolveHostFromMessage(lookupUnicastMessage(instanceFqdn, Type.SRV), targetFqdn)
?: resolveHostUnicast(targetFqdn)
?: continue
try {
val addrs = InetAddress.getAllByName(targetName).mapNotNull { it.hostAddress }
addrs.firstOrNull { !it.contains(":") } ?: addrs.firstOrNull()
} catch (_: Throwable) {
null
} ?: continue
val txtFromPtr =
recordsByName(ptrMsg, Section.ADDITIONAL)[keyName(instanceFqdn)]
.orEmpty()
.mapNotNull { it as? TXTRecord }
val txt =
if (txtFromPtr.isNotEmpty()) {
txtFromPtr
} else {
val msg = lookupUnicastMessage(instanceFqdn, Type.TXT)
records(msg, Section.ANSWER).mapNotNull { it as? TXTRecord }
}
val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain))
val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName)
val txt = lookup(instanceFqdn, Type.TXT).mapNotNull { it as? TXTRecord }
val instanceName = decodeInstanceName(instanceFqdn, domain)
val displayName = txtValue(txt, "displayName") ?: instanceName
val id = stableId(instanceName, domain)
next[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port)
}
unicastById.clear()
unicastById.putAll(next)
lastWideAreaRcode = ptrMsg.header.rcode
lastWideAreaCount = next.size
publish()
if (next.isEmpty()) {
Log.d(
logTag,
"wide-area discovery: 0 results for $ptrName (rcode=${Rcode.string(ptrMsg.header.rcode)})",
)
}
}
private fun decodeInstanceName(instanceFqdn: String, domain: String): String {
@@ -261,161 +179,15 @@ class BridgeDiscovery(
return raw.removeSuffix(".")
}
private suspend fun lookupUnicastMessage(name: String, type: Int): Message? {
val query =
try {
Message.newQuery(
org.xbill.DNS.Record.newRecord(
Name.fromString(name),
type,
DClass.IN,
),
)
} catch (_: TextParseException) {
return null
}
val system = queryViaSystemDns(query)
if (records(system, Section.ANSWER).any { it.type == type }) return system
val direct = createDirectResolver() ?: return system
private fun lookup(name: String, type: Int): List<org.xbill.DNS.Record> {
return try {
val msg = direct.send(query)
if (records(msg, Section.ANSWER).any { it.type == type }) msg else system
val out = Lookup(name, type).run() ?: return emptyList()
out.toList()
} catch (_: Throwable) {
system
emptyList()
}
}
private suspend fun queryViaSystemDns(query: Message): Message? {
val network = preferredDnsNetwork()
val bytes =
try {
rawQuery(network, query.toWire())
} catch (_: Throwable) {
return null
}
return try {
Message(bytes)
} catch (_: IOException) {
null
}
}
private fun records(msg: Message?, section: Int): List<Record> {
return msg?.getSectionArray(section)?.toList() ?: emptyList()
}
private fun keyName(raw: String): String {
return raw.trim().lowercase()
}
private fun recordsByName(msg: Message, section: Int): Map<String, List<Record>> {
val next = LinkedHashMap<String, MutableList<Record>>()
for (r in records(msg, section)) {
val name = r.name?.toString() ?: continue
next.getOrPut(keyName(name)) { mutableListOf() }.add(r)
}
return next
}
private fun recordByName(msg: Message, fqdn: String, type: Int): Record? {
val key = keyName(fqdn)
val byNameAnswer = recordsByName(msg, Section.ANSWER)
val fromAnswer = byNameAnswer[key].orEmpty().firstOrNull { it.type == type }
if (fromAnswer != null) return fromAnswer
val byNameAdditional = recordsByName(msg, Section.ADDITIONAL)
return byNameAdditional[key].orEmpty().firstOrNull { it.type == type }
}
private fun resolveHostFromMessage(msg: Message?, hostname: String): String? {
val m = msg ?: return null
val key = keyName(hostname)
val additional = recordsByName(m, Section.ADDITIONAL)[key].orEmpty()
val a = additional.mapNotNull { it as? ARecord }.mapNotNull { it.address?.hostAddress }
val aaaa = additional.mapNotNull { it as? AAAARecord }.mapNotNull { it.address?.hostAddress }
return a.firstOrNull() ?: aaaa.firstOrNull()
}
private fun preferredDnsNetwork(): android.net.Network? {
val cm = connectivity ?: return null
// Prefer VPN (Tailscale) when present; otherwise use the active network.
cm.allNetworks.firstOrNull { n ->
val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false
caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN)
}?.let { return it }
return cm.activeNetwork
}
private fun createDirectResolver(): Resolver? {
val cm = connectivity ?: return null
val candidateNetworks =
buildList {
cm.allNetworks
.firstOrNull { n ->
val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false
caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN)
}?.let(::add)
cm.activeNetwork?.let(::add)
}.distinct()
val servers =
candidateNetworks
.asSequence()
.flatMap { n ->
cm.getLinkProperties(n)?.dnsServers?.asSequence() ?: emptySequence()
}
.distinctBy { it.hostAddress ?: it.toString() }
.toList()
if (servers.isEmpty()) return null
return try {
val resolvers =
servers.mapNotNull { addr ->
try {
SimpleResolver().apply {
setAddress(InetSocketAddress(addr, 53))
setTimeout(3)
}
} catch (_: Throwable) {
null
}
}
if (resolvers.isEmpty()) return null
ExtendedResolver(resolvers.toTypedArray()).apply { setTimeout(3) }
} catch (_: Throwable) {
null
}
}
private suspend fun rawQuery(network: android.net.Network?, wireQuery: ByteArray): ByteArray =
suspendCancellableCoroutine { cont ->
val signal = CancellationSignal()
cont.invokeOnCancellation { signal.cancel() }
dns.rawQuery(
network,
wireQuery,
0,
dnsExecutor,
signal,
object : DnsResolver.Callback<ByteArray> {
override fun onAnswer(answer: ByteArray, rcode: Int) {
cont.resume(answer)
}
override fun onError(error: DnsResolver.DnsException) {
cont.resumeWithException(error)
}
},
)
}
private fun txtValue(records: List<TXTRecord>, key: String): String? {
val prefix = "$key="
for (r in records) {
@@ -426,7 +198,7 @@ class BridgeDiscovery(
emptyList()
}
for (s in strings) {
val trimmed = decodeDnsTxtString(s).trim()
val trimmed = s.trim()
if (trimmed.startsWith(prefix)) {
return trimmed.removePrefix(prefix).trim().ifEmpty { null }
}
@@ -434,33 +206,4 @@ class BridgeDiscovery(
}
return null
}
private fun decodeDnsTxtString(raw: String): String {
// dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes.
// Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible.
val bytes = raw.toByteArray(Charsets.ISO_8859_1)
val decoder =
Charsets.UTF_8
.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT)
return try {
decoder.decode(ByteBuffer.wrap(bytes)).toString()
} catch (_: Throwable) {
raw
}
}
private suspend fun resolveHostUnicast(hostname: String): String? {
val a =
records(lookupUnicastMessage(hostname, Type.A), Section.ANSWER)
.mapNotNull { it as? ARecord }
.mapNotNull { it.address?.hostAddress }
val aaaa =
records(lookupUnicastMessage(hostname, Type.AAAA), Section.ANSWER)
.mapNotNull { it as? AAAARecord }
.mapNotNull { it.address?.hostAddress }
return a.firstOrNull() ?: aaaa.firstOrNull()
}
}
@@ -3,7 +3,6 @@ package com.steipete.clawdis.node.bridge
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.JsonNull
@@ -27,8 +26,6 @@ class BridgePairingClient {
val version: String?,
val deviceFamily: String?,
val modelIdentifier: String?,
val caps: List<String>?,
val commands: List<String>?,
)
data class PairResult(val ok: Boolean, val token: String?, val error: String? = null)
@@ -62,8 +59,6 @@ class BridgePairingClient {
hello.version?.let { put("version", JsonPrimitive(it)) }
hello.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
hello.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
hello.caps?.let { put("caps", JsonArray(it.map(::JsonPrimitive))) }
hello.commands?.let { put("commands", JsonArray(it.map(::JsonPrimitive))) }
},
)
@@ -87,8 +82,6 @@ class BridgePairingClient {
hello.version?.let { put("version", JsonPrimitive(it)) }
hello.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
hello.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
hello.caps?.let { put("caps", JsonArray(it.map(::JsonPrimitive))) }
hello.commands?.let { put("commands", JsonArray(it.map(::JsonPrimitive))) }
},
)
@@ -12,7 +12,6 @@ import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonElement
@@ -42,8 +41,6 @@ class BridgeSession(
val version: String?,
val deviceFamily: String?,
val modelIdentifier: String?,
val caps: List<String>?,
val commands: List<String>?,
)
data class InvokeRequest(val id: String, val command: String, val paramsJson: String?)
@@ -77,7 +74,7 @@ class BridgeSession(
scope.launch(Dispatchers.IO) {
job?.cancelAndJoin()
job = null
onDisconnected("Offline")
onDisconnected("Disconnected")
}
}
@@ -198,8 +195,6 @@ class BridgeSession(
hello.version?.let { put("version", JsonPrimitive(it)) }
hello.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
hello.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
hello.caps?.let { put("caps", JsonArray(it.map(::JsonPrimitive))) }
hello.commands?.let { put("commands", JsonArray(it.map(::JsonPrimitive))) }
},
)
@@ -1,509 +0,0 @@
package com.steipete.clawdis.node.chat
import com.steipete.clawdis.node.bridge.BridgeSession
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
class ChatController(
private val scope: CoroutineScope,
private val session: BridgeSession,
private val json: Json,
) {
private val _sessionKey = MutableStateFlow("main")
val sessionKey: StateFlow<String> = _sessionKey.asStateFlow()
private val _sessionId = MutableStateFlow<String?>(null)
val sessionId: StateFlow<String?> = _sessionId.asStateFlow()
private val _messages = MutableStateFlow<List<ChatMessage>>(emptyList())
val messages: StateFlow<List<ChatMessage>> = _messages.asStateFlow()
private val _errorText = MutableStateFlow<String?>(null)
val errorText: StateFlow<String?> = _errorText.asStateFlow()
private val _healthOk = MutableStateFlow(false)
val healthOk: StateFlow<Boolean> = _healthOk.asStateFlow()
private val _thinkingLevel = MutableStateFlow("off")
val thinkingLevel: StateFlow<String> = _thinkingLevel.asStateFlow()
private val _pendingRunCount = MutableStateFlow(0)
val pendingRunCount: StateFlow<Int> = _pendingRunCount.asStateFlow()
private val _streamingAssistantText = MutableStateFlow<String?>(null)
val streamingAssistantText: StateFlow<String?> = _streamingAssistantText.asStateFlow()
private val pendingToolCallsById = ConcurrentHashMap<String, ChatPendingToolCall>()
private val _pendingToolCalls = MutableStateFlow<List<ChatPendingToolCall>>(emptyList())
val pendingToolCalls: StateFlow<List<ChatPendingToolCall>> = _pendingToolCalls.asStateFlow()
private val _sessions = MutableStateFlow<List<ChatSessionEntry>>(emptyList())
val sessions: StateFlow<List<ChatSessionEntry>> = _sessions.asStateFlow()
private val pendingRuns = mutableSetOf<String>()
private val pendingRunTimeoutJobs = ConcurrentHashMap<String, Job>()
private val pendingRunTimeoutMs = 120_000L
private var lastHealthPollAtMs: Long? = null
fun onDisconnected(message: String) {
_healthOk.value = false
// Not an error; keep connection status in the UI pill.
_errorText.value = null
clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
_sessionId.value = null
}
fun load(sessionKey: String = "main") {
val key = sessionKey.trim().ifEmpty { "main" }
_sessionKey.value = key
scope.launch { bootstrap(forceHealth = true) }
}
fun refresh() {
scope.launch { bootstrap(forceHealth = true) }
}
fun refreshSessions(limit: Int? = null) {
scope.launch { fetchSessions(limit = limit) }
}
fun setThinkingLevel(thinkingLevel: String) {
val normalized = normalizeThinking(thinkingLevel)
if (normalized == _thinkingLevel.value) return
_thinkingLevel.value = normalized
}
fun switchSession(sessionKey: String) {
val key = sessionKey.trim()
if (key.isEmpty()) return
if (key == _sessionKey.value) return
_sessionKey.value = key
scope.launch { bootstrap(forceHealth = true) }
}
fun sendMessage(
message: String,
thinkingLevel: String,
attachments: List<OutgoingAttachment>,
) {
val trimmed = message.trim()
if (trimmed.isEmpty() && attachments.isEmpty()) return
if (!_healthOk.value) {
_errorText.value = "Gateway health not OK; cannot send"
return
}
val runId = UUID.randomUUID().toString()
val text = if (trimmed.isEmpty() && attachments.isNotEmpty()) "See attached." else trimmed
val sessionKey = _sessionKey.value
val thinking = normalizeThinking(thinkingLevel)
// Optimistic user message.
val userContent =
buildList {
add(ChatMessageContent(type = "text", text = text))
for (att in attachments) {
add(
ChatMessageContent(
type = att.type,
mimeType = att.mimeType,
fileName = att.fileName,
base64 = att.base64,
),
)
}
}
_messages.value =
_messages.value +
ChatMessage(
id = UUID.randomUUID().toString(),
role = "user",
content = userContent,
timestampMs = System.currentTimeMillis(),
)
armPendingRunTimeout(runId)
synchronized(pendingRuns) {
pendingRuns.add(runId)
_pendingRunCount.value = pendingRuns.size
}
_errorText.value = null
_streamingAssistantText.value = null
pendingToolCallsById.clear()
publishPendingToolCalls()
scope.launch {
try {
val params =
buildJsonObject {
put("sessionKey", JsonPrimitive(sessionKey))
put("message", JsonPrimitive(text))
put("thinking", JsonPrimitive(thinking))
put("timeoutMs", JsonPrimitive(30_000))
put("idempotencyKey", JsonPrimitive(runId))
if (attachments.isNotEmpty()) {
put(
"attachments",
JsonArray(
attachments.map { att ->
buildJsonObject {
put("type", JsonPrimitive(att.type))
put("mimeType", JsonPrimitive(att.mimeType))
put("fileName", JsonPrimitive(att.fileName))
put("content", JsonPrimitive(att.base64))
}
},
),
)
}
}
val res = session.request("chat.send", params.toString())
val actualRunId = parseRunId(res) ?: runId
if (actualRunId != runId) {
clearPendingRun(runId)
armPendingRunTimeout(actualRunId)
synchronized(pendingRuns) {
pendingRuns.add(actualRunId)
_pendingRunCount.value = pendingRuns.size
}
}
} catch (err: Throwable) {
clearPendingRun(runId)
_errorText.value = err.message
}
}
}
fun abort() {
val runIds =
synchronized(pendingRuns) {
pendingRuns.toList()
}
if (runIds.isEmpty()) return
scope.launch {
for (runId in runIds) {
try {
val params =
buildJsonObject {
put("sessionKey", JsonPrimitive(_sessionKey.value))
put("runId", JsonPrimitive(runId))
}
session.request("chat.abort", params.toString())
} catch (_: Throwable) {
// best-effort
}
}
}
}
fun handleBridgeEvent(event: String, payloadJson: String?) {
when (event) {
"tick" -> {
scope.launch { pollHealthIfNeeded(force = false) }
}
"health" -> {
// If we receive a health snapshot, the gateway is reachable.
_healthOk.value = true
}
"seqGap" -> {
_errorText.value = "Event stream interrupted; try refreshing."
clearPendingRuns()
}
"chat" -> {
if (payloadJson.isNullOrBlank()) return
handleChatEvent(payloadJson)
}
"agent" -> {
if (payloadJson.isNullOrBlank()) return
handleAgentEvent(payloadJson)
}
}
}
private suspend fun bootstrap(forceHealth: Boolean) {
_errorText.value = null
_healthOk.value = false
clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
_sessionId.value = null
val key = _sessionKey.value
try {
try {
session.sendEvent("chat.subscribe", """{"sessionKey":"$key"}""")
} catch (_: Throwable) {
// best-effort
}
val historyJson = session.request("chat.history", """{"sessionKey":"$key"}""")
val history = parseHistory(historyJson, sessionKey = key)
_messages.value = history.messages
_sessionId.value = history.sessionId
history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it }
pollHealthIfNeeded(force = forceHealth)
fetchSessions(limit = 50)
} catch (err: Throwable) {
_errorText.value = err.message
}
}
private suspend fun fetchSessions(limit: Int?) {
try {
val params =
buildJsonObject {
put("includeGlobal", JsonPrimitive(true))
put("includeUnknown", JsonPrimitive(false))
if (limit != null && limit > 0) put("limit", JsonPrimitive(limit))
}
val res = session.request("sessions.list", params.toString())
_sessions.value = parseSessions(res)
} catch (_: Throwable) {
// best-effort
}
}
private suspend fun pollHealthIfNeeded(force: Boolean) {
val now = System.currentTimeMillis()
val last = lastHealthPollAtMs
if (!force && last != null && now - last < 10_000) return
lastHealthPollAtMs = now
try {
session.request("health", null)
_healthOk.value = true
} catch (_: Throwable) {
_healthOk.value = false
}
}
private fun handleChatEvent(payloadJson: String) {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val sessionKey = payload["sessionKey"].asStringOrNull()?.trim()
if (!sessionKey.isNullOrEmpty() && sessionKey != _sessionKey.value) return
val runId = payload["runId"].asStringOrNull()
if (runId != null) {
val isPending =
synchronized(pendingRuns) {
pendingRuns.contains(runId)
}
if (!isPending) return
}
val state = payload["state"].asStringOrNull()
when (state) {
"final", "aborted", "error" -> {
if (state == "error") {
_errorText.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed"
}
if (runId != null) clearPendingRun(runId) else clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
scope.launch {
try {
val historyJson =
session.request("chat.history", """{"sessionKey":"${_sessionKey.value}"}""")
val history = parseHistory(historyJson, sessionKey = _sessionKey.value)
_messages.value = history.messages
_sessionId.value = history.sessionId
history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it }
} catch (_: Throwable) {
// best-effort
}
}
}
}
}
private fun handleAgentEvent(payloadJson: String) {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val runId = payload["runId"].asStringOrNull()
val sessionId = _sessionId.value
if (sessionId != null && runId != sessionId) return
val stream = payload["stream"].asStringOrNull()
val data = payload["data"].asObjectOrNull()
when (stream) {
"assistant" -> {
val text = data?.get("text")?.asStringOrNull()
if (!text.isNullOrEmpty()) {
_streamingAssistantText.value = text
}
}
"tool" -> {
val phase = data?.get("phase")?.asStringOrNull()
val name = data?.get("name")?.asStringOrNull()
val toolCallId = data?.get("toolCallId")?.asStringOrNull()
if (phase.isNullOrEmpty() || name.isNullOrEmpty() || toolCallId.isNullOrEmpty()) return
val ts = payload["ts"].asLongOrNull() ?: System.currentTimeMillis()
if (phase == "start") {
pendingToolCallsById[toolCallId] =
ChatPendingToolCall(
toolCallId = toolCallId,
name = name,
startedAtMs = ts,
isError = null,
)
publishPendingToolCalls()
} else if (phase == "result") {
pendingToolCallsById.remove(toolCallId)
publishPendingToolCalls()
}
}
"error" -> {
_errorText.value = "Event stream interrupted; try refreshing."
clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
}
}
}
private fun publishPendingToolCalls() {
_pendingToolCalls.value =
pendingToolCallsById.values.sortedBy { it.startedAtMs }
}
private fun armPendingRunTimeout(runId: String) {
pendingRunTimeoutJobs[runId]?.cancel()
pendingRunTimeoutJobs[runId] =
scope.launch {
delay(pendingRunTimeoutMs)
val stillPending =
synchronized(pendingRuns) {
pendingRuns.contains(runId)
}
if (!stillPending) return@launch
clearPendingRun(runId)
_errorText.value = "Timed out waiting for a reply; try again or refresh."
}
}
private fun clearPendingRun(runId: String) {
pendingRunTimeoutJobs.remove(runId)?.cancel()
synchronized(pendingRuns) {
pendingRuns.remove(runId)
_pendingRunCount.value = pendingRuns.size
}
}
private fun clearPendingRuns() {
for ((_, job) in pendingRunTimeoutJobs) {
job.cancel()
}
pendingRunTimeoutJobs.clear()
synchronized(pendingRuns) {
pendingRuns.clear()
_pendingRunCount.value = 0
}
}
private fun parseHistory(historyJson: String, sessionKey: String): ChatHistory {
val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return ChatHistory(sessionKey, null, null, emptyList())
val sid = root["sessionId"].asStringOrNull()
val thinkingLevel = root["thinkingLevel"].asStringOrNull()
val array = root["messages"].asArrayOrNull() ?: JsonArray(emptyList())
val messages =
array.mapNotNull { item ->
val obj = item.asObjectOrNull() ?: return@mapNotNull null
val role = obj["role"].asStringOrNull() ?: return@mapNotNull null
val content = obj["content"].asArrayOrNull()?.mapNotNull(::parseMessageContent) ?: emptyList()
val ts = obj["timestamp"].asLongOrNull()
ChatMessage(
id = UUID.randomUUID().toString(),
role = role,
content = content,
timestampMs = ts,
)
}
return ChatHistory(sessionKey = sessionKey, sessionId = sid, thinkingLevel = thinkingLevel, messages = messages)
}
private fun parseMessageContent(el: JsonElement): ChatMessageContent? {
val obj = el.asObjectOrNull() ?: return null
val type = obj["type"].asStringOrNull() ?: "text"
return if (type == "text") {
ChatMessageContent(type = "text", text = obj["text"].asStringOrNull())
} else {
ChatMessageContent(
type = type,
mimeType = obj["mimeType"].asStringOrNull(),
fileName = obj["fileName"].asStringOrNull(),
base64 = obj["content"].asStringOrNull(),
)
}
}
private fun parseSessions(jsonString: String): List<ChatSessionEntry> {
val root = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return emptyList()
val sessions = root["sessions"].asArrayOrNull() ?: return emptyList()
return sessions.mapNotNull { item ->
val obj = item.asObjectOrNull() ?: return@mapNotNull null
val key = obj["key"].asStringOrNull()?.trim().orEmpty()
if (key.isEmpty()) return@mapNotNull null
val updatedAt = obj["updatedAt"].asLongOrNull()
ChatSessionEntry(key = key, updatedAtMs = updatedAt)
}
}
private fun parseRunId(resJson: String): String? {
return try {
json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull()
} catch (_: Throwable) {
null
}
}
private fun normalizeThinking(raw: String): String {
return when (raw.trim().lowercase()) {
"low" -> "low"
"medium" -> "medium"
"high" -> "high"
else -> "off"
}
}
}
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
private fun JsonElement?.asArrayOrNull(): JsonArray? = this as? JsonArray
private fun JsonElement?.asStringOrNull(): String? =
when (this) {
is JsonNull -> null
is JsonPrimitive -> content
else -> null
}
private fun JsonElement?.asLongOrNull(): Long? =
when (this) {
is JsonPrimitive -> content.toLongOrNull()
else -> null
}
@@ -1,42 +0,0 @@
package com.steipete.clawdis.node.chat
data class ChatMessage(
val id: String,
val role: String,
val content: List<ChatMessageContent>,
val timestampMs: Long?,
)
data class ChatMessageContent(
val type: String = "text",
val text: String? = null,
val mimeType: String? = null,
val fileName: String? = null,
val base64: String? = null,
)
data class ChatPendingToolCall(
val toolCallId: String,
val name: String,
val startedAtMs: Long,
val isError: Boolean? = null,
)
data class ChatSessionEntry(
val key: String,
val updatedAtMs: Long?,
)
data class ChatHistory(
val sessionKey: String,
val sessionId: String?,
val thinkingLevel: String?,
val messages: List<ChatMessage>,
)
data class OutgoingAttachment(
val type: String,
val mimeType: String,
val fileName: String,
val base64: String,
)
@@ -18,7 +18,6 @@ import androidx.camera.video.VideoCapture
import androidx.camera.video.VideoRecordEvent
import androidx.core.content.ContextCompat
import androidx.core.content.ContextCompat.checkSelfPermission
import com.steipete.clawdis.node.PermissionRequester
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withTimeout
@@ -33,43 +32,24 @@ class CameraCaptureManager(private val context: Context) {
data class Payload(val payloadJson: String)
@Volatile private var lifecycleOwner: LifecycleOwner? = null
@Volatile private var permissionRequester: PermissionRequester? = null
fun attachLifecycleOwner(owner: LifecycleOwner) {
lifecycleOwner = owner
}
fun attachPermissionRequester(requester: PermissionRequester) {
permissionRequester = requester
}
private suspend fun ensureCameraPermission() {
private fun requireCameraPermission() {
val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED
if (granted) return
val requester = permissionRequester
?: throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
val results = requester.requestIfMissing(listOf(Manifest.permission.CAMERA))
if (results[Manifest.permission.CAMERA] != true) {
throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
}
if (!granted) throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
}
private suspend fun ensureMicPermission() {
private fun requireMicPermission() {
val granted = checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
if (granted) return
val requester = permissionRequester
?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
val results = requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO))
if (results[Manifest.permission.RECORD_AUDIO] != true) {
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
}
if (!granted) throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
}
suspend fun snap(paramsJson: String?): Payload =
withContext(Dispatchers.Main) {
ensureCameraPermission()
requireCameraPermission()
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
val facing = parseFacing(paramsJson) ?: "front"
val quality = (parseQuality(paramsJson) ?: 0.9).coerceIn(0.1, 1.0)
@@ -110,12 +90,12 @@ class CameraCaptureManager(private val context: Context) {
suspend fun clip(paramsJson: String?): Payload =
withContext(Dispatchers.Main) {
ensureCameraPermission()
requireCameraPermission()
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
val facing = parseFacing(paramsJson) ?: "front"
val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 60_000)
val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 45_000)
val includeAudio = parseIncludeAudio(paramsJson) ?: true
if (includeAudio) ensureMicPermission()
if (includeAudio) requireMicPermission()
val provider = context.cameraProvider()
val recorder = Recorder.Builder().build()
@@ -3,63 +3,41 @@ package com.steipete.clawdis.node.node
import android.graphics.Bitmap
import android.os.Build
import android.graphics.Canvas
import android.os.Looper
import android.webkit.WebView
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import java.io.ByteArrayOutputStream
import android.util.Base64
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlin.coroutines.resume
class CanvasController {
enum class SnapshotFormat(val rawValue: String) {
Png("png"),
Jpeg("jpeg"),
}
enum class Mode { CANVAS, WEB }
@Volatile private var webView: WebView? = null
@Volatile private var url: String? = null
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
private fun clampJpegQuality(quality: Double?): Int {
val q = (quality ?: 0.82).coerceIn(0.1, 1.0)
return (q * 100.0).toInt().coerceIn(1, 100)
}
@Volatile private var mode: Mode = Mode.CANVAS
@Volatile private var url: String = ""
fun attach(webView: WebView) {
this.webView = webView
reload()
}
fun navigate(url: String) {
val trimmed = url.trim()
this.url = if (trimmed.isBlank() || trimmed == "/") null else trimmed
fun setMode(mode: Mode) {
this.mode = mode
reload()
}
private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) {
val wv = webView ?: return
if (Looper.myLooper() == Looper.getMainLooper()) {
block(wv)
} else {
wv.post { block(wv) }
}
fun navigate(url: String) {
this.url = url
reload()
}
private fun reload() {
val currentUrl = url
withWebViewOnMain { wv ->
if (currentUrl == null) {
wv.loadUrl(scaffoldAssetUrl)
} else {
wv.loadUrl(currentUrl)
}
val wv = webView ?: return
when (mode) {
Mode.WEB -> wv.loadUrl(url.trim())
Mode.CANVAS -> wv.loadDataWithBaseURL(null, canvasHtml, "text/html", "utf-8", null)
}
}
@@ -90,28 +68,6 @@ class CanvasController {
Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP)
}
suspend fun snapshotBase64(format: SnapshotFormat, quality: Double?, maxWidth: Int?): String =
withContext(Dispatchers.Main) {
val wv = webView ?: throw IllegalStateException("no webview")
val bmp = wv.captureBitmap()
val scaled =
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
Bitmap.createScaledBitmap(bmp, maxWidth, h, true)
} else {
bmp
}
val out = ByteArrayOutputStream()
val (compressFormat, compressQuality) =
when (format) {
SnapshotFormat.Png -> Bitmap.CompressFormat.PNG to 100
SnapshotFormat.Jpeg -> Bitmap.CompressFormat.JPEG to clampJpegQuality(quality)
}
scaled.compress(compressFormat, compressQuality, out)
Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP)
}
private suspend fun WebView.captureBitmap(): Bitmap =
suspendCancellableCoroutine { cont ->
val width = width.coerceAtLeast(1)
@@ -125,81 +81,162 @@ class CanvasController {
}
companion object {
data class SnapshotParams(val format: SnapshotFormat, val quality: Double?, val maxWidth: Int?)
fun parseMode(paramsJson: String?): Mode {
val raw = paramsJson ?: return Mode.CANVAS
return if (raw.contains("\"web\"")) Mode.WEB else Mode.CANVAS
}
fun parseNavigateUrl(paramsJson: String?): String {
val obj = parseParamsObject(paramsJson) ?: return ""
return obj.string("url").trim()
fun parseNavigateUrl(paramsJson: String?): String? {
val raw = paramsJson ?: return null
val key = "\"url\""
val idx = raw.indexOf(key)
if (idx < 0) return null
val start = raw.indexOf('"', idx + key.length)
if (start < 0) return null
val end = raw.indexOf('"', start + 1)
if (end < 0) return null
return raw.substring(start + 1, end)
}
fun parseEvalJs(paramsJson: String?): String? {
val obj = parseParamsObject(paramsJson) ?: return null
val js = obj.string("javaScript").trim()
return js.takeIf { it.isNotBlank() }
val raw = paramsJson ?: return null
val key = "\"javaScript\""
val idx = raw.indexOf(key)
if (idx < 0) return null
val start = raw.indexOf('"', idx + key.length)
if (start < 0) return null
val end = raw.lastIndexOf('"')
if (end <= start) return null
return raw.substring(start + 1, end)
.replace("\\n", "\n")
.replace("\\\"", "\"")
.replace("\\\\", "\\")
}
fun parseSnapshotMaxWidth(paramsJson: String?): Int? {
val obj = parseParamsObject(paramsJson) ?: return null
if (!obj.containsKey("maxWidth")) return null
val width = obj.int("maxWidth") ?: 0
return width.takeIf { it > 0 }
}
fun parseSnapshotFormat(paramsJson: String?): SnapshotFormat {
val obj = parseParamsObject(paramsJson) ?: return SnapshotFormat.Jpeg
val raw = obj.string("format").trim().lowercase()
return when (raw) {
"png" -> SnapshotFormat.Png
"jpeg", "jpg" -> SnapshotFormat.Jpeg
"" -> SnapshotFormat.Jpeg
else -> SnapshotFormat.Jpeg
}
}
fun parseSnapshotQuality(paramsJson: String?): Double? {
val obj = parseParamsObject(paramsJson) ?: return null
if (!obj.containsKey("quality")) return null
val q = obj.double("quality") ?: Double.NaN
if (!q.isFinite()) return null
return q.coerceIn(0.1, 1.0)
}
fun parseSnapshotParams(paramsJson: String?): SnapshotParams {
return SnapshotParams(
format = parseSnapshotFormat(paramsJson),
quality = parseSnapshotQuality(paramsJson),
maxWidth = parseSnapshotMaxWidth(paramsJson),
)
}
private val json = Json { ignoreUnknownKeys = true }
private fun parseParamsObject(paramsJson: String?): JsonObject? {
val raw = paramsJson?.trim().orEmpty()
if (raw.isEmpty()) return null
return try {
json.parseToJsonElement(raw).asObjectOrNull()
} catch (_: Throwable) {
null
}
}
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
private fun JsonObject.string(key: String): String {
val prim = this[key] as? JsonPrimitive ?: return ""
val raw = prim.content
return raw.takeIf { it != "null" }.orEmpty()
}
private fun JsonObject.int(key: String): Int? {
val prim = this[key] as? JsonPrimitive ?: return null
return prim.content.toIntOrNull()
}
private fun JsonObject.double(key: String): Double? {
val prim = this[key] as? JsonPrimitive ?: return null
return prim.content.toDoubleOrNull()
val raw = paramsJson ?: return null
val key = "\"maxWidth\""
val idx = raw.indexOf(key)
if (idx < 0) return null
val colon = raw.indexOf(':', idx + key.length)
if (colon < 0) return null
val tail = raw.substring(colon + 1).trimStart()
val num = tail.takeWhile { it.isDigit() }
return num.toIntOrNull()
}
}
}
private val canvasHtml =
"""
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
<title>Canvas</title>
<style>
:root { color-scheme: dark; }
html,body { height:100%; margin:0; }
body {
background: radial-gradient(1200px 900px at 15% 20%, rgba(42, 113, 255, 0.18), rgba(0,0,0,0) 55%),
radial-gradient(900px 700px at 85% 30%, rgba(255, 0, 138, 0.14), rgba(0,0,0,0) 60%),
radial-gradient(1000px 900px at 60% 90%, rgba(0, 209, 255, 0.10), rgba(0,0,0,0) 60%),
#000;
overflow: hidden;
}
body::before {
content:"";
position: fixed;
inset: -20%;
background:
repeating-linear-gradient(0deg, rgba(255,255,255,0.02) 0, rgba(255,255,255,0.02) 1px,
transparent 1px, transparent 48px),
repeating-linear-gradient(90deg, rgba(255,255,255,0.02) 0, rgba(255,255,255,0.02) 1px,
transparent 1px, transparent 48px);
transform: rotate(-7deg);
opacity: 0.55;
pointer-events: none;
}
canvas {
display:block;
width:100vw;
height:100vh;
touch-action: none;
}
#clawdis-status {
position: fixed;
inset: 0;
display: grid;
place-items: center;
pointer-events: none;
}
#clawdis-status .card {
text-align: center;
padding: 16px 18px;
border-radius: 14px;
background: rgba(18, 18, 22, 0.42);
border: 1px solid rgba(255,255,255,0.08);
box-shadow: 0 18px 60px rgba(0,0,0,0.55);
backdrop-filter: blur(14px);
}
#clawdis-status .title {
font: 600 20px system-ui, sans-serif;
letter-spacing: 0.2px;
color: rgba(255,255,255,0.92);
text-shadow: 0 0 22px rgba(42, 113, 255, 0.35);
}
#clawdis-status .subtitle {
margin-top: 6px;
font: 500 12px system-ui, sans-serif;
color: rgba(255,255,255,0.58);
}
</style>
</head>
<body>
<canvas id="clawdis-canvas"></canvas>
<div id="clawdis-status">
<div class="card">
<div class="title" id="clawdis-status-title">Ready</div>
<div class="subtitle" id="clawdis-status-subtitle">Waiting for agent</div>
</div>
</div>
<script>
(() => {
const canvas = document.getElementById('clawdis-canvas');
const ctx = canvas.getContext('2d');
const statusEl = document.getElementById('clawdis-status');
const titleEl = document.getElementById('clawdis-status-title');
const subtitleEl = document.getElementById('clawdis-status-subtitle');
function resize() {
const dpr = window.devicePixelRatio || 1;
const w = Math.max(1, Math.floor(window.innerWidth * dpr));
const h = Math.max(1, Math.floor(window.innerHeight * dpr));
canvas.width = w;
canvas.height = h;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
}
window.addEventListener('resize', resize);
resize();
window.__clawdis = {
canvas,
ctx,
setStatus: (title, subtitle) => {
if (!statusEl) return;
if (!title && !subtitle) {
statusEl.style.display = 'none';
return;
}
statusEl.style.display = 'grid';
if (titleEl && typeof title === 'string') titleEl.textContent = title;
if (subtitleEl && typeof subtitle === 'string') subtitleEl.textContent = subtitle;
}
};
})();
</script>
</body>
</html>
""".trimIndent()
@@ -1,196 +0,0 @@
package com.steipete.clawdis.node.node
import android.content.Context
import android.hardware.display.DisplayManager
import android.media.MediaRecorder
import android.media.projection.MediaProjectionManager
import android.util.Base64
import com.steipete.clawdis.node.ScreenCaptureRequester
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import java.io.File
import kotlin.math.roundToInt
class ScreenRecordManager(private val context: Context) {
data class Payload(val payloadJson: String)
@Volatile private var screenCaptureRequester: ScreenCaptureRequester? = null
@Volatile private var permissionRequester: com.steipete.clawdis.node.PermissionRequester? = null
fun attachScreenCaptureRequester(requester: ScreenCaptureRequester) {
screenCaptureRequester = requester
}
fun attachPermissionRequester(requester: com.steipete.clawdis.node.PermissionRequester) {
permissionRequester = requester
}
suspend fun record(paramsJson: String?): Payload =
withContext(Dispatchers.Default) {
val requester =
screenCaptureRequester
?: throw IllegalStateException(
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
)
val durationMs = (parseDurationMs(paramsJson) ?: 10_000).coerceIn(250, 60_000)
val fps = (parseFps(paramsJson) ?: 10.0).coerceIn(1.0, 60.0)
val fpsInt = fps.roundToInt().coerceIn(1, 60)
val screenIndex = parseScreenIndex(paramsJson)
val includeAudio = parseIncludeAudio(paramsJson) ?: true
val format = parseString(paramsJson, key = "format")
if (format != null && format.lowercase() != "mp4") {
throw IllegalArgumentException("INVALID_REQUEST: screen format must be mp4")
}
if (screenIndex != null && screenIndex != 0) {
throw IllegalArgumentException("INVALID_REQUEST: screenIndex must be 0 on Android")
}
val capture = requester.requestCapture()
?: throw IllegalStateException(
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
)
val mgr =
context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
val projection = mgr.getMediaProjection(capture.resultCode, capture.data)
?: throw IllegalStateException("UNAVAILABLE: screen capture unavailable")
val metrics = context.resources.displayMetrics
val width = metrics.widthPixels
val height = metrics.heightPixels
val densityDpi = metrics.densityDpi
val file = File.createTempFile("clawdis-screen-", ".mp4")
if (includeAudio) ensureMicPermission()
val recorder = MediaRecorder()
var virtualDisplay: android.hardware.display.VirtualDisplay? = null
try {
if (includeAudio) {
recorder.setAudioSource(MediaRecorder.AudioSource.MIC)
}
recorder.setVideoSource(MediaRecorder.VideoSource.SURFACE)
recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
recorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264)
if (includeAudio) {
recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
recorder.setAudioChannels(1)
recorder.setAudioSamplingRate(44_100)
recorder.setAudioEncodingBitRate(96_000)
}
recorder.setVideoSize(width, height)
recorder.setVideoFrameRate(fpsInt)
recorder.setVideoEncodingBitRate(estimateBitrate(width, height, fpsInt))
recorder.setOutputFile(file.absolutePath)
recorder.prepare()
val surface = recorder.surface
virtualDisplay =
projection.createVirtualDisplay(
"clawdis-screen",
width,
height,
densityDpi,
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
surface,
null,
null,
)
recorder.start()
delay(durationMs.toLong())
} finally {
try {
recorder.stop()
} catch (_: Throwable) {
// ignore
}
recorder.reset()
recorder.release()
virtualDisplay?.release()
projection.stop()
}
val bytes = withContext(Dispatchers.IO) { file.readBytes() }
file.delete()
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
Payload(
"""{"format":"mp4","base64":"$base64","durationMs":$durationMs,"fps":$fpsInt,"screenIndex":0,"hasAudio":$includeAudio}""",
)
}
private suspend fun ensureMicPermission() {
val granted =
androidx.core.content.ContextCompat.checkSelfPermission(
context,
android.Manifest.permission.RECORD_AUDIO,
) == android.content.pm.PackageManager.PERMISSION_GRANTED
if (granted) return
val requester =
permissionRequester
?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
val results = requester.requestIfMissing(listOf(android.Manifest.permission.RECORD_AUDIO))
if (results[android.Manifest.permission.RECORD_AUDIO] != true) {
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
}
}
private fun parseDurationMs(paramsJson: String?): Int? =
parseNumber(paramsJson, key = "durationMs")?.toIntOrNull()
private fun parseFps(paramsJson: String?): Double? =
parseNumber(paramsJson, key = "fps")?.toDoubleOrNull()
private fun parseScreenIndex(paramsJson: String?): Int? =
parseNumber(paramsJson, key = "screenIndex")?.toIntOrNull()
private fun parseIncludeAudio(paramsJson: String?): Boolean? {
val raw = paramsJson ?: return null
val key = "\"includeAudio\""
val idx = raw.indexOf(key)
if (idx < 0) return null
val colon = raw.indexOf(':', idx + key.length)
if (colon < 0) return null
val tail = raw.substring(colon + 1).trimStart()
return when {
tail.startsWith("true") -> true
tail.startsWith("false") -> false
else -> null
}
}
private fun parseNumber(paramsJson: String?, key: String): String? {
val raw = paramsJson ?: return null
val needle = "\"$key\""
val idx = raw.indexOf(needle)
if (idx < 0) return null
val colon = raw.indexOf(':', idx + needle.length)
if (colon < 0) return null
val tail = raw.substring(colon + 1).trimStart()
return tail.takeWhile { it.isDigit() || it == '.' || it == '-' }
}
private fun parseString(paramsJson: String?, key: String): String? {
val raw = paramsJson ?: return null
val needle = "\"$key\""
val idx = raw.indexOf(needle)
if (idx < 0) return null
val colon = raw.indexOf(':', idx + needle.length)
if (colon < 0) return null
val tail = raw.substring(colon + 1).trimStart()
if (!tail.startsWith('\"')) return null
val rest = tail.drop(1)
val end = rest.indexOf('\"')
if (end < 0) return null
return rest.substring(0, end)
}
private fun estimateBitrate(width: Int, height: Int, fps: Int): Int {
val pixels = width.toLong() * height.toLong()
val raw = (pixels * fps.toLong() * 2L).toInt()
return raw.coerceIn(1_000_000, 12_000_000)
}
}
@@ -1,49 +0,0 @@
package com.steipete.clawdis.node.protocol
object ClawdisCanvasA2UIAction {
fun sanitizeTagValue(value: String): String {
val trimmed = value.trim().ifEmpty { "-" }
val normalized = trimmed.replace(" ", "_")
val out = StringBuilder(normalized.length)
for (c in normalized) {
val ok =
c.isLetterOrDigit() ||
c == '_' ||
c == '-' ||
c == '.' ||
c == ':'
out.append(if (ok) c else '_')
}
return out.toString()
}
fun formatAgentMessage(
actionName: String,
sessionKey: String,
surfaceId: String,
sourceComponentId: String,
host: String,
instanceId: String,
contextJson: String?,
): String {
val ctxSuffix = contextJson?.takeIf { it.isNotBlank() }?.let { " ctx=$it" }.orEmpty()
return listOf(
"CANVAS_A2UI",
"action=${sanitizeTagValue(actionName)}",
"session=${sanitizeTagValue(sessionKey)}",
"surface=${sanitizeTagValue(surfaceId)}",
"component=${sanitizeTagValue(sourceComponentId)}",
"host=${sanitizeTagValue(host)}",
"instance=${sanitizeTagValue(instanceId)}$ctxSuffix",
"default=update_canvas",
).joinToString(separator = " ")
}
fun jsDispatchA2UIActionStatus(actionId: String, ok: Boolean, error: String?): String {
val err = (error ?: "").replace("\\", "\\\\").replace("\"", "\\\"")
val okLiteral = if (ok) "true" else "false"
val idEscaped = actionId.replace("\\", "\\\\").replace("\"", "\\\"")
return "window.dispatchEvent(new CustomEvent('clawdis:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));"
}
}
@@ -1,51 +0,0 @@
package com.steipete.clawdis.node.protocol
enum class ClawdisCapability(val rawValue: String) {
Canvas("canvas"),
Camera("camera"),
Screen("screen"),
VoiceWake("voiceWake"),
}
enum class ClawdisCanvasCommand(val rawValue: String) {
Present("canvas.present"),
Hide("canvas.hide"),
Navigate("canvas.navigate"),
Eval("canvas.eval"),
Snapshot("canvas.snapshot"),
;
companion object {
const val NamespacePrefix: String = "canvas."
}
}
enum class ClawdisCanvasA2UICommand(val rawValue: String) {
Push("canvas.a2ui.push"),
PushJSONL("canvas.a2ui.pushJSONL"),
Reset("canvas.a2ui.reset"),
;
companion object {
const val NamespacePrefix: String = "canvas.a2ui."
}
}
enum class ClawdisCameraCommand(val rawValue: String) {
Snap("camera.snap"),
Clip("camera.clip"),
;
companion object {
const val NamespacePrefix: String = "camera."
}
}
enum class ClawdisScreenCommand(val rawValue: String) {
Record("screen.record"),
;
companion object {
const val NamespacePrefix: String = "screen."
}
}
@@ -1,123 +0,0 @@
package com.steipete.clawdis.node.ui
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
import androidx.compose.animation.slideInVertically
import androidx.compose.animation.slideOutVertically
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.statusBarsPadding
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.CheckCircle
import androidx.compose.material.icons.filled.Error
import androidx.compose.material.icons.filled.FiberManualRecord
import androidx.compose.material.icons.filled.PhotoCamera
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.CameraHudKind
import com.steipete.clawdis.node.CameraHudState
import kotlinx.coroutines.delay
@Composable
fun CameraHudOverlay(
hud: CameraHudState?,
flashToken: Long,
modifier: Modifier = Modifier,
) {
Box(modifier = modifier.fillMaxSize()) {
CameraFlash(token = flashToken)
AnimatedVisibility(
visible = hud != null,
enter = slideInVertically(initialOffsetY = { -it / 2 }) + fadeIn(),
exit = slideOutVertically(targetOffsetY = { -it / 2 }) + fadeOut(),
modifier = Modifier.align(Alignment.TopStart).statusBarsPadding().padding(start = 12.dp, top = 58.dp),
) {
if (hud != null) {
Toast(hud = hud)
}
}
}
}
@Composable
private fun CameraFlash(token: Long) {
var alpha by remember { mutableFloatStateOf(0f) }
LaunchedEffect(token) {
if (token == 0L) return@LaunchedEffect
alpha = 0.85f
delay(110)
alpha = 0f
}
Box(
modifier =
Modifier
.fillMaxSize()
.alpha(alpha)
.background(Color.White),
)
}
@Composable
private fun Toast(hud: CameraHudState) {
Surface(
shape = RoundedCornerShape(14.dp),
color = MaterialTheme.colorScheme.surface.copy(alpha = 0.85f),
tonalElevation = 2.dp,
shadowElevation = 8.dp,
) {
Row(
modifier = Modifier.padding(vertical = 10.dp, horizontal = 12.dp),
verticalAlignment = Alignment.CenterVertically,
) {
when (hud.kind) {
CameraHudKind.Photo -> {
Icon(Icons.Default.PhotoCamera, contentDescription = null)
Spacer(Modifier.size(10.dp))
CircularProgressIndicator(modifier = Modifier.size(14.dp), strokeWidth = 2.dp)
}
CameraHudKind.Recording -> {
Icon(Icons.Default.FiberManualRecord, contentDescription = null, tint = Color.Red)
}
CameraHudKind.Success -> {
Icon(Icons.Default.CheckCircle, contentDescription = null)
}
CameraHudKind.Error -> {
Icon(Icons.Default.Error, contentDescription = null)
}
}
Spacer(Modifier.size(10.dp))
Text(
text = hud.message,
style = MaterialTheme.typography.bodyMedium,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
}
}
@@ -1,10 +1,73 @@
package com.steipete.clawdis.node.ui
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.material3.Button
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.MainViewModel
import com.steipete.clawdis.node.ui.chat.ChatSheetContent
@Composable
fun ChatSheet(viewModel: MainViewModel) {
ChatSheetContent(viewModel = viewModel)
val messages by viewModel.chatMessages.collectAsState()
val error by viewModel.chatError.collectAsState()
val pendingRunCount by viewModel.pendingRunCount.collectAsState()
var input by remember { mutableStateOf("") }
LaunchedEffect(Unit) {
viewModel.loadChat("main")
}
Column(modifier = Modifier.fillMaxSize().padding(16.dp), verticalArrangement = Arrangement.spacedBy(12.dp)) {
Text("Clawd Chat · session main")
if (!error.isNullOrBlank()) {
Text("Error: $error")
}
LazyColumn(modifier = Modifier.fillMaxWidth().weight(1f, fill = true)) {
items(messages) { msg ->
Text("${msg.role}: ${msg.text}")
}
if (pendingRunCount > 0) {
item { Text("assistant: …") }
}
}
Row(horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.fillMaxWidth()) {
OutlinedTextField(
value = input,
onValueChange = { input = it },
modifier = Modifier.weight(1f),
label = { Text("Message") },
)
Button(
onClick = {
val text = input
input = ""
viewModel.sendChat("main", text)
},
enabled = input.trim().isNotEmpty(),
) {
Text("Send")
}
}
}
}
@@ -1,32 +0,0 @@
package com.steipete.clawdis.node.ui
import androidx.compose.foundation.isSystemInDarkTheme
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.dynamicDarkColorScheme
import androidx.compose.material3.dynamicLightColorScheme
import androidx.compose.runtime.Composable
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalContext
@Composable
fun ClawdisTheme(content: @Composable () -> Unit) {
val context = LocalContext.current
val isDark = isSystemInDarkTheme()
val colorScheme = if (isDark) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
MaterialTheme(colorScheme = colorScheme, content = content)
}
@Composable
fun overlayContainerColor(): Color {
val scheme = MaterialTheme.colorScheme
val isDark = isSystemInDarkTheme()
val base = if (isDark) scheme.surfaceContainerLow else scheme.surfaceContainerHigh
// Light mode: background stays dark (canvas), so clamp overlays away from pure-white glare.
return if (isDark) base else base.copy(alpha = 0.88f)
}
@Composable
fun overlayIconColor(): Color {
return MaterialTheme.colorScheme.onSurfaceVariant
}
@@ -1,40 +1,25 @@
package com.steipete.clawdis.node.ui
import android.annotation.SuppressLint
import android.Manifest
import android.content.pm.PackageManager
import android.graphics.Color
import android.util.Log
import android.view.View
import android.webkit.JavascriptInterface
import android.webkit.WebView
import android.webkit.WebSettings
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebViewClient
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.WindowInsetsSides
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.only
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.safeDrawing
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.material3.Button
import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButtonDefaults
import androidx.compose.material3.ModalBottomSheet
import androidx.compose.material3.Text
import androidx.compose.material3.rememberModalBottomSheetState
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ChatBubble
import androidx.compose.material.icons.filled.Settings
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
@@ -46,7 +31,6 @@ import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.compose.ui.window.Popup
import androidx.compose.ui.window.PopupProperties
import androidx.core.content.ContextCompat
import com.steipete.clawdis.node.MainViewModel
@OptIn(ExperimentalMaterial3Api::class)
@@ -54,62 +38,25 @@ import com.steipete.clawdis.node.MainViewModel
fun RootScreen(viewModel: MainViewModel) {
var sheet by remember { mutableStateOf<Sheet?>(null) }
val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true)
val safeOverlayInsets = WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal)
val context = LocalContext.current
val serverName by viewModel.serverName.collectAsState()
val statusText by viewModel.statusText.collectAsState()
val cameraHud by viewModel.cameraHud.collectAsState()
val cameraFlashToken by viewModel.cameraFlashToken.collectAsState()
val bridgeState =
remember(serverName, statusText) {
when {
serverName != null -> BridgeState.Connected
statusText.contains("connecting", ignoreCase = true) ||
statusText.contains("reconnecting", ignoreCase = true) -> BridgeState.Connecting
statusText.contains("error", ignoreCase = true) -> BridgeState.Error
else -> BridgeState.Disconnected
}
}
val voiceEnabled =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
val safeButtonInsets = WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal)
Box(modifier = Modifier.fillMaxSize()) {
CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize())
}
// Camera HUD (flash + toast) must be in a Popup to render above the WebView.
Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) {
CameraHudOverlay(hud = cameraHud, flashToken = cameraFlashToken, modifier = Modifier.fillMaxSize())
}
// Keep the overlay buttons above the WebView canvas (AndroidView), otherwise they may not receive touches.
Popup(alignment = Alignment.TopStart, properties = PopupProperties(focusable = false)) {
StatusPill(
bridge = bridgeState,
voiceEnabled = voiceEnabled,
onClick = { sheet = Sheet.Settings },
modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(start = 12.dp, top = 12.dp),
)
}
Popup(alignment = Alignment.TopEnd, properties = PopupProperties(focusable = false)) {
Column(
modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(end = 12.dp, top = 12.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
horizontalAlignment = Alignment.End,
Popup(alignment = Alignment.TopCenter, properties = PopupProperties(focusable = false)) {
Row(
modifier =
Modifier
.fillMaxWidth()
.windowInsetsPadding(safeButtonInsets)
.padding(12.dp),
horizontalArrangement = Arrangement.SpaceBetween,
verticalAlignment = Alignment.CenterVertically,
) {
OverlayIconButton(
onClick = { sheet = Sheet.Chat },
icon = { Icon(Icons.Default.ChatBubble, contentDescription = "Chat") },
)
OverlayIconButton(
onClick = { sheet = Sheet.Settings },
icon = { Icon(Icons.Default.Settings, contentDescription = "Settings") },
)
Button(onClick = { sheet = Sheet.Chat }) { Text("Chat") }
Button(onClick = { sheet = Sheet.Settings }) { Text("Settings") }
}
}
@@ -132,86 +79,20 @@ private enum class Sheet {
Settings,
}
@Composable
private fun OverlayIconButton(
onClick: () -> Unit,
icon: @Composable () -> Unit,
) {
FilledTonalIconButton(
onClick = onClick,
modifier = Modifier.size(44.dp),
colors =
IconButtonDefaults.filledTonalIconButtonColors(
containerColor = overlayContainerColor(),
contentColor = overlayIconColor(),
),
) {
icon()
}
}
@SuppressLint("SetJavaScriptEnabled")
@Composable
private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier) {
val context = LocalContext.current
val isDebuggable = (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0
AndroidView(
modifier = modifier,
factory = {
WebView(context).apply {
settings.javaScriptEnabled = true
// Some embedded web UIs (incl. the "background website") use localStorage/sessionStorage.
settings.domStorageEnabled = true
settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE
webViewClient =
object : WebViewClient() {
override fun onReceivedError(
view: WebView,
request: WebResourceRequest,
error: WebResourceError,
) {
if (!isDebuggable) return
if (!request.isForMainFrame) return
Log.e("ClawdisWebView", "onReceivedError: ${error.errorCode} ${error.description} ${request.url}")
}
override fun onReceivedHttpError(
view: WebView,
request: WebResourceRequest,
errorResponse: WebResourceResponse,
) {
if (!isDebuggable) return
if (!request.isForMainFrame) return
Log.e(
"ClawdisWebView",
"onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}",
)
}
}
setBackgroundColor(Color.BLACK)
setLayerType(View.LAYER_TYPE_HARDWARE, null)
addJavascriptInterface(
CanvasA2UIActionBridge { payload ->
viewModel.handleCanvasA2UIActionFromWebView(payload)
},
CanvasA2UIActionBridge.interfaceName,
)
settings.domStorageEnabled = false
webViewClient = WebViewClient()
setBackgroundColor(0x00000000)
viewModel.canvas.attach(this)
}
},
)
}
private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) {
@JavascriptInterface
fun postMessage(payload: String?) {
val msg = payload?.trim().orEmpty()
if (msg.isEmpty()) return
onMessage(msg)
}
companion object {
const val interfaceName: String = "clawdisCanvasA2UIAction"
}
}
@@ -4,8 +4,6 @@ import android.Manifest
import android.content.pm.PackageManager
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
@@ -13,26 +11,20 @@ import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.WindowInsetsSides
import androidx.compose.foundation.layout.fillMaxHeight
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.fillMaxHeight
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.imePadding
import androidx.compose.foundation.layout.only
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.safeDrawing
import androidx.compose.foundation.layout.only
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.foundation.lazy.rememberLazyListState
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ExpandLess
import androidx.compose.material.icons.filled.ExpandMore
import androidx.compose.material3.Button
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.Icon
import androidx.compose.material3.ListItem
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.RadioButton
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
@@ -43,12 +35,10 @@ import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import com.steipete.clawdis.node.MainViewModel
import com.steipete.clawdis.node.NodeForegroundService
import com.steipete.clawdis.node.VoiceWakeMode
@Composable
fun SettingsSheet(viewModel: MainViewModel) {
@@ -56,10 +46,7 @@ fun SettingsSheet(viewModel: MainViewModel) {
val instanceId by viewModel.instanceId.collectAsState()
val displayName by viewModel.displayName.collectAsState()
val cameraEnabled by viewModel.cameraEnabled.collectAsState()
val preventSleep by viewModel.preventSleep.collectAsState()
val wakeWords by viewModel.wakeWords.collectAsState()
val voiceWakeMode by viewModel.voiceWakeMode.collectAsState()
val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState()
val isConnected by viewModel.isConnected.collectAsState()
val manualEnabled by viewModel.manualEnabled.collectAsState()
val manualHost by viewModel.manualHost.collectAsState()
@@ -68,12 +55,9 @@ fun SettingsSheet(viewModel: MainViewModel) {
val serverName by viewModel.serverName.collectAsState()
val remoteAddress by viewModel.remoteAddress.collectAsState()
val bridges by viewModel.bridges.collectAsState()
val discoveryStatusText by viewModel.discoveryStatusText.collectAsState()
val listState = rememberLazyListState()
val (wakeWordsText, setWakeWordsText) = remember { mutableStateOf("") }
val (advancedExpanded, setAdvancedExpanded) = remember { mutableStateOf(false) }
val (wakeWordsText, setWakeWordsText) = remember { mutableStateOf("") }
LaunchedEffect(wakeWords) { setWakeWordsText(wakeWords.joinToString(", ")) }
val permissionLauncher =
@@ -82,43 +66,6 @@ fun SettingsSheet(viewModel: MainViewModel) {
viewModel.setCameraEnabled(cameraOk)
}
val audioPermissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { _ ->
// Status text is handled by NodeRuntime.
}
fun setCameraEnabledChecked(checked: Boolean) {
if (!checked) {
viewModel.setCameraEnabled(false)
return
}
val cameraOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) ==
PackageManager.PERMISSION_GRANTED
if (cameraOk) {
viewModel.setCameraEnabled(true)
} else {
permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO))
}
}
val visibleBridges =
if (isConnected && remoteAddress != null) {
bridges.filterNot { "${it.host}:${it.port}" == remoteAddress }
} else {
bridges
}
val bridgeDiscoveryFooterText =
if (visibleBridges.isEmpty()) {
discoveryStatusText
} else if (isConnected) {
"Discovery active • ${visibleBridges.size} other bridge${if (visibleBridges.size == 1) "" else "s"} found"
} else {
"Discovery active • ${visibleBridges.size} bridge${if (visibleBridges.size == 1) "" else "s"} found"
}
LazyColumn(
state = listState,
modifier =
@@ -128,10 +75,9 @@ fun SettingsSheet(viewModel: MainViewModel) {
.imePadding()
.windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Bottom)),
contentPadding = PaddingValues(16.dp),
verticalArrangement = Arrangement.spacedBy(6.dp),
verticalArrangement = Arrangement.spacedBy(14.dp),
) {
// Order parity: Node → Bridge → Voice → Camera → Screen.
item { Text("Node", style = MaterialTheme.typography.titleSmall) }
item { Text("Node") }
item {
OutlinedTextField(
value = displayName,
@@ -140,200 +86,16 @@ fun SettingsSheet(viewModel: MainViewModel) {
modifier = Modifier.fillMaxWidth(),
)
}
item { Text("Instance ID: $instanceId", color = MaterialTheme.colorScheme.onSurfaceVariant) }
item { Text("Instance ID: $instanceId") }
item { HorizontalDivider() }
// Bridge
item { Text("Bridge", style = MaterialTheme.typography.titleSmall) }
item { ListItem(headlineContent = { Text("Status") }, supportingContent = { Text(statusText) }) }
if (serverName != null) {
item { ListItem(headlineContent = { Text("Server") }, supportingContent = { Text(serverName!!) }) }
}
if (remoteAddress != null) {
item { ListItem(headlineContent = { Text("Address") }, supportingContent = { Text(remoteAddress!!) }) }
}
item {
// UI sanity: "Disconnect" only when we have an active remote.
if (isConnected && remoteAddress != null) {
Button(
onClick = {
viewModel.disconnect()
NodeForegroundService.stop(context)
},
) {
Text("Disconnect")
}
}
}
item { HorizontalDivider() }
if (!isConnected || visibleBridges.isNotEmpty()) {
item {
Text(
if (isConnected) "Other Bridges" else "Discovered Bridges",
style = MaterialTheme.typography.titleSmall,
)
}
if (!isConnected && visibleBridges.isEmpty()) {
item { Text("No bridges found yet.", color = MaterialTheme.colorScheme.onSurfaceVariant) }
} else {
items(items = visibleBridges, key = { it.stableId }) { bridge ->
ListItem(
headlineContent = { Text(bridge.name) },
supportingContent = { Text("${bridge.host}:${bridge.port}") },
trailingContent = {
Button(
onClick = {
NodeForegroundService.start(context)
viewModel.connect(bridge)
},
) {
Text("Connect")
}
},
)
}
}
item {
Text(
bridgeDiscoveryFooterText,
modifier = Modifier.fillMaxWidth(),
textAlign = TextAlign.Center,
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
item { HorizontalDivider() }
item {
ListItem(
headlineContent = { Text("Advanced") },
supportingContent = { Text("Manual bridge connection") },
trailingContent = {
Icon(
imageVector = if (advancedExpanded) Icons.Filled.ExpandLess else Icons.Filled.ExpandMore,
contentDescription = if (advancedExpanded) "Collapse" else "Expand",
)
},
modifier =
Modifier.clickable {
setAdvancedExpanded(!advancedExpanded)
},
)
}
item {
AnimatedVisibility(visible = advancedExpanded) {
Column(verticalArrangement = Arrangement.spacedBy(10.dp), modifier = Modifier.fillMaxWidth()) {
ListItem(
headlineContent = { Text("Use Manual Bridge") },
supportingContent = { Text("Use this when discovery is blocked.") },
trailingContent = { Switch(checked = manualEnabled, onCheckedChange = viewModel::setManualEnabled) },
)
OutlinedTextField(
value = manualHost,
onValueChange = viewModel::setManualHost,
label = { Text("Host") },
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
)
OutlinedTextField(
value = manualPort.toString(),
onValueChange = { v -> viewModel.setManualPort(v.toIntOrNull() ?: 0) },
label = { Text("Port") },
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
)
val hostOk = manualHost.trim().isNotEmpty()
val portOk = manualPort in 1..65535
Button(
onClick = {
NodeForegroundService.start(context)
viewModel.connectManual()
},
enabled = manualEnabled && hostOk && portOk,
) {
Text("Connect (Manual)")
}
}
}
}
item { HorizontalDivider() }
// Voice
item { Text("Voice", style = MaterialTheme.typography.titleSmall) }
item {
val enabled = voiceWakeMode != VoiceWakeMode.Off
ListItem(
headlineContent = { Text("Voice Wake") },
supportingContent = { Text(voiceWakeStatusText) },
trailingContent = {
Switch(
checked = enabled,
onCheckedChange = { on ->
if (on) {
val micOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground)
} else {
viewModel.setVoiceWakeMode(VoiceWakeMode.Off)
}
},
)
},
)
}
item {
AnimatedVisibility(visible = voiceWakeMode != VoiceWakeMode.Off) {
Column(verticalArrangement = Arrangement.spacedBy(6.dp), modifier = Modifier.fillMaxWidth()) {
ListItem(
headlineContent = { Text("Foreground Only") },
supportingContent = { Text("Listens only while Clawdis is open.") },
trailingContent = {
RadioButton(
selected = voiceWakeMode == VoiceWakeMode.Foreground,
onClick = {
val micOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground)
},
)
},
)
ListItem(
headlineContent = { Text("Always") },
supportingContent = { Text("Keeps listening in the background (shows a persistent notification).") },
trailingContent = {
RadioButton(
selected = voiceWakeMode == VoiceWakeMode.Always,
onClick = {
val micOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
viewModel.setVoiceWakeMode(VoiceWakeMode.Always)
},
)
},
)
}
}
}
item { Text("Wake Words") }
item {
OutlinedTextField(
value = wakeWordsText,
onValueChange = setWakeWordsText,
label = { Text("Wake Words (comma-separated)") },
label = { Text("Comma-separated (global)") },
modifier = Modifier.fillMaxWidth(),
singleLine = true,
)
@@ -360,39 +122,123 @@ fun SettingsSheet(viewModel: MainViewModel) {
} else {
"Connect to a gateway to sync wake words globally."
},
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
item { HorizontalDivider() }
// Camera
item { Text("Camera", style = MaterialTheme.typography.titleSmall) }
item { Text("Camera") }
item {
ListItem(
headlineContent = { Text("Allow Camera") },
supportingContent = { Text("Allows the bridge to request photos or short video clips (foreground only).") },
trailingContent = { Switch(checked = cameraEnabled, onCheckedChange = ::setCameraEnabledChecked) },
)
Row(horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.fillMaxWidth()) {
Switch(
checked = cameraEnabled,
onCheckedChange = { enabled ->
if (!enabled) {
viewModel.setCameraEnabled(false)
return@Switch
}
val cameraOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) ==
PackageManager.PERMISSION_GRANTED
if (cameraOk) {
viewModel.setCameraEnabled(true)
} else {
permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO))
}
},
)
Text(if (cameraEnabled) "Allow Camera" else "Camera Disabled")
}
}
item { Text("Tip: grant Microphone permission for video clips with audio.") }
item { HorizontalDivider() }
item { Text("Bridge") }
item { Text("Status: $statusText") }
item { if (serverName != null) Text("Server: $serverName") }
item { if (remoteAddress != null) Text("Address: $remoteAddress") }
item {
Text(
"Tip: grant Microphone permission for video clips with audio.",
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
Button(
onClick = {
viewModel.disconnect()
NodeForegroundService.stop(context)
},
) {
Text("Disconnect")
}
}
}
item { HorizontalDivider() }
// Screen
item { Text("Screen", style = MaterialTheme.typography.titleSmall) }
item { Text("Advanced") }
item {
ListItem(
headlineContent = { Text("Prevent Sleep") },
supportingContent = { Text("Keeps the screen awake while Clawdis is open.") },
trailingContent = { Switch(checked = preventSleep, onCheckedChange = viewModel::setPreventSleep) },
Row(horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.fillMaxWidth()) {
Switch(checked = manualEnabled, onCheckedChange = viewModel::setManualEnabled)
Text(if (manualEnabled) "Manual Bridge Enabled" else "Manual Bridge Disabled")
}
}
item {
OutlinedTextField(
value = manualHost,
onValueChange = viewModel::setManualHost,
label = { Text("Host") },
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
)
}
item {
OutlinedTextField(
value = manualPort.toString(),
onValueChange = { v -> viewModel.setManualPort(v.toIntOrNull() ?: 0) },
label = { Text("Port") },
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
)
}
item {
Button(
onClick = {
NodeForegroundService.start(context)
viewModel.connectManual()
},
enabled = manualEnabled,
) {
Text("Connect (Manual)")
}
}
item { HorizontalDivider() }
item { Text("Discovered Bridges") }
if (bridges.isEmpty()) {
item { Text("No bridges found yet.") }
} else {
items(items = bridges, key = { it.stableId }) { bridge ->
Row(
modifier = Modifier.fillMaxWidth().padding(vertical = 8.dp),
horizontalArrangement = Arrangement.SpaceBetween,
) {
Column(modifier = Modifier.weight(1f)) {
Text(bridge.name)
Text("${bridge.host}:${bridge.port}")
}
Spacer(modifier = Modifier.padding(4.dp))
Button(
onClick = {
NodeForegroundService.start(context)
viewModel.connect(bridge)
},
) {
Text("Connect")
}
}
HorizontalDivider()
}
}
item { Spacer(modifier = Modifier.height(20.dp)) }
}
@@ -1,87 +0,0 @@
package com.steipete.clawdis.node.ui
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Mic
import androidx.compose.material.icons.filled.MicOff
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.material3.VerticalDivider
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.unit.dp
@Composable
fun StatusPill(
bridge: BridgeState,
voiceEnabled: Boolean,
onClick: () -> Unit,
modifier: Modifier = Modifier,
) {
Surface(
onClick = onClick,
modifier = modifier,
shape = RoundedCornerShape(14.dp),
color = overlayContainerColor(),
tonalElevation = 3.dp,
shadowElevation = 0.dp,
) {
Row(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp),
horizontalArrangement = Arrangement.spacedBy(10.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Row(horizontalArrangement = Arrangement.spacedBy(8.dp), verticalAlignment = Alignment.CenterVertically) {
Surface(
modifier = Modifier.size(9.dp),
shape = CircleShape,
color = bridge.color,
) {}
Text(
text = bridge.title,
style = MaterialTheme.typography.labelLarge,
)
}
VerticalDivider(
modifier = Modifier.height(14.dp).alpha(0.35f),
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
Icon(
imageVector = if (voiceEnabled) Icons.Default.Mic else Icons.Default.MicOff,
contentDescription = if (voiceEnabled) "Voice enabled" else "Voice disabled",
tint =
if (voiceEnabled) {
overlayIconColor()
} else {
MaterialTheme.colorScheme.onSurfaceVariant
},
modifier = Modifier.size(18.dp),
)
Spacer(modifier = Modifier.width(2.dp))
}
}
}
enum class BridgeState(val title: String, val color: Color) {
Connected("Connected", Color(0xFF2ECC71)),
Connecting("Connecting…", Color(0xFFF1C40F)),
Error("Error", Color(0xFFE74C3C)),
Disconnected("Offline", Color(0xFF9E9E9E)),
}
@@ -1,254 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.foundation.horizontalScroll
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ArrowUpward
import androidx.compose.material.icons.filled.AttachFile
import androidx.compose.material.icons.filled.FolderOpen
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Stop
import androidx.compose.material3.ButtonDefaults
import androidx.compose.material3.DropdownMenu
import androidx.compose.material3.DropdownMenuItem
import androidx.compose.material3.FilledTonalButton
import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButtonDefaults
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.unit.dp
@Composable
fun ChatComposer(
sessionKey: String,
healthOk: Boolean,
thinkingLevel: String,
pendingRunCount: Int,
errorText: String?,
attachments: List<PendingImageAttachment>,
onPickImages: () -> Unit,
onRemoveAttachment: (id: String) -> Unit,
onSetThinkingLevel: (level: String) -> Unit,
onShowSessions: () -> Unit,
onRefresh: () -> Unit,
onAbort: () -> Unit,
onSend: (text: String) -> Unit,
) {
var input by rememberSaveable { mutableStateOf("") }
var showThinkingMenu by remember { mutableStateOf(false) }
val canSend = pendingRunCount == 0 && (input.trim().isNotEmpty() || attachments.isNotEmpty()) && healthOk
Surface(
shape = MaterialTheme.shapes.large,
color = MaterialTheme.colorScheme.surfaceContainer,
tonalElevation = 0.dp,
shadowElevation = 0.dp,
) {
Column(modifier = Modifier.padding(10.dp), verticalArrangement = Arrangement.spacedBy(8.dp)) {
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Box {
FilledTonalButton(
onClick = { showThinkingMenu = true },
contentPadding = ButtonDefaults.ContentPadding,
) {
Text("Thinking: ${thinkingLabel(thinkingLevel)}")
}
DropdownMenu(expanded = showThinkingMenu, onDismissRequest = { showThinkingMenu = false }) {
ThinkingMenuItem("off", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
ThinkingMenuItem("low", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
ThinkingMenuItem("medium", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
ThinkingMenuItem("high", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
}
}
Spacer(modifier = Modifier.weight(1f))
FilledTonalIconButton(onClick = onShowSessions, modifier = Modifier.size(42.dp)) {
Icon(Icons.Default.FolderOpen, contentDescription = "Sessions")
}
FilledTonalIconButton(onClick = onRefresh, modifier = Modifier.size(42.dp)) {
Icon(Icons.Default.Refresh, contentDescription = "Refresh")
}
FilledTonalIconButton(onClick = onPickImages, modifier = Modifier.size(42.dp)) {
Icon(Icons.Default.AttachFile, contentDescription = "Add image")
}
}
if (attachments.isNotEmpty()) {
AttachmentsStrip(attachments = attachments, onRemoveAttachment = onRemoveAttachment)
}
OutlinedTextField(
value = input,
onValueChange = { input = it },
modifier = Modifier.fillMaxWidth(),
placeholder = { Text("Message Clawd…") },
minLines = 2,
maxLines = 6,
)
Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) {
ConnectionPill(sessionKey = sessionKey, healthOk = healthOk)
Spacer(modifier = Modifier.weight(1f))
if (pendingRunCount > 0) {
FilledTonalIconButton(
onClick = onAbort,
colors =
IconButtonDefaults.filledTonalIconButtonColors(
containerColor = Color(0x33E74C3C),
contentColor = Color(0xFFE74C3C),
),
) {
Icon(Icons.Default.Stop, contentDescription = "Abort")
}
} else {
FilledTonalIconButton(onClick = {
val text = input
input = ""
onSend(text)
}, enabled = canSend) {
Icon(Icons.Default.ArrowUpward, contentDescription = "Send")
}
}
}
if (!errorText.isNullOrBlank()) {
Text(
text = errorText,
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.error,
maxLines = 2,
)
}
}
}
}
@Composable
private fun ConnectionPill(sessionKey: String, healthOk: Boolean) {
Surface(
shape = RoundedCornerShape(999.dp),
color = MaterialTheme.colorScheme.surfaceContainerHighest,
) {
Row(
modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Surface(
modifier = Modifier.size(7.dp),
shape = androidx.compose.foundation.shape.CircleShape,
color = if (healthOk) Color(0xFF2ECC71) else Color(0xFFF39C12),
) {}
Text(sessionKey, style = MaterialTheme.typography.labelSmall)
Text(
if (healthOk) "Connected" else "Connecting…",
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
}
@Composable
private fun ThinkingMenuItem(
value: String,
current: String,
onSet: (String) -> Unit,
onDismiss: () -> Unit,
) {
DropdownMenuItem(
text = { Text(thinkingLabel(value)) },
onClick = {
onSet(value)
onDismiss()
},
trailingIcon = {
if (value == current.trim().lowercase()) {
Text("✓")
} else {
Spacer(modifier = Modifier.width(10.dp))
}
},
)
}
private fun thinkingLabel(raw: String): String {
return when (raw.trim().lowercase()) {
"low" -> "Low"
"medium" -> "Medium"
"high" -> "High"
else -> "Off"
}
}
@Composable
private fun AttachmentsStrip(
attachments: List<PendingImageAttachment>,
onRemoveAttachment: (id: String) -> Unit,
) {
Row(
modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()),
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
for (att in attachments) {
AttachmentChip(
fileName = att.fileName,
onRemove = { onRemoveAttachment(att.id) },
)
}
}
}
@Composable
private fun AttachmentChip(fileName: String, onRemove: () -> Unit) {
Surface(
shape = RoundedCornerShape(999.dp),
color = MaterialTheme.colorScheme.primary.copy(alpha = 0.10f),
) {
Row(
modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Text(text = fileName, style = MaterialTheme.typography.bodySmall, maxLines = 1)
FilledTonalIconButton(
onClick = onRemove,
modifier = Modifier.size(30.dp),
) {
Text("×")
}
}
}
}
@@ -1,214 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import android.graphics.BitmapFactory
import android.util.Base64
import androidx.compose.foundation.Image
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.text.selection.SelectionContainer
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.AnnotatedString
import androidx.compose.ui.text.SpanStyle
import androidx.compose.ui.text.buildAnnotatedString
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontStyle
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.withStyle
import androidx.compose.ui.unit.dp
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
@Composable
fun ChatMarkdown(text: String) {
val blocks = remember(text) { splitMarkdown(text) }
val inlineCodeBg = MaterialTheme.colorScheme.surfaceContainerLow
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
for (b in blocks) {
when (b) {
is ChatMarkdownBlock.Text -> {
val trimmed = b.text.trimEnd()
if (trimmed.isEmpty()) continue
Text(
text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg),
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurface,
)
}
is ChatMarkdownBlock.Code -> {
SelectionContainer(modifier = Modifier.fillMaxWidth()) {
ChatCodeBlock(code = b.code, language = b.language)
}
}
is ChatMarkdownBlock.InlineImage -> {
InlineBase64Image(base64 = b.base64, mimeType = b.mimeType)
}
}
}
}
}
private sealed interface ChatMarkdownBlock {
data class Text(val text: String) : ChatMarkdownBlock
data class Code(val code: String, val language: String?) : ChatMarkdownBlock
data class InlineImage(val mimeType: String?, val base64: String) : ChatMarkdownBlock
}
private fun splitMarkdown(raw: String): List<ChatMarkdownBlock> {
if (raw.isEmpty()) return emptyList()
val out = ArrayList<ChatMarkdownBlock>()
var idx = 0
while (idx < raw.length) {
val fenceStart = raw.indexOf("```", startIndex = idx)
if (fenceStart < 0) {
out.addAll(splitInlineImages(raw.substring(idx)))
break
}
if (fenceStart > idx) {
out.addAll(splitInlineImages(raw.substring(idx, fenceStart)))
}
val langLineStart = fenceStart + 3
val langLineEnd = raw.indexOf('\n', startIndex = langLineStart).let { if (it < 0) raw.length else it }
val language = raw.substring(langLineStart, langLineEnd).trim().ifEmpty { null }
val codeStart = if (langLineEnd < raw.length && raw[langLineEnd] == '\n') langLineEnd + 1 else langLineEnd
val fenceEnd = raw.indexOf("```", startIndex = codeStart)
if (fenceEnd < 0) {
out.addAll(splitInlineImages(raw.substring(fenceStart)))
break
}
val code = raw.substring(codeStart, fenceEnd)
out.add(ChatMarkdownBlock.Code(code = code, language = language))
idx = fenceEnd + 3
}
return out
}
private fun splitInlineImages(text: String): List<ChatMarkdownBlock> {
if (text.isEmpty()) return emptyList()
val regex = Regex("data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)")
val out = ArrayList<ChatMarkdownBlock>()
var idx = 0
while (idx < text.length) {
val m = regex.find(text, startIndex = idx) ?: break
val start = m.range.first
val end = m.range.last + 1
if (start > idx) out.add(ChatMarkdownBlock.Text(text.substring(idx, start)))
val mime = "image/" + (m.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png")
val b64 = m.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty()
if (b64.isNotEmpty()) {
out.add(ChatMarkdownBlock.InlineImage(mimeType = mime, base64 = b64))
}
idx = end
}
if (idx < text.length) out.add(ChatMarkdownBlock.Text(text.substring(idx)))
return out
}
private fun parseInlineMarkdown(text: String, inlineCodeBg: androidx.compose.ui.graphics.Color): AnnotatedString {
if (text.isEmpty()) return AnnotatedString("")
val out = buildAnnotatedString {
var i = 0
while (i < text.length) {
if (text.startsWith("**", startIndex = i)) {
val end = text.indexOf("**", startIndex = i + 2)
if (end > i + 2) {
withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) {
append(text.substring(i + 2, end))
}
i = end + 2
continue
}
}
if (text[i] == '`') {
val end = text.indexOf('`', startIndex = i + 1)
if (end > i + 1) {
withStyle(
SpanStyle(
fontFamily = FontFamily.Monospace,
background = inlineCodeBg,
),
) {
append(text.substring(i + 1, end))
}
i = end + 1
continue
}
}
if (text[i] == '*' && (i + 1 < text.length && text[i + 1] != '*')) {
val end = text.indexOf('*', startIndex = i + 1)
if (end > i + 1) {
withStyle(SpanStyle(fontStyle = FontStyle.Italic)) {
append(text.substring(i + 1, end))
}
i = end + 1
continue
}
}
append(text[i])
i += 1
}
}
return out
}
@Composable
private fun InlineBase64Image(base64: String, mimeType: String?) {
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
var failed by remember(base64) { mutableStateOf(false) }
LaunchedEffect(base64) {
failed = false
image =
withContext(Dispatchers.Default) {
try {
val bytes = Base64.decode(base64, Base64.DEFAULT)
val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null
bitmap.asImageBitmap()
} catch (_: Throwable) {
null
}
}
if (image == null) failed = true
}
if (image != null) {
Image(
bitmap = image!!,
contentDescription = mimeType ?: "image",
contentScale = ContentScale.Fit,
modifier = Modifier.fillMaxWidth(),
)
} else if (failed) {
Text(
text = "Image unavailable",
modifier = Modifier.padding(vertical = 2.dp),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
@@ -1,111 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.rememberLazyListState
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ArrowCircleDown
import androidx.compose.material3.Card
import androidx.compose.material3.CardDefaults
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.chat.ChatMessage
import com.steipete.clawdis.node.chat.ChatPendingToolCall
@Composable
fun ChatMessageListCard(
messages: List<ChatMessage>,
pendingRunCount: Int,
pendingToolCalls: List<ChatPendingToolCall>,
streamingAssistantText: String?,
modifier: Modifier = Modifier,
) {
val listState = rememberLazyListState()
LaunchedEffect(messages.size, pendingRunCount, pendingToolCalls.size, streamingAssistantText) {
val total =
messages.size +
(if (pendingRunCount > 0) 1 else 0) +
(if (pendingToolCalls.isNotEmpty()) 1 else 0) +
(if (!streamingAssistantText.isNullOrBlank()) 1 else 0)
if (total <= 0) return@LaunchedEffect
listState.animateScrollToItem(index = total - 1)
}
Card(
modifier = modifier.fillMaxWidth(),
shape = MaterialTheme.shapes.large,
colors =
CardDefaults.cardColors(
containerColor = MaterialTheme.colorScheme.surfaceContainer,
),
elevation = CardDefaults.cardElevation(defaultElevation = 0.dp),
) {
Box(modifier = Modifier.fillMaxSize()) {
LazyColumn(
modifier = Modifier.fillMaxSize(),
state = listState,
verticalArrangement = Arrangement.spacedBy(14.dp),
contentPadding = androidx.compose.foundation.layout.PaddingValues(top = 12.dp, bottom = 12.dp, start = 12.dp, end = 12.dp),
) {
items(count = messages.size, key = { idx -> messages[idx].id }) { idx ->
ChatMessageBubble(message = messages[idx])
}
if (pendingRunCount > 0) {
item(key = "typing") {
ChatTypingIndicatorBubble()
}
}
if (pendingToolCalls.isNotEmpty()) {
item(key = "tools") {
ChatPendingToolsBubble(toolCalls = pendingToolCalls)
}
}
val stream = streamingAssistantText?.trim()
if (!stream.isNullOrEmpty()) {
item(key = "stream") {
ChatStreamingAssistantBubble(text = stream)
}
}
}
if (messages.isEmpty() && pendingRunCount == 0 && pendingToolCalls.isEmpty() && streamingAssistantText.isNullOrBlank()) {
EmptyChatHint(modifier = Modifier.align(Alignment.Center))
}
}
}
}
@Composable
private fun EmptyChatHint(modifier: Modifier = Modifier) {
Row(
modifier = modifier.alpha(0.7f),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Icon(
imageVector = Icons.Default.ArrowCircleDown,
contentDescription = null,
tint = MaterialTheme.colorScheme.onSurfaceVariant,
)
Text(
text = "Message Clawd…",
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
@@ -1,218 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import android.graphics.BitmapFactory
import android.util.Base64
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.unit.dp
import androidx.compose.foundation.Image
import com.steipete.clawdis.node.chat.ChatMessage
import com.steipete.clawdis.node.chat.ChatMessageContent
import com.steipete.clawdis.node.chat.ChatPendingToolCall
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
@Composable
fun ChatMessageBubble(message: ChatMessage) {
val isUser = message.role.lowercase() == "user"
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start,
) {
Surface(
shape = RoundedCornerShape(16.dp),
tonalElevation = 0.dp,
shadowElevation = 0.dp,
color = Color.Transparent,
modifier = Modifier.fillMaxWidth(0.92f),
) {
Box(
modifier =
Modifier
.background(bubbleBackground(isUser))
.padding(horizontal = 12.dp, vertical = 10.dp),
) {
ChatMessageBody(content = message.content)
}
}
}
}
@Composable
private fun ChatMessageBody(content: List<ChatMessageContent>) {
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
for (part in content) {
when (part.type) {
"text" -> {
val text = part.text ?: continue
ChatMarkdown(text = text)
}
else -> {
val b64 = part.base64 ?: continue
ChatBase64Image(base64 = b64, mimeType = part.mimeType)
}
}
}
}
}
@Composable
fun ChatTypingIndicatorBubble() {
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) {
Surface(
shape = RoundedCornerShape(16.dp),
color = MaterialTheme.colorScheme.surfaceContainer,
) {
Row(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
DotPulse()
Text("Thinking…", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
}
}
@Composable
fun ChatPendingToolsBubble(toolCalls: List<ChatPendingToolCall>) {
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) {
Surface(
shape = RoundedCornerShape(16.dp),
color = MaterialTheme.colorScheme.surfaceContainer,
) {
Column(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), verticalArrangement = Arrangement.spacedBy(6.dp)) {
Text("Tools", style = MaterialTheme.typography.labelLarge, color = MaterialTheme.colorScheme.onSurface)
for (t in toolCalls.take(6)) {
Text("· ${t.name}", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
if (toolCalls.size > 6) {
Text("… +${toolCalls.size - 6} more", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
}
}
}
@Composable
fun ChatStreamingAssistantBubble(text: String) {
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) {
Surface(
shape = RoundedCornerShape(16.dp),
color = MaterialTheme.colorScheme.surfaceContainer,
) {
Box(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp)) {
ChatMarkdown(text = text)
}
}
}
}
@Composable
private fun bubbleBackground(isUser: Boolean): Brush {
return if (isUser) {
Brush.linearGradient(
colors = listOf(MaterialTheme.colorScheme.primary, MaterialTheme.colorScheme.primary.copy(alpha = 0.78f)),
)
} else {
Brush.linearGradient(
colors = listOf(MaterialTheme.colorScheme.surfaceContainer, MaterialTheme.colorScheme.surfaceContainerHigh),
)
}
}
@Composable
private fun ChatBase64Image(base64: String, mimeType: String?) {
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
var failed by remember(base64) { mutableStateOf(false) }
LaunchedEffect(base64) {
failed = false
image =
withContext(Dispatchers.Default) {
try {
val bytes = Base64.decode(base64, Base64.DEFAULT)
val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null
bitmap.asImageBitmap()
} catch (_: Throwable) {
null
}
}
if (image == null) failed = true
}
if (image != null) {
Image(
bitmap = image!!,
contentDescription = mimeType ?: "attachment",
contentScale = ContentScale.Fit,
modifier = Modifier.fillMaxWidth(),
)
} else if (failed) {
Text("Unsupported attachment", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
@Composable
private fun DotPulse() {
Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) {
PulseDot(alpha = 0.38f)
PulseDot(alpha = 0.62f)
PulseDot(alpha = 0.90f)
}
}
@Composable
private fun PulseDot(alpha: Float) {
Surface(
modifier = Modifier.size(6.dp).alpha(alpha),
shape = CircleShape,
color = MaterialTheme.colorScheme.onSurfaceVariant,
) {}
}
@Composable
fun ChatCodeBlock(code: String, language: String?) {
Surface(
shape = RoundedCornerShape(12.dp),
color = MaterialTheme.colorScheme.surfaceContainerLowest,
modifier = Modifier.fillMaxWidth(),
) {
Text(
text = code.trimEnd(),
modifier = Modifier.padding(10.dp),
fontFamily = FontFamily.Monospace,
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurface,
)
}
}
@@ -1,93 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material3.AlertDialog
import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.chat.ChatSessionEntry
@Composable
fun ChatSessionsDialog(
currentSessionKey: String,
sessions: List<ChatSessionEntry>,
onDismiss: () -> Unit,
onRefresh: () -> Unit,
onSelect: (sessionKey: String) -> Unit,
) {
AlertDialog(
onDismissRequest = onDismiss,
confirmButton = {},
title = {
Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth()) {
Text("Sessions", style = MaterialTheme.typography.titleMedium)
Spacer(modifier = Modifier.weight(1f))
FilledTonalIconButton(onClick = onRefresh) {
Icon(Icons.Default.Refresh, contentDescription = "Refresh")
}
}
},
text = {
if (sessions.isEmpty()) {
Text("No sessions", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
} else {
LazyColumn(verticalArrangement = Arrangement.spacedBy(8.dp)) {
items(sessions, key = { it.key }) { entry ->
SessionRow(
entry = entry,
isCurrent = entry.key == currentSessionKey,
onClick = { onSelect(entry.key) },
)
}
}
}
},
)
}
@Composable
private fun SessionRow(
entry: ChatSessionEntry,
isCurrent: Boolean,
onClick: () -> Unit,
) {
Surface(
onClick = onClick,
shape = MaterialTheme.shapes.medium,
color =
if (isCurrent) {
MaterialTheme.colorScheme.primary.copy(alpha = 0.14f)
} else {
MaterialTheme.colorScheme.surfaceContainer
},
modifier = Modifier.fillMaxWidth(),
) {
Row(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(10.dp),
) {
Text(entry.key, style = MaterialTheme.typography.bodyMedium)
Spacer(modifier = Modifier.weight(1f))
if (isCurrent) {
Text("Current", style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
}
}
@@ -1,157 +0,0 @@
package com.steipete.clawdis.node.ui.chat
import android.content.ContentResolver
import android.net.Uri
import android.util.Base64
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateListOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import com.steipete.clawdis.node.MainViewModel
import com.steipete.clawdis.node.chat.OutgoingAttachment
import java.io.ByteArrayOutputStream
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
@Composable
fun ChatSheetContent(viewModel: MainViewModel) {
val messages by viewModel.chatMessages.collectAsState()
val errorText by viewModel.chatError.collectAsState()
val pendingRunCount by viewModel.pendingRunCount.collectAsState()
val healthOk by viewModel.chatHealthOk.collectAsState()
val sessionKey by viewModel.chatSessionKey.collectAsState()
val thinkingLevel by viewModel.chatThinkingLevel.collectAsState()
val streamingAssistantText by viewModel.chatStreamingAssistantText.collectAsState()
val pendingToolCalls by viewModel.chatPendingToolCalls.collectAsState()
val sessions by viewModel.chatSessions.collectAsState()
var showSessions by remember { mutableStateOf(false) }
LaunchedEffect(Unit) {
viewModel.loadChat("main")
}
val context = LocalContext.current
val resolver = context.contentResolver
val scope = rememberCoroutineScope()
val attachments = remember { mutableStateListOf<PendingImageAttachment>() }
val pickImages =
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult
scope.launch(Dispatchers.IO) {
val next =
uris.take(8).mapNotNull { uri ->
try {
loadImageAttachment(resolver, uri)
} catch (_: Throwable) {
null
}
}
withContext(Dispatchers.Main) {
attachments.addAll(next)
}
}
}
Column(
modifier =
Modifier
.fillMaxSize()
.padding(horizontal = 12.dp, vertical = 12.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
ChatMessageListCard(
messages = messages,
pendingRunCount = pendingRunCount,
pendingToolCalls = pendingToolCalls,
streamingAssistantText = streamingAssistantText,
modifier = Modifier.weight(1f, fill = true),
)
ChatComposer(
sessionKey = sessionKey,
healthOk = healthOk,
thinkingLevel = thinkingLevel,
pendingRunCount = pendingRunCount,
errorText = errorText,
attachments = attachments,
onPickImages = { pickImages.launch("image/*") },
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) },
onShowSessions = { showSessions = true },
onRefresh = { viewModel.refreshChat() },
onAbort = { viewModel.abortChat() },
onSend = { text ->
val outgoing =
attachments.map { att ->
OutgoingAttachment(
type = "image",
mimeType = att.mimeType,
fileName = att.fileName,
base64 = att.base64,
)
}
viewModel.sendChat(message = text, thinking = thinkingLevel, attachments = outgoing)
attachments.clear()
},
)
}
if (showSessions) {
ChatSessionsDialog(
currentSessionKey = sessionKey,
sessions = sessions,
onDismiss = { showSessions = false },
onRefresh = { viewModel.refreshChatSessions(limit = 50) },
onSelect = { key ->
viewModel.switchChatSession(key)
showSessions = false
},
)
}
}
data class PendingImageAttachment(
val id: String,
val fileName: String,
val mimeType: String,
val base64: String,
)
private suspend fun loadImageAttachment(resolver: ContentResolver, uri: Uri): PendingImageAttachment {
val mimeType = resolver.getType(uri) ?: "image/*"
val fileName = (uri.lastPathSegment ?: "image").substringAfterLast('/')
val bytes =
withContext(Dispatchers.IO) {
resolver.openInputStream(uri)?.use { input ->
val out = ByteArrayOutputStream()
input.copyTo(out)
out.toByteArray()
} ?: ByteArray(0)
}
if (bytes.isEmpty()) throw IllegalStateException("empty attachment")
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
return PendingImageAttachment(
id = uri.toString() + "#" + System.currentTimeMillis().toString(),
fileName = fileName,
mimeType = mimeType,
base64 = base64,
)
}
@@ -1,40 +0,0 @@
package com.steipete.clawdis.node.voice
object VoiceWakeCommandExtractor {
fun extractCommand(text: String, triggerWords: List<String>): String? {
val raw = text.trim()
if (raw.isEmpty()) return null
val triggers =
triggerWords
.map { it.trim().lowercase() }
.filter { it.isNotEmpty() }
.distinct()
if (triggers.isEmpty()) return null
val alternation = triggers.joinToString("|") { Regex.escape(it) }
// Match: "<anything> <trigger><punct/space> <command>"
val regex = Regex("(?i)(?:^|\\s)($alternation)\\b[\\s\\p{Punct}]*([\\s\\S]+)$")
val match = regex.find(raw) ?: return null
val extracted = match.groupValues.getOrNull(2)?.trim().orEmpty()
if (extracted.isEmpty()) return null
val cleaned = extracted.trimStart { it.isWhitespace() || it.isPunctuation() }.trim()
if (cleaned.isEmpty()) return null
return cleaned
}
}
private fun Char.isPunctuation(): Boolean {
return when (Character.getType(this)) {
Character.CONNECTOR_PUNCTUATION.toInt(),
Character.DASH_PUNCTUATION.toInt(),
Character.START_PUNCTUATION.toInt(),
Character.END_PUNCTUATION.toInt(),
Character.INITIAL_QUOTE_PUNCTUATION.toInt(),
Character.FINAL_QUOTE_PUNCTUATION.toInt(),
Character.OTHER_PUNCTUATION.toInt(),
-> true
else -> false
}
}
@@ -1,173 +0,0 @@
package com.steipete.clawdis.node.voice
import android.content.Context
import android.content.Intent
import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.speech.RecognitionListener
import android.speech.RecognizerIntent
import android.speech.SpeechRecognizer
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch
class VoiceWakeManager(
private val context: Context,
private val scope: CoroutineScope,
private val onCommand: suspend (String) -> Unit,
) {
private val mainHandler = Handler(Looper.getMainLooper())
private val _isListening = MutableStateFlow(false)
val isListening: StateFlow<Boolean> = _isListening
private val _statusText = MutableStateFlow("Off")
val statusText: StateFlow<String> = _statusText
var triggerWords: List<String> = emptyList()
private set
private var recognizer: SpeechRecognizer? = null
private var restartJob: Job? = null
private var lastDispatched: String? = null
private var stopRequested = false
fun setTriggerWords(words: List<String>) {
triggerWords = words
}
fun start() {
mainHandler.post {
if (_isListening.value) return@post
stopRequested = false
if (!SpeechRecognizer.isRecognitionAvailable(context)) {
_isListening.value = false
_statusText.value = "Speech recognizer unavailable"
return@post
}
try {
recognizer?.destroy()
recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) }
startListeningInternal()
} catch (err: Throwable) {
_isListening.value = false
_statusText.value = "Start failed: ${err.message ?: err::class.simpleName}"
}
}
}
fun stop(statusText: String = "Off") {
stopRequested = true
restartJob?.cancel()
restartJob = null
mainHandler.post {
_isListening.value = false
_statusText.value = statusText
recognizer?.cancel()
recognizer?.destroy()
recognizer = null
}
}
private fun startListeningInternal() {
val r = recognizer ?: return
val intent =
Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3)
putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName)
}
_statusText.value = "Listening"
_isListening.value = true
r.startListening(intent)
}
private fun scheduleRestart(delayMs: Long = 350) {
if (stopRequested) return
restartJob?.cancel()
restartJob =
scope.launch {
delay(delayMs)
mainHandler.post {
if (stopRequested) return@post
try {
recognizer?.cancel()
startListeningInternal()
} catch (_: Throwable) {
// Will be picked up by onError and retry again.
}
}
}
}
private fun handleTranscription(text: String) {
val command = VoiceWakeCommandExtractor.extractCommand(text, triggerWords) ?: return
if (command == lastDispatched) return
lastDispatched = command
scope.launch { onCommand(command) }
_statusText.value = "Triggered"
scheduleRestart(delayMs = 650)
}
private val listener =
object : RecognitionListener {
override fun onReadyForSpeech(params: Bundle?) {
_statusText.value = "Listening"
}
override fun onBeginningOfSpeech() {}
override fun onRmsChanged(rmsdB: Float) {}
override fun onBufferReceived(buffer: ByteArray?) {}
override fun onEndOfSpeech() {
scheduleRestart()
}
override fun onError(error: Int) {
if (stopRequested) return
_isListening.value = false
if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) {
_statusText.value = "Microphone permission required"
return
}
_statusText.value =
when (error) {
SpeechRecognizer.ERROR_AUDIO -> "Audio error"
SpeechRecognizer.ERROR_CLIENT -> "Client error"
SpeechRecognizer.ERROR_NETWORK -> "Network error"
SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout"
SpeechRecognizer.ERROR_NO_MATCH -> "Listening"
SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy"
SpeechRecognizer.ERROR_SERVER -> "Server error"
SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening"
else -> "Speech error ($error)"
}
scheduleRestart(delayMs = 600)
}
override fun onResults(results: Bundle?) {
val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
list.firstOrNull()?.let(::handleTranscription)
scheduleRestart()
}
override fun onPartialResults(partialResults: Bundle?) {
val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
list.firstOrNull()?.let(::handleTranscription)
}
override fun onEvent(eventType: Int, params: Bundle?) {}
}
}
@@ -1,12 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<network-security-config>
<!-- This app is primarily used on a trusted tailnet; allow cleartext for IP-based endpoints too. -->
<base-config cleartextTrafficPermitted="true" />
<!-- Allow HTTP for tailnet/local dev endpoints (e.g. canvas/background web). -->
<domain-config cleartextTrafficPermitted="true">
<domain includeSubdomains="true">clawdis.internal</domain>
</domain-config>
<domain-config cleartextTrafficPermitted="true">
<domain includeSubdomains="true">ts.net</domain>
</domain-config>
</network-security-config>
@@ -1,19 +0,0 @@
package com.steipete.clawdis.node.bridge
import org.junit.Assert.assertEquals
import org.junit.Test
class BonjourEscapesTest {
@Test
fun decodeNoop() {
assertEquals("", BonjourEscapes.decode(""))
assertEquals("hello", BonjourEscapes.decode("hello"))
}
@Test
fun decodeDecodesDecimalEscapes() {
assertEquals("Clawdis Gateway", BonjourEscapes.decode("Clawdis\\032Gateway"))
assertEquals("A B", BonjourEscapes.decode("A\\032B"))
}
}
@@ -48,8 +48,6 @@ class BridgePairingClientTest {
version = "test",
deviceFamily = "Android",
modelIdentifier = "SM-X000",
caps = null,
commands = null,
),
)
assertTrue(res.ok)
@@ -97,8 +95,6 @@ class BridgePairingClientTest {
version = "test",
deviceFamily = "Android",
modelIdentifier = "SM-X000",
caps = null,
commands = null,
),
)
assertTrue(res.ok)
@@ -15,8 +15,6 @@ import java.io.BufferedWriter
import java.io.InputStreamReader
import java.io.OutputStreamWriter
import java.net.ServerSocket
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
class BridgeSessionTest {
@Test
@@ -69,10 +67,6 @@ class BridgeSessionTest {
token = null,
platform = "Android",
version = "test",
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
@@ -135,10 +129,6 @@ class BridgeSessionTest {
token = null,
platform = "Android",
version = "test",
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
connected.await()
@@ -187,7 +177,7 @@ class BridgeSessionTest {
writer.flush()
// Ask the node to invoke something; handler will throw.
writer.write("""{"type":"invoke","id":"i1","command":"canvas.snapshot","paramsJSON":null}""")
writer.write("""{"type":"invoke","id":"i1","command":"screen.snapshot","paramsJSON":null}""")
writer.write("\n")
writer.flush()
@@ -206,10 +196,6 @@ class BridgeSessionTest {
token = null,
platform = "Android",
version = "test",
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
connected.await()
@@ -225,74 +211,6 @@ class BridgeSessionTest {
session.disconnect()
scope.cancel()
}
@Test(timeout = 12_000)
fun reconnectsAfterBridgeClosesDuringHello() = runBlocking {
val serverSocket = ServerSocket(0)
val port = serverSocket.localPort
val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
val connected = CountDownLatch(1)
val connectionsSeen = CountDownLatch(2)
val session =
BridgeSession(
scope = scope,
onConnected = { _, _ -> connected.countDown() },
onDisconnected = { /* ignore */ },
onEvent = { _, _ -> /* ignore */ },
onInvoke = { BridgeSession.InvokeResult.ok(null) },
)
val server =
async(Dispatchers.IO) {
serverSocket.use { ss ->
// First connection: read hello, then close (no response).
val sock1 = ss.accept()
sock1.use { s ->
val reader = BufferedReader(InputStreamReader(s.getInputStream(), Charsets.UTF_8))
reader.readLine() // hello
connectionsSeen.countDown()
}
// Second connection: complete hello.
val sock2 = ss.accept()
sock2.use { s ->
val reader = BufferedReader(InputStreamReader(s.getInputStream(), Charsets.UTF_8))
val writer = BufferedWriter(OutputStreamWriter(s.getOutputStream(), Charsets.UTF_8))
reader.readLine() // hello
writer.write("""{"type":"hello-ok","serverName":"Test Bridge"}""")
writer.write("\n")
writer.flush()
connectionsSeen.countDown()
Thread.sleep(200)
}
}
}
session.connect(
endpoint = BridgeEndpoint.manual(host = "127.0.0.1", port = port),
hello =
BridgeSession.Hello(
nodeId = "node-1",
displayName = "Android Node",
token = null,
platform = "Android",
version = "test",
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
assertTrue("expected two connection attempts", connectionsSeen.await(8, TimeUnit.SECONDS))
assertTrue("expected session to connect", connected.await(8, TimeUnit.SECONDS))
session.disconnect()
scope.cancel()
server.await()
}
}
private fun extractJsonString(raw: String, key: String): String {
@@ -1,43 +0,0 @@
package com.steipete.clawdis.node.node
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
class CanvasControllerSnapshotParamsTest {
@Test
fun parseSnapshotParamsDefaultsToJpeg() {
val params = CanvasController.parseSnapshotParams(null)
assertEquals(CanvasController.SnapshotFormat.Jpeg, params.format)
assertNull(params.quality)
assertNull(params.maxWidth)
}
@Test
fun parseSnapshotParamsParsesPng() {
val params = CanvasController.parseSnapshotParams("""{"format":"png","maxWidth":900}""")
assertEquals(CanvasController.SnapshotFormat.Png, params.format)
assertEquals(900, params.maxWidth)
}
@Test
fun parseSnapshotParamsParsesJpegAliases() {
assertEquals(
CanvasController.SnapshotFormat.Jpeg,
CanvasController.parseSnapshotParams("""{"format":"jpeg"}""").format,
)
assertEquals(
CanvasController.SnapshotFormat.Jpeg,
CanvasController.parseSnapshotParams("""{"format":"jpg"}""").format,
)
}
@Test
fun parseSnapshotParamsClampsQuality() {
val low = CanvasController.parseSnapshotParams("""{"quality":0.01}""")
assertEquals(0.1, low.quality)
val high = CanvasController.parseSnapshotParams("""{"quality":5}""")
assertEquals(1.0, high.quality)
}
}
@@ -1,35 +0,0 @@
package com.steipete.clawdis.node.protocol
import org.junit.Assert.assertEquals
import org.junit.Test
class ClawdisCanvasA2UIActionTest {
@Test
fun formatAgentMessageMatchesSharedSpec() {
val msg =
ClawdisCanvasA2UIAction.formatAgentMessage(
actionName = "Get Weather",
sessionKey = "main",
surfaceId = "main",
sourceComponentId = "btnWeather",
host = "Peter’s iPad",
instanceId = "ipad16,6",
contextJson = "{\"city\":\"Vienna\"}",
)
assertEquals(
"CANVAS_A2UI action=Get_Weather session=main surface=main component=btnWeather host=Peter_s_iPad instance=ipad16_6 ctx={\"city\":\"Vienna\"} default=update_canvas",
msg,
)
}
@Test
fun jsDispatchA2uiStatusIsStable() {
val js = ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId = "a1", ok = true, error = null)
assertEquals(
"window.dispatchEvent(new CustomEvent('clawdis:a2ui-action-status', { detail: { id: \"a1\", ok: true, error: \"\" } }));",
js,
)
}
}
@@ -1,35 +0,0 @@
package com.steipete.clawdis.node.protocol
import org.junit.Assert.assertEquals
import org.junit.Test
class ClawdisProtocolConstantsTest {
@Test
fun canvasCommandsUseStableStrings() {
assertEquals("canvas.present", ClawdisCanvasCommand.Present.rawValue)
assertEquals("canvas.hide", ClawdisCanvasCommand.Hide.rawValue)
assertEquals("canvas.navigate", ClawdisCanvasCommand.Navigate.rawValue)
assertEquals("canvas.eval", ClawdisCanvasCommand.Eval.rawValue)
assertEquals("canvas.snapshot", ClawdisCanvasCommand.Snapshot.rawValue)
}
@Test
fun a2uiCommandsUseStableStrings() {
assertEquals("canvas.a2ui.push", ClawdisCanvasA2UICommand.Push.rawValue)
assertEquals("canvas.a2ui.pushJSONL", ClawdisCanvasA2UICommand.PushJSONL.rawValue)
assertEquals("canvas.a2ui.reset", ClawdisCanvasA2UICommand.Reset.rawValue)
}
@Test
fun capabilitiesUseStableStrings() {
assertEquals("canvas", ClawdisCapability.Canvas.rawValue)
assertEquals("camera", ClawdisCapability.Camera.rawValue)
assertEquals("screen", ClawdisCapability.Screen.rawValue)
assertEquals("voiceWake", ClawdisCapability.VoiceWake.rawValue)
}
@Test
fun screenCommandsUseStableStrings() {
assertEquals("screen.record", ClawdisScreenCommand.Record.rawValue)
}
}
@@ -1,26 +0,0 @@
package com.steipete.clawdis.node.voice
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
class VoiceWakeCommandExtractorTest {
@Test
fun extractsCommandAfterTriggerWord() {
val res = VoiceWakeCommandExtractor.extractCommand("Claude take a photo", listOf("clawd", "claude"))
assertEquals("take a photo", res)
}
@Test
fun extractsCommandWithPunctuation() {
val res = VoiceWakeCommandExtractor.extractCommand("hey clawd, what's the weather?", listOf("clawd"))
assertEquals("what's the weather?", res)
}
@Test
fun returnsNullWhenNoCommandProvided() {
assertNull(VoiceWakeCommandExtractor.extractCommand("claude", listOf("claude")))
assertNull(VoiceWakeCommandExtractor.extractCommand("hey claude!", listOf("claude")))
}
}
+2 -2
View File
@@ -1,4 +1,4 @@
org.gradle.jvmargs=-Xmx3g -Dfile.encoding=UTF-8 --enable-native-access=ALL-UNNAMED
org.gradle.warning.mode=none
org.gradle.jvmargs=-Xmx3g -Dfile.encoding=UTF-8
android.useAndroidX=true
android.nonTransitiveRClass=true
+1 -1
View File
@@ -200,7 +200,7 @@ fi
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m" "--enable-native-access=ALL-UNNAMED"'
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+1 -1
View File
@@ -34,7 +34,7 @@ set APP_HOME=%DIRNAME%
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" "--enable-native-access=ALL-UNNAMED"
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
+1 -3
View File
@@ -53,9 +53,7 @@ actor BridgeClient {
platform: hello.platform,
version: hello.version,
deviceFamily: hello.deviceFamily,
modelIdentifier: hello.modelIdentifier,
caps: hello.caps,
commands: hello.commands),
modelIdentifier: hello.modelIdentifier),
over: connection)
onStatus?("Waiting for approval…")
@@ -126,7 +126,7 @@ final class BridgeConnectionController {
private func makeHello(token: String) -> BridgeHello {
let defaults = UserDefaults.standard
let nodeId = defaults.string(forKey: "node.instanceId") ?? "ios-node"
let displayName = self.resolvedDisplayName(defaults: defaults)
let displayName = defaults.string(forKey: "node.displayName") ?? "iOS Node"
return BridgeHello(
nodeId: nodeId,
@@ -135,73 +135,19 @@ final class BridgeConnectionController {
platform: self.platformString(),
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands())
}
private func resolvedDisplayName(defaults: UserDefaults) -> String {
let key = "node.displayName"
let existing = defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if !existing.isEmpty, existing != "iOS Node" { return existing }
let deviceName = UIDevice.current.name.trimmingCharacters(in: .whitespacesAndNewlines)
let candidate = deviceName.isEmpty ? "iOS Node" : deviceName
if existing.isEmpty || existing == "iOS Node" {
defaults.set(candidate, forKey: key)
}
return candidate
}
private func currentCaps() -> [String] {
var caps = [ClawdisCapability.canvas.rawValue, ClawdisCapability.screen.rawValue]
// Default-on: if the key doesn't exist yet, treat it as enabled.
let cameraEnabled =
UserDefaults.standard.object(forKey: "camera.enabled") == nil
? true
: UserDefaults.standard.bool(forKey: "camera.enabled")
if cameraEnabled { caps.append(ClawdisCapability.camera.rawValue) }
let voiceWakeEnabled = UserDefaults.standard.bool(forKey: VoiceWakePreferences.enabledKey)
if voiceWakeEnabled { caps.append(ClawdisCapability.voiceWake.rawValue) }
return caps
}
private func currentCommands() -> [String] {
var commands: [String] = [
ClawdisCanvasCommand.present.rawValue,
ClawdisCanvasCommand.hide.rawValue,
ClawdisCanvasCommand.navigate.rawValue,
ClawdisCanvasCommand.evalJS.rawValue,
ClawdisCanvasCommand.snapshot.rawValue,
ClawdisCanvasA2UICommand.push.rawValue,
ClawdisCanvasA2UICommand.pushJSONL.rawValue,
ClawdisCanvasA2UICommand.reset.rawValue,
ClawdisScreenCommand.record.rawValue,
]
let caps = Set(self.currentCaps())
if caps.contains(ClawdisCapability.camera.rawValue) {
commands.append(ClawdisCameraCommand.snap.rawValue)
commands.append(ClawdisCameraCommand.clip.rawValue)
}
return commands
modelIdentifier: self.modelIdentifier())
}
private func platformString() -> String {
let v = ProcessInfo.processInfo.operatingSystemVersion
let name = switch UIDevice.current.userInterfaceIdiom {
let name: String
switch UIDevice.current.userInterfaceIdiom {
case .pad:
"iPadOS"
name = "iPadOS"
case .phone:
"iOS"
name = "iOS"
default:
"iOS"
name = "iOS"
}
return "\(name) \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"
}
@@ -209,11 +155,11 @@ final class BridgeConnectionController {
private func deviceFamily() -> String {
switch UIDevice.current.userInterfaceIdiom {
case .pad:
"iPad"
return "iPad"
case .phone:
"iPhone"
return "iPhone"
default:
"iOS"
return "iOS"
}
}
@@ -214,7 +214,7 @@ actor CameraController {
nonisolated static func clampDurationMs(_ ms: Int?) -> Int {
let v = ms ?? 3000
// Keep clips short by default; avoid huge base64 payloads on the bridge.
return min(60000, max(250, v))
return min(15000, max(250, v))
}
private nonisolated static func exportToMP4(inputURL: URL, outputURL: URL) async throws {
@@ -9,28 +9,6 @@ struct IOSBridgeChatTransport: ClawdisChatTransport, Sendable {
self.bridge = bridge
}
func abortRun(sessionKey: String, runId: String) async throws {
struct Params: Codable {
var sessionKey: String
var runId: String
}
let data = try JSONEncoder().encode(Params(sessionKey: sessionKey, runId: runId))
let json = String(data: data, encoding: .utf8)
_ = try await self.bridge.request(method: "chat.abort", paramsJSON: json, timeoutSeconds: 10)
}
func listSessions(limit: Int?) async throws -> ClawdisChatSessionsListResponse {
struct Params: Codable {
var includeGlobal: Bool
var includeUnknown: Bool
var limit: Int?
}
let data = try JSONEncoder().encode(Params(includeGlobal: true, includeUnknown: false, limit: limit))
let json = String(data: data, encoding: .utf8)
let res = try await self.bridge.request(method: "sessions.list", paramsJSON: json, timeoutSeconds: 15)
return try JSONDecoder().decode(ClawdisChatSessionsListResponse.self, from: res)
}
func setActiveSessionKey(_ sessionKey: String) async throws {
struct Subscribe: Codable { var sessionKey: String }
let data = try JSONEncoder().encode(Subscribe(sessionKey: sessionKey))
@@ -101,11 +79,6 @@ struct IOSBridgeChatTransport: ClawdisChatTransport, Sendable {
if let payload = try? JSONDecoder().decode(ClawdisChatEventPayload.self, from: data) {
continuation.yield(.chat(payload))
}
case "agent":
guard let json = evt.payloadJSON, let data = json.data(using: .utf8) else { break }
if let payload = try? JSONDecoder().decode(ClawdisAgentEventPayload.self, from: data) {
continuation.yield(.agent(payload))
}
default:
break
}
+23 -263
View File
@@ -2,23 +2,14 @@ import ClawdisKit
import Network
import Observation
import SwiftUI
import UIKit
@MainActor
@Observable
final class NodeAppModel {
enum CameraHUDKind {
case photo
case recording
case success
case error
}
var isBackgrounded: Bool = false
let screen = ScreenController()
let camera = CameraController()
private let screenRecorder = ScreenRecordService()
var bridgeStatusText: String = "Offline"
var bridgeStatusText: String = "Not connected"
var bridgeServerName: String?
var bridgeRemoteAddress: String?
var connectedBridgeID: String?
@@ -26,15 +17,10 @@ final class NodeAppModel {
private let bridge = BridgeSession()
private var bridgeTask: Task<Void, Never>?
private var voiceWakeSyncTask: Task<Void, Never>?
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
let voiceWake = VoiceWakeManager()
var bridgeSession: BridgeSession { self.bridge }
var cameraHUDText: String?
var cameraHUDKind: CameraHUDKind?
var cameraFlashNonce: Int = 0
init() {
self.voiceWake.configure { [weak self] cmd in
guard let self else { return }
@@ -57,90 +43,6 @@ final class NodeAppModel {
await self.handleDeepLink(url: url)
}
}
// Wire up A2UI action clicks (buttons, etc.)
self.screen.onA2UIAction = { [weak self] body in
guard let self else { return }
Task { @MainActor in
await self.handleCanvasA2UIAction(body: body)
}
}
}
private func handleCanvasA2UIAction(body: [String: Any]) async {
let userActionAny = body["userAction"] ?? body
let userAction: [String: Any] = {
if let dict = userActionAny as? [String: Any] { return dict }
if let dict = userActionAny as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !userAction.isEmpty else { return }
guard let name = userAction["name"] as? String, !name.isEmpty else { return }
let actionId: String = {
let id = (userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return id.isEmpty ? UUID().uuidString : id
}()
let surfaceId: String = {
let raw = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return raw.isEmpty ? "main" : raw
}()
let sourceComponentId: String = {
let raw = (userAction[
"sourceComponentId",
] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return raw.isEmpty ? "-" : raw
}()
let host = UserDefaults.standard.string(forKey: "node.displayName") ?? UIDevice.current.name
let instanceId = (UserDefaults.standard.string(forKey: "node.instanceId") ?? "ios-node").lowercased()
let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"])
let sessionKey = "main"
let message = ClawdisCanvasA2UIAction.formatAgentMessage(
actionName: name,
sessionKey: sessionKey,
surfaceId: surfaceId,
sourceComponentId: sourceComponentId,
host: host,
instanceId: instanceId,
contextJSON: contextJSON)
let ok: Bool
var errorText: String? = nil
if await !self.isBridgeConnected() {
ok = false
errorText = "bridge not connected"
} else {
do {
try await self.sendAgentRequest(link: AgentDeepLink(
message: message,
sessionKey: sessionKey,
thinking: "low",
deliver: false,
to: nil,
channel: nil,
timeoutSeconds: nil,
key: actionId))
ok = true
} catch {
ok = false
errorText = error.localizedDescription
}
}
let js = ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId: actionId, ok: ok, error: errorText)
do {
_ = try await self.screen.eval(javaScript: js)
} catch {
// ignore
}
}
func setScenePhase(_ phase: ScenePhase) {
@@ -227,7 +129,7 @@ final class NodeAppModel {
}
await MainActor.run {
self.bridgeStatusText = "Offline"
self.bridgeStatusText = "Disconnected"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeID = nil
@@ -241,7 +143,7 @@ final class NodeAppModel {
self.voiceWakeSyncTask?.cancel()
self.voiceWakeSyncTask = nil
Task { await self.bridge.disconnect() }
self.bridgeStatusText = "Offline"
self.bridgeStatusText = "Disconnected"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeID = nil
@@ -363,20 +265,16 @@ final class NodeAppModel {
}
private func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
let command = req.command
if command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen."),
self.isBackgrounded
{
if req.command.hasPrefix("screen.") || req.command.hasPrefix("camera."), self.isBackgrounded {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(
code: .backgroundUnavailable,
message: "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground"))
message: "NODE_BACKGROUND_UNAVAILABLE: screen/camera commands require foreground"))
}
if command.hasPrefix("camera."), !self.isCameraEnabled() {
if req.command.hasPrefix("camera."), !self.isCameraEnabled() {
return BridgeInvokeResponse(
id: req.id,
ok: false,
@@ -386,113 +284,37 @@ final class NodeAppModel {
}
do {
switch command {
case ClawdisCanvasCommand.present.rawValue:
let params = (try? Self.decodeParams(ClawdisCanvasPresentParams.self, from: req.paramsJSON)) ??
ClawdisCanvasPresentParams()
let url = params.url?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if url.isEmpty {
self.screen.showDefaultCanvas()
} else {
self.screen.navigate(to: url)
}
switch req.command {
case ClawdisScreenCommand.show.rawValue:
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisCanvasCommand.hide.rawValue:
case ClawdisScreenCommand.hide.rawValue:
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisCanvasCommand.navigate.rawValue:
let params = try Self.decodeParams(ClawdisCanvasNavigateParams.self, from: req.paramsJSON)
case ClawdisScreenCommand.setMode.rawValue:
let params = try Self.decodeParams(ClawdisScreenSetModeParams.self, from: req.paramsJSON)
self.screen.setMode(params.mode)
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisScreenCommand.navigate.rawValue:
let params = try Self.decodeParams(ClawdisScreenNavigateParams.self, from: req.paramsJSON)
self.screen.navigate(to: params.url)
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisCanvasCommand.evalJS.rawValue:
let params = try Self.decodeParams(ClawdisCanvasEvalParams.self, from: req.paramsJSON)
case ClawdisScreenCommand.evalJS.rawValue:
let params = try Self.decodeParams(ClawdisScreenEvalParams.self, from: req.paramsJSON)
let result = try await self.screen.eval(javaScript: params.javaScript)
let payload = try Self.encodePayload(["result": result])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisCanvasCommand.snapshot.rawValue:
let params = try? Self.decodeParams(ClawdisCanvasSnapshotParams.self, from: req.paramsJSON)
let format = params?.format ?? .jpeg
let maxWidth: CGFloat? = {
if let raw = params?.maxWidth, raw > 0 { return CGFloat(raw) }
// Keep default snapshots comfortably below the gateway client's maxPayload.
// For full-res, clients should explicitly request a larger maxWidth.
return switch format {
case .png: 900
case .jpeg: 1600
}
}()
let base64 = try await self.screen.snapshotBase64(
maxWidth: maxWidth,
format: format,
quality: params?.quality)
let payload = try Self.encodePayload([
"format": format == .jpeg ? "jpeg" : "png",
"base64": base64,
])
case ClawdisScreenCommand.snapshot.rawValue:
let params = try? Self.decodeParams(ClawdisScreenSnapshotParams.self, from: req.paramsJSON)
let maxWidth = params?.maxWidth.map { CGFloat($0) }
let base64 = try await self.screen.snapshotPNGBase64(maxWidth: maxWidth)
let payload = try Self.encodePayload(["format": "png", "base64": base64])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisCanvasA2UICommand.reset.rawValue:
self.screen.showDefaultCanvas()
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(code: .unavailable, message: "A2UI not ready"))
}
let json = try await self.screen.eval(javaScript: """
(() => {
if (!globalThis.clawdisA2UI) return JSON.stringify({ ok: false, error: "missing clawdisA2UI" });
return JSON.stringify(globalThis.clawdisA2UI.reset());
})()
""")
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
case ClawdisCanvasA2UICommand.push.rawValue, ClawdisCanvasA2UICommand.pushJSONL.rawValue:
let messages: [AnyCodable]
if command == ClawdisCanvasA2UICommand.pushJSONL.rawValue {
let params = try Self.decodeParams(ClawdisCanvasA2UIPushJSONLParams.self, from: req.paramsJSON)
messages = try ClawdisCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl)
} else {
do {
let params = try Self.decodeParams(ClawdisCanvasA2UIPushParams.self, from: req.paramsJSON)
messages = params.messages
} catch {
// Be forgiving: some clients still send JSONL payloads to `canvas.a2ui.push`.
let params = try Self.decodeParams(ClawdisCanvasA2UIPushJSONLParams.self, from: req.paramsJSON)
messages = try ClawdisCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl)
}
}
self.screen.showDefaultCanvas()
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(code: .unavailable, message: "A2UI not ready"))
}
let messagesJSON = try ClawdisCanvasA2UIJSONL.encodeMessagesJSONArray(messages)
let js = """
(() => {
try {
if (!globalThis.clawdisA2UI) return JSON.stringify({ ok: false, error: "missing clawdisA2UI" });
const messages = \(messagesJSON);
return JSON.stringify(globalThis.clawdisA2UI.applyMessages(messages));
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e) });
}
})()
"""
let resultJSON = try await self.screen.eval(javaScript: js)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON)
case ClawdisCameraCommand.snap.rawValue:
self.showCameraHUD(text: "Taking photo…", kind: .photo)
self.triggerCameraFlash()
let params = (try? Self.decodeParams(ClawdisCameraSnapParams.self, from: req.paramsJSON)) ??
ClawdisCameraSnapParams()
let res = try await self.camera.snap(params: params)
@@ -508,7 +330,6 @@ final class NodeAppModel {
base64: res.base64,
width: res.width,
height: res.height))
self.showCameraHUD(text: "Photo captured", kind: .success, autoHideSeconds: 1.6)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisCameraCommand.clip.rawValue:
@@ -518,7 +339,6 @@ final class NodeAppModel {
let suspended = (params.includeAudio ?? true) ? self.voiceWake.suspendForExternalAudioCapture() : false
defer { self.voiceWake.resumeAfterExternalAudioCapture(wasSuspended: suspended) }
self.showCameraHUD(text: "Recording…", kind: .recording)
let res = try await self.camera.clip(params: params)
struct Payload: Codable {
@@ -532,40 +352,6 @@ final class NodeAppModel {
base64: res.base64,
durationMs: res.durationMs,
hasAudio: res.hasAudio))
self.showCameraHUD(text: "Clip captured", kind: .success, autoHideSeconds: 1.8)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisScreenCommand.record.rawValue:
let params = (try? Self.decodeParams(ClawdisScreenRecordParams.self, from: req.paramsJSON)) ??
ClawdisScreenRecordParams()
if let format = params.format, format.lowercased() != "mp4" {
throw NSError(domain: "Screen", code: 30, userInfo: [
NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4",
])
}
let path = try await self.screenRecorder.record(
screenIndex: params.screenIndex,
durationMs: params.durationMs,
fps: params.fps,
includeAudio: params.includeAudio,
outPath: nil)
defer { try? FileManager.default.removeItem(atPath: path) }
let data = try Data(contentsOf: URL(fileURLWithPath: path))
struct Payload: Codable {
var format: String
var base64: String
var durationMs: Int?
var fps: Double?
var screenIndex: Int?
var hasAudio: Bool
}
let payload = try Self.encodePayload(Payload(
format: "mp4",
base64: data.base64EncodedString(),
durationMs: params.durationMs,
fps: params.fps,
screenIndex: params.screenIndex,
hasAudio: params.includeAudio ?? true))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
default:
@@ -575,10 +361,6 @@ final class NodeAppModel {
error: ClawdisNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
} catch {
if command.hasPrefix("camera.") {
let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2)
}
return BridgeInvokeResponse(
id: req.id,
ok: false,
@@ -610,26 +392,4 @@ final class NodeAppModel {
if UserDefaults.standard.object(forKey: "camera.enabled") == nil { return true }
return UserDefaults.standard.bool(forKey: "camera.enabled")
}
private func triggerCameraFlash() {
self.cameraFlashNonce &+= 1
}
private func showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) {
self.cameraHUDDismissTask?.cancel()
withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) {
self.cameraHUDText = text
self.cameraHUDKind = kind
}
guard let autoHideSeconds else { return }
self.cameraHUDDismissTask = Task { @MainActor in
try? await Task.sleep(nanoseconds: UInt64(autoHideSeconds * 1_000_000_000))
withAnimation(.easeOut(duration: 0.25)) {
self.cameraHUDText = nil
self.cameraHUDKind = nil
}
}
}
}
+34 -195
View File
@@ -1,13 +1,9 @@
import SwiftUI
import UIKit
struct RootCanvas: View {
@Environment(NodeAppModel.self) private var appModel
@Environment(VoiceWakeManager.self) private var voiceWake
@Environment(\.colorScheme) private var systemColorScheme
@Environment(\.scenePhase) private var scenePhase
@AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true
@State private var presentedSheet: PresentedSheet?
@State private var voiceWakeToastText: String?
@State private var toastDismissTask: Task<Void, Never>?
@@ -25,24 +21,37 @@ struct RootCanvas: View {
}
var body: some View {
ZStack {
CanvasContent(
systemColorScheme: self.systemColorScheme,
bridgeStatus: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
voiceWakeToastText: self.voiceWakeToastText,
cameraHUDText: self.appModel.cameraHUDText,
cameraHUDKind: self.appModel.cameraHUDKind,
openChat: {
self.presentedSheet = .chat
},
openSettings: {
self.presentedSheet = .settings
})
.preferredColorScheme(.dark)
ZStack(alignment: .topTrailing) {
ScreenTab()
if self.appModel.cameraFlashNonce != 0 {
CameraFlashOverlay(nonce: self.appModel.cameraFlashNonce)
VStack(spacing: 10) {
OverlayButton(systemImage: "text.bubble.fill") {
self.presentedSheet = .chat
}
.accessibilityLabel("Chat")
OverlayButton(systemImage: "gearshape.fill") {
self.presentedSheet = .settings
}
.accessibilityLabel("Settings")
}
.padding(.top, 10)
.padding(.trailing, 10)
}
.overlay(alignment: .topLeading) {
StatusPill(
bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
onTap: { self.presentedSheet = .settings })
.padding(.leading, 10)
.safeAreaPadding(.top, 10)
}
.overlay(alignment: .topLeading) {
if let voiceWakeToastText, !voiceWakeToastText.isEmpty {
VoiceWakeToast(command: voiceWakeToastText)
.padding(.leading, 10)
.safeAreaPadding(.top, 58)
.transition(.move(edge: .top).combined(with: .opacity))
}
}
.sheet(item: self.$presentedSheet) { sheet in
@@ -53,9 +62,7 @@ struct RootCanvas: View {
ChatSheet(bridge: self.appModel.bridgeSession)
}
}
.onAppear { self.updateIdleTimer() }
.onChange(of: self.preventSleep) { _, _ in self.updateIdleTimer() }
.onChange(of: self.scenePhase) { _, _ in self.updateIdleTimer() }
.preferredColorScheme(.dark)
.onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in
guard let newValue else { return }
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -76,7 +83,6 @@ struct RootCanvas: View {
}
}
.onDisappear {
UIApplication.shared.isIdleTimerDisabled = false
self.toastDismissTask?.cancel()
self.toastDismissTask = nil
}
@@ -98,95 +104,10 @@ struct RootCanvas: View {
return .disconnected
}
private func updateIdleTimer() {
UIApplication.shared.isIdleTimerDisabled = (self.scenePhase == .active && self.preventSleep)
}
}
private struct CanvasContent: View {
var systemColorScheme: ColorScheme
var bridgeStatus: StatusPill.BridgeState
var voiceWakeEnabled: Bool
var voiceWakeToastText: String?
var cameraHUDText: String?
var cameraHUDKind: NodeAppModel.CameraHUDKind?
var openChat: () -> Void
var openSettings: () -> Void
private var brightenButtons: Bool { self.systemColorScheme == .light }
var body: some View {
ZStack(alignment: .topTrailing) {
ScreenTab()
VStack(spacing: 10) {
OverlayButton(systemImage: "text.bubble.fill", brighten: self.brightenButtons) {
self.openChat()
}
.accessibilityLabel("Chat")
OverlayButton(systemImage: "gearshape.fill", brighten: self.brightenButtons) {
self.openSettings()
}
.accessibilityLabel("Settings")
}
.padding(.top, 10)
.padding(.trailing, 10)
}
.overlay(alignment: .topLeading) {
StatusPill(
bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
brighten: self.brightenButtons,
onTap: {
self.openSettings()
})
.padding(.leading, 10)
.safeAreaPadding(.top, 10)
}
.overlay(alignment: .topLeading) {
if let voiceWakeToastText, !voiceWakeToastText.isEmpty {
VoiceWakeToast(
command: voiceWakeToastText,
brighten: self.brightenButtons)
.padding(.leading, 10)
.safeAreaPadding(.top, 58)
.transition(.move(edge: .top).combined(with: .opacity))
}
}
.overlay(alignment: .topLeading) {
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
CameraCaptureToast(
text: cameraHUDText,
kind: self.mapCameraKind(cameraHUDKind),
brighten: self.brightenButtons)
.padding(SwiftUI.Edge.Set.leading, 10)
.safeAreaPadding(SwiftUI.Edge.Set.top, 106)
.transition(
AnyTransition.move(edge: SwiftUI.Edge.top)
.combined(with: AnyTransition.opacity))
}
}
}
private func mapCameraKind(_ kind: NodeAppModel.CameraHUDKind) -> CameraCaptureToast.Kind {
switch kind {
case .photo:
.photo
case .recording:
.recording
case .success:
.success
case .error:
.error
}
}
}
private struct OverlayButton: View {
let systemImage: String
let brighten: Bool
let action: () -> Void
var body: some View {
@@ -203,8 +124,8 @@ private struct OverlayButton: View {
.fill(
LinearGradient(
colors: [
.white.opacity(self.brighten ? 0.26 : 0.18),
.white.opacity(self.brighten ? 0.08 : 0.04),
.white.opacity(0.18),
.white.opacity(0.04),
.clear,
],
startPoint: .topLeading,
@@ -213,7 +134,7 @@ private struct OverlayButton: View {
}
.overlay {
RoundedRectangle(cornerRadius: 12, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
.strokeBorder(.white.opacity(0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.35), radius: 12, y: 6)
}
@@ -221,85 +142,3 @@ private struct OverlayButton: View {
.buttonStyle(.plain)
}
}
private struct CameraFlashOverlay: View {
var nonce: Int
@State private var opacity: CGFloat = 0
@State private var task: Task<Void, Never>?
var body: some View {
Color.white
.opacity(self.opacity)
.ignoresSafeArea()
.allowsHitTesting(false)
.onChange(of: self.nonce) { _, _ in
self.task?.cancel()
self.task = Task { @MainActor in
withAnimation(.easeOut(duration: 0.08)) {
self.opacity = 0.85
}
try? await Task.sleep(nanoseconds: 110_000_000)
withAnimation(.easeOut(duration: 0.32)) {
self.opacity = 0
}
}
}
}
}
private struct CameraCaptureToast: View {
enum Kind {
case photo
case recording
case success
case error
}
var text: String
var kind: Kind
var brighten: Bool = false
var body: some View {
HStack(spacing: 10) {
self.icon
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
Text(self.text)
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
.lineLimit(1)
.truncationMode(.tail)
}
.padding(.vertical, 10)
.padding(.horizontal, 12)
.background {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.fill(.ultraThinMaterial)
.overlay {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
}
.accessibilityLabel("Camera")
.accessibilityValue(self.text)
}
@ViewBuilder
private var icon: some View {
switch self.kind {
case .photo:
Image(systemName: "camera.fill")
case .recording:
Image(systemName: "record.circle.fill")
.symbolRenderingMode(.palette)
.foregroundStyle(.red, .primary)
case .success:
Image(systemName: "checkmark.circle.fill")
case .error:
Image(systemName: "exclamationmark.triangle.fill")
}
}
}
+158 -241
View File
@@ -8,31 +8,22 @@ import WebKit
final class ScreenController {
let webView: WKWebView
private let navigationDelegate: ScreenNavigationDelegate
private let a2uiActionHandler: CanvasA2UIActionMessageHandler
var mode: ClawdisScreenMode = .canvas
var urlString: String = ""
var errorText: String?
/// Callback invoked when a clawdis:// deep link is tapped in the canvas
var onDeepLink: ((URL) -> Void)?
/// Callback invoked when the user clicks an A2UI action (e.g. button) inside the canvas web UI.
var onA2UIAction: (([String: Any]) -> Void)?
init() {
let config = WKWebViewConfiguration()
config.websiteDataStore = .nonPersistent()
let a2uiActionHandler = CanvasA2UIActionMessageHandler()
let userContentController = WKUserContentController()
userContentController.add(a2uiActionHandler, name: CanvasA2UIActionMessageHandler.messageName)
config.userContentController = userContentController
self.navigationDelegate = ScreenNavigationDelegate()
self.a2uiActionHandler = a2uiActionHandler
self.webView = WKWebView(frame: .zero, configuration: config)
// Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays.
self.webView.isOpaque = true
self.webView.backgroundColor = .black
self.webView.scrollView.backgroundColor = .black
self.webView.isOpaque = false
self.webView.backgroundColor = .clear
self.webView.scrollView.backgroundColor = .clear
self.webView.scrollView.contentInsetAdjustmentBehavior = .never
self.webView.scrollView.contentInset = .zero
self.webView.scrollView.scrollIndicatorInsets = .zero
@@ -42,63 +33,29 @@ final class ScreenController {
self.webView.scrollView.bounces = false
self.webView.navigationDelegate = self.navigationDelegate
self.navigationDelegate.controller = self
a2uiActionHandler.controller = self
self.reload()
}
func setMode(_ mode: ClawdisScreenMode) {
self.mode = mode
self.reload()
}
func navigate(to urlString: String) {
let trimmed = urlString.trimmingCharacters(in: .whitespacesAndNewlines)
self.urlString = (trimmed == "/" ? "" : trimmed)
self.urlString = urlString
self.reload()
}
func reload() {
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
guard let url = Self.canvasScaffoldURL else { return }
self.errorText = nil
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
return
} else {
guard let url = URL(string: trimmed) else {
self.errorText = "Invalid URL: \(trimmed)"
return
}
self.errorText = nil
if url.isFileURL {
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
} else {
self.webView.load(URLRequest(url: url))
}
switch self.mode {
case .web:
guard let url = URL(string: self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)) else { return }
self.webView.load(URLRequest(url: url))
case .canvas:
self.webView.loadHTMLString(Self.canvasScaffoldHTML, baseURL: nil)
}
}
func showDefaultCanvas() {
self.urlString = ""
self.reload()
}
func waitForA2UIReady(timeoutMs: Int) async -> Bool {
let clock = ContinuousClock()
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))
while clock.now < deadline {
do {
let res = try await self.eval(javaScript: """
(() => {
try {
return !!globalThis.clawdisA2UI && typeof globalThis.clawdisA2UI.applyMessages === 'function';
} catch (_) { return false; }
})()
""")
if res == "true" { return true }
} catch {
// ignore; page likely still loading
}
try? await Task.sleep(nanoseconds: 120_000_000)
}
return false
}
func eval(javaScript: String) async throws -> String {
try await withCheckedThrowingContinuation { cont in
self.webView.evaluateJavaScript(javaScript) { result, error in
@@ -143,152 +100,151 @@ final class ScreenController {
return data.base64EncodedString()
}
func snapshotBase64(
maxWidth: CGFloat? = nil,
format: ClawdisCanvasSnapshotFormat,
quality: Double? = nil) async throws -> String
{
let config = WKSnapshotConfiguration()
if let maxWidth {
config.snapshotWidth = NSNumber(value: Double(maxWidth))
}
let image: UIImage = try await withCheckedThrowingContinuation { cont in
self.webView.takeSnapshot(with: config) { image, error in
if let error {
cont.resume(throwing: error)
return
}
guard let image else {
cont.resume(throwing: NSError(domain: "Screen", code: 2, userInfo: [
NSLocalizedDescriptionKey: "snapshot failed",
]))
return
}
cont.resume(returning: image)
private static let canvasScaffoldHTML = """
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
<title>Canvas</title>
<style>
:root { color-scheme: dark; }
@media (prefers-reduced-motion: reduce) {
body::before, body::after { animation: none !important; }
}
html,body { height:100%; margin:0; }
body {
background: radial-gradient(1200px 900px at 15% 20%, rgba(42, 113, 255, 0.18), rgba(0,0,0,0) 55%),
radial-gradient(900px 700px at 85% 30%, rgba(255, 0, 138, 0.14), rgba(0,0,0,0) 60%),
radial-gradient(1000px 900px at 60% 90%, rgba(0, 209, 255, 0.10), rgba(0,0,0,0) 60%),
#000;
overflow: hidden;
}
body::before {
content:"";
position: fixed;
inset: -20%;
background:
repeating-linear-gradient(0deg, rgba(255,255,255,0.02) 0, rgba(255,255,255,0.02) 1px,
transparent 1px, transparent 48px),
repeating-linear-gradient(90deg, rgba(255,255,255,0.02) 0, rgba(255,255,255,0.02) 1px,
transparent 1px, transparent 48px);
transform: rotate(-7deg);
opacity: 0.55;
pointer-events: none;
animation: clawdis-grid-drift 22s linear infinite;
}
body::after {
content:"";
position: fixed;
inset: -35%;
background:
radial-gradient(900px 700px at 30% 30%, rgba(42,113,255,0.16), rgba(0,0,0,0) 60%),
radial-gradient(800px 650px at 70% 35%, rgba(255,0,138,0.12), rgba(0,0,0,0) 62%),
radial-gradient(900px 800px at 55% 75%, rgba(0,209,255,0.10), rgba(0,0,0,0) 62%);
filter: blur(28px);
opacity: 0.55;
mix-blend-mode: screen;
pointer-events: none;
animation: clawdis-glow-drift 18s ease-in-out infinite alternate;
}
@keyframes clawdis-grid-drift {
0% { transform: translate3d(-18px, 12px, 0) rotate(-7deg); opacity: 0.50; }
50% { transform: translate3d( 14px,-10px, 0) rotate(-6.2deg); opacity: 0.62; }
100% { transform: translate3d(-10px, 8px, 0) rotate(-7.4deg); opacity: 0.52; }
}
@keyframes clawdis-glow-drift {
0% { transform: translate3d(-26px, 18px, 0) scale(1.02); opacity: 0.42; }
50% { transform: translate3d( 20px,-14px, 0) scale(1.05); opacity: 0.55; }
100% { transform: translate3d(-12px, 10px, 0) scale(1.03); opacity: 0.46; }
}
canvas {
display:block;
width:100vw;
height:100vh;
touch-action: none;
}
#clawdis-status {
position: fixed;
inset: 0;
display: grid;
place-items: center;
pointer-events: none;
}
#clawdis-status .card {
text-align: center;
padding: 16px 18px;
border-radius: 14px;
background: rgba(18, 18, 22, 0.42);
border: 1px solid rgba(255,255,255,0.08);
box-shadow: 0 18px 60px rgba(0,0,0,0.55);
backdrop-filter: blur(14px);
}
#clawdis-status .title {
font: 600 20px -apple-system, BlinkMacSystemFont, "SF Pro Display", "SF Pro Text", system-ui, sans-serif;
letter-spacing: 0.2px;
color: rgba(255,255,255,0.92);
text-shadow: 0 0 22px rgba(42, 113, 255, 0.35);
}
#clawdis-status .subtitle {
margin-top: 6px;
font: 500 12px -apple-system, BlinkMacSystemFont, "SF Pro Text", system-ui, sans-serif;
color: rgba(255,255,255,0.58);
}
</style>
</head>
<body>
<canvas id="clawdis-canvas"></canvas>
<div id="clawdis-status">
<div class="card">
<div class="title" id="clawdis-status-title">Ready</div>
<div class="subtitle" id="clawdis-status-subtitle">Waiting for agent</div>
</div>
</div>
<script>
(() => {
const canvas = document.getElementById('clawdis-canvas');
const ctx = canvas.getContext('2d');
const statusEl = document.getElementById('clawdis-status');
const titleEl = document.getElementById('clawdis-status-title');
const subtitleEl = document.getElementById('clawdis-status-subtitle');
function resize() {
const dpr = window.devicePixelRatio || 1;
const w = Math.max(1, Math.floor(window.innerWidth * dpr));
const h = Math.max(1, Math.floor(window.innerHeight * dpr));
canvas.width = w;
canvas.height = h;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
}
}
let data: Data?
switch format {
case .png:
data = image.pngData()
case .jpeg:
let q = (quality ?? 0.82).clamped(to: 0.1...1.0)
data = image.jpegData(compressionQuality: q)
}
guard let data else {
throw NSError(domain: "Screen", code: 1, userInfo: [
NSLocalizedDescriptionKey: "snapshot encode failed",
])
}
return data.base64EncodedString()
}
window.addEventListener('resize', resize);
resize();
private static func bundledResourceURL(
name: String,
ext: String,
subdirectory: String)
-> URL?
{
let bundle = ClawdisKitResources.bundle
return bundle.url(forResource: name, withExtension: ext, subdirectory: subdirectory)
?? bundle.url(forResource: name, withExtension: ext)
}
private static let canvasScaffoldURL: URL? = ScreenController.bundledResourceURL(
name: "scaffold",
ext: "html",
subdirectory: "CanvasScaffold")
private static let a2uiIndexURL: URL? = ScreenController.bundledResourceURL(
name: "index",
ext: "html",
subdirectory: "CanvasA2UI")
func isTrustedCanvasUIURL(_ url: URL) -> Bool {
guard url.isFileURL else { return false }
let std = url.standardizedFileURL
if let expected = Self.canvasScaffoldURL,
std == expected.standardizedFileURL
{
return true
}
if let expected = Self.a2uiIndexURL,
std == expected.standardizedFileURL
{
return true
}
return false
}
func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return false
}
guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else {
return false
}
if host == "localhost" { return true }
if host.hasSuffix(".local") { return true }
if host.hasSuffix(".ts.net") { return true }
if host.hasSuffix(".tailscale.net") { return true }
// Allow MagicDNS / LAN hostnames like "peters-mac-studio-1".
if !host.contains("."), !host.contains(":") { return true }
if let ipv4 = Self.parseIPv4(host) {
return Self.isLocalNetworkIPv4(ipv4)
}
return false
}
private static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? {
let parts = host.split(separator: ".", omittingEmptySubsequences: false)
guard parts.count == 4 else { return nil }
let bytes: [UInt8] = parts.compactMap { UInt8($0) }
guard bytes.count == 4 else { return nil }
return (bytes[0], bytes[1], bytes[2], bytes[3])
}
private static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool {
let (a, b, _, _) = ip
// 10.0.0.0/8
if a == 10 { return true }
// 172.16.0.0/12
if a == 172, (16...31).contains(Int(b)) { return true }
// 192.168.0.0/16
if a == 192, b == 168 { return true }
// 127.0.0.0/8
if a == 127 { return true }
// 169.254.0.0/16 (link-local)
if a == 169, b == 254 { return true }
// Tailscale: 100.64.0.0/10
if a == 100, (64...127).contains(Int(b)) { return true }
return false
}
nonisolated static func parseA2UIActionBody(_ body: Any) -> [String: Any]? {
if let dict = body as? [String: Any] { return dict.isEmpty ? nil : dict }
if let str = body as? String,
let data = str.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
{
return json.isEmpty ? nil : json
}
if let dict = body as? [AnyHashable: Any] {
let mapped = dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
return mapped.isEmpty ? nil : mapped
}
return nil
}
}
extension Double {
fileprivate func clamped(to range: ClosedRange<Double>) -> Double {
if self < range.lowerBound { return range.lowerBound }
if self > range.upperBound { return range.upperBound }
return self
}
window.__clawdis = {
canvas,
ctx,
setStatus: (title, subtitle) => {
if (!statusEl) return;
if (!title && !subtitle) {
statusEl.style.display = 'none';
return;
}
statusEl.style.display = 'grid';
if (titleEl && typeof title === 'string') titleEl.textContent = title;
if (subtitleEl && typeof subtitle === 'string') subtitleEl.textContent = subtitle;
// Auto-hide after 3 seconds
clearTimeout(window.__statusTimeout);
window.__statusTimeout = setTimeout(() => {
statusEl.style.display = 'none';
}, 3000);
}
};
})();
</script>
</body>
</html>
"""
}
// MARK: - Navigation Delegate
@@ -318,43 +274,4 @@ private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
decisionHandler(.allow)
}
func webView(
_: WKWebView,
didFailProvisionalNavigation _: WKNavigation?,
withError error: any Error)
{
Task { @MainActor in
self.controller?.errorText = error.localizedDescription
}
}
func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) {
Task { @MainActor in
self.controller?.errorText = error.localizedDescription
}
}
}
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "clawdisCanvasA2UIAction"
weak var controller: ScreenController?
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
guard message.name == Self.messageName else { return }
guard let controller else { return }
guard let url = message.webView?.url else { return }
if url.isFileURL {
guard controller.isTrustedCanvasUIURL(url) else { return }
} else {
// For security, only accept actions from local-network pages (e.g. the canvas host).
guard controller.isLocalNetworkCanvasURL(url) else { return }
}
guard let body = ScreenController.parseA2UIActionBody(message.body) else { return }
controller.onA2UIAction?(body)
}
}
@@ -1,193 +0,0 @@
import AVFoundation
import ReplayKit
@MainActor
final class ScreenRecordService {
enum ScreenRecordError: LocalizedError {
case invalidScreenIndex(Int)
case captureFailed(String)
case writeFailed(String)
var errorDescription: String? {
switch self {
case let .invalidScreenIndex(idx):
"Invalid screen index \(idx)"
case let .captureFailed(msg):
msg
case let .writeFailed(msg):
msg
}
}
}
func record(
screenIndex: Int?,
durationMs: Int?,
fps: Double?,
includeAudio: Bool?,
outPath: String?) async throws -> String
{
let durationMs = Self.clampDurationMs(durationMs)
let fps = Self.clampFps(fps)
let fpsInt = Int32(fps.rounded())
let fpsValue = Double(fpsInt)
let includeAudio = includeAudio ?? true
if let idx = screenIndex, idx != 0 {
throw ScreenRecordError.invalidScreenIndex(idx)
}
let outURL: URL = {
if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return URL(fileURLWithPath: outPath)
}
return FileManager.default.temporaryDirectory
.appendingPathComponent("clawdis-screen-record-\(UUID().uuidString).mp4")
}()
try? FileManager.default.removeItem(at: outURL)
let recorder = RPScreenRecorder.shared()
recorder.isMicrophoneEnabled = includeAudio
var writer: AVAssetWriter?
var videoInput: AVAssetWriterInput?
var audioInput: AVAssetWriterInput?
var started = false
var sawVideo = false
var lastVideoTime: CMTime?
var handlerError: Error?
let lock = NSLock()
func setHandlerError(_ error: Error) {
lock.lock()
defer { lock.unlock() }
if handlerError == nil { handlerError = error }
}
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
recorder.startCapture(handler: { sample, type, error in
if let error {
setHandlerError(error)
return
}
guard CMSampleBufferDataIsReady(sample) else { return }
switch type {
case .video:
let pts = CMSampleBufferGetPresentationTimeStamp(sample)
if let lastVideoTime {
let delta = CMTimeSubtract(pts, lastVideoTime)
if delta.seconds < (1.0 / fpsValue) { return }
}
if writer == nil {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
setHandlerError(ScreenRecordError.captureFailed("Missing image buffer"))
return
}
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
do {
let w = try AVAssetWriter(outputURL: outURL, fileType: .mp4)
let settings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: width,
AVVideoHeightKey: height,
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
vInput.expectsMediaDataInRealTime = true
guard w.canAdd(vInput) else {
throw ScreenRecordError.writeFailed("Cannot add video input")
}
w.add(vInput)
if includeAudio {
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
aInput.expectsMediaDataInRealTime = true
if w.canAdd(aInput) {
w.add(aInput)
audioInput = aInput
}
}
guard w.startWriting() else {
throw ScreenRecordError
.writeFailed(w.error?.localizedDescription ?? "Failed to start writer")
}
w.startSession(atSourceTime: pts)
writer = w
videoInput = vInput
started = true
} catch {
setHandlerError(error)
return
}
}
guard let vInput = videoInput, started else { return }
if vInput.isReadyForMoreMediaData {
if vInput.append(sample) {
sawVideo = true
lastVideoTime = pts
} else {
if let err = writer?.error {
setHandlerError(ScreenRecordError.writeFailed(err.localizedDescription))
}
}
}
case .audioApp, .audioMic:
guard includeAudio, let aInput = audioInput, started else { return }
if aInput.isReadyForMoreMediaData {
_ = aInput.append(sample)
}
@unknown default:
break
}
}, completionHandler: { error in
if let error { cont.resume(throwing: error) } else { cont.resume() }
})
}
try await Task.sleep(nanoseconds: UInt64(durationMs) * 1_000_000)
let stopError = await withCheckedContinuation { cont in
recorder.stopCapture { error in cont.resume(returning: error) }
}
if let stopError { throw stopError }
if let handlerError { throw handlerError }
guard let writer, let videoInput, sawVideo else {
throw ScreenRecordError.captureFailed("No frames captured")
}
videoInput.markAsFinished()
audioInput?.markAsFinished()
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
writer.finishWriting {
if let err = writer.error {
cont.resume(throwing: ScreenRecordError.writeFailed(err.localizedDescription))
} else if writer.status != .completed {
cont.resume(throwing: ScreenRecordError.writeFailed("Failed to finalize video"))
} else {
cont.resume()
}
}
}
return outURL.path
}
private nonisolated static func clampDurationMs(_ ms: Int?) -> Int {
let v = ms ?? 10000
return min(60000, max(250, v))
}
private nonisolated static func clampFps(_ fps: Double?) -> Double {
let v = fps ?? 10
if !v.isFinite { return 10 }
return min(30, max(1, v))
}
}
+1 -1
View File
@@ -21,5 +21,5 @@ struct ScreenTab: View {
}
}
// Navigation is agent-driven; no local URL bar here.
// Navigation/mode selection is agent-driven; no local controls here.
}
+26 -107
View File
@@ -21,7 +21,6 @@ struct SettingsTab: View {
@AppStorage("node.instanceId") private var instanceId: String = UUID().uuidString
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
@AppStorage("camera.enabled") private var cameraEnabled: Bool = true
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true
@AppStorage("bridge.preferredStableID") private var preferredBridgeStableID: String = ""
@AppStorage("bridge.lastDiscoveredStableID") private var lastDiscoveredBridgeStableID: String = ""
@AppStorage("bridge.manual.enabled") private var manualBridgeEnabled: Bool = false
@@ -52,6 +51,28 @@ struct SettingsTab: View {
}
}
Section("Voice") {
Toggle("Voice Wake", isOn: self.$voiceWakeEnabled)
.onChange(of: self.voiceWakeEnabled) { _, newValue in
self.appModel.setVoiceWakeEnabled(newValue)
}
NavigationLink {
VoiceWakeWordsSettingsView()
} label: {
LabeledContent(
"Wake Words",
value: VoiceWakePreferences.displayString(for: self.voiceWake.triggerWords))
}
}
Section("Camera") {
Toggle("Allow Camera", isOn: self.$cameraEnabled)
Text("Allows the bridge to request photos or short video clips (foreground only).")
.font(.footnote)
.foregroundStyle(.secondary)
}
Section("Bridge") {
LabeledContent("Discovery", value: self.bridgeController.discoveryStatusText)
LabeledContent("Status", value: self.appModel.bridgeStatusText)
@@ -144,35 +165,6 @@ struct SettingsTab: View {
}
}
}
Section("Voice") {
Toggle("Voice Wake", isOn: self.$voiceWakeEnabled)
.onChange(of: self.voiceWakeEnabled) { _, newValue in
self.appModel.setVoiceWakeEnabled(newValue)
}
NavigationLink {
VoiceWakeWordsSettingsView()
} label: {
LabeledContent(
"Wake Words",
value: VoiceWakePreferences.displayString(for: self.voiceWake.triggerWords))
}
}
Section("Camera") {
Toggle("Allow Camera", isOn: self.$cameraEnabled)
Text("Allows the bridge to request photos or short video clips (foreground only).")
.font(.footnote)
.foregroundStyle(.secondary)
}
Section("Screen") {
Toggle("Prevent Sleep", isOn: self.$preventSleep)
Text("Keeps the screen awake while Clawdis is open.")
.font(.footnote)
.foregroundStyle(.secondary)
}
}
.navigationTitle("Settings")
.toolbar {
@@ -262,63 +254,6 @@ struct SettingsTab: View {
Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"
}
private func deviceFamily() -> String {
switch UIDevice.current.userInterfaceIdiom {
case .pad:
"iPad"
case .phone:
"iPhone"
default:
"iOS"
}
}
private func modelIdentifier() -> String {
var systemInfo = utsname()
uname(&systemInfo)
let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in
String(decoding: ptr.prefix { $0 != 0 }, as: UTF8.self)
}
return machine.isEmpty ? "unknown" : machine
}
private func currentCaps() -> [String] {
var caps = [ClawdisCapability.canvas.rawValue, ClawdisCapability.screen.rawValue]
let cameraEnabled =
UserDefaults.standard.object(forKey: "camera.enabled") == nil
? true
: UserDefaults.standard.bool(forKey: "camera.enabled")
if cameraEnabled { caps.append(ClawdisCapability.camera.rawValue) }
let voiceWakeEnabled = UserDefaults.standard.bool(forKey: VoiceWakePreferences.enabledKey)
if voiceWakeEnabled { caps.append(ClawdisCapability.voiceWake.rawValue) }
return caps
}
private func currentCommands() -> [String] {
var commands: [String] = [
ClawdisCanvasCommand.present.rawValue,
ClawdisCanvasCommand.hide.rawValue,
ClawdisCanvasCommand.navigate.rawValue,
ClawdisCanvasCommand.evalJS.rawValue,
ClawdisCanvasCommand.snapshot.rawValue,
ClawdisCanvasA2UICommand.push.rawValue,
ClawdisCanvasA2UICommand.pushJSONL.rawValue,
ClawdisCanvasA2UICommand.reset.rawValue,
ClawdisScreenCommand.record.rawValue,
]
let caps = Set(self.currentCaps())
if caps.contains(ClawdisCapability.camera.rawValue) {
commands.append(ClawdisCameraCommand.snap.rawValue)
commands.append(ClawdisCameraCommand.clip.rawValue)
}
return commands
}
private func connect(_ bridge: BridgeDiscoveryModel.DiscoveredBridge) async {
self.connectingBridgeID = bridge.id
self.manualBridgeEnabled = false
@@ -342,11 +277,7 @@ struct SettingsTab: View {
displayName: self.displayName,
token: existingToken,
platform: self.platformString(),
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands())
version: self.appVersion())
let token = try await BridgeClient().pairAndHello(
endpoint: bridge.endpoint,
hello: hello,
@@ -370,11 +301,7 @@ struct SettingsTab: View {
displayName: self.displayName,
token: token,
platform: self.platformString(),
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands()))
version: self.appVersion()))
} catch {
self.connectStatus.text = "Failed: \(error.localizedDescription)"
@@ -416,11 +343,7 @@ struct SettingsTab: View {
displayName: self.displayName,
token: existingToken,
platform: self.platformString(),
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands())
version: self.appVersion())
let token = try await BridgeClient().pairAndHello(
endpoint: endpoint,
hello: hello,
@@ -444,11 +367,7 @@ struct SettingsTab: View {
displayName: self.displayName,
token: token,
platform: self.platformString(),
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands()))
version: self.appVersion()))
} catch {
self.connectStatus.text = "Failed: \(error.localizedDescription)"
+1 -2
View File
@@ -28,7 +28,6 @@ struct StatusPill: View {
var bridge: BridgeState
var voiceWakeEnabled: Bool
var brighten: Bool = false
var onTap: () -> Void
@State private var pulse: Bool = false
@@ -64,7 +63,7 @@ struct StatusPill: View {
.fill(.ultraThinMaterial)
.overlay {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
.strokeBorder(.white.opacity(0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
}
+1 -2
View File
@@ -2,7 +2,6 @@ import SwiftUI
struct VoiceWakeToast: View {
var command: String
var brighten: Bool = false
var body: some View {
HStack(spacing: 10) {
@@ -23,7 +22,7 @@ struct VoiceWakeToast: View {
.fill(.ultraThinMaterial)
.overlay {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
.strokeBorder(.white.opacity(0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
}
+2 -2
View File
@@ -6,12 +6,12 @@ import Testing
@Suite struct BridgeEndpointIDTests {
@Test func stableIDForServiceDecodesAndNormalizesName() {
let endpoint = NWEndpoint.service(
name: "Clawdis\\032Bridge \\032 Node\n",
name: "Clawdis\\032Bridge \\032 Iris\n",
type: "_clawdis-bridge._tcp",
domain: "local.",
interface: nil)
#expect(BridgeEndpointID.stableID(endpoint) == "_clawdis-bridge._tcp|local.|Clawdis Bridge Node")
#expect(BridgeEndpointID.stableID(endpoint) == "_clawdis-bridge._tcp|local.|Clawdis Bridge Iris")
}
@Test func stableIDForNonServiceUsesEndpointDescription() {
@@ -18,7 +18,7 @@ import Testing
#expect(CameraController.clampDurationMs(249) == 250)
#expect(CameraController.clampDurationMs(250) == 250)
#expect(CameraController.clampDurationMs(1000) == 1000)
#expect(CameraController.clampDurationMs(60000) == 60000)
#expect(CameraController.clampDurationMs(60001) == 60000)
#expect(CameraController.clampDurationMs(15000) == 15000)
#expect(CameraController.clampDurationMs(15001) == 15000)
}
}
+2 -2
View File
@@ -33,12 +33,12 @@ import Testing
@Test func parseAgentLinkParsesCommonFields() {
let url =
URL(string: "clawdis://agent?message=Hello&deliver=1&sessionKey=node-test&thinking=low&timeoutSeconds=30")!
URL(string: "clawdis://agent?message=Hello&deliver=1&sessionKey=node-iris&thinking=low&timeoutSeconds=30")!
#expect(
DeepLinkParser.parse(url) == .agent(
.init(
message: "Hello",
sessionKey: "node-test",
sessionKey: "node-iris",
thinking: "low",
deliver: true,
to: nil,
+8 -25
View File
@@ -6,21 +6,23 @@ import WebKit
@Test @MainActor func canvasModeConfiguresWebViewForTouch() {
let screen = ScreenController()
#expect(screen.webView.isOpaque == true)
#expect(screen.webView.backgroundColor == .black)
#expect(screen.mode == .canvas)
#expect(screen.webView.isOpaque == false)
#expect(screen.webView.backgroundColor == .clear)
let scrollView = screen.webView.scrollView
#expect(scrollView.backgroundColor == .black)
#expect(scrollView.backgroundColor == .clear)
#expect(scrollView.contentInsetAdjustmentBehavior == .never)
#expect(scrollView.isScrollEnabled == false)
#expect(scrollView.bounces == false)
}
@Test @MainActor func navigateSlashShowsDefaultCanvas() {
@Test @MainActor func webModeRejectsInvalidURLStrings() {
let screen = ScreenController()
screen.navigate(to: "/")
screen.navigate(to: "about:blank")
screen.setMode(.web)
#expect(screen.urlString.isEmpty)
#expect(screen.mode == .web)
}
@Test @MainActor func evalExecutesJavaScript() async throws {
@@ -40,23 +42,4 @@ import WebKit
}
}
}
@Test @MainActor func localNetworkCanvasURLsAreAllowed() {
let screen = ScreenController()
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://localhost:18793/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://clawd.local:18793/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://peters-mac-studio-1:18793/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "https://peters-mac-studio-1.ts.net:18793/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://192.168.0.10:18793/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://10.0.0.10:18793/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://100.123.224.76:18793/")!) == true) // Tailscale CGNAT
#expect(screen.isLocalNetworkCanvasURL(URL(string: "https://example.com/")!) == false)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://8.8.8.8/")!) == false)
}
@Test func parseA2UIActionBodyAcceptsJSONString() throws {
let body = ScreenController.parseA2UIActionBody("{\"userAction\":{\"name\":\"hello\"}}")
let userAction = try #require(body?["userAction"] as? [String: Any])
#expect(userAction["name"] as? String == "hello")
}
}
@@ -13,16 +13,6 @@ import UIKit
return window
}
@Test @MainActor func statusPillConnectingBuildsAViewHierarchy() {
let root = StatusPill(bridge: .connecting, voiceWakeEnabled: true, brighten: true) {}
_ = Self.host(root)
}
@Test @MainActor func statusPillDisconnectedBuildsAViewHierarchy() {
let root = StatusPill(bridge: .disconnected, voiceWakeEnabled: false) {}
_ = Self.host(root)
}
@Test @MainActor func settingsTabBuildsAViewHierarchy() {
let appModel = NodeAppModel()
let bridgeController = BridgeConnectionController(appModel: appModel, startDiscovery: false)
+2 -1
View File
@@ -50,7 +50,8 @@ let package = Package(
],
resources: [
.copy("Resources/Clawdis.icns"),
.copy("Resources/DeviceModels"),
.copy("Resources/WebChat"),
.copy("Resources/CanvasA2UI"),
],
swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"),
@@ -1,198 +0,0 @@
import AppKit
import Combine
import SwiftUI
@MainActor
struct AnthropicAuthControls: View {
let connectionMode: AppState.ConnectionMode
@State private var oauthStatus: PiOAuthStore.AnthropicOAuthStatus = PiOAuthStore.anthropicOAuthStatus()
@State private var pkce: AnthropicOAuth.PKCE?
@State private var code: String = ""
@State private var busy = false
@State private var statusText: String?
@State private var autoDetectClipboard = true
@State private var autoConnectClipboard = true
@State private var lastPasteboardChangeCount = NSPasteboard.general.changeCount
private static let clipboardPoll = Timer.publish(every: 0.4, on: .main, in: .common).autoconnect()
var body: some View {
VStack(alignment: .leading, spacing: 10) {
if self.connectionMode == .remote {
Text("Gateway runs remotely; OAuth must be created on the gateway host where Pi runs.")
.font(.footnote)
.foregroundStyle(.secondary)
.fixedSize(horizontal: false, vertical: true)
}
HStack(spacing: 10) {
Circle()
.fill(self.oauthStatus.isConnected ? Color.green : Color.orange)
.frame(width: 8, height: 8)
Text(self.oauthStatus.shortDescription)
.font(.footnote.weight(.semibold))
.foregroundStyle(.secondary)
Spacer()
Button("Reveal") {
NSWorkspace.shared.activateFileViewerSelecting([PiOAuthStore.oauthURL()])
}
.buttonStyle(.bordered)
.disabled(!FileManager.default.fileExists(atPath: PiOAuthStore.oauthURL().path))
Button("Refresh") {
self.refresh()
}
.buttonStyle(.bordered)
}
Text(PiOAuthStore.oauthURL().path)
.font(.caption.monospaced())
.foregroundStyle(.secondary)
.lineLimit(1)
.truncationMode(.middle)
.textSelection(.enabled)
HStack(spacing: 12) {
Button {
self.startOAuth()
} label: {
if self.busy {
ProgressView().controlSize(.small)
} else {
Text(self.oauthStatus.isConnected ? "Re-auth (OAuth)" : "Open sign-in (OAuth)")
}
}
.buttonStyle(.borderedProminent)
.disabled(self.connectionMode == .remote || self.busy)
if self.pkce != nil {
Button("Cancel") {
self.pkce = nil
self.code = ""
self.statusText = nil
}
.buttonStyle(.bordered)
.disabled(self.busy)
}
}
if self.pkce != nil {
VStack(alignment: .leading, spacing: 8) {
Text("Paste `code#state`")
.font(.footnote.weight(.semibold))
.foregroundStyle(.secondary)
TextField("code#state", text: self.$code)
.textFieldStyle(.roundedBorder)
.disabled(self.busy)
Toggle("Auto-detect from clipboard", isOn: self.$autoDetectClipboard)
.font(.footnote)
.foregroundStyle(.secondary)
.disabled(self.busy)
Toggle("Auto-connect when detected", isOn: self.$autoConnectClipboard)
.font(.footnote)
.foregroundStyle(.secondary)
.disabled(self.busy)
Button("Connect") {
Task { await self.finishOAuth() }
}
.buttonStyle(.bordered)
.disabled(self.busy || self.connectionMode == .remote || self.code
.trimmingCharacters(in: .whitespacesAndNewlines)
.isEmpty)
}
}
if let statusText, !statusText.isEmpty {
Text(statusText)
.font(.footnote)
.foregroundStyle(.secondary)
.fixedSize(horizontal: false, vertical: true)
}
}
.onAppear {
self.refresh()
}
.onReceive(Self.clipboardPoll) { _ in
self.pollClipboardIfNeeded()
}
}
private func refresh() {
self.oauthStatus = PiOAuthStore.anthropicOAuthStatus()
}
private func startOAuth() {
guard self.connectionMode == .local else { return }
guard !self.busy else { return }
self.busy = true
defer { self.busy = false }
do {
let pkce = try AnthropicOAuth.generatePKCE()
self.pkce = pkce
let url = AnthropicOAuth.buildAuthorizeURL(pkce: pkce)
NSWorkspace.shared.open(url)
self.statusText = "Browser opened. After approving, paste the `code#state` value here."
} catch {
self.statusText = "Failed to start OAuth: \(error.localizedDescription)"
}
}
@MainActor
private func finishOAuth() async {
guard self.connectionMode == .local else { return }
guard !self.busy else { return }
guard let pkce = self.pkce else { return }
self.busy = true
defer { self.busy = false }
guard let parsed = AnthropicOAuthCodeState.parse(from: self.code) else {
self.statusText = "OAuth failed: missing or invalid code/state."
return
}
do {
let creds = try await AnthropicOAuth.exchangeCode(
code: parsed.code,
state: parsed.state,
verifier: pkce.verifier)
try PiOAuthStore.saveAnthropicOAuth(creds)
self.refresh()
self.pkce = nil
self.code = ""
self.statusText = "Connected. Pi can now use Claude via OAuth."
} catch {
self.statusText = "OAuth failed: \(error.localizedDescription)"
}
}
private func pollClipboardIfNeeded() {
guard self.connectionMode == .local else { return }
guard self.pkce != nil else { return }
guard !self.busy else { return }
guard self.autoDetectClipboard else { return }
let pb = NSPasteboard.general
let changeCount = pb.changeCount
guard changeCount != self.lastPasteboardChangeCount else { return }
self.lastPasteboardChangeCount = changeCount
guard let raw = pb.string(forType: .string), !raw.isEmpty else { return }
guard let parsed = AnthropicOAuthCodeState.parse(from: raw) else { return }
guard let pkce = self.pkce, parsed.state == pkce.verifier else { return }
let next = "\(parsed.code)#\(parsed.state)"
if self.code != next {
self.code = next
self.statusText = "Detected `code#state` from clipboard."
}
guard self.autoConnectClipboard else { return }
Task { await self.finishOAuth() }
}
}
@@ -10,52 +10,6 @@ struct AnthropicOAuthCredentials: Codable {
let expires: Int64
}
enum AnthropicAuthMode: Equatable {
case oauthFile
case oauthEnv
case apiKeyEnv
case missing
var shortLabel: String {
switch self {
case .oauthFile: "OAuth (Pi token file)"
case .oauthEnv: "OAuth (env var)"
case .apiKeyEnv: "API key (env var)"
case .missing: "Missing credentials"
}
}
var isConfigured: Bool {
switch self {
case .missing: false
case .oauthFile, .oauthEnv, .apiKeyEnv: true
}
}
}
enum AnthropicAuthResolver {
static func resolve(
environment: [String: String] = ProcessInfo.processInfo.environment,
oauthStatus: PiOAuthStore.AnthropicOAuthStatus = PiOAuthStore.anthropicOAuthStatus()) -> AnthropicAuthMode
{
if oauthStatus.isConnected { return .oauthFile }
if let token = environment["ANTHROPIC_OAUTH_TOKEN"]?.trimmingCharacters(in: .whitespacesAndNewlines),
!token.isEmpty
{
return .oauthEnv
}
if let key = environment["ANTHROPIC_API_KEY"]?.trimmingCharacters(in: .whitespacesAndNewlines),
!key.isEmpty
{
return .apiKeyEnv
}
return .missing
}
}
enum AnthropicOAuth {
private static let logger = Logger(subsystem: "com.steipete.clawdis", category: "anthropic-oauth")
@@ -153,7 +107,6 @@ enum AnthropicOAuth {
enum PiOAuthStore {
static let oauthFilename = "oauth.json"
private static let providerKey = "anthropic"
private static let piAgentDirEnv = "PI_CODING_AGENT_DIR"
enum AnthropicOAuthStatus: Equatable {
case missingFile
@@ -170,26 +123,18 @@ enum PiOAuthStore {
var shortDescription: String {
switch self {
case .missingFile: "Pi OAuth token file not found"
case .unreadableFile: "Pi OAuth token file not readable"
case .invalidJSON: "Pi OAuth token file invalid"
case .missingProviderEntry: "No Anthropic entry in Pi OAuth token file"
case .missingTokens: "Anthropic entry missing tokens"
case .connected: "Pi OAuth credentials found"
case .missingFile: "oauth.json not found"
case .unreadableFile: "oauth.json not readable"
case .invalidJSON: "oauth.json invalid"
case .missingProviderEntry: "oauth.json has no anthropic entry"
case .missingTokens: "anthropic entry missing tokens"
case .connected: "OAuth credentials found"
}
}
}
static func oauthDir() -> URL {
if let override = ProcessInfo.processInfo.environment[self.piAgentDirEnv]?
.trimmingCharacters(in: .whitespacesAndNewlines),
!override.isEmpty
{
let expanded = NSString(string: override).expandingTildeInPath
return URL(fileURLWithPath: expanded, isDirectory: true)
}
return FileManager.default.homeDirectoryForCurrentUser
FileManager.default.homeDirectoryForCurrentUser
.appendingPathComponent(".pi", isDirectory: true)
.appendingPathComponent("agent", isDirectory: true)
}
@@ -1,59 +0,0 @@
import Foundation
enum AnthropicOAuthCodeState {
struct Parsed: Equatable {
let code: String
let state: String
}
/// Extracts a `code#state` payload from arbitrary text.
///
/// Supports:
/// - raw `code#state`
/// - OAuth callback URLs containing `code=` and `state=` query params
/// - surrounding text/backticks from instructions pages
static func extract(from raw: String) -> String? {
let text = raw.trimmingCharacters(in: .whitespacesAndNewlines)
.trimmingCharacters(in: CharacterSet(charactersIn: "`"))
if text.isEmpty { return nil }
if let fromURL = self.extractFromURL(text) { return fromURL }
if let fromToken = self.extractFromToken(text) { return fromToken }
return nil
}
static func parse(from raw: String) -> Parsed? {
guard let extracted = self.extract(from: raw) else { return nil }
let parts = extracted.split(separator: "#", maxSplits: 1).map(String.init)
let code = parts.first ?? ""
let state = parts.count > 1 ? parts[1] : ""
guard !code.isEmpty, !state.isEmpty else { return nil }
return Parsed(code: code, state: state)
}
private static func extractFromURL(_ text: String) -> String? {
// Users might copy the callback URL from the browser address bar.
guard let components = URLComponents(string: text),
let items = components.queryItems,
let code = items.first(where: { $0.name == "code" })?.value,
let state = items.first(where: { $0.name == "state" })?.value,
!code.isEmpty, !state.isEmpty
else { return nil }
return "\(code)#\(state)"
}
private static func extractFromToken(_ text: String) -> String? {
// Base64url-ish tokens; keep this fairly strict to avoid false positives.
let pattern = #"([A-Za-z0-9._~-]{8,})#([A-Za-z0-9._~-]{8,})"#
guard let re = try? NSRegularExpression(pattern: pattern) else { return nil }
let range = NSRange(text.startIndex..<text.endIndex, in: text)
guard let match = re.firstMatch(in: text, range: range),
match.numberOfRanges == 3,
let full = Range(match.range(at: 0), in: text)
else { return nil }
return String(text[full])
}
}
+42 -1
View File
@@ -142,6 +142,20 @@ final class AppState {
}
}
var webChatEnabled: Bool {
didSet { self.ifNotPreview { UserDefaults.standard.set(self.webChatEnabled, forKey: webChatEnabledKey) } }
}
var webChatSwiftUIEnabled: Bool {
didSet { self.ifNotPreview { UserDefaults.standard.set(
self.webChatSwiftUIEnabled,
forKey: webChatSwiftUIEnabledKey) } }
}
var webChatPort: Int {
didSet { self.ifNotPreview { UserDefaults.standard.set(self.webChatPort, forKey: webChatPortKey) } }
}
var canvasEnabled: Bool {
didSet { self.ifNotPreview { UserDefaults.standard.set(self.canvasEnabled, forKey: canvasEnabledKey) } }
}
@@ -229,6 +243,10 @@ final class AppState {
self.remoteTarget = UserDefaults.standard.string(forKey: remoteTargetKey) ?? ""
self.remoteIdentity = UserDefaults.standard.string(forKey: remoteIdentityKey) ?? ""
self.remoteProjectRoot = UserDefaults.standard.string(forKey: remoteProjectRootKey) ?? ""
self.webChatEnabled = UserDefaults.standard.object(forKey: webChatEnabledKey) as? Bool ?? true
self.webChatSwiftUIEnabled = UserDefaults.standard.object(forKey: webChatSwiftUIEnabledKey) as? Bool ?? false
let storedPort = UserDefaults.standard.integer(forKey: webChatPortKey)
self.webChatPort = storedPort > 0 ? storedPort : 18788
self.canvasEnabled = UserDefaults.standard.object(forKey: canvasEnabledKey) as? Bool ?? true
self.peekabooBridgeEnabled = UserDefaults.standard
.object(forKey: peekabooBridgeEnabledKey) as? Bool ?? true
@@ -314,7 +332,14 @@ final class AppState {
self.voiceWakeGlobalSyncTask?.cancel()
self.voiceWakeGlobalSyncTask = Task { [sanitized] in
try? await Task.sleep(nanoseconds: 650_000_000)
await GatewayConnection.shared.voiceWakeSetTriggers(sanitized)
do {
_ = try await GatewayConnection.shared.request(
method: "voicewake.set",
params: ["triggers": AnyCodable(sanitized)],
timeoutMs: 10000)
} catch {
// Best-effort only.
}
}
}
@@ -358,6 +383,9 @@ extension AppState {
state.iconOverride = .system
state.heartbeatsEnabled = true
state.connectionMode = .local
state.webChatEnabled = true
state.webChatSwiftUIEnabled = false
state.webChatPort = 18788
state.canvasEnabled = true
state.remoteTarget = "user@example.com"
state.remoteIdentity = "~/.ssh/id_ed25519"
@@ -378,6 +406,19 @@ enum AppStateStore {
}
}
static var webChatEnabled: Bool {
UserDefaults.standard.object(forKey: webChatEnabledKey) as? Bool ?? true
}
static var webChatSwiftUIEnabled: Bool {
UserDefaults.standard.object(forKey: webChatSwiftUIEnabledKey) as? Bool ?? false
}
static var webChatPort: Int {
let stored = UserDefaults.standard.integer(forKey: webChatPortKey)
return stored > 0 ? stored : 18788
}
static var canvasEnabled: Bool {
UserDefaults.standard.object(forKey: canvasEnabledKey) as? Bool ?? true
}
@@ -3,17 +3,6 @@ import Foundation
import Network
import OSLog
struct BridgeNodeInfo: Sendable {
var nodeId: String
var displayName: String?
var platform: String?
var version: String?
var deviceFamily: String?
var modelIdentifier: String?
var remoteAddress: String?
var caps: [String]?
}
actor BridgeConnectionHandler {
private let connection: NWConnection
private let logger: Logger
@@ -49,7 +38,7 @@ actor BridgeConnectionHandler {
var serverName: String
var resolveAuth: @Sendable (BridgeHello) async -> AuthResult
var handlePair: @Sendable (BridgePairRequest) async -> PairResult
var onAuthenticated: (@Sendable (BridgeNodeInfo) async -> Void)?
var onAuthenticated: (@Sendable (String) async -> Void)?
var onEvent: (@Sendable (String, BridgeEventFrame) async -> Void)?
var onRequest: (@Sendable (String, BridgeRPCRequest) async -> BridgeRPCResponse)?
}
@@ -57,7 +46,7 @@ actor BridgeConnectionHandler {
func run(
resolveAuth: @escaping @Sendable (BridgeHello) async -> AuthResult,
handlePair: @escaping @Sendable (BridgePairRequest) async -> PairResult,
onAuthenticated: (@Sendable (BridgeNodeInfo) async -> Void)? = nil,
onAuthenticated: (@Sendable (String) async -> Void)? = nil,
onDisconnected: (@Sendable (String) async -> Void)? = nil,
onEvent: (@Sendable (String, BridgeEventFrame) async -> Void)? = nil,
onRequest: (@Sendable (String, BridgeRPCRequest) async -> BridgeRPCResponse)? = nil) async
@@ -136,21 +125,11 @@ actor BridgeConnectionHandler {
{
do {
let hello = try self.decoder.decode(BridgeHello.self, from: data)
let nodeId = hello.nodeId.trimmingCharacters(in: .whitespacesAndNewlines)
self.nodeId = nodeId
self.nodeId = hello.nodeId
let result = await context.resolveAuth(hello)
await self.handleAuthResult(result, serverName: context.serverName)
if case .ok = result {
await context.onAuthenticated?(
BridgeNodeInfo(
nodeId: nodeId,
displayName: hello.displayName,
platform: hello.platform,
version: hello.version,
deviceFamily: hello.deviceFamily,
modelIdentifier: hello.modelIdentifier,
remoteAddress: self.remoteAddressString(),
caps: hello.caps))
if case .ok = result, let nodeId = self.nodeId {
await context.onAuthenticated?(nodeId)
}
} catch {
await self.sendError(code: "INVALID_REQUEST", message: error.localizedDescription)
@@ -163,33 +142,18 @@ actor BridgeConnectionHandler {
{
do {
let req = try self.decoder.decode(BridgePairRequest.self, from: data)
let nodeId = req.nodeId.trimmingCharacters(in: .whitespacesAndNewlines)
self.nodeId = nodeId
self.nodeId = req.nodeId
let enriched = BridgePairRequest(
type: req.type,
nodeId: nodeId,
nodeId: req.nodeId,
displayName: req.displayName,
platform: req.platform,
version: req.version,
deviceFamily: req.deviceFamily,
modelIdentifier: req.modelIdentifier,
caps: req.caps,
commands: req.commands,
remoteAddress: self.remoteAddressString(),
silent: req.silent)
remoteAddress: self.remoteAddressString())
let result = await context.handlePair(enriched)
await self.handlePairResult(result, serverName: context.serverName)
if case .ok = result {
await context.onAuthenticated?(
BridgeNodeInfo(
nodeId: nodeId,
displayName: enriched.displayName,
platform: enriched.platform,
version: enriched.version,
deviceFamily: enriched.deviceFamily,
modelIdentifier: enriched.modelIdentifier,
remoteAddress: enriched.remoteAddress,
caps: enriched.caps))
if case .ok = result, let nodeId = self.nodeId {
await context.onAuthenticated?(nodeId)
}
} catch {
await self.sendError(code: "INVALID_REQUEST", message: error.localizedDescription)
@@ -13,7 +13,6 @@ actor BridgeServer {
private var isRunning = false
private var store: PairedNodesStore?
private var connections: [String: BridgeConnectionHandler] = [:]
private var nodeInfoById: [String: BridgeNodeInfo] = [:]
private var presenceTasks: [String: Task<Void, Never>] = [:]
private var chatSubscriptions: [String: Set<String>] = [:]
private var gatewayPushTask: Task<Void, Never>?
@@ -82,8 +81,8 @@ actor BridgeServer {
handlePair: { [weak self] request in
await self?.pair(request: request) ?? .error(code: "UNAVAILABLE", message: "bridge unavailable")
},
onAuthenticated: { [weak self] node in
await self?.registerConnection(handler: handler, node: node)
onAuthenticated: { [weak self] nodeId in
await self?.registerConnection(handler: handler, nodeId: nodeId)
},
onDisconnected: { [weak self] nodeId in
await self?.unregisterConnection(nodeId: nodeId)
@@ -113,22 +112,10 @@ actor BridgeServer {
Array(self.connections.keys).sorted()
}
func connectedNodes() -> [BridgeNodeInfo] {
self.nodeInfoById.values.sorted { a, b in
(a.displayName ?? a.nodeId) < (b.displayName ?? b.nodeId)
}
}
func pairedNodes() async -> [PairedNode] {
guard let store = self.store else { return [] }
return await store.all()
}
private func registerConnection(handler: BridgeConnectionHandler, node: BridgeNodeInfo) async {
self.connections[node.nodeId] = handler
self.nodeInfoById[node.nodeId] = node
await self.beaconPresence(nodeId: node.nodeId, reason: "connect")
self.startPresenceTask(nodeId: node.nodeId)
private func registerConnection(handler: BridgeConnectionHandler, nodeId: String) async {
self.connections[nodeId] = handler
await self.beaconPresence(nodeId: nodeId, reason: "connect")
self.startPresenceTask(nodeId: nodeId)
self.ensureGatewayPushTask()
}
@@ -136,7 +123,6 @@ actor BridgeServer {
await self.beaconPresence(nodeId: nodeId, reason: "disconnect")
self.stopPresenceTask(nodeId: nodeId)
self.connections.removeValue(forKey: nodeId)
self.nodeInfoById.removeValue(forKey: nodeId)
self.chatSubscriptions[nodeId] = nil
self.stopGatewayPushTaskIfIdle()
}
@@ -181,13 +167,13 @@ actor BridgeServer {
let sessionKey = payload.sessionKey?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
?? "node-\(nodeId)"
_ = await GatewayConnection.shared.sendAgent(GatewayAgentInvocation(
_ = await GatewayConnection.shared.sendAgent(
message: text,
sessionKey: sessionKey,
thinking: "low",
sessionKey: sessionKey,
deliver: false,
to: nil,
channel: .last))
channel: "last")
case "agent.request":
guard let json = evt.payloadJSON, let data = json.data(using: .utf8) else {
@@ -205,15 +191,15 @@ actor BridgeServer {
?? "node-\(nodeId)"
let thinking = link.thinking?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let to = link.to?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let channel = GatewayAgentChannel(raw: link.channel)
let channel = link.channel?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
_ = await GatewayConnection.shared.sendAgent(GatewayAgentInvocation(
_ = await GatewayConnection.shared.sendAgent(
message: message,
sessionKey: sessionKey,
thinking: thinking,
sessionKey: sessionKey,
deliver: link.deliver,
to: to,
channel: channel))
channel: channel ?? "last")
default:
break
@@ -342,35 +328,39 @@ actor BridgeServer {
}
private func beaconPresence(nodeId: String, reason: String) async {
let paired = await self.store?.find(nodeId: nodeId)
let host = paired?.displayName?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
?? nodeId
let version = paired?.version?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let platform = paired?.platform?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let ip = await self.connections[nodeId]?.remoteAddress()
do {
let paired = await self.store?.find(nodeId: nodeId)
let host = paired?.displayName?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
?? nodeId
let version = paired?.version?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let platform = paired?.platform?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let ip = await self.connections[nodeId]?.remoteAddress()
var tags: [String] = ["node", "ios"]
if let platform { tags.append(platform) }
var tags: [String] = ["node", "ios"]
if let platform { tags.append(platform) }
let summary = [
"Node: \(host)\(ip.map { " (\($0))" } ?? "")",
platform.map { "platform \($0)" },
version.map { "app \($0)" },
"mode node",
"reason \(reason)",
].compactMap(\.self).joined(separator: " · ")
let summary = [
"Node: \(host)\(ip.map { " (\($0))" } ?? "")",
platform.map { "platform \($0)" },
version.map { "app \($0)" },
"mode node",
"reason \(reason)",
].compactMap(\.self).joined(separator: " · ")
var params: [String: AnyCodable] = [
"text": AnyCodable(summary),
"instanceId": AnyCodable(nodeId),
"host": AnyCodable(host),
"mode": AnyCodable("node"),
"reason": AnyCodable(reason),
"tags": AnyCodable(tags),
]
if let ip { params["ip"] = AnyCodable(ip) }
if let version { params["version"] = AnyCodable(version) }
await GatewayConnection.shared.sendSystemEvent(params)
var params: [String: Any] = [
"text": summary,
"instanceId": nodeId,
"host": host,
"mode": "node",
"reason": reason,
"tags": tags,
]
if let ip { params["ip"] = ip }
if let version { params["version"] = version }
_ = try await GatewayConnection.shared.controlRequest(method: "system-event", params: params)
} catch {
// Best-effort only.
}
}
private func startPresenceTask(nodeId: String) {
@@ -403,29 +393,7 @@ actor BridgeServer {
guard let token = hello.token, token == paired.token else {
return .unauthorized
}
do {
var updated = paired
let name = hello.displayName?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let platform = hello.platform?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let version = hello.version?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let deviceFamily = hello.deviceFamily?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
let modelIdentifier = hello.modelIdentifier?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
if updated.displayName != name { updated.displayName = name }
if updated.platform != platform { updated.platform = platform }
if updated.version != version { updated.version = version }
if updated.deviceFamily != deviceFamily { updated.deviceFamily = deviceFamily }
if updated.modelIdentifier != modelIdentifier { updated.modelIdentifier = modelIdentifier }
if updated != paired {
try await store.upsert(updated)
} else {
try await store.touchSeen(nodeId: nodeId)
}
} catch {
// ignore
}
do { try await store.touchSeen(nodeId: nodeId) } catch { /* ignore */ }
return .ok
}
@@ -451,8 +419,6 @@ actor BridgeServer {
displayName: request.displayName,
platform: request.platform,
version: request.version,
deviceFamily: request.deviceFamily,
modelIdentifier: request.modelIdentifier,
token: token,
createdAtMs: nowMs,
lastSeenAtMs: nowMs)
@@ -503,3 +469,10 @@ enum BridgePairingApprover {
}
}
}
extension String {
fileprivate var nonEmpty: String? {
let trimmed = trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? nil : trimmed
}
}
@@ -5,8 +5,6 @@ struct PairedNode: Codable, Equatable {
var displayName: String?
var platform: String?
var version: String?
var deviceFamily: String?
var modelIdentifier: String?
var token: String
var createdAtMs: Int
var lastSeenAtMs: Int?
@@ -206,7 +206,7 @@ actor CameraCaptureService {
private nonisolated static func clampDurationMs(_ ms: Int?) -> Int {
let v = ms ?? 3000
return min(60000, max(250, v))
return min(15000, max(250, v))
}
private nonisolated static func exportToMP4(inputURL: URL, outputURL: URL) async throws {
+12 -24
View File
@@ -1,15 +1,11 @@
import AppKit
import ClawdisIPC
import ClawdisKit
import Foundation
import OSLog
@MainActor
final class CanvasManager {
static let shared = CanvasManager()
private static let logger = Logger(subsystem: "com.steipete.clawdis", category: "CanvasManager")
private var panelController: CanvasWindowController?
private var panelSessionKey: String?
@@ -27,13 +23,7 @@ final class CanvasManager {
try self.showDetailed(sessionKey: sessionKey, target: path, placement: placement).directory
}
func showDetailed(
sessionKey: String,
target: String? = nil,
placement: CanvasPlacement? = nil) throws -> CanvasShowResult
{
Self.logger.debug(
"showDetailed start session=\(sessionKey, privacy: .public) target=\(target ?? "", privacy: .public) placement=\(placement != nil)")
func showDetailed(sessionKey: String, target: String? = nil, placement: CanvasPlacement? = nil) throws -> CanvasShowResult {
let anchorProvider = self.defaultAnchorProvider ?? Self.mouseAnchorProvider
let session = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
let normalizedTarget = target?
@@ -41,7 +31,6 @@ final class CanvasManager {
.nonEmpty
if let controller = self.panelController, self.panelSessionKey == session {
Self.logger.debug("showDetailed reuse existing session=\(session, privacy: .public)")
controller.onVisibilityChanged = { [weak self] visible in
self?.onPanelVisibilityChanged?(visible)
}
@@ -65,19 +54,15 @@ final class CanvasManager {
url: nil)
}
Self.logger.debug("showDetailed creating new session=\(session, privacy: .public)")
self.panelController?.close()
self.panelController = nil
self.panelSessionKey = nil
Self.logger.debug("showDetailed ensure canvas root dir")
try FileManager.default.createDirectory(at: Self.canvasRoot, withIntermediateDirectories: true)
Self.logger.debug("showDetailed init CanvasWindowController")
let controller = try CanvasWindowController(
sessionKey: session,
root: Self.canvasRoot,
presentation: .panel(anchorProvider: anchorProvider))
Self.logger.debug("showDetailed CanvasWindowController init done")
controller.onVisibilityChanged = { [weak self] visible in
self?.onPanelVisibilityChanged?(visible)
}
@@ -87,9 +72,7 @@ final class CanvasManager {
// New session: default to "/" so the user sees either the welcome page or `index.html`.
let effectiveTarget = normalizedTarget ?? "/"
Self.logger.debug("showDetailed showCanvas effectiveTarget=\(effectiveTarget, privacy: .public)")
controller.showCanvas(path: effectiveTarget)
Self.logger.debug("showDetailed showCanvas done")
return self.makeShowResult(
directory: controller.directoryPath,
@@ -182,8 +165,7 @@ final class CanvasManager {
private static func localStatus(sessionDir: URL, target: String) -> CanvasShowStatus {
let fm = FileManager.default
let trimmed = target.trimmingCharacters(in: .whitespacesAndNewlines)
let withoutQuery = trimmed.split(separator: "?", maxSplits: 1, omittingEmptySubsequences: false).first
.map(String.init) ?? trimmed
let withoutQuery = trimmed.split(separator: "?", maxSplits: 1, omittingEmptySubsequences: false).first.map(String.init) ?? trimmed
var path = withoutQuery
if path.hasPrefix("/") { path.removeFirst() }
path = path.removingPercentEncoding ?? path
@@ -226,10 +208,16 @@ final class CanvasManager {
}
private static func hasBundledA2UIShell() -> Bool {
let bundle = ClawdisKitResources.bundle
if bundle.url(forResource: "index", withExtension: "html", subdirectory: "CanvasA2UI") != nil {
return true
guard let base = Bundle.module.resourceURL?.appendingPathComponent("CanvasA2UI", isDirectory: true) else {
return false
}
return bundle.url(forResource: "index", withExtension: "html") != nil
let index = base.appendingPathComponent("index.html", isDirectory: false)
return FileManager.default.fileExists(atPath: index.path)
}
}
private extension String {
var nonEmpty: String? {
isEmpty ? nil : self
}
}
@@ -1,4 +1,3 @@
import ClawdisKit
import Foundation
import OSLog
import WebKit
@@ -205,8 +204,8 @@ final class CanvasSchemeHandler: NSObject, WKURLSchemeHandler {
}
private func a2uiShellPage(sessionRoot: URL) -> CanvasResponse {
// Default Canvas UX: when no index exists, show the built-in scaffold page.
if let data = self.loadBundledResourceData(relativePath: "CanvasScaffold/scaffold.html") {
// Default Canvas UX: when no index exists, show the built-in A2UI shell.
if let data = self.loadBundledResourceData(subdirectory: "CanvasA2UI", relativePath: "index.html") {
return CanvasResponse(mime: "text/html", data: data)
}
@@ -223,10 +222,11 @@ final class CanvasSchemeHandler: NSObject, WKURLSchemeHandler {
|| trimmed.hasPrefix(Self.builtinPrefix + "/")
else { return nil }
let relative = if trimmed == Self.builtinPrefix || trimmed == Self.builtinPrefix + "/" {
"index.html"
let relative: String
if trimmed == Self.builtinPrefix || trimmed == Self.builtinPrefix + "/" {
relative = "index.html"
} else {
String(trimmed.dropFirst((Self.builtinPrefix + "/").count))
relative = String(trimmed.dropFirst((Self.builtinPrefix + "/").count))
}
if relative.isEmpty { return self.html("Not Found", title: "Canvas: 404") }
@@ -234,7 +234,7 @@ final class CanvasSchemeHandler: NSObject, WKURLSchemeHandler {
return self.html("Forbidden", title: "Canvas: 403")
}
guard let data = self.loadBundledResourceData(relativePath: "CanvasA2UI/\(relative)") else {
guard let data = self.loadBundledResourceData(subdirectory: "CanvasA2UI", relativePath: relative) else {
return self.html("Not Found", title: "Canvas: 404")
}
@@ -243,26 +243,12 @@ final class CanvasSchemeHandler: NSObject, WKURLSchemeHandler {
return CanvasResponse(mime: mime, data: data)
}
private func loadBundledResourceData(relativePath: String) -> Data? {
let trimmed = relativePath.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
if trimmed.contains("..") || trimmed.contains("\\") { return nil }
let parts = trimmed.split(separator: "/")
guard let filename = parts.last else { return nil }
let subdirectory =
parts.count > 1 ? parts.dropLast().joined(separator: "/") : nil
let fileURL = URL(fileURLWithPath: String(filename))
let ext = fileURL.pathExtension
let name = fileURL.deletingPathExtension().lastPathComponent
guard !name.isEmpty, !ext.isEmpty else { return nil }
let bundle = ClawdisKitResources.bundle
let resourceURL =
bundle.url(forResource: name, withExtension: ext, subdirectory: subdirectory)
?? bundle.url(forResource: name, withExtension: ext)
guard let resourceURL else { return nil }
return try? Data(contentsOf: resourceURL)
private func loadBundledResourceData(subdirectory: String, relativePath: String) -> Data? {
guard let base = Bundle.module.resourceURL?.appendingPathComponent(subdirectory, isDirectory: true) else {
return nil
}
let url = base.appendingPathComponent(relativePath, isDirectory: false)
return try? Data(contentsOf: url)
}
private func textEncodingName(forMimeType mimeType: String) -> String? {
+35 -163
View File
@@ -1,6 +1,5 @@
import AppKit
import ClawdisIPC
import ClawdisKit
import Foundation
import OSLog
import QuartzCore
@@ -51,107 +50,20 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
self.root = root
self.presentation = presentation
canvasWindowLogger.debug("CanvasWindowController init start session=\(sessionKey, privacy: .public)")
let safeSessionKey = CanvasWindowController.sanitizeSessionKey(sessionKey)
canvasWindowLogger.debug("CanvasWindowController init sanitized session=\(safeSessionKey, privacy: .public)")
self.sessionDir = root.appendingPathComponent(safeSessionKey, isDirectory: true)
try FileManager.default.createDirectory(at: self.sessionDir, withIntermediateDirectories: true)
canvasWindowLogger.debug("CanvasWindowController init session dir ready")
self.schemeHandler = CanvasSchemeHandler(root: root)
canvasWindowLogger.debug("CanvasWindowController init scheme handler ready")
let config = WKWebViewConfiguration()
config.userContentController = WKUserContentController()
config.preferences.isElementFullscreenEnabled = true
config.preferences.setValue(true, forKey: "developerExtrasEnabled")
canvasWindowLogger.debug("CanvasWindowController init config ready")
config.setURLSchemeHandler(self.schemeHandler, forURLScheme: CanvasScheme.scheme)
canvasWindowLogger.debug("CanvasWindowController init scheme handler installed")
// Bridge A2UI "a2uiaction" DOM events back into the native agent loop.
//
// Prefer WKScriptMessageHandler when WebKit exposes it, otherwise fall back to an unattended deep link
// (includes the app-generated key so it won't prompt).
canvasWindowLogger.debug("CanvasWindowController init building A2UI bridge script")
let deepLinkKey = DeepLinkHandler.currentCanvasKey()
let injectedSessionKey = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "main"
let bridgeScript = """
(() => {
try {
if (location.protocol !== '\(CanvasScheme.scheme):') return;
if (globalThis.__clawdisA2UIBridgeInstalled) return;
globalThis.__clawdisA2UIBridgeInstalled = true;
const deepLinkKey = \(Self.jsStringLiteral(deepLinkKey));
const sessionKey = \(Self.jsStringLiteral(injectedSessionKey));
const machineName = \(Self.jsStringLiteral(InstanceIdentity.displayName));
const instanceId = \(Self.jsStringLiteral(InstanceIdentity.instanceId));
globalThis.addEventListener('a2uiaction', (evt) => {
try {
const payload = evt?.detail ?? evt?.payload ?? null;
if (!payload || payload.eventType !== 'a2ui.action') return;
const action = payload.action ?? null;
const name = action?.name ?? '';
if (!name) return;
const context = Array.isArray(action?.context) ? action.context : [];
const userAction = {
id: (globalThis.crypto?.randomUUID?.() ?? String(Date.now())),
name,
surfaceId: payload.surfaceId ?? 'main',
sourceComponentId: payload.sourceComponentId ?? '',
dataContextPath: payload.dataContextPath ?? '',
timestamp: new Date().toISOString(),
...(context.length ? { context } : {}),
};
const handler = globalThis.webkit?.messageHandlers?.clawdisCanvasA2UIAction;
// If the bundled A2UI shell is present, let it forward actions so we keep its richer
// context resolution (data model path lookups, surface detection, etc.).
const hasBundledA2UIHost = !!globalThis.clawdisA2UI || !!document.querySelector('clawdis-a2ui-host');
if (hasBundledA2UIHost && handler?.postMessage) return;
// Otherwise, forward directly when possible.
if (!hasBundledA2UIHost && handler?.postMessage) {
handler.postMessage({ userAction });
return;
}
const ctx = userAction.context ? (' ctx=' + JSON.stringify(userAction.context)) : '';
const message =
'CANVAS_A2UI action=' + userAction.name +
' session=' + sessionKey +
' surface=' + userAction.surfaceId +
' component=' + (userAction.sourceComponentId || '-') +
' host=' + machineName.replace(/\\s+/g, '_') +
' instance=' + instanceId +
ctx +
' default=update_canvas';
const params = new URLSearchParams();
params.set('message', message);
params.set('sessionKey', sessionKey);
params.set('thinking', 'low');
params.set('deliver', 'false');
params.set('channel', 'last');
params.set('key', deepLinkKey);
location.href = 'clawdis://agent?' + params.toString();
} catch {}
}, true);
} catch {}
})();
"""
config.userContentController.addUserScript(
WKUserScript(source: bridgeScript, injectionTime: .atDocumentStart, forMainFrameOnly: true))
canvasWindowLogger.debug("CanvasWindowController init A2UI bridge installed")
canvasWindowLogger.debug("CanvasWindowController init creating WKWebView")
self.webView = WKWebView(frame: .zero, configuration: config)
// Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays.
self.webView.setValue(true, forKey: "drawsBackground")
self.webView.setValue(false, forKey: "drawsBackground")
let sessionDir = self.sessionDir
let webView = self.webView
@@ -162,8 +74,7 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
// Only auto-reload when we are showing local canvas content.
guard webView.url?.scheme == CanvasScheme.scheme else { return }
// Avoid reloading the built-in A2UI shell due to filesystem noise (it does not depend on session
// files).
// Avoid reloading the built-in A2UI shell due to filesystem noise (it does not depend on session files).
let path = webView.url?.path ?? ""
if path.hasPrefix("/__clawdis__/a2ui") { return }
if path == "/" || path.isEmpty {
@@ -182,7 +93,6 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
self.container = HoverChromeContainerView(containing: self.webView)
let window = Self.makeWindow(for: presentation, contentView: self.container)
canvasWindowLogger.debug("CanvasWindowController init makeWindow done")
super.init(window: window)
let handler = CanvasA2UIActionMessageHandler(sessionKey: sessionKey)
@@ -196,15 +106,13 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
}
self.watcher.start()
canvasWindowLogger.debug("CanvasWindowController init done")
}
@available(*, unavailable)
required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") }
@MainActor deinit {
self.webView.configuration.userContentController
.removeScriptMessageHandler(forName: CanvasA2UIActionMessageHandler.messageName)
self.webView.configuration.userContentController.removeScriptMessageHandler(forName: CanvasA2UIActionMessageHandler.messageName)
self.watcher.stop()
}
@@ -233,8 +141,10 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
func hideCanvas() {
if case .panel = self.presentation {
self.persistFrameIfPanel()
self.window?.orderOut(nil)
} else {
self.close()
}
self.window?.orderOut(nil)
self.onVisibilityChanged?(false)
}
@@ -259,10 +169,10 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
if trimmed.hasPrefix("/") {
var isDir: ObjCBool = false
if FileManager.default.fileExists(atPath: trimmed, isDirectory: &isDir), !isDir.boolValue {
let url = URL(fileURLWithPath: trimmed)
canvasWindowLogger.debug("canvas load file \(url.absoluteString, privacy: .public)")
self.loadFile(url)
return
let url = URL(fileURLWithPath: trimmed)
canvasWindowLogger.debug("canvas load file \(url.absoluteString, privacy: .public)")
self.loadFile(url)
return
}
}
@@ -354,7 +264,6 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
backing: .buffered,
defer: false)
window.title = "Clawdis Canvas"
window.isReleasedWhenClosed = false
window.contentView = contentView
window.center()
window.minSize = NSSize(width: 880, height: 680)
@@ -500,18 +409,6 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
}
let scheme = url.scheme?.lowercased()
// Deep links: allow local Canvas content to invoke the agent without bouncing through NSWorkspace.
if scheme == "clawdis" {
if self.webView.url?.scheme == CanvasScheme.scheme {
Task { await DeepLinkHandler.shared.handle(url: url) }
} else {
canvasWindowLogger
.debug("ignoring deep link from non-canvas page \(url.absoluteString, privacy: .public)")
}
decisionHandler(.cancel)
return
}
// Keep web content inside the panel when reasonable.
// `about:blank` and friends are common internal navigations for WKWebView; never send them to NSWorkspace.
if scheme == CanvasScheme.scheme
@@ -569,11 +466,6 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
return String(scalars)
}
private static func jsStringLiteral(_ value: String) -> String {
let data = try? JSONEncoder().encode(value)
return data.flatMap { String(data: $0, encoding: .utf8) } ?? "\"\""
}
private static func storedFrameDefaultsKey(sessionKey: String) -> String {
"clawdis.canvas.frame.\(self.sanitizeSessionKey(sessionKey))"
}
@@ -612,6 +504,9 @@ private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHan
webView.url?.scheme == CanvasScheme.scheme
else { return }
let path = webView.url?.path ?? ""
guard path == "/" || path.isEmpty || path.hasPrefix("/__clawdis__/a2ui") else { return }
let body: [String: Any] = {
if let dict = message.body as? [String: Any] { return dict }
if let dict = message.body as? [AnyHashable: Any] {
@@ -638,59 +533,36 @@ private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHan
guard !userAction.isEmpty else { return }
guard let name = userAction["name"] as? String, !name.isEmpty else { return }
let actionId =
(userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
?? UUID().uuidString
let json: String = {
if let data = try? JSONSerialization.data(withJSONObject: userAction, options: [.prettyPrinted, .sortedKeys]),
let str = String(data: data, encoding: .utf8)
{
return str
}
return ""
}()
canvasWindowLogger.info("A2UI action \(name, privacy: .public) session=\(self.sessionKey, privacy: .public)")
let surfaceId = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
.nonEmpty ?? "main"
let sourceComponentId = (userAction["sourceComponentId"] as? String)?
.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "-"
let instanceId = InstanceIdentity.instanceId.lowercased()
let contextJSON = ClawdisCanvasA2UIAction.compactJSON(userAction["context"])
let text = json.isEmpty
? "A2UI action: \(name)"
: "A2UI action: \(name)\n\n```json\n\(json)\n```"
// Token-efficient and unambiguous. The agent should treat this as a UI event and (by default) update Canvas.
let text = ClawdisCanvasA2UIAction.formatAgentMessage(
actionName: name,
sessionKey: self.sessionKey,
surfaceId: surfaceId,
sourceComponentId: sourceComponentId,
host: InstanceIdentity.displayName,
instanceId: instanceId,
contextJSON: contextJSON)
Task { [weak webView] in
if AppStateStore.shared.connectionMode == .local {
GatewayProcessManager.shared.setActive(true)
}
let result = await GatewayConnection.shared.sendAgent(GatewayAgentInvocation(
Task {
let result = await GatewayConnection.shared.sendAgent(
message: text,
thinking: nil,
sessionKey: self.sessionKey,
thinking: "low",
deliver: false,
to: nil,
channel: .last,
idempotencyKey: actionId))
await MainActor.run {
guard let webView else { return }
let js = ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(
actionId: actionId,
ok: result.ok,
error: result.error)
webView.evaluateJavaScript(js) { _, _ in }
}
channel: "webchat")
if !result.ok {
canvasWindowLogger.error(
"A2UI action send failed name=\(name, privacy: .public) error=\(result.error ?? "unknown", privacy: .public)")
}
}
}
// Formatting helpers live in ClawdisKit (`ClawdisCanvasA2UIAction`).
}
// MARK: - Hover chrome container
@@ -809,8 +681,8 @@ private final class HoverChromeContainerView: NSView {
v.layer?.cornerRadius = 10
v.layer?.masksToBounds = true
v.layer?.borderWidth = 1
v.layer?.borderColor = NSColor.white.withAlphaComponent(0.22).cgColor
v.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.28).cgColor
v.layer?.borderColor = NSColor.white.withAlphaComponent(0.18).cgColor
v.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.22).cgColor
v.layer?.shadowColor = NSColor.black.withAlphaComponent(0.35).cgColor
v.layer?.shadowOpacity = 0.35
v.layer?.shadowRadius = 8
@@ -819,7 +691,7 @@ private final class HoverChromeContainerView: NSView {
}()
private let closeButton: NSButton = {
let cfg = NSImage.SymbolConfiguration(pointSize: 8, weight: .semibold)
let cfg = NSImage.SymbolConfiguration(pointSize: 10, weight: .semibold)
let img = NSImage(systemSymbolName: "xmark", accessibilityDescription: "Close")?
.withSymbolConfiguration(cfg)
?? NSImage(size: NSSize(width: 18, height: 18))
@@ -827,7 +699,7 @@ private final class HoverChromeContainerView: NSView {
btn.isBordered = false
btn.bezelStyle = .regularSquare
btn.imageScaling = .scaleProportionallyDown
btn.contentTintColor = NSColor.white.withAlphaComponent(0.92)
btn.contentTintColor = NSColor.labelColor
btn.toolTip = "Close"
return btn
}()
@@ -873,8 +745,8 @@ private final class HoverChromeContainerView: NSView {
self.closeButton.trailingAnchor.constraint(equalTo: self.trailingAnchor, constant: -8),
self.closeButton.topAnchor.constraint(equalTo: self.topAnchor, constant: 8),
self.closeButton.widthAnchor.constraint(equalToConstant: 16),
self.closeButton.heightAnchor.constraint(equalToConstant: 16),
self.closeButton.widthAnchor.constraint(equalToConstant: 20),
self.closeButton.heightAnchor.constraint(equalToConstant: 20),
self.resizeHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor),
self.resizeHandle.bottomAnchor.constraint(equalTo: self.bottomAnchor),
+200 -221
View File
@@ -3,7 +3,6 @@ import SwiftUI
@MainActor
struct ConfigSettings: View {
private let isPreview = ProcessInfo.processInfo.isPreview
private let state = AppStateStore.shared
private let labelColumnWidth: CGFloat = 120
private static let browserAttachOnlyHelp =
"When enabled, the browser server will only connect if the clawd browser is already running."
@@ -22,6 +21,8 @@ struct ConfigSettings: View {
@State private var allowAutosave = false
@State private var heartbeatMinutes: Int?
@State private var heartbeatBody: String = "HEARTBEAT"
@AppStorage(webChatEnabledKey) private var webChatEnabled: Bool = true
@AppStorage(webChatPortKey) private var webChatPort: Int = 18788
// clawd browser settings (stored in ~/.clawdis/clawdis.json under "browser")
@State private var browserEnabled: Bool = true
@@ -30,233 +31,218 @@ struct ConfigSettings: View {
@State private var browserAttachOnly: Bool = false
var body: some View {
ScrollView { self.content }
.onChange(of: self.modelCatalogPath) { _, _ in
Task { await self.loadModels() }
}
.onChange(of: self.modelCatalogReloadBump) { _, _ in
Task { await self.loadModels() }
}
.task {
guard !self.hasLoaded else { return }
guard !self.isPreview else { return }
self.hasLoaded = true
self.loadConfig()
await self.loadModels()
self.allowAutosave = true
}
}
ScrollView {
VStack(alignment: .leading, spacing: 14) {
Text("Clawdis CLI config")
.font(.title3.weight(.semibold))
Text("Edit ~/.clawdis/clawdis.json (inbound.agent / inbound.session).")
.font(.callout)
.foregroundStyle(.secondary)
private var content: some View {
VStack(alignment: .leading, spacing: 14) {
self.header
self.agentSection
self.heartbeatSection
self.browserSection
Spacer(minLength: 0)
}
.frame(maxWidth: .infinity, alignment: .leading)
.padding(.horizontal, 24)
.padding(.vertical, 18)
.groupBoxStyle(PlainSettingsGroupBoxStyle())
}
@ViewBuilder
private var header: some View {
Text("Clawdis CLI config")
.font(.title3.weight(.semibold))
Text("Edit ~/.clawdis/clawdis.json (inbound.agent / inbound.session).")
.font(.callout)
.foregroundStyle(.secondary)
}
private var agentSection: some View {
GroupBox("Agent") {
Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) {
GridRow {
self.gridLabel("Model")
VStack(alignment: .leading, spacing: 6) {
self.modelPicker
self.customModelField
self.modelMetaLabels
}
}
}
}
.frame(maxWidth: .infinity, alignment: .leading)
}
private var modelPicker: some View {
Picker("Model", selection: self.$configModel) {
ForEach(self.models) { choice in
Text("\(choice.name) — \(choice.provider.uppercased())")
.tag(choice.id)
}
Text("Manual entry…").tag("__custom__")
}
.labelsHidden()
.frame(maxWidth: .infinity)
.disabled(self.modelsLoading || (!self.modelError.isNilOrEmpty && self.models.isEmpty))
.onChange(of: self.configModel) { _, _ in
self.autosaveConfig()
}
}
@ViewBuilder
private var customModelField: some View {
if self.configModel == "__custom__" {
TextField("Enter model ID", text: self.$customModel)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
.onChange(of: self.customModel) { _, newValue in
self.configModel = newValue
self.autosaveConfig()
}
}
}
@ViewBuilder
private var modelMetaLabels: some View {
if let contextLabel = self.selectedContextLabel {
Text(contextLabel)
.font(.footnote)
.foregroundStyle(.secondary)
}
if let authMode = self.selectedAnthropicAuthMode {
HStack(spacing: 8) {
Circle()
.fill(authMode.isConfigured ? Color.green : Color.orange)
.frame(width: 8, height: 8)
Text("Anthropic auth: \(authMode.shortLabel)")
}
.font(.footnote)
.foregroundStyle(authMode.isConfigured ? Color.secondary : Color.orange)
.help(self.anthropicAuthHelpText)
AnthropicAuthControls(connectionMode: self.state.connectionMode)
}
if let modelError {
Text(modelError)
.font(.footnote)
.foregroundStyle(.secondary)
}
}
private var anthropicAuthHelpText: String {
"Determined from Pi OAuth token file (~/.pi/agent/oauth.json) " +
"or environment variables (ANTHROPIC_OAUTH_TOKEN / ANTHROPIC_API_KEY)."
}
private var heartbeatSection: some View {
GroupBox("Heartbeat") {
Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) {
GridRow {
self.gridLabel("Schedule")
VStack(alignment: .leading, spacing: 6) {
HStack(spacing: 12) {
Stepper(
value: Binding(
get: { self.heartbeatMinutes ?? 10 },
set: { self.heartbeatMinutes = $0; self.autosaveConfig() }),
in: 0...720)
{
Text("Every \(self.heartbeatMinutes ?? 10) min")
.frame(width: 150, alignment: .leading)
}
.help("Set to 0 to disable automatic heartbeats")
TextField("HEARTBEAT", text: self.$heartbeatBody)
.textFieldStyle(.roundedBorder)
GroupBox("Agent") {
Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) {
GridRow {
self.gridLabel("Model")
VStack(alignment: .leading, spacing: 6) {
Picker("Model", selection: self.$configModel) {
ForEach(self.models) { choice in
Text("\(choice.name) — \(choice.provider.uppercased())")
.tag(choice.id)
}
Text("Manual entry…").tag("__custom__")
}
.labelsHidden()
.frame(maxWidth: .infinity)
.onChange(of: self.heartbeatBody) { _, _ in
.disabled(self.modelsLoading || (!self.modelError.isNilOrEmpty && self.models.isEmpty))
.onChange(of: self.configModel) { _, _ in
self.autosaveConfig()
}
.help("Message body sent on each heartbeat")
}
Text("Heartbeats keep agent sessions warm; 0 minutes disables them.")
.font(.footnote)
.foregroundStyle(.secondary)
}
}
}
}
.frame(maxWidth: .infinity, alignment: .leading)
}
private var browserSection: some View {
GroupBox("Browser (clawd)") {
Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) {
GridRow {
self.gridLabel("Enabled")
Toggle("", isOn: self.$browserEnabled)
.labelsHidden()
.toggleStyle(.checkbox)
.onChange(of: self.browserEnabled) { _, _ in self.autosaveConfig() }
}
GridRow {
self.gridLabel("Control URL")
TextField("http://127.0.0.1:18791", text: self.$browserControlUrl)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
.disabled(!self.browserEnabled)
.onChange(of: self.browserControlUrl) { _, _ in self.autosaveConfig() }
}
GridRow {
self.gridLabel("Browser path")
VStack(alignment: .leading, spacing: 2) {
if let label = self.browserPathLabel {
Text(label)
.font(.caption.monospaced())
.foregroundStyle(.secondary)
.textSelection(.enabled)
.lineLimit(1)
.truncationMode(.middle)
} else {
Text("—")
.foregroundStyle(.secondary)
if self.configModel == "__custom__" {
TextField("Enter model ID", text: self.$customModel)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
.onChange(of: self.customModel) { _, newValue in
self.configModel = newValue
self.autosaveConfig()
}
}
if let contextLabel = self.selectedContextLabel {
Text(contextLabel)
.font(.footnote)
.foregroundStyle(.secondary)
}
if let modelError {
Text(modelError)
.font(.footnote)
.foregroundStyle(.secondary)
}
}
}
}
.frame(maxWidth: .infinity, alignment: .leading)
}
GridRow {
self.gridLabel("Accent")
HStack(spacing: 8) {
TextField("#FF4500", text: self.$browserColorHex)
.textFieldStyle(.roundedBorder)
.frame(width: 120)
.disabled(!self.browserEnabled)
.onChange(of: self.browserColorHex) { _, _ in self.autosaveConfig() }
Circle()
.fill(self.browserColor)
.frame(width: 12, height: 12)
.overlay(Circle().stroke(Color.secondary.opacity(0.25), lineWidth: 1))
Text("lobster-orange")
.font(.footnote)
.foregroundStyle(.secondary)
.frame(maxWidth: .infinity, alignment: .leading)
GroupBox("Heartbeat") {
Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) {
GridRow {
self.gridLabel("Schedule")
VStack(alignment: .leading, spacing: 6) {
HStack(spacing: 12) {
Stepper(
value: Binding(
get: { self.heartbeatMinutes ?? 10 },
set: { self.heartbeatMinutes = $0; self.autosaveConfig() }),
in: 0...720)
{
Text("Every \(self.heartbeatMinutes ?? 10) min")
.frame(width: 150, alignment: .leading)
}
.help("Set to 0 to disable automatic heartbeats")
TextField("HEARTBEAT", text: self.$heartbeatBody)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
.onChange(of: self.heartbeatBody) { _, _ in
self.autosaveConfig()
}
.help("Message body sent on each heartbeat")
}
Text("Heartbeats keep agent sessions warm; 0 minutes disables them.")
.font(.footnote)
.foregroundStyle(.secondary)
}
}
}
}
GridRow {
self.gridLabel("Attach only")
Toggle("", isOn: self.$browserAttachOnly)
.labelsHidden()
.toggleStyle(.checkbox)
.disabled(!self.browserEnabled)
.onChange(of: self.browserAttachOnly) { _, _ in self.autosaveConfig() }
.help(Self.browserAttachOnlyHelp)
.frame(maxWidth: .infinity, alignment: .leading)
GroupBox("Web Chat") {
Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) {
GridRow {
self.gridLabel("Enabled")
Toggle("", isOn: self.$webChatEnabled)
.labelsHidden()
.toggleStyle(.checkbox)
}
GridRow {
self.gridLabel("Port")
TextField("18788", value: self.$webChatPort, formatter: NumberFormatter())
.textFieldStyle(.roundedBorder)
.frame(width: 100)
.disabled(!self.webChatEnabled)
}
GridRow {
Color.clear
.frame(width: self.labelColumnWidth, height: 1)
Text(
"""
Mac app connects to the gateway’s loopback web chat on this port.
Remote mode uses SSH -L to forward it.
""")
.font(.footnote)
.foregroundStyle(.secondary)
.frame(maxWidth: .infinity, alignment: .leading)
}
}
}
GridRow {
Color.clear
.frame(width: self.labelColumnWidth, height: 1)
Text(Self.browserProfileNote)
.font(.footnote)
.foregroundStyle(.secondary)
.frame(maxWidth: .infinity, alignment: .leading)
.frame(maxWidth: .infinity, alignment: .leading)
GroupBox("Browser (clawd)") {
Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) {
GridRow {
self.gridLabel("Enabled")
Toggle("", isOn: self.$browserEnabled)
.labelsHidden()
.toggleStyle(.checkbox)
.onChange(of: self.browserEnabled) { _, _ in self.autosaveConfig() }
}
GridRow {
self.gridLabel("Control URL")
TextField("http://127.0.0.1:18791", text: self.$browserControlUrl)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
.disabled(!self.browserEnabled)
.onChange(of: self.browserControlUrl) { _, _ in self.autosaveConfig() }
}
GridRow {
self.gridLabel("Browser path")
VStack(alignment: .leading, spacing: 2) {
if let label = self.browserPathLabel {
Text(label)
.font(.caption.monospaced())
.foregroundStyle(.secondary)
.textSelection(.enabled)
.lineLimit(1)
.truncationMode(.middle)
} else {
Text("—")
.foregroundStyle(.secondary)
}
}
.frame(maxWidth: .infinity, alignment: .leading)
}
GridRow {
self.gridLabel("Accent")
HStack(spacing: 8) {
TextField("#FF4500", text: self.$browserColorHex)
.textFieldStyle(.roundedBorder)
.frame(width: 120)
.disabled(!self.browserEnabled)
.onChange(of: self.browserColorHex) { _, _ in self.autosaveConfig() }
Circle()
.fill(self.browserColor)
.frame(width: 12, height: 12)
.overlay(Circle().stroke(Color.secondary.opacity(0.25), lineWidth: 1))
Text("lobster-orange")
.font(.footnote)
.foregroundStyle(.secondary)
}
}
GridRow {
self.gridLabel("Attach only")
Toggle("", isOn: self.$browserAttachOnly)
.labelsHidden()
.toggleStyle(.checkbox)
.disabled(!self.browserEnabled)
.onChange(of: self.browserAttachOnly) { _, _ in self.autosaveConfig() }
.help(Self.browserAttachOnlyHelp)
}
GridRow {
Color.clear
.frame(width: self.labelColumnWidth, height: 1)
Text(Self.browserProfileNote)
.font(.footnote)
.foregroundStyle(.secondary)
.frame(maxWidth: .infinity, alignment: .leading)
}
}
}
.frame(maxWidth: .infinity, alignment: .leading)
Spacer(minLength: 0)
}
.frame(maxWidth: .infinity, alignment: .leading)
.padding(.horizontal, 24)
.padding(.vertical, 18)
.groupBoxStyle(PlainSettingsGroupBoxStyle())
}
.onChange(of: self.modelCatalogPath) { _, _ in
Task { await self.loadModels() }
}
.onChange(of: self.modelCatalogReloadBump) { _, _ in
Task { await self.loadModels() }
}
.task {
guard !self.hasLoaded else { return }
guard !self.isPreview else { return }
self.hasLoaded = true
self.loadConfig()
await self.loadModels()
self.allowAutosave = true
}
.frame(maxWidth: .infinity, alignment: .leading)
}
private func gridLabel(_ text: String) -> some View {
@@ -438,13 +424,6 @@ struct ConfigSettings: View {
return "Context window: \(human) tokens"
}
private var selectedAnthropicAuthMode: AnthropicAuthMode? {
let chosenId = (self.configModel == "__custom__") ? self.customModel : self.configModel
guard !chosenId.isEmpty, let choice = self.models.first(where: { $0.id == chosenId }) else { return nil }
guard choice.provider.lowercased() == "anthropic" else { return nil }
return AnthropicAuthResolver.resolve()
}
private struct PlainSettingsGroupBoxStyle: GroupBoxStyle {
func makeBody(configuration: Configuration) -> some View {
VStack(alignment: .leading, spacing: 10) {
@@ -8,7 +8,7 @@ final class ConnectionModeCoordinator {
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "connection")
/// Apply the requested connection mode by starting/stopping local gateway,
/// managing the control-channel SSH tunnel, and cleaning up chat windows/panels.
/// managing the control-channel SSH tunnel, and cleaning up WebChat tunnels.
func apply(mode: AppState.ConnectionMode, paused: Bool) async {
switch mode {
case .local:
@@ -20,6 +20,9 @@ let connectionModeKey = "clawdis.connectionMode"
let remoteTargetKey = "clawdis.remoteTarget"
let remoteIdentityKey = "clawdis.remoteIdentity"
let remoteProjectRootKey = "clawdis.remoteProjectRoot"
let webChatEnabledKey = "clawdis.webChatEnabled"
let webChatSwiftUIEnabledKey = "clawdis.webChatSwiftUIEnabled"
let webChatPortKey = "clawdis.webChatPort"
let canvasEnabledKey = "clawdis.canvasEnabled"
let cameraEnabledKey = "clawdis.cameraEnabled"
let peekabooBridgeEnabledKey = "clawdis.peekabooBridgeEnabled"
@@ -1,48 +1,9 @@
import ClawdisIPC
import ClawdisKit
import Foundation
import OSLog
enum ControlRequestHandler {
struct NodeListNode: Codable {
var nodeId: String
var displayName: String?
var platform: String?
var version: String?
var deviceFamily: String?
var modelIdentifier: String?
var remoteAddress: String?
var connected: Bool
var paired: Bool
var capabilities: [String]?
var commands: [String]?
}
struct NodeListResult: Codable {
var ts: Int
var connectedNodeIds: [String]
var pairedNodeIds: [String]
var nodes: [NodeListNode]
}
struct GatewayNodeListPayload: Decodable {
struct Node: Decodable {
var nodeId: String
var displayName: String?
var platform: String?
var version: String?
var deviceFamily: String?
var modelIdentifier: String?
var remoteIp: String?
var connected: Bool?
var paired: Bool?
var caps: [String]?
var commands: [String]?
}
var ts: Int?
var nodes: [Node]
}
private static let cameraCapture = CameraCaptureService()
static func process(
request: Request,
@@ -94,8 +55,8 @@ enum ControlRequestHandler {
deliver: deliver,
to: to)
case let .canvasPresent(session, path, placement):
return await self.handleCanvasPresent(session: session, path: path, placement: placement)
case let .canvasShow(session, path, placement):
return await self.handleCanvasShow(session: session, path: path, placement: placement)
case let .canvasHide(session):
return await self.handleCanvasHide(session: session)
@@ -112,9 +73,6 @@ enum ControlRequestHandler {
case .nodeList:
return await self.handleNodeList()
case let .nodeDescribe(nodeId):
return await self.handleNodeDescribe(nodeId: nodeId)
case let .nodeInvoke(nodeId, command, paramsJSON):
return await self.handleNodeInvoke(
nodeId: nodeId,
@@ -131,14 +89,6 @@ enum ControlRequestHandler {
durationMs: durationMs,
includeAudio: includeAudio,
outPath: outPath)
case let .screenRecord(screenIndex, durationMs, fps, includeAudio, outPath):
return await self.handleScreenRecord(
screenIndex: screenIndex,
durationMs: durationMs,
fps: fps,
includeAudio: includeAudio,
outPath: outPath)
}
}
@@ -163,7 +113,9 @@ enum ControlRequestHandler {
priority: request.priority)
return ok ? Response(ok: true) : Response(ok: false, message: "notification not authorized")
case .overlay:
await NotifyOverlayController.shared.present(title: request.title, body: request.body)
await MainActor.run {
NotifyOverlayController.shared.present(title: request.title, body: request.body)
}
return Response(ok: true)
case .auto:
let ok = await notifier.send(
@@ -172,7 +124,9 @@ enum ControlRequestHandler {
sound: chosenSound,
priority: request.priority)
if ok { return Response(ok: true) }
await NotifyOverlayController.shared.present(title: request.title, body: request.body)
await MainActor.run {
NotifyOverlayController.shared.present(title: request.title, body: request.body)
}
return Response(ok: true, message: "notification not authorized; used overlay")
}
}
@@ -215,15 +169,16 @@ enum ControlRequestHandler {
let trimmed = message.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return Response(ok: false, message: "message empty") }
let sessionKey = session ?? "main"
let invocation = GatewayAgentInvocation(
let rpcResult = await GatewayConnection.shared.sendAgent(
message: trimmed,
sessionKey: sessionKey,
thinking: thinking,
sessionKey: sessionKey,
deliver: deliver,
to: to,
channel: .last)
let rpcResult = await GatewayConnection.shared.sendAgent(invocation)
return rpcResult.ok ? Response(ok: true, message: "sent") : Response(ok: false, message: rpcResult.error)
channel: nil)
return rpcResult.ok
? Response(ok: true, message: "sent")
: Response(ok: false, message: rpcResult.error ?? "failed to send")
}
private static func canvasEnabled() -> Bool {
@@ -231,68 +186,36 @@ enum ControlRequestHandler {
}
private static func cameraEnabled() -> Bool {
UserDefaults.standard.object(forKey: cameraEnabledKey) as? Bool ?? false
UserDefaults.standard.object(forKey: cameraEnabledKey) as? Bool ?? true
}
private static func handleCanvasPresent(
private static func handleCanvasShow(
session: String,
path: String?,
placement: CanvasPlacement?) async -> Response
{
guard self.canvasEnabled() else { return Response(ok: false, message: "Canvas disabled by user") }
_ = session
do {
var params: [String: Any] = [:]
if let path, !path.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
params["url"] = path
let res = try await MainActor.run {
try CanvasManager.shared.showDetailed(sessionKey: session, target: path, placement: placement)
}
if let placement {
var placementPayload: [String: Any] = [:]
if let x = placement.x { placementPayload["x"] = x }
if let y = placement.y { placementPayload["y"] = y }
if let width = placement.width { placementPayload["width"] = width }
if let height = placement.height { placementPayload["height"] = height }
if !placementPayload.isEmpty {
params["placement"] = placementPayload
}
}
_ = try await self.invokeLocalNode(
command: ClawdisCanvasCommand.present.rawValue,
params: params.isEmpty ? nil : params,
timeoutMs: 20000)
return Response(ok: true)
let payload = try? JSONEncoder().encode(res)
return Response(ok: true, message: res.directory, payload: payload)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
}
private static func handleCanvasHide(session: String) async -> Response {
_ = session
do {
_ = try await self.invokeLocalNode(
command: ClawdisCanvasCommand.hide.rawValue,
params: nil,
timeoutMs: 10000)
return Response(ok: true)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
await MainActor.run { CanvasManager.shared.hide(sessionKey: session) }
return Response(ok: true)
}
private static func handleCanvasEval(session: String, javaScript: String) async -> Response {
guard self.canvasEnabled() else { return Response(ok: false, message: "Canvas disabled by user") }
_ = session
do {
let payload = try await self.invokeLocalNode(
command: ClawdisCanvasCommand.evalJS.rawValue,
params: ["javaScript": javaScript],
timeoutMs: 20000)
if let dict = payload as? [String: Any],
let result = dict["result"] as? String
{
return Response(ok: true, payload: Data(result.utf8))
}
return Response(ok: true)
let result = try await CanvasManager.shared.eval(sessionKey: session, javaScript: javaScript)
return Response(ok: true, payload: Data(result.utf8))
} catch {
return Response(ok: false, message: error.localizedDescription)
}
@@ -300,130 +223,184 @@ enum ControlRequestHandler {
private static func handleCanvasSnapshot(session: String, outPath: String?) async -> Response {
guard self.canvasEnabled() else { return Response(ok: false, message: "Canvas disabled by user") }
_ = session
do {
let payload = try await self.invokeLocalNode(
command: ClawdisCanvasCommand.snapshot.rawValue,
params: [:],
timeoutMs: 20000)
guard let dict = payload as? [String: Any],
let format = dict["format"] as? String,
let base64 = dict["base64"] as? String,
let data = Data(base64Encoded: base64)
else {
return Response(ok: false, message: "invalid canvas snapshot payload")
}
let ext = (format.lowercased() == "jpeg" || format.lowercased() == "jpg") ? "jpg" : "png"
let url: URL = if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
URL(fileURLWithPath: outPath)
} else {
FileManager.default.temporaryDirectory
.appendingPathComponent("clawdis-canvas-snapshot-\(UUID().uuidString).\(ext)")
}
try data.write(to: url, options: [.atomic])
return Response(ok: true, message: url.path)
let path = try await CanvasManager.shared.snapshot(sessionKey: session, outPath: outPath)
return Response(ok: true, message: path)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
}
private static func handleCanvasA2UI(
session: String,
command: CanvasA2UICommand,
jsonl: String?) async -> Response
{
private static func handleCanvasA2UI(session: String, command: CanvasA2UICommand, jsonl: String?) async -> Response {
guard self.canvasEnabled() else { return Response(ok: false, message: "Canvas disabled by user") }
_ = session
do {
// Ensure the Canvas is visible without forcing a navigation/reload.
_ = try await MainActor.run {
try CanvasManager.shared.show(sessionKey: session, path: nil)
}
// Wait for the in-page A2UI bridge. If it doesn't appear, force-load the bundled A2UI shell once.
var ready = await Self.waitForCanvasA2UI(session: session, requireBuiltinPath: false, timeoutMs: 2_000)
if !ready {
_ = try await MainActor.run {
try CanvasManager.shared.show(sessionKey: session, path: "/__clawdis__/a2ui/")
}
ready = await Self.waitForCanvasA2UI(session: session, requireBuiltinPath: true, timeoutMs: 5_000)
}
guard ready else { return Response(ok: false, message: "A2UI not ready") }
let js: String
switch command {
case .reset:
let payload = try await self.invokeLocalNode(
command: ClawdisCanvasA2UICommand.reset.rawValue,
params: nil,
timeoutMs: 20000)
if let payload {
let data = try JSONSerialization.data(withJSONObject: payload)
return Response(ok: true, payload: data)
}
return Response(ok: true)
js = """
(() => {
try {
if (!globalThis.clawdisA2UI) { return JSON.stringify({ ok: false, error: "missing clawdisA2UI" }); }
return JSON.stringify(globalThis.clawdisA2UI.reset());
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e), stack: e?.stack });
}
})()
"""
case .pushJSONL:
guard let jsonl, !jsonl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
return Response(ok: false, message: "missing jsonl")
}
let payload = try await self.invokeLocalNode(
command: ClawdisCanvasA2UICommand.pushJSONL.rawValue,
params: ["jsonl": jsonl],
timeoutMs: 30000)
if let payload {
let data = try JSONSerialization.data(withJSONObject: payload)
return Response(ok: true, payload: data)
let items: [ParsedJSONLItem]
do {
items = try Self.parseJSONL(jsonl)
} catch {
return Response(ok: false, message: "invalid jsonl: \(error.localizedDescription)")
}
return Response(ok: true)
do {
try Self.validateA2UIV0_8(items)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
let messages = items.map(\.value)
let data = try JSONSerialization.data(withJSONObject: messages, options: [])
let json = String(data: data, encoding: .utf8) ?? "[]"
js = """
(() => {
try {
if (!globalThis.clawdisA2UI) { return JSON.stringify({ ok: false, error: "missing clawdisA2UI" }); }
const messages = \(json);
return JSON.stringify(globalThis.clawdisA2UI.applyMessages(messages));
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e), stack: e?.stack });
}
})()
"""
}
let result = try await CanvasManager.shared.eval(sessionKey: session, javaScript: js)
let payload = Data(result.utf8)
if let obj = try? JSONSerialization.jsonObject(with: payload, options: []) as? [String: Any],
let ok = obj["ok"] as? Bool
{
let error = obj["error"] as? String
return Response(ok: ok, message: ok ? "" : (error ?? "A2UI error"), payload: payload)
}
return Response(ok: true, payload: payload)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
}
private struct ParsedJSONLItem {
let lineNumber: Int
let value: Any
}
private static func parseJSONL(_ text: String) throws -> [ParsedJSONLItem] {
var out: [ParsedJSONLItem] = []
var lineNumber = 0
for rawLine in text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) {
lineNumber += 1
let line = String(rawLine).trimmingCharacters(in: .whitespacesAndNewlines)
if line.isEmpty { continue }
let data = Data(line.utf8)
let obj = try JSONSerialization.jsonObject(with: data, options: [])
out.append(ParsedJSONLItem(lineNumber: lineNumber, value: obj))
}
return out
}
private static func validateA2UIV0_8(_ items: [ParsedJSONLItem]) throws {
let allowed = Set(["beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface"])
for item in items {
guard let dict = item.value as? [String: Any] else {
throw NSError(domain: "A2UI", code: 1, userInfo: [
NSLocalizedDescriptionKey: "A2UI JSONL line \(item.lineNumber): expected a JSON object",
])
}
if dict.keys.contains("createSurface") {
throw NSError(domain: "A2UI", code: 2, userInfo: [
NSLocalizedDescriptionKey: """
A2UI JSONL line \(item.lineNumber): looks like A2UI v0.9 (`createSurface`).
Canvas currently supports A2UI v0.8 server→client messages (`beginRendering`, `surfaceUpdate`, `dataModelUpdate`, `deleteSurface`).
""",
])
}
let matched = dict.keys.filter { allowed.contains($0) }
if matched.count != 1 {
let found = dict.keys.sorted().joined(separator: ", ")
throw NSError(domain: "A2UI", code: 3, userInfo: [
NSLocalizedDescriptionKey: """
A2UI JSONL line \(item.lineNumber): expected exactly one of \(allowed.sorted().joined(separator: ", ")); found: \(found)
""",
])
}
}
}
private static func waitForCanvasA2UI(session: String, requireBuiltinPath: Bool, timeoutMs: Int) async -> Bool {
let clock = ContinuousClock()
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))
while clock.now < deadline {
do {
let res = try await CanvasManager.shared.eval(
sessionKey: session,
javaScript: """
(() => {
try {
if (document?.readyState !== 'complete') { return ''; }
if (!globalThis.clawdisA2UI) { return ''; }
if (typeof globalThis.clawdisA2UI.applyMessages !== 'function') { return ''; }
if (\(requireBuiltinPath ? "true" : "false")) {
const p = String(location?.pathname ?? '');
if (!p.startsWith('/__clawdis__/a2ui')) { return ''; }
}
return 'ready';
} catch {
return '';
}
})()
""")
if res == "ready" { return true }
} catch {
// Ignore; keep waiting.
}
try? await Task.sleep(nanoseconds: 60_000_000)
}
return false
}
private static func handleNodeList() async -> Response {
do {
let data = try await GatewayConnection.shared.request(
method: "node.list",
params: [:],
timeoutMs: 10000)
let payload = try JSONDecoder().decode(GatewayNodeListPayload.self, from: data)
let result = self.buildNodeListResult(payload: payload)
let encoder = JSONEncoder()
encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
let json = (try? encoder.encode(result))
.flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
return Response(ok: true, payload: Data(json.utf8))
} catch {
return Response(ok: false, message: error.localizedDescription)
}
}
private static func handleNodeDescribe(nodeId: String) async -> Response {
do {
let data = try await GatewayConnection.shared.request(
method: "node.describe",
params: ["nodeId": AnyCodable(nodeId)],
timeoutMs: 10000)
return Response(ok: true, payload: data)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
}
static func buildNodeListResult(payload: GatewayNodeListPayload) -> NodeListResult {
let nodes = payload.nodes.map { n -> NodeListNode in
NodeListNode(
nodeId: n.nodeId,
displayName: n.displayName,
platform: n.platform,
version: n.version,
deviceFamily: n.deviceFamily,
modelIdentifier: n.modelIdentifier,
remoteAddress: n.remoteIp,
connected: n.connected == true,
paired: n.paired == true,
capabilities: n.caps,
commands: n.commands)
}
let sorted = nodes.sorted { a, b in
(a.displayName ?? a.nodeId) < (b.displayName ?? b.nodeId)
}
let pairedNodeIds = sorted.filter(\.paired).map(\.nodeId).sorted()
let connectedNodeIds = sorted.filter(\.connected).map(\.nodeId).sorted()
return NodeListResult(
ts: payload.ts ?? Int(Date().timeIntervalSince1970 * 1000),
connectedNodeIds: connectedNodeIds,
pairedNodeIds: pairedNodeIds,
nodes: sorted)
let ids = await BridgeServer.shared.connectedNodeIds()
let payload = (try? JSONSerialization.data(
withJSONObject: ["connectedNodeIds": ids],
options: [.prettyPrinted]))
.flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
return Response(ok: true, payload: Data(payload.utf8))
}
private static func handleNodeInvoke(
@@ -433,30 +410,13 @@ enum ControlRequestHandler {
logger: Logger) async -> Response
{
do {
var paramsObj: Any? = nil
let raw = (paramsJSON ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
if !raw.isEmpty {
if let data = raw.data(using: .utf8) {
paramsObj = try JSONSerialization.jsonObject(with: data)
} else {
return Response(ok: false, message: "params-json not UTF-8")
}
let res = try await BridgeServer.shared.invoke(nodeId: nodeId, command: command, paramsJSON: paramsJSON)
if res.ok {
let payload = res.payloadJSON ?? ""
return Response(ok: true, payload: Data(payload.utf8))
}
var params: [String: AnyCodable] = [
"nodeId": AnyCodable(nodeId),
"command": AnyCodable(command),
"idempotencyKey": AnyCodable(UUID().uuidString),
]
if let paramsObj {
params["params"] = AnyCodable(paramsObj)
}
let data = try await GatewayConnection.shared.request(
method: "node.invoke",
params: params,
timeoutMs: 30000)
return Response(ok: true, payload: data)
let errText = res.error?.message ?? "node invoke failed"
return Response(ok: false, message: errText)
} catch {
logger.error("node invoke failed: \(error.localizedDescription, privacy: .public)")
return Response(ok: false, message: error.localizedDescription)
@@ -471,33 +431,15 @@ enum ControlRequestHandler {
{
guard self.cameraEnabled() else { return Response(ok: false, message: "Camera disabled by user") }
do {
var params: [String: Any] = [:]
if let facing { params["facing"] = facing.rawValue }
if let maxWidth { params["maxWidth"] = maxWidth }
if let quality { params["quality"] = quality }
params["format"] = "jpg"
let payload = try await self.invokeLocalNode(
command: ClawdisCameraCommand.snap.rawValue,
params: params,
timeoutMs: 30000)
guard let dict = payload as? [String: Any],
let format = dict["format"] as? String,
let base64 = dict["base64"] as? String,
let data = Data(base64Encoded: base64)
else {
return Response(ok: false, message: "invalid camera snapshot payload")
}
let ext = (format.lowercased() == "jpeg" || format.lowercased() == "jpg") ? "jpg" : format.lowercased()
let res = try await self.cameraCapture.snap(facing: facing, maxWidth: maxWidth, quality: quality)
let url: URL = if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
URL(fileURLWithPath: outPath)
} else {
FileManager.default.temporaryDirectory
.appendingPathComponent("clawdis-camera-snap-\(UUID().uuidString).\(ext)")
.appendingPathComponent("clawdis-camera-snap-\(UUID().uuidString).jpg")
}
try data.write(to: url, options: [.atomic])
try res.data.write(to: url, options: [.atomic])
return Response(ok: true, message: url.path)
} catch {
return Response(ok: false, message: error.localizedDescription)
@@ -512,103 +454,14 @@ enum ControlRequestHandler {
{
guard self.cameraEnabled() else { return Response(ok: false, message: "Camera disabled by user") }
do {
var params: [String: Any] = ["includeAudio": includeAudio, "format": "mp4"]
if let facing { params["facing"] = facing.rawValue }
if let durationMs { params["durationMs"] = durationMs }
let payload = try await self.invokeLocalNode(
command: ClawdisCameraCommand.clip.rawValue,
params: params,
timeoutMs: 90000)
guard let dict = payload as? [String: Any],
let format = dict["format"] as? String,
let base64 = dict["base64"] as? String,
let data = Data(base64Encoded: base64)
else {
return Response(ok: false, message: "invalid camera clip payload")
}
let ext = format.lowercased()
let url: URL = if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
URL(fileURLWithPath: outPath)
} else {
FileManager.default.temporaryDirectory
.appendingPathComponent("clawdis-camera-clip-\(UUID().uuidString).\(ext)")
}
try data.write(to: url, options: [.atomic])
return Response(ok: true, message: url.path)
let res = try await self.cameraCapture.clip(
facing: facing,
durationMs: durationMs,
includeAudio: includeAudio,
outPath: outPath)
return Response(ok: true, message: res.path)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
}
private static func handleScreenRecord(
screenIndex: Int?,
durationMs: Int?,
fps: Double?,
includeAudio: Bool,
outPath: String?) async -> Response
{
do {
var params: [String: Any] = ["format": "mp4", "includeAudio": includeAudio]
if let screenIndex { params["screenIndex"] = screenIndex }
if let durationMs { params["durationMs"] = durationMs }
if let fps { params["fps"] = fps }
let payload = try await self.invokeLocalNode(
command: "screen.record",
params: params,
timeoutMs: 120_000)
guard let dict = payload as? [String: Any],
let base64 = dict["base64"] as? String,
let data = Data(base64Encoded: base64)
else {
return Response(ok: false, message: "invalid screen record payload")
}
let url: URL = if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
URL(fileURLWithPath: outPath)
} else {
FileManager.default.temporaryDirectory
.appendingPathComponent("clawdis-screen-record-\(UUID().uuidString).mp4")
}
try data.write(to: url, options: [.atomic])
return Response(ok: true, message: url.path)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
}
private static func invokeLocalNode(
command: String,
params: [String: Any]?,
timeoutMs: Double) async throws -> Any?
{
var gatewayParams: [String: AnyCodable] = [
"nodeId": AnyCodable(Self.localNodeId()),
"command": AnyCodable(command),
"idempotencyKey": AnyCodable(UUID().uuidString),
]
if let params {
gatewayParams["params"] = AnyCodable(params)
}
let data = try await GatewayConnection.shared.request(
method: "node.invoke",
params: gatewayParams,
timeoutMs: timeoutMs)
return try Self.decodeNodeInvokePayload(data: data)
}
private static func decodeNodeInvokePayload(data: Data) throws -> Any? {
let obj = try JSONSerialization.jsonObject(with: data)
guard let dict = obj as? [String: Any] else {
throw NSError(domain: "Node", code: 30, userInfo: [
NSLocalizedDescriptionKey: "invalid node invoke response",
])
}
return dict["payload"]
}
private static func localNodeId() -> String {
"mac-\(InstanceIdentity.instanceId)"
}
}
@@ -20,6 +20,7 @@ struct CritterStatusLabel: View {
@State private var nextLegWiggle = Date().addingTimeInterval(Double.random(in: 5.0...11.0))
@State private var earWiggle: CGFloat = 0
@State private var nextEarWiggle = Date().addingTimeInterval(Double.random(in: 7.0...14.0))
private let ticker = Timer.publish(every: 0.35, on: .main, in: .common).autoconnect()
private var isWorkingNow: Bool {
self.iconState.isWorking || self.isWorking
@@ -31,18 +32,34 @@ struct CritterStatusLabel: View {
.frame(width: 18, height: 18)
.rotationEffect(.degrees(self.wiggleAngle), anchor: .center)
.offset(x: self.wiggleOffset)
// Avoid Combine's TimerPublisher here: on macOS 26.2 we've seen crashes inside executor checks
// triggered by its callbacks. Drive periodic updates via a Swift-concurrency task instead.
.task(id: self.tickTaskID) {
.onReceive(self.ticker) { now in
guard self.animationsEnabled, !self.earBoostActive else {
await MainActor.run { self.resetMotion() }
self.resetMotion()
return
}
while !Task.isCancelled {
let now = Date()
await MainActor.run { self.tick(now) }
try? await Task.sleep(nanoseconds: 350_000_000)
if now >= self.nextBlink {
self.blink()
self.nextBlink = now.addingTimeInterval(Double.random(in: 3.5...8.5))
}
if now >= self.nextWiggle {
self.wiggle()
self.nextWiggle = now.addingTimeInterval(Double.random(in: 6.5...14))
}
if now >= self.nextLegWiggle {
self.wiggleLegs()
self.nextLegWiggle = now.addingTimeInterval(Double.random(in: 5.0...11.0))
}
if now >= self.nextEarWiggle {
self.wiggleEars()
self.nextEarWiggle = now.addingTimeInterval(Double.random(in: 7.0...14.0))
}
if self.isWorkingNow {
self.scurry()
}
}
.onChange(of: self.isPaused) { _, _ in self.resetMotion() }
@@ -79,42 +96,6 @@ struct CritterStatusLabel: View {
.frame(width: 18, height: 18)
}
private var tickTaskID: Int {
// Ensure SwiftUI restarts (and cancels) the task when these change.
(self.animationsEnabled ? 1 : 0) | (self.earBoostActive ? 2 : 0)
}
private func tick(_ now: Date) {
guard self.animationsEnabled, !self.earBoostActive else {
self.resetMotion()
return
}
if now >= self.nextBlink {
self.blink()
self.nextBlink = now.addingTimeInterval(Double.random(in: 3.5...8.5))
}
if now >= self.nextWiggle {
self.wiggle()
self.nextWiggle = now.addingTimeInterval(Double.random(in: 6.5...14))
}
if now >= self.nextLegWiggle {
self.wiggleLegs()
self.nextLegWiggle = now.addingTimeInterval(Double.random(in: 5.0...11.0))
}
if now >= self.nextEarWiggle {
self.wiggleEars()
self.nextEarWiggle = now.addingTimeInterval(Double.random(in: 7.0...14.0))
}
if self.isWorkingNow {
self.scurry()
}
}
private var iconImage: Image {
let badge: CritterIconRenderer.Badge? = if let prominence = self.iconState.badgeProminence, !self.isPaused {
CritterIconRenderer.Badge(
@@ -147,8 +128,7 @@ struct CritterStatusLabel: View {
private func blink() {
withAnimation(.easeInOut(duration: 0.08)) { self.blinkAmount = 1 }
Task { @MainActor in
try? await Task.sleep(nanoseconds: 160_000_000)
DispatchQueue.main.asyncAfter(deadline: .now() + 0.16) {
withAnimation(.easeOut(duration: 0.12)) { self.blinkAmount = 0 }
}
}
@@ -160,8 +140,7 @@ struct CritterStatusLabel: View {
self.wiggleAngle = targetAngle
self.wiggleOffset = targetOffset
}
Task { @MainActor in
try? await Task.sleep(nanoseconds: 360_000_000)
DispatchQueue.main.asyncAfter(deadline: .now() + 0.36) {
withAnimation(.interpolatingSpring(stiffness: 220, damping: 18)) {
self.wiggleAngle = 0
self.wiggleOffset = 0
@@ -174,8 +153,7 @@ struct CritterStatusLabel: View {
withAnimation(.easeInOut(duration: 0.14)) {
self.legWiggle = target
}
Task { @MainActor in
try? await Task.sleep(nanoseconds: 220_000_000)
DispatchQueue.main.asyncAfter(deadline: .now() + 0.22) {
withAnimation(.easeOut(duration: 0.18)) { self.legWiggle = 0 }
}
}
@@ -186,8 +164,7 @@ struct CritterStatusLabel: View {
self.legWiggle = target
self.wiggleOffset = CGFloat.random(in: -0.6...0.6)
}
Task { @MainActor in
try? await Task.sleep(nanoseconds: 180_000_000)
DispatchQueue.main.asyncAfter(deadline: .now() + 0.18) {
withAnimation(.easeOut(duration: 0.16)) {
self.legWiggle = 0.25
self.wiggleOffset = 0
@@ -200,11 +177,8 @@ struct CritterStatusLabel: View {
withAnimation(.interpolatingSpring(stiffness: 260, damping: 19)) {
self.earWiggle = target
}
Task { @MainActor in
try? await Task.sleep(nanoseconds: 320_000_000)
withAnimation(.interpolatingSpring(stiffness: 260, damping: 19)) {
self.earWiggle = 0
}
DispatchQueue.main.asyncAfter(deadline: .now() + 0.32) {
withAnimation(.interpolatingSpring(stiffness: 260, damping: 19)) { self.earWiggle = 0 }
}
}
@@ -434,8 +408,8 @@ enum CritterIconRenderer {
// Bigger, higher-contrast badge:
// - Increase diameter so tool activity is noticeable.
// - Draw a filled "puck", then knock out the symbol shape (transparent hole).
// This reads better in template-rendered menu bar icons than tiny monochrome glyphs.
// - Use a filled "puck" background with a fully-opaque SF Symbol on top.
// (The menu bar image is rendered as a template, so "knocking out" the symbol makes it invisible.)
let diameter = canvas.snapX(canvas.w * 0.52 * (0.92 + 0.08 * strength)) // ~9–10pt on an 18pt canvas
let margin = canvas.snapX(max(0.45, canvas.w * 0.03))
let rect = CGRect(
@@ -466,22 +440,19 @@ enum CritterIconRenderer {
canvas.context.strokeEllipse(in: rect.insetBy(dx: 0.45, dy: 0.45))
if let base = NSImage(systemSymbolName: badge.symbolName, accessibilityDescription: nil) {
let pointSize = max(7.0, diameter * 0.82)
let config = NSImage.SymbolConfiguration(pointSize: pointSize, weight: .black)
let pointSize = max(6.0, diameter * 0.80)
let config = NSImage.SymbolConfiguration(pointSize: pointSize, weight: .bold)
let symbol = base.withSymbolConfiguration(config) ?? base
symbol.isTemplate = true
let symbolRect = rect.insetBy(dx: diameter * 0.17, dy: diameter * 0.17)
canvas.context.saveGState()
canvas.context.setBlendMode(.clear)
let symbolRect = rect.insetBy(dx: diameter * 0.19, dy: diameter * 0.19)
symbol.draw(
in: symbolRect,
from: .zero,
operation: .sourceOver,
fraction: 1,
fraction: min(1.0, 0.96 + 0.04 * strength),
respectFlipped: true,
hints: nil)
canvas.context.restoreGState()
}
canvas.context.restoreGState()
+44 -12
View File
@@ -67,12 +67,16 @@ final class CronJobsStore {
defer { self.isLoadingJobs = false }
do {
if let status = try? await GatewayConnection.shared.cronStatus() {
if let status = try? await self.fetchCronStatus() {
self.schedulerEnabled = status.enabled
self.schedulerStorePath = status.storePath
self.schedulerNextWakeAtMs = status.nextWakeAtMs
}
self.jobs = try await GatewayConnection.shared.cronList(includeDisabled: true)
let data = try await self.request(
method: "cron.list",
params: ["includeDisabled": true])
let res = try JSONDecoder().decode(CronListResponse.self, from: data)
self.jobs = res.jobs
if self.jobs.isEmpty {
self.statusMessage = "No cron jobs yet."
}
@@ -88,7 +92,11 @@ final class CronJobsStore {
defer { self.isLoadingRuns = false }
do {
self.runEntries = try await GatewayConnection.shared.cronRuns(jobId: jobId, limit: limit)
let data = try await self.request(
method: "cron.runs",
params: ["id": jobId, "limit": limit])
let res = try JSONDecoder().decode(CronRunsResponse.self, from: data)
self.runEntries = res.entries
} catch {
self.logger.error("cron.runs failed \(error.localizedDescription, privacy: .public)")
self.lastError = error.localizedDescription
@@ -97,7 +105,10 @@ final class CronJobsStore {
func runJob(id: String, force: Bool = true) async {
do {
try await GatewayConnection.shared.cronRun(jobId: id, force: force)
_ = try await self.request(
method: "cron.run",
params: ["id": id, "mode": force ? "force" : "due"],
timeoutMs: 20000)
} catch {
self.lastError = error.localizedDescription
}
@@ -105,7 +116,7 @@ final class CronJobsStore {
func removeJob(id: String) async {
do {
try await GatewayConnection.shared.cronRemove(jobId: id)
_ = try await self.request(method: "cron.remove", params: ["id": id])
await self.refreshJobs()
if self.selectedJobId == id {
self.selectedJobId = nil
@@ -118,9 +129,9 @@ final class CronJobsStore {
func setJobEnabled(id: String, enabled: Bool) async {
do {
try await GatewayConnection.shared.cronUpdate(
jobId: id,
patch: ["enabled": AnyCodable(enabled)])
_ = try await self.request(
method: "cron.update",
params: ["id": id, "patch": ["enabled": enabled]])
await self.refreshJobs()
} catch {
self.lastError = error.localizedDescription
@@ -129,12 +140,12 @@ final class CronJobsStore {
func upsertJob(
id: String?,
payload: [String: AnyCodable]) async throws
payload: [String: Any]) async throws
{
if let id {
try await GatewayConnection.shared.cronUpdate(jobId: id, patch: payload)
_ = try await self.request(method: "cron.update", params: ["id": id, "patch": payload])
} else {
try await GatewayConnection.shared.cronAdd(payload: payload)
_ = try await self.request(method: "cron.add", params: payload)
}
await self.refreshJobs()
}
@@ -195,5 +206,26 @@ final class CronJobsStore {
}
}
// MARK: - (no additional RPC helpers)
// MARK: - RPC
private func request(
method: String,
params: [String: Any]?,
timeoutMs: Double? = nil) async throws -> Data
{
let rawParams = params?.reduce(into: [String: AnyCodable]()) { $0[$1.key] = AnyCodable($1.value) }
return try await GatewayConnection.shared.request(method: method, params: rawParams, timeoutMs: timeoutMs)
}
private func fetchCronStatus() async throws -> CronStatusResponse {
let data = try await self.request(method: "cron.status", params: nil)
return try JSONDecoder().decode(CronStatusResponse.self, from: data)
}
}
private struct CronStatusResponse: Decodable {
let enabled: Bool
let storePath: String
let jobs: Int
let nextWakeAtMs: Int?
}
+10 -10
View File
@@ -454,7 +454,7 @@ struct CronSettings: View {
return "in \(days)d"
}
private func save(payload: [String: AnyCodable]) async {
private func save(payload: [String: Any]) async {
guard !self.isSaving else { return }
self.isSaving = true
self.editorError = nil
@@ -494,7 +494,7 @@ struct CronJobEditor: View {
@Binding var isSaving: Bool
@Binding var error: String?
let onCancel: () -> Void
let onSave: ([String: AnyCodable]) -> Void
let onSave: ([String: Any]) -> Void
private let labelColumnWidth: CGFloat = 160
private static let introText =
@@ -530,7 +530,7 @@ struct CronJobEditor: View {
@State private var systemEventText: String = ""
@State private var agentMessage: String = ""
@State private var deliver: Bool = false
@State private var channel: GatewayAgentChannel = .last
@State private var channel: String = "last"
@State private var to: String = ""
@State private var thinking: String = ""
@State private var timeoutSeconds: String = ""
@@ -801,9 +801,9 @@ struct CronJobEditor: View {
GridRow {
self.gridLabel("Channel")
Picker("", selection: self.$channel) {
Text("last").tag(GatewayAgentChannel.last)
Text("whatsapp").tag(GatewayAgentChannel.whatsapp)
Text("telegram").tag(GatewayAgentChannel.telegram)
Text("last").tag("last")
Text("whatsapp").tag("whatsapp")
Text("telegram").tag("telegram")
}
.labelsHidden()
.pickerStyle(.segmented)
@@ -861,7 +861,7 @@ struct CronJobEditor: View {
self.thinking = thinking ?? ""
self.timeoutSeconds = timeoutSeconds.map(String.init) ?? ""
self.deliver = deliver ?? false
self.channel = GatewayAgentChannel(raw: channel)
self.channel = channel ?? "last"
self.to = to ?? ""
self.bestEffortDeliver = bestEffortDeliver ?? false
}
@@ -879,7 +879,7 @@ struct CronJobEditor: View {
}
}
private func buildPayload() throws -> [String: AnyCodable] {
private func buildPayload() throws -> [String: Any] {
let name = self.name.trimmingCharacters(in: .whitespacesAndNewlines)
let schedule: [String: Any]
switch self.scheduleKind {
@@ -969,7 +969,7 @@ struct CronJobEditor: View {
]
}
return root.mapValues { AnyCodable($0) }
return root
}
private func buildAgentTurnPayload() -> [String: Any] {
@@ -980,7 +980,7 @@ struct CronJobEditor: View {
if let n = Int(self.timeoutSeconds), n > 0 { payload["timeoutSeconds"] = n }
payload["deliver"] = self.deliver
if self.deliver {
payload["channel"] = self.channel.rawValue
payload["channel"] = self.channel
let to = self.to.trimmingCharacters(in: .whitespacesAndNewlines)
if !to.isEmpty { payload["to"] = to }
payload["bestEffortDeliver"] = self.bestEffortDeliver
@@ -193,6 +193,12 @@ enum DebugActions {
typealias PortListener = PortGuardian.ReportListener
typealias PortReport = PortGuardian.PortReport
@MainActor
static func openChatInBrowser() async {
let session = WebChatManager.shared.preferredSessionKey()
await WebChatManager.shared.openInBrowser(sessionKey: session)
}
static func checkGatewayPorts() async -> [PortReport] {
let mode = CommandResolver.connectionSettings().mode
return await PortGuardian.shared.diagnose(mode: mode)
+10 -7
View File
@@ -24,6 +24,7 @@ struct DebugSettings: View {
@State private var portReports: [DebugActions.PortReport] = []
@State private var portKillStatus: String?
@State private var pendingKill: DebugActions.PortListener?
@AppStorage(webChatSwiftUIEnabledKey) private var webChatSwiftUIEnabled: Bool = false
@AppStorage(attachExistingGatewayOnlyKey) private var attachExistingGatewayOnly: Bool = false
@AppStorage(debugFileLogEnabledKey) private var diagnosticsFileLogEnabled: Bool = false
@@ -277,7 +278,7 @@ struct DebugSettings: View {
}
if self.portReports.isEmpty, !self.portCheckInFlight {
Text("Check which process owns 18789 and suggest fixes.")
Text("Check which process owns 18788/18789 and suggest fixes.")
.font(.caption2)
.foregroundStyle(.secondary)
} else {
@@ -489,7 +490,7 @@ struct DebugSettings: View {
.font(.caption.monospaced())
.frame(width: 160)
Button("Show panel") {
Task { await self.canvasPresent() }
Task { await self.canvasShow() }
}
.buttonStyle(.borderedProminent)
Button("Hide panel") {
@@ -578,10 +579,12 @@ struct DebugSettings: View {
.frame(maxWidth: 280, alignment: .leading)
}
GridRow {
self.gridLabel("Chat")
Text("Native SwiftUI")
.font(.callout)
.foregroundStyle(.secondary)
self.gridLabel("Web chat")
Toggle("Use SwiftUI web chat (glass)", isOn: self.$webChatSwiftUIEnabled)
.toggleStyle(.checkbox)
.help(
"When enabled, the menu bar chat window/panel uses the native SwiftUI UI instead of the " +
"bundled WKWebView.")
}
}
}
@@ -750,7 +753,7 @@ extension DebugSettings {
// MARK: - Canvas debug actions
@MainActor
private func canvasPresent() async {
private func canvasShow() async {
self.canvasError = nil
let session = self.canvasSessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
do {
+12 -38
View File
@@ -12,11 +12,6 @@ final class DeepLinkHandler {
private var lastPromptAt: Date = .distantPast
// Ephemeral, in-memory key used for unattended deep links originating from the in-app Canvas.
// This avoids blocking Canvas init on UserDefaults and doesn't weaken the external deep-link prompt:
// outside callers can't know this randomly generated key.
private nonisolated static let canvasUnattendedKey: String = DeepLinkHandler.generateRandomKey()
func handle(url: URL) async {
guard let route = DeepLinkParser.parse(url) else {
deepLinkLogger.debug("ignored url \(url.absoluteString, privacy: .public)")
@@ -40,7 +35,7 @@ final class DeepLinkHandler {
return
}
let allowUnattended = link.key == Self.canvasUnattendedKey || link.key == Self.expectedKey()
let allowUnattended = link.key == Self.expectedKey()
if !allowUnattended {
if Date().timeIntervalSince(self.lastPromptAt) < 1.0 {
deepLinkLogger.debug("throttling deep link prompt")
@@ -59,24 +54,18 @@ final class DeepLinkHandler {
}
do {
let channel = GatewayAgentChannel(raw: link.channel)
let invocation = GatewayAgentInvocation(
message: messagePreview,
sessionKey: link.sessionKey?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "main",
thinking: link.thinking?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty,
deliver: channel.shouldDeliver(link.deliver),
to: link.to?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty,
channel: channel,
timeoutSeconds: link.timeoutSeconds,
idempotencyKey: UUID().uuidString)
var params: [String: AnyCodable] = [
"message": AnyCodable(messagePreview),
"idempotencyKey": AnyCodable(UUID().uuidString),
]
if let sessionKey = link.sessionKey, !sessionKey.isEmpty { params["sessionKey"] = AnyCodable(sessionKey) }
if let thinking = link.thinking, !thinking.isEmpty { params["thinking"] = AnyCodable(thinking) }
if let to = link.to, !to.isEmpty { params["to"] = AnyCodable(to) }
if let channel = link.channel, !channel.isEmpty { params["channel"] = AnyCodable(channel) }
if let timeout = link.timeoutSeconds { params["timeout"] = AnyCodable(timeout) }
params["deliver"] = AnyCodable(link.deliver)
let res = await GatewayConnection.shared.sendAgent(invocation)
if !res.ok {
throw NSError(
domain: "DeepLink",
code: 1,
userInfo: [NSLocalizedDescriptionKey: res.error ?? "agent request failed"])
}
_ = try await GatewayConnection.shared.request(method: "agent", params: params)
} catch {
self.presentAlert(title: "Agent request failed", message: error.localizedDescription)
}
@@ -88,10 +77,6 @@ final class DeepLinkHandler {
self.expectedKey()
}
static func currentCanvasKey() -> String {
self.canvasUnattendedKey
}
private static func expectedKey() -> String {
let defaults = UserDefaults.standard
if let key = defaults.string(forKey: deepLinkKeyKey), !key.isEmpty {
@@ -109,17 +94,6 @@ final class DeepLinkHandler {
return key
}
private nonisolated static func generateRandomKey() -> String {
var bytes = [UInt8](repeating: 0, count: 32)
_ = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes)
let data = Data(bytes)
return data
.base64EncodedString()
.replacingOccurrences(of: "+", with: "-")
.replacingOccurrences(of: "/", with: "_")
.replacingOccurrences(of: "=", with: "")
}
// MARK: - UI
private func confirm(title: String, message: String) -> Bool {
@@ -1,157 +0,0 @@
import Foundation
struct DevicePresentation: Sendable {
let title: String
let symbol: String?
}
enum DeviceModelCatalog {
private static let modelIdentifierToName: [String: String] = loadModelIdentifierToName()
static func presentation(deviceFamily: String?, modelIdentifier: String?) -> DevicePresentation? {
let family = (deviceFamily ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let model = (modelIdentifier ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let friendlyName = model.isEmpty ? nil : self.modelIdentifierToName[model]
let symbol = self.symbol(deviceFamily: family, modelIdentifier: model, friendlyName: friendlyName)
let title = if let friendlyName, !friendlyName.isEmpty {
friendlyName
} else if !family.isEmpty, !model.isEmpty {
"\(family) (\(model))"
} else if !family.isEmpty {
family
} else if !model.isEmpty {
model
} else {
""
}
if title.isEmpty { return nil }
return DevicePresentation(title: title, symbol: symbol)
}
static func symbol(
deviceFamily familyRaw: String,
modelIdentifier modelIdentifierRaw: String,
friendlyName: String?) -> String?
{
let family = familyRaw.trimmingCharacters(in: .whitespacesAndNewlines)
let modelIdentifier = modelIdentifierRaw.trimmingCharacters(in: .whitespacesAndNewlines)
return self.symbolFor(modelIdentifier: modelIdentifier, friendlyName: friendlyName)
?? self.fallbackSymbol(for: family, modelIdentifier: modelIdentifier)
}
private static func symbolFor(modelIdentifier rawModelIdentifier: String, friendlyName: String?) -> String? {
let modelIdentifier = rawModelIdentifier.trimmingCharacters(in: .whitespacesAndNewlines)
guard !modelIdentifier.isEmpty else { return nil }
let lower = modelIdentifier.lowercased()
if lower.hasPrefix("ipad") { return "ipad" }
if lower.hasPrefix("iphone") { return "iphone" }
if lower.hasPrefix("ipod") { return "iphone" }
if lower.hasPrefix("watch") { return "applewatch" }
if lower.hasPrefix("appletv") { return "appletv" }
if lower.hasPrefix("audio") || lower.hasPrefix("homepod") { return "speaker" }
if lower.hasPrefix("macbook") || lower.hasPrefix("macbookpro") || lower.hasPrefix("macbookair") {
return "laptopcomputer"
}
if lower.hasPrefix("macstudio") { return "macstudio" }
if lower.hasPrefix("macmini") { return "macmini" }
if lower.hasPrefix("imac") || lower.hasPrefix("macpro") { return "desktopcomputer" }
if lower.hasPrefix("mac"), let friendlyNameLower = friendlyName?.lowercased() {
if friendlyNameLower.contains("macbook") { return "laptopcomputer" }
if friendlyNameLower.contains("imac") { return "desktopcomputer" }
if friendlyNameLower.contains("mac mini") { return "macmini" }
if friendlyNameLower.contains("mac studio") { return "macstudio" }
if friendlyNameLower.contains("mac pro") { return "desktopcomputer" }
}
return nil
}
private static func fallbackSymbol(for familyRaw: String, modelIdentifier: String) -> String? {
let family = familyRaw.trimmingCharacters(in: .whitespacesAndNewlines)
if family.isEmpty { return nil }
switch family.lowercased() {
case "ipad":
return "ipad"
case "iphone":
return "iphone"
case "mac":
return "laptopcomputer"
case "android":
return "android"
case "linux":
return "cpu"
default:
return "cpu"
}
}
private static func loadModelIdentifierToName() -> [String: String] {
var combined: [String: String] = [:]
combined.merge(
self.loadMapping(resourceName: "ios-device-identifiers"),
uniquingKeysWith: { current, _ in current })
combined.merge(
self.loadMapping(resourceName: "mac-device-identifiers"),
uniquingKeysWith: { current, _ in current })
return combined
}
private static func loadMapping(resourceName: String) -> [String: String] {
guard let url = Bundle.module.url(
forResource: resourceName,
withExtension: "json",
subdirectory: "DeviceModels")
else {
return [:]
}
do {
let data = try Data(contentsOf: url)
let decoded = try JSONDecoder().decode([String: NameValue].self, from: data)
return decoded.compactMapValues { $0.normalizedName }
} catch {
return [:]
}
}
private enum NameValue: Decodable {
case string(String)
case stringArray([String])
init(from decoder: Decoder) throws {
let container = try decoder.singleValueContainer()
if let s = try? container.decode(String.self) {
self = .string(s)
return
}
if let arr = try? container.decode([String].self) {
self = .stringArray(arr)
return
}
throw DecodingError.typeMismatch(
String.self,
.init(codingPath: decoder.codingPath, debugDescription: "Expected string or string array"))
}
var normalizedName: String? {
switch self {
case let .string(s):
let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? nil : trimmed
case let .stringArray(arr):
let values = arr
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
.filter { !$0.isEmpty }
guard !values.isEmpty else { return nil }
return values.joined(separator: " / ")
}
}
}
}
@@ -423,12 +423,8 @@ actor GatewayChannelActor {
throw NSError(domain: "Gateway", code: 2, userInfo: [NSLocalizedDescriptionKey: "unexpected frame"])
}
if res.ok == false {
let code = res.error?["code"]?.value as? String
let msg = res.error?["message"]?.value as? String
let details: [String: AnyCodable] = (res.error ?? [:]).reduce(into: [:]) { acc, pair in
acc[pair.key] = AnyCodable(pair.value.value)
}
throw GatewayResponseError(method: method, code: code, message: msg, details: details)
let msg = (res.error?["message"]?.value as? String) ?? "gateway error"
throw NSError(domain: "Gateway", code: 3, userInfo: [NSLocalizedDescriptionKey: msg])
}
if let payload = res.payload {
// Encode back to JSON with Swift's encoder to preserve types and avoid ObjC bridging exceptions.
@@ -1,37 +1,7 @@
import ClawdisChatUI
import ClawdisProtocol
import Foundation
import OSLog
private let gatewayConnectionLogger = Logger(subsystem: "com.steipete.clawdis", category: "gateway.connection")
enum GatewayAgentChannel: String, Codable, CaseIterable, Sendable {
case last
case whatsapp
case telegram
case webchat
init(raw: String?) {
let normalized = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
self = GatewayAgentChannel(rawValue: normalized) ?? .last
}
var isDeliverable: Bool { self != .webchat }
func shouldDeliver(_ deliver: Bool) -> Bool { deliver && self.isDeliverable }
}
struct GatewayAgentInvocation: Sendable {
var message: String
var sessionKey: String = "main"
var thinking: String?
var deliver: Bool = false
var to: String?
var channel: GatewayAgentChannel = .last
var timeoutSeconds: Int?
var idempotencyKey: String = UUID().uuidString
}
/// Single, shared Gateway websocket connection for the whole app.
///
/// This owns exactly one `GatewayChannelActor` and reuses it across all callers
@@ -41,31 +11,8 @@ actor GatewayConnection {
typealias Config = (url: URL, token: String?)
enum Method: String, Sendable {
case agent
case status
case setHeartbeats = "set-heartbeats"
case systemEvent = "system-event"
case health
case chatHistory = "chat.history"
case chatSend = "chat.send"
case chatAbort = "chat.abort"
case voicewakeGet = "voicewake.get"
case voicewakeSet = "voicewake.set"
case nodePairApprove = "node.pair.approve"
case nodePairReject = "node.pair.reject"
case cronList = "cron.list"
case cronRuns = "cron.runs"
case cronRun = "cron.run"
case cronRemove = "cron.remove"
case cronUpdate = "cron.update"
case cronAdd = "cron.add"
case cronStatus = "cron.status"
}
private let configProvider: @Sendable () async throws -> Config
private let sessionBox: WebSocketSessionBox?
private let decoder = JSONDecoder()
private var client: GatewayChannelActor?
private var configuredURL: URL?
@@ -82,8 +29,6 @@ actor GatewayConnection {
self.sessionBox = sessionBox
}
// MARK: - Low-level request
func request(
method: String,
params: [String: AnyCodable]?,
@@ -94,66 +39,7 @@ actor GatewayConnection {
guard let client else {
throw NSError(domain: "Gateway", code: 0, userInfo: [NSLocalizedDescriptionKey: "gateway not configured"])
}
do {
return try await client.request(method: method, params: params, timeoutMs: timeoutMs)
} catch {
// Auto-recover in local mode by spawning/attaching a gateway and retrying a few times.
// Canvas interactions should "just work" even if the local gateway isn't running yet.
let isLocal = await MainActor.run { AppStateStore.shared.connectionMode == .local }
guard isLocal else { throw error }
await MainActor.run { GatewayProcessManager.shared.setActive(true) }
var lastError: Error = error
for delayMs in [150, 400, 900] {
try await Task.sleep(nanoseconds: UInt64(delayMs) * 1_000_000)
do {
return try await client.request(method: method, params: params, timeoutMs: timeoutMs)
} catch {
lastError = error
}
}
throw lastError
}
}
func requestRaw(
method: Method,
params: [String: AnyCodable]? = nil,
timeoutMs: Double? = nil) async throws -> Data
{
try await self.request(method: method.rawValue, params: params, timeoutMs: timeoutMs)
}
func requestRaw(
method: String,
params: [String: AnyCodable]? = nil,
timeoutMs: Double? = nil) async throws -> Data
{
try await self.request(method: method, params: params, timeoutMs: timeoutMs)
}
func requestDecoded<T: Decodable>(
method: Method,
params: [String: AnyCodable]? = nil,
timeoutMs: Double? = nil) async throws -> T
{
let data = try await self.requestRaw(method: method, params: params, timeoutMs: timeoutMs)
do {
return try self.decoder.decode(T.self, from: data)
} catch {
throw GatewayDecodingError(method: method.rawValue, message: error.localizedDescription)
}
}
func requestVoid(
method: Method,
params: [String: AnyCodable]? = nil,
timeoutMs: Double? = nil) async throws
{
_ = try await self.requestRaw(method: method, params: params, timeoutMs: timeoutMs)
return try await client.request(method: method, params: params, timeoutMs: timeoutMs)
}
/// Ensure the underlying socket is configured (and replaced if config changed).
@@ -228,13 +114,33 @@ actor GatewayConnection {
}
}
// MARK: - Typed gateway API
private let gatewayControlLogger = Logger(subsystem: "com.steipete.clawdis", category: "gateway.control")
extension GatewayConnection {
private static func wrapParams(_ raw: [String: Any]?) -> [String: AnyCodable]? {
guard let raw else { return nil }
return raw.reduce(into: [String: AnyCodable]()) { acc, pair in
acc[pair.key] = AnyCodable(pair.value)
}
}
func controlRequest(
method: String,
params: [String: Any]? = nil,
timeoutMs: Double? = nil) async throws -> Data
{
try await self.request(method: method, params: Self.wrapParams(params), timeoutMs: timeoutMs)
}
func status() async -> (ok: Bool, error: String?) {
do {
_ = try await self.requestRaw(method: .status)
return (true, nil)
let data = try await self.controlRequest(method: "status")
if let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
(obj["ok"] as? Bool) ?? true
{
return (true, nil)
}
return (false, "status error")
} catch {
return (false, error.localizedDescription)
}
@@ -242,211 +148,39 @@ extension GatewayConnection {
func setHeartbeatsEnabled(_ enabled: Bool) async -> Bool {
do {
try await self.requestVoid(method: .setHeartbeats, params: ["enabled": AnyCodable(enabled)])
_ = try await self.controlRequest(method: "set-heartbeats", params: ["enabled": enabled])
return true
} catch {
gatewayConnectionLogger.error("setHeartbeatsEnabled failed \(error.localizedDescription, privacy: .public)")
gatewayControlLogger.error("setHeartbeatsEnabled failed \(error.localizedDescription, privacy: .public)")
return false
}
}
func sendAgent(_ invocation: GatewayAgentInvocation) async -> (ok: Bool, error: String?) {
let trimmed = invocation.message.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return (false, "message empty") }
var params: [String: AnyCodable] = [
"message": AnyCodable(trimmed),
"sessionKey": AnyCodable(invocation.sessionKey),
"thinking": AnyCodable(invocation.thinking ?? "default"),
"deliver": AnyCodable(invocation.deliver),
"to": AnyCodable(invocation.to ?? ""),
"channel": AnyCodable(invocation.channel.rawValue),
"idempotencyKey": AnyCodable(invocation.idempotencyKey),
]
if let timeout = invocation.timeoutSeconds {
params["timeout"] = AnyCodable(timeout)
}
do {
try await self.requestVoid(method: .agent, params: params)
return (true, nil)
} catch {
return (false, error.localizedDescription)
}
}
func sendAgent(
message: String,
thinking: String?,
sessionKey: String,
deliver: Bool,
to: String?,
channel: GatewayAgentChannel = .last,
timeoutSeconds: Int? = nil,
channel: String? = nil,
idempotencyKey: String = UUID().uuidString) async -> (ok: Bool, error: String?)
{
await self.sendAgent(GatewayAgentInvocation(
message: message,
sessionKey: sessionKey,
thinking: thinking,
deliver: deliver,
to: to,
channel: channel,
timeoutSeconds: timeoutSeconds,
idempotencyKey: idempotencyKey))
}
func sendSystemEvent(_ params: [String: AnyCodable]) async {
let trimmed = message.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return (false, "message empty") }
do {
try await self.requestVoid(method: .systemEvent, params: params)
let params: [String: Any] = [
"message": trimmed,
"sessionKey": sessionKey,
"thinking": thinking ?? "default",
"deliver": deliver,
"to": to ?? "",
"channel": channel ?? "",
"idempotencyKey": idempotencyKey,
]
_ = try await self.controlRequest(method: "agent", params: params)
return (true, nil)
} catch {
// Best-effort only.
return (false, error.localizedDescription)
}
}
// MARK: - Health
func healthSnapshot(timeoutMs: Double? = nil) async throws -> HealthSnapshot {
let data = try await self.requestRaw(method: .health, timeoutMs: timeoutMs)
if let snap = decodeHealthSnapshot(from: data) { return snap }
throw GatewayDecodingError(method: Method.health.rawValue, message: "failed to decode health snapshot")
}
func healthOK(timeoutMs: Int = 8000) async throws -> Bool {
let data = try await self.requestRaw(method: .health, timeoutMs: Double(timeoutMs))
return (try? self.decoder.decode(ClawdisGatewayHealthOK.self, from: data))?.ok ?? true
}
// MARK: - Chat
func chatHistory(sessionKey: String) async throws -> ClawdisChatHistoryPayload {
try await self.requestDecoded(
method: .chatHistory,
params: ["sessionKey": AnyCodable(sessionKey)])
}
func chatSend(
sessionKey: String,
message: String,
thinking: String,
idempotencyKey: String,
attachments: [ClawdisChatAttachmentPayload],
timeoutMs: Int = 30000) async throws -> ClawdisChatSendResponse
{
var params: [String: AnyCodable] = [
"sessionKey": AnyCodable(sessionKey),
"message": AnyCodable(message),
"thinking": AnyCodable(thinking),
"idempotencyKey": AnyCodable(idempotencyKey),
"timeoutMs": AnyCodable(timeoutMs),
]
if !attachments.isEmpty {
let encoded = attachments.map { att in
[
"type": att.type,
"mimeType": att.mimeType,
"fileName": att.fileName,
"content": att.content,
]
}
params["attachments"] = AnyCodable(encoded)
}
return try await self.requestDecoded(method: .chatSend, params: params)
}
func chatAbort(sessionKey: String, runId: String) async throws -> Bool {
struct AbortResponse: Decodable { let ok: Bool?; let aborted: Bool? }
let res: AbortResponse = try await self.requestDecoded(
method: .chatAbort,
params: ["sessionKey": AnyCodable(sessionKey), "runId": AnyCodable(runId)])
return res.aborted ?? false
}
// MARK: - VoiceWake
func voiceWakeGetTriggers() async throws -> [String] {
struct VoiceWakePayload: Decodable { let triggers: [String] }
let payload: VoiceWakePayload = try await self.requestDecoded(method: .voicewakeGet)
return payload.triggers
}
func voiceWakeSetTriggers(_ triggers: [String]) async {
do {
try await self.requestVoid(
method: .voicewakeSet,
params: ["triggers": AnyCodable(triggers)],
timeoutMs: 10000)
} catch {
// Best-effort only.
}
}
// MARK: - Node pairing
func nodePairApprove(requestId: String) async throws {
try await self.requestVoid(
method: .nodePairApprove,
params: ["requestId": AnyCodable(requestId)],
timeoutMs: 10000)
}
func nodePairReject(requestId: String) async throws {
try await self.requestVoid(
method: .nodePairReject,
params: ["requestId": AnyCodable(requestId)],
timeoutMs: 10000)
}
// MARK: - Cron
struct CronSchedulerStatus: Decodable, Sendable {
let enabled: Bool
let storePath: String
let jobs: Int
let nextWakeAtMs: Int?
}
func cronStatus() async throws -> CronSchedulerStatus {
try await self.requestDecoded(method: .cronStatus)
}
func cronList(includeDisabled: Bool = true) async throws -> [CronJob] {
let res: CronListResponse = try await self.requestDecoded(
method: .cronList,
params: ["includeDisabled": AnyCodable(includeDisabled)])
return res.jobs
}
func cronRuns(jobId: String, limit: Int = 200) async throws -> [CronRunLogEntry] {
let res: CronRunsResponse = try await self.requestDecoded(
method: .cronRuns,
params: ["id": AnyCodable(jobId), "limit": AnyCodable(limit)])
return res.entries
}
func cronRun(jobId: String, force: Bool = true) async throws {
try await self.requestVoid(
method: .cronRun,
params: [
"id": AnyCodable(jobId),
"mode": AnyCodable(force ? "force" : "due"),
],
timeoutMs: 20000)
}
func cronRemove(jobId: String) async throws {
try await self.requestVoid(method: .cronRemove, params: ["id": AnyCodable(jobId)])
}
func cronUpdate(jobId: String, patch: [String: AnyCodable]) async throws {
try await self.requestVoid(
method: .cronUpdate,
params: ["id": AnyCodable(jobId), "patch": AnyCodable(patch)])
}
func cronAdd(payload: [String: AnyCodable]) async throws {
try await self.requestVoid(method: .cronAdd, params: payload)
}
}
@@ -1,6 +1,5 @@
import ClawdisIPC
import Foundation
import OSLog
// Lightweight SemVer helper (major.minor.patch only) for gateway compatibility checks.
struct Semver: Comparable, CustomStringConvertible, Sendable {
@@ -62,8 +61,6 @@ struct GatewayCommandResolution {
}
enum GatewayEnvironment {
private static let logger = Logger(subsystem: "com.steipete.clawdis", category: "gateway.env")
static func gatewayPort() -> Int {
let stored = UserDefaults.standard.integer(forKey: "gatewayPort")
return stored > 0 ? stored : 18789
@@ -80,15 +77,6 @@ enum GatewayEnvironment {
}
static func check() -> GatewayEnvironmentStatus {
let start = Date()
defer {
let elapsedMs = Int(Date().timeIntervalSince(start) * 1000)
if elapsedMs > 500 {
self.logger.warning("gateway env check slow (\(elapsedMs, privacy: .public)ms)")
} else {
self.logger.debug("gateway env check ok (\(elapsedMs, privacy: .public)ms)")
}
}
let expected = self.expectedGatewayVersion()
let projectRoot = CommandResolver.projectRoot()
let projectEntrypoint = CommandResolver.gatewayEntrypoint(in: projectRoot)
@@ -147,15 +135,6 @@ enum GatewayEnvironment {
}
static func resolveGatewayCommand() -> GatewayCommandResolution {
let start = Date()
defer {
let elapsedMs = Int(Date().timeIntervalSince(start) * 1000)
if elapsedMs > 500 {
self.logger.warning("gateway command resolve slow (\(elapsedMs, privacy: .public)ms)")
} else {
self.logger.debug("gateway command resolve ok (\(elapsedMs, privacy: .public)ms)")
}
}
let projectRoot = CommandResolver.projectRoot()
let projectEntrypoint = CommandResolver.gatewayEntrypoint(in: projectRoot)
let status = self.check()
@@ -212,7 +191,6 @@ enum GatewayEnvironment {
// MARK: - Internals
private static func readGatewayVersion(binary: String) -> Semver? {
let start = Date()
let process = Process()
process.executableURL = URL(fileURLWithPath: binary)
process.arguments = ["--version"]
@@ -224,21 +202,10 @@ enum GatewayEnvironment {
do {
try process.run()
process.waitUntilExit()
let elapsedMs = Int(Date().timeIntervalSince(start) * 1000)
if elapsedMs > 500 {
self.logger.warning(
"gateway --version slow (\(elapsedMs, privacy: .public)ms) bin=\(binary, privacy: .public)")
} else {
self.logger.debug(
"gateway --version ok (\(elapsedMs, privacy: .public)ms) bin=\(binary, privacy: .public)")
}
let data = pipe.fileHandleForReading.readToEndSafely()
let raw = String(data: data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines)
return Semver.parse(raw)
} catch {
let elapsedMs = Int(Date().timeIntervalSince(start) * 1000)
self.logger.error(
"gateway --version failed (\(elapsedMs, privacy: .public)ms) bin=\(binary, privacy: .public) err=\(error.localizedDescription, privacy: .public)")
return nil
}
}
Loaded 100 of 4240 files, more files were not shown because too many files have changed in this diff. Show more