From 749432fefc9543f2e22db43e262ac748a5fe6791 Mon Sep 17 00:00:00 2001 From: "Raphael Cautus (Maestro)" Date: Fri, 31 Jul 2026 14:57:24 +0100 Subject: [PATCH] feat(brain): garden, mirrors, vault-sync, provider registry consolidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .aurelio/garden/: model + agent garden (Google Cloud entries) - .aurelio/mirrors/: sync-mirrors.yaml + state tracking - .aurelio/skills/gcp/: Google Cloud skill - Consolidation audit + execution plan (2026-07-30) - vault-sync.py: Obsidian → GBrain MCP ingestion daemon - brain-to-gbrain.py: brain → GBrain migration tool - Provider registry + dist mirrors updated - .gitignore: exclude .runner, .mimocode/.cron-lock, drift/target Co-authored-by: Álvaro de Campos --- .aurelio/connectors/registry.yaml | 85 ++- .../garden/agent-garden/google-cloud.yaml | 185 +++++ .../garden/model-garden/google-cloud.yaml | 122 ++++ .aurelio/mcp_config.json | 26 +- .aurelio/mirrors/README.md | 156 +++++ .aurelio/mirrors/state/last-sync.json | 4 + .aurelio/mirrors/sync-mirrors.yaml | 159 +++++ ...-13_aurelio-omnibrain-execution-tracker.md | 654 +++--------------- .../2026-07-30_aurelio-consolidation-audit.md | 295 ++++++++ ...30_aurelio-consolidation-execution-plan.md | 346 +++++++++ .aurelio/providers/dist/dirac.providers.json | 6 +- .aurelio/providers/dist/mcp.providers.json | 17 +- .aurelio/providers/dist/portal.providers.json | 18 +- .aurelio/providers/registry.yaml | 20 +- .aurelio/skills/gcp/SKILL.md | 220 ++++++ .env.example | 39 ++ .gitignore | 3 + ...07-14_160000-music-mvp-aurelio-telegram.md | 240 +++++++ ...07-14_180000-muscriptor-mcp-weekly-midi.md | 578 ++++++++++++++++ AGENTS.md | 113 +++ scripts/brain-to-gbrain.py | 392 +++++++++++ scripts/vault-sync.py | 318 +++++++++ 22 files changed, 3406 insertions(+), 590 deletions(-) create mode 100644 .aurelio/garden/agent-garden/google-cloud.yaml create mode 100644 .aurelio/garden/model-garden/google-cloud.yaml create mode 100644 .aurelio/mirrors/README.md create mode 100644 .aurelio/mirrors/state/last-sync.json create mode 100644 .aurelio/mirrors/sync-mirrors.yaml create mode 100644 .aurelio/plans/2026-07-30_aurelio-consolidation-audit.md create mode 100644 .aurelio/plans/2026-07-30_aurelio-consolidation-execution-plan.md create mode 100644 .aurelio/skills/gcp/SKILL.md create mode 100644 .hermes/plans/2026-07-14_160000-music-mvp-aurelio-telegram.md create mode 100644 .hermes/plans/2026-07-14_180000-muscriptor-mcp-weekly-midi.md create mode 100644 scripts/brain-to-gbrain.py create mode 100644 scripts/vault-sync.py diff --git a/.aurelio/connectors/registry.yaml b/.aurelio/connectors/registry.yaml index b4c1b37e..b1c8f60f 100644 --- a/.aurelio/connectors/registry.yaml +++ b/.aurelio/connectors/registry.yaml @@ -51,16 +51,19 @@ connectors: notes: DEFAULT forge (self-hosted first policy). LAN origin 192.168.0.9:3001. CLI = tea. Tokens in Vaultwarden. - id: github - label: GitHub + label: GitHub — upstream mirror source category: devops kind: api-key auth: api_key endpoint: https://api.github.com surfaces: [mcp, theia] status: declared - scopes: [] + scopes: [repo, read:org] host: null - notes: Only for public mirrors/upstreams — Forgejo is the default forge. CLI = gh. + notes: | + Upstream source for the mirror pipeline (GitHub → Forgejo) and public mirrors. + Forgejo is the default forge. PAT (classic, repo+read:org scope) in GITHUB_MIRROR_TOKEN. + Configured in .aurelio/mirrors/sync-mirrors.yaml under targets.forgejo. CLI = gh. - id: kubernetes label: Kubernetes @@ -141,11 +144,35 @@ connectors: kind: oauth auth: oauth2 endpoint: https://cloudresourcemanager.googleapis.com - surfaces: [mcp, theia] - status: declared - scopes: [cloud-platform] + surfaces: [mcp, theia, portal] + status: provisioned + scopes: + - cloud-platform + - https://www.googleapis.com/auth/gmail.readonly + - https://www.googleapis.com/auth/calendar.readonly + - https://www.googleapis.com/auth/userinfo.profile + - https://www.googleapis.com/auth/contacts.readonly + - https://www.googleapis.com/auth/cloud-platform.read-only host: null - notes: CLI = gcloud. No real self-hosted equivalent; firebase emulators / fake-gcs-server for dev. + notes: | + PF default Google Cloud project civil-rarity-497813-r6 (€25k credits). + Vertex AI Model Garden + Gemini APIs enabled; OAuth client configured for Workspace research ingestion. + CLI = gcloud. Service-account fallback via GOOGLE_APPLICATION_CREDENTIALS for batch/background jobs. + Self-hosted dev: firebase emulators / fake-gcs-server only; no full GCP emulator. + + - id: google-cloud-vertex + label: Google Cloud Vertex AI (Model Garden) + category: cloud + kind: api-key + auth: api_key + endpoint: https://us-central1-aiplatform.googleapis.com + surfaces: [mcp, theia, portal] + status: provisioned + scopes: [] + host: null + notes: | + Vertex AI enterprise inference endpoint. Uses GOOGLE_CLOUD_API_KEY or service-account. + See .aurelio/garden/model-garden/google-cloud.yaml for model IDs and context windows. - id: azure label: Microsoft Azure @@ -234,16 +261,50 @@ connectors: notes: Entra ID app registration required; exposes Outlook/OneDrive/Calendar/Teams via Graph. - id: atlassian - label: Atlassian (Jira + Confluence) + label: Atlassian (Jira + Confluence) — upstream mirror source category: saas - kind: oauth - auth: oauth2 + kind: api-key + auth: api_key endpoint: https://api.atlassian.com surfaces: [mcp, theia] status: declared - scopes: [read:jira-work, write:jira-work, read:confluence-content.all, offline_access] + scopes: [read:jira-work, read:confluence-content.all, offline_access] host: null - notes: Atlassian OAuth 2.0 (3LO) app; Jira REST v3 + Confluence REST v2. + notes: | + Uppstream source for the mirror pipeline (Jira → Plane, Confluence → Outline). + Uses API-token auth (email + token), not OAuth, for server-side sync. + Configured in .aurelio/mirrors/sync-mirrors.yaml under targets.plane/outline. + Fetch tokens from Vaultwarden; env vars JIRA_API_TOKEN/JIRA_USER_EMAIL and + CONFLUENCE_API_TOKEN/CONFLUENCE_USER_EMAIL. + + # ── Self-hosted mirrors (receive upstream via sync-mirrors.py) ──────────── + - id: plane + label: Plane (issue tracker — Jira mirror) + category: devops + kind: api-key + auth: api_key + endpoint: https://plane.portugalfuturista.org + surfaces: [mcp, theia, portal] + status: live + scopes: [] + host: null + notes: | + Self-hosted Plane. Mirror target for Jira issues via scripts/sync-mirrors.py --sync plane. + Workspace: savearth. Token env: PLANE_API_TOKEN. REST API under /api/v1/. + + - id: outline + label: Outline (docs — Confluence mirror) + category: devops + kind: api-key + auth: api_key + endpoint: https://docs.portugalfuturista.org + surfaces: [mcp, theia, portal] + status: live + scopes: [] + host: null + notes: | + Self-hosted Outline. Mirror target for Confluence pages via scripts/sync-mirrors.py --sync outline. + Collection: "Savearth Mirror". Token env: OUTLINE_API_TOKEN. REST API under /api/. # ── Creative / CAD hosts (sidecar bridges, NOT VSIXs) ───────────────────── - id: solidworks diff --git a/.aurelio/garden/agent-garden/google-cloud.yaml b/.aurelio/garden/agent-garden/google-cloud.yaml new file mode 100644 index 00000000..d7e7649b --- /dev/null +++ b/.aurelio/garden/agent-garden/google-cloud.yaml @@ -0,0 +1,185 @@ +# PF Agent Garden — Google Cloud +# +# Agents and agentic services that run on GCP or call Google Cloud / Workspace APIs. +# Distinct from the generic `gemini` chat provider. + +platform: google-cloud +realm_default: europe-west1 +workload_identity: required + +agents: + - id: gcp-deployer + name: GCP Deployer + description: Provisions and mutates GCP resources (GKE, Cloud Run, Cloud Functions, Cloud Storage, Pub/Sub, BigQuery, Cloud SQL, Secret Manager) from declarative specs. + kind: infrastructure + surfaces: + - dirac + - portal + triggers: + - user-command + - git-push + inputs: + - terraform + - gcloud-yaml + - openapi + outputs: + - deployed-url + - resource-manifest + required_apis: + - deploymentmanager.googleapis.com + - cloudbuild.googleapis.com + - run.googleapis.com + - container.googleapis.com + + - id: vertex-coder + name: Vertex Coder + description: Coding agent backed by Vertex AI Gemini. Reads/writes code in a GKE/Cloud Run sandbox and streams results through Dirac / MCP. + kind: coding + surfaces: + - dirac + - mcp + model: gemini-2.5-pro-preview-06-05 + runtime: cloud-run + triggers: + - dirac-task + - mcp-tool-call + inputs: + - repository + - prompt + outputs: + - diff + - test-results + + - id: gcp-cost-guardian + name: GCP Cost Guardian + description: Monitors billing, budgets, and resource labels; alerts on anomalies and suggests rightsizing. + kind: observability + surfaces: + - portal + - mcp + triggers: + - schedule + - billing-alert + inputs: + - billing-export + - resource-labels + outputs: + - alert + - recommendation + required_roles: + - roles/billing.viewer + - roles/monitoring.viewer + + - id: workspace-delegator + name: Workspace Delegator + description: Calls Google Workspace APIs (Directory, Gmail, Calendar, Drive) via domain-wide delegation. Useful for tenant automation and mail/calendar agents. + kind: integration + surfaces: + - mcp + - portal + triggers: + - user-command + - workflow + inputs: + - workspace-scope + - service-account + outputs: + - api-result + - event + requires_domain_wide_delegation: true + scopes: + - https://www.googleapis.com/auth/admin.directory.user.readonly + - https://www.googleapis.com/auth/calendar + - https://www.googleapis.com/auth/gmail.modify + - https://www.googleapis.com/auth/drive + + - id: pubsub-agent-router + name: Pub/Sub Agent Router + description: Receives events from Pub/Sub and dispatches them to other PF agents running on Cloud Run, GKE, or via MCP. + kind: orchestration + surfaces: + - mcp + - portal + triggers: + - pubsub-message + inputs: + - pubsub-payload + - routing-key + outputs: + - agent-invocation + required_apis: + - pubsub.googleapis.com + - run.googleapis.com + + - id: bq-memory-analyst + name: BigQuery Memory Analyst + description: Queries structured agent memory and telemetry stored in BigQuery; returns synthesized answers with citations. + kind: memory + surfaces: + - mcp + - portal + model: gemini-2.5-flash-preview-06-05 + triggers: + - user-query + - schedule + inputs: + - bigquery-dataset + - sql-question + outputs: + - sql-query + - result-summary + required_roles: + - roles/bigquery.dataViewer + - roles/bigquery.jobUser + + - id: gcs-artifact-librarian + name: GCS Artifact Librarian + description: Indexes and retrieves artifacts from Cloud Storage buckets; supports code artifacts, documents, and media. + kind: memory + surfaces: + - mcp + - portal + triggers: + - user-query + - event + inputs: + - bucket-prefix + - metadata + outputs: + - signed-url + - object-list + required_roles: + - roles/storage.objectViewer + + - id: secret-bootstrapper + name: Secret Bootstrapper + description: Loads runtime secrets from Secret Manager into agent workloads; never logs values and rotates on schedule. + kind: security + surfaces: + - mcp + - dirac + triggers: + - deployment + - rotation-schedule + inputs: + - secret-name + - workload-identity + outputs: + - mounted-secret + required_apis: + - secretmanager.googleapis.com + +runtimes: + - name: cloud-run + use_for: stateless-http + scaling: request-driven + - name: gke-autopilot + use_for: long-running-workloads + scaling: workload-driven + - name: cloud-functions + use_for: event-driven-functions + +notes: + - All agents default to `europe-west1` unless EU residency or latency requires otherwise. + - Production agents must use Workload Identity or dedicated service accounts; no downloaded keys. + - Agents calling Workspace APIs require domain-wide delegation registered in the Google Admin console. diff --git a/.aurelio/garden/model-garden/google-cloud.yaml b/.aurelio/garden/model-garden/google-cloud.yaml new file mode 100644 index 00000000..cefb8bb9 --- /dev/null +++ b/.aurelio/garden/model-garden/google-cloud.yaml @@ -0,0 +1,122 @@ +# PF Model Garden — Google Cloud (Vertex AI) +# +# Enterprise Google Cloud models exposed through the Aurelio ecosystem. +# These are distinct from the consumer Gemini (AI Studio) provider. + +provider: google-cloud +label: Google Cloud Vertex AI +protocol: gemini +auth: + kind: service_account + env: GOOGLE_APPLICATION_CREDENTIALS + fallback_env: GOOGLE_CLOUD_API_KEY + region_default: europe-west1 + +models: + - id: gemini-2.5-pro-preview-06-05 + name: Gemini 2.5 Pro + kind: chat + publisher: google + context_window: 1000000 + surfaces: + - dirac + - mcp + - portal + pricing_proxy: + input_per_1m_tokens: 1.25 + output_per_1m_tokens: 10.00 + currency: USD + notes: Approximate; region and batch mode affect rate. Check cloud.google.com/vertex-ai/generative-ai/pricing. + + - id: gemini-2.5-flash-preview-06-05 + name: Gemini 2.5 Flash + kind: chat + publisher: google + context_window: 1000000 + surfaces: + - dirac + - mcp + - portal + pricing_proxy: + input_per_1m_tokens: 0.15 + output_per_1m_tokens: 0.60 + currency: USD + notes: Fast, low-cost; good for agentic loops and classification. + + - id: gemini-2.0-flash-001 + name: Gemini 2.0 Flash + kind: chat + publisher: google + context_window: 1000000 + surfaces: + - dirac + - mcp + - portal + pricing_proxy: + input_per_1m_tokens: 0.10 + output_per_1m_tokens: 0.40 + currency: USD + notes: Stable GA model. + + - id: imagen-3-0-generate-001 + name: Imagen 3 + kind: image + publisher: google + context_window: null + surfaces: + - mcp + - portal + pricing_proxy: + per_image: 0.04 + currency: USD + notes: Per image at 1024x1024; higher resolutions cost more. + + - id: veo-2-0-generate-001 + name: Veo 2 + kind: video + publisher: google + context_window: null + surfaces: + - mcp + - portal + pricing_proxy: + per_second: 0.05 + currency: USD + notes: Per generated video second; minimum length applies. + + - id: text-embedding-004 + name: Text Embedding 004 + kind: embedding + publisher: google + context_window: null + dimensions: 2048 + surfaces: + - mcp + - portal + pricing_proxy: + per_1m_tokens: 0.10 + currency: USD + notes: Input-only embedding model. + + - id: multimodalembedding@001 + name: Multimodal Embedding + kind: embedding + publisher: google + context_window: null + dimensions: [128, 256, 512, 1408] + surfaces: + - mcp + - portal + pricing_proxy: + per_1m_tokens: 0.20 + currency: USD + notes: Mixed image+text embeddings; dimensions selectable. + +endpoints: + chat_template: "https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{region}/publishers/google/models/{model_id}:generateContent" + stream_template: "https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{region}/publishers/google/models/{model_id}:streamGenerateContent" + +notes: + - Model availability and pricing vary by region; verify before production use. + - Use service-account auth in production; API-key fallback is for quick tests only. + - The separate `gemini` provider in .aurelio/providers/registry.yaml targets AI Studio (consumer), not Vertex AI. diff --git a/.aurelio/mcp_config.json b/.aurelio/mcp_config.json index cc55aed4..17fb4925 100644 --- a/.aurelio/mcp_config.json +++ b/.aurelio/mcp_config.json @@ -148,10 +148,32 @@ "mcp-remote", "https://stitch.googleapis.com/mcp", "--header", - "X-Goog-Api-Key: AQ.Ab8RN6KphA2R_SVCtr5-Sy1PAjPzV6lZnLecLdHjQbpdo7nGVA" + "X-Goog-Api-Key: AQ.Ab8...GVA" ], "env": {}, "_disabled": false + }, + "muscriptor-mcp": { + "command": "npx", + "args": [ + "-y", + "mcp-remote", + "http://192.168.0.104:8088/sse" + ], + "_disabled": false, + "_note": "Music transcription MCP — audio → MIDI via MuScriptor. Runs on Gigabyte (GPU) port 8088. [LAN URL]", + "disabledTools": [] + }, + "music-mcp": { + "command": "npx", + "args": [ + "-y", + "mcp-remote", + "http://192.168.0.104:8089/sse" + ], + "_disabled": false, + "_note": "Music stack MCP — Navidrome + Lidarr + Last.fm + MusicBrainz + MuScriptor. Debug mode, daily brief, MIDI conversion. Runs on Gigabyte port 8089. [LAN URL]", + "disabledTools": [] } } -} +} \ No newline at end of file diff --git a/.aurelio/mirrors/README.md b/.aurelio/mirrors/README.md new file mode 100644 index 00000000..d340bfbc --- /dev/null +++ b/.aurelio/mirrors/README.md @@ -0,0 +1,156 @@ +# Mirror Pipeline — Upstream Replication into Self-Hosted Mirrors + +Constant, one-way replication of Savearth's upstream sources into Portugal +Futurista's self-hosted mirrors. The agent fleet operates on the mirrors; the +originals stay authoritative. + +``` + UPSTREAM (source of truth) MIRROR (agents work here) + ────────────────────────── ───────────────────────── + GitHub (SavearthTech/*) ──────► Forgejo code.portugalfuturista.org + Jira Cloud ──────► Plane plane.portugalfuturista.org + Confluence Cloud ──────► Outline docs.portugalfuturista.org + │ + sync outcomes + ▼ + trajectory-rewards/mirror-sync.jsonl + │ + ▼ + GRPO/DPO RL pipeline (rl-agent-trajectory-training) +``` + +## Architecture + +Single source of truth, matching the repo's established provider/connector +pattern: + +| File | Role | +|------|------| +| `.aurelio/mirrors/sync-mirrors.yaml` | **Edit this** — declares sources, targets, auth refs | +| `.aurelio/mirrors/dist/mirrors.catalog.json` | **Generated** — catalog for downstream surfaces | +| `.aurelio/mirrors/state/last-sync.json` | **Generated** — last sync summary (for drift detection) | +| `scripts/sync-mirrors.py` | The engine: three sync modes | +| `.forgejo/workflows/mirror-sync.yml` | CI: runs every 15 min | +| `.forgejo/workflows/ci.yml` | CI: `--check` guard on every push | + +## Usage + +### 1. Provision tokens + +All tokens are API tokens/PATs. Fetch from Vaultwarden, export as env vars: + +```bash +export FORGEJO_MIRROR_TOKEN=... # Forgejo admin (manage_repo, write:org) +export GITHUB_MIRROR_TOKEN=... # GitHub PAT classic (repo, read:org) +export PLANE_API_TOKEN=... # Plane personal API token +export OUTLINE_API_TOKEN=... # Outline API token (read+write) +export JIRA_API_TOKEN=... # Atlassian API token +export JIRA_USER_EMAIL=... # Atlassian account email +export CONFLUENCE_API_TOKEN=... # Same Atlassian account +export CONFLUENCE_USER_EMAIL=... +``` + +See `.env.example` for generation instructions per service. + +### 2. Configure upstream endpoints + +Edit `.aurelio/mirrors/sync-mirrors.yaml`: + +```yaml +targets: + plane: + sources: + - id: savearth-jira + endpoint: https://savearth.atlassian.net # ← set this + projects: [SAVE, IOT, HW] # ← or null for all +``` + +The Jira and Confluence `endpoint` fields are `null` by default — the pipeline +skips them until you set the real URL. + +### 3. Run the sync + +```bash +# One-time / manual: +python3 scripts/sync-mirrors.py --sync # all targets +python3 scripts/sync-mirrors.py --sync forgejo # one target +python3 scripts/sync-mirrors.py --sync --dry-run # preview + +# Regenerate catalog after editing the YAML: +python3 scripts/sync-mirrors.py --write-in-place + +# CI guard: +python3 scripts/sync-mirrors.py --check # exit 1 if dist stale +``` + +### 4. Constant sync (CI) + +`.forgejo/workflows/mirror-sync.yml` runs every 15 minutes. Tokens must be set +as Forgejo repository secrets: + +``` +Settings → Secrets → Actions → New Secret: + FORGEJO_MIRROR_TOKEN, GITHUB_MIRROR_TOKEN, PLANE_API_TOKEN, + OUTLINE_API_TOKEN, JIRA_API_TOKEN, JIRA_USER_EMAIL, + CONFLUENCE_API_TOKEN, CONFLUENCE_USER_EMAIL +``` + +The workflow commits `last-sync.json` + reward signals back to the repo. + +## What each mirror does + +### GitHub → Forgejo (code) + +- Lists all repos in the `SavearthTech` org (+ personal matching repos). +- Creates a Forgejo migration with `mirror: true` — Forgejo polls upstream + on its own schedule (`native_interval: 8h`) after the initial clone. +- Skips forks and archived repos by default. +- Preserves private status, description, topics→labels. + +### Jira → Plane (issues) + +- Lists Jira projects, creates matching Plane projects (by key/name). +- Imports issues with field mapping (summary→name, description→description_html, + priority, status, assignee, labels). +- Converts Atlassian Document Format (ADF) to HTML for Plane. +- Idempotent: skips issues that already exist by name match. + +### Confluence → Outline (docs) + +- Lists Confluence spaces, creates an Outline collection ("Savearth Mirror"). +- Imports pages with `[SPACE_KEY] title` naming. +- Converts Confluence storage format (XHTML) to Markdown. +- Handles headings, code blocks, lists, links, bold/italic. +- Deduplicates by title within the collection. + +## RL feedback loop + +Each sync run emits reward signals to +`.aurelio/brain/trajectory-rewards/mirror-sync.jsonl`: + +| Signal | Reward | When | +|--------|--------|------| +| `sync_hit` | +0.1 | Target synced cleanly (no errors) | +| `sync_miss` | -0.2 | Target had sync errors (drift/staleness) | +| `upstream_overwrite` | -0.5 | Agent's mirror change was overwritten by upstream sync | + +These feed into the GRPO/DPO pipeline (see `rl-agent-trajectory-training` +skill). The reward shaping is configured in `sync-mirrors.yaml` under +`rl_feedback.signals`. + +## Direction: read-only mirror + +Upstream is the source of truth. Agents work on the Forgejo/Plane/Outline +copies. Changes do NOT propagate back to GitHub/Jira/Confluence. This is by +design — the pipeline is structured so bidirectional write-back can be added +per-target later without rewriting the engine. + +## Troubleshooting + +| Symptom | Fix | +|---------|-----| +| `missing env FORGEJO_MIRROR_TOKEN` | Export the token (see step 1) | +| `Jira endpoint is null` | Set `endpoint:` in `sync-mirrors.yaml` | +| `0 repos visible` from GitHub | Token lacks `read:org` or repos are private — check PAT scopes | +| Plane 401/403 | Regenerate API token in Plane settings | +| Outline OAuth error | Outline needs `ENABLE_LOCAL_AUTH=true` or Authentik OIDC (see fleet/identity.md) | diff --git a/.aurelio/mirrors/state/last-sync.json b/.aurelio/mirrors/state/last-sync.json new file mode 100644 index 00000000..2a5b2134 --- /dev/null +++ b/.aurelio/mirrors/state/last-sync.json @@ -0,0 +1,4 @@ +{ + "timestamp": "2026-07-30T22:58:11.084761+00:00", + "summaries": [] +} \ No newline at end of file diff --git a/.aurelio/mirrors/sync-mirrors.yaml b/.aurelio/mirrors/sync-mirrors.yaml new file mode 100644 index 00000000..6bbd6fe0 --- /dev/null +++ b/.aurelio/mirrors/sync-mirrors.yaml @@ -0,0 +1,159 @@ +# Mirror Pipeline — single source of truth +# +# Declares how upstream sources (GitHub, Jira, Confluence) replicate into the +# self-hosted mirrors (Forgejo, Plane, Outline). All downstream surfaces derive +# from this file via scripts/sync-mirrors.py. +# +# ONE place to edit. mirrors/dist/*.json are GENERATED (do not edit). +# +# Direction: upstream → mirror (read-only by default; write-back is a future +# toggle per-target). The agent fleet operates on the mirrors; the originals +# stay authoritative. +# +# Regenerate after editing: +# python3 scripts/sync-mirrors.py --write-in-place # regenerate dist/ +# python3 scripts/sync-mirrors.py --check # CI guard: exit 1 if stale +# python3 scripts/sync-mirrors.py --sync # run all mirror syncs +# python3 scripts/sync-mirrors.py --sync forgejo # one target +# +# Auth: tokens are NEVER in this file. They live in the environment (.env or +# shell env), referenced by the `token_env` key. Fetch from Vaultwarden. + +version: 1 + +# ── Global defaults ──────────────────────────────────────────────────────── +defaults: + # Sync cadence (minutes). Overridden by cron job schedule. + interval_minutes: 15 + # Fail the whole run if any single source errors? false = log + continue. + fail_fast: false + # User-agent for upstream API calls. + user_agent: "replica-omnisciente-mirror/1.0" + +# ── Mirror targets ───────────────────────────────────────────────────────── +# Each target is a self-hosted service that RECEIVES data from one or more +# upstream sources. A target may have multiple sources (e.g. Forgejo mirrors +# both GitHub orgs and individual repos). + +targets: + + # ── Forgejo: code mirror (GitHub → Forgejo) ──────────────────────────── + forgejo: + kind: git-mirror + label: "Forgejo (code mirror)" + endpoint: https://code.portugalfuturista.org + # Forgejo API token (admin or org-owner scope). ENV var name only. + token_env: FORGEJO_MIRROR_TOKEN + # Default owner/org for mirrored repos if the upstream org doesn't exist. + default_owner: SavearthTech + # Mirror interval in Forgejo's own scheduler (for native push mirrors). + native_interval: 8h + sources: + - id: savearthtech-github + label: "SavearthTech GitHub org" + type: github-org + # GitHub org name to mirror ALL public+private repos from. + org: SavearthTech + # GitHub PAT (classic, read scope on repo + org). ENV var name only. + token_env: GITHUB_MIRROR_TOKEN + # Clone over SSH for private repos (key must be in agent). + clone_transport: https + # Include forked repos? false = skip forks. + include_forks: false + # Map GitHub topics → Forgejo labels (optional). + topic_labels: true + status: declared + + - id: savearth-personal-github + label: "Personal Savearth repos (fabiorafaelcoutada)" + type: github-user + user: fabiorafaelcoutada + token_env: GITHUB_MIRROR_TOKEN + # Only mirror repos matching these patterns (glob). + include_patterns: + - "savearth*" + - "aws-iot-core*" + - "hardware-devices*" + - "agent-firmware*" + exclude_patterns: + - "*.github.io" + clone_transport: https + status: declared + + # ── Plane: issue tracker mirror (Jira → Plane) ───────────────────────── + plane: + kind: issue-mirror + label: "Plane (issue tracker mirror)" + endpoint: https://plane.portugalfuturista.org + token_env: PLANE_API_TOKEN + # Plane workspace slug (in the URL). Create if it doesn't exist. + workspace_slug: savearth + sources: + - id: savearth-jira + label: "Savearth Jira Cloud" + type: jira-cloud + # Jira Cloud base URL (e.g. https://savearth.atlassian.net). + # Leave null if Savearth doesn't use Jira Cloud — the pipeline will + # skip this source and report it. + endpoint: null + token_env: JIRA_API_TOKEN + # Jira user email (for API token auth). ENV var name only. + email_env: JIRA_USER_EMAIL + # Jira projects to mirror (keys, e.g. SAVE, IOT, HW). null = all. + projects: null + # Map Jira → Plane fields. Plane uses: name, description_html, + # state (group), priority (urgency), assignee, labels, cycle. + field_map: + summary: name + description: description_html + status: state + priority: priority + assignee: assignee + labels: labels + issuetype: type + status: declared + + # ── Outline: documentation mirror (Confluence → Outline) ─────────────── + outline: + kind: doc-mirror + label: "Outline (documentation mirror)" + endpoint: https://docs.portugalfuturista.org + token_env: OUTLINE_API_TOKEN + # Outline collection to mirror into (created if absent). + collection_name: Savearth Mirror + sources: + - id: savearth-confluence + label: "Savearth Confluence Cloud" + type: confluence-cloud + # Confluence Cloud base URL (e.g. https://savearth.atlassian.net/wiki). + endpoint: null + token_env: CONFLUENCE_API_TOKEN + email_env: CONFLUENCE_USER_EMAIL + # Space keys to mirror (e.g. SAVE, ENG, HW). null = all. + spaces: null + # Convert Confluence storage format → Markdown for Outline. + format: markdown + # Include attachments? (binary download + re-upload to Outline). + include_attachments: true + status: declared + +# ── RL feedback loop ────────────────────────────────────────────────────── +# Mirror sync outcomes feed into the RL trajectory pipeline as reward signals. +# When an agent operates on a mirror and the sync detects drift (upstream +# changed but mirror lagged, or an agent's change diverged from upstream), +# that becomes a negative reward signal for the agent's trajectory. +# +# See scripts/sync-mirrors.py --emit-trajectory-rewards and the +# rl-agent-trajectory-training skill for the consumption side. +rl_feedback: + enabled: true + # Where to write reward signals (trajectory.jsonl format, appended). + output: .aurelio/brain/trajectory-rewards/mirror-sync.jsonl + # Reward shaping: how much each signal type contributes. + signals: + # Agent worked on a mirror that was in sync with upstream (positive). + sync_hit: 0.1 + # Agent worked on a mirror that had drifted from upstream (negative). + sync_miss: -0.2 + # Agent's mirror change was overwritten by an upstream sync (negative). + upstream_overwrite: -0.5 diff --git a/.aurelio/plans/2026-07-13_aurelio-omnibrain-execution-tracker.md b/.aurelio/plans/2026-07-13_aurelio-omnibrain-execution-tracker.md index 65be6a91..1e258fcb 100644 --- a/.aurelio/plans/2026-07-13_aurelio-omnibrain-execution-tracker.md +++ b/.aurelio/plans/2026-07-13_aurelio-omnibrain-execution-tracker.md @@ -1,576 +1,86 @@ -# Aurelio Omnibrain — Execution Tracker +# Aurelio OmniBrain — Execution Tracker -> **Generated:** 2026-07-13 · **Source:** `~/.hermes/plans/2026-07-12_aurelio-omnibrain-master-plan.md` -> **Status:** 🔴 NOT STARTED — zero execution as of today. +**Date:** 2026-07-13 +**Status:** Draft (scaffold — confirm objectives & priorities) +**Owner:** Aurélio / Portugal Futurista +**Scope:** `replica-omnisciente` orchestration brain across realms, wired to +`mycelium` (XSD→codegen), `aurelio-theia` (IDE), `universalisos`, and `gbrain`. + +> Derived from `mycelium/.aurelio/plans/2026-07-12_portugal-futurista-12month-master-plan.md` +> and `2026-07-12_mycelium-agent-blocks-web-presence.md`. This file is a +> **starting scaffold** — replace the task list with the real program once +> confirmed. --- -## 1. Vision - -Import session history, plans, memory, and artifacts from **every coding-agent CLI/TUI/IDE** on the market into a unified **Aurelio GBrain** engine, and expose that brain as the **single knowledge backend** for the entire Aurélio fleet: - -| Surface | How it talks to GBrain | -|---------|----------------------| -| **Theia IDE** | HTTP via `GBrainClient` (Node.js backend) | -| **VS Code extension** | HTTP via `GBrainClient` (TypeScript) | -| **TUI / CLI** | HTTP via `GBrainClient` (Node.js) | -| **Hermes Agent** | MCP server (stdio) or HTTP | - -**North star:** Every coding session you've ever had — Kimi, Claude, Hermes, OpenCode, Copilot, Gemini, ZAI, ZCode, MiMo, Qwen, Antigravity — searchable from one place, available in every IDE. - ---- - -## 2. Current State (Verified 2026-07-13) - -### Brain Store -``` -~/workspace/replica-omnisciente/.aurelio/brain/ -``` -| Source | Sessions | Status | -|--------|----------|--------| -| **kimi** | 539 | ✅ Imported | -| **claude** | 127 | ✅ Imported | -| **antigravity** | 3 | ✅ Imported | -| **mimocode** | 2 | ✅ Imported | -| **hermes** | 1 | ✅ Imported | -| **legacy UUID** | ~127 | ⚠️ Pre-dating the importer (no `session--` prefix) | -| **kimi-code** | 0 | ❌ No adapter | -| **antigravity-ide** | 0 | ❌ No adapter | -| **gemini** | 0 | ❌ No adapter | -| **copilot** | 0 | ❌ No adapter | -| **opencode** | 0 | ❌ No adapter | -| **zai** | 0 | ❌ No adapter | -| **zcode** | 0 | ❌ No adapter | -| **qwen-code** | 0 | ❌ Not imported yet (adapter exists) | -| **Total** | **~800** | | - -### Code Components - -| Component | Path | Reality | -|-----------|------|---------| -| **Agent importer** | `scripts/sync-agents-to-brain.py` | ✅ Exists. Monolithic `adapters.py` (not yet split into package). 6 adapters. Idempotent. | -| **Adapters** | `scripts/agent_importers/adapters.py` | 🟡 Single file, not a package. Needs refactor before adding 5+ new adapters. | -| **Normalized contract** | `scripts/agent_importers/engine.py` | ✅ `NormalizedSession` / `NormalizedMessage` dataclasses. | -| **GBrain engine** | `aurelio-gbrain/` | 🔴 **Empty directory.** No code. | -| **Theia extension** | `aurelio-theia/AurelioTheia/` | 🔴 **Thin scaffold.** `aureilio-backend-module.ts` (hardcoded endpoints), `aurelio-frontend-module.ts` (50+ `console.log` stubs). No widgets. | -| **Gabinete Hub** | CT 208 (192.168.0.38) | 🟡 Exists as deployment target. No GBrain integration. | -| **Provider registry** | `.aurelio/providers/registry.yaml` | ✅ Covers 17+ providers. | - -### Source Directory Map - -| # | Agent / CLI / TUI | Source Directory | Adapter? | Sessions | -|---|-------------------|------------------|----------|----------| -| 1 | Kimi | `~/.kimi/sessions/` | ✅ | 539 | -| 2 | Kimi Code | `~/.kimi-code/` | ❌ NEW | ? | -| 3 | Hermes | `~/.hermes/sessions/` | ✅ | 1 | -| 4 | Claude Code | `~/.claude/projects/` | ✅ | 127 | -| 5 | Antigravity IDE | `~/.antigravity-ide/` | ❌ NEW | ? | -| 6 | Antigravity CLI | `~/.gemini/antigravity-cli/` | ✅ | 3 | -| 7 | Gemini | `~/.gemini/` | ❌ NEW | ? | -| 8 | Copilot | `~/.copilot/` | ❌ NEW | ? | -| 9 | OpenCode | `~/.opencode/` | ❌ NEW | ? | -| 10 | ZAI | `~/.zai/` | ❌ NEW | ? | -| 11 | ZCode | `~/.zcode/` | ❌ NEW | ? | -| 12 | MiMo Code | `~/.mimocode/` | ✅ | 2 | -| 13 | Qwen Code | `~/.qwen-code/` | ✅ | 0 | - ---- - -## 3. Architecture - -``` - ┌─────────────────────────────────────────────┐ - │ AURELIO FLEET │ - │ │ - │ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ - │ │ Theia │ │ VS Code │ │ TUI / │ │ - │ │ IDE │ │ ext. │ │ CLI │ │ - │ └────┬────┘ └────┬────┘ └────┬────┘ │ - │ │ │ │ │ - │ └───────────┼────────────┘ │ - │ │ HTTP │ - │ ▼ │ - │ ┌────────────────────────────────────┐ │ - │ │ AURELIO GBRAIN ENGINE │ │ - │ │ ┌──────────┐ ┌────────────────┐ │ │ - │ │ │ SQLite │ │ HTTP API │ │ │ - │ │ │ FTS5 │ │ /api/brain/* │ │ │ - │ │ │ Index │ │ (Express) │ │ │ - │ │ └──────────┘ └────────────────┘ │ │ - │ │ ┌──────────┐ ┌────────────────┐ │ │ - │ │ │ Dream │ │ MCP Server │ │ │ - │ │ │ Cycle │ │ (stdio) │ │ │ - │ │ └──────────┘ └────────────────┘ │ │ - │ └────────────────────────────────────┘ │ - │ ▲ │ - │ │ │ - │ ┌────────────────────────────────────┐ │ - │ │ AGENT IMPORTERS │ │ - │ │ sync-agents-to-brain.py │ │ - │ │ ┌─────┐ ┌──────┐ ┌───────┐ │ │ - │ │ │ kimi│ │claude│ │opencode│ ... │ │ - │ │ │code │ │ code │ │ │ │ │ - │ │ └─────┘ └──────┘ └───────┘ │ │ - │ └────────────────────────────────────┘ │ - │ ▲ │ - │ │ │ - └───────────────────┼────────────────────────┘ - │ - ┌───────────────────┼────────────────────────┐ - │ SOURCE DIRECTORIES (13 ecosystems) │ - │ ~/.kimi/ ~/.claude/ ~/.hermes/ │ - │ ~/.opencode/ ~/.copilot/ ~/.gemini/ │ - │ ~/.zai/ ~/.zcode/ ~/.mimocode/ ... │ - └────────────────────────────────────────────┘ -``` - -### Data Flow - -``` -Agent ecosystems on disk - │ - ▼ - Importers (Python) ← Normalize to NormalizedSession/Message - │ - ▼ - Brain Store (filesystem) ← session--/session.jsonl - │ - ▼ - GBrain Engine (TypeScript) ← Ingest, index (SQLite FTS5), serve - │ - ├─ HTTP API (/api/brain/*) → Theia, VS Code, TUI, CLI - └─ MCP Server (stdio) → Hermes Agent -``` - ---- - -## 4. Tech Stack - -| Layer | Technology | Version | -|-------|-----------|---------| -| **Importers** | Python | 3.13 | -| **GBrain Engine** | TypeScript + Node.js | TS 5.3, Node 20 | -| **Database** | SQLite + FTS5 | built-in | -| **HTTP Server** | Express | latest | -| **CLI Framework** | Commander | latest | -| **IDE** | Eclipse Theia | 1.73.1 | -| **Frontend** | React | 19 | -| **Agent Protocol** | MCP SDK | @modelcontextprotocol/sdk | -| **HTTP Client** | Axios | latest | - ---- - -## 5. Execution Plan - -### Track A — Agent Importers (Sessions 1–4) - -**Goal:** Cover all 13 source directories with working adapters. - -#### Phase A1: Refactor adapters into a package -**Session:** 1 · **Status:** 🔴 NOT STARTED - -| Step | Description | Files | -|------|-------------|-------| -| 1 | Create `adapters/base.py` with shared helpers (`_load_json`, `_load_jsonl`, `_file_times`, `HOME`) | NEW: `adapters/base.py` | -| 2 | Move each existing adapter into its own module | NEW: `adapters/kimi.py`, `hermes.py`, `claude_code.py`, `antigravity.py`, `qwen_code.py`, `mimocode.py` | -| 3 | Update `__init__.py` to re-export all `discover_*` functions | MODIFY: `adapters/__init__.py` | -| 4 | Regression test: `python3 scripts/sync-agents-to-brain.py --dry-run --summary` — same counts as before | VERIFY | -| 5 | Commit | | - -**Exit gate:** Same session counts as monolithic `adapters.py`. No regressions. - -#### Phase A2: Add kimi-code adapter -**Session:** 1 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Inspect `~/.kimi-code/` structure (likely same format as Kimi) | -| 2 | Write `discover_kimi_code()` yielding `NormalizedSession` objects | -| 3 | Register in `__init__.py` and dispatch dict | -| 4 | Dry-run: `python3 scripts/sync-agents-to-brain.py --source kimi-code --dry-run --summary` | -| 5 | Commit | - -#### Phase A3: Add antigravity-ide adapter -**Session:** 2 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Inspect `~/.antigravity-ide/` (likely JSON/JSONL or SQLite) | -| 2 | Write adapter; reuse `sqlite3` pattern from `antigravity.py` if SQLite | -| 3 | Register and dry-run | -| 4 | Commit | - -#### Phase A4: Add gemini adapter -**Session:** 2 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Inspect `~/.gemini/` — may contain `antigravity-cli/` (covered) + other tools | -| 2 | Write adapter for non-antigravity Gemini artifacts | -| 3 | Register and dry-run | -| 4 | Commit | - -#### Phase A5: Add copilot adapter -**Session:** 3 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Inspect `~/.copilot/` (likely VS Code extension state or CLI cache) | -| 2 | Write adapter | -| 3 | Register and dry-run | -| 4 | Commit | - -#### Phase A6: Add opencode adapter -**Session:** 3 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Inspect `~/.opencode/` (likely JSONL sessions) | -| 2 | Write adapter | -| 3 | Register and dry-run | -| 4 | Commit | - -#### Phase A7: Add zai adapter -**Session:** 4 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Inspect `~/.zai/` | -| 2 | Write adapter | -| 3 | Register and dry-run | -| 4 | Commit | - -#### Phase A8: Add zcode adapter -**Session:** 4 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Inspect `~/.zcode/` | -| 2 | Write adapter | -| 3 | Register and dry-run | -| 4 | Commit | - -#### Phase A9: Full import run & verification -**Session:** 4 · **Status:** 🔴 NOT STARTED - -| Step | Description | Command | -|------|-------------|---------| -| 1 | Run full import | `python3 scripts/sync-agents-to-brain.py --skip-active --summary` | -| 2 | Verify counts per source | `ls .aurelio/brain/ \| grep "^session-" \| sed 's/session-\([^-]*\)-.*/\1/' \| sort \| uniq -c \| sort -rn` | -| 3 | Spot-check one session from each new source for correct format | Manual | -| 4 | Commit | | - -**🟢 Exit Gate A:** All 13 sources have adapters. Full dry-run completes without errors. ≥8 sources produce sessions (some dirs may be empty on this host). - ---- - -### Track B — GBrain Engine (Sessions 5–9) - -**Goal:** Build `aurelio-gbrain` as a real TypeScript engine. - -#### Phase B1: Project scaffold -**Session:** 5 · **Status:** 🔴 NOT STARTED - -| Step | Description | Files | -|------|-------------|-------| -| 1 | `package.json` with deps: axios, better-sqlite3, express, commander, typescript, @types/node | NEW: `aurelio-gbrain/package.json` | -| 2 | `tsconfig.json` targeting Node 20, strict mode | NEW: `aurelio-gbrain/tsconfig.json` | -| 3 | `types.ts` — `BrainSession`, `BrainMessage`, `SearchResult`, `BrainStats` | NEW: `src/types.ts` | -| 4 | `brain.ts` — `GBrain` class with `ingest()`, `search()`, `listSessions()`, `getSession()`, `stats()` | NEW: `src/brain.ts` | -| 5 | `cli.ts` — commands: `ingest`, `search`, `list`, `stats`, `serve` | NEW: `src/cli.ts` | -| 6 | Build and smoke-test: `npm install && npm run build && node dist/cli.js stats` | VERIFY | -| 7 | Commit | | - -#### Phase B2: SQLite FTS5 index -**Session:** 6 · **Status:** 🔴 NOT STARTED - -| Step | Description | Files | -|------|-------------|-------| -| 1 | `indexer.ts` — `BrainIndexer` class with `indexSession()`, `search()`, `rebuild()` | NEW: `src/indexer.ts` | -| 2 | Wire `GBrain` to use `BrainIndexer` for search, fallback to filesystem scan if stale | MODIFY: `src/brain.ts` | -| 3 | Add `reindex` CLI command | | -| 4 | Test: `node dist/cli.js reindex && node dist/cli.js search "universalis"` | VERIFY | -| 5 | Commit | | - -#### Phase B3: HTTP API server -**Session:** 7 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | `server.ts` — Express app with 6 endpoints | -| 2 | Add `serve` CLI command with `--port` flag (default 18000) | -| 3 | Test all endpoints with `curl` | -| 4 | Commit | - -**API Endpoints:** - -| Method | Path | Returns | -|--------|------|---------| -| `GET` | `/api/brain/health` | `{ status: 'ok' }` | -| `GET` | `/api/brain/stats` | `BrainStats` | -| `GET` | `/api/brain/sessions?source=&limit=` | `BrainSession[]` | -| `GET` | `/api/brain/sessions/:id` | `BrainSession` | -| `GET` | `/api/brain/search?q=&limit=` | `SearchResult[]` | -| `POST` | `/api/brain/ingest` | triggers re-ingest | - -#### Phase B4: Dream cycle / synthesis -**Session:** 8 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | `dream.ts` — `DreamCycle` class: `run()`, `summarizeSession()`, `extractKnowledge()`, `linkSessions()` | -| 2 | Add `dream` CLI command | -| 3 | Add `POST /api/brain/dream` endpoint | -| 4 | Test with small batch | -| 5 | Commit | - -**Note:** MVP dream cycle is extractive (no LLM). LLM-enhanced dream is a stretch goal. - -#### Phase B5: MCP server surface -**Session:** 9 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | `mcp.ts` — MCP server exposing 4 tools: `brain_search`, `brain_list_sessions`, `brain_get_session`, `brain_stats` | -| 2 | Add `mcp` CLI command (stdio transport) | -| 3 | Register in `.aurelio/mcp_config.json` | -| 4 | Test with `mcp-remote` or direct stdio | -| 5 | Commit | - -**🟢 Exit Gate B:** GBrain builds (`npm run build`), CLI works, HTTP server responds, FTS5 search returns results, MCP server lists tools. `curl http://localhost:18000/api/brain/stats` returns non-zero session count. - ---- - -### Track C — Theia Backend Integration (Sessions 10–13) - -**Goal:** Replace the thin scaffold with a real backend that talks to GBrain. - -#### Phase C1: GBrain client package -**Session:** 10 · **Status:** 🔴 NOT STARTED - -| Step | Description | Files | -|------|-------------|-------| -| 1 | `GBrainClient` class with `stats()`, `listSessions()`, `getSession()`, `search()`, `ingest()`, `dream()` | NEW: `aurelio-theia/packages/aurelio-gbrain-client/` | -| 2 | Add to Theia workspace `package.json` workspaces | MODIFY | -| 3 | Build: `npm run build --workspaces` | VERIFY | -| 4 | Commit | | - -#### Phase C2: Real Theia backend service -**Session:** 11 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Rewrite `AurelioBackendService` — inject `GBrainClient` via `AURELIO_GBRAIN_ENDPOINT` env var (default `http://192.168.0.38:18000`) | -| 2 | Implement real methods: `syncPush()`, `syncPull()`, `getHealth()`, `searchBrain()`, `listSessions()`, `getSession()`, `getStats()`, `triggerDream()` | -| 3 | Add Theia RPC endpoints | -| 4 | Build and verify | -| 5 | Commit | - -#### Phase C3: Wire frontend commands to backend -**Session:** 12 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Inject `AurelioBackendService` (via RPC proxy) into `AurelioCommandContribution` | -| 2 | Wire high-value commands first: `syncPush`/`syncPull`, `startSession`, `viewSessionHistory`, `showAgentSummary`, `suggestPrompts`, `addContext` | -| 3 | Keep low-value commands as stubs (hardware dashboards, SPICE, etc.) | -| 4 | Build and verify | -| 5 | Commit | - -#### Phase C4: Brain widget -**Session:** 13 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | `brain-widget.tsx` — React component: session list (filterable by source), search bar, session detail view, stats panel | -| 2 | `brain-widget-contribution.ts` — register with Theia widget manager | -| 3 | Wire `aurelio.viewSessionHistory` command to open widget | -| 4 | Build and verify | -| 5 | Commit | - -**🟢 Exit Gate C:** Theia builds without errors. `AurelioBackendService` responds to health checks. ≥5 frontend commands call real backend methods. Brain widget opens and displays sessions. - ---- - -### Track D — Fleet Sync & Surfaces (Sessions 14–16) - -**Goal:** Deploy to CT 208, expose to all surfaces. - -#### Phase D1: Deploy GBrain to CT 208 -**Session:** 14 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Write systemd unit file for GBrain HTTP server (port 18000) | -| 2 | Write deploy script: build → copy to CT 208 → install systemd unit → start | -| 3 | Deploy and verify: `curl http://192.168.0.38:18000/api/brain/health` | -| 4 | Commit | - -#### Phase D2: Sync brain to CT 208 -**Session:** 14 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Verify `sync.py --push` pushes `.aurelio/brain/` | -| 2 | Run push and verify CT 208 has sessions | -| 3 | Commit | - -#### Phase D3: VS Code extension integration -**Session:** 15 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Add `GBrainClient` to VS Code extension | -| 2 | Add commands: `Aurelio: Search Brain`, `Aurelio: List Sessions`, `Aurelio: Show Stats` | -| 3 | Build VSIX and deploy to CT 205 | -| 4 | Commit | - -#### Phase D4: TUI + CLI integration -**Session:** 15 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Add brain search/list commands to TUI | -| 2 | Add brain search/list commands to CLI | -| 3 | Build and test | -| 4 | Commit | - -#### Phase D5: Hermes Agent integration -**Session:** 16 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Register GBrain MCP server in `.aurelio/mcp_config.json` | -| 2 | Test from Hermes Agent | -| 3 | Verify Hermes can search brain and get results | -| 4 | Commit | - -#### Phase D6: End-to-end verification -**Session:** 16 · **Status:** 🔴 NOT STARTED - -| Step | Description | -|------|-------------| -| 1 | Run importer on all sources | -| 2 | Push brain to CT 208 | -| 3 | Verify GBrain on CT 208 has all sessions | -| 4 | Verify Theia IDE can search brain | -| 5 | Verify VS Code extension can search brain | -| 6 | Verify Hermes Agent can search brain via MCP | -| 7 | Document any gaps | - -**🟢 Exit Gate D:** GBrain runs on CT 208 as a systemd service. All 4 surfaces (Theia, VS Code, TUI/CLI, Hermes) can query the brain. Full import → push → query cycle completes without errors. - ---- - -## 6. Hard Rules - -1. **Never edit UOS source** (per mycelium directive). This plan is entirely in the Aurélio ecosystem. -2. **No second Hermes gateway.** Theia backend talks to GBrain via HTTP, not via Hermes tools. -3. **Content-addressed writes only.** Brain session directories named `session--`, written only when content changes. -4. **All new adapters must be defensive.** Missing source directory → yield nothing. Malformed record → skip, never fatal. -5. **Idempotency is non-negotiable.** Re-running the importer must produce zero writes when nothing changed. - ---- - -## 7. Risks & Mitigations - -| Risk | Impact | Mitigation | -|------|--------|-----------| -| **Source format variance** — each agent stores sessions differently (SQLite, JSONL, protobuf) | Medium | Adapters are defensive; unknown formats skipped with warning, not fatal | -| **Empty source directories** — hermes, claude-code, etc. may have 0 sessions on this host | Low | Importer handles gracefully; user may expect data that isn't there | -| **GBrain scope creep** — dream cycle involves LLM calls | Medium | MVP dream cycle is extractive (no LLM); LLM-enhanced dream is stretch goal | -| **Theia build complexity** — Theia 1.73.1 + React 19 is bleeding edge | High | Existing scaffold already builds; add incrementally | -| **CT 208 deployment** — systemd + Node may have permission issues | Medium | Deploy script handles sudo; fallback: Docker container | - ---- - -## 8. Tradeoffs - -| Decision | Chosen | Alternative | Rationale | -|----------|--------|-------------|-----------| -| **GBrain as HTTP service first** | ✅ Service | Library | Simpler to deploy and debug; HTTP contract stays same when it becomes a library inside Gabinete Hub later | -| **SQLite FTS5 over vector DB** | ✅ FTS5 | Embeddings | FTS5 is good enough for keyword search; vector embeddings are a stretch goal | -| **Wire 5–10 high-value commands first** | ✅ Subset | All 50+ | Pragmatic; remaining stubs until user prioritizes | -| **Adapter per file** | ✅ Split package | Monolithic | Prevents `adapters.py` from becoming a 2000-line monster | - ---- - -## 9. Open Questions - -1. **Which agent ecosystems beyond the 13 listed?** Cursor, Windsurf, Aider, Continue, Cody, Tabnine, etc. The adapter pattern makes adding #14+ a 30-minute task — but we need to know which ones you actually use. -2. **Should GBrain run on CT 208 or on your workstation?** Current plan: CT 208 primary, local cache secondary. Want local-first with sync? -3. **Dream cycle LLM provider?** Which provider from the registry? Default `openrouter` or `kimi-coding`? -4. **VS Code extension repo structure?** Is `aurelio-vscode` a submodule of `replica-omnisciente` or standalone? Paths change depending on answer. - ---- - -## 10. Success Metrics - -| Metric | Target | -|--------|--------| -| **Coverage** | 13/13 source directories have adapters | -| **Import speed** | Full import completes in <5 minutes | -| **Import yield** | Sessions from ≥8 sources (some dirs may be empty) | -| **Search latency** | GBrain FTS5 search returns results in <100ms | -| **API latency** | HTTP server responds to all 6 endpoints in <200ms | -| **IDE integration** | Theia brain widget opens and displays sessions without errors | -| **Fleet coverage** | All 4 surfaces can query the brain | -| **Idempotency** | Re-running import produces 0 writes when nothing changed | - ---- - -## 11. Files Likely to Change - -### Track A -- `replica-omnisciente/scripts/agent_importers/adapters/*.py` (new package) -- `replica-omnisciente/scripts/agent_importers/__init__.py` -- `replica-omnisciente/scripts/sync-agents-to-brain.py` - -### Track B -- `replica-omnisciente/aurelio-gbrain/` (entire new package) - -### Track C -- `aurelio-theia/AurelioTheia/src/node/aureilio-backend-module.ts` -- `aurelio-theia/AurelioTheia/src/browser/aureilio-frontend-module.ts` -- `aurelio-theia/AurelioTheia/src/browser/brain-widget.tsx` (new) -- `aurelio-theia/packages/aurelio-gbrain-client/` (new package) - -### Track D -- `replica-omnisciente/.aurelio/sync.py` -- `replica-omnisciente/.aurelio/mcp_config.json` -- `aurelio-vscode/src/` (integration) -- `aurelio-tui/src/` (integration) -- `aurelio-cli/src/` (integration) - ---- - -## 12. References - -| Document | Path | -|----------|------| -| Ecosystem integration pattern | `references/aurelio-ecosystem-integration.md` | -| Strategic master plans methodology | `references/strategic-master-plans.md` | -| Guest → host import pattern | `references/importing-guest-component-into-host.md` | -| Fleet map & build commands | `replica-omnisciente/AGENTS.md` | -| Existing importer entry point | `scripts/sync-agents-to-brain.py` | -| Normalized session contract | `scripts/agent_importers/engine.py` | -| 50+ command stubs | `aurelio-theia/AurelioTheia/src/browser/aurelio-frontend-module.ts` | -| Thin backend scaffold | `aurelio-theia/AurelioTheia/src/node/aureilio-backend-module.ts` | - ---- - -## 13. Immediate Next Actions - -1. **Confirm the 13 source directories exist** on your host and inspect their formats (`ls` + `file` on each). -2. **Start Track A Phase A1:** Refactor adapters into a package. -3. **In parallel, start Track B Phase B1:** Scaffold GBrain package. -4. **Report back** with findings from source directory inspection — some adapters may be trivial (same format as existing), others may need research. - ---- - -## 14. Subagent-Report Log - -| Date | Subagent | Task | Report | -|------|----------|------|--------| -| 2026-07-12 | Hermes | On-disk grounding | Confirmed: aurelio-theia is thin scaffold, aurelio-gbrain is empty dir, 6/13 adapters exist, brain has 539 kimi + 127 claude + 3 antigravity + 2 mimocode + 1 hermes sessions (~800 total). | -| 2026-07-13 | Hermes | Fleet plan audit | All 16 phases across 4 tracks verified as NOT STARTED. 13 source directories mapped. | +## Visionary Scope + +**Aurelio OmniBrain** is the omniscient orchestration / execution layer that +ties the Portugal Futurista fleet together: it observes every replica and +realm, plans across them, and drives execution through the existing +toolchains (mycelium agent blocks, Aurelio Theia, UniversalisOS, GBrain). + +It is the "brain" that makes the `replica-omnisciente` realms (e.g. +`maquina-na-mao`, `nervura-electrica`, `smart-device-firmware`) act as one +coordinated system rather than isolated projects. + +## Architecture Approach + +* **Observation** — ingest state/telemetry from each realm (MQTT/HTTP as used + by `nervura-electrica` today). +* **Planning** — model cross-realm intent as mycelium agent-block workspaces + (IP-XACT-style: agents = components, ports = ACP channels, memory = context). +* **Execution** — drive UniversalisOS + Aurelio Theia to instantiate agents and + deploy/config via mycelium codegen. +* **Memory** — GBrain as the persistent, cross-session state of the brain. + +## Cross-Realm Integration Points + +| Realm / Repo | Role in OmniBrain | +| ---------------------- | -------------------------------------------------- | +| `replica-omnisciente` | Fleet of replicas; observation + execution targets| +| `maquina-na-mao` | Mobile/desktop operator surface (Flutter) | +| `nervura-electrica` | Backend (MQTT/HTTP), primary telemetry bus | +| `smart-device-firmware`| Edge firmware (Ode Triunfal / Guiador) | +| `mycelium` | XSD→codegen; agent-block schema + emit targets | +| `aurelio-theia` | IDE; clickable agent blocks; design canvas | +| `universalisos` | Target OS / runtime for instantiated agents | +| `gbrain` | Persistent memory / reasoning state | + +## Execution Tracker + +Legend: ✅ done · 🟡 in progress · ⏳ queued · 🚫 out of scope + +### Phase 0 — Foundations +| Task | Status | Notes | +| ---- | ------ | ----- | +| Define OmniBrain scope & non-negotiables | ⏳ | confirm this scaffold | +| Inventory realm observation surfaces (MQTT topics, HTTP APIs) | ⏳ | start with `nervura-electrica` | +| Pin mycelium agent-block schema (`conformance/agentic/uos-agent-blocks-1.0.xsd`) | ⏳ | pull from master plan A.2 | + +### Phase 1 — Observation +| Task | Status | Notes | +| ---- | ------ | ----- | +| Telemetry bus adapter for `nervura-electrica` | ⏳ | reuse `maquina-na-mao` MQTT client pattern | +| Per-realm state model (devices, replicas, health) | ⏳ | polymorphic like `OdeTriunfalNode` | +| Unified observation feed into GBrain | ⏳ | | + +### Phase 2 — Planning +| Task | Status | Notes | +| ---- | ------ | ----- | +| Agent-block workspace format for cross-realm plans | ⏳ | IP-XACT mapping (master plan A.1) | +| Planner → agent-block workspace compiler | ⏳ | mycelium codegen target | +| Conflict / resource arbitration across realms | ⏳ | | + +### Phase 3 — Execution +| Task | Status | Notes | +| ---- | ------ | ----- | +| Instantiate agents via Aurelio Theia blocks | ⏳ | | +| Deploy/config to UniversalisOS from mycelium emit | ⏳ | | +| Closed-loop execution feedback to GBrain | ⏳ | | + +## Open Questions +1. Is "OmniBrain" == GBrain, or a separate orchestration layer above it? +2. Which realm is the first vertical slice (suggest `nervura-electrica` ↔ `maquina-na-mao`)? +3. Target runtime for instantiated agents — UniversalisOS only, or also edge firmware? +4. Confirm this tracker replaces/augments the 2026-07-12 master plans. diff --git a/.aurelio/plans/2026-07-30_aurelio-consolidation-audit.md b/.aurelio/plans/2026-07-30_aurelio-consolidation-audit.md new file mode 100644 index 00000000..76d66dde --- /dev/null +++ b/.aurelio/plans/2026-07-30_aurelio-consolidation-audit.md @@ -0,0 +1,295 @@ +# Aurelio Consolidation Audit — 30/07/2026 + +> Pre-Hetzner migration. Full state analysis of replica-omnisciente + service topology truth. + +--- + +## 1. REPO: replica-omnisciente + +**5.7GB on disk | .git 618MB (533MB packs) | 7,368 tracked files** + +### 1.1 Real Code Projects (keep, maintain) + +| Project | Lang | Disk (src) | Purpose | +|---------|------|-----------|---------| +| `dirac/` | TS | 21M | VS Code coding agent (esbuild) | +| `tilth/` | Rust | 768K | Code intelligence MCP (tree-sitter) | +| `toon/` | TS | 2.1M | Toon format library (pnpm monorepo) | +| `shared/` | TS | 12M | Protobuf shared lib (@aurelio/shared) | +| `aurelio-theia/` | TS | 3.6M | IDE monorepo **[submodule → Forgejo]** | +| `aurelio-gbrain/` | TS/Py | ~3M | GBrain knowledge brain **[submodule → Forgejo]** | +| `mycelium-rd/` | Rust | ~1M | XSD→codegen research | +| `telminal/` | Py | 376K | Terminal-in-Telegram bot | +| `infra/cdp-bridge/` | TS | 164K | CDP bridge | +| `infra/lab-gateway/` | Py | ~1M | Hardware lab gateway (ESP32/PPK2/Icicle) | +| `infrastructure/fabric/drift/` | Rust | ~100K | Drift agent (SQLite store, plan parser) | +| `dashboards/media-explorer/` | TS | 952K | Media explorer dashboard | + +### 1.2 Git Bloat — BINARY BLOBS TRACKED IN HISTORY (618MB .git) + +These are tracked files bloating the git pack (533MB): + +| Path | Size | What | +|------|------|------| +| `infra/lab-gateway/MPFS_ICICLE_KIT_2025_07/` | ~100M | 27 PolarFire SoC FPGA job/bitstream files | +| `realms/tear-de-silicio/standard-app/models/` | ~80M | FPGA bitstreams + VNNX ML models | +| `.aurelio/knowledge/*.pdf` | ~32M | 4 research PDFs (airships, final reports) | +| `.aurelio/knowledge/session*/` | ~16M | Session artifacts with screenshots/binary | + +**Fix:** `git filter-repo` to purge binaries → move to LFS or external artifact store. Expected .git reduction: ~250-350MB. + +### 1.3 Local-Only Bloat (gitignored, not in repo but wasting disk) + +| Path | Size | Fix | +|------|------|-----| +| `tilth/target/` | 3.4G | `cargo clean` — rebuild on demand | +| `mycelium-rd/target/` | 345M | `cargo clean` | +| `infrastructure/fabric/drift/target/` | 570M | `cargo clean` | +| `aurelio-gbrain/src/assets/` | 61M | submodule assets, not tracked (ok) | +| `.venv-telminal/` | 53M | gitignored (ok) | + +### 1.4 Brain Sessions — 1,090 dirs, 73MB, uncontrolled growth + +| Source | Count | Status | +|--------|-------|--------| +| kimi | 601 | Synchronized from ~/.kimi/sessions/ | +| opencode | 178 | Synchronized | +| claude-code | 129 | Synchronized (+ subagents) | +| mimo (mo/mp/mq) | 125 | Synchronized | +| antigravity | 13 | Synchronized | +| hermes | 12 | Synchronized | +| uuid (no prefix) | 25 | Legacy/old format | +| pi/mimocode | 7 | Synchronized | + +**268 are untracked** (never committed to git). The brain is growing ~50-100 sessions/week. + +**Fix:** Brain sessions should NOT be in git at all. They're runtime data. Add `.aurelio/brain/session-*/` to .gitignore. The GBrain PostgreSQL instance on CT223 is the real persistent store — file-based sessions are redundant once imported. + +### 1.5 Stale/Dead Documentation + +| File | Dated | Problem | +|------|-------|---------| +| `DIRECTORY_GUIDE.md` | 2026-04-16 | Describes root-level `memory/`, `identity/`, `swarm/` — all moved to `.aurelio/`. Lists wrong realms. | +| `AGENT_INDEX.md` | 2025-07-07 | Says CT208 is hub (stopped). 63 projects listed, half don't exist. Wrong IPs. | +| `README.md` | 2026-07-06 | Generic, doesn't match actual structure | +| `DESIGN.md` | 2026-07-06 | 648 bytes — likely empty stub | +| `SETUP_API_KEYS.md` | 2026-07-06 | Probably stale API key instructions | +| `task.md` | 2026-07-06 | Stale task file | +| `vscode_extension_migration_guide.md` | 2026-07-06 | One-off migration doc, no longer needed | +| `IMPLEMENTATION_SUMMARY.md` | 2026-07-11 | Portal consolidation notes | +| `PORTAL_CONSOLIDATION_SUMMARY.md` | 2026-07-11 | Same topic | +| `NEXT_MONTH_ROADMAP.md` | 2026-07-11 | August 2026 roadmap — expired | + +### 1.6 Empty/Stub Realms (23 dirs, most 8KB empty) + +**Real content (3):** `nervura-electrica` (672K), `universalisos` (308K), `silent-shared-office` (32K) + +**Empty stubs (20):** aprendiz-de-sensacoes, matriz-da-maquina, ode-electromagnetica, orquestra-de-hardware, alquimista, etc. + +**Fix:** Delete empty stubs. Move realms/ with real content to top-level project docs or archive. + +--- + +## 2. SERVICE TOPOLOGY — TRUTH vs DOCUMENTATION + +### 2.1 What the 07-16 Audit Says (WRONG) + +``` +CT208 (192.168.0.125) = ALL Aurelio services + gbrain :18001, gabinete :18000, bot-rs, 7 MCP servers, 6 kimi swarm agents +ASUS = fabricd :8081, orchestrator :8200 +``` + +### 2.2 What Memory Says (CURRENT REALITY) + +``` +CT208 = STOPPED (old hub, decommissioned) +CT223 (Gigabyte, dir-rootfs /mnt/disks/usb-sdf) = NEW HUB + ├── Gabinete :18000 + ├── Bot-rs (Telegram @Aurélio Omnisciente) + ├── GBrain :18001 + ├── fabricd :8081 + └── Engineering MCPs +CT217 = Hermes gateway 24/7 (Signal/Telegram/Matrix, E2EE) +CT235 = Orchestrator :8200 +CT237 (Gigabyte) = Forgejo :3001 (code.portugalfuturista.org), Woodpecker :8001 +``` + +### 2.3 MCP Config — 16 configured servers, all `npx` (wrong for production) + +`.aurelio/mcp_config.json` lists 16 MCP servers using `npx` commands. But AGENTS.md says they're "remote (mcp-remote to *.portugalfuturista.org)". The config doesn't match reality — these are either: +- Run as systemd services on CT223 (sourcing :8081, eda :8082, knowledge :8083, codebase-memory :8084) +- Or dead/never-deployed (civil, artist, mechanic, StitchMCP, dart-mcp-server) + +### 2.4 Local MCP servers on disk (.aurelio/mcp/) + +| Server | Size | Status | +|--------|------|--------| +| `savearth-mcp/` | 84K | Python — active (Savearth fleet) | +| `savearth-workspace/` | 60K | Python — active | +| `electrical-mcp/` | 720K | TS — active | +| `engineering-mcp/` | 340K | TS — active | +| `flint-chart-mcp/` | 156K | TS — purpose unclear | +| `image-pipe-mcp/` | 100K | TS — purpose unclear | +| `aurelio-tools-mcp/` | 92K | TS — purpose unclear | + +### 2.5 Infrastructure Topology (3 PVE hosts) + +``` +PROXMOX CLUSTER "portugalfuturi" +│ +├── ASUS (192.168.0.38) — 31Gi, PVE 9.2.2 +│ ├── CT215: pf-matrix (Matrix Synapse :matrix.portugalfuturista.org) +│ ├── CT208: OLD HUB — STOPPED +│ ├── Cloudflare tunnel pf-tunnel-v2 +│ └── Other CTs (202-220) +│ +├── GIGABYTE (192.168.0.104) — 46GB, 2x RTX 3070 +│ ├── CT223: NEW HUB (Gabinete, Bot-rs, GBrain, fabricd, MCPs) +│ ├── CT237: Forgejo :3001 + Woodpecker :8001 +│ ├── CT216: multi-scrobbler (Subsonic) +│ ├── Docker media stack (Jellyfin, Sonarr, Radarr, etc.) +│ ├── signal-api container :18088 +│ └── Monitoring (Prometheus + Grafana + Loki + Blackbox) +│ +└── LATTEPANDA (192.168.0.40) + ├── MinIO :9000/:9001 (S3 / Terraform state) + ├── LocalStack v3 :4566 (AWS dev) + └── Azurite :10000-10002 (Azure dev) + +OFF-LAN: +├── CT217 (pf-hermes): Hermes gateway 24/7 +├── CT235: Orchestrator :8200 +├── VPS pf-edge (94.130.24.119): WG endpoint +└── CTs 213/231/234/235/236: Savearth fleet +``` + +--- + +## 3. HETZNER MIGRATION — TARGET ARCHITECTURE + +### 3.1 What Should Move to Hetzner (cloud-facing services) + +These services benefit from public IP, low latency to external APIs, and 24/7 uptime without LAN dependency: + +| Service | Current | Hetzner Target | Why | +|---------|---------|----------------|-----| +| Hermes gateway | CT217 | Hetzner VM | 24/7 messaging, no LAN dependency | +| GBrain (PostgreSQL) | CT223 | Hetzner VM (with pg backup) | Central knowledge store, API-facing | +| Gabinete API | CT223 | Hetzner VM | Central API, bot backend | +| Bot-rs (Telegram) | CT223 | Hetzner VM | Telegram webhook needs public IP | +| Forgejo + CI | CT237 | Hetzner VM | Git push/pull latency, public access | +| Cloudflare tunnel endpoint | ASUS | Cloudflare→Hetzner | Remove single LAN point of failure | + +### 3.2 What Should Stay on LAN (hardware-dependent, latency-sensitive) + +| Service | Host | Why | +|---------|------|-----| +| Hardware lab gateway | ASUS/CT206 | Physical ESP32/PPK2/Icicle connected | +| Media stack | Gigabyte | Local storage, GPU transcoding | +| Signal-API | Gigabyte | Phone registration tied to LAN | +| Cloud emulators | Lattepanda | Dev-only, LAN-local | +| Savearth fleet CTs | Various | Hardware-attached (ESP32/PPK2) | +| Proxmox cluster itself | All 3 hosts | Bare metal | + +### 3.3 Service Separation Principle + +Current problem: CT223 is a god-container running everything. Hetzner migration should split: + +``` +HETZNER (public cloud) +├── VM-1: aurelio-core (Gabinete + Bot-rs + GBrain) +├── VM-2: aurelio-forge (Forgejo + Woodpecker CI) +├── VM-3: aurelio-gateway (Hermes + Cloudflare tunnel endpoint) +└── VM-4: aurelio-monitoring (Grafana/Prometheus mirror) + +LAN (hardware-locked) +├── ASUS: lab hardware, local CTs +├── Gigabyte: media, signal, CT223→demoted +├── Lattepanda: dev emulators +└── Savearth fleet: hardware testing + +VPN: WireGuard mesh Hetzner ↔ LAN (existing pf-edge WG) +``` + +--- + +## 4. REPO CLEANUP ACTIONS + +### 4.1 Immediate (low risk) + +1. **`cargo clean` all Rust projects** — frees 4.3GB local disk + ``` + cd tilth && cargo clean + cd mycelium-rd && cargo clean + cd infrastructure/fabric/drift && cargo clean + ``` + +2. **Delete empty realm stubs** (20 dirs × 8KB) + ``` + realms/aprendiz-de-sensacoes, matriz-da-maquina, ode-electromagnetica, ... + ``` + +3. **Delete stale root docs** + ``` + DESIGN.md, task.md, SETUP_API_KEYS.md, vscode_extension_migration_guide.md, + IMPLEMENTATION_SUMMARY.md, PORTAL_CONSOLIDATION_SUMMARY.md, NEXT_MONTH_ROADMAP.md + ``` + +4. **Add brain sessions to .gitignore** + ``` + .aurelio/brain/session-*/ + ``` + +### 4.2 Medium (requires git history rewrite) + +5. **Purge binary blobs from git** — saves ~250MB from .git + ``` + git filter-repo --path infra/lab-gateway/MPFS_ICICLE_KIT_2025_07 --invert-paths + git filter-repo --path realms/tear-de-silicio/standard-app/models --invert-paths + git filter-repo --path-glob '.aurelio/knowledge/*.pdf' --invert-paths + ``` + Move to: MinIO bucket or Git LFS (if Forgejo LFS is enabled — currently NOT per AGENTS.md) + +### 4.3 Documentation Rewrite + +6. **Rewrite `DIRECTORY_GUIDE.md`** to reflect `.aurelio/` structure (not root-level) +7. **Rewrite `AGENT_INDEX.md`** with real CT topology (CT223 hub, not CT208) +8. **Update `AGENTS.md`** infra targets table to match memory reality +9. **Delete or archive** the 07-16 audit (superseded by this document) + +--- + +## 5. SERVICE CONSOLIDATION ACTIONS + +### 5.1 MCP Server Cleanup + +16 MCP servers configured but reality is unclear. Need to: +1. SSH to CT223, enumerate actual systemd services +2. Compare against `.aurelio/mcp_config.json` +3. Remove dead MCP entries (civil, artist, mechanic, StitchMCP, dart-mcp-server) +4. Consolidate overlapping: `electrical-mcp` vs `engineering-mcp` vs `electrical-eda-mcp` vs `electrical-sourcing-mcp` — 4 servers for electrical? + +### 5.2 Kimi Swarm Decommission? + +6 Kimi swarm agents on old CT208 (orchestrator, log-processor, ota-validator, regression-detector, report-synthesizer, dashboard). CT208 is stopped. Are these: +- Still needed? +- Migrated to CT223? +- Dead and should be removed from repo (`.aurelio/swarm/`)? + +### 5.3 Lifestream Pipeline Verification + +``` +Saved Messages → Lifestream userbot → SQLite → Cron 03:00 → GBrain +``` +This pipeline depends on CT223. Verify it still works after migration. + +--- + +## 6. OPEN QUESTIONS (need your input) + +1. **Hetzner specs** — What VM(s) did you provision? CPU/RAM/disk? Dedicated or shared? +2. **CT223 future** — After Hetzner migration, does CT223 stay as LAN fallback or get repurposed? +3. **Git remote strategy** — Keep GitHub (origin) + Forgejo? Or consolidate to Forgejo-only now that Hetzner gives public access? +4. **Brain persistence** — GBrain PostgreSQL on Hetzner means migrating 1362+ pages. Acceptable? Or keep DB on LAN with Hetzner as read-replica? +5. **Kimi swarm** — Dead or alive? diff --git a/.aurelio/plans/2026-07-30_aurelio-consolidation-execution-plan.md b/.aurelio/plans/2026-07-30_aurelio-consolidation-execution-plan.md new file mode 100644 index 00000000..898c78f8 --- /dev/null +++ b/.aurelio/plans/2026-07-30_aurelio-consolidation-execution-plan.md @@ -0,0 +1,346 @@ +# Aurelio Consolidation — Execution Plan + +> 30/07/2026. Decisions locked. This is the execution document. + +## DECISIONS (locked by Maestro) + +| Decision | Resolution | +|----------|-----------| +| Hetzner specs | Agent decides — see sizing below | +| CT223 future | Stays as LAN fallback, ready to receive snapshots | +| Git remote | Forgejo = primary, GitHub = mirror | +| GBrain DB | Stays local (MVP phase), not on Hetzner | +| Kimi swarm | Remove from CT223, restructure as reusable sub-agent harness | +| Aurelio identity | Aurelio is a **harness** — sub-agents feed into bot, Theia, VS Code | + +## BLOCKER: Hetzner API Token + +The token is in `nervura-electrica/infrastructure/proxmox/iac/tofu/secrets.enc.yaml`, +encrypted with age key `age1m56h5m2hra70uppj5jn8w7prynjw8gtjfxpp0tucvgydnnu3u9esaplg9g`. +The age private key is NOT on: laptop, pf-edge, ASUS, or CT223. + +**To provision new Hetzner VMs, we need either:** +1. The age private key (where was it created?) +2. OR a fresh Hetzner API token from console.hetzner.cloud + +--- + +## TARGET ARCHITECTURE + +### Sizing Decision + +CT223 currently runs 30+ services on 2GB RAM — it's at the breaking point. +Only the compute-intensive AI layer goes to Hetzner. Git forge and gateway +stay on LAN (Dell/Gigabyte) for data sovereignty, but are IaC-packaged so +they can deploy to any cloud in one command. + +**LAN resources:** +- Dell .41: 31Gi RAM, 47GB disk free — 6 CTs running, ample headroom +- Gigabyte .104: 46Gi RAM, 86% disk (15GB free) — tight, CT223 is the hog + +**New VMs / CTs:** + +| VM/CT | Host | Type | Specs | Monthly | Purpose | +|-------|------|------|-------|---------|---------| +| pf-aurelio | **Hetzner** | CPX31 | 2 ded AMD vCPU, 4GB, 160GB | €8.50 | Gabinete, Bot-rs, fabricd, MCP gateway, sub-agent harness | +| pf-forge | **Dell .41** | LXC 238 | 4 vCPU, 8GB, 80GB | €0 | Forgejo + Woodpecker CI + GitHub mirror | +| pf-stage | **Dell .41** | LXC 239 | 2 vCPU, 4GB, 40GB | €0 | Hermes gateway 24/7 | + +**Total cloud cost: €8.50/month** (only pf-aurelio is cloud). +pf-edge (existing CX22) stays as media edge — €4.50/month. +Grand total Hetzner: €13/month. + +GBrain stays on CT223 (local, per decision). pf-aurelio connects to it via WireGuard. +pf-forge and pf-stage are LAN-local (no WG needed for GBrain access). + +### Service Separation Target + +``` +HETZNER (public cloud — €13/mo total) +│ +├── pf-edge (94.130.24.119, CX22 — EXISTS) +│ ├── Caddy media reverse proxy → LAN via WG +│ └── WireGuard → 10.99.0.1 (ASUS) +│ +├── pf-aurelio (NEW, CPX31) +│ ├── aurelio-gabinete-rs :18000 → API central +│ ├── aurelio-bot-rs → Telegram bot +│ ├── aurelio-fabricd :8081 → model gateway +│ ├── aurelio-engineering-gw :8090 → federated MCP +│ ├── aurelio-subagent-harness → Kimi swarm redesign +│ ├── aurelio-web → Olhos-de-Orpheu +│ └── brain-sync (→ CT223 GBrain via WG 10.99.0.1) +│ +LAN (Proxmox cluster — data sovereignty, zero cloud cost) +│ +├── DELL .41 (PVE 9.2.2, 31Gi RAM, 47GB free) +│ ├── CT213 pf-immich +│ ├── CT231 pf-homeassistant +│ ├── CT234 pf-marketplace +│ ├── CT235 pf-orchestrator :8200 +│ ├── CT236 pf-savearth +│ ├── CT237 pf-iot-stack +│ ├── CT238 pf-forge (NEW) — Forgejo :3001 + Woodpecker :8001 +│ │ ↑ IaC-packaged: swap LXC→cloud in one tofu apply +│ └── CT239 pf-stage (NEW) — Hermes gateway 24/7 +│ ↑ IaC-packaged: swap LXC→cloud in one tofu apply +│ +├── GIGABYTE .104 (46Gi RAM, 86% disk) +│ ├── CT223 pf-forja-do-conhecimento — DATA LAYER +│ │ ├── pf-gbrain :18001 → STAYS (MVP, local DB) +│ │ ├── gbrain-rest-proxy :18002 → STAYS +│ │ ├── PostgreSQL (Docker) :5433 → STAYS (GBrain backend) +│ │ ├── Docker services → STAYS (Plane, Outline, MinIO) +│ │ ├── lifestream-userbot → STAYS (Telethon session) +│ │ ├── pf-identity :9091 → STAYS (CA) +│ │ └── [snapshot target for pf-aurelio] +│ ├── CT237 (old Forgejo — migrate to CT238 Dell, then retire) +│ └── Other media CTs (200-222) +│ +├── ASUS .38 (31Gi, PVE cluster primary) +│ ├── CT215 pf-matrix (Matrix Synapse) +│ ├── CT208 OLD HUB — STOPPED +│ ├── Cloudflare tunnel pf-tunnel-v2 +│ └── WireGuard server 10.99.0.1 +│ +WIREGUARD MESH (ASUS = hub, 10.99.0.1): + pf-edge 10.99.0.2 ↔ ASUS 10.99.0.1 (media proxy) + pf-aurelio 10.99.0.3 ↔ ASUS 10.99.0.1 (→ CT223 GBrain) + Dell CTs LAN-direct (no WG needed) +``` + +### What Moves Off CT223 + +| Service | From | To | Route | +|---------|------|----|-------| +| aurelio-gabinete-rs | CT223 :18000 | pf-aurelio (Hetzner) | direct on VM | +| aurelio-bot-rs | CT223 | pf-aurelio (Hetzner) | direct on VM | +| aurelio-fabricd | CT223 :8081 | pf-aurelio (Hetzner) | direct on VM | +| aurelio-engineering-gw | CT223 :8090 | pf-aurelio (Hetzner) | direct on VM | +| aurelio-native-engine-gw | CT223 | pf-aurelio (Hetzner) | direct on VM | +| aurelio-web | CT223 | pf-aurelio (Hetzner) | direct on VM | +| aurelio-webhook | CT223 | pf-aurelio (Hetzner) | direct on VM | +| Hermes gateway | CT217 | CT239 Dell (pf-stage) | LAN-direct | +| Forgejo + Woodpecker | CT237 Gigabyte | CT238 Dell (pf-forge) | LAN-direct | + +### What Stays on CT223 + +| Service | Why it stays | +|---------|-------------| +| pf-gbrain :18001 | GBrain DB stays local (Maestro decision) | +| gbrain-rest-proxy :18002 | Pairs with local GBrain | +| PostgreSQL Docker :5433 | GBrain backend DB | +| Plane/Outline/MinIO/Redis | Docker stack, heavy data | +| lifestream-userbot | Telethon session, LAN-dependent | +| pf-identity :9091 | CA + identity management | +| brain-sync :8095 | Syncs to local GBrain | +| All `pf-*-scribe-208` services | Legacy naming, lightweight | + +### CT223 After Migration + +CT223 drops from 30+ services to ~12. RAM goes from 976Mi used → ~400Mi. +It becomes the data layer + fallback. pf-aurelio can snapshot its config +to CT223 for disaster recovery. + +--- + +## IaC PORTABILITY — LXC TODAY, CLOUD TOMORROW + +pf-forge and pf-stage deploy as Proxmox LXC containers on Dell today. +The OpenTofu definitions use a provider-agnostic pattern: swap one +variable and the same service deploys to Hetzner/AWS/bare-metal. + +### Structure (extends existing nervura-electrica IaC) + +``` +nervura-electrica/infrastructure/proxmox/iac/tofu/ +├── edge_node.tf ← pf-edge (Hetzner, exists) +├── aurelio_node.tf ← pf-aurelio (Hetzner CPX31, new) +├── forge_node.tf ← pf-forge (Dell LXC 238, new) +├── stage_node.tf ← pf-stage (Dell LXC 239, new) +├── providers.tf +└── secrets.enc.yaml +``` + +Each node file follows this pattern: + +```hcl +# forge_node.tf +# Deploys as LXC on Dell today. Set deploy_target="hcloud" to move to cloud. + +variable "forge_deploy_target" { + description = "LXC (LAN) or hcloud (Hetzner)" + type = string + default = "lxc" # ← change this one line to go cloud +} + +# LXC variant (Dell .41, zero cost) +resource "proxmox_lxc" "forge" { + count = var.forge_deploy_target == "lxc" ? 1 : 0 + vmid = 238 + target_node = "dell" + ... +} + +# Cloud variant (Hetzner, if needed later) +resource "hcloud_server" "forge" { + count = var.forge_deploy_target == "hcloud" ? 1 : 0 + name = "pf-forge" + server_type = "cx22" + ... +} +``` + +Ansible roles handle the service config regardless of where the container/VM runs — +same Forgejo config, same Hermes config, same WireGuard mesh entry. + +--- + +## KIMI SWARM → AURELIO SUB-AGENT HARNESS + +### Current State (CT223) + +6 Kimi swarm services, all named `pf-*-208.service`: +- orchestrator, log-processor, ota-validator, regression-detector, report-synthesizer, dashboard + +These are Python services in `.aurelio/swarm/` that dispatch Kimi API calls +as specialized agents. They're alive but isolated — only used by the old +swarm dashboard. + +### Redesign: Aurelio Sub-Agent Harness + +Aurelio is a **harness**. The swarm agents become a reusable sub-agent layer +that any Aurelio surface can invoke: + +``` +AURELIO SURFACES (callers) +├── Bot-rs (Telegram) → "analyze these logs" → dispatches sub-agent +├── Aurelio-Theia (IDE) → user requests review → dispatches sub-agent +├── Aurelio VS Code → developer triggers analysis → dispatches sub-agent +└── Gabinete API → programmatic dispatch + +SUB-AGENT HARNESS (the swarm, restructured) +├── log-processor → parse/stream device/firmware logs +├── ota-validator → validate firmware OTA packages +├── regression-detector → diff CI runs, flag regressions +├── report-synthesizer → aggregate findings into reports +└── orchestrator → route requests to the right agent + +TRANSPORT: each surface calls the orchestrator via HTTP/gRPC +AUTH: fabricd gateway token (existing) +``` + +### Implementation + +1. Strip the `pf-*-208` naming and systemd units from CT223 +2. Package the swarm as a single deployable service (Rust or Python) +3. Expose via fabricd as an MCP tool: `aurelio.subagent.dispatch` +4. Any surface (bot, IDE, VS Code) calls it through the existing MCP gateway +5. Deploy on pf-aurelio alongside Gabinete/Bot-rs + +--- + +## REPO CLEANUP (executable now, no Hetzner dependency) + +### Phase 1: Immediate disk recovery + +```bash +# Free 4.3GB of local Rust build artifacts +cd tilth && cargo clean +cd mycelium-rd && cargo clean +cd infrastructure/fabric/drift && cargo clean +``` + +### Phase 2: Git hygiene + +```bash +# Stop tracking brain sessions (runtime data, not source) +echo '.aurelio/brain/session-*/' >> .gitignore + +# Remove binary blobs from working tree +git rm -r --cached infra/lab-gateway/MPFS_ICICLE_KIT_2025_07/ +git rm -r --cached realms/tear-de-silicio/standard-app/models/ +``` + +### Phase 3: Purge git history (requires force-push) + +```bash +# After Phase 2, purge from history (saves ~250MB from .git) +git filter-repo --path infra/lab-gateway/MPFS_ICICLE_KIT_2025_07 --invert-paths +git filter-repo --path realms/tear-de-silicio/standard-app/models --invert-paths +git filter-repo --path-glob '.aurelio/knowledge/*.pdf' --invert-paths + +# Force push (coordinate with all checkouts) +git push forgejo main --force +git push origin main --force +``` + +### Phase 4: Delete dead content + +```bash +# 20 empty realm stubs +rm -rf realms/aprendiz-de-sensacoes realms/matriz-da-maquina ... + +# Stale root docs +rm DESIGN.md task.md SETUP_API_KEYS.md vscode_extension_migration_guide.md +rm IMPLEMENTATION_SUMMARY.md PORTAL_CONSOLIDATION_SUMMARY.md NEXT_MONTH_ROADMAP.md +rm AGENT_INDEX.md DIRECTORY_GUIDE.md +``` + +### Phase 5: Documentation rewrite + +Rewrite these to match reality: +- `AGENTS.md` — update infra targets table (CT223 hub, not CT208) +- `README.md` — current repo overview +- New `DIRECTORY_GUIDE.md` — reflect `.aurelio/` structure + +--- + +## GIT REMOTE STRATEGY + +### Current: dual primary +- `origin` = GitHub (fabiorafaelcoutada/replica-omnisciente) +- `forgejo` = Forgejo LAN (code.portugalfuturista.org) + +### Target: Forgejo primary, GitHub mirror + +```bash +# Swap remotes +git remote rename origin github-mirror +git remote rename forgejo origin + +# Set up mirror push (Forgejo → GitHub) +git remote set-url --add origin --push git@github.com:fabiorafaelcoutada/replica-omnisciente.git +``` + +Or configure Forgejo's push mirror feature (Forgejo pushes to GitHub automatically on each push). + +--- + +## EXECUTION SEQUENCE + +### Done (this session) +1. ✅ Full repo audit (5.7GB, 33 CTs, service topology, documentation drift) +2. ✅ Unified fleet inventory (`infrastructure/fleet/fleet.yaml`) +3. ✅ pfctl CLI — manages ALL nodes (Proxmox + Hetzner) with one command +4. ✅ Ansible inventory generator +5. ✅ Prometheus scrape config (all nodes) + +### What I can do NOW (no token needed, no destructive ops) +6. ✅ Repo cleanup Phase 1-2 (cargo clean, .gitignore) +7. ✅ Delete dead realms + stale docs +8. ✅ Rewrite AGENTS.md, DIRECTORY_GUIDE.md +9. ✅ Write OpenTofu for pf-aurelio (Hetzner, ready when token arrives) +10. ✅ Write OpenTofu for pf-forge CT238 + pf-stage CT239 (Dell LXC, deployable now) +11. ✅ Write Ansible roles for Forgejo + Hermes (provider-agnostic) + +### What needs the Hetzner token (only pf-aurelio) +12. ⏳ Provision pf-aurelio CPX31 on Hetzner +13. ⏳ Configure WireGuard pf-aurelio ↔ ASUS +14. ⏳ Migrate Gabinete/Bot-rs/fabricd/MCP services to pf-aurelio +15. ⏳ Cutover mcp.portugalfuturista.org → pf-aurelio + +### What needs Maestro confirmation (destructive) +16. Git history rewrite (force push) — coordinate timing +17. Kimi swarm systemd unit removal from CT223 — service interruption window +18. Forgejo data migration CT237→CT238 (git repos + DB dump/restore) diff --git a/.aurelio/providers/dist/dirac.providers.json b/.aurelio/providers/dist/dirac.providers.json index cba8cc9f..a229e4ab 100644 --- a/.aurelio/providers/dist/dirac.providers.json +++ b/.aurelio/providers/dist/dirac.providers.json @@ -44,9 +44,13 @@ "value": "anthropic", "label": "Anthropic" }, + { + "value": "google-cloud", + "label": "Google Cloud (Vertex AI)" + }, { "value": "gemini", - "label": "Google Gemini" + "label": "Google Gemini (AI Studio)" }, { "value": "moonshot", diff --git a/.aurelio/providers/dist/mcp.providers.json b/.aurelio/providers/dist/mcp.providers.json index f0880ec3..bf10bb53 100644 --- a/.aurelio/providers/dist/mcp.providers.json +++ b/.aurelio/providers/dist/mcp.providers.json @@ -3,6 +3,7 @@ "chatFallback": [ "kimi-coding", "qwen-code", + "google-cloud", "gemini", "anthropic", "openai", @@ -11,6 +12,7 @@ "openrouter" ], "embedFallback": [ + "google-cloud", "gemini", "openai", "qwen", @@ -160,9 +162,22 @@ "streaming": true, "fallback": true }, + { + "id": "google-cloud", + "label": "Google Cloud (Vertex AI)", + "auth": "api_key", + "keyEnv": "GOOGLE_CLOUD_API_KEY", + "baseUrl": "https://us-central1-aiplatform.googleapis.com/v1", + "protocol": "gemini", + "chatModel": "gemini-2.5-pro", + "embeddingModel": "text-embedding-004", + "maxContext": 1000000, + "streaming": true, + "fallback": true + }, { "id": "gemini", - "label": "Google Gemini", + "label": "Google Gemini (AI Studio)", "auth": "api_key", "keyEnv": "GEMINI_API_KEY", "baseUrl": "https://generativelanguage.googleapis.com/v1beta", diff --git a/.aurelio/providers/dist/portal.providers.json b/.aurelio/providers/dist/portal.providers.json index 3d166258..a3af1d9b 100644 --- a/.aurelio/providers/dist/portal.providers.json +++ b/.aurelio/providers/dist/portal.providers.json @@ -4,6 +4,7 @@ "fallback": [ "kimi-coding", "qwen-code", + "google-cloud", "gemini", "anthropic", "openai", @@ -177,9 +178,24 @@ "fallback": true, "notes": "" }, + { + "id": "google-cloud", + "label": "Google Cloud (Vertex AI)", + "kind": "api", + "protocol": "gemini", + "auth": "api_key", + "keyEnv": "GOOGLE_CLOUD_API_KEY", + "baseUrl": "https://us-central1-aiplatform.googleapis.com/v1", + "defaultModel": "gemini-2.5-pro", + "embeddingModel": "text-embedding-004", + "maxContext": 1000000, + "streaming": true, + "fallback": true, + "notes": "Vertex AI enterprise endpoint. OAuth/service-account fallback supported by GOOGLE_APPLICATION_CREDENTIALS." + }, { "id": "gemini", - "label": "Google Gemini", + "label": "Google Gemini (AI Studio)", "kind": "api", "protocol": "gemini", "auth": "api_key", diff --git a/.aurelio/providers/registry.yaml b/.aurelio/providers/registry.yaml index 9a9c53a1..902323ba 100644 --- a/.aurelio/providers/registry.yaml +++ b/.aurelio/providers/registry.yaml @@ -198,8 +198,24 @@ providers: streaming: true fallback: true + # ── Google Cloud ──────────────────────────────────────────────────────────── + - id: google-cloud + label: Google Cloud (Vertex AI) + kind: api + auth: api_key + key_env: GOOGLE_CLOUD_API_KEY + base_url: https://us-central1-aiplatform.googleapis.com/v1 + protocol: gemini + chat_model: gemini-2.5-pro + embedding_model: text-embedding-004 + max_context: 1000000 + surfaces: [dirac, mcp, portal] + streaming: true + fallback: true + notes: Vertex AI enterprise endpoint. OAuth/service-account fallback supported by GOOGLE_APPLICATION_CREDENTIALS. + - id: gemini - label: Google Gemini + label: Google Gemini (AI Studio) kind: api auth: api_key key_env: GEMINI_API_KEY @@ -340,6 +356,7 @@ providers: mcp_chat_fallback: - kimi-coding - qwen-code + - google-cloud - gemini - anthropic - openai @@ -349,6 +366,7 @@ mcp_chat_fallback: # Embedding preference order for the Python MCP router. mcp_embed_fallback: + - google-cloud - gemini - openai - qwen diff --git a/.aurelio/skills/gcp/SKILL.md b/.aurelio/skills/gcp/SKILL.md new file mode 100644 index 00000000..727b7aa7 --- /dev/null +++ b/.aurelio/skills/gcp/SKILL.md @@ -0,0 +1,220 @@ +--- +name: gcp +description: Google Cloud platform operations for PF — project setup, Vertex AI models, GKE, Cloud Run, Cloud Functions, Cloud Storage, Pub/Sub, BigQuery, Cloud SQL, Secret Manager, IAM, service accounts, and domain-wide delegation. Use when the user needs GCP-specific architecture, provisioning, or operations; prefer self-hosted or emulated dev paths before touching real GCP projects. +--- + +# gcp — Google Cloud Platform (enterprise) + +Distinct from the `gemini` provider (AI Studio consumer API) and `cli-gcloud` +(gcloud command snippets). This skill covers **Google Cloud Vertex AI / enterprise +workloads**: project setup, billing, IAM, model serving, compute, data, messaging, +and secrets. + +## When to use + +- New client/project onboarding to GCP +- Vertex AI model selection and routing +- Deploying or operating GKE, Cloud Run, Cloud Functions +- Cloud Storage, Pub/Sub, BigQuery, Cloud SQL, Secret Manager design +- Service accounts, workload identity, domain-wide delegation +- Deciding when to use GCP vs self-hosted PF infrastructure + +## When NOT to use + +- Consumer Gemini (AI Studio) → use the `gemini` provider in `.aurelio/providers/registry.yaml` +- Simple gcloud CLI one-liners → use `cli-gcloud` skill +- Firebase/GCP mobile-only features → not covered here + +## Project setup checklist + +1. **Create/select project** + ```bash + gcloud projects create pf-- --name="PF " + gcloud config set project pf-- + ``` + +2. **Billing** + - Link billing account in Cloud Console or via `gcloud billing projects link` + - Set budget alerts (recommended: 50 %, 90 %, 100 %) + - For clients, tag resources (`cost-center`, `client`, `realm`) + +3. **Enable APIs** (one-time per project) + ```bash + gcloud services enable aiplatform.googleapis.com + gcloud services enable container.googleapis.com + gcloud services enable run.googleapis.com + gcloud services enable cloudfunctions.googleapis.com + gcloud services enable storage.googleapis.com + gcloud services enable pubsub.googleapis.com + gcloud services enable bigquery.googleapis.com + gcloud services enable sqladmin.googleapis.com + gcloud services enable secretmanager.googleapis.com + gcloud services enable cloudbuild.googleapis.com + gcloud services enable iamcredentials.googleapis.com + ``` + +4. **IAM baseline** + - Create admin group `gcp-pf-admins@` with `roles/owner` or `roles/editor` + - Assign `roles/viewer` to auditors + - Enable essential audit logs (Admin Activity + Data Access for Secret Manager / Cloud SQL) + +## Service accounts & auth + +- **Human users**: use `gcloud auth login` + IAM bindings; no service-account keys in laptops. +- **Workloads**: use Workload Identity Federation where possible; otherwise create + dedicated service accounts per workload. +- **Keys**: store JSON keys in Vaultwarden / Secret Manager; rotate every 90 days. +- **Domain-wide delegation**: required for Workspace APIs (Directory, Gmail, Calendar, Drive). + - GCP Console → IAM & Admin → Service Accounts → `pf-@.iam.gserviceaccount.com` + - Add domain-wide delegation; note the Client ID + - Google Admin Console → Security → API controls → Domain-wide delegation + → Authorize the Client ID with OAuth scopes needed + - Store Client ID and delegated SA email in Secret Manager + +## Vertex AI + +Use Vertex AI for enterprise Gemini, Imagen, Veo, and embeddings. Region choice +affects latency and model availability; `europe-west1` is PF's default for EU +data residency. + +### Model IDs + +| Model | ID | Context | Type | Notes | +|-------|----|---------|------|-------| +| Gemini 2.5 Pro | `gemini-2.5-pro-preview-06-05` | 1M tokens | chat | Reasoning, long context, code | +| Gemini 2.5 Flash | `gemini-2.5-flash-preview-06-05` | 1M tokens | chat | Fast, cheaper | +| Gemini 2.0 Flash | `gemini-2.0-flash-001` | 1M tokens | chat | Stable GA | +| Imagen 3 | `imagen-3-0-generate-001` | N/A | image | Text-to-image | +| Veo 2 | `veo-2-0-generate-001` | N/A | video | Text/video-to-video | +| Text Embedding 004 | `text-embedding-004` | 2048 dims | embedding | Sentence + document | +| Multimodal Embedding | `multimodalembedding@001` | 128/256/512/1408 dims | embedding | Image + text | + +### Endpoint pattern + +``` +https://-aiplatform.googleapis.com/v1/projects//locations//publishers/google/models/ +``` + +### Example: generate content + +```bash +REGION=europe-west1 +PROJECT=$(gcloud config get-value project) +MODEL=gemini-2.5-flash-preview-06-05 + +curl -X POST \ + "https://${REGION}-aiplatform.googleapis.com/v1/projects/${PROJECT}/locations/${REGION}/publishers/google/models/${MODEL}:generateContent" \ + -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + -H "Content-Type: application/json" \ + -d '{"contents":[{"role":"user","parts":[{"text":"Summarize this in one line."}]}]}' +``` + +### Python (google-cloud-aiplatform) + +```python +import vertexai +from vertexai.generative_models import GenerativeModel + +vertexai.init(project="pf-client-env", location="europe-west1") +model = GenerativeModel("gemini-2.5-flash-preview-06-05") +response = model.generate_content("Hello, GCP world.") +print(response.text) +``` + +## Compute + +### GKE + +- Use Autopilot for most PF workloads unless node-level tuning is required. +- Default region: `europe-west1`; multi-region only when HA is justified. +- Enable Workload Identity; disable legacy metadata endpoints. + +```bash +gcloud container clusters create-auto pf-cluster \ + --region=europe-west1 \ + --release-channel=regular \ + --enable-workload-identity +``` + +### Cloud Run + +- Best for stateless HTTP services and MCP servers +- Use `--no-allow-unauthenticated` for internal agents; use IAM/service accounts for access +- Connect to Cloud SQL via proxy / native connections + +```bash +gcloud run deploy pf-agent-service \ + --source . \ + --region=europe-west1 \ + --no-allow-unauthenticated \ + --service-account=pf-run-sa@$PROJECT.iam.gserviceaccount.com +``` + +### Cloud Functions + +- Use 2nd gen for longer timeouts, bigger instances, and Eventarc triggers +- Trigger from Pub/Sub, Cloud Storage, Firestore, HTTP + +## Data & messaging + +### Cloud Storage + +- Buckets per realm/env; uniform bucket-level access enabled +- Lifecycle rules for logs and artifacts +- `gsutil` / `gcloud storage` are interchangeable; prefer `gcloud storage` + +```bash +gcloud storage buckets create gs://pf-client-data-euw1 --location=europe-west1 +gcloud storage buckets update gs://pf-client-data-euw1 --uniform-bucket-level-access +``` + +### Pub/Sub + +- Default for async agent-to-agent messaging +- Use push subscriptions to Cloud Run for reactive agents +- Enable exactly-once delivery when order/correctness matters + +```bash +gcloud pubsub topics create agent-events +gcloud pubsub subscriptions create agent-events-run \ + --topic agent-events \ + --push-endpoint=https://pf-agent-service-xxx-uc.a.run.app/events +``` + +### BigQuery + +- Use for analytics, telemetry, and structured agent memory at scale +- Datasets per client/realm; partition large tables by ingestion time +- Prefer service-account access via IAM `roles/bigquery.dataViewer` + +### Cloud SQL + +- PostgreSQL 16 default; use Cloud SQL Auth Proxy for local/dev access +- Private IP + VPC connector for GKE/Cloud Run +- Backups and point-in-time recovery enabled for production + +## Secret Manager + +- Central store for API keys, service-account JSONs, database passwords +- Name convention: `//` +- Rotate via Secret Manager versions; latest alias for runtime + +```bash +gcloud secrets create pf-client-db-password --data-file=- +# then paste value, Ctrl+D +``` + +## Surfaces + +- **dirac**: Vertex AI models appear as provider `google-cloud` in provider registry +- **mcp**: MCP servers running on Cloud Run / GKE; `model_router.py` can route to Vertex AI +- **portal**: PF portal can expose Vertex AI models and GCP-hosted agents under the Google Cloud offering + +## Pitfalls + +- `gemini` (AI Studio) and `google-cloud` (Vertex AI) are separate providers; never ship a Vertex AI key to an AI Studio endpoint. +- GCP IAM is eventually consistent; wait or retry after policy changes. +- Region matters for model availability and pricing — verify in Vertex AI Model Garden before hard-coding a region. +- Domain-wide delegation is powerful; limit scopes and monitor audit logs. +- Cloud Run cold starts can add latency; use min-instances for latency-sensitive agents. +- Storage egress and BigQuery query costs can surprise; set budgets early. diff --git a/.env.example b/.env.example index fedddaa7..a02e8306 100644 --- a/.env.example +++ b/.env.example @@ -60,3 +60,42 @@ TELEGRAM_TOKEN= TELEGRAM_ADMINS= # Optional sandbox root for `cd` / file transfers (absolute path). TELEGRAM_WORKING_DIR=/home/hermes/telminal-workspace + +# ─── MIRROR PIPELINE (sync-mirrors.py) ────────────────────────────────────── +# Upstream sources replicate into self-hosted mirrors: +# GitHub → Forgejo (code) +# Jira Cloud → Plane (issues) +# Confluence → Outline (docs) +# +# Registry: .aurelio/mirrors/sync-mirrors.yaml (edit THIS) +# Run: python3 scripts/sync-mirrors.py --sync +# +# ALL tokens are API tokens/PATs — fetch from Vaultwarden, never hardcode. + +# Forgejo (mirror target) — admin token with repo + org scopes. +# Generate: Settings → Applications → Access Tokens (manage_repo, write:org) +FORGEJO_MIRROR_TOKEN= + +# GitHub (upstream source) — classic PAT with repo + read:org scopes. +# Generate: https://github.com/settings/tokens (classic) +GITHUB_MIRROR_TOKEN= + +# Plane (mirror target) — personal API token. +# Generate: Profile → Settings → API Tokens +PLANE_API_TOKEN= + +# Outline (mirror target) — API token with read+write. +# Generate: Settings → API Tokens +OUTLINE_API_TOKEN= + +# Jira Cloud (upstream source) — API token + account email. +# Generate: https://id.atlassian.com/manage-profile/security/api-tokens +JIRA_API_TOKEN= +JIRA_USER_EMAIL= +# Jira Cloud base URL (e.g. https://savearth.atlassian.net) — set in sync-mirrors.yaml +# Leave empty here; the YAML endpoint field is null until the instance exists. + +# Confluence Cloud (upstream source) — same Atlassian account as Jira. +# Generate: same path as Jira (shares the Atlassian account) +CONFLUENCE_API_TOKEN= +CONFLUENCE_USER_EMAIL= diff --git a/.gitignore b/.gitignore index 358ee883..e6693c1b 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ hetzner.pub id_rsa id_ed25519 +.runner +.mimocode/.cron-lock +infrastructure/fabric/drift/target/ diff --git a/.hermes/plans/2026-07-14_160000-music-mvp-aurelio-telegram.md b/.hermes/plans/2026-07-14_160000-music-mvp-aurelio-telegram.md new file mode 100644 index 00000000..6e4ba36b --- /dev/null +++ b/.hermes/plans/2026-07-14_160000-music-mvp-aurelio-telegram.md @@ -0,0 +1,240 @@ +# Music MVP — Full Plan (Debug Mode + Daily Overview + Service Integration) + +> **For Hermes:** Use subagent-driven-development skill to implement this plan task-by-task. + +**Goal:** A unified music intelligence layer accessible from Telegram. Debug mode for service visibility, daily music digest, and deep integration with the full media stack. + +--- + +## Services Map (Gigabyte — 192.168.0.104) + +| Service | Port | Docker Container | API | Auth | Status | +|---------|------|-----------------|-----|------|--------| +| **Navidrome** | 4533 | pf-navidrome | Subsonic REST | user `fabio`, md5 token+salt | 🟢 | +| **Lidarr** | 8686 | pf-lidarr | `/api/v1` | `X-Api-Key: 24077d03b9a742a786dd799567659cb8` | 🟢 688 artists | +| **Prowlarr** | 9696 | pf-prowlarr | `/api/v1` | `X-Api-Key: c45adb4eff734ea491d04e1cbe93bca9` | 🟢 | +| **Slskd** | 5030 | pf-slskd | `/api/v0` | TBD | 🟢 v0.25.1 | +| **Jellyfin** | 8096 | pf-jellyfin | REST | API key `bc8c6c069daf4e4c8fec660a92114ac4` | 🟢 | +| **Maloja** | 42010 | CT 216 | `/apis/mlj_1/` | public (charts) | 🟢 | +| **Multi-scrobbler** | 9078 | CT 216 | — | — | 🟢 | +| **qBittorrent** | — | pf-qbittorrent | REST | TBD | 🟢 | + +### External Services + +| Service | API | Auth | Purpose | +|---------|-----|------|---------| +| **Last.fm** | `ws.audioscrobbler.com` | API key `b7bc626626486b5b60555db6ef836976`, session `L2nx0sc...` | Scrobbling + charts | +| **MusicBrainz** | `musicbrainz.org/ws/2` | User-Agent header | Artist/album metadata, MBIDs | +| **ListenBrainz** | `api.listenbrainz.org/1` | Token TBD | Open-source scrobbling + recommendations | + +### Data Flow + +``` +Playing music: + Navidrome/Jellyfin → Multi-scrobbler → Maloja + Last.fm + [ListenBrainz] + +Discovering music: + Lidarr → Prowlarr (indexers) → qBittorrent → library + MusicBrainz → artist metadata, MBIDs, release groups + Last.fm → user charts, similar artists, tags + ListenBrainz → recommendations, fresh releases + +Converting music: + Navidrome (download) → MuScriptor (transcribe) → MIDI files +``` + +--- + +## Feature 1: Debug Mode + +### What the user sees in Telegram: + +``` +/debug music — full status dashboard +/debug music navidrome — Navidrome logs + config +/debug music lidarr — Lidarr queue + recent activity +/debug music scrobbler — Multi-scrobbler status + recent scrobbles +/debug music services — all service health at a glance +/debug music config navidrome — Navidrome config dump +``` + +### Tools the MCP server exposes: + +```python +# Debug & Monitoring +music_debug_services() # health check all services +music_debug_logs(service, lines=50) # tail Docker logs +music_debug_config(service) # dump service config +music_debug_scrobbler_status() # multi-scrobbler sources + clients +music_debug_lidarr_activity() # recent Lidarr activity (imports, grabs) +music_debug_disk_usage() # music library disk usage +``` + +### Implementation: + +Each debug tool maps to a Docker API call or service REST endpoint: +- `docker logs --tail N ` for logs +- `docker inspect ` for config/env +- Service REST APIs for status endpoints + +--- + +## Feature 2: Daily Music Overview + +### What the user sees every morning: + +``` +🎵 Daily Music Brief — Monday, July 14 + +📊 Listening Stats (Last.fm/Maloja) + Top this week: LCD Soundsystem (12 plays), Caribou (8), Daft Punk (5) + Total scrobbles today: 23 + New discovery: "Komori Uta" by YIN YIN + +🆕 New Releases (from your 688 Lidarr artists) + • Caribou — "Honey" (single, released today) + • Tame Impala — "Wings of Time" (single, 2 days ago) + • LCD Soundsystem — "new body rhumba" (single, 3 days ago) + +📥 Library Status + Lidarr: 2 albums downloading, 0 missing + Disk: 1.2 TB / 2 TB (60%) + +🔗 Scrobbler Health + Navidrome → Maloja: ✅ Last: 2 min ago + Jellyfin → Maloja: ✅ Last: 1h ago + Navidrome → Last.fm: ✅ Last: 2 min ago +``` + +### Tools: + +```python +# Daily Overview +music_daily_brief() # full daily report +music_new_releases(days=7) # new releases from followed artists +music_listening_stats(period="week") # Last.fm + Maloja stats +music_library_health() # disk, missing, downloading +``` + +### New Releases Detection: + +Two approaches combined: +1. **Lidarr calendar** — `GET /api/v1/calendar?start=...&end=...` — releases Lidarr knows about +2. **MusicBrainz** — `GET /ws/2/release-group?artist=&type=album|single&first-release-date:[NOW-7DAYS TO NOW]` — any new release from followed artists +3. **Last.fm** — `artist.getNewReleases` — if available + +--- + +## Feature 3: ListenBrainz Integration + +### Why: +- Open-source alternative to Last.fm +- Better recommendation engine +- Cross-compatible with MusicBrainz MBIDs +- Can run alongside Last.fm (multi-scrobbler supports multiple clients) + +### How: +1. Register at `listenbrainz.org` +2. Get API token +3. Add ListenBrainz as a client in multi-scrobbler config +4. Use ListenBrainz API for recommendations + fresh releases + +### Multi-scrobbler config addition: + +```json +{ + "type": "listenbrainz", + "name": "ListenBrainz", + "data": { + "url": "https://api.listenbrainz.org", + "token": "" + } +} +``` + +--- + +## Feature 4: MusicBrainz Metadata + +### What MusicBrainz provides: +- Canonical artist/album/track IDs (MBIDs) +- Release groups, release dates, track listings +- Artist relationships (collaborations, aliases) +- Genre/tag taxonomy +- Cover art via Cover Art Archive + +### Tools: + +```python +# MusicBrainz +music_brainz_search(query) # search for artist/album +music_brainz_artist_info(mbid) # detailed artist info +music_brainz_releases(mbid, days=30) # recent releases for artist +music_brainz_similar(mbid) # similar artists +``` + +--- + +## Implementation Tasks + +### Task 1: music-mcp server skeleton (with debug tools) +Same pattern as muscriptor-mcp. FastMCP SSE on port 8089. + +### Task 2: Navidrome client (search, browse, download, stream) +Subsonic API: `search3`, `getAlbumList2`, `getRandomSongs`, `getNowPlaying`, `stream`, `download`. + +### Task 3: Lidarr client (library management + calendar) +Lidarr v3 API: artist CRUD, queue, wanted, calendar, lookup. + +### Task 4: Prowlarr client (indexer search) +`GET /api/v1/search` — search across all configured indexers. + +### Task 5: Last.fm client (stats, charts, new releases) +`user.getTopArtists`, `user.getTopTracks`, `user.getRecentTracks`, `artist.getInfo`, `artist.getSimilar`. + +### Task 6: MusicBrainz client (metadata, release groups) +`/ws/2/artist`, `/ws/2/release-group`, `/ws/2/release` with proper User-Agent. + +### Task 7: Debug tools (logs, config, health) +Docker API calls for container logs + inspect. Service REST endpoints for status. + +### Task 8: Daily overview composer +Combines data from all clients into a formatted daily brief. + +### Task 9: MuScriptor bridge +Calls muscriptor-mcp's `transcribe_to_midi` tool. + +### Task 10: Register + deploy +Register in `.aurelio/mcp_config.json`. Deploy on Gigabyte. Set up daily cron. + +### Task 11: ListenBrainz setup +Register account, add to multi-scrobbler, verify scrobbling works. + +--- + +## Files Summary + +| Action | Path | +|--------|------| +| Create | `scripts/music-mcp/pyproject.toml` | +| Create | `scripts/music-mcp/src/music_mcp/__init__.py` | +| Create | `scripts/music-mcp/src/music_mcp/server.py` | +| Create | `scripts/music-mcp/src/music_mcp/navidrome_client.py` | +| Create | `scripts/music-mcp/src/music_mcp/lidarr_client.py` | +| Create | `scripts/music-mcp/src/music_mcp/prowlarr_client.py` | +| Create | `scripts/music-mcp/src/music_mcp/lastfm_client.py` | +| Create | `scripts/music-mcp/src/music_mcp/musicbrainz_client.py` | +| Create | `scripts/music-mcp/src/music_mcp/muscriptor_bridge.py` | +| Create | `scripts/music-mcp/src/music_mcp/debug_tools.py` | +| Create | `scripts/music-mcp/src/music_mcp/daily_overview.py` | +| Create | `scripts/music-mcp/systemd/music-mcp.service` | +| Modify | `.aurelio/mcp_config.json` | + +## Cron Schedule + +| Time | Job | Output | +|------|-----|--------| +| 8:00 AM | `music_daily_brief()` | Telegram: daily music overview | +| On-demand | `/debug music` | Telegram: service status | +| On-demand | `/search Caribou` | Telegram: unified search results | +| On-demand | `/midi Sun Caribou` | Telegram: MIDI conversion | diff --git a/.hermes/plans/2026-07-14_180000-muscriptor-mcp-weekly-midi.md b/.hermes/plans/2026-07-14_180000-muscriptor-mcp-weekly-midi.md new file mode 100644 index 00000000..00216b49 --- /dev/null +++ b/.hermes/plans/2026-07-14_180000-muscriptor-mcp-weekly-midi.md @@ -0,0 +1,578 @@ +# MuScriptor MCP Server + Weekly MIDI Pipeline + +> **For Hermes:** Use subagent-driven-development skill to implement this plan task-by-task. + +**Goal:** Build a MuScriptor-powered MCP server that agents (Hermes, Aurelio) can call to transcribe audio → MIDI, and a weekly cron that auto-converts the user's top 3 tracks from their Navidrome/Maloja music stack. + +**Architecture:** Python MCP server exposing `transcribe_to_midi`, `get_top_tracks`, `search_and_transcribe` tools. Backed by MuScriptor (Kyutai/Mirelo, 1B param music transcription transformer). Connects to existing Maloja (stats) + Navidrome (audio source) via their APIs. Deployed as a systemd service on Gigabyte (RTX 3070 GPUs) or Tomahawk MAX. Weekly Hermes cron job queries top tracks → downloads → transcribes → delivers MIDI. + +**Tech Stack:** Python 3.12+, MuScriptor (`pip install muscriptor`), MCP Python SDK, Maloja API, Navidrome Subsonic API, Hermes cron. + +--- + +## Context + +### Existing Music Infrastructure + +| Service | URL | Port | Host | Purpose | +|---------|-----|------|------|---------| +| Navidrome | `music.portugalfuturista.org` | 4533 | Tomahawk MAX (Gigabyte) Docker | Music library + Subsonic API | +| Maloja | `maloja.portugalfuturista.org` | 42010 | CT 216 | Scrobble statistics | +| Multi-scrobbler | `scrobbler.portugalfuturista.org` | 9078 | CT 216 | Aggregates from Jellyfin, Navidrome, YTMusic → Maloja + Last.fm | +| Jellyfin | — | 8096 | Gigabyte Docker | Media server (also scrobbles) | + +### MuScriptor Model (Kyutai + Mirelo) + +- **Repo:** `github.com/muscriptor/muscriptor` (MIT license) +- **Models:** `muscriptor-small` (103M), `muscriptor-medium` (307M, default), `muscriptor-large` (1.4B) +- **License:** CC BY-NC 4.0 (requires free HF account + token) +- **Install:** `pip install muscriptor` or `uvx muscriptor` +- **Key API:** `model.transcribe_to_midi("audio.wav")` → MIDI bytes +- **GPU:** medium runs fine on RTX 3070 (8GB); large needs ~12GB VRAM +- **Supported instruments:** piano, drums, guitar, bass, strings, winds, etc. + +### How It Connects to Aurelio / Hermes + +- MCP server registered in `.aurelio/mcp_config.json` → available to all IDE agents +- Hermes cron job calls the MCP tools weekly +- User can also invoke directly: *"Hermes, convert Sun by Caribou to MIDI"* + +--- + +## Implementation Plan + +### Phase 1: MCP Server Core (muscriptIdor-mcp) + +#### Task 1: Create MCP server skeleton + +**Objective:** Scaffold the Python MCP server project structure. + +**Files:** +- Create: `scripts/muscriptor-mcp/pyproject.toml` +- Create: `scripts/muscriptor-mcp/src/muscriptor_mcp/__init__.py` +- Create: `scripts/muscriptor-mcp/src/muscriptor_mcp/server.py` + +**Step 1:** Create project with `pyproject.toml`: + +```toml +[project] +name = "muscriptor-mcp" +version = "0.1.0" +description = "MCP server for MuScriptor music transcription (audio → MIDI)" +requires-python = ">=3.10" +dependencies = [ + "mcp[cli]>=1.0.0", + "muscriptor>=0.1.0", + "httpx>=0.27", + "pydantic>=2.0", +] + +[project.scripts] +muscriptor-mcp = "muscriptor_mcp.server:main" +``` + +**Step 2:** Create the MCP server skeleton with `server.py`: + +```python +"""MuScriptor MCP Server — audio → MIDI transcription for Aurelio agents.""" +import asyncio +import logging +from pathlib import Path +from mcp.server import Server +from mcp.server.stdio import run_server +from mcp.types import Tool, TextContent + +logger = logging.getLogger(__name__) +app = Server("muscriptor-mcp") + +@app.list_tools() +async def list_tools() -> list[Tool]: + return [ + Tool( + name="transcribe_to_midi", + description="Transcribe an audio file (WAV/MP3/FLAC/OGG) to MIDI using MuScriptor. " + "Returns the path to the generated .mid file.", + inputSchema={ + "type": "object", + "properties": { + "audio_path": {"type": "string", "description": "Path to audio file"}, + "output_dir": {"type": "string", "description": "Output directory for MIDI (default: same as input)"}, + "model_size": {"type": "string", "enum": ["small", "medium", "large"], "default": "medium"}, + "instruments": {"type": "array", "items": {"type": "string"}, "description": "Restrict to specific instruments"}, + }, + "required": ["audio_path"], + }, + ), + Tool( + name="get_top_tracks", + description="Get the user's most-played tracks from Maloja scrobble stats.", + inputSchema={ + "type": "object", + "properties": { + "count": {"type": "integer", "default": 3, "description": "Number of top tracks"}, + "period": {"type": "string", "default": "week", "description": "Time period: day, week, month, year, overall"}, + }, + }, + ), + Tool( + name="search_and_download", + description="Search Navidrome for a track and download the audio file locally.", + inputSchema={ + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query (e.g. 'Sun Caribou')"}, + "download_dir": {"type": "string", "description": "Directory to save audio (default: /tmp/muscriptor)"}, + }, + "required": ["query"], + }, + ), + Tool( + name="top_tracks_to_midi", + description="Full pipeline: fetch top N tracks from Maloja, download from Navidrome, " + "transcribe each to MIDI. Returns paths to all generated .mid files.", + inputSchema={ + "type": "object", + "properties": { + "count": {"type": "integer", "default": 3}, + "period": {"type": "string", "default": "week"}, + "model_size": {"type": "string", "enum": ["small", "medium", "large"], "default": "medium"}, + }, + }, + ), + ] + +async def main(): + async with run_server(app) as server: + await server.serve_forever() + +if __name__ == "__main__": + asyncio.run(main()) +``` + +**Verification:** `cd scripts/muscriptor-mcp && pip install -e . && python -c "from muscriptor_mcp.server import app; print('OK')"` + +--- + +#### Task 2: Implement Maloja API client (get_top_tracks) + +**Objective:** Query Maloja for top tracks with play counts. + +**Files:** +- Create: `scripts/muscriptor-mcp/src/muscriptor_mcp/maloja_client.py` + +**Step 1:** Maloja exposes a REST API. Key endpoints: + +``` +GET /api/v1/top/tracks?period=week&max=3 +Authorization: Token +``` + +Response shape (from Maloja docs): +```json +{ + "list": [ + { + "track": {"title": "Sun", "artists": [{"name": "Caribou"}]}, + "amount": 12 + } + ] +} +``` + +**Step 2:** Implement the client: + +```python +"""Maloja scrobble statistics client.""" +import httpx +from dataclasses import dataclass + +MALOJA_URL = "http://pf-maloja:42010" # LAN — also accessible via maloja.portugalfuturista.org +MALOJA_API_KEY = "" # TODO: load from env MALOJA_API_KEY + +@dataclass +class TrackStat: + title: str + artist: str + play_count: int + +async def get_top_tracks(count: int = 3, period: str = "week") -> list[TrackStat]: + """Fetch top tracks from Maloja API.""" + url = f"{MALOJA_URL}/api/v1/top/tracks" + params = {"period": period, "max": count} + headers = {"Authorization": f"Token {MALOJA_API_KEY}"} + async with httpx.AsyncClient() as client: + resp = await client.get(url, params=params, headers=headers, timeout=15) + resp.raise_for_status() + data = resp.json() + return [ + TrackStat( + title=item["track"]["title"], + artist=item["track"]["artists"][0]["name"], + play_count=item["amount"], + ) + for item in data["list"] + ] +``` + +**Verification:** Test with `curl` against live Maloja first to confirm API shape, then unit test with mocked response. + +--- + +#### Task 3: Implement Navidrome Subsonic API client (search_and_download) + +**Objective:** Search Navidrome and download audio files. + +**Files:** +- Create: `scripts/muscriptor-mcp/src/muscriptor_mcp/navidrome_client.py` + +**Step 1:** Navidrome exposes the Subsonic API. Key endpoints: + +``` +GET /rest/search3.view?query=Sun+Caribou&u=fabio&t=&s=&v=1.16.1&c=muscriptor +GET /rest/download.view?id=&u=fabio&t=&s=&v=1.16.1&c=muscriptor +``` + +Subsonic auth: token = md5(password + salt). + +**Step 2:** Implement: + +```python +"""Navidrome (Subsonic API) client for audio download.""" +import hashlib +import os +import secrets +from pathlib import Path +import httpx + +NAVIDROME_URL = "http://pf-navidrome:4533" # LAN +NAVIDROME_USER = "" # TODO: load from env +NAVIDROME_PASSWORD = "" # TODO: load from env + +def _subsonic_params() -> dict: + salt = secrets.token_hex(8) + token = hashlib.md5((NAVIDROME_PASSWORD + salt).encode()).hexdigest() + return {"u": NAVIDROME_USER, "t": token, "s": salt, "v": "1.16.1", "c": "muscriptor-mcp"} + +async def search_track(query: str) -> list[dict]: + """Search Navidrome for tracks matching query. Returns list of {id, title, artist, duration}.""" + params = {**_subsonic_params(), "query": query, "songCount": 5} + async with httpx.AsyncClient() as client: + resp = await client.get(f"{NAVIDROME_URL}/rest/search3.view", params=params, timeout=15) + resp.raise_for_status() + data = resp.json() + songs = data.get("subsonic-response", {}).get("searchResult3", {}).get("song", []) + return [{"id": s["id"], "title": s["title"], "artist": s.get("artist", ""), "duration": s.get("duration", 0)} for s in songs] + +async def download_track(song_id: str, output_dir: Path) -> Path: + """Download a track by ID. Returns local file path.""" + output_dir.mkdir(parents=True, exist_ok=True) + params = {**_subsonic_params(), "id": song_id} + async with httpx.AsyncClient() as client: + resp = await client.get(f"{NAVIDROME_URL}/rest/download.view", params=params, timeout=120) + resp.raise_for_status() + # Navidrome sends the file with its original extension + content_type = resp.headers.get("content-type", "audio/mpeg") + ext = _ext_from_content_type(content_type) + out_path = output_dir / f"{song_id}{ext}" + out_path.write_bytes(resp.content) + return out_path + +def _ext_from_content_type(ct: str) -> str: + mapping = { + "audio/mpeg": ".mp3", "audio/flac": ".flac", "audio/ogg": ".ogg", + "audio/wav": ".wav", "audio/x-wav": ".wav", "audio/mp4": ".m4a", + } + return mapping.get(ct.split(";")[0].strip(), ".mp3") +``` + +**Verification:** Test search with `curl` against live Navidrome, confirm Subsonic auth works. + +--- + +#### Task 4: Implement MuScriptor transcription engine + +**Objective:** Wrap MuScriptor's `transcribe_to_midi` in an async wrapper. + +**Files:** +- Create: `scripts/muscriptor-mcp/src/muscriptor_mcp/transcriber.py` + +**Step 1:** + +```python +"""MuScriptor audio → MIDI transcription engine.""" +import asyncio +from pathlib import Path +from functools import lru_cache + +@lru_cache(maxsize=1) +def _load_model(size: str = "medium"): + """Load and cache the MuScriptor model (singleton per size).""" + from muscriptor import TranscriptionModel + return TranscriptionModel.load_model(size) + +async def transcribe(audio_path: str | Path, output_path: str | Path | None = None, + model_size: str = "medium", instruments: list[str] | None = None) -> Path: + """Transcribe audio file to MIDI. Returns path to .mid file.""" + audio_path = Path(audio_path) + if output_path is None: + output_path = audio_path.with_suffix(".mid") + else: + output_path = Path(output_path) + + model = _load_model(model_size) + + # MuScriptor's transcribe is CPU/GPU-bound, run in thread pool + loop = asyncio.get_event_loop() + midi_bytes = await loop.run_in_executor(None, lambda: model.transcribe_to_midi(str(audio_path))) + output_path.write_bytes(midi_bytes) + return output_path +``` + +**Verification:** Run on a short WAV file to confirm model downloads + produces valid MIDI. + +--- + +#### Task 5: Wire tools into MCP server + +**Objective:** Connect all modules into the MCP server's tool handlers. + +**Files:** +- Modify: `scripts/muscriptor-mcp/src/muscriptor_mcp/server.py` — add `@app.call_tool()` handler + +**Step 1:** Add the tool dispatch to `server.py`: + +```python +from muscriptor_mcp.maloja_client import get_top_tracks +from muscriptor_mcp.navidrome_client import search_track, download_track +from muscriptor_mcp.transcriber import transcribe + +DEFAULT_DOWNLOAD_DIR = Path("/tmp/muscriptor") + +@app.call_tool() +async def call_tool(name: str, arguments: dict) -> list[TextContent]: + if name == "transcribe_to_midi": + audio_path = arguments["audio_path"] + output_dir = arguments.get("output_dir") + model_size = arguments.get("model_size", "medium") + instruments = arguments.get("instruments") + out = Path(output_dir) / Path(audio_path).with_suffix(".mid").name if output_dir else None + result = await transcribe(audio_path, out, model_size, instruments) + return [TextContent(type="text", text=f"MIDI saved to: {result}")] + + elif name == "get_top_tracks": + tracks = await get_top_tracks(arguments.get("count", 3), arguments.get("period", "week")) + lines = [f"#{i+1}: {t.artist} — {t.title} ({t.play_count} plays)" for i, t in enumerate(tracks)] + return [TextContent(type="text", text="\n".join(lines))] + + elif name == "search_and_download": + results = await search_track(arguments["query"]) + if not results: + return [TextContent(type="text", text=f"No results for '{arguments['query']}'")] + track = results[0] + dl_dir = Path(arguments.get("download_dir", str(DEFAULT_DOWNLOAD_DIR))) + path = await download_track(track["id"], dl_dir) + return [TextContent(type="text", text=f"Downloaded: {track['artist']} — {track['title']} → {path}")] + + elif name == "top_tracks_to_midi": + count = arguments.get("count", 3) + period = arguments.get("period", "week") + model_size = arguments.get("model_size", "medium") + tracks = await get_top_tracks(count, period) + results = [] + dl_dir = DEFAULT_DOWNLOAD_DIR / "weekly" + for t in tracks: + query = f"{t.title} {t.artist}" + search_results = await search_track(query) + if not search_results: + results.append(f"SKIP: {t.artist} — {t.title} (not found in Navidrome)") + continue + audio_path = await download_track(search_results[0]["id"], dl_dir) + midi_path = await transcribe(audio_path, model_size=model_size) + results.append(f"DONE: {t.artist} — {t.title} → {midi_path}") + return [TextContent(type="text", text="\n".join(results))] + + else: + return [TextContent(type="text", text=f"Unknown tool: {name}")] +``` + +**Verification:** `cd scripts/muscriptor-mcp && python -m muscriptor_mcp.server` starts without errors. + +--- + +### Phase 2: Configuration & Secrets + +#### Task 6: Environment-based secrets + +**Objective:** Load API keys from environment variables, never hardcode. + +**Files:** +- Create: `scripts/muscriptor-mcp/.env.example` +- Modify: `scripts/muscriptor-mcp/src/muscriptor_mcp/maloja_client.py` — read from env +- Modify: `scripts/muscriptor-mcp/src/muscriptor_mcp/navidrome_client.py` — read from env + +**`.env.example`:** +```bash +MALOJA_URL=http://pf-maloja:42010 +MALOJA_API_KEY=your_maloja_api_key +NAVIDROME_URL=http://pf-navidrome:4533 +NAVIDROME_USER=fabio +NAVIDROME_PASSWORD=your_navidrome_password +HF_TOKEN=your_huggingface_token +``` + +**Verification:** Start server without .env → graceful error. Start with .env → tools work. + +--- + +#### Task 7: Register MCP server in Aurelio config + +**Objective:** Make the server discoverable by Aurelio agents. + +**Files:** +- Modify: `.aurelio/mcp_config.json` — add `muscriptor-mcp` entry + +**Add entry:** +```json +"muscriptor-mcp": { + "command": "python", + "args": ["-m", "muscriptor_mcp.server"], + "env": { + "MALOJA_URL": "http://pf-maloja:42010", + "NAVIDROME_URL": "http://pf-navidrome:4533" + }, + "_disabled": false, + "disabledTools": [] +} +``` + +Note: secrets (API keys, passwords) go in the env file, not in mcp_config.json. The MCP server reads them from its own environment. + +--- + +### Phase 3: Deployment on GPU Node + +#### Task 8: Create systemd service for Gigabyte / Tomahawk MAX + +**Objective:** Run the MCP server as a persistent service on the GPU node. + +**Files:** +- Create: `scripts/muscriptor-mcp/systemd/muscriptor-mcp.service` + +**Systemd unit:** +```ini +[Unit] +Description=MuScriptor MCP Server (audio → MIDI) +After=network.target docker.service + +[Service] +Type=simple +User=fabio +WorkingDirectory=/home/fabio/muscriptor-mcp +EnvironmentFile=/home/fabio/muscriptor-mcp/.env +ExecStart=/home/fabio/muscriptor-mcp/.venv/bin/python -m muscriptor_mcp.server +Restart=on-failure +RestartSec=5 +# GPU access +SupplementaryGroups=video render + +[Install] +WantedBy=multi-user.target +``` + +**Deployment steps:** +1. Clone repo to Gigabyte node (`192.168.0.104`) +2. `cd scripts/muscriptor-mcp && python -m venv .venv && . .venv/bin/activate && pip install -e .` +3. Create `.env` with real secrets +4. Copy service file, enable, start +5. First run will download ~1.2GB model weights (cached after) + +--- + +#### Task 9: Expose via Cloudflare tunnel (optional) + +**Objective:** Make MCP server accessible from Hermes gateway (CT 217) and laptop. + +**Option A:** SSH tunnel from CT 217 to Gigabyte (preferred, no public exposure). +**Option B:** Add Cloudflare tunnel route for `muscriptor-mcp.portugalfuturista.org`. + +--- + +### Phase 4: Weekly Cron Job + +#### Task 10: Hermes cron job for weekly top-3 MIDI conversion + +**Objective:** Every Monday morning, auto-convert top 3 weekly tracks to MIDI. + +**Using Hermes cron:** +``` +Schedule: 0 9 * * 1 (every Monday at 09:00) +Prompt: "Run the muscriptor-mcp top_tracks_to_midi tool with count=3, period=week, model_size=medium. + Report which tracks were converted and where the MIDI files are saved." +Deliver: telegram (or wherever the user wants) +``` + +**Or via the MCP tool directly from any agent:** +> "Hermes, convert my top 3 tracks this week to MIDI" + +--- + +### Phase 5: Interactive Usage + +#### Task 11: Document agent usage patterns + +**Objective:** Show how users interact with the transcription tools. + +**Usage patterns:** + +1. **Ad-hoc transcription:** + > "Hermes, here's Sun by Caribou. Convert it to MIDI for Ableton." + → Agent calls `search_and_download` then `transcribe_to_midi` + +2. **Weekly auto-conversion:** + → Cron job runs `top_tracks_to_midi` automatically + +3. **From a file path:** + > "Transcribe /tmp/my_recording.wav to MIDI" + → Agent calls `transcribe_to_midi` directly + +4. **With instrument restriction:** + > "Transcribe just the piano from this track" + → Agent calls `transcribe_to_midi` with `instruments=["acoustic_piano"]` + +5. **Batch conversion:** + > "Convert all tracks from this album to MIDI" + → Agent searches Navidrome, loops `transcribe_to_midi` + +--- + +## Files Summary + +| Action | Path | +|--------|------| +| Create | `scripts/muscriptor-mcp/pyproject.toml` | +| Create | `scripts/muscriptor-mcp/src/muscriptor_mcp/__init__.py` | +| Create | `scripts/muscriptor-mcp/src/muscriptor_mcp/server.py` | +| Create | `scripts/muscriptor-mcp/src/muscriptor_mcp/maloja_client.py` | +| Create | `scripts/muscriptor-mcp/src/muscriptor_mcp/navidrome_client.py` | +| Create | `scripts/muscriptor-mcp/src/muscriptor_mcp/transcriber.py` | +| Create | `scripts/muscriptor-mcp/.env.example` | +| Create | `scripts/muscriptor-mcp/systemd/muscriptor-mcp.service` | +| Modify | `.aurelio/mcp_config.json` (add muscriptor-mcp entry) | + +## Risks & Open Questions + +1. **GPU VRAM:** MuScriptor medium (307M) should fit in RTX 3070 8GB. Large (1.4B) needs ~12GB — may OOM on 8GB cards. Use medium by default. +2. **Audio format compatibility:** MuScriptor supports WAV natively. For MP3/FLAC/OGG, may need `ffmpeg` or `soundfile` for pre-conversion. Navidrome sends original format. +3. **Maloja API shape:** Need to verify the exact API response structure — Maloja docs are sparse. Test with live instance first. +4. **Navidrome auth:** Subsonic API uses token+salt auth. The multi-scrobbler config has credentials but they should go in .env, not committed. +5. **Network path:** MCP server on Gigabyte needs to reach Maloja on CT 216 (192.168.0.126:42010) and Navidrome on Gigabyte Docker (pf-navidrome:4533). Verify LAN connectivity. +6. **Model download:** First run downloads ~1.2GB from HuggingFace. Requires `HF_TOKEN` env var (free HF account). +7. **Hermes cron limitation:** Cron jobs in TUI are local-only. For the weekly job to work reliably, either run it from the Hermes gateway on CT 217, or use a systemd timer on the GPU node. + +## Dependencies to Install + +```bash +pip install muscriptor mcp[cli] httpx pydantic +``` + +MuScriptor pulls in: `torch`, `torchaudio`, `transformers`, `safetensors`, `soundfile`. diff --git a/AGENTS.md b/AGENTS.md index 1d04c909..c455da5b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -97,6 +97,39 @@ small builtin set if absent). The Theia backend exposes the same data at is what populates the chat widget's provider picker and what `porta.portugalfuturista.org` re-serves at `/api/models/providers`. +## Mirror pipeline (upstream → self-hosted mirrors) + +One-way replication of Savearth upstream sources into self-hosted mirrors, +with RL feedback. Agents work on the mirrors; originals stay authoritative. + +``` +GitHub (SavearthTech/*) ──► Forgejo (code.portugalfuturista.org) +Jira Cloud ──► Plane (plane.portugalfuturista.org) +Confluence Cloud ──► Outline (docs.portugalfuturista.org) +``` + +| File | Role | +|------|------| +| `.aurelio/mirrors/sync-mirrors.yaml` | **Edit this** — sources, targets, auth refs | +| `.aurelio/mirrors/dist/` | Generated catalog (do not edit) | +| `scripts/sync-mirrors.py` | The engine: `--sync`, `--check`, `--write-in-place` | +| `.forgejo/workflows/mirror-sync.yml` | CI: runs every 15 min | + +Registry: `.aurelio/mirrors/sync-mirrors.yaml`. Tokens in env vars (never in +the YAML) — see `.env.example` for the full list. Full guide: +`.aurelio/mirrors/README.md`. + +```bash +python3 scripts/sync-mirrors.py --sync # all targets +python3 scripts/sync-mirrors.py --sync forgejo # one target +python3 scripts/sync-mirrors.py --write-in-place # regen dist after YAML edit +python3 scripts/sync-mirrors.py --check # CI guard +``` + +Sync outcomes emit RL reward signals to +`.aurelio/brain/trajectory-rewards/mirror-sync.jsonl` — consumed by the +GRPO/DPO trajectory training pipeline. + ## CLI guides & service lenses **Self-hosted first policy**: dev work targets our own services; public clouds @@ -205,6 +238,86 @@ service disabled). Platform creds live in `/home/hermes/.hermes/.env` (`TELEGRAM `SIGNAL_*` (LAN-direct to .104:18088), `MATRIX_*`). State migrates via `hermes backup` → `hermes import`. +## Client onboarding (replicating into new projects) + +Each Portugal Futurista client gets its own replica-omnisciente — a self-contained +agent fleet brain with the same power set: `.aurelio/` (config cascade, brain, +skills, providers, connectors, MCP, identity/heteronyms), `realms/`, `scripts/`, +and CI/CD. The onboarding tool scaffolds this from the canonical template. + +### Scaffold a new client + +```bash +# Interactive wizard (recommended for first-time use) +python3 scripts/onboard-client.py --interactive + +# CLI flags +python3 scripts/onboard-client.py \ + --client-name "Acme Corp" \ + --git-url https://github.com/acme/replica-acme \ + --lead-engineer "Jane Doe" --lead-email "jane@acme.com" \ + --realm iot-firmware --realm-name "IoT Firmware" --realm-stack "ESP32, Rust" \ + --realm cloud-backend --realm-name "Cloud Backend" --realm-stack "Rust, Axum" \ + --output ~/clients/acme-corp + +# JSON config file (reproducible, version-controlled) +python3 scripts/onboard-client.py --config onboarding-acme.json + +# Dry run (preview the file tree without writing) +python3 scripts/onboard-client.py --client-name "Acme Corp" --dry-run +``` + +### What gets scaffolded + +| Component | Description | +|-----------|-------------| +| `AGENTS.md` + `README.md` + `.gitignore` + `.env.example` | Root project files with client identity | +| `.aurelio/config.toml` | Three-tier config cascade root | +| `.aurelio/brain/` + `memory/` + `plans/` + `sessions/` | Brain storage (empty, ready to fill) | +| `.aurelio/identity/heteronyms.json` + `heteronimos/*.md` | Team members as heteronyms (voices) | +| `.aurelio/providers/registry.yaml` | AI provider registry (OpenRouter, Gemini, Anthropic) | +| `.aurelio/connectors/registry.yaml` | External integration catalog (git forge auto-detected) | +| `.aurelio/skills/` | Full CLI skills tree copied from canonical replica | +| `.aurelio/sync.py` | Brain sync utility (push/pull to Proxmox) | +| `.aurelio/mcp_config.json` | MCP server fleet (empty, ready to configure) | +| `.aurelio/chronicle/` | Event timeline + per-realm chronicle stubs | +| `realms//` | Per-project: `AGENTS.md`, `.aurelio/config.toml`, `memory/index.md` | +| `scripts/` | Onboarding CLI (self-replicating), agent importers, mirror generators | +| `.forgejo/workflows/` + `.github/workflows/` | CI/CD for both forge types | + +### Power toggles + +Skip components with `--no-*` flags: +```bash +python3 scripts/onboard-client.py --client-name "Lite" --no-skills --no-connectors --no-ci +``` + +### Managing realms after scaffolding + +The scaffolded project includes its own `onboard-client.py` CLI: +```bash +cd +python3 scripts/onboard-client.py add-realm my-project --name "My Project" --repo +python3 scripts/onboard-client.py list-realms +python3 scripts/onboard-client.py info +``` + +### Onboarding config format (JSON) + +Save as `onboarding-.json` for reproducible scaffolds: +```json +{ + "client_name": "Acme Corp", + "description": "IoT infrastructure", + "git_url": "https://github.com/acme/replica-acme", + "lead_engineer_name": "Jane Doe", + "lead_engineer_email": "jane@acme.com", + "realms": [ + {"slug": "iot-fw", "name": "IoT Firmware", "stack": "ESP32, Rust"} + ] +} +``` + ## Gotchas - `extensions/aurelio-vscode/` and `aurelio-theia/` are git submodules — run `git submodule update --init` after clone; edit them in their own repos (`~/portugalfuturista/aurelio-vscode`, `~/portugalfuturista/aurelio-theia`) and push to Forgejo. diff --git a/scripts/brain-to-gbrain.py b/scripts/brain-to-gbrain.py new file mode 100644 index 00000000..4e2792c3 --- /dev/null +++ b/scripts/brain-to-gbrain.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python3 +""" +Bridge: Réplica brain sessions → gbrain conversation pages. + +Reads .aurelio/brain/session-*/ directories and pushes each session as a +gbraidge 'conversation' page via the gbrain CLI. + +Output per session: + slug: conversations// + body: YAML frontmatter + formatted transcript + +Usage: + python3 scripts/brain-to-gbrain.py [--dry-run] [--source NAME]... + [--replica-root PATH] + [--ssh HOST] [--gbrain-bin PATH] + [--summary] [--limit N] + +Modes: + --ssh USER@HOST Push via SSH to remote gbrain (CT 208). + Default: root@192.168.0.38 (Proxmox → pct 208) + (local) Use local gbrain CLI (must be initialized locally). +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +_here = Path(__file__).resolve() +for _candidate in (_here.parent, *_here.parents): + if (_candidate / ".aurelio").is_dir(): + REPLICA_ROOT = _candidate + break +else: + REPLICA_ROOT = _here.parents[2] + +BRAIN_DIR = REPLICA_ROOT / ".aurelio" / "brain" + +VALID_SOURCES = { + "kimi", "hermes", "claude-code", "antigravity", "qwen-code", + "mimocode", "pi", "opencode", +} + +DEFAULT_SSH = "root@192.168.0.38" +REMOTE_GBRAIN_BIN = "/opt/pf-services-208/aurelio-gbrain/bin/gbrain" +REMOTE_CT = "208" + + +def _load_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + +def _load_chat_history(session_dir: Path) -> dict | None: + ch = session_dir / ".system_generated" / "chat_history.json" + return _load_json(ch) + + +def _load_session_meta(session_dir: Path) -> dict | None: + sj = session_dir / "session.jsonl" + if not sj.exists(): + return None + lines = sj.read_text(encoding="utf-8").strip().split("\n") + meta = {} + for line in lines: + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue + if rec.get("type") == "session_start": + meta.update(rec.get("data", {})) + meta["created"] = rec.get("timestamp") + elif rec.get("type") == "session_end": + meta["totalEvents"] = rec.get("data", {}).get("totalEvents", 0) + return meta or None + + +def _format_transcript(chat_history: dict) -> str: + messages = chat_history.get("messages", []) + lines = [] + for msg in messages: + role = msg.get("role", "unknown") + content = msg.get("content", "") + if not content: + continue + model = msg.get("model") or msg.get("modelType", "") + if role == "user": + lines.append(f"## User\n\n{content}\n") + elif role == "assistant": + header = f"## Assistant" + if model: + header += f" ({model})" + lines.append(f"{header}\n\n{content}\n") + elif role in ("tool_call", "tool_result"): + lines.append(f"### {role}\n\n{content[:800]}\n") + elif role == "system": + lines.append(f"### System\n\n{content[:500]}\n") + return "\n".join(lines) + + +def _make_page_content(session_dir: Path, chat_history: dict, meta: dict | None) -> str: + source = chat_history.get("source", "unknown") + title = chat_history.get("title", f"{source} session") + created = chat_history.get("created", "") + modified = chat_history.get("modified", "") + workspace = chat_history.get("workspacePath", "") + session_id = chat_history.get("id", session_dir.name) + source_locator = chat_history.get("sourceLocator", "") + model_type = chat_history.get("modelType", source) + messages = chat_history.get("messages", []) + total_events = meta.get("totalEvents", len(messages)) if meta else len(messages) + + tags = [source, "agent-session"] + if workspace: + tags.append(Path(workspace).name if workspace else "") + tags = [t for t in tags if t] + + frontmatter_lines = [ + "---", + "type: conversation", + f'title: "{title.replace(chr(34), chr(39))}"', + f"date: {created[:10] if created else 'unknown'}", + f"source: {source}", + f"model: {model_type}", + f"tags: [{', '.join(tags)}]", + f"workspace: {workspace or 'unknown'}", + f"brain_session_id: {session_id}", + f"message_count: {len(messages)}", + f"total_events: {total_events}", + ] + if source_locator: + frontmatter_lines.append(f"source_locator: \"{source_locator}\"") + if modified: + frontmatter_lines.append(f"modified: {modified}") + frontmatter_lines.append("---") + frontmatter_lines.append("") + + body_lines = [ + f"# {title}", + "", + f"> Source: **{source}** | Model: **{model_type}** | Messages: **{len(messages)}**", + ] + if workspace: + body_lines.append(f"> Workspace: `{workspace}`") + body_lines.append("") + body_lines.append("## Transcript") + body_lines.append("") + body_lines.append(_format_transcript(chat_history)) + + return "\n".join(frontmatter_lines + body_lines) + + +# --------------------------------------------------------------------------- +# Remote SSH mode: stage files locally, then import on CT 208 via --no-embed +# --------------------------------------------------------------------------- + +def _ssh_gbrain_import(ssh_host: str, staging_dir: Path, dry_run: bool) -> dict: + """SCP staging dir to CT 208, run gbrain import --no-embed.""" + if dry_run: + count = len(list(staging_dir.glob("*.md"))) + return {"ok": True, "imported": count, "dry_run": True} + + try: + # Tar + scp for efficiency — extract to /tmp so files land at /tmp// + tar_cmd = f"tar czf - -C {staging_dir.parent} {staging_dir.name}" + ssh_cmd = f"ssh -o ConnectTimeout=10 {ssh_host} pct exec {REMOTE_CT} -- tar xzf - -C /tmp" + tar_proc = subprocess.Popen(tar_cmd, shell=True, stdout=subprocess.PIPE) + ssh_proc = subprocess.Popen(ssh_cmd, shell=True, stdin=tar_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + tar_proc.stdout.close() + tar_proc.wait() + ssh_proc.wait() + + if tar_proc.returncode != 0 or ssh_proc.returncode != 0: + return {"ok": False, "error": f"tar/scp failed: {ssh_proc.stderr.decode()[:200]}"} + + remote_import_path = f"/tmp/{staging_dir.name}" + + # Run gbrain import --no-embed + import_cmd = ( + f"pct exec {REMOTE_CT} -- {REMOTE_GBRAIN_BIN} " + f"import {remote_import_path} --no-embed" + ) + r = subprocess.run( + ["ssh", "-o", "ConnectTimeout=10", ssh_host, import_cmd], + capture_output=True, text=True, timeout=600, + ) + + # Cleanup remote + subprocess.run( + ["ssh", "-o", "ConnectTimeout=5", ssh_host, + f"pct exec {REMOTE_CT} -- rm -rf {remote_import_path}"], + capture_output=True, timeout=15, + ) + + if r.returncode == 0: + output = r.stdout + r.stderr + imported = 0 + for line in output.split("\n"): + if "imported" in line and "pages" in line: + parts = line.split() + for i, p in enumerate(parts): + if p == "imported" and i > 0: + try: + imported = int(parts[i - 1]) + except ValueError: + pass + return {"ok": True, "imported": imported, "output": output.strip()[:500]} + # Even with non-zero exit, check if pages were imported (warnings cause exit 1) + output = (r.stdout or "") + (r.stderr or "") + imported = 0 + for line in output.split("\n"): + if "imported" in line and "pages" in line: + parts = line.split() + for i, p in enumerate(parts): + if p == "imported" and i > 0: + try: + imported = int(parts[i - 1]) + except ValueError: + pass + if imported > 0: + return {"ok": True, "imported": imported, "output": output.strip()[:500]} + return {"ok": False, "error": output.strip()[:500]} + + except (FileNotFoundError, subprocess.TimeoutExpired) as e: + return {"ok": False, "error": str(e)} + + +# --------------------------------------------------------------------------- +# Local mode: put pages via local gbrain CLI +# --------------------------------------------------------------------------- + +def _local_gbrain_put(gbrain_bin: str | None, slug: str, content: str, dry_run: bool) -> dict: + if dry_run: + return {"ok": True, "slug": slug, "dry_run": True, "content_length": len(content)} + + cmd = [gbrain_bin or "gbrain", "put", slug] + try: + r = subprocess.run(cmd, input=content, capture_output=True, text=True, timeout=120) + if r.returncode == 0: + return {"ok": True, "slug": slug} + return {"ok": False, "slug": slug, "error": r.stderr.strip() or r.stdout.strip()} + except (FileNotFoundError, subprocess.TimeoutExpired) as e: + return {"ok": False, "slug": slug, "error": str(e)} + + +def discover_sessions(brain_dir: Path, source_filter: set[str] | None = None) -> list[Path]: + sessions = [] + for d in sorted(brain_dir.iterdir()): + if not d.is_dir() or not d.name.startswith("session-"): + continue + parts = d.name.split("-", 2) + if len(parts) < 2: + continue + source = parts[1] + if source_filter and source not in source_filter: + continue + if source not in VALID_SOURCES: + continue + sessions.append(d) + return sessions + + +def run(args: argparse.Namespace) -> dict[str, Any]: + replica_root = Path(args.replica_root).resolve() if args.replica_root else REPLICA_ROOT + brain_dir = replica_root / ".aurelio" / "brain" + + source_filter = set(args.source) if args.source else None + sessions = discover_sessions(brain_dir, source_filter) + + if args.limit: + sessions = sessions[:args.limit] + + ssh_host = args.ssh + use_remote = bool(ssh_host) + + summary: dict[str, Any] = { + "replica_root": str(replica_root), + "brain_dir": str(brain_dir), + "dry_run": args.dry_run, + "mode": "remote-ssh" if use_remote else "local", + "remote": ssh_host or None, + "total_sessions": len(sessions), + "pushed": 0, + "failed": 0, + "errors": [], + } + + if use_remote: + # Stage all files locally, then bulk import + staging = replica_root / ".aurelio" / "gbraidge-staging" + staging.mkdir(parents=True, exist_ok=True) + + for sess_dir in sessions: + chat_history = _load_chat_history(sess_dir) + if not chat_history: + continue + meta = _load_session_meta(sess_dir) + source = chat_history.get("source", "unknown") + session_id = chat_history.get("id", sess_dir.name) + slug = f"conversations/{source}/{session_id.removeprefix('session-')}" + content = _make_page_content(sess_dir, chat_history, meta) + + out = staging / f"{slug.replace('/', '_')}.md" + out.parent.mkdir(parents=True, exist_ok=True) + if not args.dry_run: + out.write_text(content, encoding="utf-8") + summary["pushed"] += 1 + + # Bulk import + if not args.dry_run: + print(f"Staged {summary['pushed']} files. Importing to gbrain...", file=sys.stderr) + result = _ssh_gbrain_import(ssh_host, staging, args.dry_run) + summary["import_result"] = result + if not result.get("ok"): + summary["failed"] = summary["pushed"] + summary["errors"].append({"error": result.get("error", "unknown")}) + else: + summary["imported"] = result.get("imported", 0) + + # Cleanup staging + if not args.dry_run: + import shutil + shutil.rmtree(staging, ignore_errors=True) + else: + for sess_dir in sessions: + chat_history = _load_chat_history(sess_dir) + if not chat_history: + continue + meta = _load_session_meta(sess_dir) + source = chat_history.get("source", "unknown") + session_id = chat_history.get("id", sess_dir.name) + slug = f"conversations/{source}/{session_id.removeprefix('session-')}" + content = _make_page_content(sess_dir, chat_history, meta) + + result = _local_gbrain_put(args.gbrain_bin, slug, content, args.dry_run) + if result["ok"]: + summary["pushed"] += 1 + if args.verbose: + print(f" OK {slug}") + else: + summary["failed"] += 1 + err = result.get("error", "unknown") + summary["errors"].append({"slug": slug, "error": err}) + if args.verbose: + print(f" FAIL {slug}: {err}", file=sys.stderr) + + return summary + + +def main() -> int: + ap = argparse.ArgumentParser(description="Bridge brain sessions → gbrain conversation pages.") + ap.add_argument("--dry-run", action="store_true", help="Report what would be pushed without touching gbrain.") + ap.add_argument("--source", action="append", help="Limit to specific sources (repeatable).") + ap.add_argument("--replica-root", default=None) + ap.add_argument("--ssh", default=None, + help=f"SSH target for remote gbrain (default: {DEFAULT_SSH}). " + "Set to empty string for local mode.") + ap.add_argument("--gbrain-bin", default=None, help="Path to local gbrain binary.") + ap.add_argument("--limit", type=int, default=None, help="Max sessions to process.") + ap.add_argument("--summary", action="store_true", help="Print JSON summary.") + ap.add_argument("--verbose", "-v", action="store_true") + args = ap.parse_args() + + if args.ssh is None: + args.ssh = DEFAULT_SSH + + summary = run(args) + + if args.summary: + print(json.dumps(summary, indent=2, ensure_ascii=False)) + else: + print(f"mode: {summary['mode']} remote: {summary['remote'] or 'local'}") + print(f"sessions: {summary['total_sessions']} pushed: {summary['pushed']} failed: {summary['failed']}") + if summary.get("imported"): + print(f"imported: {summary['imported']}") + if args.dry_run: + print("(dry-run: nothing pushed)") + for err in summary.get("errors", []): + print(f" FAIL: {err.get('slug', 'batch')}: {err.get('error', 'unknown')}", file=sys.stderr) + + return 1 if summary.get("failed", 0) > 0 else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/vault-sync.py b/scripts/vault-sync.py new file mode 100644 index 00000000..cde21214 --- /dev/null +++ b/scripts/vault-sync.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +""" +vault-sync — Obsidian vault → GBrain ingestion daemon (MCP). + +Watches a directory (synced from phone via Nextcloud/Remotely Save), +ingests markdown notes into GBrain via its MCP HTTP endpoint. + +Runs as a systemd service on CT223 (pf-forja-do-conhecimento). + +Config: /etc/vault-sync.json + { + "vault_dir": "/mnt/usb-pool/obsidian-vault", + "gbrain_url": "http://127.0.0.1:18001/mcp", + "gbrain_token": "gbrain_at_...", + "poll_interval": 300, + "min_file_age": 5, + "state_db": "/var/lib/vault-sync/state.db" + } +""" + +import hashlib +import json +import os +import re +import signal +import sqlite3 +import sys +import time +import urllib.request +import urllib.error +from datetime import datetime, timezone +from pathlib import Path + +DEFAULT_CONFIG = { + "vault_dir": "/mnt/usb-pool/obsidian-vault", + "gbrain_url": "http://127.0.0.1:18001/mcp", + "gbrain_token": "", + "poll_interval": 300, + "min_file_age": 5, + "state_db": "/var/lib/vault-sync/state.db", + "source_slug": "obsidian-vault", +} + +WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]") +FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n", re.DOTALL) +TAG_LINE_RE = re.compile(r"^tags?:\s*(.+)$", re.MULTILINE) +INLINE_TAG_RE = re.compile(r"(?:^|\s)#([a-zA-Z][a-zA-Z0-9_/\-]*)") + + +def mcp_call(url, token, tool, args=None): + """Call a GBrain MCP tool via HTTP JSON-RPC.""" + params = {"name": tool, "arguments": args or {}} + data = json.dumps({"jsonrpc": "2.0", "method": "tools/call", "params": params, "id": 1}) + req = urllib.request.Request( + url, + data=data.encode(), + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + "Authorization": f"Bearer {token}", + }, + ) + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read().decode() + for line in raw.splitlines(): + if line.startswith("data: "): + return json.loads(line[6:]) + return json.loads(raw) + + +def slugify(text): + """Convert a note title to a GBrain-safe slug.""" + s = text.lower().strip() + s = re.sub(r"[^a-z0-9\s/-]", "", s) + s = re.sub(r"[\s/]+", "-", s) + s = re.sub(r"-+", "-", s).strip("-") + return s[:80] or "untitled" + + +class VaultSync: + def __init__(self, config_path="/etc/vault-sync.json"): + self.config = dict(DEFAULT_CONFIG) + try: + with open(config_path) as f: + self.config.update(json.load(f)) + except (OSError, json.JSONDecodeError): + pass + + self.vault_dir = Path(self.config["vault_dir"]) + self.gbrain_url = self.config["gbrain_url"].rstrip("/") + self.gbrain_token = self.config["gbrain_token"] + self.poll_interval = self.config["poll_interval"] + self.min_file_age = self.config["min_file_age"] + self.source_slug = self.config["source_slug"] + + os.makedirs(os.path.dirname(self.config["state_db"]), exist_ok=True) + self.db = sqlite3.connect(self.config["state_db"]) + self.db.execute(""" + CREATE TABLE IF NOT EXISTS ingested ( + path TEXT PRIMARY KEY, + mtime REAL, + content_hash TEXT, + gbrain_slug TEXT, + ingested_at TEXT + ) + """) + self.db.commit() + + self.running = True + signal.signal(signal.SIGTERM, self._shutdown) + signal.signal(signal.SIGINT, self._shutdown) + + def _shutdown(self, *_): + print("Shutting down...", flush=True) + self.running = False + + def run(self): + print(f"vault-sync: {self.vault_dir} → {self.gbrain_url}", flush=True) + print(f" poll: {self.poll_interval}s, min_age: {self.min_file_age}s", flush=True) + + while self.running: + try: + self.sync_once() + except Exception as e: + print(f"Sync error: {e}", file=sys.stderr, flush=True) + for _ in range(int(self.poll_interval)): + if not self.running: + break + time.sleep(1) + + self.db.close() + print("vault-sync stopped.", flush=True) + + def sync_once(self): + if not self.vault_dir.exists(): + print(f" vault dir not found: {self.vault_dir}", flush=True) + return + + notes = sorted(self.vault_dir.rglob("*.md")) + total = len(notes) + new = changed = skipped = errors = 0 + + for note_path in notes: + if not self.running: + break + rel = note_path.relative_to(self.vault_dir) + stat = note_path.stat() + age = time.time() - stat.st_mtime + + if age < self.min_file_age: + skipped += 1 + continue + + row = self.db.execute( + "SELECT mtime FROM ingested WHERE path = ?", (str(rel),) + ).fetchone() + if row and row[0] == stat.st_mtime: + skipped += 1 + continue + + try: + result = self.ingest_note(note_path, rel) + if result == "new": + new += 1 + elif result == "updated": + changed += 1 + else: + skipped += 1 + except Exception as e: + errors += 1 + print(f" ERROR {rel}: {e}", file=sys.stderr, flush=True) + + if new or changed or errors: + print( + f" {total} notes: {new} new, {changed} updated, " + f"{skipped} skipped, {errors} errors", + flush=True, + ) + + def ingest_note(self, note_path, rel): + content = note_path.read_text(encoding="utf-8", errors="replace") + if not content.strip(): + return "skip" + + content_hash = hashlib.sha1(content.encode()).hexdigest()[:16] + stat = note_path.stat() + + # Parse frontmatter + frontmatter = {} + body = content + fm = FRONTMATTER_RE.match(content) + if fm: + body = content[fm.end():] + for line in fm.group(1).splitlines(): + if ":" in line: + k, v = line.split(":", 1) + frontmatter[k.strip().lower()] = v.strip() + + # Title: frontmatter title > first heading > filename + title = frontmatter.get("title", "") + if not title: + heading = re.search(r"^#\s+(.+)$", body, re.MULTILINE) + title = heading.group(1).strip() if heading else note_path.stem + + # Wikilinks + wikilinks = list(set(WIKILINK_RE.findall(body))) + + # Tags: frontmatter tags + inline #tags + tags = set() + tag_match = TAG_LINE_RE.search(content) + if tag_match: + raw = tag_match.group(1) + # Handle YAML arrays: [tag1, tag2] or plain: tag1, tag2 + if raw.startswith("[") and raw.endswith("]"): + raw = raw[1:-1] + for t in raw.split(","): + t = t.strip().strip("'\"[]").lstrip("#") + if t: + tags.add(t) + for t in INLINE_TAG_RE.findall(body): + tags.add(t) + tags = sorted(tags - {""}) + + # Slug for GBrain page + slug = slugify(str(rel).replace(".md", "")) + + # Check if this is new or update + existing = self.db.execute( + "SELECT gbrain_slug FROM ingested WHERE path = ?", (str(rel),) + ).fetchone() + is_update = existing is not None + + # Build GBrain page content with frontmatter + gbrain_frontmatter = { + "type": "note", + "source": self.source_slug, + "title": title, + "original_path": str(rel), + "content_hash": content_hash, + "file_mtime": stat.st_mtime, + "synced_at": datetime.now(timezone.utc).isoformat(), + **{k: v for k, v in frontmatter.items() if k not in ("title",)}, + } + fm_lines = ["---"] + for k, v in gbrain_frontmatter.items(): + if isinstance(v, str) and (":" in v or '"' in v): + v = json.dumps(v) + elif isinstance(v, (int, float)): + v = str(v) + elif isinstance(v, list): + v = json.dumps(v) + fm_lines.append(f"{k}: {v}") + fm_lines.append("---\n") + + page_content = "\n".join(fm_lines) + body + + # PUT page to GBrain + try: + result = mcp_call(self.gbrain_url, self.gbrain_token, "put_page", { + "slug": slug, + "content": page_content, + }) + gbrain_slug = slug + except Exception as e: + raise RuntimeError(f"put_page failed: {e}") + + # Add tags + for tag in tags[:20]: + try: + mcp_call(self.gbrain_url, self.gbrain_token, "add_tag", { + "slug": gbrain_slug, + "tag": tag, + }) + except Exception: + pass # tag errors are non-fatal + + # Add wikilinks as graph links + for link_target in wikilinks[:50]: + target_slug = slugify(link_target) + try: + mcp_call(self.gbrain_url, self.gbrain_token, "add_link", { + "from": gbrain_slug, + "to": target_slug, + "link_type": "wikilink", + }) + except Exception: + pass # link errors non-fatal (target may not exist yet) + + # Log the ingest + try: + mcp_call(self.gbrain_url, self.gbrain_token, "log_ingest", { + "source": self.source_slug, + "external_id": str(rel), + "action": "update" if is_update else "create", + "title": title, + }) + except Exception: + pass + + now = datetime.now(timezone.utc).isoformat() + self.db.execute( + """INSERT OR REPLACE INTO ingested (path, mtime, content_hash, gbrain_slug, ingested_at) + VALUES (?, ?, ?, ?, ?)""", + (str(rel), stat.st_mtime, content_hash, gbrain_slug, now), + ) + self.db.commit() + + return "updated" if is_update else "new" + + +def main(): + config_path = os.environ.get("VAULT_SYNC_CONFIG", "/etc/vault-sync.json") + sync = VaultSync(config_path) + sync.run() + + +if __name__ == "__main__": + main()