From c96b6631ce303d05cdebca75ec3cef4f75ab15d1 Mon Sep 17 00:00:00 2001 From: "Raphael Cautus (Maestro)" Date: Fri, 31 Jul 2026 00:07:40 +0100 Subject: [PATCH] =?UTF-8?q?feat(scripts):=20agent=20importer=20=E2=80=94?= =?UTF-8?q?=20new=20adapters,=20RL=20trajectory=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends sync-agents-to-brain.py with: - cursor, zed, continue, aider, cline adapters (11 total now) - RL trajectory extraction (state/action/observation triples) - Improved content flattening for OpenAI/Anthropic multipart content - Engine: stable per-message timestamps, session_memory.md rendering Co-authored-by: Álvaro de Campos --- scripts/agent_importers/adapters.py | 897 +++++++++++++++++++++++++++- scripts/agent_importers/engine.py | 56 +- scripts/sync-agents-to-brain.py | 8 + 3 files changed, 925 insertions(+), 36 deletions(-) diff --git a/scripts/agent_importers/adapters.py b/scripts/agent_importers/adapters.py index a7dc0ec5..6e0c80a4 100644 --- a/scripts/agent_importers/adapters.py +++ b/scripts/agent_importers/adapters.py @@ -8,6 +8,7 @@ abort a fleet-wide import. from __future__ import annotations +import hashlib import json import os import sqlite3 @@ -118,6 +119,33 @@ def discover_hermes(hermes_dir: Optional[Path] = None) -> Iterable[NormalizedSes role = m.get("role") if role not in ("user", "assistant", "tool", "system"): continue + # Hermes assistant messages may carry OpenAI-style tool_calls + if role == "assistant": + tc = m.get("tool_calls") + if isinstance(tc, list) and tc: + for j, t in enumerate(tc): + fn = t.get("function", {}) if isinstance(t, dict) else {} + name = fn.get("name", "?") + args = fn.get("arguments", "") + if isinstance(args, str) and len(args) > 500: + args = args[:500] + msgs.append(NormalizedMessage( + role="tool_call", + content=f"{name}({args})", + timestamp=to_iso_z(m.get("timestamp")), + model=m.get("model") or model, + id=f"hc-{i:06d}-{j}", + )) + content = flatten_content(m.get("content")) + if content: + msgs.append(NormalizedMessage( + role="assistant", + content=content, + timestamp=to_iso_z(m.get("timestamp")), + model=m.get("model") or model, + id=f"h-{i:06d}", + )) + continue norm_role = "tool_result" if role == "tool" else role content = flatten_content(m.get("content")) if not content and norm_role in ("user", "assistant"): @@ -198,7 +226,39 @@ def discover_claude_code(claude_dir: Optional[Path] = None) -> Iterable[Normaliz if not isinstance(message, dict): continue role = message.get("role") or rtype - content = flatten_content(message.get("content")) + content_raw = message.get("content") + + # Claude assistant messages may contain tool_use parts — emit as tool_call + if isinstance(content_raw, list): + tool_use_parts = [p for p in content_raw if isinstance(p, dict) and p.get("type") == "tool_use"] + if tool_use_parts: + for j, part in enumerate(tool_use_parts): + name = part.get("name", "?") + inp = json.dumps(part.get("input", {}), ensure_ascii=False)[:500] + msgs.append(NormalizedMessage( + role="tool_call", + content=f"{name}({inp})", + timestamp=to_iso_z(rec.get("timestamp")), + model=message.get("model"), + id=f"cc-{rec.get('uuid', 'x')}-t{j}", + )) + # Also emit any text content as assistant + text_parts = [p.get("text", "") for p in content_raw if isinstance(p, dict) and p.get("type") == "text"] + content = "\n".join(t for t in text_parts if t) + if content: + if role == "assistant": + model = model or message.get("model") + norm_role = "assistant" if role == "assistant" else ("user" if role == "user" else "system") + msgs.append(NormalizedMessage( + role=norm_role, + content=content, + timestamp=to_iso_z(rec.get("timestamp")), + model=message.get("model") if norm_role == "assistant" else None, + id=rec.get("uuid") or None, + )) + continue + + content = flatten_content(content_raw) if role == "assistant" and not content: continue if role == "assistant": @@ -243,6 +303,31 @@ def discover_claude_code(claude_dir: Optional[Path] = None) -> Iterable[Normaliz # status, task_details, step_payload, ...) # --------------------------------------------------------------------------- +def _antigravity_extract_json(blob: bytes | str) -> Optional[dict]: + """Extract the first JSON object from a protobuf blob that embeds JSON.""" + if isinstance(blob, bytes): + blob = blob.decode("utf-8", "replace") + start = blob.find("{") + if start < 0: + return None + depth = 0 + for i in range(start, len(blob)): + if blob[i] == "{": depth += 1 + elif blob[i] == "}": + depth -= 1 + if depth == 0: + try: + return json.loads(blob[start:i + 1]) + except json.JSONDecodeError: + return None + return None + + +# Antigravity step_type enum (from protobuf field numbers observed in the wild) +_AG_TOOL_TYPES = {9, 21} # tool call steps (list_dir, run_command, etc.) +_AG_RESPONSE_TYPES = {15, 90} # assistant/planner response steps +_AG_USER_TYPES = {5, 14} # user input steps + def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]: msgs: list[NormalizedMessage] = [] try: @@ -256,54 +341,87 @@ def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]: except sqlite3.Error: return msgs for row in rows: - step_type = row["step_type"] or "" + step_type = row["step_type"] + if not isinstance(step_type, int): + continue payload_raw = row["step_payload"] task_raw = row["task_details"] text = "" + tool_name = "" + tool_input = "" + for blob in (payload_raw, task_raw): if not blob: continue - if isinstance(blob, bytes): - try: - blob = blob.decode("utf-8", "replace") - except Exception: - continue - if isinstance(blob, str): - stripped = blob.strip() + obj = _antigravity_extract_json(blob) + if obj: + # Tool call extraction + if step_type in _AG_TOOL_TYPES: + tool_name = ( + obj.get("toolAction", "").split(" ")[0].lower() + or obj.get("name", "") + or obj.get("tool", "") + or obj.get("CommandLine", "").split(" ")[0] + ) + tool_input = json.dumps(obj, ensure_ascii=False)[:500] + text = tool_input + else: + text = ( + obj.get("text") or obj.get("content") + or obj.get("message") or obj.get("prompt") + or obj.get("plan") or "" + ) + if not text and isinstance(obj.get("messages"), list): + text = "\n".join( + flatten_content(m.get("content")) for m in obj["messages"] + if isinstance(m, dict) + ) + if text: + break + elif isinstance(blob, (bytes, str)): + # Fallback: raw text extraction + raw = blob.decode("utf-8", "replace") if isinstance(blob, bytes) else blob + stripped = raw.strip() if stripped.startswith("{") or stripped.startswith("["): try: obj = json.loads(stripped) - # common fields that carry human text text = ( obj.get("text") or obj.get("content") or obj.get("message") or obj.get("prompt") or obj.get("plan") or "" ) - if not text and isinstance(obj.get("messages"), list): - text = "\n".join( - flatten_content(m.get("content")) for m in obj["messages"] - if isinstance(m, dict) - ) except json.JSONDecodeError: text = stripped else: - text = stripped - if text: - break - st = str(step_type).lower() - if "user" in st or "input" in st: - role = "user" - elif "planner" in st or "agent" in st or "assistant" in st or "response" in st: - role = "assistant" - elif "tool" in st or "command" in st or "action" in st: + # Skip binary protobuf blobs that aren't JSON + if any(c.isprintable() for c in stripped[:50]): + text = stripped + if text: + break + + if step_type in _AG_TOOL_TYPES: role = "tool_call" + if not tool_name: + # Try to extract tool name from the protobuf text + raw = payload_raw.decode("utf-8", "replace") if isinstance(payload_raw, bytes) else str(payload_raw or "") + for marker in ("run_command", "list_dir", "read_file", "write_file", "search", "edit_file"): + if marker in raw: + tool_name = marker + break + content = f"{tool_name}({tool_input})" if tool_input else (tool_name or f"[step {step_type}]") + elif step_type in _AG_USER_TYPES: + role = "user" + content = text[:2000] if text else "[user input]" + elif step_type in _AG_RESPONSE_TYPES: + role = "assistant" + content = text[:2000] if text else f"[step {step_type}]" else: role = "assistant" - if not text: - text = f"[step {step_type}]" + content = text[:2000] if text else f"[step {step_type}]" + msgs.append(NormalizedMessage( role=role, - content=text[:2000], + content=content, id=f"ag-{row['idx']:06d}", )) finally: @@ -465,6 +583,317 @@ def discover_mimocode(repo_root: Optional[Path] = None, home: Optional[Path] = N ) +# --------------------------------------------------------------------------- +# Pi (~/.pi/agent/sessions//_.jsonl) +# --------------------------------------------------------------------------- + +def discover_pi(pi_dir: Optional[Path] = None) -> Iterable[NormalizedSession]: + root = pi_dir or (HOME / ".pi" / "agent") + sessions_dir = root / "sessions" + if not sessions_dir.exists(): + return + + for jsonl in sorted(sessions_dir.rglob("*.jsonl")): + records = _load_jsonl(jsonl) + if not records: + continue + + session_id = None + cwd = None + model = None + msgs: list[NormalizedMessage] = [] + + for rec in records: + if not isinstance(rec, dict): + continue + rtype = rec.get("type") + + if rtype == "session": + session_id = rec.get("id") + cwd = rec.get("cwd") + continue + + if rtype == "model_change": + provider = rec.get("provider", "") + model_id = rec.get("modelId", "") + model = f"{provider}/{model_id}" if provider and model_id else (model_id or provider or None) + continue + + if rtype in ("thinking_level_change",): + continue + + if rtype != "message": + continue + + message = rec.get("message") + if not isinstance(message, dict): + continue + + role = message.get("role") + if role not in ("user", "assistant", "toolResult"): + continue + + content_raw = message.get("content") + + # Pi assistant messages may contain toolCall content parts + if isinstance(content_raw, list): + tool_use_parts = [p for p in content_raw if isinstance(p, dict) and p.get("type") in ("tool_use", "toolCall")] + if tool_use_parts: + ts = to_iso_z(message.get("timestamp") or rec.get("timestamp")) + for j, part in enumerate(tool_use_parts): + name = part.get("name", "?") + inp_raw = part.get("input") or part.get("arguments", {}) + inp = json.dumps(inp_raw, ensure_ascii=False)[:500] + msgs.append(NormalizedMessage( + role="tool_call", + content=f"{name}({inp})", + timestamp=ts, + model=message.get("model") or model, + id=f"pi-{rec.get('id', 'x')}-t{j}", + )) + # Also emit text content + text_parts = [p.get("text", "") for p in content_raw if isinstance(p, dict) and p.get("type") == "text"] + content = "\n".join(t for t in text_parts if t) + if content: + norm_role = "assistant" if role == "assistant" else ("user" if role == "user" else "system") + msgs.append(NormalizedMessage( + role=norm_role, + content=content, + timestamp=ts, + model=(message.get("model") or model) if norm_role == "assistant" else None, + id=str(rec.get("id") or f"pi-{len(msgs):06d}"), + )) + continue + + norm_role = "tool_result" if role == "toolResult" else role + content = flatten_content(content_raw) + if not content and norm_role in ("user", "assistant"): + continue + + msg_model = None + if norm_role == "assistant": + msg_model = message.get("model") or model + + msgs.append(NormalizedMessage( + role=norm_role, + content=content, + timestamp=to_iso_z(message.get("timestamp") or rec.get("timestamp")), + model=msg_model, + id=str(rec.get("id") or f"pi-{len(msgs):06d}"), + )) + + if not msgs: + continue + + created, modified = _file_times(jsonl) + brain_session_id = str(session_id or jsonl.stem) + yield NormalizedSession( + source="pi", + session_id=source_id_for("pi", brain_session_id), + title="", + created=to_iso_z(next( + (r.get("timestamp") for r in records + if isinstance(r, dict) and r.get("type") == "session"), + None, + )) or created, + modified=modified, + workspace_path=cwd, + messages=msgs, + source_locator=str(jsonl), + model_type=model or "pi", + ) + + +# --------------------------------------------------------------------------- +# OpenCode (~/.local/share/opencode/opencode.db — SQLite) +# --------------------------------------------------------------------------- + +def discover_opencode(db_path: Optional[Path] = None) -> Iterable[NormalizedSession]: + db = db_path or (HOME / ".local" / "share" / "opencode" / "opencode.db") + if not db.exists(): + return + + try: + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + except sqlite3.Error: + return + + try: + con.row_factory = sqlite3.Row + + try: + sessions = con.execute( + "SELECT id, title, directory, agent, model, parent_id, " + "time_created, time_updated " + "FROM session ORDER BY time_created" + ).fetchall() + except sqlite3.Error: + return + + for sess in sessions: + sess_id = sess["id"] or "" + parent_id = sess["parent_id"] + title = sess["title"] or "" + directory = sess["directory"] or "" + agent = sess["agent"] + model_raw = sess["model"] + created_ms = sess["time_created"] or 0 + updated_ms = sess["time_updated"] or 0 + + model_name = None + if model_raw: + try: + mobj = json.loads(model_raw) if isinstance(model_raw, str) else model_raw + if isinstance(mobj, dict): + provider = mobj.get("providerID", "") + model_id = mobj.get("id", "") + model_name = f"{provider}/{model_id}" if provider and model_id else (model_id or provider or None) + except (json.JSONDecodeError, TypeError): + pass + + created = to_iso_z(created_ms / 1000.0) if created_ms else None + modified = to_iso_z(updated_ms / 1000.0) if updated_ms else None + if not created: + created, modified = _file_times(db) + + try: + rows = con.execute( + "SELECT id, data FROM message WHERE session_id = ? ORDER BY time_created", + (sess_id,), + ).fetchall() + except sqlite3.Error: + continue + + msgs: list[NormalizedMessage] = [] + for row in rows: + msg_data_raw = row["data"] + if not msg_data_raw: + continue + try: + msg_data = json.loads(msg_data_raw) if isinstance(msg_data_raw, str) else msg_data_raw + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(msg_data, dict): + continue + + role = msg_data.get("role") + if role not in ("user", "assistant", "tool", "toolResult"): + continue + + norm_role = "tool_result" if role in ("tool", "toolResult") else role + msg_model = None + if norm_role == "assistant": + m = msg_data.get("model") + if isinstance(m, dict): + msg_model = m.get("id") or m.get("providerID") or model_name + else: + msg_model = model_name + + try: + part_rows = con.execute( + "SELECT data FROM part WHERE message_id = ? ORDER BY time_created", + (row["id"],), + ).fetchall() + except sqlite3.Error: + part_rows = [] + + content_parts: list[str] = [] + tool_call_parts: list[tuple[str, str, str]] = [] # (name, input_json, output) + for pr in part_rows: + pd = pr["data"] + if not pd: + continue + try: + pdata = json.loads(pd) if isinstance(pd, str) else pd + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(pdata, dict): + continue + ptype = pdata.get("type") + if ptype == "text": + t = pdata.get("text", "") + if t: + content_parts.append(t) + elif ptype == "tool": + # OpenCode native tool format: {type:"tool", tool:"read", state:{input,output}} + name = pdata.get("tool", "?") + state = pdata.get("state", {}) + inp = json.dumps(state.get("input", {}), ensure_ascii=False)[:500] + out = str(state.get("output", ""))[:2000] + tool_call_parts.append((name, inp, out)) + elif ptype == "tool_use": + name = pdata.get("name", "?") + inp = json.dumps(pdata.get("input", {}), ensure_ascii=False)[:500] + tool_call_parts.append((name, inp, "")) + elif ptype == "tool_result": + tr = pdata.get("content", "") + if isinstance(tr, list): + tr = " ".join(x.get("text", "") for x in tr if isinstance(x, dict)) + content_parts.append(f"[tool_result] {str(tr)[:500]}") + + # Compute timestamp before tool_call emission + msg_ts = msg_data.get("time", {}) + ts = None + if isinstance(msg_ts, dict): + ts = to_iso_z(msg_ts.get("created")) + if not ts: + ts = to_iso_z(row["time_created"] / 1000.0) if row["time_created"] else None + + # Emit tool_call messages for OpenCode tool parts + for j, (name, inp, out) in enumerate(tool_call_parts): + msgs.append(NormalizedMessage( + role="tool_call", + content=f"{name}({inp})", + timestamp=ts, + model=msg_model, + id=f"oc-{row['id']}-t{j}", + )) + if out: + msgs.append(NormalizedMessage( + role="tool_result", + content=out[:2000], + timestamp=ts, + id=f"oc-{row['id']}-r{j}", + )) + + content = "\n".join(content_parts) + if not content and norm_role in ("user", "assistant"): + continue + + msgs.append(NormalizedMessage( + role=norm_role, + content=content, + timestamp=ts, + model=msg_model, + id=row["id"], + )) + + if not msgs: + continue + + extra: dict = {} + if parent_id: + extra["parentSessionId"] = parent_id + if agent: + extra["agent"] = agent + + brain_session_id = sess_id.removeprefix("ses_") if sess_id.startswith("ses_") else sess_id + yield NormalizedSession( + source="opencode", + session_id=source_id_for("oc", brain_session_id), + title=title, + created=created, + modified=modified, + workspace_path=directory or None, + messages=msgs, + source_locator=f"{db}#{sess_id}", + model_type=model_name or "opencode", + extra=extra, + ) + finally: + con.close() + + # --------------------------------------------------------------------------- # Kimi (re-export of the proven layout so the unified driver covers it too) # --------------------------------------------------------------------------- @@ -501,11 +930,37 @@ def discover_kimi(kimi_dir: Optional[Path] = None) -> Iterable[NormalizedSession role = rec.get("role") if role in (None, "_system_prompt", "_checkpoint", "_usage"): continue - if role == "tool_call": - msgs.append(NormalizedMessage(role="tool_call", content=flatten_content(rec.get("content"))[:500], id=f"kc-{i:06d}")) - continue - if role == "tool_result": - msgs.append(NormalizedMessage(role="tool_result", content=flatten_content(rec.get("content"))[:500], id=f"kr-{i:06d}")) + # Kimi assistant messages may carry tool_calls — emit as tool_call + if role == "assistant": + tc = rec.get("tool_calls") + if isinstance(tc, list) and tc: + for j, t in enumerate(tc): + fn = t.get("function", {}) if isinstance(t, dict) else {} + name = fn.get("name", "?") + args = fn.get("arguments", "") + if isinstance(args, str) and len(args) > 500: + args = args[:500] + msgs.append(NormalizedMessage( + role="tool_call", + content=f"{name}({args})", + id=f"kc-{i:06d}-{j}", + )) + # Also emit any text content as assistant + content = flatten_content(rec.get("content")) + if content: + msgs.append(NormalizedMessage( + role="assistant", + content=content, + id=f"k-{i:06d}", + )) + continue + # Kimi tool results use role=tool (not tool_result) + if role in ("tool", "tool_result"): + msgs.append(NormalizedMessage( + role="tool_result", + content=flatten_content(rec.get("content"))[:2000], + id=f"kr-{i:06d}", + )) continue if role not in ("user", "model"): continue @@ -530,6 +985,377 @@ def discover_kimi(kimi_dir: Optional[Path] = None) -> Iterable[NormalizedSession ) +# --------------------------------------------------------------------------- +# Cursor (~/.config/Cursor/User/workspaceStorage/*/state.vscdb + globalStorage) +# Chat sessions are in the globalStorage state.vscdb under +# 'chat.ChatSessionStore.index' and per-workspace state.vscdb keys. +# Composer/agent sessions are in globalStorage/cursor.chatSessions or similar. +# --------------------------------------------------------------------------- + +def discover_cursor(cursor_dir: Optional[Path] = None) -> Iterable[NormalizedSession]: + root = cursor_dir or (HOME / ".config" / "Cursor") + if not root.exists(): + return + + # Cursor stores chat data in workspace-level state.vscdb SQLite DBs. + # The composer (agent) history is in globalStorage. + for db_path in sorted(root.rglob("state.vscdb")): + try: + con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + except sqlite3.Error: + continue + try: + con.row_factory = sqlite3.Row + try: + rows = con.execute( + "SELECT key, value FROM ItemTable WHERE " + "key LIKE '%chatSession%' OR key LIKE '%composer%' " + "OR key LIKE '%chat%session%' OR key LIKE '%aichat%'" + ).fetchall() + except sqlite3.Error: + continue + + for row in rows: + key = row["key"] or "" + raw = row["value"] + if not raw: + continue + try: + data = json.loads(raw) if isinstance(raw, str) else raw + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(data, dict): + continue + + # Extract messages from common Cursor chat formats + messages_raw = ( + data.get("messages") or data.get("entries") + or data.get("history") or data.get("turns") + ) + if not isinstance(messages_raw, list) or not messages_raw: + continue + + session_id = ( + data.get("sessionId") or data.get("id") + or (hashlib.sha1(raw.encode()).hexdigest()[:16] if isinstance(raw, str) else db_path.stem) + ) + workspace = str(db_path.parent.parent) if "workspaceStorage" in str(db_path) else None + + msgs: list[NormalizedMessage] = [] + for i, m in enumerate(messages_raw): + if not isinstance(m, dict): + continue + role = m.get("role") or m.get("type", "") + content = flatten_content(m.get("content") or m.get("text") or m.get("message")) + if not content: + continue + norm_role = "user" if "user" in role.lower() else ( + "tool_call" if "tool" in role.lower() and "result" not in role.lower() else ( + "tool_result" if "result" in role.lower() else "assistant" + ) + ) + msgs.append(NormalizedMessage( + role=norm_role, + content=content, + timestamp=to_iso_z(m.get("timestamp") or m.get("createdAt")), + model=m.get("model") or data.get("model"), + id=m.get("id") or f"cur-{i:06d}", + )) + + if not msgs: + continue + created, modified = _file_times(db_path) + yield NormalizedSession( + source="cursor", + session_id=source_id_for("cursor", str(session_id)), + title=data.get("title") or "", + created=created, + modified=modified, + workspace_path=workspace, + messages=msgs, + source_locator=f"{db_path}#{key}", + model_type=data.get("model") or "cursor", + ) + finally: + con.close() + + +# --------------------------------------------------------------------------- +# Zed (~/.local/share/zed/threads/*.json + ~/.config/zed/) +# Thread files are JSON with entries[].role + entries[].content. +# Agent conversations in ~/.local/share/zed/assistant2/ or threads/. +# --------------------------------------------------------------------------- + +def discover_zed(zed_dir: Optional[Path] = None) -> Iterable[NormalizedSession]: + roots = [zed_dir] if zed_dir else [ + HOME / ".local" / "share" / "zed", + HOME / ".config" / "zed", + ] + for root in roots: + if not root or not root.exists(): + continue + for sub in ("threads", "assistant2", "assistant", "conversations"): + thread_dir = root / sub + if not thread_dir.exists(): + continue + for f in sorted(thread_dir.rglob("*.json")): + data = _load_json(f) + if not isinstance(data, dict): + continue + entries = data.get("entries") or data.get("messages") or data.get("turns") + if not isinstance(entries, list) or not entries: + continue + + session_id = data.get("id") or data.get("threadId") or f.stem + msgs: list[NormalizedMessage] = [] + for i, entry in enumerate(entries): + if not isinstance(entry, dict): + continue + role = entry.get("role", "") + content = flatten_content(entry.get("content") or entry.get("text") or entry.get("message")) + if not content: + continue + norm_role = "user" if role == "user" else ( + "tool_call" if "tool" in role.lower() else "assistant" + ) + msgs.append(NormalizedMessage( + role=norm_role, + content=content, + timestamp=to_iso_z(entry.get("timestamp")), + model=entry.get("model"), + id=f"zed-{i:06d}", + )) + + if not msgs: + continue + created, modified = _file_times(f) + yield NormalizedSession( + source="zed", + session_id=source_id_for("zed", str(session_id)), + title=data.get("title") or data.get("summary") or "", + created=created, + modified=modified, + workspace_path=data.get("workspace"), + messages=msgs, + source_locator=str(f), + model_type=data.get("model") or "zed", + ) + + +# --------------------------------------------------------------------------- +# Continue.dev (~/.continue/sessions/*.json) +# Each session is a JSON file with history[] containing {role, content}. +# --------------------------------------------------------------------------- + +def discover_continue(continue_dir: Optional[Path] = None) -> Iterable[NormalizedSession]: + root = continue_dir or (HOME / ".continue") + sessions_dir = root / "sessions" + if not sessions_dir.exists(): + return + + for f in sorted(sessions_dir.rglob("*.json")): + data = _load_json(f) + if not isinstance(data, dict): + continue + history = data.get("history") or data.get("messages") + if not isinstance(history, list) or not history: + continue + + session_id = data.get("sessionId") or data.get("id") or f.stem + msgs: list[NormalizedMessage] = [] + for i, entry in enumerate(history): + if not isinstance(entry, dict): + continue + role = entry.get("role", "") + content = flatten_content(entry.get("content") or entry.get("message")) + if not content: + continue + norm_role = "user" if role == "user" else ( + "tool_call" if "tool" in role.lower() else "assistant" + ) + msgs.append(NormalizedMessage( + role=norm_role, + content=content, + timestamp=to_iso_z(entry.get("timestamp")), + model=entry.get("model") or data.get("model"), + id=f"cont-{i:06d}", + )) + + if not msgs: + continue + created, modified = _file_times(f) + yield NormalizedSession( + source="continue", + session_id=source_id_for("continue", str(session_id)), + title=data.get("title") or "", + created=created, + modified=modified, + workspace_path=data.get("workspaceDirectory"), + messages=msgs, + source_locator=str(f), + model_type=data.get("model") or "continue", + ) + + +# --------------------------------------------------------------------------- +# Aider (.aider.chat.history.md in each repo root) +# Markdown chat log with # aider chat started at ... headers. +# --------------------------------------------------------------------------- + +def discover_aider(home: Optional[Path] = None) -> Iterable[NormalizedSession]: + # Aider writes .aider.chat.history.md in the repo root where it runs. + # We scan the home directory for repos with aider history. + # Skip hidden dirs and common non-repo dirs. + _SKIP = {".cache", ".config", ".local", ".npm", ".vscode", ".git", + "node_modules", "__pycache__", ".mozilla", ".thunderbird"} + home_dir = home or HOME + + for history_file in sorted(home_dir.rglob(".aider.chat.history.md")): + # Skip if inside a hidden dir (other than the file itself) + parts = history_file.relative_to(home_dir).parts + if any(p.startswith(".") and p != ".aider.chat.history.md" for p in parts[:-1]): + if not any(p in _SKIP for p in parts[:-1]): + # Allow .aurelio, .aider, etc. + pass + else: + continue + + try: + text = history_file.read_text(encoding="utf-8") + except OSError: + continue + if not text.strip(): + continue + + # Parse aider markdown: user messages start with ####, assistant with > or plain + msgs: list[NormalizedMessage] = [] + current_role: Optional[str] = None + current_lines: list[str] = [] + + def flush(): + nonlocal current_role, current_lines + if current_role and current_lines: + content = "\n".join(current_lines).strip() + if content: + msgs.append(NormalizedMessage( + role=current_role, + content=content, + id=f"aider-{len(msgs):06d}", + )) + current_role = None + current_lines = [] + + for line in text.splitlines(): + if line.startswith("# aider chat started at"): + flush() + continue + if line.startswith("#### "): + flush() + current_role = "user" + current_lines = [line[5:]] + elif line.startswith("> "): + if current_role != "assistant": + flush() + current_role = "assistant" + current_lines.append(line[2:]) + elif line.strip() and current_role: + current_lines.append(line) + + flush() + if not msgs: + continue + + workspace = str(history_file.parent) + created, modified = _file_times(history_file) + session_id = hashlib.sha1(workspace.encode()).hexdigest()[:16] + yield NormalizedSession( + source="aider", + session_id=source_id_for("aider", session_id), + title=f"Aider in {Path(workspace).name}", + created=created, + modified=modified, + workspace_path=workspace, + messages=msgs, + source_locator=str(history_file), + model_type="aider", + ) + + +# --------------------------------------------------------------------------- +# Cline / Roo Code (VS Code globalStorage/./tasks/) +# Task files are JSON with messages[] containing {role, content, toolUses}. +# --------------------------------------------------------------------------- + +_CLINE_PUBLISHERS = [ + "saoudrizwan.claude-dev", + "rooveterinaryinc.roo-cline", +] + +def discover_cline(vscode_storage: Optional[Path] = None) -> Iterable[NormalizedSession]: + roots = [vscode_storage] if vscode_storage else [ + HOME / ".config" / "Code" / "User" / "globalStorage", + HOME / ".config" / "Code - Insiders" / "User" / "globalStorage", + HOME / ".vscode" / "extensions", + ] + for root in roots: + if not root or not root.exists(): + continue + for publisher in _CLINE_PUBLISHERS: + ext_dir = root / publisher + if not ext_dir.exists(): + continue + # Cline stores tasks in tasks/ subdirectory + for task_dir_name in ("tasks", "taskHistory", "history"): + task_dir = ext_dir / task_dir_name + if not task_dir.exists(): + continue + for f in sorted(task_dir.rglob("*.json")): + data = _load_json(f) + if not isinstance(data, dict): + continue + messages_raw = data.get("messages") or data.get("apiConversationHistory") + if not isinstance(messages_raw, list) or not messages_raw: + continue + + session_id = data.get("taskId") or data.get("id") or f.stem + source = "cline" if "claude-dev" in publisher else "roo-code" + msgs: list[NormalizedMessage] = [] + for i, m in enumerate(messages_raw): + if not isinstance(m, dict): + continue + role = m.get("role", "") + content = flatten_content(m.get("content")) + if not content: + continue + norm_role = "user" if role == "user" else ( + "tool_call" if "tool" in role.lower() and "result" not in role.lower() else ( + "tool_result" if "result" in role.lower() else "assistant" + ) + ) + msgs.append(NormalizedMessage( + role=norm_role, + content=content, + timestamp=to_iso_z(m.get("timestamp")), + model=m.get("model"), + id=f"{source[:2]}-{i:06d}", + )) + + if not msgs: + continue + created, modified = _file_times(f) + yield NormalizedSession( + source=source, + session_id=source_id_for(source, str(session_id)), + title=data.get("title") or data.get("task", "")[:80], + created=created, + modified=modified, + workspace_path=data.get("workspace"), + messages=msgs, + source_locator=str(f), + model_type=data.get("model") or source, + ) + + ADAPTERS = { "kimi": discover_kimi, "hermes": discover_hermes, @@ -537,4 +1363,11 @@ ADAPTERS = { "antigravity": discover_antigravity_cli, "qwen-code": discover_qwen_code, "mimocode": discover_mimocode, + "pi": discover_pi, + "opencode": discover_opencode, + "cursor": discover_cursor, + "zed": discover_zed, + "continue": discover_continue, + "aider": discover_aider, + "cline": discover_cline, } diff --git a/scripts/agent_importers/engine.py b/scripts/agent_importers/engine.py index aec8893a..05c681be 100644 --- a/scripts/agent_importers/engine.py +++ b/scripts/agent_importers/engine.py @@ -306,7 +306,7 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A **({"extra": sess.extra} if sess.extra else {}), } - session_jsonl = [ + session_jsonl_lines = [ json.dumps({ "type": "session_start", "timestamp": sess.created, @@ -317,12 +317,58 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A "sourceSessionId": sess.session_id, }, }, ensure_ascii=False), + ] + # Full trajectory: one event per message, preserving tool_call/tool_result + # sequence for RL training. This is the primary training data artifact. + for idx, (m, ts) in enumerate(zip(sess.messages, timestamps)): + event: dict[str, Any] = { + "type": "message", + "timestamp": ts, + "data": { + "id": m.id or f"m{idx:06d}", + "role": m.role, + "content": m.content, + }, + } + if m.model: + event["data"]["model"] = m.model + session_jsonl_lines.append(json.dumps(event, ensure_ascii=False)) + session_jsonl_lines.append( json.dumps({ "type": "session_end", "timestamp": sess.modified, - "data": {"sessionId": sess.brain_id, "totalEvents": len(chat_messages)}, + "data": {"sessionId": sess.brain_id, "totalEvents": len(sess.messages)}, }, ensure_ascii=False), - ] + ) + + # RL trajectory: compact (state, action, reward) triples derived from the + # message sequence. Each tool_call is an action; the preceding user/assistant + # message is the state; the tool_result is the immediate observation. + trajectory_lines: list[str] = [] + for idx, m in enumerate(sess.messages): + if m.role != "tool_call": + continue + # Find the nearest preceding user/assistant message as state + state = "" + for j in range(idx - 1, -1, -1): + if sess.messages[j].role in ("user", "assistant"): + state = sess.messages[j].content[:2000] + break + # Find the tool_result that follows + observation = "" + if idx + 1 < len(sess.messages) and sess.messages[idx + 1].role == "tool_result": + observation = sess.messages[idx + 1].content[:2000] + traj_event = { + "type": "trajectory_step", + "session_id": sess.brain_id, + "step": idx, + "timestamp": timestamps[idx] if idx < len(timestamps) else None, + "state": state, + "action": m.content[:2000], + "observation": observation, + "model": m.model or sess.model_type, + } + trajectory_lines.append(json.dumps(traj_event, ensure_ascii=False)) overview_txt = ( f"# Conversation: {title}\n" @@ -334,11 +380,13 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A ) files = { - target_dir / "session.jsonl": "\n".join(session_jsonl) + "\n", + target_dir / "session.jsonl": "\n".join(session_jsonl_lines) + "\n", target_dir / "session_memory.md": render_session_memory(sess, import_ts), target_dir / ".system_generated" / "chat_history.json": json.dumps(chat_history, indent=2, ensure_ascii=False) + "\n", target_dir / ".system_generated" / "logs" / "overview.txt": overview_txt, } + if trajectory_lines: + files[target_dir / ".system_generated" / "trajectory.jsonl"] = "\n".join(trajectory_lines) + "\n" if dry_run: result["would_write"] = [str(p.relative_to(REPLICA_ROOT)) for p in files] diff --git a/scripts/sync-agents-to-brain.py b/scripts/sync-agents-to-brain.py index 3ac7fabd..2a83bc82 100755 --- a/scripts/sync-agents-to-brain.py +++ b/scripts/sync-agents-to-brain.py @@ -12,6 +12,8 @@ bridge (`scripts/sync-kimi-to-brain.py`) but for all sources: antigravity ~/.gemini/antigravity-{cli,ide}/conversations/*.db (sqlite) qwen-code ~/.qwen[-code]/**/{context.jsonl,*.jsonl} (no-op if absent) mimocode ./.mimocode/plans/*.md (+ future *.jsonl) + pi ~/.pi/agent/sessions/**/*.jsonl + opencode ~/.local/share/opencode/opencode.db (sqlite) Output contract per session (see agent_importers/engine.py): .aurelio/brain/session--/{session.jsonl, session_memory.md, @@ -52,6 +54,8 @@ from agent_importers.adapters import ( # noqa: E402 discover_antigravity_cli, discover_qwen_code, discover_mimocode, + discover_pi, + discover_opencode, ) @@ -89,6 +93,8 @@ def run(args: argparse.Namespace) -> dict[str, Any]: "antigravity": lambda: discover_antigravity_cli(Path(args.gemini_dir) if args.gemini_dir else None), "qwen-code": lambda: discover_qwen_code(Path(args.qwen_dir) if args.qwen_dir else None), "mimocode": lambda: discover_mimocode(repo_root=replica_root), + "pi": lambda: discover_pi(Path(args.pi_dir) if args.pi_dir else None), + "opencode": lambda: discover_opencode(Path(args.opencode_db) if args.opencode_db else None), } for name in requested: @@ -139,6 +145,8 @@ def main() -> int: ap.add_argument("--claude-dir", default=None) ap.add_argument("--gemini-dir", default=None) ap.add_argument("--qwen-dir", default=None) + ap.add_argument("--pi-dir", default=None) + ap.add_argument("--opencode-db", default=None) ap.add_argument("--summary", action="store_true", help="Print JSON summary.") ap.add_argument("--verbose", "-v", action="store_true") args = ap.parse_args()