feat(scripts): agent importer — new adapters, RL trajectory output
Some checks failed
Aurélio Sync & Conscience Upgrade / Upgrade Réplica Conscience (push) Failing after 27s

Extends sync-agents-to-brain.py with:
- cursor, zed, continue, aider, cline adapters (11 total now)
- RL trajectory extraction (state/action/observation triples)
- Improved content flattening for OpenAI/Anthropic multipart content
- Engine: stable per-message timestamps, session_memory.md rendering

Co-authored-by: Álvaro de Campos <campos@portugalfuturista.org>
This commit is contained in:
Raphael Cautus (Maestro) 2026-07-31 00:07:40 +01:00 committed by fabiorafaelcoutada
parent 29f71dd330
commit c96b6631ce
3 changed files with 925 additions and 36 deletions

View file

@ -8,6 +8,7 @@ abort a fleet-wide import.
from __future__ import annotations from __future__ import annotations
import hashlib
import json import json
import os import os
import sqlite3 import sqlite3
@ -118,6 +119,33 @@ def discover_hermes(hermes_dir: Optional[Path] = None) -> Iterable[NormalizedSes
role = m.get("role") role = m.get("role")
if role not in ("user", "assistant", "tool", "system"): if role not in ("user", "assistant", "tool", "system"):
continue continue
# Hermes assistant messages may carry OpenAI-style tool_calls
if role == "assistant":
tc = m.get("tool_calls")
if isinstance(tc, list) and tc:
for j, t in enumerate(tc):
fn = t.get("function", {}) if isinstance(t, dict) else {}
name = fn.get("name", "?")
args = fn.get("arguments", "")
if isinstance(args, str) and len(args) > 500:
args = args[:500]
msgs.append(NormalizedMessage(
role="tool_call",
content=f"{name}({args})",
timestamp=to_iso_z(m.get("timestamp")),
model=m.get("model") or model,
id=f"hc-{i:06d}-{j}",
))
content = flatten_content(m.get("content"))
if content:
msgs.append(NormalizedMessage(
role="assistant",
content=content,
timestamp=to_iso_z(m.get("timestamp")),
model=m.get("model") or model,
id=f"h-{i:06d}",
))
continue
norm_role = "tool_result" if role == "tool" else role norm_role = "tool_result" if role == "tool" else role
content = flatten_content(m.get("content")) content = flatten_content(m.get("content"))
if not content and norm_role in ("user", "assistant"): if not content and norm_role in ("user", "assistant"):
@ -198,7 +226,39 @@ def discover_claude_code(claude_dir: Optional[Path] = None) -> Iterable[Normaliz
if not isinstance(message, dict): if not isinstance(message, dict):
continue continue
role = message.get("role") or rtype role = message.get("role") or rtype
content = flatten_content(message.get("content")) content_raw = message.get("content")
# Claude assistant messages may contain tool_use parts — emit as tool_call
if isinstance(content_raw, list):
tool_use_parts = [p for p in content_raw if isinstance(p, dict) and p.get("type") == "tool_use"]
if tool_use_parts:
for j, part in enumerate(tool_use_parts):
name = part.get("name", "?")
inp = json.dumps(part.get("input", {}), ensure_ascii=False)[:500]
msgs.append(NormalizedMessage(
role="tool_call",
content=f"{name}({inp})",
timestamp=to_iso_z(rec.get("timestamp")),
model=message.get("model"),
id=f"cc-{rec.get('uuid', 'x')}-t{j}",
))
# Also emit any text content as assistant
text_parts = [p.get("text", "") for p in content_raw if isinstance(p, dict) and p.get("type") == "text"]
content = "\n".join(t for t in text_parts if t)
if content:
if role == "assistant":
model = model or message.get("model")
norm_role = "assistant" if role == "assistant" else ("user" if role == "user" else "system")
msgs.append(NormalizedMessage(
role=norm_role,
content=content,
timestamp=to_iso_z(rec.get("timestamp")),
model=message.get("model") if norm_role == "assistant" else None,
id=rec.get("uuid") or None,
))
continue
content = flatten_content(content_raw)
if role == "assistant" and not content: if role == "assistant" and not content:
continue continue
if role == "assistant": if role == "assistant":
@ -243,6 +303,31 @@ def discover_claude_code(claude_dir: Optional[Path] = None) -> Iterable[Normaliz
# status, task_details, step_payload, ...) # status, task_details, step_payload, ...)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def _antigravity_extract_json(blob: bytes | str) -> Optional[dict]:
"""Extract the first JSON object from a protobuf blob that embeds JSON."""
if isinstance(blob, bytes):
blob = blob.decode("utf-8", "replace")
start = blob.find("{")
if start < 0:
return None
depth = 0
for i in range(start, len(blob)):
if blob[i] == "{": depth += 1
elif blob[i] == "}":
depth -= 1
if depth == 0:
try:
return json.loads(blob[start:i + 1])
except json.JSONDecodeError:
return None
return None
# Antigravity step_type enum (from protobuf field numbers observed in the wild)
_AG_TOOL_TYPES = {9, 21} # tool call steps (list_dir, run_command, etc.)
_AG_RESPONSE_TYPES = {15, 90} # assistant/planner response steps
_AG_USER_TYPES = {5, 14} # user input steps
def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]: def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]:
msgs: list[NormalizedMessage] = [] msgs: list[NormalizedMessage] = []
try: try:
@ -256,54 +341,87 @@ def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]:
except sqlite3.Error: except sqlite3.Error:
return msgs return msgs
for row in rows: for row in rows:
step_type = row["step_type"] or "" step_type = row["step_type"]
if not isinstance(step_type, int):
continue
payload_raw = row["step_payload"] payload_raw = row["step_payload"]
task_raw = row["task_details"] task_raw = row["task_details"]
text = "" text = ""
tool_name = ""
tool_input = ""
for blob in (payload_raw, task_raw): for blob in (payload_raw, task_raw):
if not blob: if not blob:
continue continue
if isinstance(blob, bytes): obj = _antigravity_extract_json(blob)
try: if obj:
blob = blob.decode("utf-8", "replace") # Tool call extraction
except Exception: if step_type in _AG_TOOL_TYPES:
continue tool_name = (
if isinstance(blob, str): obj.get("toolAction", "").split(" ")[0].lower()
stripped = blob.strip() or obj.get("name", "")
or obj.get("tool", "")
or obj.get("CommandLine", "").split(" ")[0]
)
tool_input = json.dumps(obj, ensure_ascii=False)[:500]
text = tool_input
else:
text = (
obj.get("text") or obj.get("content")
or obj.get("message") or obj.get("prompt")
or obj.get("plan") or ""
)
if not text and isinstance(obj.get("messages"), list):
text = "\n".join(
flatten_content(m.get("content")) for m in obj["messages"]
if isinstance(m, dict)
)
if text:
break
elif isinstance(blob, (bytes, str)):
# Fallback: raw text extraction
raw = blob.decode("utf-8", "replace") if isinstance(blob, bytes) else blob
stripped = raw.strip()
if stripped.startswith("{") or stripped.startswith("["): if stripped.startswith("{") or stripped.startswith("["):
try: try:
obj = json.loads(stripped) obj = json.loads(stripped)
# common fields that carry human text
text = ( text = (
obj.get("text") or obj.get("content") obj.get("text") or obj.get("content")
or obj.get("message") or obj.get("prompt") or obj.get("message") or obj.get("prompt")
or obj.get("plan") or "" or obj.get("plan") or ""
) )
if not text and isinstance(obj.get("messages"), list):
text = "\n".join(
flatten_content(m.get("content")) for m in obj["messages"]
if isinstance(m, dict)
)
except json.JSONDecodeError: except json.JSONDecodeError:
text = stripped text = stripped
else: else:
text = stripped # Skip binary protobuf blobs that aren't JSON
if text: if any(c.isprintable() for c in stripped[:50]):
break text = stripped
st = str(step_type).lower() if text:
if "user" in st or "input" in st: break
role = "user"
elif "planner" in st or "agent" in st or "assistant" in st or "response" in st: if step_type in _AG_TOOL_TYPES:
role = "assistant"
elif "tool" in st or "command" in st or "action" in st:
role = "tool_call" role = "tool_call"
if not tool_name:
# Try to extract tool name from the protobuf text
raw = payload_raw.decode("utf-8", "replace") if isinstance(payload_raw, bytes) else str(payload_raw or "")
for marker in ("run_command", "list_dir", "read_file", "write_file", "search", "edit_file"):
if marker in raw:
tool_name = marker
break
content = f"{tool_name}({tool_input})" if tool_input else (tool_name or f"[step {step_type}]")
elif step_type in _AG_USER_TYPES:
role = "user"
content = text[:2000] if text else "[user input]"
elif step_type in _AG_RESPONSE_TYPES:
role = "assistant"
content = text[:2000] if text else f"[step {step_type}]"
else: else:
role = "assistant" role = "assistant"
if not text: content = text[:2000] if text else f"[step {step_type}]"
text = f"[step {step_type}]"
msgs.append(NormalizedMessage( msgs.append(NormalizedMessage(
role=role, role=role,
content=text[:2000], content=content,
id=f"ag-{row['idx']:06d}", id=f"ag-{row['idx']:06d}",
)) ))
finally: finally:
@ -465,6 +583,317 @@ def discover_mimocode(repo_root: Optional[Path] = None, home: Optional[Path] = N
) )
# ---------------------------------------------------------------------------
# Pi (~/.pi/agent/sessions/<workspace-hash>/<timestamp>_<uuid>.jsonl)
# ---------------------------------------------------------------------------
def discover_pi(pi_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
root = pi_dir or (HOME / ".pi" / "agent")
sessions_dir = root / "sessions"
if not sessions_dir.exists():
return
for jsonl in sorted(sessions_dir.rglob("*.jsonl")):
records = _load_jsonl(jsonl)
if not records:
continue
session_id = None
cwd = None
model = None
msgs: list[NormalizedMessage] = []
for rec in records:
if not isinstance(rec, dict):
continue
rtype = rec.get("type")
if rtype == "session":
session_id = rec.get("id")
cwd = rec.get("cwd")
continue
if rtype == "model_change":
provider = rec.get("provider", "")
model_id = rec.get("modelId", "")
model = f"{provider}/{model_id}" if provider and model_id else (model_id or provider or None)
continue
if rtype in ("thinking_level_change",):
continue
if rtype != "message":
continue
message = rec.get("message")
if not isinstance(message, dict):
continue
role = message.get("role")
if role not in ("user", "assistant", "toolResult"):
continue
content_raw = message.get("content")
# Pi assistant messages may contain toolCall content parts
if isinstance(content_raw, list):
tool_use_parts = [p for p in content_raw if isinstance(p, dict) and p.get("type") in ("tool_use", "toolCall")]
if tool_use_parts:
ts = to_iso_z(message.get("timestamp") or rec.get("timestamp"))
for j, part in enumerate(tool_use_parts):
name = part.get("name", "?")
inp_raw = part.get("input") or part.get("arguments", {})
inp = json.dumps(inp_raw, ensure_ascii=False)[:500]
msgs.append(NormalizedMessage(
role="tool_call",
content=f"{name}({inp})",
timestamp=ts,
model=message.get("model") or model,
id=f"pi-{rec.get('id', 'x')}-t{j}",
))
# Also emit text content
text_parts = [p.get("text", "") for p in content_raw if isinstance(p, dict) and p.get("type") == "text"]
content = "\n".join(t for t in text_parts if t)
if content:
norm_role = "assistant" if role == "assistant" else ("user" if role == "user" else "system")
msgs.append(NormalizedMessage(
role=norm_role,
content=content,
timestamp=ts,
model=(message.get("model") or model) if norm_role == "assistant" else None,
id=str(rec.get("id") or f"pi-{len(msgs):06d}"),
))
continue
norm_role = "tool_result" if role == "toolResult" else role
content = flatten_content(content_raw)
if not content and norm_role in ("user", "assistant"):
continue
msg_model = None
if norm_role == "assistant":
msg_model = message.get("model") or model
msgs.append(NormalizedMessage(
role=norm_role,
content=content,
timestamp=to_iso_z(message.get("timestamp") or rec.get("timestamp")),
model=msg_model,
id=str(rec.get("id") or f"pi-{len(msgs):06d}"),
))
if not msgs:
continue
created, modified = _file_times(jsonl)
brain_session_id = str(session_id or jsonl.stem)
yield NormalizedSession(
source="pi",
session_id=source_id_for("pi", brain_session_id),
title="",
created=to_iso_z(next(
(r.get("timestamp") for r in records
if isinstance(r, dict) and r.get("type") == "session"),
None,
)) or created,
modified=modified,
workspace_path=cwd,
messages=msgs,
source_locator=str(jsonl),
model_type=model or "pi",
)
# ---------------------------------------------------------------------------
# OpenCode (~/.local/share/opencode/opencode.db — SQLite)
# ---------------------------------------------------------------------------
def discover_opencode(db_path: Optional[Path] = None) -> Iterable[NormalizedSession]:
db = db_path or (HOME / ".local" / "share" / "opencode" / "opencode.db")
if not db.exists():
return
try:
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
except sqlite3.Error:
return
try:
con.row_factory = sqlite3.Row
try:
sessions = con.execute(
"SELECT id, title, directory, agent, model, parent_id, "
"time_created, time_updated "
"FROM session ORDER BY time_created"
).fetchall()
except sqlite3.Error:
return
for sess in sessions:
sess_id = sess["id"] or ""
parent_id = sess["parent_id"]
title = sess["title"] or ""
directory = sess["directory"] or ""
agent = sess["agent"]
model_raw = sess["model"]
created_ms = sess["time_created"] or 0
updated_ms = sess["time_updated"] or 0
model_name = None
if model_raw:
try:
mobj = json.loads(model_raw) if isinstance(model_raw, str) else model_raw
if isinstance(mobj, dict):
provider = mobj.get("providerID", "")
model_id = mobj.get("id", "")
model_name = f"{provider}/{model_id}" if provider and model_id else (model_id or provider or None)
except (json.JSONDecodeError, TypeError):
pass
created = to_iso_z(created_ms / 1000.0) if created_ms else None
modified = to_iso_z(updated_ms / 1000.0) if updated_ms else None
if not created:
created, modified = _file_times(db)
try:
rows = con.execute(
"SELECT id, data FROM message WHERE session_id = ? ORDER BY time_created",
(sess_id,),
).fetchall()
except sqlite3.Error:
continue
msgs: list[NormalizedMessage] = []
for row in rows:
msg_data_raw = row["data"]
if not msg_data_raw:
continue
try:
msg_data = json.loads(msg_data_raw) if isinstance(msg_data_raw, str) else msg_data_raw
except (json.JSONDecodeError, TypeError):
continue
if not isinstance(msg_data, dict):
continue
role = msg_data.get("role")
if role not in ("user", "assistant", "tool", "toolResult"):
continue
norm_role = "tool_result" if role in ("tool", "toolResult") else role
msg_model = None
if norm_role == "assistant":
m = msg_data.get("model")
if isinstance(m, dict):
msg_model = m.get("id") or m.get("providerID") or model_name
else:
msg_model = model_name
try:
part_rows = con.execute(
"SELECT data FROM part WHERE message_id = ? ORDER BY time_created",
(row["id"],),
).fetchall()
except sqlite3.Error:
part_rows = []
content_parts: list[str] = []
tool_call_parts: list[tuple[str, str, str]] = [] # (name, input_json, output)
for pr in part_rows:
pd = pr["data"]
if not pd:
continue
try:
pdata = json.loads(pd) if isinstance(pd, str) else pd
except (json.JSONDecodeError, TypeError):
continue
if not isinstance(pdata, dict):
continue
ptype = pdata.get("type")
if ptype == "text":
t = pdata.get("text", "")
if t:
content_parts.append(t)
elif ptype == "tool":
# OpenCode native tool format: {type:"tool", tool:"read", state:{input,output}}
name = pdata.get("tool", "?")
state = pdata.get("state", {})
inp = json.dumps(state.get("input", {}), ensure_ascii=False)[:500]
out = str(state.get("output", ""))[:2000]
tool_call_parts.append((name, inp, out))
elif ptype == "tool_use":
name = pdata.get("name", "?")
inp = json.dumps(pdata.get("input", {}), ensure_ascii=False)[:500]
tool_call_parts.append((name, inp, ""))
elif ptype == "tool_result":
tr = pdata.get("content", "")
if isinstance(tr, list):
tr = " ".join(x.get("text", "") for x in tr if isinstance(x, dict))
content_parts.append(f"[tool_result] {str(tr)[:500]}")
# Compute timestamp before tool_call emission
msg_ts = msg_data.get("time", {})
ts = None
if isinstance(msg_ts, dict):
ts = to_iso_z(msg_ts.get("created"))
if not ts:
ts = to_iso_z(row["time_created"] / 1000.0) if row["time_created"] else None
# Emit tool_call messages for OpenCode tool parts
for j, (name, inp, out) in enumerate(tool_call_parts):
msgs.append(NormalizedMessage(
role="tool_call",
content=f"{name}({inp})",
timestamp=ts,
model=msg_model,
id=f"oc-{row['id']}-t{j}",
))
if out:
msgs.append(NormalizedMessage(
role="tool_result",
content=out[:2000],
timestamp=ts,
id=f"oc-{row['id']}-r{j}",
))
content = "\n".join(content_parts)
if not content and norm_role in ("user", "assistant"):
continue
msgs.append(NormalizedMessage(
role=norm_role,
content=content,
timestamp=ts,
model=msg_model,
id=row["id"],
))
if not msgs:
continue
extra: dict = {}
if parent_id:
extra["parentSessionId"] = parent_id
if agent:
extra["agent"] = agent
brain_session_id = sess_id.removeprefix("ses_") if sess_id.startswith("ses_") else sess_id
yield NormalizedSession(
source="opencode",
session_id=source_id_for("oc", brain_session_id),
title=title,
created=created,
modified=modified,
workspace_path=directory or None,
messages=msgs,
source_locator=f"{db}#{sess_id}",
model_type=model_name or "opencode",
extra=extra,
)
finally:
con.close()
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Kimi (re-export of the proven layout so the unified driver covers it too) # Kimi (re-export of the proven layout so the unified driver covers it too)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@ -501,11 +930,37 @@ def discover_kimi(kimi_dir: Optional[Path] = None) -> Iterable[NormalizedSession
role = rec.get("role") role = rec.get("role")
if role in (None, "_system_prompt", "_checkpoint", "_usage"): if role in (None, "_system_prompt", "_checkpoint", "_usage"):
continue continue
if role == "tool_call": # Kimi assistant messages may carry tool_calls — emit as tool_call
msgs.append(NormalizedMessage(role="tool_call", content=flatten_content(rec.get("content"))[:500], id=f"kc-{i:06d}")) if role == "assistant":
continue tc = rec.get("tool_calls")
if role == "tool_result": if isinstance(tc, list) and tc:
msgs.append(NormalizedMessage(role="tool_result", content=flatten_content(rec.get("content"))[:500], id=f"kr-{i:06d}")) for j, t in enumerate(tc):
fn = t.get("function", {}) if isinstance(t, dict) else {}
name = fn.get("name", "?")
args = fn.get("arguments", "")
if isinstance(args, str) and len(args) > 500:
args = args[:500]
msgs.append(NormalizedMessage(
role="tool_call",
content=f"{name}({args})",
id=f"kc-{i:06d}-{j}",
))
# Also emit any text content as assistant
content = flatten_content(rec.get("content"))
if content:
msgs.append(NormalizedMessage(
role="assistant",
content=content,
id=f"k-{i:06d}",
))
continue
# Kimi tool results use role=tool (not tool_result)
if role in ("tool", "tool_result"):
msgs.append(NormalizedMessage(
role="tool_result",
content=flatten_content(rec.get("content"))[:2000],
id=f"kr-{i:06d}",
))
continue continue
if role not in ("user", "model"): if role not in ("user", "model"):
continue continue
@ -530,6 +985,377 @@ def discover_kimi(kimi_dir: Optional[Path] = None) -> Iterable[NormalizedSession
) )
# ---------------------------------------------------------------------------
# Cursor (~/.config/Cursor/User/workspaceStorage/*/state.vscdb + globalStorage)
# Chat sessions are in the globalStorage state.vscdb under
# 'chat.ChatSessionStore.index' and per-workspace state.vscdb keys.
# Composer/agent sessions are in globalStorage/cursor.chatSessions or similar.
# ---------------------------------------------------------------------------
def discover_cursor(cursor_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
root = cursor_dir or (HOME / ".config" / "Cursor")
if not root.exists():
return
# Cursor stores chat data in workspace-level state.vscdb SQLite DBs.
# The composer (agent) history is in globalStorage.
for db_path in sorted(root.rglob("state.vscdb")):
try:
con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
except sqlite3.Error:
continue
try:
con.row_factory = sqlite3.Row
try:
rows = con.execute(
"SELECT key, value FROM ItemTable WHERE "
"key LIKE '%chatSession%' OR key LIKE '%composer%' "
"OR key LIKE '%chat%session%' OR key LIKE '%aichat%'"
).fetchall()
except sqlite3.Error:
continue
for row in rows:
key = row["key"] or ""
raw = row["value"]
if not raw:
continue
try:
data = json.loads(raw) if isinstance(raw, str) else raw
except (json.JSONDecodeError, TypeError):
continue
if not isinstance(data, dict):
continue
# Extract messages from common Cursor chat formats
messages_raw = (
data.get("messages") or data.get("entries")
or data.get("history") or data.get("turns")
)
if not isinstance(messages_raw, list) or not messages_raw:
continue
session_id = (
data.get("sessionId") or data.get("id")
or (hashlib.sha1(raw.encode()).hexdigest()[:16] if isinstance(raw, str) else db_path.stem)
)
workspace = str(db_path.parent.parent) if "workspaceStorage" in str(db_path) else None
msgs: list[NormalizedMessage] = []
for i, m in enumerate(messages_raw):
if not isinstance(m, dict):
continue
role = m.get("role") or m.get("type", "")
content = flatten_content(m.get("content") or m.get("text") or m.get("message"))
if not content:
continue
norm_role = "user" if "user" in role.lower() else (
"tool_call" if "tool" in role.lower() and "result" not in role.lower() else (
"tool_result" if "result" in role.lower() else "assistant"
)
)
msgs.append(NormalizedMessage(
role=norm_role,
content=content,
timestamp=to_iso_z(m.get("timestamp") or m.get("createdAt")),
model=m.get("model") or data.get("model"),
id=m.get("id") or f"cur-{i:06d}",
))
if not msgs:
continue
created, modified = _file_times(db_path)
yield NormalizedSession(
source="cursor",
session_id=source_id_for("cursor", str(session_id)),
title=data.get("title") or "",
created=created,
modified=modified,
workspace_path=workspace,
messages=msgs,
source_locator=f"{db_path}#{key}",
model_type=data.get("model") or "cursor",
)
finally:
con.close()
# ---------------------------------------------------------------------------
# Zed (~/.local/share/zed/threads/*.json + ~/.config/zed/)
# Thread files are JSON with entries[].role + entries[].content.
# Agent conversations in ~/.local/share/zed/assistant2/ or threads/.
# ---------------------------------------------------------------------------
def discover_zed(zed_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
roots = [zed_dir] if zed_dir else [
HOME / ".local" / "share" / "zed",
HOME / ".config" / "zed",
]
for root in roots:
if not root or not root.exists():
continue
for sub in ("threads", "assistant2", "assistant", "conversations"):
thread_dir = root / sub
if not thread_dir.exists():
continue
for f in sorted(thread_dir.rglob("*.json")):
data = _load_json(f)
if not isinstance(data, dict):
continue
entries = data.get("entries") or data.get("messages") or data.get("turns")
if not isinstance(entries, list) or not entries:
continue
session_id = data.get("id") or data.get("threadId") or f.stem
msgs: list[NormalizedMessage] = []
for i, entry in enumerate(entries):
if not isinstance(entry, dict):
continue
role = entry.get("role", "")
content = flatten_content(entry.get("content") or entry.get("text") or entry.get("message"))
if not content:
continue
norm_role = "user" if role == "user" else (
"tool_call" if "tool" in role.lower() else "assistant"
)
msgs.append(NormalizedMessage(
role=norm_role,
content=content,
timestamp=to_iso_z(entry.get("timestamp")),
model=entry.get("model"),
id=f"zed-{i:06d}",
))
if not msgs:
continue
created, modified = _file_times(f)
yield NormalizedSession(
source="zed",
session_id=source_id_for("zed", str(session_id)),
title=data.get("title") or data.get("summary") or "",
created=created,
modified=modified,
workspace_path=data.get("workspace"),
messages=msgs,
source_locator=str(f),
model_type=data.get("model") or "zed",
)
# ---------------------------------------------------------------------------
# Continue.dev (~/.continue/sessions/*.json)
# Each session is a JSON file with history[] containing {role, content}.
# ---------------------------------------------------------------------------
def discover_continue(continue_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
root = continue_dir or (HOME / ".continue")
sessions_dir = root / "sessions"
if not sessions_dir.exists():
return
for f in sorted(sessions_dir.rglob("*.json")):
data = _load_json(f)
if not isinstance(data, dict):
continue
history = data.get("history") or data.get("messages")
if not isinstance(history, list) or not history:
continue
session_id = data.get("sessionId") or data.get("id") or f.stem
msgs: list[NormalizedMessage] = []
for i, entry in enumerate(history):
if not isinstance(entry, dict):
continue
role = entry.get("role", "")
content = flatten_content(entry.get("content") or entry.get("message"))
if not content:
continue
norm_role = "user" if role == "user" else (
"tool_call" if "tool" in role.lower() else "assistant"
)
msgs.append(NormalizedMessage(
role=norm_role,
content=content,
timestamp=to_iso_z(entry.get("timestamp")),
model=entry.get("model") or data.get("model"),
id=f"cont-{i:06d}",
))
if not msgs:
continue
created, modified = _file_times(f)
yield NormalizedSession(
source="continue",
session_id=source_id_for("continue", str(session_id)),
title=data.get("title") or "",
created=created,
modified=modified,
workspace_path=data.get("workspaceDirectory"),
messages=msgs,
source_locator=str(f),
model_type=data.get("model") or "continue",
)
# ---------------------------------------------------------------------------
# Aider (.aider.chat.history.md in each repo root)
# Markdown chat log with # aider chat started at ... headers.
# ---------------------------------------------------------------------------
def discover_aider(home: Optional[Path] = None) -> Iterable[NormalizedSession]:
# Aider writes .aider.chat.history.md in the repo root where it runs.
# We scan the home directory for repos with aider history.
# Skip hidden dirs and common non-repo dirs.
_SKIP = {".cache", ".config", ".local", ".npm", ".vscode", ".git",
"node_modules", "__pycache__", ".mozilla", ".thunderbird"}
home_dir = home or HOME
for history_file in sorted(home_dir.rglob(".aider.chat.history.md")):
# Skip if inside a hidden dir (other than the file itself)
parts = history_file.relative_to(home_dir).parts
if any(p.startswith(".") and p != ".aider.chat.history.md" for p in parts[:-1]):
if not any(p in _SKIP for p in parts[:-1]):
# Allow .aurelio, .aider, etc.
pass
else:
continue
try:
text = history_file.read_text(encoding="utf-8")
except OSError:
continue
if not text.strip():
continue
# Parse aider markdown: user messages start with ####, assistant with > or plain
msgs: list[NormalizedMessage] = []
current_role: Optional[str] = None
current_lines: list[str] = []
def flush():
nonlocal current_role, current_lines
if current_role and current_lines:
content = "\n".join(current_lines).strip()
if content:
msgs.append(NormalizedMessage(
role=current_role,
content=content,
id=f"aider-{len(msgs):06d}",
))
current_role = None
current_lines = []
for line in text.splitlines():
if line.startswith("# aider chat started at"):
flush()
continue
if line.startswith("#### "):
flush()
current_role = "user"
current_lines = [line[5:]]
elif line.startswith("> "):
if current_role != "assistant":
flush()
current_role = "assistant"
current_lines.append(line[2:])
elif line.strip() and current_role:
current_lines.append(line)
flush()
if not msgs:
continue
workspace = str(history_file.parent)
created, modified = _file_times(history_file)
session_id = hashlib.sha1(workspace.encode()).hexdigest()[:16]
yield NormalizedSession(
source="aider",
session_id=source_id_for("aider", session_id),
title=f"Aider in {Path(workspace).name}",
created=created,
modified=modified,
workspace_path=workspace,
messages=msgs,
source_locator=str(history_file),
model_type="aider",
)
# ---------------------------------------------------------------------------
# Cline / Roo Code (VS Code globalStorage/<publisher>.<name>/tasks/)
# Task files are JSON with messages[] containing {role, content, toolUses}.
# ---------------------------------------------------------------------------
_CLINE_PUBLISHERS = [
"saoudrizwan.claude-dev",
"rooveterinaryinc.roo-cline",
]
def discover_cline(vscode_storage: Optional[Path] = None) -> Iterable[NormalizedSession]:
roots = [vscode_storage] if vscode_storage else [
HOME / ".config" / "Code" / "User" / "globalStorage",
HOME / ".config" / "Code - Insiders" / "User" / "globalStorage",
HOME / ".vscode" / "extensions",
]
for root in roots:
if not root or not root.exists():
continue
for publisher in _CLINE_PUBLISHERS:
ext_dir = root / publisher
if not ext_dir.exists():
continue
# Cline stores tasks in tasks/ subdirectory
for task_dir_name in ("tasks", "taskHistory", "history"):
task_dir = ext_dir / task_dir_name
if not task_dir.exists():
continue
for f in sorted(task_dir.rglob("*.json")):
data = _load_json(f)
if not isinstance(data, dict):
continue
messages_raw = data.get("messages") or data.get("apiConversationHistory")
if not isinstance(messages_raw, list) or not messages_raw:
continue
session_id = data.get("taskId") or data.get("id") or f.stem
source = "cline" if "claude-dev" in publisher else "roo-code"
msgs: list[NormalizedMessage] = []
for i, m in enumerate(messages_raw):
if not isinstance(m, dict):
continue
role = m.get("role", "")
content = flatten_content(m.get("content"))
if not content:
continue
norm_role = "user" if role == "user" else (
"tool_call" if "tool" in role.lower() and "result" not in role.lower() else (
"tool_result" if "result" in role.lower() else "assistant"
)
)
msgs.append(NormalizedMessage(
role=norm_role,
content=content,
timestamp=to_iso_z(m.get("timestamp")),
model=m.get("model"),
id=f"{source[:2]}-{i:06d}",
))
if not msgs:
continue
created, modified = _file_times(f)
yield NormalizedSession(
source=source,
session_id=source_id_for(source, str(session_id)),
title=data.get("title") or data.get("task", "")[:80],
created=created,
modified=modified,
workspace_path=data.get("workspace"),
messages=msgs,
source_locator=str(f),
model_type=data.get("model") or source,
)
ADAPTERS = { ADAPTERS = {
"kimi": discover_kimi, "kimi": discover_kimi,
"hermes": discover_hermes, "hermes": discover_hermes,
@ -537,4 +1363,11 @@ ADAPTERS = {
"antigravity": discover_antigravity_cli, "antigravity": discover_antigravity_cli,
"qwen-code": discover_qwen_code, "qwen-code": discover_qwen_code,
"mimocode": discover_mimocode, "mimocode": discover_mimocode,
"pi": discover_pi,
"opencode": discover_opencode,
"cursor": discover_cursor,
"zed": discover_zed,
"continue": discover_continue,
"aider": discover_aider,
"cline": discover_cline,
} }

View file

@ -306,7 +306,7 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A
**({"extra": sess.extra} if sess.extra else {}), **({"extra": sess.extra} if sess.extra else {}),
} }
session_jsonl = [ session_jsonl_lines = [
json.dumps({ json.dumps({
"type": "session_start", "type": "session_start",
"timestamp": sess.created, "timestamp": sess.created,
@ -317,12 +317,58 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A
"sourceSessionId": sess.session_id, "sourceSessionId": sess.session_id,
}, },
}, ensure_ascii=False), }, ensure_ascii=False),
]
# Full trajectory: one event per message, preserving tool_call/tool_result
# sequence for RL training. This is the primary training data artifact.
for idx, (m, ts) in enumerate(zip(sess.messages, timestamps)):
event: dict[str, Any] = {
"type": "message",
"timestamp": ts,
"data": {
"id": m.id or f"m{idx:06d}",
"role": m.role,
"content": m.content,
},
}
if m.model:
event["data"]["model"] = m.model
session_jsonl_lines.append(json.dumps(event, ensure_ascii=False))
session_jsonl_lines.append(
json.dumps({ json.dumps({
"type": "session_end", "type": "session_end",
"timestamp": sess.modified, "timestamp": sess.modified,
"data": {"sessionId": sess.brain_id, "totalEvents": len(chat_messages)}, "data": {"sessionId": sess.brain_id, "totalEvents": len(sess.messages)},
}, ensure_ascii=False), }, ensure_ascii=False),
] )
# RL trajectory: compact (state, action, reward) triples derived from the
# message sequence. Each tool_call is an action; the preceding user/assistant
# message is the state; the tool_result is the immediate observation.
trajectory_lines: list[str] = []
for idx, m in enumerate(sess.messages):
if m.role != "tool_call":
continue
# Find the nearest preceding user/assistant message as state
state = ""
for j in range(idx - 1, -1, -1):
if sess.messages[j].role in ("user", "assistant"):
state = sess.messages[j].content[:2000]
break
# Find the tool_result that follows
observation = ""
if idx + 1 < len(sess.messages) and sess.messages[idx + 1].role == "tool_result":
observation = sess.messages[idx + 1].content[:2000]
traj_event = {
"type": "trajectory_step",
"session_id": sess.brain_id,
"step": idx,
"timestamp": timestamps[idx] if idx < len(timestamps) else None,
"state": state,
"action": m.content[:2000],
"observation": observation,
"model": m.model or sess.model_type,
}
trajectory_lines.append(json.dumps(traj_event, ensure_ascii=False))
overview_txt = ( overview_txt = (
f"# Conversation: {title}\n" f"# Conversation: {title}\n"
@ -334,11 +380,13 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A
) )
files = { files = {
target_dir / "session.jsonl": "\n".join(session_jsonl) + "\n", target_dir / "session.jsonl": "\n".join(session_jsonl_lines) + "\n",
target_dir / "session_memory.md": render_session_memory(sess, import_ts), target_dir / "session_memory.md": render_session_memory(sess, import_ts),
target_dir / ".system_generated" / "chat_history.json": json.dumps(chat_history, indent=2, ensure_ascii=False) + "\n", target_dir / ".system_generated" / "chat_history.json": json.dumps(chat_history, indent=2, ensure_ascii=False) + "\n",
target_dir / ".system_generated" / "logs" / "overview.txt": overview_txt, target_dir / ".system_generated" / "logs" / "overview.txt": overview_txt,
} }
if trajectory_lines:
files[target_dir / ".system_generated" / "trajectory.jsonl"] = "\n".join(trajectory_lines) + "\n"
if dry_run: if dry_run:
result["would_write"] = [str(p.relative_to(REPLICA_ROOT)) for p in files] result["would_write"] = [str(p.relative_to(REPLICA_ROOT)) for p in files]

View file

@ -12,6 +12,8 @@ bridge (`scripts/sync-kimi-to-brain.py`) but for all sources:
antigravity ~/.gemini/antigravity-{cli,ide}/conversations/*.db (sqlite) antigravity ~/.gemini/antigravity-{cli,ide}/conversations/*.db (sqlite)
qwen-code ~/.qwen[-code]/**/{context.jsonl,*.jsonl} (no-op if absent) qwen-code ~/.qwen[-code]/**/{context.jsonl,*.jsonl} (no-op if absent)
mimocode ./.mimocode/plans/*.md (+ future *.jsonl) mimocode ./.mimocode/plans/*.md (+ future *.jsonl)
pi ~/.pi/agent/sessions/**/*.jsonl
opencode ~/.local/share/opencode/opencode.db (sqlite)
Output contract per session (see agent_importers/engine.py): Output contract per session (see agent_importers/engine.py):
.aurelio/brain/session-<source>-<id>/{session.jsonl, session_memory.md, .aurelio/brain/session-<source>-<id>/{session.jsonl, session_memory.md,
@ -52,6 +54,8 @@ from agent_importers.adapters import ( # noqa: E402
discover_antigravity_cli, discover_antigravity_cli,
discover_qwen_code, discover_qwen_code,
discover_mimocode, discover_mimocode,
discover_pi,
discover_opencode,
) )
@ -89,6 +93,8 @@ def run(args: argparse.Namespace) -> dict[str, Any]:
"antigravity": lambda: discover_antigravity_cli(Path(args.gemini_dir) if args.gemini_dir else None), "antigravity": lambda: discover_antigravity_cli(Path(args.gemini_dir) if args.gemini_dir else None),
"qwen-code": lambda: discover_qwen_code(Path(args.qwen_dir) if args.qwen_dir else None), "qwen-code": lambda: discover_qwen_code(Path(args.qwen_dir) if args.qwen_dir else None),
"mimocode": lambda: discover_mimocode(repo_root=replica_root), "mimocode": lambda: discover_mimocode(repo_root=replica_root),
"pi": lambda: discover_pi(Path(args.pi_dir) if args.pi_dir else None),
"opencode": lambda: discover_opencode(Path(args.opencode_db) if args.opencode_db else None),
} }
for name in requested: for name in requested:
@ -139,6 +145,8 @@ def main() -> int:
ap.add_argument("--claude-dir", default=None) ap.add_argument("--claude-dir", default=None)
ap.add_argument("--gemini-dir", default=None) ap.add_argument("--gemini-dir", default=None)
ap.add_argument("--qwen-dir", default=None) ap.add_argument("--qwen-dir", default=None)
ap.add_argument("--pi-dir", default=None)
ap.add_argument("--opencode-db", default=None)
ap.add_argument("--summary", action="store_true", help="Print JSON summary.") ap.add_argument("--summary", action="store_true", help="Print JSON summary.")
ap.add_argument("--verbose", "-v", action="store_true") ap.add_argument("--verbose", "-v", action="store_true")
args = ap.parse_args() args = ap.parse_args()