feat(scripts): agent importer — new adapters, RL trajectory output
Some checks failed
Aurélio Sync & Conscience Upgrade / Upgrade Réplica Conscience (push) Failing after 27s
Some checks failed
Aurélio Sync & Conscience Upgrade / Upgrade Réplica Conscience (push) Failing after 27s
Extends sync-agents-to-brain.py with: - cursor, zed, continue, aider, cline adapters (11 total now) - RL trajectory extraction (state/action/observation triples) - Improved content flattening for OpenAI/Anthropic multipart content - Engine: stable per-message timestamps, session_memory.md rendering Co-authored-by: Álvaro de Campos <campos@portugalfuturista.org>
This commit is contained in:
parent
29f71dd330
commit
c96b6631ce
3 changed files with 925 additions and 36 deletions
|
|
@ -8,6 +8,7 @@ abort a fleet-wide import.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
|
|
@ -118,6 +119,33 @@ def discover_hermes(hermes_dir: Optional[Path] = None) -> Iterable[NormalizedSes
|
|||
role = m.get("role")
|
||||
if role not in ("user", "assistant", "tool", "system"):
|
||||
continue
|
||||
# Hermes assistant messages may carry OpenAI-style tool_calls
|
||||
if role == "assistant":
|
||||
tc = m.get("tool_calls")
|
||||
if isinstance(tc, list) and tc:
|
||||
for j, t in enumerate(tc):
|
||||
fn = t.get("function", {}) if isinstance(t, dict) else {}
|
||||
name = fn.get("name", "?")
|
||||
args = fn.get("arguments", "")
|
||||
if isinstance(args, str) and len(args) > 500:
|
||||
args = args[:500]
|
||||
msgs.append(NormalizedMessage(
|
||||
role="tool_call",
|
||||
content=f"{name}({args})",
|
||||
timestamp=to_iso_z(m.get("timestamp")),
|
||||
model=m.get("model") or model,
|
||||
id=f"hc-{i:06d}-{j}",
|
||||
))
|
||||
content = flatten_content(m.get("content"))
|
||||
if content:
|
||||
msgs.append(NormalizedMessage(
|
||||
role="assistant",
|
||||
content=content,
|
||||
timestamp=to_iso_z(m.get("timestamp")),
|
||||
model=m.get("model") or model,
|
||||
id=f"h-{i:06d}",
|
||||
))
|
||||
continue
|
||||
norm_role = "tool_result" if role == "tool" else role
|
||||
content = flatten_content(m.get("content"))
|
||||
if not content and norm_role in ("user", "assistant"):
|
||||
|
|
@ -198,7 +226,39 @@ def discover_claude_code(claude_dir: Optional[Path] = None) -> Iterable[Normaliz
|
|||
if not isinstance(message, dict):
|
||||
continue
|
||||
role = message.get("role") or rtype
|
||||
content = flatten_content(message.get("content"))
|
||||
content_raw = message.get("content")
|
||||
|
||||
# Claude assistant messages may contain tool_use parts — emit as tool_call
|
||||
if isinstance(content_raw, list):
|
||||
tool_use_parts = [p for p in content_raw if isinstance(p, dict) and p.get("type") == "tool_use"]
|
||||
if tool_use_parts:
|
||||
for j, part in enumerate(tool_use_parts):
|
||||
name = part.get("name", "?")
|
||||
inp = json.dumps(part.get("input", {}), ensure_ascii=False)[:500]
|
||||
msgs.append(NormalizedMessage(
|
||||
role="tool_call",
|
||||
content=f"{name}({inp})",
|
||||
timestamp=to_iso_z(rec.get("timestamp")),
|
||||
model=message.get("model"),
|
||||
id=f"cc-{rec.get('uuid', 'x')}-t{j}",
|
||||
))
|
||||
# Also emit any text content as assistant
|
||||
text_parts = [p.get("text", "") for p in content_raw if isinstance(p, dict) and p.get("type") == "text"]
|
||||
content = "\n".join(t for t in text_parts if t)
|
||||
if content:
|
||||
if role == "assistant":
|
||||
model = model or message.get("model")
|
||||
norm_role = "assistant" if role == "assistant" else ("user" if role == "user" else "system")
|
||||
msgs.append(NormalizedMessage(
|
||||
role=norm_role,
|
||||
content=content,
|
||||
timestamp=to_iso_z(rec.get("timestamp")),
|
||||
model=message.get("model") if norm_role == "assistant" else None,
|
||||
id=rec.get("uuid") or None,
|
||||
))
|
||||
continue
|
||||
|
||||
content = flatten_content(content_raw)
|
||||
if role == "assistant" and not content:
|
||||
continue
|
||||
if role == "assistant":
|
||||
|
|
@ -243,6 +303,31 @@ def discover_claude_code(claude_dir: Optional[Path] = None) -> Iterable[Normaliz
|
|||
# status, task_details, step_payload, ...)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _antigravity_extract_json(blob: bytes | str) -> Optional[dict]:
|
||||
"""Extract the first JSON object from a protobuf blob that embeds JSON."""
|
||||
if isinstance(blob, bytes):
|
||||
blob = blob.decode("utf-8", "replace")
|
||||
start = blob.find("{")
|
||||
if start < 0:
|
||||
return None
|
||||
depth = 0
|
||||
for i in range(start, len(blob)):
|
||||
if blob[i] == "{": depth += 1
|
||||
elif blob[i] == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
try:
|
||||
return json.loads(blob[start:i + 1])
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
# Antigravity step_type enum (from protobuf field numbers observed in the wild)
|
||||
_AG_TOOL_TYPES = {9, 21} # tool call steps (list_dir, run_command, etc.)
|
||||
_AG_RESPONSE_TYPES = {15, 90} # assistant/planner response steps
|
||||
_AG_USER_TYPES = {5, 14} # user input steps
|
||||
|
||||
def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]:
|
||||
msgs: list[NormalizedMessage] = []
|
||||
try:
|
||||
|
|
@ -256,54 +341,87 @@ def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]:
|
|||
except sqlite3.Error:
|
||||
return msgs
|
||||
for row in rows:
|
||||
step_type = row["step_type"] or ""
|
||||
step_type = row["step_type"]
|
||||
if not isinstance(step_type, int):
|
||||
continue
|
||||
payload_raw = row["step_payload"]
|
||||
task_raw = row["task_details"]
|
||||
text = ""
|
||||
tool_name = ""
|
||||
tool_input = ""
|
||||
|
||||
for blob in (payload_raw, task_raw):
|
||||
if not blob:
|
||||
continue
|
||||
if isinstance(blob, bytes):
|
||||
try:
|
||||
blob = blob.decode("utf-8", "replace")
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(blob, str):
|
||||
stripped = blob.strip()
|
||||
obj = _antigravity_extract_json(blob)
|
||||
if obj:
|
||||
# Tool call extraction
|
||||
if step_type in _AG_TOOL_TYPES:
|
||||
tool_name = (
|
||||
obj.get("toolAction", "").split(" ")[0].lower()
|
||||
or obj.get("name", "")
|
||||
or obj.get("tool", "")
|
||||
or obj.get("CommandLine", "").split(" ")[0]
|
||||
)
|
||||
tool_input = json.dumps(obj, ensure_ascii=False)[:500]
|
||||
text = tool_input
|
||||
else:
|
||||
text = (
|
||||
obj.get("text") or obj.get("content")
|
||||
or obj.get("message") or obj.get("prompt")
|
||||
or obj.get("plan") or ""
|
||||
)
|
||||
if not text and isinstance(obj.get("messages"), list):
|
||||
text = "\n".join(
|
||||
flatten_content(m.get("content")) for m in obj["messages"]
|
||||
if isinstance(m, dict)
|
||||
)
|
||||
if text:
|
||||
break
|
||||
elif isinstance(blob, (bytes, str)):
|
||||
# Fallback: raw text extraction
|
||||
raw = blob.decode("utf-8", "replace") if isinstance(blob, bytes) else blob
|
||||
stripped = raw.strip()
|
||||
if stripped.startswith("{") or stripped.startswith("["):
|
||||
try:
|
||||
obj = json.loads(stripped)
|
||||
# common fields that carry human text
|
||||
text = (
|
||||
obj.get("text") or obj.get("content")
|
||||
or obj.get("message") or obj.get("prompt")
|
||||
or obj.get("plan") or ""
|
||||
)
|
||||
if not text and isinstance(obj.get("messages"), list):
|
||||
text = "\n".join(
|
||||
flatten_content(m.get("content")) for m in obj["messages"]
|
||||
if isinstance(m, dict)
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
text = stripped
|
||||
else:
|
||||
text = stripped
|
||||
if text:
|
||||
break
|
||||
st = str(step_type).lower()
|
||||
if "user" in st or "input" in st:
|
||||
role = "user"
|
||||
elif "planner" in st or "agent" in st or "assistant" in st or "response" in st:
|
||||
role = "assistant"
|
||||
elif "tool" in st or "command" in st or "action" in st:
|
||||
# Skip binary protobuf blobs that aren't JSON
|
||||
if any(c.isprintable() for c in stripped[:50]):
|
||||
text = stripped
|
||||
if text:
|
||||
break
|
||||
|
||||
if step_type in _AG_TOOL_TYPES:
|
||||
role = "tool_call"
|
||||
if not tool_name:
|
||||
# Try to extract tool name from the protobuf text
|
||||
raw = payload_raw.decode("utf-8", "replace") if isinstance(payload_raw, bytes) else str(payload_raw or "")
|
||||
for marker in ("run_command", "list_dir", "read_file", "write_file", "search", "edit_file"):
|
||||
if marker in raw:
|
||||
tool_name = marker
|
||||
break
|
||||
content = f"{tool_name}({tool_input})" if tool_input else (tool_name or f"[step {step_type}]")
|
||||
elif step_type in _AG_USER_TYPES:
|
||||
role = "user"
|
||||
content = text[:2000] if text else "[user input]"
|
||||
elif step_type in _AG_RESPONSE_TYPES:
|
||||
role = "assistant"
|
||||
content = text[:2000] if text else f"[step {step_type}]"
|
||||
else:
|
||||
role = "assistant"
|
||||
if not text:
|
||||
text = f"[step {step_type}]"
|
||||
content = text[:2000] if text else f"[step {step_type}]"
|
||||
|
||||
msgs.append(NormalizedMessage(
|
||||
role=role,
|
||||
content=text[:2000],
|
||||
content=content,
|
||||
id=f"ag-{row['idx']:06d}",
|
||||
))
|
||||
finally:
|
||||
|
|
@ -465,6 +583,317 @@ def discover_mimocode(repo_root: Optional[Path] = None, home: Optional[Path] = N
|
|||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pi (~/.pi/agent/sessions/<workspace-hash>/<timestamp>_<uuid>.jsonl)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def discover_pi(pi_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||
root = pi_dir or (HOME / ".pi" / "agent")
|
||||
sessions_dir = root / "sessions"
|
||||
if not sessions_dir.exists():
|
||||
return
|
||||
|
||||
for jsonl in sorted(sessions_dir.rglob("*.jsonl")):
|
||||
records = _load_jsonl(jsonl)
|
||||
if not records:
|
||||
continue
|
||||
|
||||
session_id = None
|
||||
cwd = None
|
||||
model = None
|
||||
msgs: list[NormalizedMessage] = []
|
||||
|
||||
for rec in records:
|
||||
if not isinstance(rec, dict):
|
||||
continue
|
||||
rtype = rec.get("type")
|
||||
|
||||
if rtype == "session":
|
||||
session_id = rec.get("id")
|
||||
cwd = rec.get("cwd")
|
||||
continue
|
||||
|
||||
if rtype == "model_change":
|
||||
provider = rec.get("provider", "")
|
||||
model_id = rec.get("modelId", "")
|
||||
model = f"{provider}/{model_id}" if provider and model_id else (model_id or provider or None)
|
||||
continue
|
||||
|
||||
if rtype in ("thinking_level_change",):
|
||||
continue
|
||||
|
||||
if rtype != "message":
|
||||
continue
|
||||
|
||||
message = rec.get("message")
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
|
||||
role = message.get("role")
|
||||
if role not in ("user", "assistant", "toolResult"):
|
||||
continue
|
||||
|
||||
content_raw = message.get("content")
|
||||
|
||||
# Pi assistant messages may contain toolCall content parts
|
||||
if isinstance(content_raw, list):
|
||||
tool_use_parts = [p for p in content_raw if isinstance(p, dict) and p.get("type") in ("tool_use", "toolCall")]
|
||||
if tool_use_parts:
|
||||
ts = to_iso_z(message.get("timestamp") or rec.get("timestamp"))
|
||||
for j, part in enumerate(tool_use_parts):
|
||||
name = part.get("name", "?")
|
||||
inp_raw = part.get("input") or part.get("arguments", {})
|
||||
inp = json.dumps(inp_raw, ensure_ascii=False)[:500]
|
||||
msgs.append(NormalizedMessage(
|
||||
role="tool_call",
|
||||
content=f"{name}({inp})",
|
||||
timestamp=ts,
|
||||
model=message.get("model") or model,
|
||||
id=f"pi-{rec.get('id', 'x')}-t{j}",
|
||||
))
|
||||
# Also emit text content
|
||||
text_parts = [p.get("text", "") for p in content_raw if isinstance(p, dict) and p.get("type") == "text"]
|
||||
content = "\n".join(t for t in text_parts if t)
|
||||
if content:
|
||||
norm_role = "assistant" if role == "assistant" else ("user" if role == "user" else "system")
|
||||
msgs.append(NormalizedMessage(
|
||||
role=norm_role,
|
||||
content=content,
|
||||
timestamp=ts,
|
||||
model=(message.get("model") or model) if norm_role == "assistant" else None,
|
||||
id=str(rec.get("id") or f"pi-{len(msgs):06d}"),
|
||||
))
|
||||
continue
|
||||
|
||||
norm_role = "tool_result" if role == "toolResult" else role
|
||||
content = flatten_content(content_raw)
|
||||
if not content and norm_role in ("user", "assistant"):
|
||||
continue
|
||||
|
||||
msg_model = None
|
||||
if norm_role == "assistant":
|
||||
msg_model = message.get("model") or model
|
||||
|
||||
msgs.append(NormalizedMessage(
|
||||
role=norm_role,
|
||||
content=content,
|
||||
timestamp=to_iso_z(message.get("timestamp") or rec.get("timestamp")),
|
||||
model=msg_model,
|
||||
id=str(rec.get("id") or f"pi-{len(msgs):06d}"),
|
||||
))
|
||||
|
||||
if not msgs:
|
||||
continue
|
||||
|
||||
created, modified = _file_times(jsonl)
|
||||
brain_session_id = str(session_id or jsonl.stem)
|
||||
yield NormalizedSession(
|
||||
source="pi",
|
||||
session_id=source_id_for("pi", brain_session_id),
|
||||
title="",
|
||||
created=to_iso_z(next(
|
||||
(r.get("timestamp") for r in records
|
||||
if isinstance(r, dict) and r.get("type") == "session"),
|
||||
None,
|
||||
)) or created,
|
||||
modified=modified,
|
||||
workspace_path=cwd,
|
||||
messages=msgs,
|
||||
source_locator=str(jsonl),
|
||||
model_type=model or "pi",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenCode (~/.local/share/opencode/opencode.db — SQLite)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def discover_opencode(db_path: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||
db = db_path or (HOME / ".local" / "share" / "opencode" / "opencode.db")
|
||||
if not db.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||
except sqlite3.Error:
|
||||
return
|
||||
|
||||
try:
|
||||
con.row_factory = sqlite3.Row
|
||||
|
||||
try:
|
||||
sessions = con.execute(
|
||||
"SELECT id, title, directory, agent, model, parent_id, "
|
||||
"time_created, time_updated "
|
||||
"FROM session ORDER BY time_created"
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
return
|
||||
|
||||
for sess in sessions:
|
||||
sess_id = sess["id"] or ""
|
||||
parent_id = sess["parent_id"]
|
||||
title = sess["title"] or ""
|
||||
directory = sess["directory"] or ""
|
||||
agent = sess["agent"]
|
||||
model_raw = sess["model"]
|
||||
created_ms = sess["time_created"] or 0
|
||||
updated_ms = sess["time_updated"] or 0
|
||||
|
||||
model_name = None
|
||||
if model_raw:
|
||||
try:
|
||||
mobj = json.loads(model_raw) if isinstance(model_raw, str) else model_raw
|
||||
if isinstance(mobj, dict):
|
||||
provider = mobj.get("providerID", "")
|
||||
model_id = mobj.get("id", "")
|
||||
model_name = f"{provider}/{model_id}" if provider and model_id else (model_id or provider or None)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
created = to_iso_z(created_ms / 1000.0) if created_ms else None
|
||||
modified = to_iso_z(updated_ms / 1000.0) if updated_ms else None
|
||||
if not created:
|
||||
created, modified = _file_times(db)
|
||||
|
||||
try:
|
||||
rows = con.execute(
|
||||
"SELECT id, data FROM message WHERE session_id = ? ORDER BY time_created",
|
||||
(sess_id,),
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
continue
|
||||
|
||||
msgs: list[NormalizedMessage] = []
|
||||
for row in rows:
|
||||
msg_data_raw = row["data"]
|
||||
if not msg_data_raw:
|
||||
continue
|
||||
try:
|
||||
msg_data = json.loads(msg_data_raw) if isinstance(msg_data_raw, str) else msg_data_raw
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
if not isinstance(msg_data, dict):
|
||||
continue
|
||||
|
||||
role = msg_data.get("role")
|
||||
if role not in ("user", "assistant", "tool", "toolResult"):
|
||||
continue
|
||||
|
||||
norm_role = "tool_result" if role in ("tool", "toolResult") else role
|
||||
msg_model = None
|
||||
if norm_role == "assistant":
|
||||
m = msg_data.get("model")
|
||||
if isinstance(m, dict):
|
||||
msg_model = m.get("id") or m.get("providerID") or model_name
|
||||
else:
|
||||
msg_model = model_name
|
||||
|
||||
try:
|
||||
part_rows = con.execute(
|
||||
"SELECT data FROM part WHERE message_id = ? ORDER BY time_created",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
part_rows = []
|
||||
|
||||
content_parts: list[str] = []
|
||||
tool_call_parts: list[tuple[str, str, str]] = [] # (name, input_json, output)
|
||||
for pr in part_rows:
|
||||
pd = pr["data"]
|
||||
if not pd:
|
||||
continue
|
||||
try:
|
||||
pdata = json.loads(pd) if isinstance(pd, str) else pd
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
if not isinstance(pdata, dict):
|
||||
continue
|
||||
ptype = pdata.get("type")
|
||||
if ptype == "text":
|
||||
t = pdata.get("text", "")
|
||||
if t:
|
||||
content_parts.append(t)
|
||||
elif ptype == "tool":
|
||||
# OpenCode native tool format: {type:"tool", tool:"read", state:{input,output}}
|
||||
name = pdata.get("tool", "?")
|
||||
state = pdata.get("state", {})
|
||||
inp = json.dumps(state.get("input", {}), ensure_ascii=False)[:500]
|
||||
out = str(state.get("output", ""))[:2000]
|
||||
tool_call_parts.append((name, inp, out))
|
||||
elif ptype == "tool_use":
|
||||
name = pdata.get("name", "?")
|
||||
inp = json.dumps(pdata.get("input", {}), ensure_ascii=False)[:500]
|
||||
tool_call_parts.append((name, inp, ""))
|
||||
elif ptype == "tool_result":
|
||||
tr = pdata.get("content", "")
|
||||
if isinstance(tr, list):
|
||||
tr = " ".join(x.get("text", "") for x in tr if isinstance(x, dict))
|
||||
content_parts.append(f"[tool_result] {str(tr)[:500]}")
|
||||
|
||||
# Compute timestamp before tool_call emission
|
||||
msg_ts = msg_data.get("time", {})
|
||||
ts = None
|
||||
if isinstance(msg_ts, dict):
|
||||
ts = to_iso_z(msg_ts.get("created"))
|
||||
if not ts:
|
||||
ts = to_iso_z(row["time_created"] / 1000.0) if row["time_created"] else None
|
||||
|
||||
# Emit tool_call messages for OpenCode tool parts
|
||||
for j, (name, inp, out) in enumerate(tool_call_parts):
|
||||
msgs.append(NormalizedMessage(
|
||||
role="tool_call",
|
||||
content=f"{name}({inp})",
|
||||
timestamp=ts,
|
||||
model=msg_model,
|
||||
id=f"oc-{row['id']}-t{j}",
|
||||
))
|
||||
if out:
|
||||
msgs.append(NormalizedMessage(
|
||||
role="tool_result",
|
||||
content=out[:2000],
|
||||
timestamp=ts,
|
||||
id=f"oc-{row['id']}-r{j}",
|
||||
))
|
||||
|
||||
content = "\n".join(content_parts)
|
||||
if not content and norm_role in ("user", "assistant"):
|
||||
continue
|
||||
|
||||
msgs.append(NormalizedMessage(
|
||||
role=norm_role,
|
||||
content=content,
|
||||
timestamp=ts,
|
||||
model=msg_model,
|
||||
id=row["id"],
|
||||
))
|
||||
|
||||
if not msgs:
|
||||
continue
|
||||
|
||||
extra: dict = {}
|
||||
if parent_id:
|
||||
extra["parentSessionId"] = parent_id
|
||||
if agent:
|
||||
extra["agent"] = agent
|
||||
|
||||
brain_session_id = sess_id.removeprefix("ses_") if sess_id.startswith("ses_") else sess_id
|
||||
yield NormalizedSession(
|
||||
source="opencode",
|
||||
session_id=source_id_for("oc", brain_session_id),
|
||||
title=title,
|
||||
created=created,
|
||||
modified=modified,
|
||||
workspace_path=directory or None,
|
||||
messages=msgs,
|
||||
source_locator=f"{db}#{sess_id}",
|
||||
model_type=model_name or "opencode",
|
||||
extra=extra,
|
||||
)
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kimi (re-export of the proven layout so the unified driver covers it too)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -501,11 +930,37 @@ def discover_kimi(kimi_dir: Optional[Path] = None) -> Iterable[NormalizedSession
|
|||
role = rec.get("role")
|
||||
if role in (None, "_system_prompt", "_checkpoint", "_usage"):
|
||||
continue
|
||||
if role == "tool_call":
|
||||
msgs.append(NormalizedMessage(role="tool_call", content=flatten_content(rec.get("content"))[:500], id=f"kc-{i:06d}"))
|
||||
continue
|
||||
if role == "tool_result":
|
||||
msgs.append(NormalizedMessage(role="tool_result", content=flatten_content(rec.get("content"))[:500], id=f"kr-{i:06d}"))
|
||||
# Kimi assistant messages may carry tool_calls — emit as tool_call
|
||||
if role == "assistant":
|
||||
tc = rec.get("tool_calls")
|
||||
if isinstance(tc, list) and tc:
|
||||
for j, t in enumerate(tc):
|
||||
fn = t.get("function", {}) if isinstance(t, dict) else {}
|
||||
name = fn.get("name", "?")
|
||||
args = fn.get("arguments", "")
|
||||
if isinstance(args, str) and len(args) > 500:
|
||||
args = args[:500]
|
||||
msgs.append(NormalizedMessage(
|
||||
role="tool_call",
|
||||
content=f"{name}({args})",
|
||||
id=f"kc-{i:06d}-{j}",
|
||||
))
|
||||
# Also emit any text content as assistant
|
||||
content = flatten_content(rec.get("content"))
|
||||
if content:
|
||||
msgs.append(NormalizedMessage(
|
||||
role="assistant",
|
||||
content=content,
|
||||
id=f"k-{i:06d}",
|
||||
))
|
||||
continue
|
||||
# Kimi tool results use role=tool (not tool_result)
|
||||
if role in ("tool", "tool_result"):
|
||||
msgs.append(NormalizedMessage(
|
||||
role="tool_result",
|
||||
content=flatten_content(rec.get("content"))[:2000],
|
||||
id=f"kr-{i:06d}",
|
||||
))
|
||||
continue
|
||||
if role not in ("user", "model"):
|
||||
continue
|
||||
|
|
@ -530,6 +985,377 @@ def discover_kimi(kimi_dir: Optional[Path] = None) -> Iterable[NormalizedSession
|
|||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cursor (~/.config/Cursor/User/workspaceStorage/*/state.vscdb + globalStorage)
|
||||
# Chat sessions are in the globalStorage state.vscdb under
|
||||
# 'chat.ChatSessionStore.index' and per-workspace state.vscdb keys.
|
||||
# Composer/agent sessions are in globalStorage/cursor.chatSessions or similar.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def discover_cursor(cursor_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||
root = cursor_dir or (HOME / ".config" / "Cursor")
|
||||
if not root.exists():
|
||||
return
|
||||
|
||||
# Cursor stores chat data in workspace-level state.vscdb SQLite DBs.
|
||||
# The composer (agent) history is in globalStorage.
|
||||
for db_path in sorted(root.rglob("state.vscdb")):
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
||||
except sqlite3.Error:
|
||||
continue
|
||||
try:
|
||||
con.row_factory = sqlite3.Row
|
||||
try:
|
||||
rows = con.execute(
|
||||
"SELECT key, value FROM ItemTable WHERE "
|
||||
"key LIKE '%chatSession%' OR key LIKE '%composer%' "
|
||||
"OR key LIKE '%chat%session%' OR key LIKE '%aichat%'"
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
continue
|
||||
|
||||
for row in rows:
|
||||
key = row["key"] or ""
|
||||
raw = row["value"]
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
data = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
|
||||
# Extract messages from common Cursor chat formats
|
||||
messages_raw = (
|
||||
data.get("messages") or data.get("entries")
|
||||
or data.get("history") or data.get("turns")
|
||||
)
|
||||
if not isinstance(messages_raw, list) or not messages_raw:
|
||||
continue
|
||||
|
||||
session_id = (
|
||||
data.get("sessionId") or data.get("id")
|
||||
or (hashlib.sha1(raw.encode()).hexdigest()[:16] if isinstance(raw, str) else db_path.stem)
|
||||
)
|
||||
workspace = str(db_path.parent.parent) if "workspaceStorage" in str(db_path) else None
|
||||
|
||||
msgs: list[NormalizedMessage] = []
|
||||
for i, m in enumerate(messages_raw):
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
role = m.get("role") or m.get("type", "")
|
||||
content = flatten_content(m.get("content") or m.get("text") or m.get("message"))
|
||||
if not content:
|
||||
continue
|
||||
norm_role = "user" if "user" in role.lower() else (
|
||||
"tool_call" if "tool" in role.lower() and "result" not in role.lower() else (
|
||||
"tool_result" if "result" in role.lower() else "assistant"
|
||||
)
|
||||
)
|
||||
msgs.append(NormalizedMessage(
|
||||
role=norm_role,
|
||||
content=content,
|
||||
timestamp=to_iso_z(m.get("timestamp") or m.get("createdAt")),
|
||||
model=m.get("model") or data.get("model"),
|
||||
id=m.get("id") or f"cur-{i:06d}",
|
||||
))
|
||||
|
||||
if not msgs:
|
||||
continue
|
||||
created, modified = _file_times(db_path)
|
||||
yield NormalizedSession(
|
||||
source="cursor",
|
||||
session_id=source_id_for("cursor", str(session_id)),
|
||||
title=data.get("title") or "",
|
||||
created=created,
|
||||
modified=modified,
|
||||
workspace_path=workspace,
|
||||
messages=msgs,
|
||||
source_locator=f"{db_path}#{key}",
|
||||
model_type=data.get("model") or "cursor",
|
||||
)
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Zed (~/.local/share/zed/threads/*.json + ~/.config/zed/)
|
||||
# Thread files are JSON with entries[].role + entries[].content.
|
||||
# Agent conversations in ~/.local/share/zed/assistant2/ or threads/.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def discover_zed(zed_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||
roots = [zed_dir] if zed_dir else [
|
||||
HOME / ".local" / "share" / "zed",
|
||||
HOME / ".config" / "zed",
|
||||
]
|
||||
for root in roots:
|
||||
if not root or not root.exists():
|
||||
continue
|
||||
for sub in ("threads", "assistant2", "assistant", "conversations"):
|
||||
thread_dir = root / sub
|
||||
if not thread_dir.exists():
|
||||
continue
|
||||
for f in sorted(thread_dir.rglob("*.json")):
|
||||
data = _load_json(f)
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
entries = data.get("entries") or data.get("messages") or data.get("turns")
|
||||
if not isinstance(entries, list) or not entries:
|
||||
continue
|
||||
|
||||
session_id = data.get("id") or data.get("threadId") or f.stem
|
||||
msgs: list[NormalizedMessage] = []
|
||||
for i, entry in enumerate(entries):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
role = entry.get("role", "")
|
||||
content = flatten_content(entry.get("content") or entry.get("text") or entry.get("message"))
|
||||
if not content:
|
||||
continue
|
||||
norm_role = "user" if role == "user" else (
|
||||
"tool_call" if "tool" in role.lower() else "assistant"
|
||||
)
|
||||
msgs.append(NormalizedMessage(
|
||||
role=norm_role,
|
||||
content=content,
|
||||
timestamp=to_iso_z(entry.get("timestamp")),
|
||||
model=entry.get("model"),
|
||||
id=f"zed-{i:06d}",
|
||||
))
|
||||
|
||||
if not msgs:
|
||||
continue
|
||||
created, modified = _file_times(f)
|
||||
yield NormalizedSession(
|
||||
source="zed",
|
||||
session_id=source_id_for("zed", str(session_id)),
|
||||
title=data.get("title") or data.get("summary") or "",
|
||||
created=created,
|
||||
modified=modified,
|
||||
workspace_path=data.get("workspace"),
|
||||
messages=msgs,
|
||||
source_locator=str(f),
|
||||
model_type=data.get("model") or "zed",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Continue.dev (~/.continue/sessions/*.json)
|
||||
# Each session is a JSON file with history[] containing {role, content}.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def discover_continue(continue_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||
root = continue_dir or (HOME / ".continue")
|
||||
sessions_dir = root / "sessions"
|
||||
if not sessions_dir.exists():
|
||||
return
|
||||
|
||||
for f in sorted(sessions_dir.rglob("*.json")):
|
||||
data = _load_json(f)
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
history = data.get("history") or data.get("messages")
|
||||
if not isinstance(history, list) or not history:
|
||||
continue
|
||||
|
||||
session_id = data.get("sessionId") or data.get("id") or f.stem
|
||||
msgs: list[NormalizedMessage] = []
|
||||
for i, entry in enumerate(history):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
role = entry.get("role", "")
|
||||
content = flatten_content(entry.get("content") or entry.get("message"))
|
||||
if not content:
|
||||
continue
|
||||
norm_role = "user" if role == "user" else (
|
||||
"tool_call" if "tool" in role.lower() else "assistant"
|
||||
)
|
||||
msgs.append(NormalizedMessage(
|
||||
role=norm_role,
|
||||
content=content,
|
||||
timestamp=to_iso_z(entry.get("timestamp")),
|
||||
model=entry.get("model") or data.get("model"),
|
||||
id=f"cont-{i:06d}",
|
||||
))
|
||||
|
||||
if not msgs:
|
||||
continue
|
||||
created, modified = _file_times(f)
|
||||
yield NormalizedSession(
|
||||
source="continue",
|
||||
session_id=source_id_for("continue", str(session_id)),
|
||||
title=data.get("title") or "",
|
||||
created=created,
|
||||
modified=modified,
|
||||
workspace_path=data.get("workspaceDirectory"),
|
||||
messages=msgs,
|
||||
source_locator=str(f),
|
||||
model_type=data.get("model") or "continue",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Aider (.aider.chat.history.md in each repo root)
|
||||
# Markdown chat log with # aider chat started at ... headers.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def discover_aider(home: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||
# Aider writes .aider.chat.history.md in the repo root where it runs.
|
||||
# We scan the home directory for repos with aider history.
|
||||
# Skip hidden dirs and common non-repo dirs.
|
||||
_SKIP = {".cache", ".config", ".local", ".npm", ".vscode", ".git",
|
||||
"node_modules", "__pycache__", ".mozilla", ".thunderbird"}
|
||||
home_dir = home or HOME
|
||||
|
||||
for history_file in sorted(home_dir.rglob(".aider.chat.history.md")):
|
||||
# Skip if inside a hidden dir (other than the file itself)
|
||||
parts = history_file.relative_to(home_dir).parts
|
||||
if any(p.startswith(".") and p != ".aider.chat.history.md" for p in parts[:-1]):
|
||||
if not any(p in _SKIP for p in parts[:-1]):
|
||||
# Allow .aurelio, .aider, etc.
|
||||
pass
|
||||
else:
|
||||
continue
|
||||
|
||||
try:
|
||||
text = history_file.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
if not text.strip():
|
||||
continue
|
||||
|
||||
# Parse aider markdown: user messages start with ####, assistant with > or plain
|
||||
msgs: list[NormalizedMessage] = []
|
||||
current_role: Optional[str] = None
|
||||
current_lines: list[str] = []
|
||||
|
||||
def flush():
|
||||
nonlocal current_role, current_lines
|
||||
if current_role and current_lines:
|
||||
content = "\n".join(current_lines).strip()
|
||||
if content:
|
||||
msgs.append(NormalizedMessage(
|
||||
role=current_role,
|
||||
content=content,
|
||||
id=f"aider-{len(msgs):06d}",
|
||||
))
|
||||
current_role = None
|
||||
current_lines = []
|
||||
|
||||
for line in text.splitlines():
|
||||
if line.startswith("# aider chat started at"):
|
||||
flush()
|
||||
continue
|
||||
if line.startswith("#### "):
|
||||
flush()
|
||||
current_role = "user"
|
||||
current_lines = [line[5:]]
|
||||
elif line.startswith("> "):
|
||||
if current_role != "assistant":
|
||||
flush()
|
||||
current_role = "assistant"
|
||||
current_lines.append(line[2:])
|
||||
elif line.strip() and current_role:
|
||||
current_lines.append(line)
|
||||
|
||||
flush()
|
||||
if not msgs:
|
||||
continue
|
||||
|
||||
workspace = str(history_file.parent)
|
||||
created, modified = _file_times(history_file)
|
||||
session_id = hashlib.sha1(workspace.encode()).hexdigest()[:16]
|
||||
yield NormalizedSession(
|
||||
source="aider",
|
||||
session_id=source_id_for("aider", session_id),
|
||||
title=f"Aider in {Path(workspace).name}",
|
||||
created=created,
|
||||
modified=modified,
|
||||
workspace_path=workspace,
|
||||
messages=msgs,
|
||||
source_locator=str(history_file),
|
||||
model_type="aider",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cline / Roo Code (VS Code globalStorage/<publisher>.<name>/tasks/)
|
||||
# Task files are JSON with messages[] containing {role, content, toolUses}.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CLINE_PUBLISHERS = [
|
||||
"saoudrizwan.claude-dev",
|
||||
"rooveterinaryinc.roo-cline",
|
||||
]
|
||||
|
||||
def discover_cline(vscode_storage: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||
roots = [vscode_storage] if vscode_storage else [
|
||||
HOME / ".config" / "Code" / "User" / "globalStorage",
|
||||
HOME / ".config" / "Code - Insiders" / "User" / "globalStorage",
|
||||
HOME / ".vscode" / "extensions",
|
||||
]
|
||||
for root in roots:
|
||||
if not root or not root.exists():
|
||||
continue
|
||||
for publisher in _CLINE_PUBLISHERS:
|
||||
ext_dir = root / publisher
|
||||
if not ext_dir.exists():
|
||||
continue
|
||||
# Cline stores tasks in tasks/ subdirectory
|
||||
for task_dir_name in ("tasks", "taskHistory", "history"):
|
||||
task_dir = ext_dir / task_dir_name
|
||||
if not task_dir.exists():
|
||||
continue
|
||||
for f in sorted(task_dir.rglob("*.json")):
|
||||
data = _load_json(f)
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
messages_raw = data.get("messages") or data.get("apiConversationHistory")
|
||||
if not isinstance(messages_raw, list) or not messages_raw:
|
||||
continue
|
||||
|
||||
session_id = data.get("taskId") or data.get("id") or f.stem
|
||||
source = "cline" if "claude-dev" in publisher else "roo-code"
|
||||
msgs: list[NormalizedMessage] = []
|
||||
for i, m in enumerate(messages_raw):
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
role = m.get("role", "")
|
||||
content = flatten_content(m.get("content"))
|
||||
if not content:
|
||||
continue
|
||||
norm_role = "user" if role == "user" else (
|
||||
"tool_call" if "tool" in role.lower() and "result" not in role.lower() else (
|
||||
"tool_result" if "result" in role.lower() else "assistant"
|
||||
)
|
||||
)
|
||||
msgs.append(NormalizedMessage(
|
||||
role=norm_role,
|
||||
content=content,
|
||||
timestamp=to_iso_z(m.get("timestamp")),
|
||||
model=m.get("model"),
|
||||
id=f"{source[:2]}-{i:06d}",
|
||||
))
|
||||
|
||||
if not msgs:
|
||||
continue
|
||||
created, modified = _file_times(f)
|
||||
yield NormalizedSession(
|
||||
source=source,
|
||||
session_id=source_id_for(source, str(session_id)),
|
||||
title=data.get("title") or data.get("task", "")[:80],
|
||||
created=created,
|
||||
modified=modified,
|
||||
workspace_path=data.get("workspace"),
|
||||
messages=msgs,
|
||||
source_locator=str(f),
|
||||
model_type=data.get("model") or source,
|
||||
)
|
||||
|
||||
|
||||
ADAPTERS = {
|
||||
"kimi": discover_kimi,
|
||||
"hermes": discover_hermes,
|
||||
|
|
@ -537,4 +1363,11 @@ ADAPTERS = {
|
|||
"antigravity": discover_antigravity_cli,
|
||||
"qwen-code": discover_qwen_code,
|
||||
"mimocode": discover_mimocode,
|
||||
"pi": discover_pi,
|
||||
"opencode": discover_opencode,
|
||||
"cursor": discover_cursor,
|
||||
"zed": discover_zed,
|
||||
"continue": discover_continue,
|
||||
"aider": discover_aider,
|
||||
"cline": discover_cline,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -306,7 +306,7 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A
|
|||
**({"extra": sess.extra} if sess.extra else {}),
|
||||
}
|
||||
|
||||
session_jsonl = [
|
||||
session_jsonl_lines = [
|
||||
json.dumps({
|
||||
"type": "session_start",
|
||||
"timestamp": sess.created,
|
||||
|
|
@ -317,12 +317,58 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A
|
|||
"sourceSessionId": sess.session_id,
|
||||
},
|
||||
}, ensure_ascii=False),
|
||||
]
|
||||
# Full trajectory: one event per message, preserving tool_call/tool_result
|
||||
# sequence for RL training. This is the primary training data artifact.
|
||||
for idx, (m, ts) in enumerate(zip(sess.messages, timestamps)):
|
||||
event: dict[str, Any] = {
|
||||
"type": "message",
|
||||
"timestamp": ts,
|
||||
"data": {
|
||||
"id": m.id or f"m{idx:06d}",
|
||||
"role": m.role,
|
||||
"content": m.content,
|
||||
},
|
||||
}
|
||||
if m.model:
|
||||
event["data"]["model"] = m.model
|
||||
session_jsonl_lines.append(json.dumps(event, ensure_ascii=False))
|
||||
session_jsonl_lines.append(
|
||||
json.dumps({
|
||||
"type": "session_end",
|
||||
"timestamp": sess.modified,
|
||||
"data": {"sessionId": sess.brain_id, "totalEvents": len(chat_messages)},
|
||||
"data": {"sessionId": sess.brain_id, "totalEvents": len(sess.messages)},
|
||||
}, ensure_ascii=False),
|
||||
]
|
||||
)
|
||||
|
||||
# RL trajectory: compact (state, action, reward) triples derived from the
|
||||
# message sequence. Each tool_call is an action; the preceding user/assistant
|
||||
# message is the state; the tool_result is the immediate observation.
|
||||
trajectory_lines: list[str] = []
|
||||
for idx, m in enumerate(sess.messages):
|
||||
if m.role != "tool_call":
|
||||
continue
|
||||
# Find the nearest preceding user/assistant message as state
|
||||
state = ""
|
||||
for j in range(idx - 1, -1, -1):
|
||||
if sess.messages[j].role in ("user", "assistant"):
|
||||
state = sess.messages[j].content[:2000]
|
||||
break
|
||||
# Find the tool_result that follows
|
||||
observation = ""
|
||||
if idx + 1 < len(sess.messages) and sess.messages[idx + 1].role == "tool_result":
|
||||
observation = sess.messages[idx + 1].content[:2000]
|
||||
traj_event = {
|
||||
"type": "trajectory_step",
|
||||
"session_id": sess.brain_id,
|
||||
"step": idx,
|
||||
"timestamp": timestamps[idx] if idx < len(timestamps) else None,
|
||||
"state": state,
|
||||
"action": m.content[:2000],
|
||||
"observation": observation,
|
||||
"model": m.model or sess.model_type,
|
||||
}
|
||||
trajectory_lines.append(json.dumps(traj_event, ensure_ascii=False))
|
||||
|
||||
overview_txt = (
|
||||
f"# Conversation: {title}\n"
|
||||
|
|
@ -334,11 +380,13 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A
|
|||
)
|
||||
|
||||
files = {
|
||||
target_dir / "session.jsonl": "\n".join(session_jsonl) + "\n",
|
||||
target_dir / "session.jsonl": "\n".join(session_jsonl_lines) + "\n",
|
||||
target_dir / "session_memory.md": render_session_memory(sess, import_ts),
|
||||
target_dir / ".system_generated" / "chat_history.json": json.dumps(chat_history, indent=2, ensure_ascii=False) + "\n",
|
||||
target_dir / ".system_generated" / "logs" / "overview.txt": overview_txt,
|
||||
}
|
||||
if trajectory_lines:
|
||||
files[target_dir / ".system_generated" / "trajectory.jsonl"] = "\n".join(trajectory_lines) + "\n"
|
||||
|
||||
if dry_run:
|
||||
result["would_write"] = [str(p.relative_to(REPLICA_ROOT)) for p in files]
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ bridge (`scripts/sync-kimi-to-brain.py`) but for all sources:
|
|||
antigravity ~/.gemini/antigravity-{cli,ide}/conversations/*.db (sqlite)
|
||||
qwen-code ~/.qwen[-code]/**/{context.jsonl,*.jsonl} (no-op if absent)
|
||||
mimocode ./.mimocode/plans/*.md (+ future *.jsonl)
|
||||
pi ~/.pi/agent/sessions/**/*.jsonl
|
||||
opencode ~/.local/share/opencode/opencode.db (sqlite)
|
||||
|
||||
Output contract per session (see agent_importers/engine.py):
|
||||
.aurelio/brain/session-<source>-<id>/{session.jsonl, session_memory.md,
|
||||
|
|
@ -52,6 +54,8 @@ from agent_importers.adapters import ( # noqa: E402
|
|||
discover_antigravity_cli,
|
||||
discover_qwen_code,
|
||||
discover_mimocode,
|
||||
discover_pi,
|
||||
discover_opencode,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -89,6 +93,8 @@ def run(args: argparse.Namespace) -> dict[str, Any]:
|
|||
"antigravity": lambda: discover_antigravity_cli(Path(args.gemini_dir) if args.gemini_dir else None),
|
||||
"qwen-code": lambda: discover_qwen_code(Path(args.qwen_dir) if args.qwen_dir else None),
|
||||
"mimocode": lambda: discover_mimocode(repo_root=replica_root),
|
||||
"pi": lambda: discover_pi(Path(args.pi_dir) if args.pi_dir else None),
|
||||
"opencode": lambda: discover_opencode(Path(args.opencode_db) if args.opencode_db else None),
|
||||
}
|
||||
|
||||
for name in requested:
|
||||
|
|
@ -139,6 +145,8 @@ def main() -> int:
|
|||
ap.add_argument("--claude-dir", default=None)
|
||||
ap.add_argument("--gemini-dir", default=None)
|
||||
ap.add_argument("--qwen-dir", default=None)
|
||||
ap.add_argument("--pi-dir", default=None)
|
||||
ap.add_argument("--opencode-db", default=None)
|
||||
ap.add_argument("--summary", action="store_true", help="Print JSON summary.")
|
||||
ap.add_argument("--verbose", "-v", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
|
|
|||
Loading…
Reference in a new issue