feat(scripts): agent importer — new adapters, RL trajectory output
Some checks failed
Aurélio Sync & Conscience Upgrade / Upgrade Réplica Conscience (push) Failing after 27s
Some checks failed
Aurélio Sync & Conscience Upgrade / Upgrade Réplica Conscience (push) Failing after 27s
Extends sync-agents-to-brain.py with: - cursor, zed, continue, aider, cline adapters (11 total now) - RL trajectory extraction (state/action/observation triples) - Improved content flattening for OpenAI/Anthropic multipart content - Engine: stable per-message timestamps, session_memory.md rendering Co-authored-by: Álvaro de Campos <campos@portugalfuturista.org>
This commit is contained in:
parent
29f71dd330
commit
c96b6631ce
3 changed files with 925 additions and 36 deletions
|
|
@ -8,6 +8,7 @@ abort a fleet-wide import.
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
|
@ -118,6 +119,33 @@ def discover_hermes(hermes_dir: Optional[Path] = None) -> Iterable[NormalizedSes
|
||||||
role = m.get("role")
|
role = m.get("role")
|
||||||
if role not in ("user", "assistant", "tool", "system"):
|
if role not in ("user", "assistant", "tool", "system"):
|
||||||
continue
|
continue
|
||||||
|
# Hermes assistant messages may carry OpenAI-style tool_calls
|
||||||
|
if role == "assistant":
|
||||||
|
tc = m.get("tool_calls")
|
||||||
|
if isinstance(tc, list) and tc:
|
||||||
|
for j, t in enumerate(tc):
|
||||||
|
fn = t.get("function", {}) if isinstance(t, dict) else {}
|
||||||
|
name = fn.get("name", "?")
|
||||||
|
args = fn.get("arguments", "")
|
||||||
|
if isinstance(args, str) and len(args) > 500:
|
||||||
|
args = args[:500]
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role="tool_call",
|
||||||
|
content=f"{name}({args})",
|
||||||
|
timestamp=to_iso_z(m.get("timestamp")),
|
||||||
|
model=m.get("model") or model,
|
||||||
|
id=f"hc-{i:06d}-{j}",
|
||||||
|
))
|
||||||
|
content = flatten_content(m.get("content"))
|
||||||
|
if content:
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role="assistant",
|
||||||
|
content=content,
|
||||||
|
timestamp=to_iso_z(m.get("timestamp")),
|
||||||
|
model=m.get("model") or model,
|
||||||
|
id=f"h-{i:06d}",
|
||||||
|
))
|
||||||
|
continue
|
||||||
norm_role = "tool_result" if role == "tool" else role
|
norm_role = "tool_result" if role == "tool" else role
|
||||||
content = flatten_content(m.get("content"))
|
content = flatten_content(m.get("content"))
|
||||||
if not content and norm_role in ("user", "assistant"):
|
if not content and norm_role in ("user", "assistant"):
|
||||||
|
|
@ -198,7 +226,39 @@ def discover_claude_code(claude_dir: Optional[Path] = None) -> Iterable[Normaliz
|
||||||
if not isinstance(message, dict):
|
if not isinstance(message, dict):
|
||||||
continue
|
continue
|
||||||
role = message.get("role") or rtype
|
role = message.get("role") or rtype
|
||||||
content = flatten_content(message.get("content"))
|
content_raw = message.get("content")
|
||||||
|
|
||||||
|
# Claude assistant messages may contain tool_use parts — emit as tool_call
|
||||||
|
if isinstance(content_raw, list):
|
||||||
|
tool_use_parts = [p for p in content_raw if isinstance(p, dict) and p.get("type") == "tool_use"]
|
||||||
|
if tool_use_parts:
|
||||||
|
for j, part in enumerate(tool_use_parts):
|
||||||
|
name = part.get("name", "?")
|
||||||
|
inp = json.dumps(part.get("input", {}), ensure_ascii=False)[:500]
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role="tool_call",
|
||||||
|
content=f"{name}({inp})",
|
||||||
|
timestamp=to_iso_z(rec.get("timestamp")),
|
||||||
|
model=message.get("model"),
|
||||||
|
id=f"cc-{rec.get('uuid', 'x')}-t{j}",
|
||||||
|
))
|
||||||
|
# Also emit any text content as assistant
|
||||||
|
text_parts = [p.get("text", "") for p in content_raw if isinstance(p, dict) and p.get("type") == "text"]
|
||||||
|
content = "\n".join(t for t in text_parts if t)
|
||||||
|
if content:
|
||||||
|
if role == "assistant":
|
||||||
|
model = model or message.get("model")
|
||||||
|
norm_role = "assistant" if role == "assistant" else ("user" if role == "user" else "system")
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role=norm_role,
|
||||||
|
content=content,
|
||||||
|
timestamp=to_iso_z(rec.get("timestamp")),
|
||||||
|
model=message.get("model") if norm_role == "assistant" else None,
|
||||||
|
id=rec.get("uuid") or None,
|
||||||
|
))
|
||||||
|
continue
|
||||||
|
|
||||||
|
content = flatten_content(content_raw)
|
||||||
if role == "assistant" and not content:
|
if role == "assistant" and not content:
|
||||||
continue
|
continue
|
||||||
if role == "assistant":
|
if role == "assistant":
|
||||||
|
|
@ -243,6 +303,31 @@ def discover_claude_code(claude_dir: Optional[Path] = None) -> Iterable[Normaliz
|
||||||
# status, task_details, step_payload, ...)
|
# status, task_details, step_payload, ...)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _antigravity_extract_json(blob: bytes | str) -> Optional[dict]:
|
||||||
|
"""Extract the first JSON object from a protobuf blob that embeds JSON."""
|
||||||
|
if isinstance(blob, bytes):
|
||||||
|
blob = blob.decode("utf-8", "replace")
|
||||||
|
start = blob.find("{")
|
||||||
|
if start < 0:
|
||||||
|
return None
|
||||||
|
depth = 0
|
||||||
|
for i in range(start, len(blob)):
|
||||||
|
if blob[i] == "{": depth += 1
|
||||||
|
elif blob[i] == "}":
|
||||||
|
depth -= 1
|
||||||
|
if depth == 0:
|
||||||
|
try:
|
||||||
|
return json.loads(blob[start:i + 1])
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Antigravity step_type enum (from protobuf field numbers observed in the wild)
|
||||||
|
_AG_TOOL_TYPES = {9, 21} # tool call steps (list_dir, run_command, etc.)
|
||||||
|
_AG_RESPONSE_TYPES = {15, 90} # assistant/planner response steps
|
||||||
|
_AG_USER_TYPES = {5, 14} # user input steps
|
||||||
|
|
||||||
def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]:
|
def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]:
|
||||||
msgs: list[NormalizedMessage] = []
|
msgs: list[NormalizedMessage] = []
|
||||||
try:
|
try:
|
||||||
|
|
@ -256,54 +341,87 @@ def _antigravity_steps_to_messages(db_path: Path) -> list[NormalizedMessage]:
|
||||||
except sqlite3.Error:
|
except sqlite3.Error:
|
||||||
return msgs
|
return msgs
|
||||||
for row in rows:
|
for row in rows:
|
||||||
step_type = row["step_type"] or ""
|
step_type = row["step_type"]
|
||||||
|
if not isinstance(step_type, int):
|
||||||
|
continue
|
||||||
payload_raw = row["step_payload"]
|
payload_raw = row["step_payload"]
|
||||||
task_raw = row["task_details"]
|
task_raw = row["task_details"]
|
||||||
text = ""
|
text = ""
|
||||||
|
tool_name = ""
|
||||||
|
tool_input = ""
|
||||||
|
|
||||||
for blob in (payload_raw, task_raw):
|
for blob in (payload_raw, task_raw):
|
||||||
if not blob:
|
if not blob:
|
||||||
continue
|
continue
|
||||||
if isinstance(blob, bytes):
|
obj = _antigravity_extract_json(blob)
|
||||||
try:
|
if obj:
|
||||||
blob = blob.decode("utf-8", "replace")
|
# Tool call extraction
|
||||||
except Exception:
|
if step_type in _AG_TOOL_TYPES:
|
||||||
continue
|
tool_name = (
|
||||||
if isinstance(blob, str):
|
obj.get("toolAction", "").split(" ")[0].lower()
|
||||||
stripped = blob.strip()
|
or obj.get("name", "")
|
||||||
|
or obj.get("tool", "")
|
||||||
|
or obj.get("CommandLine", "").split(" ")[0]
|
||||||
|
)
|
||||||
|
tool_input = json.dumps(obj, ensure_ascii=False)[:500]
|
||||||
|
text = tool_input
|
||||||
|
else:
|
||||||
|
text = (
|
||||||
|
obj.get("text") or obj.get("content")
|
||||||
|
or obj.get("message") or obj.get("prompt")
|
||||||
|
or obj.get("plan") or ""
|
||||||
|
)
|
||||||
|
if not text and isinstance(obj.get("messages"), list):
|
||||||
|
text = "\n".join(
|
||||||
|
flatten_content(m.get("content")) for m in obj["messages"]
|
||||||
|
if isinstance(m, dict)
|
||||||
|
)
|
||||||
|
if text:
|
||||||
|
break
|
||||||
|
elif isinstance(blob, (bytes, str)):
|
||||||
|
# Fallback: raw text extraction
|
||||||
|
raw = blob.decode("utf-8", "replace") if isinstance(blob, bytes) else blob
|
||||||
|
stripped = raw.strip()
|
||||||
if stripped.startswith("{") or stripped.startswith("["):
|
if stripped.startswith("{") or stripped.startswith("["):
|
||||||
try:
|
try:
|
||||||
obj = json.loads(stripped)
|
obj = json.loads(stripped)
|
||||||
# common fields that carry human text
|
|
||||||
text = (
|
text = (
|
||||||
obj.get("text") or obj.get("content")
|
obj.get("text") or obj.get("content")
|
||||||
or obj.get("message") or obj.get("prompt")
|
or obj.get("message") or obj.get("prompt")
|
||||||
or obj.get("plan") or ""
|
or obj.get("plan") or ""
|
||||||
)
|
)
|
||||||
if not text and isinstance(obj.get("messages"), list):
|
|
||||||
text = "\n".join(
|
|
||||||
flatten_content(m.get("content")) for m in obj["messages"]
|
|
||||||
if isinstance(m, dict)
|
|
||||||
)
|
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
text = stripped
|
text = stripped
|
||||||
else:
|
else:
|
||||||
text = stripped
|
# Skip binary protobuf blobs that aren't JSON
|
||||||
if text:
|
if any(c.isprintable() for c in stripped[:50]):
|
||||||
break
|
text = stripped
|
||||||
st = str(step_type).lower()
|
if text:
|
||||||
if "user" in st or "input" in st:
|
break
|
||||||
role = "user"
|
|
||||||
elif "planner" in st or "agent" in st or "assistant" in st or "response" in st:
|
if step_type in _AG_TOOL_TYPES:
|
||||||
role = "assistant"
|
|
||||||
elif "tool" in st or "command" in st or "action" in st:
|
|
||||||
role = "tool_call"
|
role = "tool_call"
|
||||||
|
if not tool_name:
|
||||||
|
# Try to extract tool name from the protobuf text
|
||||||
|
raw = payload_raw.decode("utf-8", "replace") if isinstance(payload_raw, bytes) else str(payload_raw or "")
|
||||||
|
for marker in ("run_command", "list_dir", "read_file", "write_file", "search", "edit_file"):
|
||||||
|
if marker in raw:
|
||||||
|
tool_name = marker
|
||||||
|
break
|
||||||
|
content = f"{tool_name}({tool_input})" if tool_input else (tool_name or f"[step {step_type}]")
|
||||||
|
elif step_type in _AG_USER_TYPES:
|
||||||
|
role = "user"
|
||||||
|
content = text[:2000] if text else "[user input]"
|
||||||
|
elif step_type in _AG_RESPONSE_TYPES:
|
||||||
|
role = "assistant"
|
||||||
|
content = text[:2000] if text else f"[step {step_type}]"
|
||||||
else:
|
else:
|
||||||
role = "assistant"
|
role = "assistant"
|
||||||
if not text:
|
content = text[:2000] if text else f"[step {step_type}]"
|
||||||
text = f"[step {step_type}]"
|
|
||||||
msgs.append(NormalizedMessage(
|
msgs.append(NormalizedMessage(
|
||||||
role=role,
|
role=role,
|
||||||
content=text[:2000],
|
content=content,
|
||||||
id=f"ag-{row['idx']:06d}",
|
id=f"ag-{row['idx']:06d}",
|
||||||
))
|
))
|
||||||
finally:
|
finally:
|
||||||
|
|
@ -465,6 +583,317 @@ def discover_mimocode(repo_root: Optional[Path] = None, home: Optional[Path] = N
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Pi (~/.pi/agent/sessions/<workspace-hash>/<timestamp>_<uuid>.jsonl)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def discover_pi(pi_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||||
|
root = pi_dir or (HOME / ".pi" / "agent")
|
||||||
|
sessions_dir = root / "sessions"
|
||||||
|
if not sessions_dir.exists():
|
||||||
|
return
|
||||||
|
|
||||||
|
for jsonl in sorted(sessions_dir.rglob("*.jsonl")):
|
||||||
|
records = _load_jsonl(jsonl)
|
||||||
|
if not records:
|
||||||
|
continue
|
||||||
|
|
||||||
|
session_id = None
|
||||||
|
cwd = None
|
||||||
|
model = None
|
||||||
|
msgs: list[NormalizedMessage] = []
|
||||||
|
|
||||||
|
for rec in records:
|
||||||
|
if not isinstance(rec, dict):
|
||||||
|
continue
|
||||||
|
rtype = rec.get("type")
|
||||||
|
|
||||||
|
if rtype == "session":
|
||||||
|
session_id = rec.get("id")
|
||||||
|
cwd = rec.get("cwd")
|
||||||
|
continue
|
||||||
|
|
||||||
|
if rtype == "model_change":
|
||||||
|
provider = rec.get("provider", "")
|
||||||
|
model_id = rec.get("modelId", "")
|
||||||
|
model = f"{provider}/{model_id}" if provider and model_id else (model_id or provider or None)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if rtype in ("thinking_level_change",):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if rtype != "message":
|
||||||
|
continue
|
||||||
|
|
||||||
|
message = rec.get("message")
|
||||||
|
if not isinstance(message, dict):
|
||||||
|
continue
|
||||||
|
|
||||||
|
role = message.get("role")
|
||||||
|
if role not in ("user", "assistant", "toolResult"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
content_raw = message.get("content")
|
||||||
|
|
||||||
|
# Pi assistant messages may contain toolCall content parts
|
||||||
|
if isinstance(content_raw, list):
|
||||||
|
tool_use_parts = [p for p in content_raw if isinstance(p, dict) and p.get("type") in ("tool_use", "toolCall")]
|
||||||
|
if tool_use_parts:
|
||||||
|
ts = to_iso_z(message.get("timestamp") or rec.get("timestamp"))
|
||||||
|
for j, part in enumerate(tool_use_parts):
|
||||||
|
name = part.get("name", "?")
|
||||||
|
inp_raw = part.get("input") or part.get("arguments", {})
|
||||||
|
inp = json.dumps(inp_raw, ensure_ascii=False)[:500]
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role="tool_call",
|
||||||
|
content=f"{name}({inp})",
|
||||||
|
timestamp=ts,
|
||||||
|
model=message.get("model") or model,
|
||||||
|
id=f"pi-{rec.get('id', 'x')}-t{j}",
|
||||||
|
))
|
||||||
|
# Also emit text content
|
||||||
|
text_parts = [p.get("text", "") for p in content_raw if isinstance(p, dict) and p.get("type") == "text"]
|
||||||
|
content = "\n".join(t for t in text_parts if t)
|
||||||
|
if content:
|
||||||
|
norm_role = "assistant" if role == "assistant" else ("user" if role == "user" else "system")
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role=norm_role,
|
||||||
|
content=content,
|
||||||
|
timestamp=ts,
|
||||||
|
model=(message.get("model") or model) if norm_role == "assistant" else None,
|
||||||
|
id=str(rec.get("id") or f"pi-{len(msgs):06d}"),
|
||||||
|
))
|
||||||
|
continue
|
||||||
|
|
||||||
|
norm_role = "tool_result" if role == "toolResult" else role
|
||||||
|
content = flatten_content(content_raw)
|
||||||
|
if not content and norm_role in ("user", "assistant"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
msg_model = None
|
||||||
|
if norm_role == "assistant":
|
||||||
|
msg_model = message.get("model") or model
|
||||||
|
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role=norm_role,
|
||||||
|
content=content,
|
||||||
|
timestamp=to_iso_z(message.get("timestamp") or rec.get("timestamp")),
|
||||||
|
model=msg_model,
|
||||||
|
id=str(rec.get("id") or f"pi-{len(msgs):06d}"),
|
||||||
|
))
|
||||||
|
|
||||||
|
if not msgs:
|
||||||
|
continue
|
||||||
|
|
||||||
|
created, modified = _file_times(jsonl)
|
||||||
|
brain_session_id = str(session_id or jsonl.stem)
|
||||||
|
yield NormalizedSession(
|
||||||
|
source="pi",
|
||||||
|
session_id=source_id_for("pi", brain_session_id),
|
||||||
|
title="",
|
||||||
|
created=to_iso_z(next(
|
||||||
|
(r.get("timestamp") for r in records
|
||||||
|
if isinstance(r, dict) and r.get("type") == "session"),
|
||||||
|
None,
|
||||||
|
)) or created,
|
||||||
|
modified=modified,
|
||||||
|
workspace_path=cwd,
|
||||||
|
messages=msgs,
|
||||||
|
source_locator=str(jsonl),
|
||||||
|
model_type=model or "pi",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# OpenCode (~/.local/share/opencode/opencode.db — SQLite)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def discover_opencode(db_path: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||||
|
db = db_path or (HOME / ".local" / "share" / "opencode" / "opencode.db")
|
||||||
|
if not db.exists():
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||||
|
except sqlite3.Error:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
con.row_factory = sqlite3.Row
|
||||||
|
|
||||||
|
try:
|
||||||
|
sessions = con.execute(
|
||||||
|
"SELECT id, title, directory, agent, model, parent_id, "
|
||||||
|
"time_created, time_updated "
|
||||||
|
"FROM session ORDER BY time_created"
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.Error:
|
||||||
|
return
|
||||||
|
|
||||||
|
for sess in sessions:
|
||||||
|
sess_id = sess["id"] or ""
|
||||||
|
parent_id = sess["parent_id"]
|
||||||
|
title = sess["title"] or ""
|
||||||
|
directory = sess["directory"] or ""
|
||||||
|
agent = sess["agent"]
|
||||||
|
model_raw = sess["model"]
|
||||||
|
created_ms = sess["time_created"] or 0
|
||||||
|
updated_ms = sess["time_updated"] or 0
|
||||||
|
|
||||||
|
model_name = None
|
||||||
|
if model_raw:
|
||||||
|
try:
|
||||||
|
mobj = json.loads(model_raw) if isinstance(model_raw, str) else model_raw
|
||||||
|
if isinstance(mobj, dict):
|
||||||
|
provider = mobj.get("providerID", "")
|
||||||
|
model_id = mobj.get("id", "")
|
||||||
|
model_name = f"{provider}/{model_id}" if provider and model_id else (model_id or provider or None)
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
created = to_iso_z(created_ms / 1000.0) if created_ms else None
|
||||||
|
modified = to_iso_z(updated_ms / 1000.0) if updated_ms else None
|
||||||
|
if not created:
|
||||||
|
created, modified = _file_times(db)
|
||||||
|
|
||||||
|
try:
|
||||||
|
rows = con.execute(
|
||||||
|
"SELECT id, data FROM message WHERE session_id = ? ORDER BY time_created",
|
||||||
|
(sess_id,),
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.Error:
|
||||||
|
continue
|
||||||
|
|
||||||
|
msgs: list[NormalizedMessage] = []
|
||||||
|
for row in rows:
|
||||||
|
msg_data_raw = row["data"]
|
||||||
|
if not msg_data_raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
msg_data = json.loads(msg_data_raw) if isinstance(msg_data_raw, str) else msg_data_raw
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
continue
|
||||||
|
if not isinstance(msg_data, dict):
|
||||||
|
continue
|
||||||
|
|
||||||
|
role = msg_data.get("role")
|
||||||
|
if role not in ("user", "assistant", "tool", "toolResult"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
norm_role = "tool_result" if role in ("tool", "toolResult") else role
|
||||||
|
msg_model = None
|
||||||
|
if norm_role == "assistant":
|
||||||
|
m = msg_data.get("model")
|
||||||
|
if isinstance(m, dict):
|
||||||
|
msg_model = m.get("id") or m.get("providerID") or model_name
|
||||||
|
else:
|
||||||
|
msg_model = model_name
|
||||||
|
|
||||||
|
try:
|
||||||
|
part_rows = con.execute(
|
||||||
|
"SELECT data FROM part WHERE message_id = ? ORDER BY time_created",
|
||||||
|
(row["id"],),
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.Error:
|
||||||
|
part_rows = []
|
||||||
|
|
||||||
|
content_parts: list[str] = []
|
||||||
|
tool_call_parts: list[tuple[str, str, str]] = [] # (name, input_json, output)
|
||||||
|
for pr in part_rows:
|
||||||
|
pd = pr["data"]
|
||||||
|
if not pd:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
pdata = json.loads(pd) if isinstance(pd, str) else pd
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
continue
|
||||||
|
if not isinstance(pdata, dict):
|
||||||
|
continue
|
||||||
|
ptype = pdata.get("type")
|
||||||
|
if ptype == "text":
|
||||||
|
t = pdata.get("text", "")
|
||||||
|
if t:
|
||||||
|
content_parts.append(t)
|
||||||
|
elif ptype == "tool":
|
||||||
|
# OpenCode native tool format: {type:"tool", tool:"read", state:{input,output}}
|
||||||
|
name = pdata.get("tool", "?")
|
||||||
|
state = pdata.get("state", {})
|
||||||
|
inp = json.dumps(state.get("input", {}), ensure_ascii=False)[:500]
|
||||||
|
out = str(state.get("output", ""))[:2000]
|
||||||
|
tool_call_parts.append((name, inp, out))
|
||||||
|
elif ptype == "tool_use":
|
||||||
|
name = pdata.get("name", "?")
|
||||||
|
inp = json.dumps(pdata.get("input", {}), ensure_ascii=False)[:500]
|
||||||
|
tool_call_parts.append((name, inp, ""))
|
||||||
|
elif ptype == "tool_result":
|
||||||
|
tr = pdata.get("content", "")
|
||||||
|
if isinstance(tr, list):
|
||||||
|
tr = " ".join(x.get("text", "") for x in tr if isinstance(x, dict))
|
||||||
|
content_parts.append(f"[tool_result] {str(tr)[:500]}")
|
||||||
|
|
||||||
|
# Compute timestamp before tool_call emission
|
||||||
|
msg_ts = msg_data.get("time", {})
|
||||||
|
ts = None
|
||||||
|
if isinstance(msg_ts, dict):
|
||||||
|
ts = to_iso_z(msg_ts.get("created"))
|
||||||
|
if not ts:
|
||||||
|
ts = to_iso_z(row["time_created"] / 1000.0) if row["time_created"] else None
|
||||||
|
|
||||||
|
# Emit tool_call messages for OpenCode tool parts
|
||||||
|
for j, (name, inp, out) in enumerate(tool_call_parts):
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role="tool_call",
|
||||||
|
content=f"{name}({inp})",
|
||||||
|
timestamp=ts,
|
||||||
|
model=msg_model,
|
||||||
|
id=f"oc-{row['id']}-t{j}",
|
||||||
|
))
|
||||||
|
if out:
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role="tool_result",
|
||||||
|
content=out[:2000],
|
||||||
|
timestamp=ts,
|
||||||
|
id=f"oc-{row['id']}-r{j}",
|
||||||
|
))
|
||||||
|
|
||||||
|
content = "\n".join(content_parts)
|
||||||
|
if not content and norm_role in ("user", "assistant"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role=norm_role,
|
||||||
|
content=content,
|
||||||
|
timestamp=ts,
|
||||||
|
model=msg_model,
|
||||||
|
id=row["id"],
|
||||||
|
))
|
||||||
|
|
||||||
|
if not msgs:
|
||||||
|
continue
|
||||||
|
|
||||||
|
extra: dict = {}
|
||||||
|
if parent_id:
|
||||||
|
extra["parentSessionId"] = parent_id
|
||||||
|
if agent:
|
||||||
|
extra["agent"] = agent
|
||||||
|
|
||||||
|
brain_session_id = sess_id.removeprefix("ses_") if sess_id.startswith("ses_") else sess_id
|
||||||
|
yield NormalizedSession(
|
||||||
|
source="opencode",
|
||||||
|
session_id=source_id_for("oc", brain_session_id),
|
||||||
|
title=title,
|
||||||
|
created=created,
|
||||||
|
modified=modified,
|
||||||
|
workspace_path=directory or None,
|
||||||
|
messages=msgs,
|
||||||
|
source_locator=f"{db}#{sess_id}",
|
||||||
|
model_type=model_name or "opencode",
|
||||||
|
extra=extra,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Kimi (re-export of the proven layout so the unified driver covers it too)
|
# Kimi (re-export of the proven layout so the unified driver covers it too)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
@ -501,11 +930,37 @@ def discover_kimi(kimi_dir: Optional[Path] = None) -> Iterable[NormalizedSession
|
||||||
role = rec.get("role")
|
role = rec.get("role")
|
||||||
if role in (None, "_system_prompt", "_checkpoint", "_usage"):
|
if role in (None, "_system_prompt", "_checkpoint", "_usage"):
|
||||||
continue
|
continue
|
||||||
if role == "tool_call":
|
# Kimi assistant messages may carry tool_calls — emit as tool_call
|
||||||
msgs.append(NormalizedMessage(role="tool_call", content=flatten_content(rec.get("content"))[:500], id=f"kc-{i:06d}"))
|
if role == "assistant":
|
||||||
continue
|
tc = rec.get("tool_calls")
|
||||||
if role == "tool_result":
|
if isinstance(tc, list) and tc:
|
||||||
msgs.append(NormalizedMessage(role="tool_result", content=flatten_content(rec.get("content"))[:500], id=f"kr-{i:06d}"))
|
for j, t in enumerate(tc):
|
||||||
|
fn = t.get("function", {}) if isinstance(t, dict) else {}
|
||||||
|
name = fn.get("name", "?")
|
||||||
|
args = fn.get("arguments", "")
|
||||||
|
if isinstance(args, str) and len(args) > 500:
|
||||||
|
args = args[:500]
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role="tool_call",
|
||||||
|
content=f"{name}({args})",
|
||||||
|
id=f"kc-{i:06d}-{j}",
|
||||||
|
))
|
||||||
|
# Also emit any text content as assistant
|
||||||
|
content = flatten_content(rec.get("content"))
|
||||||
|
if content:
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role="assistant",
|
||||||
|
content=content,
|
||||||
|
id=f"k-{i:06d}",
|
||||||
|
))
|
||||||
|
continue
|
||||||
|
# Kimi tool results use role=tool (not tool_result)
|
||||||
|
if role in ("tool", "tool_result"):
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role="tool_result",
|
||||||
|
content=flatten_content(rec.get("content"))[:2000],
|
||||||
|
id=f"kr-{i:06d}",
|
||||||
|
))
|
||||||
continue
|
continue
|
||||||
if role not in ("user", "model"):
|
if role not in ("user", "model"):
|
||||||
continue
|
continue
|
||||||
|
|
@ -530,6 +985,377 @@ def discover_kimi(kimi_dir: Optional[Path] = None) -> Iterable[NormalizedSession
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Cursor (~/.config/Cursor/User/workspaceStorage/*/state.vscdb + globalStorage)
|
||||||
|
# Chat sessions are in the globalStorage state.vscdb under
|
||||||
|
# 'chat.ChatSessionStore.index' and per-workspace state.vscdb keys.
|
||||||
|
# Composer/agent sessions are in globalStorage/cursor.chatSessions or similar.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def discover_cursor(cursor_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||||
|
root = cursor_dir or (HOME / ".config" / "Cursor")
|
||||||
|
if not root.exists():
|
||||||
|
return
|
||||||
|
|
||||||
|
# Cursor stores chat data in workspace-level state.vscdb SQLite DBs.
|
||||||
|
# The composer (agent) history is in globalStorage.
|
||||||
|
for db_path in sorted(root.rglob("state.vscdb")):
|
||||||
|
try:
|
||||||
|
con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
||||||
|
except sqlite3.Error:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
con.row_factory = sqlite3.Row
|
||||||
|
try:
|
||||||
|
rows = con.execute(
|
||||||
|
"SELECT key, value FROM ItemTable WHERE "
|
||||||
|
"key LIKE '%chatSession%' OR key LIKE '%composer%' "
|
||||||
|
"OR key LIKE '%chat%session%' OR key LIKE '%aichat%'"
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.Error:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for row in rows:
|
||||||
|
key = row["key"] or ""
|
||||||
|
raw = row["value"]
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = json.loads(raw) if isinstance(raw, str) else raw
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
continue
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Extract messages from common Cursor chat formats
|
||||||
|
messages_raw = (
|
||||||
|
data.get("messages") or data.get("entries")
|
||||||
|
or data.get("history") or data.get("turns")
|
||||||
|
)
|
||||||
|
if not isinstance(messages_raw, list) or not messages_raw:
|
||||||
|
continue
|
||||||
|
|
||||||
|
session_id = (
|
||||||
|
data.get("sessionId") or data.get("id")
|
||||||
|
or (hashlib.sha1(raw.encode()).hexdigest()[:16] if isinstance(raw, str) else db_path.stem)
|
||||||
|
)
|
||||||
|
workspace = str(db_path.parent.parent) if "workspaceStorage" in str(db_path) else None
|
||||||
|
|
||||||
|
msgs: list[NormalizedMessage] = []
|
||||||
|
for i, m in enumerate(messages_raw):
|
||||||
|
if not isinstance(m, dict):
|
||||||
|
continue
|
||||||
|
role = m.get("role") or m.get("type", "")
|
||||||
|
content = flatten_content(m.get("content") or m.get("text") or m.get("message"))
|
||||||
|
if not content:
|
||||||
|
continue
|
||||||
|
norm_role = "user" if "user" in role.lower() else (
|
||||||
|
"tool_call" if "tool" in role.lower() and "result" not in role.lower() else (
|
||||||
|
"tool_result" if "result" in role.lower() else "assistant"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role=norm_role,
|
||||||
|
content=content,
|
||||||
|
timestamp=to_iso_z(m.get("timestamp") or m.get("createdAt")),
|
||||||
|
model=m.get("model") or data.get("model"),
|
||||||
|
id=m.get("id") or f"cur-{i:06d}",
|
||||||
|
))
|
||||||
|
|
||||||
|
if not msgs:
|
||||||
|
continue
|
||||||
|
created, modified = _file_times(db_path)
|
||||||
|
yield NormalizedSession(
|
||||||
|
source="cursor",
|
||||||
|
session_id=source_id_for("cursor", str(session_id)),
|
||||||
|
title=data.get("title") or "",
|
||||||
|
created=created,
|
||||||
|
modified=modified,
|
||||||
|
workspace_path=workspace,
|
||||||
|
messages=msgs,
|
||||||
|
source_locator=f"{db_path}#{key}",
|
||||||
|
model_type=data.get("model") or "cursor",
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Zed (~/.local/share/zed/threads/*.json + ~/.config/zed/)
|
||||||
|
# Thread files are JSON with entries[].role + entries[].content.
|
||||||
|
# Agent conversations in ~/.local/share/zed/assistant2/ or threads/.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def discover_zed(zed_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||||
|
roots = [zed_dir] if zed_dir else [
|
||||||
|
HOME / ".local" / "share" / "zed",
|
||||||
|
HOME / ".config" / "zed",
|
||||||
|
]
|
||||||
|
for root in roots:
|
||||||
|
if not root or not root.exists():
|
||||||
|
continue
|
||||||
|
for sub in ("threads", "assistant2", "assistant", "conversations"):
|
||||||
|
thread_dir = root / sub
|
||||||
|
if not thread_dir.exists():
|
||||||
|
continue
|
||||||
|
for f in sorted(thread_dir.rglob("*.json")):
|
||||||
|
data = _load_json(f)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
continue
|
||||||
|
entries = data.get("entries") or data.get("messages") or data.get("turns")
|
||||||
|
if not isinstance(entries, list) or not entries:
|
||||||
|
continue
|
||||||
|
|
||||||
|
session_id = data.get("id") or data.get("threadId") or f.stem
|
||||||
|
msgs: list[NormalizedMessage] = []
|
||||||
|
for i, entry in enumerate(entries):
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
role = entry.get("role", "")
|
||||||
|
content = flatten_content(entry.get("content") or entry.get("text") or entry.get("message"))
|
||||||
|
if not content:
|
||||||
|
continue
|
||||||
|
norm_role = "user" if role == "user" else (
|
||||||
|
"tool_call" if "tool" in role.lower() else "assistant"
|
||||||
|
)
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role=norm_role,
|
||||||
|
content=content,
|
||||||
|
timestamp=to_iso_z(entry.get("timestamp")),
|
||||||
|
model=entry.get("model"),
|
||||||
|
id=f"zed-{i:06d}",
|
||||||
|
))
|
||||||
|
|
||||||
|
if not msgs:
|
||||||
|
continue
|
||||||
|
created, modified = _file_times(f)
|
||||||
|
yield NormalizedSession(
|
||||||
|
source="zed",
|
||||||
|
session_id=source_id_for("zed", str(session_id)),
|
||||||
|
title=data.get("title") or data.get("summary") or "",
|
||||||
|
created=created,
|
||||||
|
modified=modified,
|
||||||
|
workspace_path=data.get("workspace"),
|
||||||
|
messages=msgs,
|
||||||
|
source_locator=str(f),
|
||||||
|
model_type=data.get("model") or "zed",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Continue.dev (~/.continue/sessions/*.json)
|
||||||
|
# Each session is a JSON file with history[] containing {role, content}.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def discover_continue(continue_dir: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||||
|
root = continue_dir or (HOME / ".continue")
|
||||||
|
sessions_dir = root / "sessions"
|
||||||
|
if not sessions_dir.exists():
|
||||||
|
return
|
||||||
|
|
||||||
|
for f in sorted(sessions_dir.rglob("*.json")):
|
||||||
|
data = _load_json(f)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
continue
|
||||||
|
history = data.get("history") or data.get("messages")
|
||||||
|
if not isinstance(history, list) or not history:
|
||||||
|
continue
|
||||||
|
|
||||||
|
session_id = data.get("sessionId") or data.get("id") or f.stem
|
||||||
|
msgs: list[NormalizedMessage] = []
|
||||||
|
for i, entry in enumerate(history):
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
role = entry.get("role", "")
|
||||||
|
content = flatten_content(entry.get("content") or entry.get("message"))
|
||||||
|
if not content:
|
||||||
|
continue
|
||||||
|
norm_role = "user" if role == "user" else (
|
||||||
|
"tool_call" if "tool" in role.lower() else "assistant"
|
||||||
|
)
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role=norm_role,
|
||||||
|
content=content,
|
||||||
|
timestamp=to_iso_z(entry.get("timestamp")),
|
||||||
|
model=entry.get("model") or data.get("model"),
|
||||||
|
id=f"cont-{i:06d}",
|
||||||
|
))
|
||||||
|
|
||||||
|
if not msgs:
|
||||||
|
continue
|
||||||
|
created, modified = _file_times(f)
|
||||||
|
yield NormalizedSession(
|
||||||
|
source="continue",
|
||||||
|
session_id=source_id_for("continue", str(session_id)),
|
||||||
|
title=data.get("title") or "",
|
||||||
|
created=created,
|
||||||
|
modified=modified,
|
||||||
|
workspace_path=data.get("workspaceDirectory"),
|
||||||
|
messages=msgs,
|
||||||
|
source_locator=str(f),
|
||||||
|
model_type=data.get("model") or "continue",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Aider (.aider.chat.history.md in each repo root)
|
||||||
|
# Markdown chat log with # aider chat started at ... headers.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def discover_aider(home: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||||
|
# Aider writes .aider.chat.history.md in the repo root where it runs.
|
||||||
|
# We scan the home directory for repos with aider history.
|
||||||
|
# Skip hidden dirs and common non-repo dirs.
|
||||||
|
_SKIP = {".cache", ".config", ".local", ".npm", ".vscode", ".git",
|
||||||
|
"node_modules", "__pycache__", ".mozilla", ".thunderbird"}
|
||||||
|
home_dir = home or HOME
|
||||||
|
|
||||||
|
for history_file in sorted(home_dir.rglob(".aider.chat.history.md")):
|
||||||
|
# Skip if inside a hidden dir (other than the file itself)
|
||||||
|
parts = history_file.relative_to(home_dir).parts
|
||||||
|
if any(p.startswith(".") and p != ".aider.chat.history.md" for p in parts[:-1]):
|
||||||
|
if not any(p in _SKIP for p in parts[:-1]):
|
||||||
|
# Allow .aurelio, .aider, etc.
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
text = history_file.read_text(encoding="utf-8")
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
if not text.strip():
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Parse aider markdown: user messages start with ####, assistant with > or plain
|
||||||
|
msgs: list[NormalizedMessage] = []
|
||||||
|
current_role: Optional[str] = None
|
||||||
|
current_lines: list[str] = []
|
||||||
|
|
||||||
|
def flush():
|
||||||
|
nonlocal current_role, current_lines
|
||||||
|
if current_role and current_lines:
|
||||||
|
content = "\n".join(current_lines).strip()
|
||||||
|
if content:
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role=current_role,
|
||||||
|
content=content,
|
||||||
|
id=f"aider-{len(msgs):06d}",
|
||||||
|
))
|
||||||
|
current_role = None
|
||||||
|
current_lines = []
|
||||||
|
|
||||||
|
for line in text.splitlines():
|
||||||
|
if line.startswith("# aider chat started at"):
|
||||||
|
flush()
|
||||||
|
continue
|
||||||
|
if line.startswith("#### "):
|
||||||
|
flush()
|
||||||
|
current_role = "user"
|
||||||
|
current_lines = [line[5:]]
|
||||||
|
elif line.startswith("> "):
|
||||||
|
if current_role != "assistant":
|
||||||
|
flush()
|
||||||
|
current_role = "assistant"
|
||||||
|
current_lines.append(line[2:])
|
||||||
|
elif line.strip() and current_role:
|
||||||
|
current_lines.append(line)
|
||||||
|
|
||||||
|
flush()
|
||||||
|
if not msgs:
|
||||||
|
continue
|
||||||
|
|
||||||
|
workspace = str(history_file.parent)
|
||||||
|
created, modified = _file_times(history_file)
|
||||||
|
session_id = hashlib.sha1(workspace.encode()).hexdigest()[:16]
|
||||||
|
yield NormalizedSession(
|
||||||
|
source="aider",
|
||||||
|
session_id=source_id_for("aider", session_id),
|
||||||
|
title=f"Aider in {Path(workspace).name}",
|
||||||
|
created=created,
|
||||||
|
modified=modified,
|
||||||
|
workspace_path=workspace,
|
||||||
|
messages=msgs,
|
||||||
|
source_locator=str(history_file),
|
||||||
|
model_type="aider",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Cline / Roo Code (VS Code globalStorage/<publisher>.<name>/tasks/)
|
||||||
|
# Task files are JSON with messages[] containing {role, content, toolUses}.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_CLINE_PUBLISHERS = [
|
||||||
|
"saoudrizwan.claude-dev",
|
||||||
|
"rooveterinaryinc.roo-cline",
|
||||||
|
]
|
||||||
|
|
||||||
|
def discover_cline(vscode_storage: Optional[Path] = None) -> Iterable[NormalizedSession]:
|
||||||
|
roots = [vscode_storage] if vscode_storage else [
|
||||||
|
HOME / ".config" / "Code" / "User" / "globalStorage",
|
||||||
|
HOME / ".config" / "Code - Insiders" / "User" / "globalStorage",
|
||||||
|
HOME / ".vscode" / "extensions",
|
||||||
|
]
|
||||||
|
for root in roots:
|
||||||
|
if not root or not root.exists():
|
||||||
|
continue
|
||||||
|
for publisher in _CLINE_PUBLISHERS:
|
||||||
|
ext_dir = root / publisher
|
||||||
|
if not ext_dir.exists():
|
||||||
|
continue
|
||||||
|
# Cline stores tasks in tasks/ subdirectory
|
||||||
|
for task_dir_name in ("tasks", "taskHistory", "history"):
|
||||||
|
task_dir = ext_dir / task_dir_name
|
||||||
|
if not task_dir.exists():
|
||||||
|
continue
|
||||||
|
for f in sorted(task_dir.rglob("*.json")):
|
||||||
|
data = _load_json(f)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
continue
|
||||||
|
messages_raw = data.get("messages") or data.get("apiConversationHistory")
|
||||||
|
if not isinstance(messages_raw, list) or not messages_raw:
|
||||||
|
continue
|
||||||
|
|
||||||
|
session_id = data.get("taskId") or data.get("id") or f.stem
|
||||||
|
source = "cline" if "claude-dev" in publisher else "roo-code"
|
||||||
|
msgs: list[NormalizedMessage] = []
|
||||||
|
for i, m in enumerate(messages_raw):
|
||||||
|
if not isinstance(m, dict):
|
||||||
|
continue
|
||||||
|
role = m.get("role", "")
|
||||||
|
content = flatten_content(m.get("content"))
|
||||||
|
if not content:
|
||||||
|
continue
|
||||||
|
norm_role = "user" if role == "user" else (
|
||||||
|
"tool_call" if "tool" in role.lower() and "result" not in role.lower() else (
|
||||||
|
"tool_result" if "result" in role.lower() else "assistant"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
msgs.append(NormalizedMessage(
|
||||||
|
role=norm_role,
|
||||||
|
content=content,
|
||||||
|
timestamp=to_iso_z(m.get("timestamp")),
|
||||||
|
model=m.get("model"),
|
||||||
|
id=f"{source[:2]}-{i:06d}",
|
||||||
|
))
|
||||||
|
|
||||||
|
if not msgs:
|
||||||
|
continue
|
||||||
|
created, modified = _file_times(f)
|
||||||
|
yield NormalizedSession(
|
||||||
|
source=source,
|
||||||
|
session_id=source_id_for(source, str(session_id)),
|
||||||
|
title=data.get("title") or data.get("task", "")[:80],
|
||||||
|
created=created,
|
||||||
|
modified=modified,
|
||||||
|
workspace_path=data.get("workspace"),
|
||||||
|
messages=msgs,
|
||||||
|
source_locator=str(f),
|
||||||
|
model_type=data.get("model") or source,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
ADAPTERS = {
|
ADAPTERS = {
|
||||||
"kimi": discover_kimi,
|
"kimi": discover_kimi,
|
||||||
"hermes": discover_hermes,
|
"hermes": discover_hermes,
|
||||||
|
|
@ -537,4 +1363,11 @@ ADAPTERS = {
|
||||||
"antigravity": discover_antigravity_cli,
|
"antigravity": discover_antigravity_cli,
|
||||||
"qwen-code": discover_qwen_code,
|
"qwen-code": discover_qwen_code,
|
||||||
"mimocode": discover_mimocode,
|
"mimocode": discover_mimocode,
|
||||||
|
"pi": discover_pi,
|
||||||
|
"opencode": discover_opencode,
|
||||||
|
"cursor": discover_cursor,
|
||||||
|
"zed": discover_zed,
|
||||||
|
"continue": discover_continue,
|
||||||
|
"aider": discover_aider,
|
||||||
|
"cline": discover_cline,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -306,7 +306,7 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A
|
||||||
**({"extra": sess.extra} if sess.extra else {}),
|
**({"extra": sess.extra} if sess.extra else {}),
|
||||||
}
|
}
|
||||||
|
|
||||||
session_jsonl = [
|
session_jsonl_lines = [
|
||||||
json.dumps({
|
json.dumps({
|
||||||
"type": "session_start",
|
"type": "session_start",
|
||||||
"timestamp": sess.created,
|
"timestamp": sess.created,
|
||||||
|
|
@ -317,12 +317,58 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A
|
||||||
"sourceSessionId": sess.session_id,
|
"sourceSessionId": sess.session_id,
|
||||||
},
|
},
|
||||||
}, ensure_ascii=False),
|
}, ensure_ascii=False),
|
||||||
|
]
|
||||||
|
# Full trajectory: one event per message, preserving tool_call/tool_result
|
||||||
|
# sequence for RL training. This is the primary training data artifact.
|
||||||
|
for idx, (m, ts) in enumerate(zip(sess.messages, timestamps)):
|
||||||
|
event: dict[str, Any] = {
|
||||||
|
"type": "message",
|
||||||
|
"timestamp": ts,
|
||||||
|
"data": {
|
||||||
|
"id": m.id or f"m{idx:06d}",
|
||||||
|
"role": m.role,
|
||||||
|
"content": m.content,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if m.model:
|
||||||
|
event["data"]["model"] = m.model
|
||||||
|
session_jsonl_lines.append(json.dumps(event, ensure_ascii=False))
|
||||||
|
session_jsonl_lines.append(
|
||||||
json.dumps({
|
json.dumps({
|
||||||
"type": "session_end",
|
"type": "session_end",
|
||||||
"timestamp": sess.modified,
|
"timestamp": sess.modified,
|
||||||
"data": {"sessionId": sess.brain_id, "totalEvents": len(chat_messages)},
|
"data": {"sessionId": sess.brain_id, "totalEvents": len(sess.messages)},
|
||||||
}, ensure_ascii=False),
|
}, ensure_ascii=False),
|
||||||
]
|
)
|
||||||
|
|
||||||
|
# RL trajectory: compact (state, action, reward) triples derived from the
|
||||||
|
# message sequence. Each tool_call is an action; the preceding user/assistant
|
||||||
|
# message is the state; the tool_result is the immediate observation.
|
||||||
|
trajectory_lines: list[str] = []
|
||||||
|
for idx, m in enumerate(sess.messages):
|
||||||
|
if m.role != "tool_call":
|
||||||
|
continue
|
||||||
|
# Find the nearest preceding user/assistant message as state
|
||||||
|
state = ""
|
||||||
|
for j in range(idx - 1, -1, -1):
|
||||||
|
if sess.messages[j].role in ("user", "assistant"):
|
||||||
|
state = sess.messages[j].content[:2000]
|
||||||
|
break
|
||||||
|
# Find the tool_result that follows
|
||||||
|
observation = ""
|
||||||
|
if idx + 1 < len(sess.messages) and sess.messages[idx + 1].role == "tool_result":
|
||||||
|
observation = sess.messages[idx + 1].content[:2000]
|
||||||
|
traj_event = {
|
||||||
|
"type": "trajectory_step",
|
||||||
|
"session_id": sess.brain_id,
|
||||||
|
"step": idx,
|
||||||
|
"timestamp": timestamps[idx] if idx < len(timestamps) else None,
|
||||||
|
"state": state,
|
||||||
|
"action": m.content[:2000],
|
||||||
|
"observation": observation,
|
||||||
|
"model": m.model or sess.model_type,
|
||||||
|
}
|
||||||
|
trajectory_lines.append(json.dumps(traj_event, ensure_ascii=False))
|
||||||
|
|
||||||
overview_txt = (
|
overview_txt = (
|
||||||
f"# Conversation: {title}\n"
|
f"# Conversation: {title}\n"
|
||||||
|
|
@ -334,11 +380,13 @@ def write_session(sess: NormalizedSession, dry_run: bool = False) -> dict[str, A
|
||||||
)
|
)
|
||||||
|
|
||||||
files = {
|
files = {
|
||||||
target_dir / "session.jsonl": "\n".join(session_jsonl) + "\n",
|
target_dir / "session.jsonl": "\n".join(session_jsonl_lines) + "\n",
|
||||||
target_dir / "session_memory.md": render_session_memory(sess, import_ts),
|
target_dir / "session_memory.md": render_session_memory(sess, import_ts),
|
||||||
target_dir / ".system_generated" / "chat_history.json": json.dumps(chat_history, indent=2, ensure_ascii=False) + "\n",
|
target_dir / ".system_generated" / "chat_history.json": json.dumps(chat_history, indent=2, ensure_ascii=False) + "\n",
|
||||||
target_dir / ".system_generated" / "logs" / "overview.txt": overview_txt,
|
target_dir / ".system_generated" / "logs" / "overview.txt": overview_txt,
|
||||||
}
|
}
|
||||||
|
if trajectory_lines:
|
||||||
|
files[target_dir / ".system_generated" / "trajectory.jsonl"] = "\n".join(trajectory_lines) + "\n"
|
||||||
|
|
||||||
if dry_run:
|
if dry_run:
|
||||||
result["would_write"] = [str(p.relative_to(REPLICA_ROOT)) for p in files]
|
result["would_write"] = [str(p.relative_to(REPLICA_ROOT)) for p in files]
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,8 @@ bridge (`scripts/sync-kimi-to-brain.py`) but for all sources:
|
||||||
antigravity ~/.gemini/antigravity-{cli,ide}/conversations/*.db (sqlite)
|
antigravity ~/.gemini/antigravity-{cli,ide}/conversations/*.db (sqlite)
|
||||||
qwen-code ~/.qwen[-code]/**/{context.jsonl,*.jsonl} (no-op if absent)
|
qwen-code ~/.qwen[-code]/**/{context.jsonl,*.jsonl} (no-op if absent)
|
||||||
mimocode ./.mimocode/plans/*.md (+ future *.jsonl)
|
mimocode ./.mimocode/plans/*.md (+ future *.jsonl)
|
||||||
|
pi ~/.pi/agent/sessions/**/*.jsonl
|
||||||
|
opencode ~/.local/share/opencode/opencode.db (sqlite)
|
||||||
|
|
||||||
Output contract per session (see agent_importers/engine.py):
|
Output contract per session (see agent_importers/engine.py):
|
||||||
.aurelio/brain/session-<source>-<id>/{session.jsonl, session_memory.md,
|
.aurelio/brain/session-<source>-<id>/{session.jsonl, session_memory.md,
|
||||||
|
|
@ -52,6 +54,8 @@ from agent_importers.adapters import ( # noqa: E402
|
||||||
discover_antigravity_cli,
|
discover_antigravity_cli,
|
||||||
discover_qwen_code,
|
discover_qwen_code,
|
||||||
discover_mimocode,
|
discover_mimocode,
|
||||||
|
discover_pi,
|
||||||
|
discover_opencode,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -89,6 +93,8 @@ def run(args: argparse.Namespace) -> dict[str, Any]:
|
||||||
"antigravity": lambda: discover_antigravity_cli(Path(args.gemini_dir) if args.gemini_dir else None),
|
"antigravity": lambda: discover_antigravity_cli(Path(args.gemini_dir) if args.gemini_dir else None),
|
||||||
"qwen-code": lambda: discover_qwen_code(Path(args.qwen_dir) if args.qwen_dir else None),
|
"qwen-code": lambda: discover_qwen_code(Path(args.qwen_dir) if args.qwen_dir else None),
|
||||||
"mimocode": lambda: discover_mimocode(repo_root=replica_root),
|
"mimocode": lambda: discover_mimocode(repo_root=replica_root),
|
||||||
|
"pi": lambda: discover_pi(Path(args.pi_dir) if args.pi_dir else None),
|
||||||
|
"opencode": lambda: discover_opencode(Path(args.opencode_db) if args.opencode_db else None),
|
||||||
}
|
}
|
||||||
|
|
||||||
for name in requested:
|
for name in requested:
|
||||||
|
|
@ -139,6 +145,8 @@ def main() -> int:
|
||||||
ap.add_argument("--claude-dir", default=None)
|
ap.add_argument("--claude-dir", default=None)
|
||||||
ap.add_argument("--gemini-dir", default=None)
|
ap.add_argument("--gemini-dir", default=None)
|
||||||
ap.add_argument("--qwen-dir", default=None)
|
ap.add_argument("--qwen-dir", default=None)
|
||||||
|
ap.add_argument("--pi-dir", default=None)
|
||||||
|
ap.add_argument("--opencode-db", default=None)
|
||||||
ap.add_argument("--summary", action="store_true", help="Print JSON summary.")
|
ap.add_argument("--summary", action="store_true", help="Print JSON summary.")
|
||||||
ap.add_argument("--verbose", "-v", action="store_true")
|
ap.add_argument("--verbose", "-v", action="store_true")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue