564 lines
20 KiB
Python
Executable file
564 lines
20 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""
|
|
Kimi → Réplica Omnisciente Brain Sync Bridge
|
|
|
|
Imports Kimi Code CLI session history from ~/.kimi/sessions/ and plans from
|
|
~/.kimi/plans/ into the central Réplica brain at
|
|
<replica>/.aurelio/brain/session-kimi-<uuid>/.
|
|
|
|
This makes Kimi conversations discoverable alongside Antigravity and Aurélio
|
|
sessions in the unified brain.
|
|
|
|
Usage:
|
|
python3 scripts/sync-kimi-to-brain.py [--dry-run] [--skip-active]
|
|
|
|
The script is idempotent: it only writes files when the content would change.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
HOME = Path.home()
|
|
KIMI_DIR = HOME / ".kimi"
|
|
SESSIONS_DIR = KIMI_DIR / "sessions"
|
|
PLANS_DIR = KIMI_DIR / "plans"
|
|
KIMI_REGISTRY = KIMI_DIR / "kimi.json"
|
|
|
|
# Resolve replica root relative to this script: replica/scripts/sync-kimi-to-brain.py
|
|
REPLICA_ROOT = Path(__file__).resolve().parent.parent
|
|
BRAIN_DIR = REPLICA_ROOT / ".aurelio" / "brain"
|
|
|
|
# Roles in Kimi context.jsonl that carry meaningful conversational content
|
|
CHAT_ROLES = {"user", "model"}
|
|
SKIP_ROLES = {"_system_prompt", "_checkpoint", "_usage"}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def md5(text: str) -> str:
|
|
return hashlib.md5(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def load_json(path: Path) -> Any:
|
|
if not path.exists():
|
|
return None
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
except (json.JSONDecodeError, OSError):
|
|
return None
|
|
|
|
|
|
def load_jsonl(path: Path) -> list[dict]:
|
|
"""Stream a JSONL file, skipping malformed lines."""
|
|
records: list[dict] = []
|
|
if not path.exists():
|
|
return records
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
records.append(json.loads(line))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
except OSError:
|
|
pass
|
|
return records
|
|
|
|
|
|
def write_if_changed(path: Path, content: str | bytes) -> bool:
|
|
"""Write a file only if content differs. Returns True if written."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
binary = isinstance(content, bytes)
|
|
current: bytes | None = None
|
|
if path.exists():
|
|
with open(path, "rb") as f:
|
|
current = f.read()
|
|
new = content if binary else content.encode("utf-8")
|
|
if current == new:
|
|
return False
|
|
with open(path, "wb") as f:
|
|
f.write(new)
|
|
return True
|
|
|
|
|
|
def iso_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def parse_timestamp(ts: str | None) -> str:
|
|
if not ts:
|
|
return iso_now()
|
|
# Kimi does not currently emit per-message timestamps in context.jsonl,
|
|
# so we return now() as a fallback. If state.json had a created_at in
|
|
# the future, we would prefer it.
|
|
return ts
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Kimi registry mapping: path -> workspace hash
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def build_workspace_map(registry_path: Path) -> dict[str, str]:
|
|
"""Return {workspace_hash: absolute_path} for all registered Kimi workspaces."""
|
|
mapping: dict[str, str] = {}
|
|
data = load_json(registry_path)
|
|
if not isinstance(data, dict):
|
|
return mapping
|
|
for entry in data.get("work_dirs", []):
|
|
path = entry.get("path")
|
|
if isinstance(path, str) and path:
|
|
mapping[md5(path)] = path
|
|
return mapping
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Conversation parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_chat_messages(context: list[dict], base_ts: str) -> list[dict]:
|
|
"""Convert Kimi context.jsonl records to Aurelio chat_history.json messages.
|
|
|
|
Each message gets a deterministic timestamp offset from base_ts by its index
|
|
in the context stream, so repeated imports are idempotent.
|
|
"""
|
|
messages: list[dict] = []
|
|
base_dt = datetime.fromisoformat(base_ts.replace("Z", "+00:00"))
|
|
|
|
for idx, record in enumerate(context):
|
|
role = record.get("role")
|
|
if role in SKIP_ROLES:
|
|
continue
|
|
|
|
ts = (base_dt.replace(tzinfo=timezone.utc) + timedelta(seconds=idx)).isoformat().replace("+00:00", "Z")
|
|
|
|
if role == "tool_call":
|
|
# Summarise tool call to keep history readable
|
|
content = record.get("content", "")
|
|
if isinstance(content, list):
|
|
content = json.dumps(content, ensure_ascii=False)
|
|
messages.append({
|
|
"id": f"tc-{idx:06d}",
|
|
"role": "tool_call",
|
|
"content": f"[tool_call] {content[:500]}",
|
|
"timestamp": ts,
|
|
})
|
|
continue
|
|
if role == "tool_result":
|
|
content = record.get("content", "")
|
|
if isinstance(content, list):
|
|
content = json.dumps(content, ensure_ascii=False)
|
|
messages.append({
|
|
"id": f"tr-{idx:06d}",
|
|
"role": "tool_result",
|
|
"content": f"[tool_result] {content[:500]}",
|
|
"timestamp": ts,
|
|
})
|
|
continue
|
|
if role not in CHAT_ROLES:
|
|
continue
|
|
|
|
content = record.get("content", "")
|
|
if isinstance(content, list):
|
|
# Multi-modal content; flatten text parts
|
|
text_parts = []
|
|
for part in content:
|
|
if isinstance(part, dict):
|
|
if part.get("type") == "text":
|
|
text_parts.append(part.get("text", ""))
|
|
else:
|
|
text_parts.append(f"[{part.get('type', 'media')}]")
|
|
else:
|
|
text_parts.append(str(part))
|
|
content = "\n".join(text_parts)
|
|
|
|
mapped_role = "assistant" if role == "model" else "user"
|
|
messages.append({
|
|
"id": f"m{idx:06d}",
|
|
"role": mapped_role,
|
|
"content": content,
|
|
"timestamp": ts,
|
|
"modelType": "kimi",
|
|
})
|
|
return messages
|
|
|
|
|
|
def derive_title(messages: list[dict], workspace_path: str | None) -> str:
|
|
"""Derive a human-readable title from the first user message."""
|
|
first_user = next((m for m in messages if m.get("role") == "user"), None)
|
|
if first_user:
|
|
text = first_user.get("content", "").strip()
|
|
# Remove system/compaction tags
|
|
text = re.sub(r"<system>.*?</system>", "", text, flags=re.S).strip()
|
|
text = re.sub(r"<current_focus>.*?</current_focus>", "", text, flags=re.S).strip()
|
|
text = re.sub(r"<[^>]+>", "", text).strip()
|
|
if text:
|
|
# Take first sentence or first 80 chars
|
|
title = text.split("\n")[0][:80]
|
|
if len(title) == 80:
|
|
title += "..."
|
|
return title
|
|
if workspace_path:
|
|
return f"Kimi session in {Path(workspace_path).name}"
|
|
return "Kimi session"
|
|
|
|
|
|
def generate_memory_md(session_id: str, workspace_path: str | None, messages: list[dict], import_ts: str | None = None, source_path: str | None = None) -> str:
|
|
"""Generate an Aurelio-compatible session_memory.md from Kimi messages."""
|
|
if import_ts is None:
|
|
import_ts = iso_now()
|
|
title = derive_title(messages, workspace_path)
|
|
user_msgs = [m for m in messages if m.get("role") == "user"]
|
|
assistant_msgs = [m for m in messages if m.get("role") == "assistant"]
|
|
last_user = user_msgs[-1]["content"][:300].replace("\n", " ") if user_msgs else "_(none)_"
|
|
last_assistant = assistant_msgs[-1]["content"][:300].replace("\n", " ") if assistant_msgs else "_(none)_"
|
|
|
|
# Extract code/file references heuristically
|
|
code_refs: set[str] = set()
|
|
file_refs: set[str] = set()
|
|
for m in messages:
|
|
content = m.get("content", "")
|
|
code_refs.update(re.findall(r"```(\w+)", content))
|
|
file_refs.update(re.findall(r"`([^`]+\.(?:py|js|ts|tsx|jsx|c|cpp|h|hpp|md|yaml|yml|json|toml|rs|go|java|kt|swift|dart))`", content, re.I))
|
|
file_refs.update(re.findall(r"(?:file|path)[:\s]+([~./]?[\w\-/.]+\.[\w]+)", content, re.I))
|
|
|
|
if source_path:
|
|
source_line = f"- Original chat: `~/.kimi/sessions/{source_path}`"
|
|
else:
|
|
source_line = f"- Original chat: `~/.kimi/sessions/<workspace_hash>/{session_id}/context.jsonl`"
|
|
|
|
lines = [
|
|
"# Session Memory",
|
|
"",
|
|
"_Imported from Kimi Code CLI session history._",
|
|
"",
|
|
"---",
|
|
"",
|
|
"## Session Goal",
|
|
f"{title}",
|
|
"",
|
|
"## Workspace",
|
|
f"- **Path**: `{workspace_path or 'unknown'}`",
|
|
f"- **Kimi session ID**: `{session_id}`",
|
|
"",
|
|
"## Key Facts",
|
|
f"- **Total messages**: {len(messages)}",
|
|
f"- **User messages**: {len(user_msgs)}",
|
|
f"- **Assistant/tool messages**: {len(assistant_msgs)}",
|
|
"",
|
|
"## Source",
|
|
source_line,
|
|
]
|
|
|
|
if code_refs:
|
|
lines += ["", "## Languages / Technologies Mentioned", ""]
|
|
lines += [f"- `{lang}`" for lang in sorted(code_refs)]
|
|
|
|
if file_refs:
|
|
lines += ["", "## Files Referenced", ""]
|
|
lines += [f"- `{f}`" for f in sorted(set(file_refs))[:30]]
|
|
|
|
lines += [
|
|
"",
|
|
"## Last Exchange",
|
|
"",
|
|
f"**User**: {last_user}",
|
|
"",
|
|
f"**Assistant**: {last_assistant}",
|
|
"",
|
|
"## Source",
|
|
source_line,
|
|
f"- Imported at: {import_ts}",
|
|
]
|
|
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-session sync
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def discover_sessions():
|
|
"""Yield (context_path, session_id, workspace_path) for every Kimi session.
|
|
|
|
Kimi stores sessions as:
|
|
sessions/<workspace_hash>/<session_uuid>/context.jsonl
|
|
Some stores (e.g. the professional/Antigravity isolated store) also keep
|
|
subagent sessions at:
|
|
sessions/<workspace_hash>/<session_uuid>/subagents/<agent_hash>/context.jsonl
|
|
|
|
Main sessions use the UUID as the brain session id. Subagent sessions use
|
|
``<session_uuid>-<agent_hash>`` so they remain unique and traceable.
|
|
"""
|
|
if not SESSIONS_DIR.exists():
|
|
return
|
|
|
|
workspace_map = build_workspace_map(KIMI_REGISTRY)
|
|
|
|
for context_path in sorted(SESSIONS_DIR.rglob("context.jsonl")):
|
|
rel = context_path.relative_to(SESSIONS_DIR)
|
|
parts = rel.parts
|
|
if len(parts) < 3:
|
|
continue
|
|
|
|
workspace_hash = parts[0]
|
|
session_uuid = parts[1]
|
|
workspace_path = workspace_map.get(workspace_hash)
|
|
|
|
# Nested subagent session
|
|
if len(parts) >= 5 and parts[-2] != session_uuid:
|
|
agent_hash = parts[-2]
|
|
session_id = f"{session_uuid}-{agent_hash}"
|
|
else:
|
|
session_id = session_uuid
|
|
|
|
yield context_path, session_id, workspace_path
|
|
|
|
|
|
def sync_session(
|
|
context_path: Path,
|
|
session_id: str,
|
|
workspace_path: str | None,
|
|
dry_run: bool = False,
|
|
skip_active: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Sync a single Kimi session into the Réplica brain."""
|
|
result = {"session_id": session_id, "workspace": workspace_path, "written": [], "skipped_empty": False}
|
|
|
|
state_path = context_path.parent / "state.json"
|
|
|
|
if not context_path.exists():
|
|
result["skipped_empty"] = True
|
|
return result
|
|
|
|
context = load_jsonl(context_path)
|
|
if not context:
|
|
result["skipped_empty"] = True
|
|
return result
|
|
|
|
# Skip the session that is currently running this script to avoid churn.
|
|
# The current session's context.jsonl is being appended to live.
|
|
if skip_active:
|
|
# Heuristic: if the context file mtime is within the last minute, skip it.
|
|
stat = context_path.stat()
|
|
age_seconds = datetime.now().timestamp() - stat.st_mtime
|
|
if age_seconds < 60:
|
|
result["skipped_active"] = True
|
|
return result
|
|
|
|
# Derive stable timestamps from the source context.jsonl mtime.
|
|
context_stat = context_path.stat()
|
|
created = modified = datetime.fromtimestamp(context_stat.st_mtime, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
|
if hasattr(context_stat, "st_birthtime") and context_stat.st_birthtime > 0:
|
|
created = datetime.fromtimestamp(context_stat.st_birthtime, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
messages = extract_chat_messages(context, created)
|
|
title = derive_title(messages, workspace_path)
|
|
|
|
target_dir = BRAIN_DIR / f"session-kimi-{session_id}"
|
|
sys_gen_dir = target_dir / ".system_generated" / "logs"
|
|
|
|
# Provenance: include subagent path if this is a nested context.
|
|
source_path = context_path.relative_to(SESSIONS_DIR) if context_path.is_relative_to(SESSIONS_DIR) else context_path
|
|
|
|
chat_history = {
|
|
"id": f"session-kimi-{session_id}",
|
|
"title": title,
|
|
"created": created,
|
|
"source": "kimi",
|
|
"workspacePath": workspace_path,
|
|
"messages": messages,
|
|
"modelType": "kimi",
|
|
}
|
|
|
|
session_jsonl = [
|
|
json.dumps({
|
|
"type": "session_start",
|
|
"timestamp": created,
|
|
"data": {
|
|
"sessionId": f"session-kimi-{session_id}",
|
|
"workspacePath": workspace_path,
|
|
"source": "kimi",
|
|
"kimiSessionId": session_id,
|
|
},
|
|
}, ensure_ascii=False),
|
|
json.dumps({
|
|
"type": "session_end",
|
|
"timestamp": modified,
|
|
"data": {
|
|
"sessionId": f"session-kimi-{session_id}",
|
|
"totalEvents": len(messages),
|
|
},
|
|
}, ensure_ascii=False),
|
|
]
|
|
|
|
# Use a stable import timestamp derived from the source file mtime.
|
|
import_ts = modified
|
|
memory_md = generate_memory_md(session_id, workspace_path, messages, import_ts=import_ts, source_path=str(source_path))
|
|
overview_txt = f"# Conversation: {title}\n# Created: {created}\n# ID: session-kimi-{session_id}\n# Source: Kimi Code CLI\n# Workspace: {workspace_path or 'unknown'}\n\n"
|
|
|
|
files_to_write = {
|
|
target_dir / "session.jsonl": "\n".join(session_jsonl) + "\n",
|
|
target_dir / "session_memory.md": memory_md,
|
|
target_dir / ".system_generated" / "chat_history.json": json.dumps(chat_history, indent=2, ensure_ascii=False) + "\n",
|
|
sys_gen_dir / "overview.txt": overview_txt,
|
|
}
|
|
|
|
if dry_run:
|
|
result["would_write"] = [str(p.relative_to(REPLICA_ROOT)) for p in files_to_write]
|
|
return result
|
|
|
|
for path, content in files_to_write.items():
|
|
if write_if_changed(path, content):
|
|
result["written"].append(str(path.relative_to(REPLICA_ROOT)))
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Plans sync
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def sync_plans(dry_run: bool = False) -> dict[str, Any]:
|
|
"""Copy Kimi plans into the brain as read-only reference material."""
|
|
result: dict[str, Any] = {"source_dir": str(PLANS_DIR), "written": [], "skipped": []}
|
|
if not PLANS_DIR.exists():
|
|
return result
|
|
|
|
target_dir = BRAIN_DIR / "kimi-plans"
|
|
for plan_path in sorted(PLANS_DIR.glob("*.md")):
|
|
relative = plan_path.relative_to(PLANS_DIR)
|
|
target = target_dir / relative
|
|
try:
|
|
content = plan_path.read_text(encoding="utf-8")
|
|
except OSError:
|
|
continue
|
|
|
|
# Inject a small header with provenance (stable once added)
|
|
if not content.startswith("<!-- Imported from"):
|
|
header = f"<!-- Imported from {plan_path} at {datetime.fromtimestamp(plan_path.stat().st_mtime, tz=timezone.utc).isoformat().replace('+00:00', 'Z')} -->\n"
|
|
content = header + content
|
|
|
|
if dry_run:
|
|
result["skipped"].append(str(target.relative_to(REPLICA_ROOT)))
|
|
continue
|
|
|
|
if write_if_changed(target, content):
|
|
result["written"].append(str(target.relative_to(REPLICA_ROOT)))
|
|
else:
|
|
result["skipped"].append(str(target.relative_to(REPLICA_ROOT)))
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Sync Kimi Code CLI sessions/plans into Réplica Omnisciente brain.",
|
|
)
|
|
parser.add_argument("--dry-run", action="store_true", help="Show what would be synced without writing.")
|
|
parser.add_argument("--skip-active", action="store_true", help="Skip sessions modified in the last 60s (likely active).")
|
|
parser.add_argument("--kimi-dir", type=Path, default=KIMI_DIR, help="Override ~/.kimi location.")
|
|
parser.add_argument("--replica-root", type=Path, default=REPLICA_ROOT, help="Override replica repo root.")
|
|
args = parser.parse_args()
|
|
|
|
global SESSIONS_DIR, PLANS_DIR, KIMI_REGISTRY, BRAIN_DIR
|
|
SESSIONS_DIR = args.kimi_dir / "sessions"
|
|
PLANS_DIR = args.kimi_dir / "plans"
|
|
KIMI_REGISTRY = args.kimi_dir / "kimi.json"
|
|
BRAIN_DIR = args.replica_root / ".aurelio" / "brain"
|
|
|
|
if not SESSIONS_DIR.exists():
|
|
print(f"❌ Kimi sessions directory not found: {SESSIONS_DIR}", file=sys.stderr)
|
|
return 1
|
|
|
|
print(f"📚 Registered Kimi workspaces: {len(build_workspace_map(KIMI_REGISTRY))}")
|
|
print(f"🧠 Target brain: {BRAIN_DIR}")
|
|
print()
|
|
|
|
stats = {
|
|
"sessions_scanned": 0,
|
|
"sessions_written": 0,
|
|
"sessions_empty": 0,
|
|
"sessions_active_skipped": 0,
|
|
"files_written": 0,
|
|
}
|
|
|
|
# Discover all Kimi contexts (main + nested subagent sessions)
|
|
for context_path, session_id, workspace_path in discover_sessions():
|
|
stats["sessions_scanned"] += 1
|
|
|
|
result = sync_session(
|
|
context_path,
|
|
session_id,
|
|
workspace_path,
|
|
dry_run=args.dry_run,
|
|
skip_active=args.skip_active,
|
|
)
|
|
|
|
if result.get("skipped_empty"):
|
|
stats["sessions_empty"] += 1
|
|
continue
|
|
if result.get("skipped_active"):
|
|
stats["sessions_active_skipped"] += 1
|
|
continue
|
|
|
|
written = result.get("written", [])
|
|
if written:
|
|
stats["sessions_written"] += 1
|
|
stats["files_written"] += len(written)
|
|
label = "subagent" if "-" in session_id and len(session_id) > 36 else "session"
|
|
print(f"📝 {label} session-kimi-{session_id} ({Path(workspace_path).name if workspace_path else 'unknown'})")
|
|
for f in written:
|
|
print(f" → {f}")
|
|
|
|
# Sync plans
|
|
plans_result = sync_plans(dry_run=args.dry_run)
|
|
plan_files_written = plans_result.get("written", [])
|
|
if plan_files_written:
|
|
stats["files_written"] += len(plan_files_written)
|
|
print(f"\n📋 Plans synced:")
|
|
for f in plan_files_written:
|
|
print(f" → {f}")
|
|
|
|
print("\n" + "=" * 60)
|
|
print("Summary")
|
|
print("=" * 60)
|
|
print(f" Sessions scanned: {stats['sessions_scanned']}")
|
|
print(f" Sessions written: {stats['sessions_written']}")
|
|
print(f" Sessions empty: {stats['sessions_empty']}")
|
|
print(f" Active skipped: {stats['sessions_active_skipped']}")
|
|
print(f" Total files written: {stats['files_written']}")
|
|
if args.dry_run:
|
|
print("\n⚠️ Dry run — no files were actually written.")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|