oc/plugins/memory/extract.ts

// Automatic extraction: turn a finished turn into candidate memories.
//
// Runs on `session.idle`. The point is that memory capture must not depend on
// the model remembering to call `memory_write` — models forget. This is the
// safety net.
//
// Two deliberate limits keep it from becoming a garbage generator:
//
//  1. A separate, cheap model does the extraction. It never touches the main
//     conversation, so a bad extraction cannot derail the user's work.
//  2. Hard rules are never written automatically. The extractor may *propose*
//     one, but only a human confirms it. Letting a model write "iron rules"
//     would let a hallucination become permanent policy.

import type { Context } from "@opencode/plugin"
import { tokenize } from "./recall.ts"

export interface ExtractOptions {
  enabled: boolean
  /** Minimum turns in a session before extraction kicks in. */
  minTurns: number
  /** Cap on how many messages are handed to the extractor. */
  maxMessages: number
  model?: { providerID: string; id: string }
}

export const DEFAULT_EXTRACT: ExtractOptions = {
  enabled: true,
  minTurns: 4,
  maxMessages: 24,
}

export interface Candidate {
  category: string
  slug: string
  title: string
  summary: string
  highlight?: string
  body: string
  kind?: string
  loadWhen?: string[]
  /** Rules the model believes should be hard rules, needing human confirmation. */
  proposeRule?: boolean
}

const SYSTEM = `You extract durable memories from a coding conversation.

Return ONLY a JSON array. Each element:
{
  "category": "preferences|decisions|pitfalls|facts",
  "slug": "kebab-case-identifier",
  "title": "short title",
  "summary": "ONE sentence, under 100 chars. This is what shows in an index.",
  "highlight": "the single most important detail",
  "body": "markdown detail, 2-6 sentences",
  "loadWhen": ["trigger words or situations where this is relevant"],
  "proposeRule": false
}

Only record what stays true after the session ends:
  GOOD  A stated preference, an architectural decision and its reason,
        a non-obvious gotcha and its root cause, a project convention.
  BAD   What the code currently looks like, task state, git branch names,
        file paths, anything already obvious from the repo, one-off chatter.

NEVER set proposeRule to true unless the user stated an absolute rule about how
to behave ("always", "never", "必须", "禁止"). Doubtful -> omit the memory.

If nothing durable was learned, return [].`

/**
 * Pull plain text out of a session message without depending on the exact
 * message shape: the plugin package ships types for a schema the host may
 * evolve, so this walks defensively and collects `text` fields.
 */
export function collectText(value: unknown, depth = 0, out: string[] = []): string[] {
  if (depth > 6 || out.length > 400) return out
  if (typeof value === "string") {
    if (value.trim()) out.push(value)
    return out
  }
  if (Array.isArray(value)) {
    for (const item of value) collectText(item, depth + 1, out)
    return out
  }
  if (value && typeof value === "object") {
    const record = value as Record<string, unknown>
    // Prefer a part's own text over its metadata noise.
    if (typeof record.text === "string" && record.text.trim()) out.push(record.text)
    for (const [key, child] of Object.entries(record)) {
      if (key === "text") continue
      if (key === "synthetic" || key === "cache" || key === "providerMetadata") continue
      collectText(child, depth + 1, out)
    }
  }
  return out
}

export function userAndAssistantText(messages: readonly unknown[]): string {
  const parts: string[] = []
  for (const message of messages) {
    const record = message as Record<string, unknown> | null
    if (!record) continue
    const role = record.role
    if (role !== "user" && role !== "assistant") continue
    const texts = collectText(record.content)
    const joined = texts.join("\n").trim()
    if (joined) parts.push(`${role === "user" ? "USER" : "ASSISTANT"}: ${joined}`)
  }
  return parts.join("\n\n")
}

/** `generate.text` is documented as returning text; accept a few shapes. */
function coerceText(result: unknown): string {
  if (typeof result === "string") return result
  if (result && typeof result === "object") {
    const record = result as Record<string, unknown>
    for (const key of ["text", "content", "output", "result"]) {
      const value = record[key]
      if (typeof value === "string") return value
      if (Array.isArray(value)) {
        const joined = collectText(value).join("\n")
        if (joined) return joined
      }
    }
  }
  return ""
}

/** Parse the extractor's JSON array, tolerating prose and code fences. */
export function parseCandidates(raw: string): Candidate[] {
  const text = raw.trim()
  if (!text) return []
  const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(text)
  const body = fenced?.[1] ?? text
  const start = body.indexOf("[")
  const end = body.lastIndexOf("]")
  if (start < 0 || end <= start) return []

  let parsed: unknown
  try {
    parsed = JSON.parse(body.slice(start, end + 1))
  } catch {
    return []
  }
  if (!Array.isArray(parsed)) return []

  const out: Candidate[] = []
  for (const entry of parsed) {
    if (!entry || typeof entry !== "object") continue
    const record = entry as Record<string, unknown>
    const title = str(record.title)
    const bodyText = str(record.body)
    if (!title || !bodyText) continue

    const category = str(record.category).toLowerCase() || "facts"
    const slug =
      slugify(str(record.slug)) || slugify(title)
    if (!slug) continue

    out.push({
      category: category.replace(/[^a-z0-9._-]/g, "") || "facts",
      slug,
      title,
      summary: str(record.summary) || title,
      highlight: str(record.highlight) || undefined,
      body: bodyText,
      kind: str(record.kind) || undefined,
      loadWhen: Array.isArray(record.loadWhen)
        ? record.loadWhen.map((v) => str(v)).filter(Boolean).slice(0, 8)
        : undefined,
      proposeRule: record.proposeRule === true,
    })
  }
  return out
}

function str(value: unknown): string {
  return typeof value === "string" ? value.trim() : ""
}

export function slugify(input: string): string {
  const ascii = input
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, "-")
    .replace(/^-+|-+$/g, "")
    .slice(0, 48)
  if (ascii) return ascii
  // CJK titles have no ASCII to work with; fall back to a hash-free stable id.
  return [...input].map((ch) => ch.codePointAt(0)!.toString(36)).join("").slice(0, 24)
}

/** Secrets that must never reach a git-tracked memory file. */
const SECRET_PATTERNS: RegExp[] = [
  /\b(?:sk|pk|rk)-[A-Za-z0-9_-]{16,}/g,
  /\bgh[pousr]_[A-Za-z0-9]{16,}/g,
  /\bAKIA[0-9A-Z]{16}\b/g,
  /\bxox[baprs]-[A-Za-z0-9-]{10,}/g,
  /\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
  /\b(?:password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token)\s*[:=]\s*\S+/gi,
  /-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----/g,
]

export function hasSecret(text: string): string | undefined {
  for (const pattern of SECRET_PATTERNS) {
    const match = pattern.exec(text)
    if (match) return match[0].slice(0, 24)
  }
  return undefined
}

export function redactSecrets(text: string): string {
  let out = text
  for (const pattern of SECRET_PATTERNS) out = out.replace(pattern, "[REDACTED]")
  return out
}

export interface ExtractionOutcome {
  candidates: Candidate[]
  rejected: { candidate: Candidate; reason: string }[]
  /**
   * Set when the round-trip itself failed. Absent means the model answered.
   * Without this, a broken call and a conversation with nothing worth keeping
   * both come back as zero candidates — which is how a permanently broken
   * extractor can look like a healthy one that simply rarely fires.
   */
  error?: string
  /** Raw model text, truncated, for diagnosing a surprising empty result. */
  rawText?: string
}

/**
 * Decide which candidates are worth writing, using existing memories for
 * dedupe.
 *
 * Dedupe uses term COVERAGE of the candidate, not the retrieval score. The
 * score carries idf and scope priors that shift as the corpus grows, so a fixed
 * score threshold silently stops rejecting duplicates as the library fills up.
 * Coverage answers the question we actually care about: "does an existing
 * memory already say this?"
 */
export function screenCandidates(
  candidates: Candidate[],
  existing: readonly import("./store.ts").MemoryFile[],
  options: { coverage?: number } = {},
): ExtractionOutcome {
  const threshold = options.coverage ?? 0.55
  const accepted: Candidate[] = []
  const rejected: { candidate: Candidate; reason: string }[] = []

  for (const candidate of candidates) {
    const secret = hasSecret(`${candidate.title}\n${candidate.summary}\n${candidate.body}`)
    if (secret) {
      rejected.push({ candidate, reason: `含疑似密钥 (${secret}…)` })
      continue
    }
    if (candidate.proposeRule) {
      // Not rejected outright — a proposal is surfaced for a human, which is
      // handled by the caller. Flag it here so it is written as a normal
      // memory under `proposals` rather than promoted to policy.
      candidate.category = "rules"
      candidate.kind = "rule-proposal"
    }

    const probe = `${candidate.title}\n${candidate.summary}\n${candidate.body}`
    const best = bestCoverage(probe, existing)
    if (best.file && best.coverage >= threshold) {
      rejected.push({
        candidate,
        reason: `与已有记忆重复 ${(best.coverage * 100).toFixed(0)}% (${best.file.rel})`,
      })
      continue
    }
    accepted.push(candidate)
  }

  return { candidates: accepted, rejected }
}

/**
 * Fraction of the candidate's distinctive terms that an existing memory also
 * contains. The candidate's own terms are the query; the best match wins.
 */
function bestCoverage(
  probe: string,
  existing: readonly import("./store.ts").MemoryFile[],
): { file?: import("./store.ts").MemoryFile; coverage: number } {
  const probeTerms = new Set(tokenize(probe).filter((t) => t.length >= 2))
  if (probeTerms.size === 0) return { coverage: 0 }

  let best: { file?: import("./store.ts").MemoryFile; coverage: number } = { coverage: 0 }
  for (const file of existing) {
    if (file.fm.superseded_by) continue
    // Same slug in the same root is the same memory, whatever the wording.
    const doc = new Set(
      tokenize([file.title, file.summary, file.highlight, file.body].join("\n")).filter((t) => t.length >= 2),
    )
    let shared = 0
    for (const term of probeTerms) if (doc.has(term)) shared++
    const coverage = shared / probeTerms.size
    if (coverage > best.coverage) best = { file, coverage }
  }
  return best
}

export interface RunExtractionInput {
  ctx: Context
  model: { providerID: string; id: string }
  transcript: string
  existing: readonly import("./store.ts").MemoryFile[]
  signal?: AbortSignal
}

/**
 * One model round-trip. Returns screened candidates and why things were dropped.
 *
 * A transport failure is reported in `error` rather than collapsed into an
 * empty result: the caller's only other signal would be "no candidates", which
 * is also what a correct answer looks like when nothing was worth keeping.
 */
export async function extract(input: RunExtractionInput): Promise<ExtractionOutcome> {
  const prompt = `${SYSTEM}\n\n---\n\n${input.transcript}`
  let result: unknown
  try {
    // `generate.text` has no cancellation documented on this surface, so the
    // signal is forwarded when accepted and simply ignored when not.
    result = await (input.ctx.generate as { text: (input: unknown) => Promise<unknown> }).text({
      model: input.model,
      prompt,
      ...(input.signal ? { signal: input.signal } : {}),
    })
  } catch (error) {
    // An extraction failure must never disturb the user's session, but it must
    // not be indistinguishable from success either.
    return {
      candidates: [],
      rejected: [],
      error: `模型调用失败：${(error as Error).message.slice(0, 160)}`,
    }
  }

  const text = coerceText(result)
  // An empty string here means the call "succeeded" but returned nothing usable.
  // An auth or provider error can arrive as a JSON error body rather than a
  // thrown error, so surface what actually came back instead of reporting it
  // as "nothing worth remembering".
  if (!text.trim()) {
    const preview =
      result && typeof result === "object"
        ? JSON.stringify(result).slice(0, 160)
        : String(result ?? "").slice(0, 160)
    return {
      candidates: [],
      rejected: [],
      error: `模型返回空内容（响应：${preview || "空"}）`,
    }
  }

  const candidates = parseCandidates(text)
  const screened = screenCandidates(candidates, input.existing)
  return { ...screened, rawText: text.slice(0, 400) }
}