oc/plugins/memory/extract.ts

// Automatic extraction: turn a finished turn into candidate memories.
//
// Runs on `session.idle`. The point is that memory capture must not depend on
// the model remembering to call `memory_write` — models forget. This is the
// safety net.
//
// Two deliberate limits keep it from becoming a garbage generator:
//
//  1. A separate, cheap model does the extraction. It never touches the main
//     conversation, so a bad extraction cannot derail the user's work.
//  2. Hard rules are never written automatically. The extractor may *propose*
//     one, but only a human confirms it. Letting a model write "iron rules"
//     would let a hallucination become permanent policy.

type Context = import("@opencode/plugin").Plugin.Context
import { tokenize } from "./recall.ts"

export interface ExtractOptions {
  enabled: boolean
  /** Minimum turns in a session before extraction kicks in. */
  minTurns: number
  /** Cap on how many messages are handed to the extractor. */
  maxMessages: number
  model?: { providerID: string; id: string }
}

export const DEFAULT_EXTRACT: ExtractOptions = {
  enabled: true,
  minTurns: 4,
  maxMessages: 24,
}

export interface Candidate {
  category: string
  slug: string
  title: string
  summary: string
  highlight?: string
  body: string
  kind?: string
  loadWhen?: string[]
  /** Rules the model believes should be hard rules, needing human confirmation. */
  proposeRule?: boolean
}

const SYSTEM = `You extract durable memories from a coding conversation.

Return ONLY a JSON array. Each element:
{
  "category": "preferences|decisions|pitfalls|facts",
  "slug": "kebab-case-identifier",
  "title": "short title",
  "summary": "ONE sentence, under 100 chars. This is what shows in an index.",
  "highlight": "the single most important detail",
  "body": "markdown detail, 2-6 sentences",
  "loadWhen": ["trigger words or situations where this is relevant"],
  "proposeRule": false
}

Only record what stays true after the session ends:
  GOOD  A stated preference, an architectural decision and its reason,
        a non-obvious gotcha and its root cause, a project convention.
  BAD   What the code currently looks like, task state, git branch names,
        file paths, anything already obvious from the repo, one-off chatter.

NEVER set proposeRule to true unless the user stated an absolute rule about how
to behave ("always", "never", "必须", "禁止"). Doubtful -> omit the memory.

If nothing durable was learned, return [].`

/**
 * Pull plain text out of a session message without depending on the exact
 * message shape: the plugin package ships types for a schema the host may
 * evolve, so this walks defensively and collects `text` fields.
 *
 * The 400-part cap was removed: it silently truncated long transcripts, so a
 * 300-turn session would only contribute its first ~50 turns to extraction.
 * Depth still bounds recursion on malformed input.
 */
export function collectText(value: unknown, depth = 0, out: string[] = []): string[] {
  if (depth > 6) return out
  if (typeof value === "string") {
    if (value.trim()) out.push(value)
    return out
  }
  if (Array.isArray(value)) {
    for (const item of value) collectText(item, depth + 1, out)
    return out
  }
  if (value && typeof value === "object") {
    const record = value as Record<string, unknown>
    // Prefer a part's own text over its metadata noise.
    if (typeof record.text === "string" && record.text.trim()) out.push(record.text)
    for (const [key, child] of Object.entries(record)) {
      if (key === "text") continue
      if (key === "synthetic" || key === "cache" || key === "providerMetadata") continue
      collectText(child, depth + 1, out)
    }
  }
  return out
}

export function userAndAssistantText(messages: readonly unknown[]): string {
  const parts: string[] = []
  for (const message of messages) {
    const record = message as Record<string, unknown> | null
    if (!record) continue
    // The role sits at the top level or under `message`, depending on whether
    // the host returned `Session.Message.Info` or its inner message. Reading
    // only `.role` produced an empty transcript, which then failed the length
    // gate and made extraction look permanently inert.
    const nested = record.message as Record<string, unknown> | undefined
    const role =
      typeof record.role === "string"
        ? record.role
        : typeof nested?.role === "string"
          ? nested.role
          : undefined
    if (role !== "user" && role !== "assistant") continue
    const texts = collectText(record.content ?? nested)
    const joined = texts.join("\n").trim()
    if (joined) parts.push(`${role === "user" ? "USER" : "ASSISTANT"}: ${joined}`)
  }
  return parts.join("\n\n")
}

/** `generate.text` is documented as returning text; accept a few shapes. */
function coerceText(result: unknown): string {
  if (typeof result === "string") return result
  if (result && typeof result === "object") {
    const record = result as Record<string, unknown>
    for (const key of ["text", "content", "output", "result"]) {
      const value = record[key]
      if (typeof value === "string") return value
      if (Array.isArray(value)) {
        const joined = collectText(value).join("\n")
        if (joined) return joined
      }
    }
  }
  return ""
}

/** Parse the extractor's JSON array, tolerating prose and code fences. */
export function parseCandidates(raw: string): Candidate[] {
  const text = raw.trim()
  if (!text) return []
  const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(text)
  const body = fenced?.[1] ?? text
  const start = body.indexOf("[")
  const end = body.lastIndexOf("]")
  if (start < 0 || end <= start) return []

  let parsed: unknown
  try {
    parsed = JSON.parse(body.slice(start, end + 1))
  } catch {
    return []
  }
  if (!Array.isArray(parsed)) return []

  const out: Candidate[] = []
  for (const entry of parsed) {
    if (!entry || typeof entry !== "object") continue
    const record = entry as Record<string, unknown>
    const title = str(record.title)
    const bodyText = str(record.body)
    if (!title || !bodyText) continue

    const category = str(record.category).toLowerCase() || "facts"
    const slug =
      slugify(str(record.slug)) || slugify(title)
    if (!slug) continue

    out.push({
      category: category.replace(/[^a-z0-9._-]/g, "") || "facts",
      slug,
      title,
      summary: str(record.summary) || title,
      highlight: str(record.highlight) || undefined,
      body: bodyText,
      kind: str(record.kind) || undefined,
      loadWhen: Array.isArray(record.loadWhen)
        ? record.loadWhen.map((v) => str(v)).filter(Boolean).slice(0, 8)
        : undefined,
      proposeRule: record.proposeRule === true,
    })
  }
  return out
}

function str(value: unknown): string {
  return typeof value === "string" ? value.trim() : ""
}

export function slugify(input: string): string {
  const ascii = input
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, "-")
    .replace(/^-+|-+$/g, "")
    .slice(0, 48)
  if (ascii) return ascii
  // CJK titles have no ASCII to work with; fall back to a hash-free stable id.
  // `codePointAt(0)` returns undefined for an empty string, so guard with ?? 0.
  return [...input].map((ch) => (ch.codePointAt(0) ?? 0).toString(36)).join("").slice(0, 24)
}

/** Secrets that must never reach a git-tracked memory file. */
const SECRET_PATTERNS: RegExp[] = [
  /\b(?:sk|pk|rk)-[A-Za-z0-9_-]{16,}/g,
  /\bgh[pousr]_[A-Za-z0-9]{16,}/g,
  /\bAKIA[0-9A-Z]{16}\b/g,
  /\bxox[baprs]-[A-Za-z0-9-]{10,}/g,
  /\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
  /\b(?:password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token)\s*[:=]\s*\S+/gi,
  /-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----/g,
  // CJK variants: \b does not work on Chinese text, so match the keyword
  // directly with a separator and a non-space value.
  /(?:密码|口令|密钥|秘钥|api[_-]?key|access[_-]?token|auth[_-]?token|secret)\s*[:=：]\s*\S+/gi,
]

export function hasSecret(text: string): string | undefined {
  for (const pattern of SECRET_PATTERNS) {
    const match = pattern.exec(text)
    if (match) return match[0].slice(0, 24)
  }
  return undefined
}

export function redactSecrets(text: string): string {
  let out = text
  for (const pattern of SECRET_PATTERNS) out = out.replace(pattern, "[REDACTED]")
  return out
}

export interface ExtractionOutcome {
  candidates: Candidate[]
  rejected: { candidate: Candidate; reason: string }[]
  /**
   * Set when every model call failed. Absent means one answered.
   * Without this, a broken call and a conversation with nothing worth keeping
   * both come back as zero candidates — which is how a permanently broken
   * extractor can look like a healthy one that simply rarely fires.
   */
  error?: string
  /** Raw model text, truncated, for diagnosing a surprising empty result. */
  rawText?: string
  /** The model that actually answered, when not the first candidate. */
  modelUsed?: string
  /** Models that failed before one succeeded. */
  notes?: string
}

/**
 * Decide which candidates are worth writing, using existing memories for
 * dedupe.
 *
 * Dedupe uses term COVERAGE of the candidate, not the retrieval score. The
 * score carries idf and scope priors that shift as the corpus grows, so a fixed
 * score threshold silently stops rejecting duplicates as the library fills up.
 * Coverage answers the question we actually care about: "does an existing
 * memory already say this?"
 */
/** Floor on matched signals, so coincidental overlap cannot reject a memory. */
const MIN_SHARED_TERMS = 6

export function screenCandidates(
  candidates: Candidate[],
  existing: readonly import("./store.ts").MemoryFile[],
  options: { coverage?: number } = {},
): ExtractionOutcome {
  const threshold = options.coverage ?? 0.5
  const accepted: Candidate[] = []
  const rejected: { candidate: Candidate; reason: string }[] = []

  for (const candidate of candidates) {
    const secret = hasSecret(`${candidate.title}\n${candidate.summary}\n${candidate.body}`)
    if (secret) {
      rejected.push({ candidate, reason: `含疑似密钥 (${secret}…)` })
      continue
    }
    if (candidate.proposeRule) {
      // Not rejected outright — a proposal is surfaced for a human, which is
      // handled by the caller. Flag it here so it is written as a normal
      // memory under `proposals` rather than promoted to policy.
      candidate.category = "rules"
      candidate.kind = "rule-proposal"
    }

    const best = bestCoverage(candidate, existing)
    // Both conditions: most of the candidate's declared subjects are already
    // covered by an existing memory, AND enough terms matched in total that the
    // overlap is not a coincidence.
    if (best.file && best.coverage >= threshold && best.shared >= MIN_SHARED_TERMS) {
      rejected.push({
        candidate,
        reason: `与已有记忆主题重复 ${(best.coverage * 100).toFixed(0)}%（${best.shared} 项重合，${best.file.rel}）`,
      })
      continue
    }
    accepted.push(candidate)
  }

  return { candidates: accepted, rejected }
}

/**
 * How much of a candidate is already said elsewhere, 0..1. Best match wins.
 *
 * Measured against the real library, plain bigram overlap does NOT work, and no
 * threshold fixes it. A genuine restatement scored only 0.43 because the model
 * writes in different words than a human does: existing "走两阶段、禁止 drop
 * 列" versus extracted "遵循 expand/contract：先以向后兼容方式扩展结构…收缩
 * 变更". Nearly every non-topic bigram differs; the topic terms are identical.
 *
 * So overlap is measured over KEY TERMS only — latin words and the terms the
 * extractor itself declared in `loadWhen`. Those are the vocabulary both sides
 * are forced to share, which is why they survive paraphrasing. Bigrams are
 * still used to find an existing memory's key terms, just not as the score.
 */
function keyTerms(text: string): Set<string> {
  return new Set(tokenize(text).filter((t) => t.length >= 2 && !/[㐀-鿿]/.test(t[0])))
}

function bestCoverage(
  candidate: Candidate,
  existing: readonly import("./store.ts").MemoryFile[],
): { file?: import("./store.ts").MemoryFile; coverage: number; shared: number } {
  // The topic vocabulary: what the extractor said this memory is about, plus
  // the technical terms in its text.
  const subject = new Set<string>([
    ...(candidate.loadWhen ?? []).map((s) => s.trim().toLowerCase()),
    ...candidate.title.split(/[\s、，,。：:（）()]+/).filter((s) => /[A-Za-z0-9]/.test(s)),
  ])
  // Every term in the candidate, used only to look a document up.
  const probeTerms = new Set(tokenize(`${candidate.title}\n${candidate.summary}\n${candidate.body}`))

  if (subject.size === 0) return { coverage: 0, shared: 0 }

  let best: { file?: import("./store.ts").MemoryFile; coverage: number; shared: number } = {
    coverage: 0,
    shared: 0,
  }

  for (const file of existing) {
    if (file.fm.superseded_by) continue

    // A document's subject vocabulary: its own declared triggers plus its
    // technical terms. Human-written and model-written memories land here on
    // the same footing, which is the whole point.
    const docSubject = new Set<string>([
      ...file.loadWhen.map((s) => s.trim().toLowerCase()),
      ...keyTerms(`${file.title}\n${file.summary}\n${file.highlight}\n${file.body}`),
    ])

    // Overlap on the candidate's declared subjects first: two memories about
    // "数据库迁移" and "删除列" are the same subject even if worded apart.
    let shared = 0
    for (const term of subject) {
      if (!term) continue
      if (docSubject.has(term)) {
        shared++
        continue
      }
      // Fall back to a partial hit, so 迁移 matches 数据库迁移.
      for (const docTerm of docSubject) {
        if (docTerm.length >= 2 && term.length >= 2 && (term.includes(docTerm) || docTerm.includes(term))) {
          shared++
          break
        }
      }
    }

    // Second signal: technical terms the two share (expand/contract, drop,
    // rollback). Requires any lexical overlap so unrelated topics never merge.
    const docAll = new Set(tokenize(`${file.title}\n${file.summary}\n${file.body}`))
    let lexical = 0
    for (const term of probeTerms) if (docAll.has(term)) lexical++

    const coverage = shared / subject.size
    const total = shared + lexical
    if (total > best.shared || (total === best.shared && coverage > best.coverage)) {
      best = { file, coverage, shared: total }
    }
  }
  return best
}

export interface RunExtractionInput {
  ctx: Context
  model: { providerID: string; id: string }
  /** Tried in order after `model` fails. */
  fallbackModels?: readonly { providerID: string; id: string }[]
  transcript: string
  existing: readonly import("./store.ts").MemoryFile[]
  signal?: AbortSignal
}

/**
 * One model round-trip. Returns screened candidates and why things were dropped.
 *
 * A transport failure is reported in `error` rather than collapsed into an
 * empty result: the caller's only other signal would be "no candidates", which
 * is also what a correct answer looks like when nothing was worth keeping.
 */
export async function extract(input: RunExtractionInput): Promise<ExtractionOutcome> {
  const prompt = `${SYSTEM}\n\n---\n\n${input.transcript}`

  // Walk the candidates on failure. The model registry is not authoritative:
  // it lists `opencode/exo-free` as active while the service refuses the call
  // with "has been deprecated". No amount of reading metadata catches that, so
  // the only reliable signal is trying the next one.
  const primary = input.model
  const alternates = (input.fallbackModels ?? []).filter(
    (m) => m.providerID !== primary.providerID || m.id !== primary.id,
  )
  const attempts = [primary, ...alternates]

  const failures: string[] = []

  for (let i = 0; i < attempts.length; i++) {
    const model = attempts[i]
    let result: unknown
    try {
      // `generate.text` has no cancellation documented on this surface, so the
      // signal is forwarded when accepted and simply ignored when not.
      result = await (input.ctx.generate as { text: (i: unknown) => Promise<unknown> }).text({
        model,
        prompt,
        ...(input.signal ? { signal: input.signal } : {}),
      })
    } catch (error) {
      failures.push(`${model.providerID}/${model.id}: ${(error as Error).message.slice(0, 90)}`)
      continue
    }

    const text = coerceText(result)
    // An empty string means the call "succeeded" but returned nothing usable.
    // An auth or provider error can arrive as a JSON error body rather than a
    // thrown error, so surface what came back instead of reporting it as
    // "nothing worth remembering" — and try another model before giving up.
    if (!text.trim()) {
      const preview =
        result && typeof result === "object"
          ? JSON.stringify(result).slice(0, 120)
          : String(result ?? "").slice(0, 120)
      failures.push(`${model.providerID}/${model.id}: 空内容（${preview || "空"}）`)
      continue
    }

    const candidates = parseCandidates(text)
    const screened = screenCandidates(candidates, input.existing)
    return {
      ...screened,
      rawText: text.slice(0, 400),
      modelUsed: `${model.providerID}/${model.id}`,
      ...(failures.length ? { notes: `已跳过不可用模型：${failures.join("; ")}` } : {}),
    }
  }

  return {
    candidates: [],
    rejected: [],
    error: `所有候选模型都调用失败：${failures.join("; ") || "无可用模型"}`,
  }
}