memory: 修注入开销、idf 排序与抽取诊断

注入侧 - context 在每次模型请求都触发，改为按 session + 最新用户消息去重：一个 turn 里 n 次工具调用不再重复注入 n 个 memory 块 - 默认注入从 1200 字符 / 内联 4 条正文，降到 600 字符 / 内联 0 条：大纲已说明存在什么，正文交给 memory_recall 按需取一次，避免每次请求都为同一批正文付费 - halfLifeDays 此前被静默丢弃，注入始终按 14 天半衰期排序，现已接通 - 召回缓存 2s -> 500ms，同一工具链里 remember -> recall 能读到刚写的文件排序 - idf 原先拿「单文件匹配数」当分子、「总文件数」当分母，每个文件得到相同 idf，排序退化为纯词频；改为按文档频率计算的真实 idf 抽取 - 移除 400 段上限：它会静默截断长对话，300 轮只贡献前 ~50 轮 - role 在顶层或 message 下两种返回结构都读，之前只读 .role 得到空转录，长度门槛不过，抽取看起来永远不触发 - 去重改按关键词重叠计分，不再用 bigram 重叠：模型改写与人工措辞几乎不共享 bigram，真实改写只拿到 0.43；关键词是双方被迫共用的词汇 - 新增 memory_extract_dryrun：对真实模型跑完整抽取链路，不必改 minTurns 或等长会话 - 新增 memory_inspect：诊断记录按 scope 存入 storage key，并保留历史与 fires/injected 计数，去重是否生效可被证伪其他 - Context 改从 Plugin 命名空间导入，此前类型退化为 unknown - store 暴露 kind 字段；compileRootTables 内联进 compileRoot - frontmatter 列表元素统一 String()，index 不再出现数字验证：由仓库所有者端到端验证（真实模型调用、真实会话）；提交者未复跑，以上为 diff 复核结论。
2026-10-12 18:10:40 +08:00
parent 8dc02b6b35
commit cb9db666ba
5 changed files with 702 additions and 220 deletions
@@ -12,7 +12,7 @@
 //     one, but only a human confirms it. Letting a model write "iron rules"
 //     would let a hallucination become permanent policy.

-import type { Context } from "@opencode/plugin"
+type Context = import("@opencode/plugin").Plugin.Context
 import { tokenize } from "./recall.ts"

 export interface ExtractOptions {
@@ -72,9 +72,13 @@ If nothing durable was learned, return [].`
 * Pull plain text out of a session message without depending on the exact
 * message shape: the plugin package ships types for a schema the host may
 * evolve, so this walks defensively and collects `text` fields.
+ *
+ * The 400-part cap was removed: it silently truncated long transcripts, so a
+ * 300-turn session would only contribute its first ~50 turns to extraction.
+ * Depth still bounds recursion on malformed input.
 */
 export function collectText(value: unknown, depth = 0, out: string[] = []): string[] {
-  if (depth > 6 || out.length > 400) return out
+  if (depth > 6) return out
  if (typeof value === "string") {
    if (value.trim()) out.push(value)
    return out
@@ -101,9 +105,19 @@ export function userAndAssistantText(messages: readonly unknown[]): string {
  for (const message of messages) {
    const record = message as Record<string, unknown> | null
    if (!record) continue
-    const role = record.role
+    // The role sits at the top level or under `message`, depending on whether
+    // the host returned `Session.Message.Info` or its inner message. Reading
+    // only `.role` produced an empty transcript, which then failed the length
+    // gate and made extraction look permanently inert.
+    const nested = record.message as Record<string, unknown> | undefined
+    const role =
+      typeof record.role === "string"
+        ? record.role
+        : typeof nested?.role === "string"
+          ? nested.role
+          : undefined
    if (role !== "user" && role !== "assistant") continue
-    const texts = collectText(record.content)
+    const texts = collectText(record.content ?? nested)
    const joined = texts.join("\n").trim()
    if (joined) parts.push(`${role === "user" ? "USER" : "ASSISTANT"}: ${joined}`)
  }
@@ -187,7 +201,8 @@ export function slugify(input: string): string {
    .slice(0, 48)
  if (ascii) return ascii
  // CJK titles have no ASCII to work with; fall back to a hash-free stable id.
-  return [...input].map((ch) => ch.codePointAt(0)!.toString(36)).join("").slice(0, 24)
+  // `codePointAt(0)` returns undefined for an empty string, so guard with ?? 0.
+  return [...input].map((ch) => (ch.codePointAt(0) ?? 0).toString(36)).join("").slice(0, 24)
 }

 /** Secrets that must never reach a git-tracked memory file. */
@@ -199,6 +214,9 @@ const SECRET_PATTERNS: RegExp[] = [
  /\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
  /\b(?:password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token)\s*[:=]\s*\S+/gi,
  /-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----/g,
+  // CJK variants: \b does not work on Chinese text, so match the keyword
+  // directly with a separator and a non-space value.
+  /(?:密码|口令|密钥|秘钥|api[_-]?key|access[_-]?token|auth[_-]?token|secret)\s*[:=：]\s*\S+/gi,
 ]

 export function hasSecret(text: string): string | undefined {
@@ -219,7 +237,7 @@ export interface ExtractionOutcome {
  candidates: Candidate[]
  rejected: { candidate: Candidate; reason: string }[]
  /**
-   * Set when the round-trip itself failed. Absent means the model answered.
+   * Set when every model call failed. Absent means one answered.
   * Without this, a broken call and a conversation with nothing worth keeping
   * both come back as zero candidates — which is how a permanently broken
   * extractor can look like a healthy one that simply rarely fires.
@@ -227,6 +245,10 @@ export interface ExtractionOutcome {
  error?: string
  /** Raw model text, truncated, for diagnosing a surprising empty result. */
  rawText?: string
+  /** The model that actually answered, when not the first candidate. */
+  modelUsed?: string
+  /** Models that failed before one succeeded. */
+  notes?: string
 }

 /**
@@ -239,12 +261,15 @@ export interface ExtractionOutcome {
 * Coverage answers the question we actually care about: "does an existing
 * memory already say this?"
 */
+/** Floor on matched signals, so coincidental overlap cannot reject a memory. */
+const MIN_SHARED_TERMS = 6
+
 export function screenCandidates(
  candidates: Candidate[],
  existing: readonly import("./store.ts").MemoryFile[],
  options: { coverage?: number } = {},
 ): ExtractionOutcome {
-  const threshold = options.coverage ?? 0.55
+  const threshold = options.coverage ?? 0.5
  const accepted: Candidate[] = []
  const rejected: { candidate: Candidate; reason: string }[] = []

@@ -262,12 +287,14 @@ export function screenCandidates(
      candidate.kind = "rule-proposal"
    }

-    const probe = `${candidate.title}\n${candidate.summary}\n${candidate.body}`
-    const best = bestCoverage(probe, existing)
-    if (best.file && best.coverage >= threshold) {
+    const best = bestCoverage(candidate, existing)
+    // Both conditions: most of the candidate's declared subjects are already
+    // covered by an existing memory, AND enough terms matched in total that the
+    // overlap is not a coincidence.
+    if (best.file && best.coverage >= threshold && best.shared >= MIN_SHARED_TERMS) {
      rejected.push({
        candidate,
-        reason: `与已有记忆重复 ${(best.coverage * 100).toFixed(0)}% (${best.file.rel})`,
+        reason: `与已有记忆主题重复 ${(best.coverage * 100).toFixed(0)}%（${best.shared} 项重合，${best.file.rel}）`,
      })
      continue
    }
@@ -278,27 +305,83 @@ export function screenCandidates(
 }

 /**
- * Fraction of the candidate's distinctive terms that an existing memory also
- * contains. The candidate's own terms are the query; the best match wins.
+ * How much of a candidate is already said elsewhere, 0..1. Best match wins.
+ *
+ * Measured against the real library, plain bigram overlap does NOT work, and no
+ * threshold fixes it. A genuine restatement scored only 0.43 because the model
+ * writes in different words than a human does: existing "走两阶段、禁止 drop
+ * 列" versus extracted "遵循 expand/contract：先以向后兼容方式扩展结构…收缩
+ * 变更". Nearly every non-topic bigram differs; the topic terms are identical.
+ *
+ * So overlap is measured over KEY TERMS only — latin words and the terms the
+ * extractor itself declared in `loadWhen`. Those are the vocabulary both sides
+ * are forced to share, which is why they survive paraphrasing. Bigrams are
+ * still used to find an existing memory's key terms, just not as the score.
 */
-function bestCoverage(
-  probe: string,
-  existing: readonly import("./store.ts").MemoryFile[],
-): { file?: import("./store.ts").MemoryFile; coverage: number } {
-  const probeTerms = new Set(tokenize(probe).filter((t) => t.length >= 2))
-  if (probeTerms.size === 0) return { coverage: 0 }
+function keyTerms(text: string): Set<string> {
+  return new Set(tokenize(text).filter((t) => t.length >= 2 && !/[㐀-鿿]/.test(t[0])))
+}
+
+function bestCoverage(
+  candidate: Candidate,
+  existing: readonly import("./store.ts").MemoryFile[],
+): { file?: import("./store.ts").MemoryFile; coverage: number; shared: number } {
+  // The topic vocabulary: what the extractor said this memory is about, plus
+  // the technical terms in its text.
+  const subject = new Set<string>([
+    ...(candidate.loadWhen ?? []).map((s) => s.trim().toLowerCase()),
+    ...candidate.title.split(/[\s、，,。：:（）()]+/).filter((s) => /[A-Za-z0-9]/.test(s)),
+  ])
+  // Every term in the candidate, used only to look a document up.
+  const probeTerms = new Set(tokenize(`${candidate.title}\n${candidate.summary}\n${candidate.body}`))
+
+  if (subject.size === 0) return { coverage: 0, shared: 0 }
+
+  let best: { file?: import("./store.ts").MemoryFile; coverage: number; shared: number } = {
+    coverage: 0,
+    shared: 0,
+  }

-  let best: { file?: import("./store.ts").MemoryFile; coverage: number } = { coverage: 0 }
  for (const file of existing) {
    if (file.fm.superseded_by) continue
-    // Same slug in the same root is the same memory, whatever the wording.
-    const doc = new Set(
-      tokenize([file.title, file.summary, file.highlight, file.body].join("\n")).filter((t) => t.length >= 2),
-    )
+
+    // A document's subject vocabulary: its own declared triggers plus its
+    // technical terms. Human-written and model-written memories land here on
+    // the same footing, which is the whole point.
+    const docSubject = new Set<string>([
+      ...file.loadWhen.map((s) => s.trim().toLowerCase()),
+      ...keyTerms(`${file.title}\n${file.summary}\n${file.highlight}\n${file.body}`),
+    ])
+
+    // Overlap on the candidate's declared subjects first: two memories about
+    // "数据库迁移" and "删除列" are the same subject even if worded apart.
    let shared = 0
-    for (const term of probeTerms) if (doc.has(term)) shared++
-    const coverage = shared / probeTerms.size
-    if (coverage > best.coverage) best = { file, coverage }
+    for (const term of subject) {
+      if (!term) continue
+      if (docSubject.has(term)) {
+        shared++
+        continue
+      }
+      // Fall back to a partial hit, so 迁移 matches 数据库迁移.
+      for (const docTerm of docSubject) {
+        if (docTerm.length >= 2 && term.length >= 2 && (term.includes(docTerm) || docTerm.includes(term))) {
+          shared++
+          break
+        }
+      }
+    }
+
+    // Second signal: technical terms the two share (expand/contract, drop,
+    // rollback). Requires any lexical overlap so unrelated topics never merge.
+    const docAll = new Set(tokenize(`${file.title}\n${file.summary}\n${file.body}`))
+    let lexical = 0
+    for (const term of probeTerms) if (docAll.has(term)) lexical++
+
+    const coverage = shared / subject.size
+    const total = shared + lexical
+    if (total > best.shared || (total === best.shared && coverage > best.coverage)) {
+      best = { file, coverage, shared: total }
+    }
  }
  return best
 }
@@ -306,6 +389,8 @@ function bestCoverage(
 export interface RunExtractionInput {
  ctx: Context
  model: { providerID: string; id: string }
+  /** Tried in order after `model` fails. */
+  fallbackModels?: readonly { providerID: string; id: string }[]
  transcript: string
  existing: readonly import("./store.ts").MemoryFile[]
  signal?: AbortSignal
@@ -320,43 +405,62 @@ export interface RunExtractionInput {
 */
 export async function extract(input: RunExtractionInput): Promise<ExtractionOutcome> {
  const prompt = `${SYSTEM}\n\n---\n\n${input.transcript}`
-  let result: unknown
-  try {
-    // `generate.text` has no cancellation documented on this surface, so the
-    // signal is forwarded when accepted and simply ignored when not.
-    result = await (input.ctx.generate as { text: (input: unknown) => Promise<unknown> }).text({
-      model: input.model,
-      prompt,
-      ...(input.signal ? { signal: input.signal } : {}),
-    })
-  } catch (error) {
-    // An extraction failure must never disturb the user's session, but it must
-    // not be indistinguishable from success either.
+
+  // Walk the candidates on failure. The model registry is not authoritative:
+  // it lists `opencode/exo-free` as active while the service refuses the call
+  // with "has been deprecated". No amount of reading metadata catches that, so
+  // the only reliable signal is trying the next one.
+  const primary = input.model
+  const alternates = (input.fallbackModels ?? []).filter(
+    (m) => m.providerID !== primary.providerID || m.id !== primary.id,
+  )
+  const attempts = [primary, ...alternates]
+
+  const failures: string[] = []
+
+  for (let i = 0; i < attempts.length; i++) {
+    const model = attempts[i]
+    let result: unknown
+    try {
+      // `generate.text` has no cancellation documented on this surface, so the
+      // signal is forwarded when accepted and simply ignored when not.
+      result = await (input.ctx.generate as { text: (i: unknown) => Promise<unknown> }).text({
+        model,
+        prompt,
+        ...(input.signal ? { signal: input.signal } : {}),
+      })
+    } catch (error) {
+      failures.push(`${model.providerID}/${model.id}: ${(error as Error).message.slice(0, 90)}`)
+      continue
+    }
+
+    const text = coerceText(result)
+    // An empty string means the call "succeeded" but returned nothing usable.
+    // An auth or provider error can arrive as a JSON error body rather than a
+    // thrown error, so surface what came back instead of reporting it as
+    // "nothing worth remembering" — and try another model before giving up.
+    if (!text.trim()) {
+      const preview =
+        result && typeof result === "object"
+          ? JSON.stringify(result).slice(0, 120)
+          : String(result ?? "").slice(0, 120)
+      failures.push(`${model.providerID}/${model.id}: 空内容（${preview || "空"}）`)
+      continue
+    }
+
+    const candidates = parseCandidates(text)
+    const screened = screenCandidates(candidates, input.existing)
    return {
-      candidates: [],
-      rejected: [],
-      error: `模型调用失败：${(error as Error).message.slice(0, 160)}`,
+      ...screened,
+      rawText: text.slice(0, 400),
+      modelUsed: `${model.providerID}/${model.id}`,
+      ...(failures.length ? { notes: `已跳过不可用模型：${failures.join("; ")}` } : {}),
    }
  }

-  const text = coerceText(result)
-  // An empty string here means the call "succeeded" but returned nothing usable.
-  // An auth or provider error can arrive as a JSON error body rather than a
-  // thrown error, so surface what actually came back instead of reporting it
-  // as "nothing worth remembering".
-  if (!text.trim()) {
-    const preview =
-      result && typeof result === "object"
-        ? JSON.stringify(result).slice(0, 160)
-        : String(result ?? "").slice(0, 160)
-    return {
-      candidates: [],
-      rejected: [],
-      error: `模型返回空内容（响应：${preview || "空"}）`,
-    }
+  return {
+    candidates: [],
+    rejected: [],
+    error: `所有候选模型都调用失败：${failures.join("; ") || "无可用模型"}`,
  }
-
-  const candidates = parseCandidates(text)
-  const screened = screenCandidates(candidates, input.existing)
-  return { ...screened, rawText: text.slice(0, 400) }
 }