memory: 修注入开销、idf 排序与抽取诊断

注入侧
- context 在每次模型请求都触发,改为按 session + 最新用户消息去重:
  一个 turn 里 n 次工具调用不再重复注入 n 个 memory 块
- 默认注入从 1200 字符 / 内联 4 条正文,降到 600 字符 / 内联 0 条:
  大纲已说明存在什么,正文交给 memory_recall 按需取一次,避免每次
  请求都为同一批正文付费
- halfLifeDays 此前被静默丢弃,注入始终按 14 天半衰期排序,现已接通
- 召回缓存 2s -> 500ms,同一工具链里 remember -> recall 能读到刚写的文件

排序
- idf 原先拿「单文件匹配数」当分子、「总文件数」当分母,每个文件得到
  相同 idf,排序退化为纯词频;改为按文档频率计算的真实 idf

抽取
- 移除 400 段上限:它会静默截断长对话,300 轮只贡献前 ~50 轮
- role 在顶层或 message 下两种返回结构都读,之前只读 .role 得到空转录,
  长度门槛不过,抽取看起来永远不触发
- 去重改按关键词重叠计分,不再用 bigram 重叠:模型改写与人工措辞几乎
  不共享 bigram,真实改写只拿到 0.43;关键词是双方被迫共用的词汇
- 新增 memory_extract_dryrun:对真实模型跑完整抽取链路,不必改
  minTurns 或等长会话
- 新增 memory_inspect:诊断记录按 scope 存入 storage key,并保留历史
  与 fires/injected 计数,去重是否生效可被证伪

其他
- Context 改从 Plugin 命名空间导入,此前类型退化为 unknown
- store 暴露 kind 字段;compileRootTables 内联进 compileRoot
- frontmatter 列表元素统一 String(),index 不再出现数字

验证:由仓库所有者端到端验证(真实模型调用、真实会话);提交者未复跑,
以上为 diff 复核结论。
This commit is contained in:
2026-10-12 18:10:40 +08:00
parent 8dc02b6b35
commit cb9db666ba
5 changed files with 702 additions and 220 deletions
+51 -29
View File
@@ -47,7 +47,7 @@ export function tokenize(text: string): string[] {
// CJK: unigrams plus bigrams. Bigrams carry the matching power; unigrams
// rescue one-character queries that have no bigram at all.
const runs = text.match(/[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]+/g) ?? []
const runs = text.match(new RegExp(CJK.source, "g")) ?? []
for (const run of runs) {
const chars = [...run]
for (let i = 0; i < chars.length - 1; i++) {
@@ -84,19 +84,33 @@ export function scoreFiles(
const halfLife = (options.halfLifeDays ?? 14) * 86_400_000
const queryTokens = tokenize(query)
if (queryTokens.length === 0) return []
if (files.length === 0) return []
const querySet = new Map<string, number>()
for (const token of queryTokens) querySet.set(token, (querySet.get(token) ?? 0) + 1)
// Document frequency for idf: how many files contain each term.
// The previous version passed a per-file match count into a total-files
// denominator, which gave every file the same idf and reduced ranking to
// raw term frequency.
const docFreq = new Map<string, number>()
const docTokenList: Set<string>[] = []
for (const file of files) {
const haystack = [file.title, file.summary, file.highlight, file.loadWhen.join(" "), file.body].join("\n")
const docTokens = new Set(tokenize(haystack))
docTokenList.push(docTokens)
for (const token of docTokens) docFreq.set(token, (docFreq.get(token) ?? 0) + 1)
}
const results: ScoredFile[] = []
for (const file of files) {
for (let i = 0; i < files.length; i++) {
const file = files[i]
// A superseded memory stays on disk for audit but must never be injected.
if (file.fm.superseded_by) continue
const state = states[file.rel]
const haystack = [file.title, file.summary, file.highlight, file.loadWhen.join(" "), file.body].join("\n")
const docTokens = new Set(tokenize(haystack))
const docTokens = docTokenList[i]
let overlap = 0
const matched: string[] = []
@@ -108,8 +122,18 @@ export function scoreFiles(
}
if (overlap <= 0) continue
// Relative term rarity across the corpus: a term in every file is useless.
const idf = idfFor(tokenScore(querySet, docTokens), files.length)
// Inverse document frequency: a term appearing in every file is useless.
let idfSum = 0
let idfCount = 0
for (const token of querySet.keys()) {
if (!docTokens.has(token)) continue
const df = docFreq.get(token) ?? 1
idfSum += Math.log(1 + files.length / df)
idfCount++
}
if (idfCount === 0) continue
const idf = idfSum / idfCount
let score = overlap * idf
// Scope prior.
@@ -137,23 +161,18 @@ export function scoreFiles(
return results.sort((a, b) => b.score - a.score)
}
function tokenScore(querySet: Map<string, number>, docTokens: Set<string>): number {
let n = 0
for (const token of querySet.keys()) if (docTokens.has(token)) n++
return n
}
/** Inverse document frequency over the candidate set. */
function idfFor(matchedTerms: number, total: number): number {
if (matchedTerms <= 0 || total <= 0) return 0
return Math.log(1 + total / matchedTerms)
}
export interface InjectionOptions {
/** Hard cap on the injected block, in characters. */
maxChars?: number
/** Max number of memory bodies to inline. */
maxEntries?: number
/**
* Half-life in days for the recency decay. 0 disables decay.
* Declared here because the caller passes it: it was silently dropped when
* this option was missing, so injection ranked with the default 14-day
* half-life regardless of configuration.
*/
halfLifeDays?: number
now?: number
}
@@ -175,36 +194,36 @@ export interface InjectionInput {
*/
export function buildInjection(input: InjectionInput): string {
const { roots, filesByRoot, statesByRoot, query } = input
const maxChars = input.options?.maxChars ?? 1200
const maxEntries = input.options?.maxEntries ?? 4
const maxChars = input.options?.maxChars ?? 600
const maxEntries = input.options?.maxEntries ?? 0
const now = input.options?.now ?? Date.now()
const blocks: string[] = []
// Tier 1 — hard rules. Resident regardless of query: these are the user's
// non-negotiables and must never be outranked by relevance.
// Body first line only: the title usually restates it, and this block is
// re-sent on every model request.
const ruleLines: string[] = []
for (const root of roots) {
for (const file of filesByRoot.get(root.key) ?? []) {
const isRule = file.category === "rules" || String(file.fm.kind ?? "").toLowerCase() === "rule"
if (!isRule || file.fm.superseded_by) continue
ruleLines.push(`- [${root.label}] ${file.title}${file.summary ? ` — ${file.summary}` : ""}`)
const firstLine = file.body.split(/\r?\n/).find((l) => l.trim())?.trim() ?? ""
if (firstLine) ruleLines.push(`- [${root.label}] ${firstLine}`)
}
}
// Tier 2 — the outline: what exists, and when each part is worth loading.
// This is the ROOT.md idea: one small file answers "what do I know and when
// do I need it", instead of dumping every memory into every request.
//
// Goals only. The slug list was the single most expensive part of the block
// and the model does not need it: `memory_recall` discovers files by query.
const outline: string[] = []
for (const root of roots) {
const files = filesByRoot.get(root.key) ?? []
if (files.length === 0) continue
const goals = readGoal(root.dir)
if (goals) outline.push(`目标(${root.label}): ${goals}`)
for (const [category, list] of groupCategories(files)) {
const bits = list.map((f) => `${f.slug}(${f.summary.slice(0, 28)})`)
outline.push(`${root.label}/${category}: ${bits.join(", ")}`)
}
}
// Tier 3 — full bodies for the top matches.
@@ -266,9 +285,12 @@ function groupCategories(files: readonly import("./store.ts").MemoryFile[]) {
export function readGoal(rootDir: string): string | undefined {
try {
const text = fs.readFileSync(path.join(rootDir, "ROOT.md"), "utf8")
const m = /^##\s*当前目标\s*\n([\s\S]*?)(?=\n##\s|\n<!--|$)/m.exec(text)
// Match until the next section header or auto marker. The previous regex
// stopped at any `##` or `<!--`, which truncated content containing
// sub-headings or HTML comments inside the goals section.
const m = /^##\s*当前目标\s*\n([\s\S]*?)(?=\n##\s|\n<!-- auto:begin|$)/m.exec(text)
const body = m?.[1]
?.replace(/<!--[\s\S]*?-->/g, "")
?.replace(/<!--(?!auto:)[\s\S]*?-->/g, "")
.split(/\r?\n/)
.map((l) => l.trim())
.filter((l) => l && !l.startsWith("<!--"))