memory: 修注入开销、idf 排序与抽取诊断
注入侧 - context 在每次模型请求都触发,改为按 session + 最新用户消息去重: 一个 turn 里 n 次工具调用不再重复注入 n 个 memory 块 - 默认注入从 1200 字符 / 内联 4 条正文,降到 600 字符 / 内联 0 条: 大纲已说明存在什么,正文交给 memory_recall 按需取一次,避免每次 请求都为同一批正文付费 - halfLifeDays 此前被静默丢弃,注入始终按 14 天半衰期排序,现已接通 - 召回缓存 2s -> 500ms,同一工具链里 remember -> recall 能读到刚写的文件 排序 - idf 原先拿「单文件匹配数」当分子、「总文件数」当分母,每个文件得到 相同 idf,排序退化为纯词频;改为按文档频率计算的真实 idf 抽取 - 移除 400 段上限:它会静默截断长对话,300 轮只贡献前 ~50 轮 - role 在顶层或 message 下两种返回结构都读,之前只读 .role 得到空转录, 长度门槛不过,抽取看起来永远不触发 - 去重改按关键词重叠计分,不再用 bigram 重叠:模型改写与人工措辞几乎 不共享 bigram,真实改写只拿到 0.43;关键词是双方被迫共用的词汇 - 新增 memory_extract_dryrun:对真实模型跑完整抽取链路,不必改 minTurns 或等长会话 - 新增 memory_inspect:诊断记录按 scope 存入 storage key,并保留历史 与 fires/injected 计数,去重是否生效可被证伪 其他 - Context 改从 Plugin 命名空间导入,此前类型退化为 unknown - store 暴露 kind 字段;compileRootTables 内联进 compileRoot - frontmatter 列表元素统一 String(),index 不再出现数字 验证:由仓库所有者端到端验证(真实模型调用、真实会话);提交者未复跑, 以上为 diff 复核结论。
This commit is contained in:
+51
-29
@@ -47,7 +47,7 @@ export function tokenize(text: string): string[] {
|
||||
|
||||
// CJK: unigrams plus bigrams. Bigrams carry the matching power; unigrams
|
||||
// rescue one-character queries that have no bigram at all.
|
||||
const runs = text.match(/[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]+/g) ?? []
|
||||
const runs = text.match(new RegExp(CJK.source, "g")) ?? []
|
||||
for (const run of runs) {
|
||||
const chars = [...run]
|
||||
for (let i = 0; i < chars.length - 1; i++) {
|
||||
@@ -84,19 +84,33 @@ export function scoreFiles(
|
||||
const halfLife = (options.halfLifeDays ?? 14) * 86_400_000
|
||||
const queryTokens = tokenize(query)
|
||||
if (queryTokens.length === 0) return []
|
||||
if (files.length === 0) return []
|
||||
|
||||
const querySet = new Map<string, number>()
|
||||
for (const token of queryTokens) querySet.set(token, (querySet.get(token) ?? 0) + 1)
|
||||
|
||||
// Document frequency for idf: how many files contain each term.
|
||||
// The previous version passed a per-file match count into a total-files
|
||||
// denominator, which gave every file the same idf and reduced ranking to
|
||||
// raw term frequency.
|
||||
const docFreq = new Map<string, number>()
|
||||
const docTokenList: Set<string>[] = []
|
||||
for (const file of files) {
|
||||
const haystack = [file.title, file.summary, file.highlight, file.loadWhen.join(" "), file.body].join("\n")
|
||||
const docTokens = new Set(tokenize(haystack))
|
||||
docTokenList.push(docTokens)
|
||||
for (const token of docTokens) docFreq.set(token, (docFreq.get(token) ?? 0) + 1)
|
||||
}
|
||||
|
||||
const results: ScoredFile[] = []
|
||||
|
||||
for (const file of files) {
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i]
|
||||
// A superseded memory stays on disk for audit but must never be injected.
|
||||
if (file.fm.superseded_by) continue
|
||||
|
||||
const state = states[file.rel]
|
||||
const haystack = [file.title, file.summary, file.highlight, file.loadWhen.join(" "), file.body].join("\n")
|
||||
const docTokens = new Set(tokenize(haystack))
|
||||
const docTokens = docTokenList[i]
|
||||
|
||||
let overlap = 0
|
||||
const matched: string[] = []
|
||||
@@ -108,8 +122,18 @@ export function scoreFiles(
|
||||
}
|
||||
if (overlap <= 0) continue
|
||||
|
||||
// Relative term rarity across the corpus: a term in every file is useless.
|
||||
const idf = idfFor(tokenScore(querySet, docTokens), files.length)
|
||||
// Inverse document frequency: a term appearing in every file is useless.
|
||||
let idfSum = 0
|
||||
let idfCount = 0
|
||||
for (const token of querySet.keys()) {
|
||||
if (!docTokens.has(token)) continue
|
||||
const df = docFreq.get(token) ?? 1
|
||||
idfSum += Math.log(1 + files.length / df)
|
||||
idfCount++
|
||||
}
|
||||
if (idfCount === 0) continue
|
||||
const idf = idfSum / idfCount
|
||||
|
||||
let score = overlap * idf
|
||||
|
||||
// Scope prior.
|
||||
@@ -137,23 +161,18 @@ export function scoreFiles(
|
||||
return results.sort((a, b) => b.score - a.score)
|
||||
}
|
||||
|
||||
function tokenScore(querySet: Map<string, number>, docTokens: Set<string>): number {
|
||||
let n = 0
|
||||
for (const token of querySet.keys()) if (docTokens.has(token)) n++
|
||||
return n
|
||||
}
|
||||
|
||||
/** Inverse document frequency over the candidate set. */
|
||||
function idfFor(matchedTerms: number, total: number): number {
|
||||
if (matchedTerms <= 0 || total <= 0) return 0
|
||||
return Math.log(1 + total / matchedTerms)
|
||||
}
|
||||
|
||||
export interface InjectionOptions {
|
||||
/** Hard cap on the injected block, in characters. */
|
||||
maxChars?: number
|
||||
/** Max number of memory bodies to inline. */
|
||||
maxEntries?: number
|
||||
/**
|
||||
* Half-life in days for the recency decay. 0 disables decay.
|
||||
* Declared here because the caller passes it: it was silently dropped when
|
||||
* this option was missing, so injection ranked with the default 14-day
|
||||
* half-life regardless of configuration.
|
||||
*/
|
||||
halfLifeDays?: number
|
||||
now?: number
|
||||
}
|
||||
|
||||
@@ -175,36 +194,36 @@ export interface InjectionInput {
|
||||
*/
|
||||
export function buildInjection(input: InjectionInput): string {
|
||||
const { roots, filesByRoot, statesByRoot, query } = input
|
||||
const maxChars = input.options?.maxChars ?? 1200
|
||||
const maxEntries = input.options?.maxEntries ?? 4
|
||||
const maxChars = input.options?.maxChars ?? 600
|
||||
const maxEntries = input.options?.maxEntries ?? 0
|
||||
const now = input.options?.now ?? Date.now()
|
||||
|
||||
const blocks: string[] = []
|
||||
|
||||
// Tier 1 — hard rules. Resident regardless of query: these are the user's
|
||||
// non-negotiables and must never be outranked by relevance.
|
||||
// Body first line only: the title usually restates it, and this block is
|
||||
// re-sent on every model request.
|
||||
const ruleLines: string[] = []
|
||||
for (const root of roots) {
|
||||
for (const file of filesByRoot.get(root.key) ?? []) {
|
||||
const isRule = file.category === "rules" || String(file.fm.kind ?? "").toLowerCase() === "rule"
|
||||
if (!isRule || file.fm.superseded_by) continue
|
||||
ruleLines.push(`- [${root.label}] ${file.title}${file.summary ? ` — ${file.summary}` : ""}`)
|
||||
const firstLine = file.body.split(/\r?\n/).find((l) => l.trim())?.trim() ?? ""
|
||||
if (firstLine) ruleLines.push(`- [${root.label}] ${firstLine}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Tier 2 — the outline: what exists, and when each part is worth loading.
|
||||
// This is the ROOT.md idea: one small file answers "what do I know and when
|
||||
// do I need it", instead of dumping every memory into every request.
|
||||
//
|
||||
// Goals only. The slug list was the single most expensive part of the block
|
||||
// and the model does not need it: `memory_recall` discovers files by query.
|
||||
const outline: string[] = []
|
||||
for (const root of roots) {
|
||||
const files = filesByRoot.get(root.key) ?? []
|
||||
if (files.length === 0) continue
|
||||
const goals = readGoal(root.dir)
|
||||
if (goals) outline.push(`目标(${root.label}): ${goals}`)
|
||||
for (const [category, list] of groupCategories(files)) {
|
||||
const bits = list.map((f) => `${f.slug}(${f.summary.slice(0, 28)})`)
|
||||
outline.push(`${root.label}/${category}: ${bits.join(", ")}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Tier 3 — full bodies for the top matches.
|
||||
@@ -266,9 +285,12 @@ function groupCategories(files: readonly import("./store.ts").MemoryFile[]) {
|
||||
export function readGoal(rootDir: string): string | undefined {
|
||||
try {
|
||||
const text = fs.readFileSync(path.join(rootDir, "ROOT.md"), "utf8")
|
||||
const m = /^##\s*当前目标\s*\n([\s\S]*?)(?=\n##\s|\n<!--|$)/m.exec(text)
|
||||
// Match until the next section header or auto marker. The previous regex
|
||||
// stopped at any `##` or `<!--`, which truncated content containing
|
||||
// sub-headings or HTML comments inside the goals section.
|
||||
const m = /^##\s*当前目标\s*\n([\s\S]*?)(?=\n##\s|\n<!-- auto:begin|$)/m.exec(text)
|
||||
const body = m?.[1]
|
||||
?.replace(/<!--[\s\S]*?-->/g, "")
|
||||
?.replace(/<!--(?!auto:)[\s\S]*?-->/g, "")
|
||||
.split(/\r?\n/)
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l && !l.startsWith("<!--"))
|
||||
|
||||
Reference in New Issue
Block a user