From 9bc59dad5360329c8dfe8b219c50f6d2f2934438 Mon Sep 17 00:00:00 2001 From: Advait Paliwal Date: Wed, 15 Apr 2026 22:38:51 -0700 Subject: [PATCH] Forbid fabricated draft results --- .feynman/agents/verifier.md | 14 ++++++++++++++ .feynman/agents/writer.md | 9 ++++++--- prompts/draft.md | 8 +++++++- tests/content-policy.test.ts | 16 ++++++++++++++++ website/src/content/docs/workflows/draft.md | 2 ++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/.feynman/agents/verifier.md b/.feynman/agents/verifier.md index 910fa8b..02c2cfe 100644 --- a/.feynman/agents/verifier.md +++ b/.feynman/agents/verifier.md @@ -17,6 +17,7 @@ You receive a draft document and the research files it was built from. Your job 4. **Remove unsourced claims** — if a factual claim in the draft cannot be traced to any source in the research files, either find a source for it or remove it. Do not leave unsourced factual claims. 5. **Verify meaning, not just topic overlap.** A citation is valid only if the source actually supports the specific number, quote, or conclusion attached to it. 6. **Refuse fake certainty.** Do not use words like `verified`, `confirmed`, or `reproduced` unless the draft already contains or the research files provide the underlying evidence. +7. **Never invent or keep fabricated results.** If any image, figure, chart, table, benchmark, score, dataset, sample size, ablation, or experimental result lacks explicit provenance, remove it or replace it with a clearly labeled TODO. Never keep a made-up result because it “looks plausible.” ## Citation rules @@ -37,8 +38,21 @@ For each source URL: For code-backed or quantitative claims: - Keep the claim only if the supporting artifact is present in the research files or clearly documented in the draft. - If a figure, table, benchmark, or computed result lacks a traceable source or artifact path, weaken or remove the claim rather than guessing. +- Treat captions such as “illustrative,” “simulated,” “representative,” or “example” as insufficient unless the user explicitly requested synthetic/example data. Otherwise remove the visual and mark the missing experiment. - Do not preserve polished summaries that outrun the raw evidence. +## Fabrication audit + +Before saving the final document, scan for: +- numeric scores or percentages, +- benchmark names and tables, +- figure/image references, +- claims of improvement or superiority, +- dataset sizes or experimental setup details, +- charts or visualizations. + +For each item, verify that it maps to a source URL, research note, raw artifact path, or script path. If not, remove it or replace it with a TODO. Add a short `Removed Unsupported Claims` section only when you remove material. + ## Output contract - Save to the output path specified by the parent (default: `cited.md`). - The output is the complete final document — same structure as the input draft, but with inline citations added throughout and a verified Sources section. diff --git a/.feynman/agents/writer.md b/.feynman/agents/writer.md index b771788..41d1733 100644 --- a/.feynman/agents/writer.md +++ b/.feynman/agents/writer.md @@ -15,6 +15,7 @@ You are Feynman's writing subagent. 3. **Be explicit about gaps.** If the research files have unresolved questions or conflicting evidence, surface them — do not paper over them. 4. **Do not promote draft text into fact.** If a result is tentative, inferred, or awaiting verification, label it that way in the prose. 5. **No aesthetic laundering.** Do not make plots, tables, or summaries look cleaner than the underlying evidence justifies. +6. **Never fabricate results.** Do not invent experimental scores, datasets, sample sizes, ablations, benchmark tables, charts, image captions, or figures. If evidence is missing, write `No results are available yet` or `TODO: run experiment` rather than producing plausible-looking data. ## Output structure @@ -36,9 +37,10 @@ Unresolved issues, disagreements between sources, gaps in evidence. ## Visuals - When the research contains quantitative data (benchmarks, comparisons, trends over time), generate charts using the `pi-charts` package to embed them in the draft. -- When explaining architectures, pipelines, or multi-step processes, use Mermaid diagrams. -- When a comparison across multiple dimensions would benefit from an interactive view, use `pi-generative-ui`. -- Every visual must have a descriptive caption and reference the data it's based on. +- Do not create charts from invented or example data. If values are missing, describe the planned measurement instead. +- When explaining architectures, pipelines, or multi-step processes, use Mermaid diagrams only when the structure is supported by the supplied evidence. +- When a comparison across multiple dimensions would benefit from an interactive view, use `pi-generative-ui` only for source-backed data. +- Every visual must have a descriptive caption and reference the data, source URL, research file, raw artifact, or script it is based on. - Do not add visuals for decoration — only when they materially improve understanding of the evidence. ## Operating rules @@ -48,6 +50,7 @@ Unresolved issues, disagreements between sources, gaps in evidence. - Do NOT add inline citations — the verifier agent handles that as a separate post-processing step. - Do NOT add a Sources section — the verifier agent builds that. - Before finishing, do a claim sweep: every strong factual statement in the draft should have an obvious source home in the research files. +- Before finishing, do a fake-result sweep: remove or replace any numeric result, figure, chart, benchmark, table, or image that lacks explicit provenance. ## Output contract - Save the main artifact to the specified output path (default: `draft.md`). diff --git a/prompts/draft.md b/prompts/draft.md index 0c64cc5..d739b0d 100644 --- a/prompts/draft.md +++ b/prompts/draft.md @@ -12,8 +12,14 @@ Requirements: - Before writing, outline the draft structure: proposed title, sections, key claims to make, source material to draw from, and a verification log for the critical claims, figures, and calculations. Write the outline to `outputs/.plans/.md`. Present the outline to the user. If this is an unattended or one-shot run, continue automatically. If the user is actively interacting, give them a brief chance to request changes before proceeding. - Use the `writer` subagent when the draft should be produced from already-collected notes, then use the `verifier` subagent to add inline citations and verify sources. - Include at minimum: title, abstract, problem statement, related work, method or synthesis, evidence or experiments, limitations, conclusion. +- **Never invent experimental results, scores, figures, images, charts, tables, datasets, or benchmarks.** If no raw artifact, cited source, or prior research note provides the value, write a clearly labeled placeholder such as `TODO: run experiment` or `No experimental results are available yet` instead of fabricating plausible numbers. +- The `evidence or experiments` section must contain only one of: + - cited results from primary sources, + - results computed from explicit raw artifacts/scripts already present in the workspace, + - a proposed experimental plan with no claimed outcomes. +- Every figure, chart, image, or table must have provenance in its caption: source URL, research-file reference, raw artifact path, or script path. If provenance is missing, omit the figure. - Use clean Markdown with LaTeX where equations materially help. -- Generate charts with `pi-charts` for quantitative data, benchmarks, and comparisons. Use Mermaid for architectures and pipelines. Every figure needs a caption. +- Generate charts with `pi-charts` only for quantitative data, benchmarks, and comparisons that already exist in the source material or raw artifacts. Use Mermaid for architectures and pipelines only when the structure is supported by sources. Every figure needs a provenance-bearing caption. - Before delivery, sweep the draft for any claim that sounds stronger than its support. Mark tentative results as tentative and remove unsupported numerics instead of letting the verifier discover them later. - Save exactly one draft to `papers/.md`. - End with a `Sources` appendix with direct URLs for all primary references. diff --git a/tests/content-policy.test.ts b/tests/content-policy.test.ts index 806f591..cf24a54 100644 --- a/tests/content-policy.test.ts +++ b/tests/content-policy.test.ts @@ -30,3 +30,19 @@ test("bundled prompts and skills do not contain blocked promotional product cont } } }); + +test("draft workflow explicitly forbids fabricated results and unproven figures", () => { + const draftPrompt = readFileSync(join(repoRoot, "prompts", "draft.md"), "utf8"); + const writerPrompt = readFileSync(join(repoRoot, ".feynman", "agents", "writer.md"), "utf8"); + const verifierPrompt = readFileSync(join(repoRoot, ".feynman", "agents", "verifier.md"), "utf8"); + + for (const [label, content] of [ + ["draft prompt", draftPrompt], + ["writer prompt", writerPrompt], + ["verifier prompt", verifierPrompt], + ] as const) { + assert.match(content, /Never (invent|fabricate)/i, `${label} must explicitly forbid invented or fabricated results`); + assert.match(content, /(figure|chart|image|table)/i, `${label} must cover visual/table provenance`); + assert.match(content, /(provenance|source|artifact|script|raw)/i, `${label} must require traceable support`); + } +}); diff --git a/website/src/content/docs/workflows/draft.md b/website/src/content/docs/workflows/draft.md index 25632a4..1115c46 100644 --- a/website/src/content/docs/workflows/draft.md +++ b/website/src/content/docs/workflows/draft.md @@ -35,6 +35,8 @@ When working from existing session context (after a deep research or literature The writer pays attention to academic conventions: claims are attributed to their sources with inline citations, methodology sections describe procedures precisely, and limitations are discussed honestly. The draft includes placeholder sections for any content the writer cannot generate from available sources, clearly marking what needs human input. +The draft workflow must not invent experimental results, scores, figures, images, tables, or benchmark data. When no source material or raw artifact supports a result, Feynman should leave a clearly labeled placeholder such as `No experimental results are available yet` or `TODO: run experiment` instead of producing plausible-looking data. + ## Output format The draft follows standard academic structure: