Files
feynman/tests/model-harness.test.ts
Advait Paliwal f5570b4e5a Rename .pi to .feynman, rename citation agent to verifier, add website, skills, and docs
- Rename project config dir from .pi/ to .feynman/ (Pi supports this via piConfig.configDir)
- Rename citation agent to verifier across all prompts, agents, skills, and docs
- Add website with homepage and 24 doc pages (Astro + Tailwind)
- Add skills for all workflows (deep-research, lit, review, audit, replicate, compare, draft, autoresearch, watch, jobs, session-log, agentcomputer)
- Add Pi-native prompt frontmatter (args, section, topLevelCli) and read at runtime
- Remove sync-docs generation layer — docs are standalone
- Remove metadata/prompts.mjs and metadata/packages.mjs — not needed at runtime
- Rewrite README and homepage copy
- Add environment selection to /replicate before executing
- Add prompts/delegate.md and AGENTS.md

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-23 17:35:35 -07:00

69 lines
2.8 KiB
TypeScript

import test from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { resolveInitialPrompt } from "../src/cli.js";
import { buildModelStatusSnapshotFromRecords, chooseRecommendedModel } from "../src/model/catalog.js";
import { setDefaultModelSpec } from "../src/model/commands.js";
function createAuthPath(contents: Record<string, unknown>): string {
const root = mkdtempSync(join(tmpdir(), "feynman-auth-"));
const authPath = join(root, "auth.json");
writeFileSync(authPath, JSON.stringify(contents, null, 2) + "\n", "utf8");
return authPath;
}
test("chooseRecommendedModel prefers the strongest authenticated research model", () => {
const authPath = createAuthPath({
openai: { type: "api_key", key: "openai-test-key" },
anthropic: { type: "api_key", key: "anthropic-test-key" },
});
const recommendation = chooseRecommendedModel(authPath);
assert.equal(recommendation?.spec, "anthropic/claude-opus-4-6");
});
test("setDefaultModelSpec accepts a unique bare model id from authenticated models", () => {
const authPath = createAuthPath({
openai: { type: "api_key", key: "openai-test-key" },
});
const settingsPath = join(mkdtempSync(join(tmpdir(), "feynman-settings-")), "settings.json");
setDefaultModelSpec(settingsPath, authPath, "gpt-5.4");
const settings = JSON.parse(readFileSync(settingsPath, "utf8")) as {
defaultProvider?: string;
defaultModel?: string;
};
assert.equal(settings.defaultProvider, "openai");
assert.equal(settings.defaultModel, "gpt-5.4");
});
test("buildModelStatusSnapshotFromRecords flags an invalid current model and suggests a replacement", () => {
const snapshot = buildModelStatusSnapshotFromRecords(
[
{ provider: "anthropic", id: "claude-opus-4-6" },
{ provider: "openai", id: "gpt-5.4" },
],
[{ provider: "openai", id: "gpt-5.4" }],
"anthropic/claude-opus-4-6",
);
assert.equal(snapshot.currentValid, false);
assert.equal(snapshot.recommended, "openai/gpt-5.4");
assert.ok(snapshot.guidance.some((line) => line.includes("Configured default model is unavailable")));
});
test("resolveInitialPrompt maps top-level research commands to Pi slash workflows", () => {
const workflows = new Set(["lit", "watch", "jobs", "deepresearch"]);
assert.equal(resolveInitialPrompt("lit", ["tool-using", "agents"], undefined, workflows), "/lit tool-using agents");
assert.equal(resolveInitialPrompt("watch", ["openai"], undefined, workflows), "/watch openai");
assert.equal(resolveInitialPrompt("jobs", [], undefined, workflows), "/jobs");
assert.equal(resolveInitialPrompt("chat", ["hello"], undefined, workflows), "hello");
assert.equal(resolveInitialPrompt("unknown", ["topic"], undefined, workflows), "unknown topic");
});