Upgrade Feynman research runtime and setup

src/feynman-prompt.ts

@@ -1,6 +1,7 @@
-export const FEYNMAN_SYSTEM_PROMPT = `You are Feynman, a research-first AI agent.
+export function buildFeynmanSystemPrompt(): string {
+	return `You are Feynman, a research-first AI agent.
 
-Your job is to investigate questions, read primary sources, design experiments, run them when useful, and produce reproducible written artifacts.
+Your job is to investigate questions, read primary sources, compare evidence, design experiments when useful, and produce reproducible written artifacts.
 
 Operating rules:
 - Evidence over fluency.
@@ -9,18 +10,32 @@ Operating rules:
 - State uncertainty explicitly.
 - When a claim depends on recent literature or unstable facts, use tools before answering.
 - When discussing papers, cite title, year, and identifier or URL when possible.
-- Use the alpha-backed research tools first for literature search, paper reading, paper Q&A, and persistent annotations.
-- Use the installed Pi research packages for broader web/PDF access, document parsing, session recall, background processes, experiment tracking, citations, and delegated subtasks when they reduce friction.
+- Use the alpha-backed research tools for academic paper search, paper reading, paper Q&A, repository inspection, and persistent annotations.
+- Use \`web_search\`, \`fetch_content\`, and \`get_search_content\` first for current topics: products, companies, markets, regulations, software releases, model availability, model pricing, benchmarks, docs, or anything phrased as latest/current/recent/today.
+- For mixed topics, combine both: use web sources for current reality and paper sources for background literature.
+- Never answer a latest/current question from arXiv or alpha-backed paper search alone.
+- For AI model or product claims, prefer official docs/vendor pages plus recent web sources over old papers.
+- Use the installed Pi research packages for broader web/PDF access, document parsing, citation workflows, background processes, memory, session recall, and delegated subtasks when they reduce friction.
+- Use the visualization packages when a chart, diagram, or interactive widget would materially improve understanding. Prefer charts for quantitative comparisons, Mermaid for simple process/architecture diagrams, and interactive HTML widgets for exploratory visual explanations.
+- Persistent memory is package-backed. Use \`memory_search\` to recall prior preferences and lessons, \`memory_remember\` to store explicit durable facts, and \`memory_lessons\` when prior corrections matter.
+- If the user says "remember", states a stable preference, or asks for something to be the default in future sessions, call \`memory_remember\`. Do not just say you will remember it.
+- Session recall is package-backed. Use \`session_search\` when the user references prior work, asks what has been done before, or when you suspect relevant past context exists.
+- Feynman is intended to support always-on research work. Use the scheduling package when recurring or deferred work is appropriate instead of telling the user to remember manually.
+- Use \`schedule_prompt\` for recurring scans, delayed follow-ups, reminders, and periodic research jobs.
+- If the user asks you to remind, check later, run something nightly, or keep watching something over time, call \`schedule_prompt\`. Do not just promise to do it later.
+- Prefer the smallest investigation or experiment that can materially reduce uncertainty before escalating to broader work.
 - When an experiment is warranted, write the code or scripts, run them, capture outputs, and save artifacts to disk.
 - Treat polished scientific communication as part of the job: structure reports cleanly, use Markdown deliberately, and use LaTeX math when equations clarify the argument.
 - For any source-based answer, include an explicit Sources section with direct URLs, not just paper titles.
 - When citing papers from alpha-backed tools, prefer direct arXiv or alphaXiv links and include the arXiv ID.
+- After writing a polished artifact, use \`preview_file\` when the user wants to review it in a browser or PDF viewer.
+- Default toward delivering a concrete artifact when the task naturally calls for one: reading list, memo, audit, experiment log, or draft.
 - Default artifact locations:
   - outputs/ for reviews, reading lists, and summaries
   - experiments/ for runnable experiment code and result logs
   - notes/ for scratch notes and intermediate synthesis
   - papers/ for polished paper-style drafts and writeups
-- Default deliverables should include: summary, strongest evidence, disagreements or gaps, open questions, and recommended next steps.
+- Default deliverables should include: summary, strongest evidence, disagreements or gaps, open questions, recommended next steps, and links to the source material.
 
 Default workflow:
 1. Clarify the research objective if needed.
@@ -33,4 +48,6 @@ Default workflow:
 Style:
 - Concise, skeptical, and explicit.
 - Avoid fake certainty.
-- Do not present unverified claims as facts.`;
+- Do not present unverified claims as facts.
+- When greeting, introducing yourself, or answering "who are you", identify yourself explicitly as Feynman.`;
+}

src/index.ts

@@ -1,9 +1,11 @@
 import "dotenv/config";
 
-import { spawn } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { dirname, resolve } from "node:path";
+import { stdin as input, stdout as output } from "node:process";
+import { createInterface } from "node:readline/promises";
 import { parseArgs } from "node:util";
 import { fileURLToPath } from "node:url";
 
@@ -18,7 +20,7 @@ import {
 	AuthStorage,
 } from "@mariozechner/pi-coding-agent";
 
-import { FEYNMAN_SYSTEM_PROMPT } from "./feynman-prompt.js";
+import { buildFeynmanSystemPrompt } from "./feynman-prompt.js";
 
 type ThinkingLevel = "off" | "low" | "medium" | "high";
 
@@ -34,6 +36,8 @@ function printHelp(): void {
 	  /replicate <paper>        Expand the replication prompt template
 	  /reading-list <topic>     Expand the reading list prompt template
 	  /research-memo <topic>    Expand the general research memo prompt template
+	  /deepresearch <topic>     Expand the thorough source-heavy research prompt template
+	  /autoresearch <idea>      Expand the idea-to-paper autoresearch prompt template
 	  /compare-sources <topic>  Expand the source comparison prompt template
 	  /paper-code-audit <item>  Expand the paper/code audit prompt template
 	  /paper-draft <topic>      Expand the paper-style writing prompt template
@@ -46,7 +50,15 @@ function printHelp(): void {
   --model provider:model    Force a specific model
   --thinking level          off | low | medium | high
   --cwd /path/to/workdir    Working directory for tools
-  --session-dir /path       Session storage directory`);
+  --session-dir /path       Session storage directory
+  --doctor                  Check Feynman auth, models, preview tools, and paths
+  --setup-preview           Install preview dependencies when possible
+
+Top-level:
+  feynman setup             Configure alpha login, web search, and preview deps
+  feynman setup alpha       Configure alphaXiv login
+  feynman setup web         Configure web search provider
+  feynman setup preview     Install preview dependencies`);
 }
 
 function parseModelSpec(spec: string, modelRegistry: ModelRegistry) {
@@ -78,9 +90,32 @@ function normalizeThinkingLevel(value: string | undefined): ThinkingLevel | unde
 	return undefined;
 }
 
+function resolveExecutable(name: string, fallbackPaths: string[] = []): string | undefined {
+	for (const candidate of fallbackPaths) {
+		if (existsSync(candidate)) {
+			return candidate;
+		}
+	}
+
+	const result = spawnSync("sh", ["-lc", `command -v ${name}`], {
+		encoding: "utf8",
+		stdio: ["ignore", "pipe", "ignore"],
+	});
+
+	if (result.status === 0) {
+		const resolved = result.stdout.trim();
+		if (resolved) {
+			return resolved;
+		}
+	}
+
+	return undefined;
+}
+
 function patchEmbeddedPiBranding(piPackageRoot: string): void {
 	const packageJsonPath = resolve(piPackageRoot, "package.json");
 	const cliPath = resolve(piPackageRoot, "dist", "cli.js");
+	const interactiveModePath = resolve(piPackageRoot, "dist", "modes", "interactive", "interactive-mode.js");
 
 	if (existsSync(packageJsonPath)) {
 		const pkg = JSON.parse(readFileSync(packageJsonPath, "utf8")) as {
@@ -101,6 +136,77 @@ function patchEmbeddedPiBranding(piPackageRoot: string): void {
 			writeFileSync(cliPath, cliSource.replace('process.title = "pi";', 'process.title = "feynman";'), "utf8");
 		}
 	}
+
+	if (existsSync(interactiveModePath)) {
+		const interactiveModeSource = readFileSync(interactiveModePath, "utf8");
+		if (interactiveModeSource.includes("`π - ${sessionName} - ${cwdBasename}`")) {
+			writeFileSync(
+				interactiveModePath,
+				interactiveModeSource
+					.replace("`π - ${sessionName} - ${cwdBasename}`", "`feynman - ${sessionName} - ${cwdBasename}`")
+					.replace("`π - ${cwdBasename}`", "`feynman - ${cwdBasename}`"),
+				"utf8",
+			);
+		}
+	}
+}
+
+function patchPackageWorkspace(appRoot: string): void {
+	const workspaceRoot = resolve(appRoot, ".pi", "npm", "node_modules");
+	const webAccessPath = resolve(workspaceRoot, "pi-web-access", "index.ts");
+	const sessionSearchIndexerPath = resolve(
+		workspaceRoot,
+		"@kaiserlich-dev",
+		"pi-session-search",
+		"extensions",
+		"indexer.ts",
+	);
+	const piMemoryPath = resolve(workspaceRoot, "@samfp", "pi-memory", "src", "index.ts");
+
+	if (existsSync(webAccessPath)) {
+		const source = readFileSync(webAccessPath, "utf8");
+		if (source.includes('pi.registerCommand("search",')) {
+			writeFileSync(
+				webAccessPath,
+				source.replace('pi.registerCommand("search",', 'pi.registerCommand("web-results",'),
+				"utf8",
+			);
+		}
+	}
+
+	if (existsSync(sessionSearchIndexerPath)) {
+		const source = readFileSync(sessionSearchIndexerPath, "utf8");
+		const original = 'const sessionsDir = path.join(os.homedir(), ".pi", "agent", "sessions");';
+		const replacement =
+			'const sessionsDir = process.env.FEYNMAN_SESSION_DIR ?? process.env.PI_SESSION_DIR ?? path.join(os.homedir(), ".pi", "agent", "sessions");';
+		if (source.includes(original)) {
+			writeFileSync(sessionSearchIndexerPath, source.replace(original, replacement), "utf8");
+		}
+	}
+
+	if (existsSync(piMemoryPath)) {
+		let source = readFileSync(piMemoryPath, "utf8");
+		const memoryOriginal = 'const MEMORY_DIR = join(homedir(), ".pi", "memory");';
+		const memoryReplacement =
+			'const MEMORY_DIR = process.env.FEYNMAN_MEMORY_DIR ?? process.env.PI_MEMORY_DIR ?? join(homedir(), ".pi", "memory");';
+		if (source.includes(memoryOriginal)) {
+			source = source.replace(memoryOriginal, memoryReplacement);
+		}
+
+		const execOriginal = 'const result = await pi.exec("pi", ["-p", prompt, "--print"], {';
+		const execReplacement = [
+			'const execBinary = process.env.FEYNMAN_NODE_EXECUTABLE || process.env.FEYNMAN_EXECUTABLE || "pi";',
+			'      const execArgs = process.env.FEYNMAN_BIN_PATH',
+			'        ? [process.env.FEYNMAN_BIN_PATH, "--prompt", prompt]',
+			'        : ["-p", prompt, "--print"];',
+			'      const result = await pi.exec(execBinary, execArgs, {',
+		].join("\n");
+		if (source.includes(execOriginal)) {
+			source = source.replace(execOriginal, execReplacement);
+		}
+
+		writeFileSync(piMemoryPath, source, "utf8");
+	}
 }
 
 function choosePreferredModel(
@@ -149,12 +255,6 @@ function normalizeFeynmanSettings(
 		}
 	}
 
-	if (Array.isArray(settings.packages)) {
-		settings.packages = settings.packages.filter(
-			(entry) => entry !== "npm:@kaiserlich-dev/pi-session-search",
-		);
-	}
-
 	if (!settings.defaultThinkingLevel) {
 		settings.defaultThinkingLevel = defaultThinkingLevel;
 	}
@@ -180,6 +280,217 @@ function normalizeFeynmanSettings(
 	writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + "\n", "utf8");
 }
 
+function readJson(path: string): Record<string, unknown> {
+	if (!existsSync(path)) {
+		return {};
+	}
+
+	try {
+		return JSON.parse(readFileSync(path, "utf8"));
+	} catch {
+		return {};
+	}
+}
+
+function getWebSearchConfigPath(): string {
+	return resolve(homedir(), ".pi", "web-search.json");
+}
+
+function loadWebSearchConfig(): Record<string, unknown> {
+	return readJson(getWebSearchConfigPath());
+}
+
+function saveWebSearchConfig(config: Record<string, unknown>): void {
+	const path = getWebSearchConfigPath();
+	mkdirSync(dirname(path), { recursive: true });
+	writeFileSync(path, JSON.stringify(config, null, 2) + "\n", "utf8");
+}
+
+function hasConfiguredWebProvider(): boolean {
+	const config = loadWebSearchConfig();
+	return typeof config.perplexityApiKey === "string" && config.perplexityApiKey.trim().length > 0
+		|| typeof config.geminiApiKey === "string" && config.geminiApiKey.trim().length > 0;
+}
+
+async function promptText(question: string, defaultValue = ""): Promise<string> {
+	if (!input.isTTY || !output.isTTY) {
+		throw new Error("feynman setup requires an interactive terminal.");
+	}
+	const rl = createInterface({ input, output });
+	try {
+		const suffix = defaultValue ? ` [${defaultValue}]` : "";
+		const value = (await rl.question(`${question}${suffix}: `)).trim();
+		return value || defaultValue;
+	} finally {
+		rl.close();
+	}
+}
+
+async function promptChoice(question: string, choices: string[], defaultIndex = 0): Promise<number> {
+	console.log(question);
+	for (const [index, choice] of choices.entries()) {
+		const marker = index === defaultIndex ? "*" : " ";
+		console.log(`  ${marker} ${index + 1}. ${choice}`);
+	}
+	const answer = await promptText("Select", String(defaultIndex + 1));
+	const parsed = Number(answer);
+	if (!Number.isFinite(parsed) || parsed < 1 || parsed > choices.length) {
+		return defaultIndex;
+	}
+	return parsed - 1;
+}
+
+async function setupWebProvider(): Promise<void> {
+	const config = loadWebSearchConfig();
+	const choices = [
+		"Gemini API key",
+		"Perplexity API key",
+		"Browser Gemini (manual sign-in only)",
+		"Skip",
+	];
+	const selection = await promptChoice("Choose a web search provider for Feynman:", choices, hasConfiguredWebProvider() ? 3 : 0);
+
+	if (selection === 0) {
+		const key = await promptText("Gemini API key");
+		if (key) {
+			config.geminiApiKey = key;
+			delete config.perplexityApiKey;
+			saveWebSearchConfig(config);
+			console.log("Saved Gemini API key to ~/.pi/web-search.json");
+		}
+		return;
+	}
+
+	if (selection === 1) {
+		const key = await promptText("Perplexity API key");
+		if (key) {
+			config.perplexityApiKey = key;
+			delete config.geminiApiKey;
+			saveWebSearchConfig(config);
+			console.log("Saved Perplexity API key to ~/.pi/web-search.json");
+		}
+		return;
+	}
+
+	if (selection === 2) {
+		console.log("Sign into gemini.google.com in Chrome, Chromium, Brave, or Edge, then restart Feynman.");
+		return;
+	}
+}
+
+async function runSetup(
+	section: string | undefined,
+	settingsPath: string,
+	bundledSettingsPath: string,
+	authPath: string,
+	workingDir: string,
+	sessionDir: string,
+): Promise<void> {
+	if (section === "alpha" || !section) {
+		if (!isAlphaLoggedIn()) {
+			await loginAlpha();
+			console.log("alphaXiv login complete");
+		} else {
+			console.log("alphaXiv login already configured");
+		}
+		if (section === "alpha") return;
+	}
+
+	if (section === "web" || !section) {
+		await setupWebProvider();
+		if (section === "web") return;
+	}
+
+	if (section === "preview" || !section) {
+		setupPreviewDependencies();
+		if (section === "preview") return;
+	}
+
+	normalizeFeynmanSettings(settingsPath, bundledSettingsPath, "medium", authPath);
+	runDoctor(settingsPath, authPath, sessionDir, workingDir);
+}
+
+function runDoctor(
+	settingsPath: string,
+	authPath: string,
+	sessionDir: string,
+	workingDir: string,
+): void {
+	const settings = readJson(settingsPath);
+	const modelRegistry = new ModelRegistry(AuthStorage.create(authPath));
+	const availableModels = modelRegistry.getAvailable();
+	const pandocPath = resolveExecutable("pandoc", [
+		"/opt/homebrew/bin/pandoc",
+		"/usr/local/bin/pandoc",
+	]);
+	const browserPath =
+		process.env.PUPPETEER_EXECUTABLE_PATH ??
+		resolveExecutable("google-chrome", [
+			"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+			"/Applications/Chromium.app/Contents/MacOS/Chromium",
+			"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
+			"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
+		]);
+
+	console.log("Feynman doctor");
+	console.log("");
+	console.log(`working dir: ${workingDir}`);
+	console.log(`session dir: ${sessionDir}`);
+	console.log("");
+	console.log(`alphaXiv auth: ${isAlphaLoggedIn() ? "ok" : "missing"}`);
+	if (isAlphaLoggedIn()) {
+		const name = getAlphaUserName();
+		if (name) {
+			console.log(`  user: ${name}`);
+		}
+	}
+	console.log(`models available: ${availableModels.length}`);
+	if (availableModels.length > 0) {
+		const sample = availableModels
+			.slice(0, 6)
+			.map((model) => `${model.provider}/${model.id}`)
+			.join(", ");
+		console.log(`  sample: ${sample}`);
+	}
+	console.log(
+		`default model: ${typeof settings.defaultProvider === "string" && typeof settings.defaultModel === "string"
+			? `${settings.defaultProvider}/${settings.defaultModel}`
+			: "not set"}`,
+	);
+	console.log(`pandoc: ${pandocPath ?? "missing"}`);
+	console.log(`browser preview runtime: ${browserPath ?? "missing"}`);
+	console.log(`web research provider: ${hasConfiguredWebProvider() ? "configured" : "missing"}`);
+	console.log(`quiet startup: ${settings.quietStartup === true ? "enabled" : "disabled"}`);
+	console.log(`theme: ${typeof settings.theme === "string" ? settings.theme : "not set"}`);
+	console.log(`setup hint: feynman setup`);
+}
+
+function setupPreviewDependencies(): void {
+	const pandocPath = resolveExecutable("pandoc", [
+		"/opt/homebrew/bin/pandoc",
+		"/usr/local/bin/pandoc",
+	]);
+	if (pandocPath) {
+		console.log(`pandoc already installed at ${pandocPath}`);
+		return;
+	}
+
+	const brewPath = resolveExecutable("brew", [
+		"/opt/homebrew/bin/brew",
+		"/usr/local/bin/brew",
+	]);
+	if (process.platform === "darwin" && brewPath) {
+		const result = spawnSync(brewPath, ["install", "pandoc"], { stdio: "inherit" });
+		if (result.status !== 0) {
+			throw new Error("Failed to install pandoc via Homebrew.");
+		}
+		console.log("Preview dependency installed: pandoc");
+		return;
+	}
+
+	throw new Error("Automatic preview setup is only supported on macOS with Homebrew right now.");
+}
+
 function syncFeynmanTheme(appRoot: string, agentDir: string): void {
 	const sourceThemePath = resolve(appRoot, ".pi", "themes", "feynman.json");
 	const targetThemeDir = resolve(agentDir, "themes");
@@ -201,11 +512,13 @@ async function main(): Promise<void> {
 	const feynmanAgentDir = resolve(homedir(), ".feynman", "agent");
 	const bundledSettingsPath = resolve(appRoot, ".pi", "settings.json");
 	patchEmbeddedPiBranding(piPackageRoot);
+	patchPackageWorkspace(appRoot);
 
 	const { values, positionals } = parseArgs({
 		allowPositionals: true,
 		options: {
 			cwd: { type: "string" },
+			doctor: { type: "boolean" },
 			help: { type: "boolean" },
 			"alpha-login": { type: "boolean" },
 			"alpha-logout": { type: "boolean" },
@@ -214,6 +527,7 @@ async function main(): Promise<void> {
 			"new-session": { type: "boolean" },
 			prompt: { type: "string" },
 			"session-dir": { type: "string" },
+			"setup-preview": { type: "boolean" },
 			thinking: { type: "string" },
 		},
 	});
@@ -233,6 +547,21 @@ async function main(): Promise<void> {
 	const thinkingLevel = normalizeThinkingLevel(values.thinking ?? process.env.FEYNMAN_THINKING) ?? "medium";
 	normalizeFeynmanSettings(feynmanSettingsPath, bundledSettingsPath, thinkingLevel, feynmanAuthPath);
 
+	if (positionals[0] === "setup") {
+		await runSetup(positionals[1], feynmanSettingsPath, bundledSettingsPath, feynmanAuthPath, workingDir, sessionDir);
+		return;
+	}
+
+	if (values.doctor) {
+		runDoctor(feynmanSettingsPath, feynmanAuthPath, sessionDir, workingDir);
+		return;
+	}
+
+	if (values["setup-preview"]) {
+		setupPreviewDependencies();
+		return;
+	}
+
 	if (values["alpha-login"]) {
 		const result = await loginAlpha();
 		normalizeFeynmanSettings(feynmanSettingsPath, bundledSettingsPath, thinkingLevel, feynmanAuthPath);
@@ -273,6 +602,7 @@ async function main(): Promise<void> {
 	}
 	const oneShotPrompt = values.prompt;
 	const initialPrompt = oneShotPrompt ?? (positionals.length > 0 ? positionals.join(" ") : undefined);
+	const systemPrompt = buildFeynmanSystemPrompt();
 
 	const piArgs = [
 		"--session-dir",
@@ -284,7 +614,7 @@ async function main(): Promise<void> {
 		"--prompt-template",
 		resolve(appRoot, "prompts"),
 		"--system-prompt",
-		FEYNMAN_SYSTEM_PROMPT,
+		systemPrompt,
 	];
 
 	if (explicitModelSpec) {
@@ -307,6 +637,33 @@ async function main(): Promise<void> {
 			...process.env,
 			PI_CODING_AGENT_DIR: feynmanAgentDir,
 			FEYNMAN_CODING_AGENT_DIR: feynmanAgentDir,
+			FEYNMAN_PI_NPM_ROOT: resolve(appRoot, ".pi", "npm", "node_modules"),
+			FEYNMAN_SESSION_DIR: sessionDir,
+			PI_SESSION_DIR: sessionDir,
+			FEYNMAN_MEMORY_DIR: resolve(dirname(feynmanAgentDir), "memory"),
+			FEYNMAN_NODE_EXECUTABLE: process.execPath,
+			FEYNMAN_BIN_PATH: resolve(appRoot, "bin", "feynman.js"),
+			PANDOC_PATH:
+				process.env.PANDOC_PATH ??
+				resolveExecutable("pandoc", [
+					"/opt/homebrew/bin/pandoc",
+					"/usr/local/bin/pandoc",
+				]),
+			PI_SKIP_VERSION_CHECK: process.env.PI_SKIP_VERSION_CHECK ?? "1",
+			MERMAID_CLI_PATH:
+				process.env.MERMAID_CLI_PATH ??
+				resolveExecutable("mmdc", [
+					"/opt/homebrew/bin/mmdc",
+					"/usr/local/bin/mmdc",
+				]),
+			PUPPETEER_EXECUTABLE_PATH:
+				process.env.PUPPETEER_EXECUTABLE_PATH ??
+				resolveExecutable("google-chrome", [
+					"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+					"/Applications/Chromium.app/Contents/MacOS/Chromium",
+					"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
+					"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
+				]),
 		},
 	});