diff --git a/build.py b/build.py index f11a3b1..157f5f8 100755 --- a/build.py +++ b/build.py @@ -349,6 +349,25 @@ DEFAULT_SKILL_PERSONA_MAP = { # Infrastructure skills → personas "docker-essentials": ["architect"], "session-logs": ["architect"], + # Agent tooling / dev-environment skills → personas + "opencode-cli": ["forge", "architect"], + "feynman-cli": ["scholar", "forge", "oracle"], + "anythingllm-manager": ["architect", "forge"], + "jira": ["architect", "forge"], + "obsidian-tasks": ["architect", "scribe"], + "vercel-react-best-practices": ["forge"], + "ui-ux-pro-max": ["forge"], + # Document / archival → personas + "foia-tool": ["scribe", "scholar"], + "librarian": ["scribe", "scholar"], + "notebooklm": ["scholar", "oracle", "scribe"], + # Intelligence / strategic analysis → personas + "intel-briefing": ["frodo", "oracle", "sentinel"], + "waha-whatsapp": ["oracle", "frodo"], + # Reporting / pentest workflow → personas + "pentest-reporter": ["neo", "phantom", "bastion"], + # Marketing / business → personas + "marketing-strategist": ["herald", "ledger"], # Document processing → personas "image-ocr": ["oracle", "scribe"], "mistral-ocr": ["oracle", "scribe"], @@ -1472,12 +1491,12 @@ def _classify_skill_topic(name: str, fm: dict) -> str: ("ops-sysadmin", r"^(session-log|session-search|preview|watch|jobs|contributing)$"), ("coding-frontend", r"^(react|nextjs|next-|angular|vue-|svelte|tailwind|shadcn|vercel|expo|remotion|frontend|ui-ux|accessibility|canvas-|stitch|framer)"), ("coding-backend", r"^(python|java-|csharp|dotnet|aspnet|kotlin|swift|rust-|golang|go-|ruby-|php-|nodejs|node-|bash-|cli-|bazel|async-|architecting-|aspire-)"), - ("coding-tools", r"^(commit|changelog|debug-|refactor|test-driven|tdd|bdd|git-|github-|gitlab-|bats|copilot|codeql|code-review|linting|formatting|add-|adr-|agent-browser|mcp-)"), - ("ai-llm-dev", r"^(ai-|agentic|claude-|mcp|openai|anthropic|llm|rag-|embedding|fine-tun|prompt|anythingllm|olla|huggingface|elevenlabs|crawl-for-ai|agent-tools|agent-ui|agent-governance|para-memory|knowledge-hub)"), + ("coding-tools", r"^(commit|changelog|debug-|refactor|test-driven|tdd|bdd|git-|github-|gitlab-|bats|copilot|codeql|code-review|linting|formatting|add-|adr-|agent-browser|mcp-|opencode|jira)"), + ("ai-llm-dev", r"^(ai-|agentic|claude-|mcp|openai|anthropic|llm|rag-|embedding|fine-tun|prompt|anythingllm|olla|notebooklm|feynman-|huggingface|elevenlabs|crawl-for-ai|agent-tools|agent-ui|agent-governance|para-memory|knowledge-hub)"), ("cloud-infra", r"^(aws|azure|gcp|kubernetes|docker|terraform|cloudflare|vercel|netlify|supabase|firebase|k8s|iac|devops|cicd|ansible|helm|bigquery|airflow|az-)"), ("database", r"^(sql-|postgres|mysql|mongodb|redis)"), ("browser-scrape", r"^(browser|playwright|puppeteer|firecrawl|stealth|scrape|crawl|use-my-browser)"), - ("osint-intel", r"^(osint|recon|intel-|foia|seithar|deep-scraper|stealth-browser|social-trust|news-crawler|proudguard|gov-cyber|tavily|session-logs|youtube-transcript)"), + ("osint-intel", r"^(osint|recon|intel-|foia|seithar|deep-scraper|stealth-browser|social-trust|news-crawler|proudguard|gov-cyber|tavily|session-logs|youtube-transcript|waha-)"), ("marketing-content", r"^(copywriting|content-|seo-|blog-|article-|marketing-|ad-(creative|campaign)|brand-|banner|churn|billing|gtm-|competitive|backlink|boost|twitter|ai-social|ai-marketing|ai-content|ai-podcast|ai-music|ai-avatar|ai-automation|ai-image|ai-video|impeccable)"), ("ops-sysadmin", r"^(healthcheck|sysadmin|dns-networking|network-|nmap-|pcap-|tmux|freshrss|obsidian-|librarian|pdf-|image-ocr|mistral-ocr|analyze|weather|node-connect|clawflow|skill-creator|devops-engineer)"), ("business-pm", r"^(ceo-|cfo-|product-manager|marketing-strategist|marketing-psychology|qa-testing|design-md|persona-customer|product-|gtm-|arize|dataverse|power-|microsoft-)"), diff --git a/personas/_shared/community-skills/anythingllm-manager/SKILL.md b/personas/_shared/community-skills/anythingllm-manager/SKILL.md new file mode 100644 index 0000000..36b2eda --- /dev/null +++ b/personas/_shared/community-skills/anythingllm-manager/SKILL.md @@ -0,0 +1,115 @@ +--- +name: anythingllm-manager +description: Manage AnythingLLM persona workspaces, document uploads, vector re-embedding, and recovery. Use when the user mentions AnythingLLM, persona workspaces, vector DB recovery, document embedding, RAG workspace management, or needs to re-assign documents to persona workspaces. +--- + +# AnythingLLM Persona Manager + +## Overview + +Manages a 29-persona RAG system in AnythingLLM Desktop with workspace creation, document upload/OCR, and vector embedding assignment via the AnythingLLM REST API. Library: 39,754 files (67 GB) across 88 mapped paths. + +## Architecture + +- **AnythingLLM API:** `http://localhost:3001/api/v1` +- **API Key:** `SXQGXH3-AQ64B8E-KQNMDWC-WZBQAFW` +- **Integration scripts:** `~/.config/anythingllm-desktop/integration/` +- **Config:** `~/.config/anythingllm-desktop/integration/config.yaml` +- **Progress tracker:** `~/.config/anythingllm-desktop/integration/upload_progress.json` +- **Book library:** `/mnt/storage/Common/Books/` (39,754 files, 67 GB) +- **Vector DB:** LanceDB at `~/.config/anythingllm-desktop/storage/lancedb/` +- **Git repo:** `https:////anything-llm-rag.git` + +## 5 Persona Clusters (29 total) + +| Cluster | Personas | +|---------|----------| +| intel | frodo, echo, ghost, oracle, wraith, scribe, polyglot | +| cyber | neo, bastion, sentinel, specter, phantom, cipher, vortex | +| military | marshal, centurion, corsair, warden, medic | +| humanities | chronos, tribune, arbiter, ledger, sage, herald, scholar, gambit | +| engineering | forge, architect | + +## Common Operations + +### Check Status +```bash +cd ~/.config/anythingllm-desktop/integration +python3 setup.py --status +``` + +### Create/Update Workspaces with Persona Prompts +```bash +python3 setup.py --create-workspaces +python3 setup.py --create-workspaces --persona frodo +``` + +### Full Upload Pipeline (upload + OCR + embed) +```bash +python3 setup.py --upload-documents --resume +python3 setup.py --upload-documents --cluster cyber --resume +python3 setup.py --upload-documents --persona neo --priority 1 --resume +``` + +### Re-assign (fast vector recovery — no disk scan) +```bash +# Preview +python3 setup.py --reassign --reset --dry-run + +# Reset + re-embed all workspaces +python3 setup.py --reassign --reset + +# Single persona or cluster +python3 setup.py --reassign --reset --persona frodo +python3 setup.py --reassign --reset --cluster intel + +# Only fill missing assignments (no reset) +python3 setup.py --reassign +``` + +### Dry Run (preview without changes) +```bash +python3 setup.py --upload-documents --dry-run +``` + +## Recovery Procedures + +### Vector DB Deleted (documents still imported) +```bash +python3 setup.py --reassign --reset +``` + +### Single Persona Vector Recovery +```bash +python3 setup.py --reassign --reset --persona frodo +``` + +### Full Reset (everything from scratch) +```bash +rm upload_progress.json +python3 setup.py --all +``` + +## API Quick Reference + +Check API health: +```bash +curl -s http://localhost:3001/api/v1/auth -H "Authorization: Bearer SXQGXH3-AQ64B8E-KQNMDWC-WZBQAFW" +``` + +List workspaces: +```bash +curl -s http://localhost:3001/api/v1/workspaces -H "Authorization: Bearer SXQGXH3-AQ64B8E-KQNMDWC-WZBQAFW" +``` + +## Important Notes + +- Library: 39,754 files, 67 GB, 88 mapped paths, 0 unmapped content folders +- Batch processing: 50 files/batch, 5s delay to avoid rate limits +- OCR: ocrmypdf with Turkish+English support +- `direct-uploads/` symlinked to `/mnt/storage/anythingllm/direct-uploads/` (SSD savings) +- Embedding: Google Gemini (`gemini-embedding-001`) +- LLM: Ollama local (`qwen3:14b`) +- `--reassign` skips disk scan — uses upload_progress.json directly for fast recovery +- `--resume` continues interrupted upload operations +- Skipped extensions: .bin .gz .zip .html .php .jpg .pptx .ppt .doc .json .log .py .djvu .mobi .azw3 diff --git a/personas/_shared/community-skills/feynman-cli/SKILL.md b/personas/_shared/community-skills/feynman-cli/SKILL.md new file mode 100644 index 0000000..11348a4 --- /dev/null +++ b/personas/_shared/community-skills/feynman-cli/SKILL.md @@ -0,0 +1,504 @@ +--- +name: feynman-cli +description: Install, configure, and operate the Feynman research-assistant CLI by Companion, Inc. Covers one-line installer, npm install, Windows PowerShell, feynman setup wizard, provider auth (Anthropic/OpenAI/Bedrock/Gemini/local Ollama+LM Studio+LiteLLM), ~/.feynman/settings.json, model selection + per-subagent overrides, thinking levels, env vars (FEYNMAN_MODEL/FEYNMAN_HOME/FEYNMAN_THINKING/TAVILY_API_KEY/SERPER_API_KEY), feynman doctor, every CLI subcommand and flag, REPL slash commands (/deepresearch /lit /review /audit /replicate /compare /draft /autoresearch /watch /log /jobs /init /outputs /search /preview /feynman-model /help), the four built-in agents (researcher/reviewer/writer/verifier), the four tools (alphaxiv/web-search/session-search/preview), 11 core packages + generative-ui, and version pinning + update + uninstall. Use when the user asks to install Feynman, run feynman setup, change the default model, authenticate alphaXiv, configure web search routing, troubleshoot Pi runtime, run a one-shot prompt, schedule a /watch, or any feynman.is question. +license: MIT +compatibility: claude-code, opencode +metadata: + audience: researchers, scientists, engineers + source: https://www.feynman.is/docs + vendor: Companion, Inc. +--- + +# Feynman CLI — Install, Configure, Operate + +Reference for **using** the Feynman research CLI. This skill covers the tool +itself; the nine research **workflows** (`/deepresearch`, `/lit`, `/review`, +`/audit`, `/replicate`, `/compare`, `/draft`, `/autoresearch`, `/watch`) each +have their own dedicated skill in this repo's `feynman-skills/` set — see the +"Workflows index" below. + +> Skill spec: `name` matches `^[a-z0-9]+(-[a-z0-9]+)*$` (≤64). `description` +> ≤1024. + +## References (load on demand) + +- `references/installation-setup.md` — every install path, prerequisites, post-install, doctor, update/uninstall, version pinning +- `references/configuration.md` — real `~/.feynman/` layout (verified from working install), `agent/settings.json`, env vars, thinking levels, session dirs, web-search routing +- `references/cli-reference.md` — every `feynman ` with flags + examples +- `references/repl-and-slash.md` — REPL behavior, slash commands, file-ref + shell expansion +- `references/agents-tools-packages.md` — 4 built-in agents, 4 tools (alphaxiv/web-search/session-search/preview), 11 core + 1 optional package +- `references/custom-agents.md` — full pi-subagents agent frontmatter spec, scope priority (builtin < user < project), `agentOverrides` in `settings.json`, prompt-assembly modes, MCP tool selection + +## What is it + +Feynman is an AI research-assistant CLI by **Companion, Inc.** It dispatches +specialized subagents (researcher, reviewer, writer, verifier) to run +deep-research, literature review, peer review, code audit, replication, +source comparison, draft writing, and autonomous research workflows. The +runtime is built on the Pi package stack (`pi-subagents`, `pi-docparser`, +`pi-web-access`, `pi-markdown-preview`, `pi-charts`, `pi-zotero`, +`pi-schedule-prompt`, etc.). + +GitHub: not directly linked; vendor site: . + +## Install (cheatsheet) + +```bash +# macOS / Linux — recommended +curl -fsSL https://feynman.is/install | bash + +# Pin a specific version +curl -fsSL https://feynman.is/install | bash -s -- 0.2.39 + +# Skills only (no runtime) +curl -fsSL https://feynman.is/install-skills | bash + +# Windows PowerShell (as Administrator) +irm https://feynman.is/install.ps1 | iex + +# npm (cross-platform; needs Node ≥20.19.0 <25) +npm install -g @companion-ai/feynman +``` + +Install locations: + +| Layer | macOS / Linux | Windows | +| ---------------- | -------------------------------- | ------------------------------------ | +| Launcher binary | `~/.local/bin/feynman` | `%LOCALAPPDATA%\Programs\feynman\` | +| Runtime | `~/.local/share/feynman` | `%LOCALAPPDATA%\Programs\feynman\` | +| Config / sessions| `~/.feynman/` | `~/.feynman/` | +| Auth state | `~/.ahub/` | `~/.ahub/` | + +Post-install: + +```bash +feynman setup # guided wizard: model, auth, optional packages +feynman --version +feynman doctor # validates config, auth, Pi runtime, preview deps +feynman status # one-line summary (model + auth + packages) +``` + +Update: + +```bash +# Standalone — rerun installer +curl -fsSL https://feynman.is/install | bash +# npm +npm install -g @companion-ai/feynman@latest +# Single package +feynman update +``` + +Uninstall (npm): `npm uninstall -g @companion-ai/feynman`. Standalone: remove +`~/.local/bin/feynman` + `~/.local/share/feynman` + `~/.feynman` + `~/.ahub`. + +## Provider auth + +`feynman setup` walks you through provider auth. After install you can +manage providers individually: + +```bash +feynman model login anthropic # OAuth ("Pi") flow → ~/.feynman/auth/ +feynman model login openai # OAuth or API key +feynman model login google # Gemini +feynman model login amazon-bedrock # uses AWS credential chain +feynman model logout +feynman model list # all configured models +feynman model set anthropic/claude-opus-4-20250514 +``` + +Local providers (Ollama, LM Studio, LiteLLM, vLLM): pick "Custom provider +(baseUrl + API key)" in `feynman setup` and supply a base URL like +`http://localhost:11434/v1`. LM Studio default `http://localhost:1234/v1`, +LiteLLM `http://localhost:4000/v1`. + +API keys also accepted via env (provider-config-files override env): + +``` +ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY AWS_PROFILE +TAVILY_API_KEY SERPER_API_KEY +``` + +## REPL launch + one-shot + +```bash +feynman # interactive REPL (in cwd) +feynman --cwd ~/papers # REPL scoped to a directory +feynman --new-session # fresh session +feynman --session-dir ~/proj/.fey # custom session storage + +feynman --prompt "Summarize Attention Is All You Need" # one-shot +feynman deepresearch "transformer architectures for protein folding" +feynman lit "agentic LLM benchmarks" +feynman review draft.md +feynman audit 2401.12345 +feynman replicate "scaling-law claim" +feynman compare "post-training vs RLHF" +feynman draft "literature notes on mechanistic interpretability" +``` + +Override per call: `--model anthropic/claude-sonnet-4-20250514`, +`--thinking high` (`off|minimal|low|medium|high|xhigh`). + +## Slash commands (REPL) + +Research workflows: + +| Command | Purpose | +| ----------------- | ----------------------------------------------------------- | +| `/deepresearch` | Source-heavy multi-agent investigation → research brief | +| `/lit` | Structured literature review (consensus / disagreement) | +| `/review` | Peer-review of an artifact with severity-graded feedback | +| `/audit` | Compare a paper's claims against its public code | +| `/replicate` | Plan or execute a replication for a paper / claim / benchmark | +| `/compare` | Agreement / disagreement matrix across multiple sources | +| `/draft` | Paper-style draft from research findings | +| `/autoresearch` | Autonomous experiment loop optimizing toward a goal | +| `/watch` | Recurring monitoring on a topic | + +Project / session: + +| Command | Purpose | +| ----------------- | -------------------------------------------------------- | +| `/help` | Grouped command list | +| `/init` | Bootstrap `AGENTS.md` + session-log folders for a project| +| `/log` | Durable session log: work done / findings / next steps | +| `/jobs` | Active background work (running / scheduled / watches) | +| `/outputs` | Browse research artifacts | +| `/search` | Search prior session transcripts (semantic + keyword) | +| `/preview` | Render artifact as HTML or PDF | +| `/feynman-model` | Open the model picker (default + per-subagent overrides) | + +Body templating in REPL: `@` to inject file content, `!` to run +a shell snippet inline. + +## Built-in agents + +| Agent | Role | Invoked by | +| ----------- | --------------------------------------------- | --------------------------------------------------------- | +| `researcher`| Search / read / extract / organize sources | `/deepresearch`, `/lit`, `/review`, `/audit`, `/replicate`, `/compare`, `/draft` | +| `reviewer` | Peer-review with Critical/Major/Minor/Nits | `/review`, `/audit`, `/compare` | +| `writer` | Synthesize notes → brief, lit-review, paper, report | `/deepresearch`, `/lit`, `/draft`, `/compare` | +| `verifier` | Citation/URL/code verification | `/deepresearch`, `/audit`, `/replicate` | + +Override a model per agent: `/feynman-model` in REPL. Persists to +`~/.feynman/agent/agents/.md` as a `model:` frontmatter field. + +## Built-in tools + +| Tool | Purpose | +| ---------------- | ---------------------------------------------------------- | +| `alphaxiv` | Academic search; arXiv ID lookup; PDF fetch + section extract; Q&A | +| `web-search` | Routes between auto/Perplexity/Gemini per `~/.feynman/web-search.json` | +| `session-search` | Indexed semantic + keyword recall over `~/.feynman/sessions/` | +| `preview` | Render artifact via pandoc → HTML/PDF (KaTeX, code highlighting) | + +AlphaXiv auth: `feynman alpha login` / `feynman alpha status` / +`feynman alpha logout`. Without it Feynman falls back to general web search +and loses citation metadata. + +Web search backends: +- **auto** (default) — Gemini-grounded via Chromium profile (zero-config on + macOS/Linux), falls back to Perplexity when configured +- **perplexity** — forces Perplexity Sonar (needs `perplexityApiKey`) +- **gemini** — forces Gemini grounding + +Configure via `~/.feynman/web-search.json`: +```json +{ "route": "auto", "perplexityApiKey": "{env:PERPLEXITY_KEY}", "geminiApiKey": "{env:GEMINI_API_KEY}" } +``` + +Preview deps: `feynman setup preview` installs pandoc (Homebrew on macOS, +package manager on Linux). Verify with `feynman doctor`. + +## Packages (11 core + 1 optional) + +Core (auto-installed): +`pi-subagents`, `pi-docparser`, `pi-web-access`, `pi-markdown-preview`, +`pi-mermaid`, `@walterra/pi-charts`, `pi-zotero`, +`@kaiserlich-dev/pi-session-search`, `pi-schedule-prompt`, plus background-task ++ memory-persistence + long-running-loop helpers. + +Optional: +- `generative-ui` — interactive HTML widgets (macOS only) + +Manage: +```bash +feynman packages list +feynman packages install generative-ui +feynman update # all +feynman update pi-docparser # one +``` + +`generative-ui` is hidden from `feynman packages list` on Linux/Windows. + +## Configuration locations (verified from working install) + +| Path | Purpose | +| ------------------------------------- | -------------------------------------------------- | +| `~/.feynman/agent/settings.json` | core config (model, provider, packages, theme) | +| `~/.feynman/agent/auth.json` | OAuth tokens + API keys | +| `~/.feynman/agent/agents/.md` | builtin + custom agent prompts (markdown + frontmatter) | +| `~/.feynman/agent/skills//` | installed skills (deep-research, lit-review, …) | +| `~/.feynman/agent/themes/.json`| color themes (bundled `feynman.json`) | +| `~/.feynman/agent/bin/` | bundled binaries (`fd`, `rg`) | +| `~/.feynman/agent/run-history.jsonl` | append-only run log | +| `~/.feynman/sessions/` | persisted REPL sessions | +| `~/.feynman/memory/` | long-term memory store (`pi-memory` package) | +| `~/.feynman/web-search.json` | search-backend routing | +| `~/.feynman/.state/` | runtime state | +| `~/.feynman/npm-global/` | sandboxed npm prefix for Pi packages | +| `~/.ahub/` | Pi auth-hub state (separate from feynman/agent) | + +> **Heads-up:** the public docs site shows simpler paths like +> `~/.feynman/settings.json`. The real layout (above) prefixes most things +> with `agent/` because Feynman wraps the Pi runtime. When in doubt, trust +> what `feynman doctor` reports. + +`agent/settings.json` real shape (taken from a working install): +```json +{ + "packages": [ + "npm:@companion-ai/alpha-hub", + "npm:pi-subagents", + "npm:pi-btw", + "npm:pi-docparser", + "npm:pi-web-access", + "npm:pi-markdown-preview", + "npm:@walterra/pi-charts", + "npm:pi-mermaid", + "npm:@aliou/pi-processes", + "npm:pi-zotero", + "npm:@kaiserlich-dev/pi-session-search", + "npm:pi-schedule-prompt", + "npm:@samfp/pi-memory", + "npm:@tmustier/pi-ralph-wiggum" + ], + "quietStartup": true, + "collapseChangelog": true, + "defaultThinkingLevel": "medium", + "editorPaddingX": 1, + "theme": "feynman", + "defaultProvider": "github-copilot", + "defaultModel": "gpt-5.3-codex", + "lastChangelogVersion": "0.67.6" +} +``` + +> Documented `defaultProvider`/`defaultModel`/`defaultThinkingLevel` are real; +> the rest (`packages`, `theme`, `quietStartup`, etc.) are what an actual +> install writes. Edit by hand or rerun `feynman setup`. + +## Custom agents (the right way) + +Feynman runs on the **pi-subagents** runtime — same agent format as Pi. There +are three scopes; later layers override earlier ones: + +| Scope | Path | Priority | +| -------- | ----------------------------------------------- | -------- | +| Builtin | bundled inside the npm package's `agents/` | lowest | +| User | `~/.feynman/agent/agents/.md` | medium | +| Project | `.feynman/agents/.md` (walks up tree) | highest | + +The four bundled-by-Feynman agents (`researcher`, `reviewer`, `writer`, +`verifier`) ship as full files in `~/.feynman/agent/agents/` — edit them +in-place to customize, or copy to `.feynman/agents/.md` to scope the +change to one project. + +Minimal custom agent (`~/.feynman/agent/agents/security-auditor.md`): + +```yaml +--- +name: security-auditor +description: Audit code for OWASP Top 10 + secret leaks; FATAL/MAJOR/MINOR rubric. +thinking: high +tools: read, grep, find, ls, bash, web_search, fetch_content +output: audit.md +defaultProgress: true +inheritProjectContext: false # specialist — don't bias with repo conventions +inheritSkills: false +systemPromptMode: replace +--- + +You are an adversarial security auditor. Find vulnerabilities; do not +fix them. Output sections: ## Critical, ## High, ## Medium, ## Low, +## Out-of-scope. Quote exact line ranges. Refuse to speculate when a +file isn't in scope. +``` + +Then dispatch from REPL: `/run security-auditor "audit src/auth/"`. Or chain: +`/chain researcher "find recent CVEs in node-jsonwebtoken" -> security-auditor "check our usage"`. + +**Override a builtin without copying it** — write into +`agent/settings.json`: +```json +{ + "subagents": { + "agentOverrides": { + "reviewer": { + "model": "anthropic/claude-opus-4-5", + "thinking": "high", + "inheritProjectContext": true + } + } + } +} +``` +Override fields: `model`, `fallbackModels`, `thinking`, `systemPromptMode`, +`inheritProjectContext`, `inheritSkills`, `disabled`, `skills`, `tools`, +`systemPrompt`. Project scope (`.feynman/settings.json`) beats user scope. + +Disable a builtin: `disabled: true` inside the override (still listed in +`/agents` so you can re-enable). Disable all builtins: +`subagents.disableBuiltins: true`. + +Frontmatter cheat-sheet (full spec → `references/custom-agents.md`): + +| Field | Purpose | +| ---------------------- | ---------------------------------------------------- | +| `name` | required, kebab-case | +| `description` | required, agent-picker tooltip | +| `tools` | comma list of builtin tools + `mcp:server[/tool]` | +| `extensions` | absent=all, empty=none, csv=allowlist | +| `model` | `provider/model-id` | +| `fallbackModels` | csv ordered list of backups | +| `thinking` | `off|minimal|low|medium|high|xhigh` | +| `systemPromptMode` | `replace` (default) / `append` | +| `inheritProjectContext`| custom default `false`; builtins default `true` | +| `inheritSkills` | default `false` | +| `skills` | csv of skill names to inject | +| `output` | default artifact filename | +| `defaultReads` | csv of files to auto-read | +| `defaultProgress` | maintain `progress.md` | +| `maxSubagentDepth` | nested-delegation limit for this agent's children | + +Env vars override: + +| Var | Effect | +| ---------------------- | ------------------------------------- | +| `FEYNMAN_MODEL` | Override default model for the run | +| `FEYNMAN_HOME` | Override `~/.feynman` location | +| `FEYNMAN_THINKING` | Override default thinking level | +| `ANTHROPIC_API_KEY` | Anthropic auth | +| `OPENAI_API_KEY` | OpenAI auth | +| `GEMINI_API_KEY` | Google auth | +| `AWS_PROFILE` | Bedrock profile selection | +| `TAVILY_API_KEY` | Tavily web-search backend | +| `SERPER_API_KEY` | Serper web-search backend | + +## Workflows index (this repo) + +The detailed how-to for each workflow lives in +`personas/_shared/feynman-skills/`. This skill explains **the CLI**; +those skills explain **the methodology**. Existing dedicated skills: + +- `deep-research/SKILL.md` +- `literature-review/SKILL.md` +- `peer-review/SKILL.md` +- `paper-code-audit/SKILL.md` +- `replication/SKILL.md` +- `source-comparison/SKILL.md` +- `paper-writing/SKILL.md` — `/draft` +- `autoresearch/SKILL.md` +- `watch/SKILL.md` +- `summarize/SKILL.md`, `eli5/SKILL.md`, `alpha-research/SKILL.md`, + `paper-code-audit/SKILL.md`, `source-comparison/SKILL.md`, + `session-log/SKILL.md`, `session-search/SKILL.md`, `preview/SKILL.md`, + `jobs/SKILL.md`, `contributing/SKILL.md`, `docker/SKILL.md`, + `modal-compute/SKILL.md`, `runpod-compute/SKILL.md` + +Cross-platform note: `_platform-mapping.md` in `feynman-skills/` maps the +four roles (researcher/reviewer/writer/verifier) to host-platform equivalents +(Claude Code `Task` + persona subagent_types, OpenCode `task` + agent names) +when these skills run **outside** Feynman itself. + +## Quick recipes + +**First-time setup on a fresh machine:** +```bash +curl -fsSL https://feynman.is/install | bash +exec $SHELL # pick up ~/.local/bin +feynman setup # interactive +feynman alpha login # academic search +feynman setup preview # pandoc for /preview +feynman doctor # confirm everything green +``` + +**Run a one-shot deep-research and save the brief:** +```bash +feynman --cwd ~/research --prompt "/deepresearch agentic memory persistence patterns 2024-2026" +ls ~/research/outputs/ +``` + +**Schedule a recurring watch from REPL:** +``` +feynman +> /watch agentic-memory weekly +> /jobs +``` + +**Lock everyone on the team to one provider:** +```json +// ~/.feynman/settings.json (committed in your team's dotfiles repo) +{ + "defaultProvider": "anthropic", + "defaultModel": "claude-sonnet-4-20250514", + "defaultThinkingLevel": "high" +} +``` +Plus deny other providers via `feynman model logout ` per machine. + +**Use a self-hosted local model (vLLM):** +```bash +feynman setup +# Choose: Custom provider (baseUrl + API key) +# baseURL: http://gpu-host:8000/v1 +# api key: any non-empty string +# model id: +``` + +**CI / scripted invocation (one-shot, no auth prompts):** +```bash +ANTHROPIC_API_KEY=$KEY \ +FEYNMAN_THINKING=low \ +feynman --prompt "Summarize $(cat README.md | head -200)" > brief.md +``` + +## Pitfalls + +- `~/.local/bin` may not be on `$PATH` after install — restart the shell or + `export PATH="$HOME/.local/bin:$PATH"`. +- `feynman setup` overwrites `settings.json`. Keep a backup if you've + hand-edited it. +- AlphaXiv-less mode silently degrades research quality. Always run + `feynman alpha login` for serious work. +- Web-search "auto" mode requires a logged-in Chromium profile on + macOS/Linux for Gemini grounding — without one it errors instead of + falling through to Perplexity unless `perplexityApiKey` is set. +- `--thinking high` on cheap models (e.g. Haiku) can hit token limits before + the run completes. Pair high thinking with high-context models. +- `generative-ui` package is macOS-only; running `feynman packages install + generative-ui` on Linux is a no-op (hidden from list). +- `feynman doctor` reports OK even when the model is logged-out as long as + the env var is set — env auth bypasses the auth-store check. +- Per-subagent model overrides in `~/.feynman/agent/agents/*.md` shadow the + global default for **just that role**; check with `/feynman-model` if a + subagent suddenly costs more. +- `--session-dir` doesn't migrate existing sessions; previous transcripts + stay in the original location. +- Pinning to a specific version with the bash installer is sticky — running + the bare installer later will upgrade to latest. Re-pin to stay locked. +- npm install needs Node `>=20.19.0 <25`; on Node 25+ you must use the + standalone installer. + +## Operational checklist + +- [ ] Installer ran and `feynman --version` prints something +- [ ] `feynman setup` completed (model picked, auth done) +- [ ] `feynman alpha login` for academic workflows +- [ ] `feynman setup preview` if you'll use `/preview` +- [ ] `feynman doctor` reports all green +- [ ] `~/.feynman/settings.json` reviewed (model, thinking level) +- [ ] `~/.feynman/web-search.json` configured if not on macOS/Linux Chromium +- [ ] Optional packages installed (`feynman packages list` to audit) +- [ ] Sessions dir backed up if research is durable diff --git a/personas/_shared/community-skills/feynman-cli/references/agents-tools-packages.md b/personas/_shared/community-skills/feynman-cli/references/agents-tools-packages.md new file mode 100644 index 0000000..1f7774e --- /dev/null +++ b/personas/_shared/community-skills/feynman-cli/references/agents-tools-packages.md @@ -0,0 +1,143 @@ +# Built-in Agents, Tools, and Packages + +## Built-in agents (Feynman bundle) + +Feynman ships four primary research roles. Each lives as a full markdown +file in `~/.feynman/agent/agents/.md` — edit in place to customize, or +override fields surgically via `agent/settings.json` → +`subagents.agentOverrides.`. + +### researcher + +Gathers primary evidence across papers, web sources, repos, and local files. +The most-invoked agent. + +- **Frontmatter (real):** `thinking: high`, default tools `read, write, edit, bash, grep, find, ls, web_search, fetch_content, get_search_content`, output `research.md`. +- **Invoked by:** `/deepresearch`, `/lit`, `/review`, `/audit`, `/replicate`, `/compare`, `/draft`. +- **Search strategy:** start wide (2–4 varied queries simultaneously via `web_search.queries`), evaluate, narrow. Cross-source `web_search` + `alpha`. Use `recencyFilter` for fast-moving topics, `includeContent: true` for top results. +- **Output:** evidence table (numbered IDs, source / URL / claim / type / confidence), findings with inline `[1]`-style references, numbered Sources section, Coverage Status. + +### reviewer + +Acts as a skeptical-but-fair peer reviewer. + +- **Frontmatter:** `thinking: high`, output `review.md`. +- **Severity rubric:** FATAL (publication blocker), MAJOR (substantive flaw), MINOR (improvement), Nits (style). +- **Examines:** claims-vs-evidence, methodology, baselines, ablations, reproducibility, related-work coverage, statistical strength, benchmark leakage. +- **Output:** Structured Review (Summary / Strengths / Weaknesses / Questions / Verdict / Revision Plan) + Inline Annotations (quoted passages with `[W#]` references). +- **Invoked by:** `/review`, `/audit`, `/compare`. + +### writer + +Synthesizes research findings into a structured artifact. + +- **Frontmatter:** `thinking: medium`, default tools `read, bash, grep, find, ls, write, edit`, output `draft.md`. +- **Output styles:** research brief, literature review, paper draft, comparison report, summary. +- **Discipline:** writes only from supplied evidence, preserves caveats and disagreements, no inline citations (verifier handles that), no Sources section (verifier builds), no aesthetic laundering. +- **Visuals:** `pi-charts` for data, Mermaid for diagrams (only when supported by evidence). Captions reference source data. +- **Invoked by:** `/deepresearch`, `/lit`, `/draft`, `/compare`. + +### verifier + +Adds inline citations + verifies every URL. + +- **Frontmatter:** `thinking: medium`, default tools `read, bash, grep, find, ls, write, edit, web_search, fetch_content, get_search_content`, output `cited.md`. +- **Process:** anchor every factual claim → `[N]` citation, fetch every URL to confirm content, build merged Sources section, remove unsourced claims, audit numeric/figure provenance. +- **Refuses:** "verified", "confirmed", "reproduced" without underlying evidence in research files. +- **Invoked by:** `/deepresearch`, `/audit`, `/replicate`. + +## Other pi-subagents agents (available via `/run`) + +These ship in `pi-subagents` and are usable for non-research workflows. They +load at lowest priority — overridable from `~/.feynman/agent/agents/`. + +| Agent | Purpose | +| ----------------- | ------------------------------------------------ | +| `scout` | Fast codebase recon (read-grep-find heavy) | +| `planner` | Make a step-by-step implementation plan | +| `worker` | Execute against a plan | +| `delegate` | Orchestrate child agents (`systemPromptMode: append`) | +| `context-builder` | Build a focused context bundle for a task | + +Use via pi-subagents commands: `/run planner "design X"`, +`/chain scout "scan auth" -> planner -> worker`. + +## Built-in tools + +### alphaxiv + +Primary academic search and retrieval. + +- **Setup:** `feynman alpha login`, status `feynman alpha status`. +- **Auth store:** `~/.feynman/agent/auth.json`. +- **Capabilities:** topic / author / keyword search, arXiv ID lookup, PDF download + parsing, section extraction (abstract / methodology / results), paper Q&A, GitHub repo file inspection, persistent local annotations. +- **Without auth:** falls back to web search; loses citation metadata, threads, full-text. Strongly recommend logging in for serious research. + +### web-search + +Routes between backends per `~/.feynman/web-search.json`. + +- **Modes:** `auto` (Perplexity if configured else Gemini), `perplexity` (force Perplexity Sonar), `gemini` (force Gemini grounding). +- **Default zero-config path:** Gemini grounding via signed-in Chromium profile (macOS / Linux Chrome). +- **Features:** multi-query (`queries: [...]`), `domainFilter`, `recencyFilter`, `includeContent: true` for full-page fetch. +- **Failure mode:** without Chromium profile and no `perplexityApiKey`, errors instead of degrading. + +### session-search + +Indexes prior session transcripts for recall. + +- **Package:** `@kaiserlich-dev/pi-session-search` (core). +- **Storage:** `~/.feynman/sessions/` (or `--session-dir `). +- **Invocation:** explicit `/search transformer scaling laws` or natural reference to past work — Feynman auto-triggers. +- **Algorithm:** keyword + semantic similarity. Returns session IDs, timestamps, excerpts. +- **Index:** incremental, fast even at hundreds of sessions. + +### preview + +Renders artifacts to HTML / PDF via pandoc + KaTeX. + +- **Package:** `pi-markdown-preview` (core). +- **Setup:** `feynman setup preview` (installs pandoc via Homebrew on macOS, package manager on Linux). +- **REPL:** `/preview` (last artifact), `/preview outputs/foo.md` (specific). +- **Formats:** Markdown → HTML/PDF (KaTeX `$…$` and `$$…$$`, syntax-highlighted code, tables, citations); HTML opens directly; PDF via pandoc + LaTeX. +- **Verify:** `feynman doctor` reports preview-deps status. + +## Packages (real install lists 14) + +Loaded at startup per `agent/settings.json` → `packages` array. Default +install includes: + +| Package | Purpose | +| ------------------------------------------------- | ---------------------------------------------- | +| `npm:@companion-ai/alpha-hub` | AlphaXiv adapter | +| `npm:pi-subagents` | Subagent orchestration (researcher/reviewer/…)| +| `npm:pi-btw` | Inline aside / footnotes for outputs | +| `npm:pi-docparser` | PDF / Office / image parsing | +| `npm:pi-web-access` | Web fetch, browse, GitHub integration | +| `npm:pi-markdown-preview` | HTML / PDF render | +| `npm:@walterra/pi-charts` | Quantitative charts | +| `npm:pi-mermaid` | Mermaid diagrams | +| `npm:@aliou/pi-processes` | Background process management | +| `npm:pi-zotero` | Zotero integration | +| `npm:@kaiserlich-dev/pi-session-search` | Session search | +| `npm:pi-schedule-prompt` | `/watch` cron-style scheduler | +| `npm:@samfp/pi-memory` | Long-term memory | +| `npm:@tmustier/pi-ralph-wiggum` | Long-running agent loop helper | + +Optional preset: +- `generative-ui` — interactive HTML widgets (macOS only). Hidden from + `feynman packages list` on Linux/Windows. + +Manage: +```bash +feynman packages list +feynman packages install generative-ui +feynman update # all packages +feynman update pi-docparser # one package +``` + +## Notes + +- The four "official" agents (`researcher`, `reviewer`, `writer`, `verifier`) are bundled with Feynman, not with `pi-subagents`. The pi-subagents npm package contains `scout`, `planner`, `worker`, `delegate`, `context-builder`, plus its own `researcher` / `reviewer` (lower priority — overridden by Feynman's versions). +- Custom agents take precedence over both layers — see `references/custom-agents.md`. +- "Tools" in agent frontmatter are builtin tool names (`read`, `bash`, `web_search`, `fetch_content`, `get_search_content`). MCP server tools use `mcp:server-name` or `mcp:server/tool-name` and require `pi-mcp-adapter`. diff --git a/personas/_shared/community-skills/feynman-cli/references/cli-reference.md b/personas/_shared/community-skills/feynman-cli/references/cli-reference.md new file mode 100644 index 0000000..2f346cd --- /dev/null +++ b/personas/_shared/community-skills/feynman-cli/references/cli-reference.md @@ -0,0 +1,147 @@ +# CLI Reference + +## Core commands + +| Command | Purpose | +| -------------------------------- | -------------------------------------------------------- | +| `feynman` | Launch interactive REPL in cwd | +| `feynman chat [prompt]` | Start chat explicitly, optional initial prompt | +| `feynman --prompt ""` | One-shot prompt: print response, exit | +| `feynman help` | Show CLI help | +| `feynman setup` | Guided setup wizard (model, auth, packages) | +| `feynman setup preview` | Install / verify pandoc for `/preview` | +| `feynman doctor` | Diagnose config, auth, Pi runtime, preview deps | +| `feynman status` | Current setup summary (model + auth + packages) | +| `feynman --version` | Print version | + +## Model management + +```bash +feynman model list # list every reachable model +feynman model login # OAuth or API key +feynman model logout # clear stored auth +feynman model set # set default; accepts + # provider/model-name + # provider:model-name +``` + +Examples: +```bash +feynman model set anthropic/claude-sonnet-4-20250514 +feynman model set openai:gpt-4o +feynman model set github-copilot/gpt-5.3-codex +``` + +## AlphaXiv (academic search) + +| Command | Purpose | +| ------------------------ | ------------------------------------ | +| `feynman alpha login` | Sign in to alphaXiv | +| `feynman alpha logout` | Clear alphaXiv auth | +| `feynman alpha status` | Check alphaXiv auth status | + +Without alphaXiv auth Feynman falls back to general web search and loses +citation metadata, discussion threads, and full-text PDFs. Strongly +recommend running `feynman alpha login` before serious research. + +## Packages + +| Command | Purpose | +| ------------------------------------- | ---------------------------------- | +| `feynman packages list` | All packages + install status | +| `feynman packages install ` | Install optional package preset | +| `feynman update` | Update all packages | +| `feynman update ` | Update one package | + +Optional preset today: `generative-ui` (macOS only — interactive HTML +widgets). Other names hidden on Linux/Windows. + +## Workflow shortcuts (CLI form) + +These are equivalent to the REPL slash commands but run as one-shots from +the shell: + +```bash +feynman deepresearch "topic" +feynman lit "topic" # literature review +feynman review draft.md # peer review of an artifact +feynman audit 2401.12345 # paper-vs-code audit (arXiv id) +feynman replicate "claim or paper" +feynman compare "topic across sources" +feynman draft "paper-style draft topic" +``` + +CLI form auto-creates a session, runs the workflow, prints/saves the +artifact, and exits. Pair with `--cwd` to scope research to a directory: +```bash +feynman --cwd ~/research/llm-evals lit "agentic LLM benchmarks" +``` + +## Search + +| Command | Purpose | +| ------------------------- | ------------------------------------ | +| `feynman search status` | Show Pi web-access status | + +## Global flags + +| Flag | Effect | +| -------------------------- | --------------------------------------------------- | +| `--prompt ""` | One-shot prompt → exit | +| `--model ` | Override default model for this run | +| `--thinking ` | `off|minimal|low|medium|high|xhigh` | +| `--cwd ` | Run with a specific working directory | +| `--session-dir ` | Custom session storage path | +| `--new-session` | Force fresh session | +| `--alpha-login` | Alias for `feynman alpha login` | +| `--alpha-logout` | Alias for `feynman alpha logout` | +| `--alpha-status` | Alias for `feynman alpha status` | +| `--doctor` | Alias for `feynman doctor` | +| `--setup-preview` | Alias for `feynman setup preview` | + +## CI / scripted invocation + +```bash +ANTHROPIC_API_KEY=$KEY \ +FEYNMAN_THINKING=low \ +feynman --prompt "Summarize $(cat README.md | head -200)" > brief.md +``` + +Notes: +- Pure CLI runs print the model output to stdout; pipe + redirect freely. +- `--thinking low` keeps cost down for batch jobs. +- Set the appropriate `*_API_KEY` env var so you don't depend on + `auth.json` being warm (env auth bypasses interactive prompts). + +## Examples by goal + +**Quickly summarize a paper:** +```bash +feynman --prompt "Summarize the key findings of Attention Is All You Need" +``` + +**Run a deep-research and save the brief:** +```bash +feynman --cwd ~/research deepresearch "agentic memory persistence patterns" +ls ~/research/outputs/ +``` + +**Switch model just for this run:** +```bash +feynman --model anthropic/claude-opus-4-5 --thinking high +``` + +**Verify install + doctor in one shot (CI):** +```bash +feynman --version && feynman doctor +``` + +## Commands not present (yet) + +The docs claim some, the CLI lacks others. Verify with `feynman help` if a +command isn't in the reference above. Missing as of this skill snapshot: +`feynman run`, `feynman version` (use `--version`), `feynman uninstall`, +`feynman session`, `feynman auth`, `feynman config`. Use the alternates: +- session control → REPL `/sessions` or `--session-dir` +- auth → `feynman model login/logout` and `feynman alpha login/logout` +- config → edit `~/.feynman/agent/settings.json` directly diff --git a/personas/_shared/community-skills/feynman-cli/references/configuration.md b/personas/_shared/community-skills/feynman-cli/references/configuration.md new file mode 100644 index 0000000..7412586 --- /dev/null +++ b/personas/_shared/community-skills/feynman-cli/references/configuration.md @@ -0,0 +1,182 @@ +# Configuration Deep Dive + +Real layout (verified from a working install). Public docs show simpler paths +without the `agent/` subdir; the actual install nests most things under +`~/.feynman/agent/` because Feynman wraps the Pi runtime. + +## Directory tree + +``` +~/.feynman/ +├── agent/ # the Pi-runtime root +│ ├── settings.json # core config +│ ├── auth.json # OAuth tokens + API keys +│ ├── agents/ # builtin + custom agent prompts +│ │ ├── researcher.md +│ │ ├── reviewer.md +│ │ ├── writer.md +│ │ └── verifier.md +│ ├── skills/ # installed feynman-skills +│ │ ├── deep-research/ +│ │ ├── literature-review/ +│ │ ├── peer-review/ +│ │ └── … +│ ├── themes/ +│ │ └── feynman.json +│ ├── bin/ # bundled tools +│ │ ├── fd +│ │ └── rg +│ └── run-history.jsonl +├── sessions/ # persisted REPL sessions +├── memory/ # pi-memory long-term store +├── web-search.json # search backend routing +├── .state/ # runtime state +└── npm-global/ # sandboxed npm prefix for Pi packages +~/.ahub/ # Pi auth-hub state (separate) +``` + +## `agent/settings.json` — full real shape + +```json +{ + "packages": [ + "npm:@companion-ai/alpha-hub", + "npm:pi-subagents", + "npm:pi-btw", + "npm:pi-docparser", + "npm:pi-web-access", + "npm:pi-markdown-preview", + "npm:@walterra/pi-charts", + "npm:pi-mermaid", + "npm:@aliou/pi-processes", + "npm:pi-zotero", + "npm:@kaiserlich-dev/pi-session-search", + "npm:pi-schedule-prompt", + "npm:@samfp/pi-memory", + "npm:@tmustier/pi-ralph-wiggum" + ], + "quietStartup": true, + "collapseChangelog": true, + "defaultThinkingLevel": "medium", + "editorPaddingX": 1, + "theme": "feynman", + "defaultProvider": "github-copilot", + "defaultModel": "gpt-5.3-codex", + "lastChangelogVersion": "0.67.6", + "subagents": { + "agentOverrides": { + "reviewer": { "thinking": "high" } + }, + "disableBuiltins": false + } +} +``` + +| Key | Purpose | +| ------------------------- | ------------------------------------------------------ | +| `packages` | Pi packages to load at startup | +| `quietStartup` | suppress banner | +| `collapseChangelog` | don't expand changelog on launch | +| `defaultProvider` | fallback provider | +| `defaultModel` | fallback model id | +| `defaultThinkingLevel` | `off|minimal|low|medium|high|xhigh` | +| `theme` | name of theme JSON in `agent/themes/` | +| `editorPaddingX` | TUI editor padding | +| `subagents.agentOverrides`| per-agent field overrides without copying the file | +| `subagents.disableBuiltins`| hide all builtins from `/agents` | +| `lastChangelogVersion` | last seen version (Feynman manages this) | + +Edit by hand or rerun `feynman setup` (which overwrites the file). + +Project-level scope: `.feynman/settings.json` (walks up tree). Project +scope wins on conflicting keys. + +## Thinking levels + +| Level | Effect | +| --------- | ------------------------------------- | +| `off` | No extended-thinking | +| `minimal` | Smallest reasoning budget | +| `low` | | +| `medium` | Default | +| `high` | More chain-of-thought | +| `xhigh` | Maximum | + +Override per run: `feynman --thinking high`. Override per agent in +frontmatter `thinking: high`. Suffix on model id: `claude-sonnet-4-5:high`. + +## `web-search.json` + +```json +{ + "route": "auto", + "perplexityApiKey": "{env:PERPLEXITY_API_KEY}", + "geminiApiKey": "{env:GEMINI_API_KEY}" +} +``` + +`route` ∈ `auto | perplexity | gemini`. +- `auto` (default): prefer Perplexity if configured, else Gemini grounding via signed-in Chromium profile (zero-config on macOS/Linux Chrome). +- `perplexity`: always Perplexity Sonar (needs `perplexityApiKey`). +- `gemini`: force Gemini grounding (Chromium profile required, or `geminiApiKey`). + +Without a Chromium profile on `auto` and no `perplexityApiKey`, web search +errors instead of degrading silently. + +## Sessions + +```bash +feynman # current session in cwd +feynman --new-session # fresh +feynman --session-dir ~/proj/.fey # custom dir (existing sessions don't migrate) +``` + +Session storage: `~/.feynman/sessions//` by default, or wherever +`--session-dir` points. The `pi-session-search` package indexes the dir +for `/search`. + +## Environment variables + +| Variable | Effect | +| ----------------------- | ---------------------------------------------- | +| `FEYNMAN_MODEL` | Override default model | +| `FEYNMAN_HOME` | Override `~/.feynman` location | +| `FEYNMAN_THINKING` | Override default thinking level | +| `ANTHROPIC_API_KEY` | Anthropic auth | +| `OPENAI_API_KEY` | OpenAI auth | +| `GEMINI_API_KEY` | Google auth | +| `AWS_PROFILE` | Bedrock profile | +| `TAVILY_API_KEY` | Optional Tavily backend (supplemental) | +| `SERPER_API_KEY` | Optional Serper backend (supplemental) | +| `PERPLEXITY_API_KEY` | Used by `web-search.json` `perplexityApiKey` | + +Provider config (`agent/settings.json`) takes precedence over env vars. + +## Themes + +Bundled: `~/.feynman/agent/themes/feynman.json`. Custom themes drop in the +same directory. Switch via `theme: ""` in `agent/settings.json`. + +## Auth storage + +`~/.feynman/agent/auth.json` is a single JSON file that holds OAuth tokens +and API keys for every authed provider. Don't commit it. To reset auth for +one provider: `feynman model logout ` then `feynman model login `. +To nuke everything: `rm ~/.feynman/agent/auth.json && feynman setup`. + +## Project-scope config + +A research project can override user-scope settings: + +``` +my-research/ +├── .feynman/ +│ ├── settings.json # project overrides (subagents, theme, etc.) +│ └── agents/ +│ └── domain-expert.md # project-only agent +└── outputs/ + └── … +``` + +`feynman --cwd my-research/` (or running from inside the dir) makes Feynman +walk up the tree, find `.feynman/`, and merge. diff --git a/personas/_shared/community-skills/feynman-cli/references/custom-agents.md b/personas/_shared/community-skills/feynman-cli/references/custom-agents.md new file mode 100644 index 0000000..c6076bd --- /dev/null +++ b/personas/_shared/community-skills/feynman-cli/references/custom-agents.md @@ -0,0 +1,266 @@ +# Custom Agents — Full Spec + +Feynman runs on **pi-subagents**, the Pi extension that handles agent +discovery, prompt assembly, and dispatch. The agent format here is the +authoritative spec — same one pi-subagents reads. + +## Three scopes (later wins) + +| Scope | Path | Priority | +| -------- | ------------------------------------------------------------------- | -------- | +| Builtin | inside the package: `~/.feynman/npm-global/lib/node_modules/pi-subagents/agents/` (and Feynman's bundled `~/.feynman/agent/agents/.md` for the four "official" roles) | lowest | +| User | `~/.feynman/agent/agents/.md` | medium | +| Project | `.feynman/agents/.md` (walks up from cwd to git root) | highest | + +> Project discovery also reads legacy `.agents/.md` for compatibility. +> If both `.agents/` and `.feynman/agents/` define the same name, +> `.feynman/agents/` wins. + +`agentScope` parameter (when calling pi-subagents tools programmatically): +`"user"`, `"project"`, or `"both"` (default; project wins on conflict). + +## Frontmatter — full spec + +```yaml +--- +name: scout # required, kebab-case +description: Fast codebase recon # required, agent-picker tooltip +tools: read, grep, find, ls, bash, mcp:chrome-devtools +extensions: # absent=all, empty=none, csv=allowlist +model: claude-haiku-4-5 # provider/model-id (or bare id resolved against current provider) +fallbackModels: openai/gpt-5-mini, anthropic/claude-sonnet-4 +thinking: high # off|minimal|low|medium|high|xhigh +systemPromptMode: replace # replace (default) | append +inheritProjectContext: false # custom default false; builtins default true +inheritSkills: false # default false +skills: safe-bash, chrome-devtools # csv of skills to inject +output: context.md # default artifact filename → {chain_dir}/context.md +defaultReads: context.md # csv of files to auto-read +defaultProgress: true # maintain progress.md +interactive: true # parsed but not enforced in v1 +maxSubagentDepth: 1 # tighten nested delegation +--- + +Your system prompt goes here (markdown body after frontmatter). +``` + +### Field semantics + +**`name`** — required, kebab-case `^[a-z0-9]+(-[a-z0-9]+)*$`. + +**`description`** — required, used in agent-picker UI and by orchestrators. + +**`tools`** — comma-separated list controlling builtin-tool allowlisting: +- **omitted** → child gets Pi's default builtins (no `--tools` flag passed) +- **listed** → explicit allowlist (e.g. `tools: read, bash` restricts to those two) +- `mcp:server` or `mcp:server/tool` entries → forwarded as MCP-tool selections (requires `pi-mcp-adapter`) +- path-like entries ending in `.ts` / `.js` → treated as tool-extension paths +- `tools: mcp:chrome-devtools` (no regular tools listed) → child gets all default builtins **plus** chrome-devtools tools (additive) + +**`extensions`** — extension sandboxing: +- absent → all extensions load +- empty (`extensions:`) → `--no-extensions` (none) +- csv → `--no-extensions --extension a --extension b` (allowlist) +- when present, takes precedence over extension paths implied by `tools` + +**`model`** — `provider/model-id`. Bare ids first prefer the current session's provider if it serves them, then fall back to a unique registry match. Ambiguous bare ids stay bare. + +**`fallbackModels`** — ordered list of backups for **provider/model availability** failures only (quota, auth, timeout, model unavailable). Ordinary `bash`-or-tool errors don't trigger a hop. CSV in markdown frontmatter; CSV or array in JSON `config` objects. + +**`thinking`** — extended-thinking budget. Appended to model id at runtime as `:level` suffix (e.g. `claude-sonnet-4-5:high`). If the model already has a thinking suffix (from chain-clarify override), agent default is not double-applied. + +**`systemPromptMode`** — how the agent's markdown body is injected: +- `replace` (default for custom agents) — body becomes the entire system prompt; clean slate, no Pi base prompt baggage +- `append` — body appended to Pi's base prompt; child gets full Pi capabilities + your additions + +The bundled `delegate` agent stays on `append` because it orchestrates within the parent workflow. + +**`inheritProjectContext`** — keep Pi's inherited project-instructions block (built from `AGENTS.md` / `CLAUDE.md` etc.): +- `false` (custom default) — strip; child runs without repo conventions +- `true` (builtin default) — keep; child follows project-specific rules + +Does **not** affect repo access, cwd, tools, or task — only those instruction-file blocks. + +**`inheritSkills`** — keep Pi's discovered skills catalog: +- `false` (default) — stripped (focused agents) +- `true` — full catalog visible (general helpers) + +The `skills` field works independently — it injects specific skills regardless of `inheritSkills`. + +**`skills`** — csv of skill names injected directly into the agent prompt. + +**`output`** — default artifact filename. The parent reads this file to retrieve the agent's work. Defaults shipped: researcher → `research.md`, reviewer → `review.md`, writer → `draft.md`, verifier → `cited.md`. + +**`defaultReads`** — csv of files the agent auto-reads on start (typically the parent's research file). + +**`defaultProgress`** — maintain a `progress.md` so the parent can poll status. + +**`maxSubagentDepth`** — limit nested delegation depth from this agent's children. + +## Common recipes (frontmatter combos) + +| Goal | `systemPromptMode` | `inheritProjectContext` | `inheritSkills` | +| -------------------------------------------------------- | ------------------ | ----------------------- | --------------- | +| Fully isolated specialist (custom-agent default) | `replace` | `false` | `false` | +| Specialist that should follow project instruction files | `replace` | `true` | `false` | +| Pi-plus-extensions (general helper with extra prompt) | `append` | `true` | `true` | + +Worked examples: +- **Security auditor** — fully isolated; checks objectively without project bias. +- **Architecture planner** — repo-aware; respects `AGENTS.md` constraints. +- **Generic helper** — append mode + full inheritance; behaves like a slightly-customized Pi. + +## Override builtins without copying the file + +In `~/.feynman/agent/settings.json` (user scope) or `.feynman/settings.json` +(project scope; wins on conflict): + +```json +{ + "subagents": { + "agentOverrides": { + "reviewer": { + "model": "anthropic/claude-opus-4-5", + "thinking": "high", + "inheritProjectContext": true + }, + "researcher": { + "tools": "read, web_search, fetch_content", + "skills": "alpha-research" + }, + "experimental-agent": { + "disabled": true + } + } + } +} +``` + +Override fields: `model`, `fallbackModels`, `thinking`, `systemPromptMode`, +`inheritProjectContext`, `inheritSkills`, `disabled`, `skills`, `tools`, +`systemPrompt`. Project overrides beat user overrides. + +Disable all builtins at once: `subagents.disableBuiltins: true`. Re-enable +one with an explicit override that sets `disabled: false`. + +UI alternative: `/agents` → press `e` on a builtin → toggle a field → save. + +Override badges: `[builtin+user]` / `[builtin+project]` / `off`. + +## MCP tools (optional, requires `pi-mcp-adapter`) + +```yaml +# All tools from a server +tools: read, bash, mcp:chrome-devtools + +# Specific tools +tools: read, bash, mcp:github/search_repositories, mcp:github/get_file_contents +``` + +| Syntax | Effect | +| ---------------------------- | ---------------------------- | +| `mcp:server-name` | All tools from that MCP server| +| `mcp:server-name/tool_name` | One specific tool | + +`mcp:` items are additive — `tools: mcp:chrome-devtools` (no regular tools) +gives the agent all default builtins **plus** chrome-devtools tools. To +restrict: list builtins explicitly. + +> First-run caveat: MCP adapter caches tool metadata at startup. The first +> session to a new MCP server only has access through the generic `mcp` +> proxy. Restart Pi after that first connection to enable direct tools. + +Without `pi-mcp-adapter` installed, all `mcp:` entries are ignored silently. + +## Calling custom agents + +``` +/run security-auditor "audit src/auth/" +/chain researcher "find recent CVEs in node-jsonwebtoken" -> security-auditor "check our usage" +/parallel scanner "find OWASP issues" -> reviewer "check style" +``` + +Per-step task delimiters (`->` separates steps): +- Quotes (`"…"` or `'…'`) define each step's task +- `--` works as task delimiter: `/chain scout -- scan code -> planner -- analyze auth` +- Bare step inherits behavior from execution mode: + - chain → uses `{previous}` (prior step's output) + - parallel → uses first-available task + +Programmatic call (from JS via pi-subagents): +```js +await subagent({ + agent: "security-auditor", + task: "audit src/auth/", + agentScope: "both", // user | project | both (default) + config: { // step override > frontmatter + model: "anthropic/claude-opus-4-5", + fallbackModels: ["openai/gpt-5-mini"], + thinking: "xhigh" + } +}) +``` + +Resolution priority (highest wins): step override > agent frontmatter > disabled flag. + +## Worked example: project-scoped reviewer + +``` +my-research/ +└── .feynman/ + └── agents/ + └── ml-reviewer.md +``` + +```yaml +--- +name: ml-reviewer +description: Reviews ML/AI papers with reproducibility focus. +thinking: xhigh +systemPromptMode: replace +inheritProjectContext: true # keep my repo's AGENTS.md +inheritSkills: false +tools: read, write, edit, web_search, fetch_content +output: review.md +defaultReads: research.md, draft.md +--- + +You are a senior ML reviewer. Apply the FATAL/MAJOR/MINOR/Nits rubric. +Specifically scrutinize: +- claimed reproducibility vs released artifacts (code, hyperparams, data) +- statistical strength of headline numbers +- ablation completeness +- benchmark contamination + +Output sections: Summary, Strengths, Weaknesses (FATAL/MAJOR/MINOR/Nits), +Reproducibility Audit, Verdict, Revision Plan. +``` + +When you run `/review draft.md` from inside `my-research/`, this agent +overrides the global `reviewer`. From any other directory, the global one +runs. + +## Pitfalls + +- `name` mismatch with filename → agent picker silently skips. Filename and + `name:` should match. +- `tools` field is a CSV string in markdown frontmatter, not a YAML array + (`tools: read, bash` not `tools: [read, bash]`). +- Bare model ids (`gpt-5.3-codex`) only resolve cleanly when the current + session provider serves them. Use `provider/model-id` for portability. +- `fallbackModels` triggers only on provider-availability errors. Ordinary + failures (bad bash, missing files) propagate; the agent fails on the + primary model. +- `inheritProjectContext: false` for custom agents is the default but + surprises users porting from generic chat-agent setups. If your custom + agent should respect the repo's `AGENTS.md`, opt in explicitly. +- Per-agent `thinking` is appended as `:level` suffix to the model id at + runtime — model strings already containing a suffix (from a chain + override) are not double-suffixed. +- `mcp:` tools require `pi-mcp-adapter` AND a second startup before direct + tools are populated (first run only has the `mcp` proxy). +- Project agent at `.feynman/agents/foo.md` only loads when cwd or an + ancestor contains `.feynman/`. Run from inside the project, or pass + `--cwd `. +- `agent/settings.json` overrides + a custom file with the same name → the + custom file wins entirely (it's a higher-priority scope, not a merge). diff --git a/personas/_shared/community-skills/feynman-cli/references/installation-setup.md b/personas/_shared/community-skills/feynman-cli/references/installation-setup.md new file mode 100644 index 0000000..6751034 --- /dev/null +++ b/personas/_shared/community-skills/feynman-cli/references/installation-setup.md @@ -0,0 +1,147 @@ +# Installation & Setup + +## Install methods + +```bash +# 1. macOS / Linux — recommended +curl -fsSL https://feynman.is/install | bash + +# 2. macOS / Linux — pin a specific version (example 0.2.39) +curl -fsSL https://feynman.is/install | bash -s -- 0.2.39 + +# 3. Windows PowerShell (run as Administrator) +irm https://feynman.is/install.ps1 | iex + +# 4. npm — cross-platform (needs Node ≥20.19.0 <25) +npm install -g @companion-ai/feynman + +# 5. Skills only (no runtime — for plugging into another agent host) +curl -fsSL https://feynman.is/install-skills | bash +``` + +The bash installer ends with the version it just pinned. To follow latest, +re-run the bare installer. + +## Prerequisites + +- macOS 12+, recent Linux glibc, or Windows 10+ +- For npm path: Node.js `>=20.19.0 <25` (Node 25+ is unsupported as of v0.2.x) +- For `/preview`: pandoc — install via `feynman setup preview` + +## Install locations + +| Layer | macOS / Linux | Windows | +| ---------------- | ------------------------------------- | -------------------------------------- | +| Launcher binary | `~/.local/bin/feynman` | `%LOCALAPPDATA%\Programs\feynman\` | +| Runtime | `~/.local/share/feynman/` | `%LOCALAPPDATA%\Programs\feynman\` | +| User config | `~/.feynman/` | `~/.feynman/` | +| Pi auth-hub | `~/.ahub/` | `~/.ahub/` | + +Make sure `~/.local/bin` is on `$PATH`. If not, restart the shell or add it +manually. + +## Post-install sequence + +```bash +exec $SHELL # pick up new $PATH +feynman --version # smoke test +feynman setup # interactive wizard +feynman alpha login # academic search (strongly recommended) +feynman setup preview # install pandoc for /preview rendering +feynman doctor # validate everything +feynman status # one-line summary +``` + +What `feynman setup` does: +1. Lists provider+model options. Pick a default (e.g. `anthropic:claude-sonnet-4-20250514`, `openai:gpt-4o`, `github-copilot:gpt-5.3-codex`). +2. Authenticates via OAuth ("Pi") or accepts an API key. +3. For Bedrock, validates AWS credential chain (AWS_PROFILE, ~/.aws, SSO, ECS/IRSA, EC2 instance roles). +4. Optionally installs `generative-ui` (macOS only). +5. Writes `~/.feynman/agent/settings.json`. + +Re-running `feynman setup` overwrites `agent/settings.json`. Back it up if +you've hand-edited. + +## Provider authentication individually + +```bash +feynman model login anthropic # Pi OAuth → ~/.feynman/agent/auth.json +feynman model login openai # OAuth or API key +feynman model login google # Gemini +feynman model login amazon-bedrock # uses AWS chain +feynman model logout # clear stored auth +feynman model list # all reachable models +feynman model set anthropic/claude-opus-4-5 +``` + +Local providers (Ollama, LM Studio, LiteLLM, vLLM): in `feynman setup` choose +"Custom provider (baseUrl + API key)" and supply a base URL. Defaults: +- LM Studio: `http://localhost:1234/v1` +- LiteLLM proxy: `http://localhost:4000/v1` +- Ollama: `http://localhost:11434/v1` +- vLLM: depends on your serve config + +## Diagnostics + +```bash +feynman doctor # validates config, auth, Pi runtime, preview deps +feynman status # quick summary +feynman --version +``` + +`feynman doctor` reports green even when a provider is logged-out as long as +its env var is set. Don't rely on doctor to confirm OAuth tokens are still +valid — auth-store check is bypassed when the env var is present. + +## Updating + +```bash +# Standalone — rerun the installer +curl -fsSL https://feynman.is/install | bash + +# Pinned standalone — rerun with version +curl -fsSL https://feynman.is/install | bash -s -- 0.3.0 + +# npm +npm install -g @companion-ai/feynman@latest + +# Single Pi package +feynman update + +# All packages +feynman update +``` + +Release cadence: extremely active during development — multiple per day. Pin +versions if you need reproducibility. Recent fix focus: thinking/reasoning +persistence in TUI, section-focused PDF extraction, summarization controls. + +## Uninstall + +```bash +# npm +npm uninstall -g @companion-ai/feynman + +# Standalone (no uninstaller bundled) +rm ~/.local/bin/feynman +rm -rf ~/.local/share/feynman +rm -rf ~/.feynman +rm -rf ~/.ahub +``` + +Sessions, run history, and skills live in `~/.feynman/`. Back them up before +removing if you've done long-running research there. + +## Verify the install (sanity script) + +```bash +set -e +feynman --version +feynman doctor +feynman model list | head +ls ~/.feynman/agent/agents/ # should list researcher reviewer writer verifier +ls ~/.feynman/agent/skills/ # should include deep-research literature-review etc. +test -f ~/.feynman/agent/settings.json +test -f ~/.feynman/agent/auth.json +echo "Feynman OK" +``` diff --git a/personas/_shared/community-skills/feynman-cli/references/repl-and-slash.md b/personas/_shared/community-skills/feynman-cli/references/repl-and-slash.md new file mode 100644 index 0000000..3cfdf7c --- /dev/null +++ b/personas/_shared/community-skills/feynman-cli/references/repl-and-slash.md @@ -0,0 +1,90 @@ +# REPL & Slash Commands + +## REPL flow + +`feynman` (no args) launches the REPL. Conversational: type prompt, Enter, +read response. The REPL retains session memory, indexes artifacts to +`outputs/` (when running in a project), and exposes slash commands. + +REPL templating: +- `@` — inject file contents into the prompt (e.g. `@papers/foo.pdf`) +- `!` — run a shell command and inject its stdout (e.g. `!ls outputs/`) + +Quit: Ctrl-D or `/exit` (when supported by your version). + +## Slash commands — research workflows + +| Command | What it does | +| --------------------------- | ------------------------------------------------------------- | +| `/deepresearch ` | Multi-agent investigation → research brief in `outputs/` | +| `/lit ` | Structured literature review (consensus / disagreement / open Qs) | +| `/review ` | Peer review with severity-graded feedback + inline annotations | +| `/audit `| Compare a paper's claims against its public code | +| `/replicate ` | Plan or execute a replication for a paper / claim / benchmark | +| `/compare ` | Build agreement / disagreement matrix across sources | +| `/draft ` | Paper-style draft from research findings | +| `/autoresearch ` | Autonomous loop iteratively optimizing toward a goal | +| `/watch [cadence]` | Recurring research monitoring on a topic | + +## Slash commands — project / session / control + +| Command | What it does | +| ------------------ | ------------------------------------------------------------- | +| `/help` | Grouped command list | +| `/init` | Bootstrap `AGENTS.md` + session-log folders for a new project | +| `/log` | Durable session log: work / findings / open Qs / next steps | +| `/jobs` | Active background work — running, scheduled, watches | +| `/outputs` | Browse research artifacts | +| `/search ` | Search prior session transcripts (semantic + keyword) | +| `/preview [path]` | Render artifact as HTML or PDF | +| `/feynman-model` | Model picker (default model + per-subagent overrides) | + +## Slash commands — pi-subagents (when installed) + +`pi-subagents` adds direct agent control: + +| Command | What it does | +| ---------------------------------------------------- | ----------------------------------------- | +| `/run ` | Run one agent with a task | +| `/chain "task" -> "task"` | Sequential chain | +| `/parallel "task" -> "task"` | Parallel run | +| `/subagents-status` | Async status overlay | +| `/agents` | Agents Manager overlay (edit / override) | + +Per-step task forms: +- `/chain scout "scan code" -> planner "make plan"` — each step has its own task +- `/chain scout -- scan code -> planner -- analyze auth` — `--` delimiter +- `/chain scout "analyze auth" -> planner -> implementer` — bare steps inherit `{previous}` (chain) or first task (parallel) +- Shared task: `/parallel scout reviewer -- audit src/` — same task to both + +Quotes (`"…"` or `'…'`) and `--` are interchangeable. + +## Sample REPL session + +``` +$ feynman +Feynman 0.2.39 — github-copilot/gpt-5.3-codex (medium thinking) +> /deepresearch Current approaches to mechanistic interpretability in LLMs +[plan] Reading 12 source candidates… +[researcher] Fetching arXiv:2401.… +[writer] Drafting brief +✓ outputs/mechanistic-interp-brief.md (2,341 lines, 47 sources) +> /preview outputs/mechanistic-interp-brief.md +✓ Opened in browser +> /log +✓ Wrote outputs/session-log-2026-05-01.md +``` + +## Tips + +- Long-running workflows (`/deepresearch`, `/lit`) run for minutes. Use + `/jobs` from another window to monitor. +- `/watch weekly` schedules via `pi-schedule-prompt`. List with + `/jobs`. Cancel by editing the schedule (no built-in unwatch verb). +- `/preview` requires pandoc (`feynman setup preview` once). +- `/feynman-model` lets you set different models per role — + e.g. cheap Haiku for `researcher` (high token volume), Opus for `reviewer`. +- `@` works with directories — Feynman recursively reads supported + formats (`pi-docparser` handles PDFs/Office/etc). +- Slash commands shadow built-ins of the same name when defined as a + custom command. Project-scope shadows user-scope. diff --git a/personas/_shared/community-skills/foia-tool/SKILL.md b/personas/_shared/community-skills/foia-tool/SKILL.md new file mode 100644 index 0000000..0dd973c --- /dev/null +++ b/personas/_shared/community-skills/foia-tool/SKILL.md @@ -0,0 +1,520 @@ +--- +name: foia-tool +description: Use when downloading, scraping, analyzing, or annotating declassified government documents with the foiacquire/foia Rust tool. Covers config writing, CLI commands, PostgreSQL setup, DocumentCloud API, OCR pipeline, Komga integration, foia-monitor web panel, and LLM annotation. +--- + +# FOIA Tool — Complete Reference + +## Tool Location + +| Item | Path | +|------|------| +| Repo | `~/Documents/foia/` | +| Binary | `~/Documents/foia/target/release/foia` | +| Active config | `/mnt/storage/Common/Books/FOIA/foia.json` | +| Data dir | `/mnt/storage/Common/Books/FOIA/` | +| Docs | `~/Documents/foia/docs/` | +| README | `/mnt/storage/Common/Books/FOIA/README.md` | +| Monitor | `~/Documents/foia-monitor/` → http://localhost:3031 | + +**Always run with `-d` flag pointing to data dir:** +```bash +FOIA_DIRECT=1 ~/Documents/foia/target/release/foia -d /mnt/storage/Common/Books/FOIA/ --no-tor-warning +``` + +--- + +## Building + +Default build = SQLite only. For PostgreSQL support: +```bash +cd ~/Documents/foia +cargo build --release --features postgres -p foia-cli +``` + +Other optional features: `--features gis` (PostGIS spatial search), `--features browser` (headless browser). + +--- + +## Database + +### SQLite (default) +```json +{ "database": "foia.db" } +``` +**Problem:** Deadlocks with 4+ concurrent workers. Use `--workers 2` or switch to PostgreSQL. + +### PostgreSQL (Docker container: `foia_postgres`) +```json +{ "database": "postgresql://foia_user:@localhost:5432/foia_db" } +``` +Start container: `docker start foia_postgres` +Init DB: `foia -d /mnt/storage/Common/Books/FOIA/ init --no-tor-warning` + +**⚠️ PostgreSQL migration bug:** Some migrations use multi-statement SQL that fails. If `init` errors on a migration, manually apply SQL via `docker exec foia_postgres psql -U foia_user -d foia_db` then mark done: +```sql +INSERT INTO __cetane_migrations (name) VALUES ('MIGRATION_NAME') ON CONFLICT DO NOTHING; +``` + +**⚠️ New scrapers not auto-synced:** `foia init` does NOT add new scrapers from foia.json to `scraper_configs` table. Must INSERT the **full config JSON** manually — empty `{}` causes the scraper to find 0 URLs and exit silently: +```sql +INSERT INTO scraper_configs (source_id, config, created_at, updated_at) +VALUES ('new_source', '{"discovery":{"type":"api_paginated",...}}', NOW()::TEXT, NOW()::TEXT) +ON CONFLICT DO NOTHING; +``` + +**Quick bulk-sync from foia.json** (Python): +```python +import json, subprocess +with open('/mnt/storage/Common/Books/FOIA/foia.json') as f: + cfg = json.load(f) +for source_id, config in cfg['scrapers'].items(): + config_json = json.dumps(config).replace("'", "''") + sql = f"INSERT INTO scraper_configs (source_id, config, created_at, updated_at) VALUES ('{source_id}', '{config_json}', NOW()::TEXT, NOW()::TEXT) ON CONFLICT (source_id) DO UPDATE SET config=EXCLUDED.config, updated_at=NOW()::TEXT;" + subprocess.run(['docker', 'exec', 'foia_postgres', 'psql', '-U', 'foia_user', '-d', 'foia_db', '-c', sql]) +``` + +**Actual schema** (NOT `config_json`/`is_active`): +```sql +-- source_id TEXT PK, config TEXT, created_at TEXT, updated_at TEXT +``` + +--- + +## foia.json — Full Config Reference + +```json +{ + "target": "/mnt/storage/Common/Books/FOIA/", + "database": "postgresql://...", + "request_delay_ms": 1500, + "request_timeout": 45, + "user_agent": "Mozilla/5.0 ...", + "default_refresh_ttl_days": 30, + + "llm": { "enabled": false }, + + "analysis": { + "ocr_backends": ["tesseract", "pdftotext"], + "pdf_text_enabled": true, + "extract_urls": false + }, + + "privacy": { "direct": true }, + + "scrapers": { "SOURCE_ID": { ... } } +} +``` + +--- + +## Discovery Types + +### 1. `html_crawl` +```json +"discovery": { + "type": "html_crawl", + "base_url": "https://vault.fbi.gov", + "start_paths": ["/alphabetical-index"], + "document_links": ["a[href*='/vault/']"], + "document_patterns": ["\\.pdf$"], + "max_depth": 3, + "use_browser": false, + "pagination": { + "next_selectors": ["a[rel='next']", "a.pager-next"], + "max_pages": 100 + } +} +``` + +**With search queries** (crawl site search pages): +```json +"search_queries": ["CIA Turkey", "CIA KGB"], +"search_url_template": "/search?q={query}" +``` + +**With LLM query expansion:** +```json +"expand_search_terms": true +``` + +### 2. `api_paginated` +```json +"discovery": { + "type": "api_paginated", + "api": { + "base_url": "https://api.www.documentcloud.org", + "endpoint": "/api/documents/search/?q=CIA+Turkey+declassified", + "pagination": { + "page_param": "page", + "page_size_param": "per_page", + "page_size": 25, + "results_path": "results" + }, + "url_extraction": { + "url_field": "_none", + "url_template": "https://assets.documentcloud.org/documents/{id}/{slug}.pdf" + } + } +} +``` + +### 3. `api_cursor` +```json +"discovery": { + "type": "api_cursor", + "api": { + "base_url": "https://api.example.com", + "endpoint": "/search?q=FOIA&limit=100", + "pagination": { + "cursor_param": "cursor", + "cursor_response_path": "pagination.next_cursor", + "results_path": "results", + "page_size": 100 + }, + "url_extraction": { "url_field": "download_url" } + } +} +``` + +### 4. `api_nested` (parent→child) +For APIs where documents are nested (e.g., FOIA requests → communications → files): +```json +"discovery": { + "type": "api_nested", + "api": { + "base_url": "https://www.muckrock.com/api_v1", + "parent": { + "endpoint": "/foia/?status=done&per_page=100", + "pagination": { "page_param": "page", "results_path": "results" }, + "id_path": "id" + }, + "child": { + "endpoint_template": "/communication/?foia={id}", + "results_path": "results", + "url_extraction": { + "url_field": "url", + "nested_arrays": ["files"] + } + } + } +} +``` + +--- + +## URL Extraction Options + +```json +"url_extraction": { + "url_field": "download_url", // Field containing URL (use "_none" for templates) + "url_template": "https://example.com/docs/{id}/{slug}.pdf", // Template with {field} vars + "fallback_field": "pdf_url", // Try this if url_field is empty + "items_path": "data.documents", // JSON path to array if not top-level + "nested_arrays": ["files", "attachments"] // Traverse nested arrays +} +``` + +--- + +## External Discovery (sitemap, Wayback, search engines) + +Add to any `discovery` block: +```json +"external": { + "enable_sitemap": true, // Parse sitemap.xml and robots.txt + "enable_wayback": true, // Query Wayback Machine CDX API + "common_paths": ["/foia/docs"], // Extra paths to enumerate + "search_engines": [ + { + "engine": "duckduckgo", // "duckduckgo" | "google" | "bing" | "brave" + "enabled": true, + "queries": ["CIA declassified site:vault.fbi.gov"] + } + ] +} +``` + +--- + +## Per-Source Overrides + +```json +"my_source": { + "request_delay_ms": 2000, // Override global delay + "request_timeout": 60, // Override global timeout + "refresh_ttl_days": 7, // Re-scrape interval + "user_agent": "...", // Override user-agent + "via": { // CDN proxy rewrites + "assets.documentcloud.org": "https://proxy.example.com/dc/" + }, + "via_mode": "fallback", // "strict" | "fallback" | "priority" + "privacy": { "direct": true } // Per-source privacy override +} +``` + +--- + +## Browser Config (requires `--features browser`) + +```json +"browser": { + "enabled": true, + "engine": "stealth", // "standard" | "stealth" | "cookies" + "headless": true, + "timeout": 60, + "wait_for_selector": ".content-loaded", + "cookies_file": "./site_cookies.json", // For "cookies" engine + "proxy": "socks5://127.0.0.1:9050" +} +``` + +Get cookies interactively: +```bash +foia browser-test https://example.gov/login --headed --save-cookies cookies.json +``` + +**Binary fetch** (for Akamai/Cloudflare-protected PDF endpoints): +```json +"fetch": { "use_browser": true, "binary_fetch": true } +``` + +--- + +## CLI Commands + +### Status & Init +```bash +foia status # Queue/doc counts, running services +foia init # Init DB and dirs (run after changing DB) +foia state clear SOURCE_ID # Clear stale scraper status +``` + +### Discovery & Scraping +```bash +foia crawl SOURCE_ID --limit 50 # Discover URLs only (test config) +foia scrape SOURCE_ID --workers 2 # Crawl + download combined +foia scrape --all --workers 2 --daemon --interval 3600 # All sources, continuous +foia scrape SOURCE_ID --reload=inplace # Hot-reload config without restart +``` + +### Downloading +```bash +foia download --workers 2 --limit 100 # Process download queue (2 workers = safe for SQLite) +foia download SOURCE_ID --workers 4 # Download specific source (PostgreSQL: 4 workers OK) +``` + +### Import +```bash +foia import url "https://..." --source SOURCE_ID --title "Title" +foia import warc file.warc.gz --source SOURCE_ID --dry-run +foia import urls --file urls.txt --source SOURCE_ID +``` + +### Processing Pipeline +```bash +foia analyze --all --workers 4 # OCR + text extraction +foia analyze SOURCE_ID --daemon --interval 300 +foia annotate --all # LLM summaries + tags +foia extract-entities SOURCE_ID # NER: people, orgs, locations, file numbers +foia detect-dates SOURCE_ID # Estimate document dates with LLM +``` + +### Search & Browse +```bash +foia ls --source SOURCE_ID --limit 50 --format table +foia search "Turkey intelligence 1980" +foia search-entities "Kissinger" --type person +foia info DOC_ID +foia read DOC_ID --text +foia serve # Web UI at http://127.0.0.1:3030 +``` + +### Database & Config +```bash +foia db copy sqlite:old.db postgres://user:pass@host/db --progress +foia config recover # Generate skeleton config from existing DB +foia config restore # Restore last saved config +``` + +--- + +## Pipeline Flow + +``` +crawl/scrape → download → analyze (OCR) → annotate (LLM) → search/serve +``` + +Documents stored at: `documents/{sha256[0:2]}/{sha256[2:10]}/{filename}` +Filename format: `{slug}-{hash[0:8]}.pdf` + +**⚠️ `foia analyze` has no `--all` flag:** +```bash +foia analyze --workers 4 # no source = all sources (correct) +foia analyze --all --workers 4 # ERROR — --all is not valid +``` + +--- + +## Komga Integration + +### komga_organize.py +Moves downloaded PDFs from hash-based `documents/` into a Komga folder hierarchy by source ID. + +**Critical:** After moving, creates symlinks at the original hash path so `foia analyze` can still find files: +``` +documents/{hash[0:2]}/{hash[2:10]}/file.pdf → /mnt/storage/Common/Books/Istihbarat/CIA/file.pdf +``` + +Without symlinks, `foia analyze` can't find moved files (`file_path` is always NULL in DB; path is reconstructed from `content_hash`). + +```bash +python3 /mnt/storage/Common/Books/FOIA/komga_organize.py +# or via .zshrc alias: foia-organize +``` + +### ocrmypdf_batch.py +Embeds invisible text layer into scanned PDFs in Komga folders — makes them searchable inside Komga. + +```bash +python3 /mnt/storage/Common/Books/FOIA/ocrmypdf_batch.py +``` + +- Uses `--skip-text` (skips pages that already have text) +- 4 parallel workers, replaces in-place (temp file → shutil.move) +- Exit code 2 = already has text (counted as "skipped", not error) +- Error log: `/tmp/ocrmypdf_batch.log` +- Processes files from KOMGA_DIRS (not from `documents/`) + +**Source → Komga folder map** (23 sources): +``` +cia_reading_room → Istihbarat/CIA/ +cia_turkey → Istihbarat/FOIA-CIA-Turkey/ +fbi_vault → Istihbarat/FOIA-FBI-Vault/ +cia_nato → NATO/FOIA-NATO/ +military_doctrine → AskeriDoktrin/FOIA-Pentagon/ +cyber_warfare → SiberGuvenlik/FOIA-SiberSavas/ +ia_cia_cold_war → Istihbarat/FOIA-IA-CIA-SogukSavas/ +# ... see komga_organize.py for full map +``` + +--- + +## foia-monitor Web Panel + +**Repo:** `~/Documents/foia-monitor/` +**Start:** `foia-monitor` (.zshrc alias) or `bash ~/Documents/foia-monitor/start.sh` +**URL:** http://localhost:3031 + +Architecture: Express.js server (port 3031) serves built React frontend + REST API + WebSocket. + +**Tabs:** +- **MONITOR** — live log stream, start/stop scraper & OCR, per-source quick-scrape buttons +- **SOURCES** — per-source document counts with progress bars +- **DOCUMENTS** — full-text search, source filter dropdown, paginated results, PDF links (proxied via foia serve API) +- **ORGANIZE** — run komga_organize.py from the UI + +**API endpoints:** +``` +GET /api/stats → {total, diskCount, queuePending, sources[], scraperRunning} +POST /api/command → {cmd: start|stop|restart|start-ocr|stop-ocr|organize} +POST /api/scrape-source → {source: source_id} +``` + +**Build after UI changes:** +```bash +cd ~/Documents/foia-monitor/client && npm run build +# then restart: fuser -k 3031/tcp && cd server && node index.js & +``` + +--- + +## Environment Variables + +| Variable | Purpose | +|----------|---------| +| `FOIA_DIRECT=1` | Disable Tor (direct connection) | +| `SOCKS_PROXY=socks5://127.0.0.1:9050` | External SOCKS5/Tor proxy | +| `RUST_LOG=debug` | Verbose logging | +| `LLM_PROVIDER=ollama` | Override LLM provider | +| `LLM_MODEL=llama3.1:8b` | Override LLM model | +| `LLM_ENDPOINT=http://localhost:11434` | Override LLM endpoint | +| `GROQ_API_KEY=...` | Use Groq for OCR/annotation | +| `OPENAI_API_KEY=...` | Use OpenAI | +| `DATABASE_URL=postgres://...` | Override database (full URL) | +| `BROWSER_URL=ws://localhost:9222` | Remote Chrome DevTools | + +--- + +## Available OCR Backends + +| Backend | Best For | +|---------|---------| +| `tesseract` | Scanned PDFs (eng + tur installed) | +| `ocrs` | Pure Rust, offline | +| `paddle` | CJK languages, complex layouts | +| `groq` | API-based, fast (requires key) | + +pdftotext (poppler) handles digital PDFs automatically. + +--- + +## Site Compatibility Notes + +| Site | Method | Notes | +|------|--------|-------| +| DocumentCloud | `api_paginated` | Rate limits heavily; 503 = backoff 60s. IP-blocks Tor exits. | +| FBI Vault | `html_crawl` | Works well, occasional 503 backoff | +| CIA Reading Room | `html_crawl` + `use_browser: true` | Requires browser feature | +| Wilson Center | — | **Cloudflare-blocked**, cannot scrape | +| NSA.gov | — | JS-heavy, no static PDF links | +| MuckRock | `api_nested` | `/api_v1/foia/` → `/api_v1/communication/` chain | +| NARA/archives.gov | — | JS-heavy search, static PDFs need specific paths | + +--- + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| `params` field in config | Embed query directly in `endpoint` string | +| `endpoints` array format | Use single `endpoint` string | +| `url_field: "pdf_url"` for DocumentCloud | Use `url_template` with `{id}/{slug}` | +| SQLite deadlock with 4 workers | Use `--workers 2` or switch to PostgreSQL | +| Binary compiled without postgres | `cargo build --release --features postgres -p foia-cli` | +| Status command errors "database is locked" | Too many concurrent processes, kill extras | +| `foia status` shows stale scrapers | `foia state clear SOURCE_ID` | +| New source shows "No scraper configured" or 0 URLs found | INSERT into `scraper_configs` with FULL config JSON (not `{}`); schema: `source_id, config, created_at, updated_at` | +| DocumentCloud "error sending request" via Tor | DC blocks Tor exit nodes; use `FOIA_DIRECT=1` | + +--- + +## Quick Workflow: Add New DocumentCloud Source + +1. Add to `/mnt/storage/Common/Books/FOIA/foia.json` under `scrapers`: +```json +"new_source": { + "discovery": { + "type": "api_paginated", + "api": { + "base_url": "https://api.www.documentcloud.org", + "endpoint": "/api/documents/search/?q=YOUR+QUERY+HERE", + "pagination": { "page_param": "page", "page_size_param": "per_page", "page_size": 25, "results_path": "results" }, + "url_extraction": { "url_field": "_none", "url_template": "https://assets.documentcloud.org/documents/{id}/{slug}.pdf" } + } + }, + "fetch": { "use_browser": false } +} +``` +2. Sync to DB (full config, NOT empty `{}`): +```python +import json, subprocess +cfg = json.load(open('/mnt/storage/Common/Books/FOIA/foia.json')) +for sid, config in cfg['scrapers'].items(): + cj = json.dumps(config).replace("'", "''") + sql = f"INSERT INTO scraper_configs (source_id, config, created_at, updated_at) VALUES ('{sid}', '{cj}', NOW()::TEXT, NOW()::TEXT) ON CONFLICT (source_id) DO UPDATE SET config=EXCLUDED.config, updated_at=NOW()::TEXT;" + subprocess.run(['docker', 'exec', 'foia_postgres', 'psql', '-U', 'foia_user', '-d', 'foia_db', '-c', sql]) +``` +3. Test: `FOIA_DIRECT=1 foia crawl new_source --limit 5 --no-tor-warning` +4. Scrape: `FOIA_DIRECT=1 foia scrape new_source --workers 2 --no-tor-warning` +5. Analyze: `FOIA_DIRECT=1 foia analyze --workers 4 --no-tor-warning` diff --git a/personas/_shared/community-skills/intel-briefing/SKILL.md b/personas/_shared/community-skills/intel-briefing/SKILL.md new file mode 100644 index 0000000..d38871e --- /dev/null +++ b/personas/_shared/community-skills/intel-briefing/SKILL.md @@ -0,0 +1,126 @@ +--- +name: intel-briefing +description: Generate structured intelligence briefings in IC format. Supports EXEC_SUMMARY, FULL_INTEL_REPORT, JSON, and NEED_VISUAL output modes. Uses UAP methodology, ACH-over-ToT for competing hypotheses, IC confidence levels. Triggers on briefing, intelligence report, assessment, geopolitical analysis, country analysis, threat assessment, PDB format. +--- + +# Intel Briefing — Intelligence Product Generator + +## When to Use + +| Trigger | Action | +|---------|--------| +| "Iran briefing" / "Russia assessment" | Full intel briefing on topic | +| "[EXEC_SUMMARY] Turkey defense" | 1-page BLUF executive summary | +| "[FULL_INTEL_REPORT] nuclear" | Multi-section deep analysis | +| "Threat assessment for [region]" | Structured threat evaluation | +| "Compare hypotheses on [event]" | ACH-over-ToT analysis | + +## Output Modes + +### [EXEC_SUMMARY] — Executive Summary +```markdown +## BLUF +[1 paragraph — the bottom line assessment with confidence level] + +## Key Findings +1. [Finding with confidence: High/Moderate/Low] +2. [Finding] +3. [Finding] + +## Outlook +- Most Likely: [scenario] +- Wild Card: [low-probability high-impact] + +## Indicators to Watch +- [What would change this assessment] +``` + +### [FULL_INTEL_REPORT] — Complete Assessment +```markdown +# Intelligence Assessment: [Topic] +Report ID: IA-[YYYY]-[NNN] +Classification: UNCLASSIFIED +Date: [Date] + +## BLUF +[Bottom line in 1 paragraph] + +## Background +[Context, history, relevant developments] + +## Key Findings +[Evidence-based observations with source citations] + +## Analysis +### Hypothesis 1: [Description] +- Supporting evidence: [...] +- Contradicting evidence: [...] +- Confidence: [High/Moderate/Low] — [percentage] + +### Hypothesis 2: [Alternative] +- Supporting evidence: [...] +- Confidence: [...] + +### ACH Matrix +| Evidence | H1 | H2 | H3 | +|----------|-----|-----|-----| +| [fact] | ++ | - | N | + +## Outlook & Scenarios +| Scenario | Probability | Key Indicators | +|----------|------------|----------------| +| Most Likely | [%] | [what to watch] | +| Best Case | [%] | [triggers] | +| Worst Case | [%] | [triggers] | +| Wild Card | [%] | [triggers] | + +## Assumptions & Gaps +- [Key assumption that if wrong changes everything] +- [Collection gap that limits confidence] + +## Sources & Methodology +- [Source list with reliability ratings] +- SATs applied: [ACH, Red Hat, Key Assumptions Check, etc.] +``` + +### [JSON_OUTPUT] — Structured Data +```json +{ + "report_id": "IA-2026-001", + "topic": "...", + "bluf": "...", + "confidence": "Moderate", + "confidence_pct": 72, + "hypotheses": [...], + "findings": [...], + "scenarios": [...], + "indicators": [...], + "sources": [...] +} +``` + +## Methodology: UAP (Universal Analytical Protocol) + +1. **Direction** — Define KIQs, select response mode, scope boundaries +2. **Collection** — OSINT sweep, RAG knowledge retrieval, source evaluation (Admiralty Code) +3. **Analysis** — ACH-over-ToT, multi-source triangulation, SATs, Devil's Advocate +4. **Production** — Format per output mode, BLUF first, mandatory confidence levels +5. **Dissemination** — Deliver with caveats, handling instructions, feedback loop + +## Confidence Language (IC Standard) + +| Level | Probability | Language | +|-------|------------|---------| +| High | 90-100% | "almost certainly", "we assess with high confidence" | +| Moderate | 60-89% | "likely", "we assess with moderate confidence" | +| Low | 50-59% | "roughly even chance", "we assess with low confidence" | +| Very Low | <50% | "unlikely", "remote possibility" | + +## Rules + +- BLUF first, always — if the reader stops after 1 paragraph, they have the core judgment +- Competing hypotheses mandatory — never single-explanation analysis +- Distinguish FACTS (verified) from ASSESSMENTS (judgments) from SPECULATION (conjecture) +- Cite sources with reliability rating (A-F reliability, 1-6 credibility) +- Minimum 3 independent sources per major claim +- State assumptions explicitly — the assumption you don't question is the one that burns you diff --git a/personas/_shared/community-skills/jira/SKILL.md b/personas/_shared/community-skills/jira/SKILL.md new file mode 100644 index 0000000..f3b11a4 --- /dev/null +++ b/personas/_shared/community-skills/jira/SKILL.md @@ -0,0 +1,325 @@ +--- +name: jira +description: Use when the user mentions Jira, tasks, tickets, sprints, boards, or wants to view/create/update/transition issues. Triggers on any Jira-related request including listing tasks, checking status, creating tickets, updating progress, or managing sprints on Atlassian Jira. +--- + +# Jira — Atlassian Jira Integration + +## Overview + +Interact with Atlassian Jira via REST API. View boards, list issues, create/update tickets, transition statuses, add comments, and manage sprints. Uses `curl` with Basic Auth (email + API token). + +## Authentication + +**Credentials** (stored securely, never log tokens): +- **Instance**: `.atlassian.net` +- **Email**: `` +- **API Token**: Read from `~/.jira-token` file (one line, no newline) + +If `~/.jira-token` doesn't exist, ask the user to create it: +```bash +echo -n "YOUR_API_TOKEN" > ~/.jira-token +chmod 600 ~/.jira-token +``` + +**Base URL**: `https://.atlassian.net/rest/api/3` +**Auth Header**: Basic Auth with `email:token` + +## API Helper + +For ALL Jira API calls, use this pattern: + +```bash +JIRA_EMAIL="" +JIRA_TOKEN=$(cat ~/.jira-token) +JIRA_BASE="https://.atlassian.net/rest/api/3" + +# GET request +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" "$JIRA_BASE/endpoint" + +# POST request (search, create) +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X POST "$JIRA_BASE/endpoint" \ + -H "Content-Type: application/json" \ + -d '{"key": "value"}' + +# PUT request (update) +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X PUT "$JIRA_BASE/endpoint" \ + -H "Content-Type: application/json" \ + -d '{"fields": {"summary": "new title"}}' +``` + +## Projects + +Known projects on this instance: +| Key | Name | Description | +|-----|------|-------------| +| CTI | ProudGuard-CTI | Pentests, exposure scans, customer engagements | +| LZY | proudguard | ASM Aygan development, internal tasks | +| DEVOPS | proudguard-devops | Infrastructure, deployment | +| PB | proudguard-Backlog | Product backlog | +| SELF | DukkanıKapatanlar | Side projects | + +## Common Operations + +### 1. List My Tasks +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X POST "$JIRA_BASE/search/jql" \ + -H "Content-Type: application/json" \ + -d '{"jql":"assignee=currentUser() AND status NOT IN (Tamam, Done) ORDER BY updated DESC","maxResults":30,"fields":["summary","status","priority","project","updated"]}' +``` + +### 2. List Tasks in Project +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X POST "$JIRA_BASE/search/jql" \ + -H "Content-Type: application/json" \ + -d '{"jql":"project=CTI ORDER BY updated DESC","maxResults":30,"fields":["summary","status","priority","assignee","updated"]}' +``` + +### 3. Get Single Issue Detail +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" "$JIRA_BASE/issue/CTI-40" +``` + +### 4. Create Issue +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X POST "$JIRA_BASE/issue" \ + -H "Content-Type: application/json" \ + -d '{ + "fields": { + "project": {"key": "CTI"}, + "summary": "Issue title", + "description": {"type":"doc","version":1,"content":[{"type":"paragraph","content":[{"type":"text","text":"Description here"}]}]}, + "issuetype": {"name": "Task"}, + "priority": {"name": "Medium"} + } + }' +``` + +### 5. Update Issue Status (Transition) + +**CRITICAL RULE: NEVER transition to `Tamam` (Done, id=31).** Only managers close tickets. We move tasks to `İNCELEMEDE` (In Review, id=2) or `Test` (Testing, id=3) only. If a task is already in İNCELEMEDE or Test, just add a comment — don't change status. + +Allowed transitions: +- `id=2` — İNCELEMEDE (In Review) — use after completing work +- `id=3` — Test (Testing) — use when ready for QA +- `id=21` — Devam Ediyor (In Progress) — use when starting work +- `id=11` — Yapılacaklar (To Do) — use to reset +- ~~`id=31` — Tamam (Done)~~ — **FORBIDDEN** — manager only + +First get available transitions: +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" "$JIRA_BASE/issue/CTI-40/transitions" +``` + +Then transition (NEVER use id=31): +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X POST "$JIRA_BASE/issue/CTI-40/transitions" \ + -H "Content-Type: application/json" \ + -d '{"transition": {"id": "2"}}' +``` + +### 6. Add Comment +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X POST "$JIRA_BASE/issue/CTI-40/comment" \ + -H "Content-Type: application/json" \ + -d '{"body":{"type":"doc","version":1,"content":[{"type":"paragraph","content":[{"type":"text","text":"Comment text here"}]}]}}' +``` + +### 7. Update Issue Fields +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X PUT "$JIRA_BASE/issue/CTI-40" \ + -H "Content-Type: application/json" \ + -d '{"fields": {"summary": "Updated title", "priority": {"name": "High"}}}' +``` + +### 8. Assign Issue +```bash +# First find account ID +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" "$JIRA_BASE/user/search?query=ali" + +# Then assign +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X PUT "$JIRA_BASE/issue/CTI-40/assignee" \ + -H "Content-Type: application/json" \ + -d '{"accountId": "ACCOUNT_ID"}' +``` + +### 9. Get Board/Sprint Info +```bash +# List boards +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" "https://.atlassian.net/rest/agile/1.0/board" + +# Get sprint issues +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" "https://.atlassian.net/rest/agile/1.0/board/BOARD_ID/sprint/SPRINT_ID/issue" +``` + +### 10. Search with JQL +```bash +# Issues updated in last 7 days +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X POST "$JIRA_BASE/search/jql" \ + -H "Content-Type: application/json" \ + -d '{"jql":"updated >= -7d ORDER BY updated DESC","maxResults":20}' + +# Unresolved high priority +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X POST "$JIRA_BASE/search/jql" \ + -H "Content-Type: application/json" \ + -d '{"jql":"priority = High AND resolution = Unresolved","maxResults":20}' +``` + +## Status Values (Turkish) + +Known status names on this instance: +- `Yapılacaklar` — To Do +- `Devam Ediyor` — In Progress +- `İNCELEMEDE` — In Review +- `Test` — Testing +- `Tamam` — Done +- `Daily` — Recurring + +## Output Formatting + +When displaying issues, use this table format: +``` +Key Project Status Priority Summary +CTI-40 CTI Devam Ediyor Medium Tailwind Airlines EXP + ASM +``` + +When displaying a single issue, show: +- Key, Summary, Status, Priority, Assignee +- Description (first 200 chars) +- Comments (last 3) +- Subtasks if any +- Links if any + +## Important Notes + +- **API version**: Use `/rest/api/3` (v3), NOT v2 +- **Search endpoint**: Use `POST /rest/api/3/search/jql` (NOT GET `/rest/api/3/search`) +- **Description format**: Atlassian Document Format (ADF), not plain text +- **Rate limit**: 100 requests per minute per user +- **Never log or display the API token** — always read from `~/.jira-token` +- **Transition IDs vary** — always fetch transitions first before attempting status change + +## Comment Hygiene (CRITICAL) + +**Don't spam multiple comments on the same ticket.** If you've already commented and new info arrives, UPDATE the existing comment instead of stacking new ones. User preference: clean single-source-of-truth comment per ticket. + +### Update an existing comment +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X PUT "$JIRA_BASE/issue/LZY-228/comment/12864" \ + -H "Content-Type: application/json" \ + -d '{"body":{"type":"doc","version":1,"content":[{"type":"paragraph","content":[{"type":"text","text":"Updated consolidated content..."}]}]}}' +# Returns 200 on success; Jira adds "edited" badge automatically +``` + +### Delete a redundant comment +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X DELETE "$JIRA_BASE/issue/LZY-228/comment/12873" +# Returns 204 on success +``` + +### List comments with IDs (needed for update/delete) +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" "$JIRA_BASE/issue/LZY-228/comment?orderBy=-created&maxResults=5" \ + | jq -r '.comments[] | "\(.id)\t\(.created[0:19])\t\(.author.displayName)\t\(.body.content[0].content[0].text[0:70] // "null")"' +``` + +### Update issue description (when rewriting scope) +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" -X PUT "$JIRA_BASE/issue/LZY-228" \ + -H "Content-Type: application/json" \ + -d '{"fields":{"description":{"type":"doc","version":1,"content":[...]}}}' +``` + +**Rule of thumb:** if you're about to post a second Ali-comment with a correction or additional context on the same ticket, stop — update the first one instead. Only post a new comment if it's a genuinely new event (a follow-up status change days later, a reply to someone else). + +## Verified Transitions (ProudGuard Jira — LZY/CTI/PB/DEVOPS projects) + +Confirmed via `GET /issue/LZY-230/transitions` (2026-04-18). These IDs are stable across the instance: + +| ID | Name | Target Status | +|----|------|---------------| +| 11 | Yapılacaklar | To Do | +| 21 | Devam Ediyor | In Progress | +| 3 | Testing | Test | +| 2 | In Review | İNCELEMEDE | +| 31 | Tamam | Done — **FORBIDDEN** | + +**Reminder:** ID=31 (Tamam) is manager-only. Never transition to it even if you're certain the work is done. Max-state is id=2 (İNCELEMEDE) or id=3 (Test). If already at İNCELEMEDE or Test, just add a comment or update the existing one — don't change status. + +## ADF Formatting Patterns (working examples) + +### Paragraph with bold prefix +```json +{"type":"paragraph","content":[ + {"type":"text","text":"Durum (2026-04-18): ","marks":[{"type":"strong"}]}, + {"type":"text","text":"Faz tamamlandı."} +]} +``` + +### Full-bold paragraph +```json +{"type":"paragraph","content":[ + {"type":"text","text":"Heading text","marks":[{"type":"strong"}]} +]} +``` + +### Inline code +```json +{"type":"text","text":"severity_engine.py","marks":[{"type":"code"}]} +``` + +### Code block +```json +{"type":"codeBlock","attrs":{"language":"bash"},"content":[ + {"type":"text","text":"python -m pytest tests/ -v"} +]} +``` + +### Bullet list +```json +{"type":"bulletList","content":[ + {"type":"listItem","content":[{"type":"paragraph","content":[{"type":"text","text":"item 1"}]}]}, + {"type":"listItem","content":[{"type":"paragraph","content":[{"type":"text","text":"item 2"}]}]} +]} +``` + +### Safe-quote escaping in curl -d JSON payloads +Use `\u2019` for right single quote (`'`) inside JSON string literals when shell-escaping gets hairy: +- `Aygan\u2019ın ayrı ürün` renders as `Aygan'ın ayrı ürün` +- Backslash escaping single quotes (`'\''`) works but is error-prone in long payloads + +## Professional Comment Standard (ProudStar/Aygan projects) + +When commenting on a work ticket, include: +1. **Status line with date**: `**Durum (2026-04-18): **` +2. **Commit references**: `commit abc1234 — ` (use 7-char short hashes) +3. **File references**: `shared/services/severity_engine.py:L61` (file:line format) +4. **Test metrics**: `49 unit tests + 63 regression tests passed` +5. **Production evidence where applicable**: doc counts, live data snapshots +6. **Next action / recommended status change** + +Avoid generic "milestone closed" blurbs — every comment should cite specific code or live-data evidence. A manager skimming the ticket should be able to verify the claim without asking follow-up questions. + +## Parsing ADF Descriptions Safely + +Descriptions come back as nested ADF. Extract text with jq: +```bash +jq -r '.fields.description.content // [] | map(.content // [] | map(.text // "") | join(" ")) | join("\n")' +``` +This survives missing descriptions, empty paragraphs, and mixed content types (image nodes without text). For richer extraction (tables, code blocks), walk `.content` recursively. + +## Getting currentUser accountId + +For assign/search operations you sometimes need the raw accountId: +```bash +curl -s -u "$JIRA_EMAIL:$JIRA_TOKEN" "$JIRA_BASE/myself" | jq -r '.accountId' +# Example output: 712020:fcf72a82-bfec-4cf9-9948-7c8f44480c37 +``` + +## Common Mistakes (learned the hard way) + +1. **`jq -w` is not a flag** — use `curl -w "HTTP: %{http_code}\n"` for HTTP status, pipe separately to jq. +2. **`search/jql` needs POST, not GET** — GET returns 405. Body is JSON with `{"jql":"...","maxResults":N,"fields":[...]}`. +3. **`fields:["comment"]` in search returns only count+last 1** — for full comment history use `GET /issue/{key}/comment?maxResults=50` as a separate call. +4. **Comment author displayName may be a bot or integration user** — always check `.author.accountId` matches yours before editing/deleting. Don't delete someone else's comment. +5. **Spamming comments triggers ticket noise and manager frustration** — the "Updates" tab on each ticket shows every edit; 5 Ali-comments in 10 minutes looks worse than 1 consolidated comment. diff --git a/personas/_shared/community-skills/librarian/SKILL.md b/personas/_shared/community-skills/librarian/SKILL.md new file mode 100644 index 0000000..85464e6 --- /dev/null +++ b/personas/_shared/community-skills/librarian/SKILL.md @@ -0,0 +1,251 @@ +--- +name: librarian +description: Use when organizing, categorizing, renaming, or moving PDF/epub books in a library. Triggers on book classification, library reorganization, fixing garbled filenames, sorting documents into topic folders, Komga library maintenance, or any "organize these books/PDFs" request. +--- + +# Librarian - PDF Library Organizer + +## Overview + +Systematically classify, rename, and organize PDF/epub files into a topic-based folder hierarchy by **reading actual content** (not just filenames). Every file lands in the right place with a clean name. Uses Claude Code tools (Read, Bash, Glob, Grep, Agent) natively. + +## Core Principles + +1. **READ BEFORE CLASSIFY** — Read minimum 20 pages of each PDF via `Read` tool with `pages: "1-20"`. Filenames lie (OCR artifacts, download garbage, wrong metadata). Only the actual content is truth. +2. **NEVER DELETE** — Move unwanted files to `Arsiv/` subfolder. Deletion decisions belong to the user only. +3. **NO SPACES IN FOLDER NAMES** — Use CamelCase: `AskeriTarih/StratejiVeSavasSanati/` +4. **CLEAN FILENAMES** — Format: `Author - Title (Year).pdf`. Fix OCR garbage, encoding issues, truncated names. +5. **FLAT TOPICS** — Each major topic is a top-level folder under library root. Subtopics one level below. Max 2 levels deep. +6. **ASK BEFORE DESTRUCTIVE OPS** — Never rm, never force-overwrite. Always confirm with user. + +## Tool Usage Map + +| Task | Tool | How | +|---|---|---| +| Find PDFs | `Glob` | `pattern: "**/*.pdf"` | +| Count files per folder | `Bash` | `find DIR -name "*.pdf" \| wc -l` | +| Read PDF content | `Read` | `file_path: "/path/to.pdf", pages: "1-20"` | +| Read PDF metadata | `Bash` | `pdfinfo "/path/to.pdf"` (title, author, pages, creator) | +| Search text in PDFs | `Grep` | `pattern: "keyword", path: "/library/"` | +| Move/rename files | `Bash` | `mkdir -p TARGET && mv "SRC" "TARGET/NewName.pdf"` | +| Parallel processing | `Agent` | Dispatch subagents for 30-file batches | +| Check duplicates | `Bash` | `ls -la FILE1 FILE2` (compare sizes) | + +## Workflow + +```dot +digraph librarian { + rankdir=TB; + "Survey library" -> "Plan folder structure"; + "Plan folder structure" -> "User approves?"; + "User approves?" -> "Batch files by folder" [label="yes"]; + "User approves?" -> "Revise plan" [label="no"]; + "Revise plan" -> "User approves?"; + "Batch files by folder" -> "Dispatch parallel agents"; + "Dispatch parallel agents" -> "Each agent: read 20pg, classify, rename, move"; + "Each agent: read 20pg, classify, rename, move" -> "Collect reports"; + "Collect reports" -> "Verify & cleanup empty dirs"; +} +``` + +### Phase 1: Survey + +```python +# Use Glob to find all PDFs +Glob(pattern="**/*.pdf", path="/library/root/") + +# Use Bash to count per folder +Bash("find /library -type d -exec sh -c 'echo \"$(find \"$1\" -maxdepth 1 -name \"*.pdf\" | wc -l) $1\"' _ {} \\;") +``` + +### Phase 2: Plan Structure + +Present proposed folder tree to user. Get approval before any moves. Example: + +``` +Books/ +├── AskeriTarih/ (military history) +│ ├── StratejiVeSavasSanati/ +│ └── GenelAskeriTarih/ +├── Istihbarat/ (intelligence) +│ ├── TeoriVeAnaliz/ +│ ├── RusIstihbarati/ +│ └── CIA/ +├── SiberGuvenlik/ (cybersecurity) +└── Arsiv/ (user-decides-later) +``` + +### Phase 3: Dispatch Parallel Agents + +**REQUIRED:** Use `superpowers:dispatching-parallel-agents` pattern. + +Split work into independent batches of ~30 files. Each agent works on one folder or file batch. + +**Agent prompt template:** + +``` +You are a LIBRARIAN. Working directory: {LIBRARY_ROOT} + +RULES (non-negotiable): +- Read FIRST 20 PAGES of each PDF: Read tool, file_path, pages "1-20" +- Also run: Bash("pdfinfo 'FILE'") for metadata +- Rename to "Author - Title (Year).pdf" +- NEVER delete any file. Unwanted → {LIBRARY_ROOT}/Arsiv/ +- No spaces in folder names +- mkdir -p before every mv + +Process these files from {SOURCE_DIR}: +{FILE_LIST} + +Target folder mapping: +{FOLDER_MAP_TABLE} + +For EACH file: +1. Bash: pdfinfo "FILE" → extract Title, Author, Pages, CreationDate +2. Read: file_path="FILE", pages="1-20" → verify author, title, topic +3. If ambiguous after 20 pages → read pages "20-40" +4. Classify → determine target folder +5. Bash: mkdir -p "TARGET" && mv "OLD" "TARGET/Author - Title (Year).pdf" + +Report format per file: +[TYPE] old_name.pdf → Author - Title (Year).pdf → TargetFolder/ +``` + +**Launching agents:** + +```python +# Send ALL agent calls in a SINGLE message for parallel execution +Agent( + name="lib-batch1", + prompt="...(files 1-30)...", + mode="bypassPermissions", + run_in_background=True +) +Agent( + name="lib-batch2", + prompt="...(files 31-60)...", + mode="bypassPermissions", + run_in_background=True +) +# ... up to 10 parallel agents +``` + +### Phase 4: Read & Classify Each PDF + +**Step 1 - Metadata check (fast):** +```bash +pdfinfo "file.pdf" +# Gives: Title, Author, Creator, Producer, CreationDate, Pages +``` + +**Step 2 - Content read (thorough):** +``` +Read(file_path="/path/to/file.pdf", pages="1-20") +``` + +**Extract from content:** +- **Real author** (not editor, not advisor, not publisher, not translator) +- **Real title** (from title page, not OCR garbage in filename) +- **Publication year** (from copyright page, not filename date) +- **Type**: Kitap / Makale / Rapor / Tez / El Kitabı / Dergi / Teknik Doküman +- **Topic**: Specific subject for classification +- **Language**: TR / EN / DE / RU / etc. + +**Step 3 - Ambiguity resolution:** +If 20 pages insufficient → `Read(file_path, pages="20-40")` +If still unclear → `Grep(pattern="keywords", path="file.pdf")` + +### Phase 5: Rename + +**Format:** `Author - Title (Year).pdf` + +**Common OCR/download fixes:** + +| Problem | Example | Fix | +|---|---|---| +| OCR junk prefix | `Dağıtım GAMEDA - Tanzimat` | Read title page → real author | +| Download artifact | `Book (PDFDrive).pdf` | Remove `(PDFDrive)` | +| Hash/ID in name | `[#131337]-112799.pdf` | Remove, use real title | +| Underscores | `Yusuf_Hikmet_Bayur` | Replace with spaces | +| Wrong year | `(1882)` = birth year | Check copyright page | +| Truncated title | `Sertan Kolat - Web Tabanli Uygulamalarda Otomatik Guvenlik Denetim Yazilimlarinin Iyilestirilmes` | Complete from content | +| Advisor as author | `Güngör ŞAHİN - MİLLÎ SAVUNMA ÜNİVERSİTESİ (1920)` | Read → find student name | +| Typos | `Terraki`, `Asleri` | Fix: `Terakki`, `Askeri` | +| Encoding issues | `Te__kilat`, `LLÎGÜVENLİ` | Read → reconstruct | + +### Phase 6: Move + +```bash +# Always mkdir -p first +mkdir -p "/TargetTopic/SubTopic/" +mv "/old/path/garbled_name.pdf" "/TargetTopic/SubTopic/Author - Title (Year).pdf" +``` + +### Phase 7: Report & Cleanup + +```bash +# Remove empty old directories (safe - only removes if empty) +find /old/library/path -type d -empty -delete + +# Final count +find -name "*.pdf" | wc -l +``` + +## Duplicate Handling + +```bash +# Compare sizes +ls -la "file1.pdf" "file2.pdf" +# Keep LARGER (better scan quality) +# Move smaller to Arsiv/Duplikatlar/ (NEVER delete) +mkdir -p Arsiv/Duplikatlar +mv "smaller_file.pdf" Arsiv/Duplikatlar/ +``` + +## Library Root + +**`/mnt/storage/Common/Books/`** — Komga-compatible hierarchy. Max 2 levels deep. + +## Classification Reference Table + +| Content | Top Folder | Subtopic Examples | +|---|---|---| +| Askeri strateji, savaş teorisi | `AskeriTarih/` | `StratejiVeSavasSanati/`, `GenelAskeriTarih/`, `DenizHarpTarihi/` | +| Field manual, doktrin | `AskeriDoktrin/` | `ABD-FieldManual/`, `FOIA-Pentagon/`, `ElKitaplari/` | +| NATO dokümanları | `NATO/` | `FOIA-NATO/`, `Teknik/`, `Tatbikat/`, `Idari/` | +| İstihbarat genel | `Istihbarat/` | `TeoriVeAnaliz/`, `CIA/`, `RusIstihbarati/`, `TurkIstihbarati/` | +| **FOIA CIA genel (okuma odası)** | `Istihbarat/CIA/` | *(doğrudan dosya, alt klasör yok)* | +| **FOIA CIA Türkiye** | `Istihbarat/FOIA-CIA-Turkey/` | *(doğrudan dosya)* | +| **FOIA CIA Orta Doğu** | `Istihbarat/FOIA-CIA-OrtaDogu/` | *(doğrudan dosya)* | +| **FOIA CIA Pakistan/Afgan** | `Istihbarat/FOIA-CIA-Pakistan/` | *(doğrudan dosya)* | +| **FOIA CIA Çin** | `Istihbarat/FOIA-CIA-Cin/` | *(doğrudan dosya)* | +| **FOIA CIA Afrika** | `Istihbarat/FOIA-CIA-Afrika/` | *(doğrudan dosya)* | +| **FOIA CIA İsrail/Filistin** | `Istihbarat/FOIA-CIA-Israil-Filistin/` | *(doğrudan dosya)* | +| **FOIA CIA Hindistan** | `Istihbarat/FOIA-CIA-Hindistan/` | *(doğrudan dosya)* | +| **FOIA FBI Vault** | `Istihbarat/FOIA-FBI-Vault/` | *(doğrudan dosya)* | +| **FOIA Rusya/KGB/FSB** | `Istihbarat/RusIstihbarati/` | *(doğrudan dosya)* | +| **FOIA Terörle Mücadele** | `GuvenlikStratejileri/FOIA-CIA-Teror/` | *(doğrudan dosya)* | +| **FOIA Siber Savaş** | `SiberGuvenlik/FOIA-SiberSavas/` | *(doğrudan dosya)* | +| Osmanlı tarihi | `OsmanliTarihi/` | `IttihatVeTerakki/`, `JonTurkHareketi/`, `AbdulhamidDonemi/` | +| Cumhuriyet tarihi | `CumhuriyetTarihi/` | `MilliMucadele/`, `SiyasiDusunce/`, `Donemler/` | +| Dünya tarihi | `DunyaTarihi/` | `AntikTarih/`, `AvrupaTarihi/`, `DersMateryalleri/` | +| Siber güvenlik | `SiberGuvenlik/` | `AgGuvenligi/`, `WebGuvenligi/`, `SaldiriTeknikleri/` | +| Roman/kurgu | `FelsefeVeEdebiyat/` | `Edebiyat/` | +| Dergi sayıları | `GuvenlikStratejileri/` | `Dergi/` | +| Kişisel/idari doküman | `Arsiv/` | `Duplikatlar/` | +| Sınıflandırılamayan | `Arsiv/` | `Siniflandirilmamis/` | + +## Common Mistakes + +| Mistake | Fix | +|---|---| +| Classifying by filename alone | ALWAYS read 20+ pages AND pdfinfo first | +| Deleting "junk" files | Move to Arsiv/, let user decide | +| Spaces in folder names | CamelCase or hyphens only | +| Using advisor name as author (theses) | Read title page for student name | +| Using topic date as publication year | Check copyright/publishing page | +| Moving without mkdir -p | Always create target dir first | +| Too many files per agent | Max 30 files per subagent | +| Single agent for 100+ files | MUST use parallel agents | +| Not checking pdfinfo first | pdfinfo is fast metadata; Read is thorough content | +| Forgetting epub files | Glob for both `*.pdf` and `*.epub` | diff --git a/personas/_shared/community-skills/marketing-strategist/SKILL.md b/personas/_shared/community-skills/marketing-strategist/SKILL.md new file mode 100644 index 0000000..89bd4e8 --- /dev/null +++ b/personas/_shared/community-skills/marketing-strategist/SKILL.md @@ -0,0 +1,597 @@ +--- +name: marketing-strategist +description: Use when analyzing products for market fit, creating go-to-market strategies, pricing decisions, competitive positioning, customer segmentation, landing page copy, brand storytelling, or any "how do we sell this" question. Triggers on marketing strategy, product launch, sales planning, market research, pricing, branding, or commercialization tasks. +--- + +# Marketing Strategist — Alihan Framework Edition + +An AI marketing strategist that combines Alihan Aydin's proprietary frameworks with classical marketing theory to analyze products, identify market opportunities, and create actionable go-to-market strategies. + +## Source Knowledge Base + +This skill synthesizes insights from 13 documents in `/`: + +### Alihan's Original Frameworks +- **W2P / H2W (Where to Play / How to Win)** — Lafley & Martin's Playing to Win adapted for tech products +- **Fizibilite Cercevesi** — Feasibility framework for product-market fit +- **Hedef Kitle Duygu Analiz Framework'u** — Audience emotion analysis +- **Beklenti Paradigmasi (Expectation Paradigm)** — Marketing as anticipation management +- **8 Haftalik TR Marketing Stratejisti Plan** — Turkey-specific marketing curriculum +- **Pazarlama Calisma Rehberi 1 & 2** — 4-module marketing workbook (Strategy, Analytics, Storytelling, Trends) + +### Classical References (Key Concepts Extracted) +- **Kahneman (Thinking Fast & Slow)** — System 1/2, loss aversion, anchoring, framing effects +- **Ariely (Predictably Irrational)** — Decoy effect, free pricing, social vs market norms, expectation bias +- **Sinek (Start With Why)** — Golden Circle, WHY before WHAT, limbic brain decisions +- **Godin (This Is Marketing)** — Smallest viable audience, status roles, tension & relief, permission marketing +- **Sutherland (Alchemy)** — Psycho-logic vs logic, perception > reality, irrational value creation + +--- + +## Core Strategic Model: Playing to Win (W2P/H2W) + +Every product analysis MUST pass through these 5 cascading questions: + +``` +1. WINNING ASPIRATION — "Ne icin kazanmak istiyoruz?" + → Not "var olmak" but "kazanmak" — what does winning mean? + +2. WHERE TO PLAY (W2P) — "Nerede oynayacagiz?" + → 5 dimensions: Musteri Segmenti, Cografya, Urun/Kategori, Kanal, Fiyat/Deger + → W2P = Kaynaklar + Rekabet + Firsatlar ucgeninde karar + → "Her yeri hedefleyen strateji, hicbir yere varamaz." + +3. HOW TO WIN (H2W) — "Nasil kazanacagiz?" + → Cost Advantage (maliyet liderligi) vs Differentiation (farklilasmma) + → Modern variants: Network, Brand, Capability, Platform, Data advantage + → H2W mantigi: Deger + Farklilik + Savunulabilirlik + → Value = (Fayda + Duygu + Fark) - Maliyet + +4. CAPABILITIES — "Bunu gerceklestirecek yetkinlikler neler?" + → Core (gunluk), Strategic (kopyalanamaz), Dynamic (uyumlama) + → "Strateji, sahip olmadigin kaslarla yapilamaz." + +5. MANAGEMENT SYSTEMS — "Stratejiyi nasil surdurecegiz?" + → KPI/OKR, Quarterly Review, Decision Rights, Learning Loops, Culture + → "Strateji PowerPoint'te degil, rutinlerde yasar." +``` + +**FIT RULE:** All 5 must align. Wrong W2P = advantage doesn't work. Wrong Capability = strategy fails. + +--- + +## Analytical Frameworks + +### W2P Analysis Techniques +- **Segmentation (Bolumleme)** — demografik, davranissal, cografi, psikografik, ihtiyac-temelli +- **Attractiveness Analysis (Cekicilik)** — pazar buyuklugu, rekabet yogunlugu, musteri sadakati, giris bariyeri +- **Competitor Mapping (Rakip Haritalama)** — white space vs red ocean +- **Capability-Fit Analysis** — beceriler o segmentte ise yariyor mu? + +### H2W Analytical Techniques +- **Value Curve (Deger Egrisi)** — her faktorde rakiplerle karsilastir, overinvested/underinvested bul +- **VRIO Framework** — Valuable, Rare, Inimitable, Organized +- **Value Discipline Alignment** — urun-musteri-operasyon uclusu tutarli mi? +- **Playbook Mapping** — Speed Play, Scale Play, Brand Play, Niche Play, Platform Play +- **Stratejik Moat Turleri** — Brand, Network, Data, Switching Cost, Scale, Ecosystem + +--- + +## Customer Analysis: 3 Katmanli Model + +For EVERY product, analyze the customer through: + +``` +1. Insan (Insight) — "Kimin hayatina dokunuyoruz?" +2. Problem (Pain Point) — "Onun icin hangi problemi cozuyoruz?" +3. Deger (Value Prop) — "Bizim farkimiz ne?" +``` + +### 4 Veri Katmani (Data Layers) +1. **Tanimlayici (Descriptive)** — Demografik, cografi, sosyoekonomik, psikografik +2. **Davranissal (Behavioral)** — Kampanya etkilesimi, web davranisi, satin alma, loyalty/churn +3. **Deger Temelli (Value/RFM)** — Recency, Frequency, Monetary +4. **Tutum & Algi (Attitudinal)** — Anket/NPS, sosyal dinleme, musteri yorumlari + +### Segmentation (3 Eksen) +- **Deger (Value)** — RFM, CLV, sadakat orani +- **Davranis (Behavior)** — kampanya duyarliligi, kanal tercihi +- **Tutum (Attitude)** — savunucu mu, kararsiz mi, riskli mi? + +--- + +## Behavioral Marketing Principles + +### Kahneman (System 1/2) +- Most buying decisions are System 1 (fast, emotional, automatic) +- Loss aversion: losing hurts 2x more than gaining feels good +- Anchoring: first price seen sets the reference frame +- Framing: "95% fat-free" vs "5% fat" — same product, different perception +- **Application:** Lead with emotion, justify with logic. Price anchoring in tiers. + +### Ariely (Predictably Irrational) +- **Decoy Effect:** Add a worse option to make the target option look better +- **Free Effect:** "Free" triggers irrational excitement — use free tier strategically +- **Social vs Market Norms:** Don't mix — once you introduce money, goodwill framework breaks +- **Expectation Bias:** What people expect to experience shapes what they actually experience +- **Application:** Freemium design, pricing tier architecture, trial period psychology + +### Sutherland (Alchemy / Psycho-logic) +- Perception IS reality in marketing — objective quality matters less than perceived quality +- Irrational value creation: Red Bull tastes bad ON PURPOSE (signals potency) +- The opposite of a good idea can also be a good idea +- **Application:** Don't compete on features, compete on meaning. "Bu bir kahve makinesi degil, ritual baslaticisi." + +--- + +## Storytelling Framework + +### Brand Story Structure (5 Steps) +1. **Kahraman** — Hedef kitle (NOT the brand) +2. **Sorun** — Kahramanin karsilastigi problem +3. **Cozum** — Markanin sundugu cozum +4. **Sonuc** — Kahramanin yasadigi degisim +5. **Mesaj** — Hikayenin tasidigi ana mesaj + +### Sinek's Golden Circle +``` +WHY → Neden variz? (limbik beyin, karar) +HOW → Bunu nasil yapiyoruz? (farklilik) +WHAT → Ne sunuyoruz? (urun/hizmet) +``` +**Rule:** Always communicate from inside out. WHY first. + +### Godin's This Is Marketing +- "Insanlar senin urunune ihtiyac duymuyor. Onlarin hikayesine uygun olan seyi ariyorlar." +- Find the **smallest viable audience** — serve them obsessively +- Marketing creates **tension** (between where they are and where they could be) +- Two status roles: Affiliation (belong to group) vs Dominance (stand out from group) +- Permission > Interruption: Earn attention, don't buy it + +--- + +## Expectation Paradigm (Beklenti Yonetimi) + +### 3 Katman +1. **Kognitif** — Insan ne bekledigini dusunur → Konumlandirma & Mesajlama +2. **Duygusal** — Insan ne hissetmeyi umar → Storytelling & Marka deneyimi +3. **Sosyal** — Insan basklarinin ne bekledigini gozler → Sosyal kanit & influencer + +### Beklenti Yonetim Araclari +1. **Framing (Cerceveleme)** — Sunusu degistir, algiyi degistir +2. **Anchor (Referans Noktasi)** — Ilk deger sonraki algiyi sabitler +3. **Expectation Gap** — Az vaat et, surprizle genislet +4. **Reputation Feedback Loop** — Kullanici geri bildirimleri yeni beklentiler uretir + +### Turkiye Ozeli +- Tuketicinin guven duygusu dusuk +- Fiyat dalgalanmalari beklentileri surekli degistiriyor +- "Fazla vaat"e duyarlilar ama "surpriz tatmine" asiri pozitif tepki veriyorlar +- **Turkiye'de marka buyutmek = surprizle guveni harmanlamak** +- En iyi TR markalari "samimi" veya "adil" hissi yaratabilenlerdir (Tadelle, Beko, Trendyol) + +--- + +## Turkey-Specific Marketing Intelligence + +### 8 Haftalik Stratejist Plani +1. **Turkiye'de Pazarlamanin Anatomisi** — duygusal davranis + anlik trend + satis baskisi ucgeni +2. **Insan Davranisi Okuma (Behavioral Marketing)** — tetikleyiciler ve marka yonlendirme +3. **Marka Psikolojisi ve Arketipler** — "Markanin karakteri kim?" +4. **Storytelling ve Anlam Tasarimi** — hikaye satis yapmaz, anlam yaratir +5. **Turkiye'de Kultur ve Trend Okuma** — kulturle catisma degil, kulturden dogma +6. **Veriyle Hikaye Kurmak (Analitik)** — Google Trends, SimilarWeb, Statista +7. **Stratejik Planlama Kasini Gelistir** — "Strateji plan degil, tercihler butunudur" +8. **Kisisel Marka & Gorunurluk** — LinkedIn/Medium seri, dusunce liderligi + +### Kanal Secimi (Audience Fit) +- Genc & yaratici → Instagram, TikTok, YouTube +- Profesyonel & B2B → LinkedIn +- Haber tuketen → X (Twitter) +- Satin alma odakli → Google Search / E-commerce + +### Altin Kural +``` +Strateji → Mesaj → Icerik → Kanal → KPI +(Tersine islem yapmak kaos getirir) +``` + +--- + +## Fizibilite Cercevesi (Feasibility Framework) + +When evaluating whether to commercialize a product, score on: +1. **Teknik Hazirlik** — Deploy edilebilir mi? Demo var mi? +2. **Pazar Buyuklugu** — TAM/SAM/SOM ne kadar? +3. **Rekabet Yogunlugu** — Red ocean mi blue ocean mi? +4. **Kaynak Gereksinimleri** — Ne kadar zaman/para/insan lazim? +5. **Monetizasyon Netliigi** — Gelir modeli net mi? +6. **Ilk Musteri Erisimi** — Ilk 10 musteri nereden gelecek? + +--- + +## How to Use This Skill + +### For Product Analysis +1. Apply W2P/H2W cascade to the product +2. Run 3 Katmanli Customer Analysis (Insight/Problem/Value) +3. Identify market position using Competitor Mapping +4. Score with Fizibilite Cercevesi +5. Recommend: SELL NOW / NEEDS WORK / PIVOT / KILL + +### For Go-to-Market Strategy +1. Define WHY (Sinek Golden Circle) +2. Choose W2P (segment, geography, channel, price) +3. Define H2W (differentiation or cost advantage) +4. Create Brand Story (5-step structure) +5. Select channels (Audience Fit + Behavior Fit + Content Fit) +6. Set KPI chain: Awareness → Engagement → Conversion +7. Apply Beklenti Yonetimi for Turkish market + +### For Pricing +1. Anchor with premium tier first +2. Use Ariely's Decoy Effect for tier design +3. Apply Kahneman's loss aversion for churn prevention +4. Consider TR market: "adil fiyat" algisi kritik + +### For Landing Page / Sales Copy +1. Lead with WHY, not WHAT +2. Hero = Customer (not brand) +3. Frame the transformation (before → after) +4. Social proof (logos, numbers, testimonials) +5. Manage expectations (vaat az, surpriz cok) +6. CTA with tension (what they lose by not acting) + +--- + +## Behavioral Conversion Model (Rehber 2 — Alihan) + +### Master Formula +``` +Istek → Engel → Tetikleyici → Odul → Tekrar +``` +This is the core behavioral marketing engine. Apply to EVERY product launch. + +### 5 Engel Turu (Barriers to Purchase) +1. **Bilgi Engeli** — Customer doesn't know what to do → Fix with education content +2. **Yetkinlik Engeli** — "I can't do this" → Fix with onboarding, demo, white-glove +3. **Finansal Engel** — Price too high → Fix with freemium, payment plans +4. **Sosyal Engel** — Fear of judgment → Fix with social proof, community +5. **Aliskanlik Engeli** — "I've always done it this way" → Fix with switching incentives + +> **Davranis = Istek - Engeller** — Even high desire fails if barriers remain. + +### 3 Decision Drivers (Karar Tetikleyicileri) +1. **Benlik** — "Bu secim beni nasil biri yapar?" (identity) +2. **Sosyal** — "Beni kim nasil gorecek?" (status) +3. **Duygusal** — "Bana nasil hissettiriyor?" (feeling) + +Find the dominant driver per persona with 3 questions: +1. Onlar icin basari ne demek? +2. Basari oldugunda kim fark etmeli? +3. Basardiklarinda nasil hissetmeliler? + +### Behavior Change Tactics +1. **Kanitla rahatlatarak** (Proof) — "1000+ user", reviews, case studies +2. **Is yukunu azaltarak** (Ease) — 1-click deploy, instant demo, no-config +3. **Duygu ile tetikleyerek** (Emotion) — "Your attack surface is exposed right now" + +### Funnel KPIs +- **Activation Rate** — signup → first meaningful action +- **First Value Time** — time to first "aha" moment (MOST IMPORTANT KPI) +- **Messaging/Engagement Rate** — first 24h activity +- **Retention D7** — 7-day return rate + +--- + +## Alchemy — Rory Sutherland (Key Concepts) + +### Psycho-Logic vs Logic +- Human decisions follow psychological logic, not rational logic +- Objective improvement often matters less than perceived improvement +- **Red Bull tastes bad ON PURPOSE** — signals potency through unpleasantness + +### The Four S-es of Context-Dependent Decision Making +1. **Signaling** — What does choosing this say about me? +2. **Subconscious** — What automatic associations does this trigger? +3. **Satisficing** — Is this "good enough" vs optimal? +4. **Suggestion** — What default/framing am I being nudged toward? + +### Application to +- Don't sell "82 microservices" — sell "sovereign intelligence" +- A complex setup process can SIGNAL sophistication (Alchemy principle) +- Price above competitors to signal quality (Veblen effect) + +--- + +## Predictably Irrational — Dan Ariely (Key Concepts) + +### Decoy Effect (Asymmetric Dominance) +Add a clearly inferior option to make the target option look superior. +- Economist example: Print-only $59, Digital+Print $125, **Print-only+Digital $125** → everyone picks combo +- **Apply:** In pricing tiers, make Starter deliberately weak so Pro looks obvious + +### Zero Price Effect +"Free" is not just a price — it's an emotional trigger. Free removes decision anxiety entirely. +- **Apply:** Always have a free tier. The conversion from free→paid is easier than no-account→paid. + +### Anchoring +First number seen becomes the reference point for all future judgments. +- **Apply:** Show competitor prices ($50K+/yr) BEFORE showing your price ($15K/yr) + +### Self-Herding +People look at their own past behavior to decide future behavior. +- **Apply:** Get users to take ANY small action first (sign up, star repo, follow). Future buying follows. + +--- + +## Alihan's 5-Layer Feasibility Model (Fizibilite Cercevesi) + +Before commercializing ANY product, score it across these 5 layers: + +### Layer A: Problem Reality (Problemin Gercekligi) +- How often does this problem occur? +- How severe is the impact? (time, money, missed opportunity, stress, reputation) +- How are people currently solving it? +- Why is the current solution insufficient? +- Is the problem deferrable or does ignoring it create cost? + +### Layer B: Causality & Mechanism (Nedensellik) +- What root mechanism creates this problem? +- Is it a process, behavior, knowledge gap, coordination, or visibility issue? +- Does the product address the exact break point? + +### Layer C: Solution-Market Fit (Cozum-Pazar Uyumu) +- Can you explain in 30 seconds what it does and why they need it? +- Is there a repeatable sales motion? +- Does the unit economics work? + +### Layer D: Behavioral & Identity Value (Davranissal Deger) +Not just "who uses it" but **"who does the user WANT TO BE by using this?"** +- **Identity signal**: Does using this product make them feel competent, elite, in-control? +- **Social signal**: Does it raise their status among peers? +- **Emotional signal**: Does it reduce anxiety, increase confidence? + +### Layer E: Execution Feasibility (Uygulama Kapasitesi) +- Team capability, distribution channels, time, technical debt +- Can you ship an MVP in 2 weeks? + +### Alihan's 6-Axis Quick Filter +1. Pain Test: "Bu aci gercekten yakiyor mu, tekrar ediyor mu, para kaybettiriyor mu?" +2. Existing Solutions: "Insanlar bunu zaten cozmeye calisiyor mu?" +3. Explainability: "Kisa surede anlatabilir miyim?" +4. Execution Capacity: "Teknik kapasite yeterli mi?" +5. Founder Belief: "Kendim de inanabilecegim bir sey mi?" +6. Behavioral Value: "Bu urunu kullanan kisi kim olmak istiyor?" + +--- + +## Hedef Kitle Duygu Analiz Framework'u (Alihan) + +### Emotion Mapping for Target Audience +For each customer segment, map: +1. **Dominant emotion** before purchase (anxiety, frustration, curiosity, ambition) +2. **Desired emotion** after purchase (relief, confidence, pride, control) +3. **Fear** that blocks purchase (waste of money, complexity, judgment) +4. **Social proof type** that resolves fear (peer testimonials, authority endorsement, numbers) + +### The Emotion → Action Bridge +``` +Emotion (System 1) → Rationalization (System 2) → Action → Post-purchase emotion → Loyalty loop +``` + +--- + +## Seth Godin — This Is Marketing (Deep Framework) + +### The 5 Steps of Marketing +1. **Invent** — Create something valuable with a compelling story +2. **Design** — Build it for a specific niche that truly benefits +3. **Tell a Story** — Align narrative with aspirations of that small market +4. **Spread the Word** — Actively promote and advocate +5. **Show Up** — Consistently engage to foster trust and community + +### Smallest Viable Audience (SVA) +The most critical concept. Do NOT try to reach everyone. +- **Ask**: "What change am I trying to make?" +- **Ask**: "Who am I seeking to change?" +- **Identify** the smallest group sharing beliefs and worldviews +- **Focus** all effort on delighting them + +> "My product is for people who believe ___. I will promise them ___ and deliver ___." + +> "Coloring the ocean purple" — dye a pool (visible) not the ocean (invisible). Target the pool. + +### Status Roles: Affiliation vs Dominance +Every purchase decision involves status: +- **Affiliation** — "I belong to this group" (community, shared identity) +- **Dominance** — "I stand out from others" (premium, exclusive) +- Determine which role your product serves per segment + +### Tension Creates Action +Marketing creates **tension** between where customer is and where they could be. +- Tension ≠ Fear. Fear freezes. Tension mobilizes. +- "What will you miss if you don't act?" + +### Pricing as Signal +- Price tells a story about who this is for +- Cheapness = fear, not generosity +- "Free" changes the dynamic entirely (see Ariely) + +### Permission > Interruption +Earn attention, don't buy it. Build a permission asset over time. + +--- + +## Rory Sutherland — Alchemy (Deep Framework) + +### 12 Counterintuitive Principles +1. **Opposite of good idea can be good** — Red Bull: high price + bad taste = signal of potency +2. **Value is in the mind, not the object** — Rebrand, don't rebuild +3. **Costly signals > cheap signals** — Expensive = trustworthy +4. **Efficiency destroys meaning** — "Pick any two: Efficiency, Logic, Meaning" +5. **Making things harder makes them better** — IKEA Effect, cake mix + egg +6. **Context is everything** — Same product, different frame = different perception +7. **Vaguely right > precisely wrong** — Gut + data beats data alone +8. **Small changes, big effects** — Trivial modifications can have disproportionate impact +9. **Test before you theorize** — Behavior contradicts theory constantly +10. **Control perception, not product** — Marketing > engineering for satisfaction +11. **Unpredictable rewards beat predictable ones** — Surprise drives loyalty +12. **What you remove matters as much as what you add** — Subtraction creates value + +### Signalling Theory (5 Types) +1. **Costly Signalling** — Expensive ads signal financial health +2. **Repetitive Signalling** — Consistency = reliability (Coca-Cola repeats forever) +3. **Handicap Signalling** — Self-imposed constraints prove quality (peacock's tail) +4. **Creative Signalling** — Novel approaches substitute cost with ingenuity +5. **Social Signalling** — Brand choice broadcasts identity to others + +### The Placebo Effect in Marketing +- Wine tastes better when told it's expensive +- Pain relievers work better at higher prices +- **Your price IS your product** — don't undercut yourself + +--- + +## Daniel Kahneman — Thinking Fast & Slow (Deep Framework) + +### System 1 vs System 2 — Complete Table +| Dimension | System 1 (Fast) | System 2 (Slow) | +|-----------|----------------|-----------------| +| Speed | Automatic, instant | Deliberate, effortful | +| Effort | None | Requires attention, depletes energy | +| Awareness | Below conscious | Fully conscious | +| Errors | Systematic biases | Can correct but often endorses System 1 | +| Default | Always on | Activated only when needed | + +> **Most purchase decisions are System 1.** Target emotions first, give logic for justification. + +### Critical Cognitive Biases for Marketing +1. **WYSIATI** (What You See Is All There Is) — Control the information environment. First thing seen = entire frame +2. **Anchoring** — First number sets reference. Show competitor price ($50K) before yours ($15K) +3. **Loss Aversion** — Losing hurts 2x more than gaining. Frame as "what you lose by not acting" +4. **Framing Effect** — "95% effective" vs "5% fail rate" — same data, opposite reaction +5. **Halo Effect** — One good impression colors everything else +6. **Availability Heuristic** — Recent/vivid examples dominate judgment. Use case studies, breach news +7. **Endowment Effect** — People overvalue what they already own. Free trials create ownership +8. **Peak-End Rule** — People remember the peak moment and the end. Design onboarding peaks + +### Prospect Theory for Pricing +- Losses loom larger than gains (2:1 ratio) +- People are risk-seeking in losses, risk-averse in gains +- **Bundle losses** (one painful payment vs many small ones) +- **Unbundle gains** (many small delights vs one big one) +- **Frame discounts as avoiding loss**, not gaining savings + +--- + +## Dan Ariely — Predictably Irrational (Complete) + +### Chapter-by-Chapter Actionable Principles + +1. **Decoy Effect** — Always offer 3 tiers. Middle = target. Add deliberately inferior option. +2. **Anchoring** — First price seen becomes reference forever. Show high anchor first. +3. **Zero Price Effect** — "Free" triggers irrational excitement. Free tier removes decision anxiety. +4. **Social vs Market Norms** — Don't mix! Once you introduce money, goodwill breaks. Community ≠ transaction. +5. **Arousal & Decision-Making** — Emotional state changes preferences. Urgency/fear/excitement = different choices. +6. **Procrastination** — People defer even beneficial actions. Add deadlines, limited offers. +7. **Endowment Effect** — Free trials create psychological ownership. Hard to cancel what you "own." +8. **Keeping Doors Open** — People hate losing options. "Limited time" works because it threatens option loss. +9. **Expectations Shape Experience** — What people expect to experience shapes actual experience. Set expectations high. +10. **Placebo Pricing** — Higher price = perceived higher quality. Don't compete on price; compete on perception. +11. **The Cycle of Distrust** — Once trust breaks, it's nearly impossible to rebuild. Guard reputation obsessively. + +### Master Pricing Architecture (from Ariely) +``` +Tier 1: FREE (removes anxiety, creates self-herding) +Tier 2: STARTER (deliberate decoy — limited, slightly frustrating) +Tier 3: PRO (target — obviously best value vs Starter) +Tier 4: ENTERPRISE (anchor — makes Pro look reasonable) +``` + +--- + +## Portfolio — Applied Intelligence + +### Product Priority Matrix (from 20-Agent Analysis) +``` +SELL NOW (this week): +1. Personas Library → Gumroad/PromptBase, $50-500/sale +2. ollama-scanner → Blog post + Black Hat Arsenal +3. killer-claude → PTaaS to MSSPs, $5K-35K/engagement + +LAUNCH (2 weeks): +4. DaveIntel → Substack $15/mo + Telegram premium +5. iNTERCEPT → Open-core, Apache 2.0 + +SCALE (month 2-3): +6. ASM → TR banks (BDDK mandate), $10-40K/yr +7. FrodoIntel → Sovereign CTI SaaS, $15-50K/yr +8. Suite → Bundle, $3-10K/mo +``` + +### Positioning Cheat Sheet +| Product | Tagline | Archetype | W2P | +|---------|---------|-----------|-----| +| ASM | "See what attackers see" | Guardian | TR FIRST | +| killer-claude | "Pentest. Prove. Repeat." | Outlaw | BOTH | +| FrodoIntel | "Intelligence. Sovereign." | Sage-Ruler | TR FIRST | +| ollama-scanner | "Shodan for AI" | Hero | GLOBAL | +| Shadowbroker | "Connect every dot" | Magician | GLOBAL | +| Mission Control | "Command your agents" | Ruler | GLOBAL | +| DaveIntel | "The signal in the noise" | Sage | BOTH | +| iNTERCEPT | "Listen to everything" | Explorer | BOTH | +| Personas | "29 minds, one library" | Creator | GLOBAL | + +### Revenue Projections (Year 1) +- Conservative: $349K +- Expected: $580K +- Optimistic: $980K +- Highest driver: killer-claude PTaaS ($120-420K) +- Fastest: Personas (days to first sale) +- Highest margin: Personas (95%+) + +### Growth Loop +``` +ollama-scanner (free, viral) + → discovers exposed LLMs + → leads need security assessment + → killer-claude (paid pentest) + → findings need continuous monitoring + → ASM (subscription) + → threat context needed + → FrodoIntel (platform) + → all feeds into Suite (bundle) +``` + +### Turkey Strategic Advantage +- **Law 7545** (March 2025): Mandatory cybersecurity compliance for all public entities +- **BDDK**: Annual independent pentest mandate for banks +- **KVKK**: Fines up to TRY 17M per violation +- **SSB**: Domestic vendor preference for defense/intelligence +- **Currency arbitrage**: Earn USD, spend TRY → 5-10x purchasing power advantage +- **Zero domestic competition**: No Turkish ASM, CTI, or OSINT platform exists + +--- + +## Key Quotes to Internalize + +> "Strateji bir plan degil, secimler sistemidir." — Lafley & Martin + +> "Her yerde iyi olmak aslinda hicbir yerde kazanmamaktir." + +> "Value = (Fayda + Duygu + Fark) - Maliyet" + +> "Pazarlama = Geregi yonetmek degil, beklentiyi tasarlamaktir." — Alihan + +> "Insanlar markayi degil, hikayeyi hatirlar." + +> "Trade-off yapmayanin stratejisi yoktur." — Porter + +> "Turkiye'de marka buyutmek = surprizle guveni harmanlamak." + +> "Start with Why." — Sinek + +> "People don't buy what you do, they buy why you do it." + +> "Find the smallest viable audience and serve them so well they tell others." — Godin diff --git a/personas/_shared/community-skills/notebooklm/AUTHENTICATION.md b/personas/_shared/community-skills/notebooklm/AUTHENTICATION.md new file mode 100755 index 0000000..5d9de88 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/AUTHENTICATION.md @@ -0,0 +1,154 @@ +# Authentication Architecture + +## Overview + +This skill uses a **hybrid authentication approach** that combines the best of both worlds: + +1. **Persistent Browser Profile** (`user_data_dir`) for consistent browser fingerprinting +2. **Manual Cookie Injection** from `state.json` for reliable session cookie persistence + +## Why This Approach? + +### The Problem + +Playwright/Patchright has a known bug ([#36139](https://github.com/microsoft/playwright/issues/36139)) where **session cookies** (cookies without an `Expires` attribute) do not persist correctly when using `launch_persistent_context()` with `user_data_dir`. + +**What happens:** +- ✅ Persistent cookies (with `Expires` date) → Saved correctly to browser profile +- ❌ Session cookies (without `Expires`) → **Lost after browser restarts** + +**Impact:** +- Some Google auth cookies are session cookies +- Users experience random authentication failures +- "Works on my machine" syndrome (depends on which cookies Google uses) + +### TypeScript vs Python + +The **MCP Server** (TypeScript) can work around this by passing `storage_state` as a parameter: + +```typescript +// TypeScript - works! +const context = await chromium.launchPersistentContext(userDataDir, { + storageState: "state.json", // ← Loads cookies including session cookies + channel: "chrome" +}); +``` + +But **Python's Playwright API doesn't support this** ([#14949](https://github.com/microsoft/playwright/issues/14949)): + +```python +# Python - NOT SUPPORTED! +context = playwright.chromium.launch_persistent_context( + user_data_dir=profile_dir, + storage_state="state.json", # ← Parameter not available in Python! + channel="chrome" +) +``` + +## Our Solution: Hybrid Approach + +We use a **two-phase authentication system**: + +### Phase 1: Setup (`auth_manager.py setup`) + +1. Launch persistent context with `user_data_dir` +2. User logs in manually +3. **Save state to TWO places:** + - Browser profile directory (automatic, for fingerprint + persistent cookies) + - `state.json` file (explicit save, for session cookies) + +```python +context = playwright.chromium.launch_persistent_context( + user_data_dir="browser_profile/", + channel="chrome" +) +# User logs in... +context.storage_state(path="state.json") # Save all cookies +``` + +### Phase 2: Runtime (`ask_question.py`) + +1. Launch persistent context with `user_data_dir` (loads fingerprint + persistent cookies) +2. **Manually inject cookies** from `state.json` (adds session cookies) + +```python +# Step 1: Launch with browser profile +context = playwright.chromium.launch_persistent_context( + user_data_dir="browser_profile/", + channel="chrome" +) + +# Step 2: Manually inject cookies from state.json +with open("state.json", 'r') as f: + state = json.load(f) + context.add_cookies(state['cookies']) # ← Workaround for session cookies! +``` + +## Benefits + +| Feature | Our Approach | Pure `user_data_dir` | Pure `storage_state` | +|---------|--------------|----------------------|----------------------| +| **Browser Fingerprint Consistency** | ✅ Same across restarts | ✅ Same | ❌ Changes each time | +| **Session Cookie Persistence** | ✅ Manual injection | ❌ Lost (bug) | ✅ Native support | +| **Persistent Cookie Persistence** | ✅ Automatic | ✅ Automatic | ✅ Native support | +| **Google Trust** | ✅ High (same browser) | ✅ High | ❌ Low (new browser) | +| **Cross-platform Reliability** | ✅ Chrome required | ⚠️ Chromium issues | ✅ Portable | +| **Cache Performance** | ✅ Keeps cache | ✅ Keeps cache | ❌ No cache | + +## File Structure + +``` +~/.claude/skills/notebooklm/data/ +├── auth_info.json # Metadata about authentication +├── browser_state/ +│ ├── state.json # Cookies + localStorage (for manual injection) +│ └── browser_profile/ # Chrome user profile (for fingerprint + cache) +│ ├── Default/ +│ │ ├── Cookies # Persistent cookies only (session cookies missing!) +│ │ ├── Local Storage/ +│ │ └── Cache/ +│ └── ... +``` + +## Why `state.json` is Critical + +Even though we use `user_data_dir`, we **still need `state.json`** because: + +1. **Session cookies** are not saved to the browser profile (Playwright bug) +2. **Manual injection** is the only reliable way to load session cookies +3. **Validation** - we can check if cookies are expired before launching + +## Code References + +**Setup:** `scripts/auth_manager.py:94-120` +- Lines 100-113: Launch persistent context with `channel="chrome"` +- Line 167: Save to `state.json` via `context.storage_state()` + +**Runtime:** `scripts/ask_question.py:77-118` +- Lines 86-99: Launch persistent context +- Lines 101-118: Manual cookie injection workaround + +**Validation:** `scripts/auth_manager.py:236-298` +- Lines 262-275: Launch persistent context +- Lines 277-287: Manual cookie injection for validation + +## Related Issues + +- [microsoft/playwright#36139](https://github.com/microsoft/playwright/issues/36139) - Session cookies not persisting +- [microsoft/playwright#14949](https://github.com/microsoft/playwright/issues/14949) - Storage state with persistent context +- [StackOverflow Question](https://stackoverflow.com/questions/79641481/) - Session cookie persistence issue + +## Future Improvements + +If Playwright adds support for `storage_state` parameter in Python's `launch_persistent_context()`, we can simplify to: + +```python +# Future (when Python API supports it): +context = playwright.chromium.launch_persistent_context( + user_data_dir="browser_profile/", + storage_state="state.json", # ← Would handle everything automatically! + channel="chrome" +) +``` + +Until then, our hybrid approach is the most reliable solution. diff --git a/personas/_shared/community-skills/notebooklm/CHANGELOG.md b/personas/_shared/community-skills/notebooklm/CHANGELOG.md new file mode 100755 index 0000000..a60278e --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/CHANGELOG.md @@ -0,0 +1,44 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.3.0] - 2025-11-21 + +### Added +- **Modular Architecture** - Refactored codebase for better maintainability + - New `config.py` - Centralized configuration (paths, selectors, timeouts) + - New `browser_utils.py` - BrowserFactory and StealthUtils classes + - Cleaner separation of concerns across all scripts + +### Changed +- **Timeout increased to 120 seconds** - Long queries no longer timeout prematurely + - `ask_question.py`: 30s → 120s + - `browser_session.py`: 30s → 120s + - Resolves Issue #4 + +### Fixed +- **Thinking Message Detection** - Fixed incomplete answers showing placeholder text + - Now waits for `div.thinking-message` element to disappear before reading answer + - Answers like "Reviewing the content..." or "Looking for answers..." no longer returned prematurely + - Works reliably across all languages and NotebookLM UI changes + +- **Correct CSS Selectors** - Updated to match current NotebookLM UI + - Changed from `.response-content, .message-content` to `.to-user-container .message-text-content` + - Consistent selectors across all scripts + +- **Stability Detection** - Improved answer completeness check + - Now requires 3 consecutive stable polls instead of 1 second wait + - Prevents truncated responses during streaming + +## [1.2.0] - 2025-10-28 + +### Added +- Initial public release +- NotebookLM integration via browser automation +- Session-based conversations with Gemini 2.5 +- Notebook library management +- Knowledge base preparation tools +- Google authentication with persistent sessions diff --git a/personas/_shared/community-skills/notebooklm/LICENSE b/personas/_shared/community-skills/notebooklm/LICENSE new file mode 100755 index 0000000..5b2d751 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Please Prompto! + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/personas/_shared/community-skills/notebooklm/README.md b/personas/_shared/community-skills/notebooklm/README.md new file mode 100755 index 0000000..0a46e9c --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/README.md @@ -0,0 +1,412 @@ +
+ +# NotebookLM Claude Code Skill + +**Let [Claude Code](https://github.com/anthropics/claude-code) chat directly with NotebookLM for source-grounded answers based exclusively on your uploaded documents** + +[![Python](https://img.shields.io/badge/Python-3.8+-blue.svg)](https://www.python.org/) +[![Claude Code Skill](https://img.shields.io/badge/Claude%20Code-Skill-purple.svg)](https://www.anthropic.com/news/skills) +[![Based on](https://img.shields.io/badge/Based%20on-NotebookLM%20MCP-green.svg)](https://github.com/PleasePrompto/notebooklm-mcp) +[![GitHub](https://img.shields.io/github/stars/PleasePrompto/notebooklm-skill?style=social)](https://github.com/PleasePrompto/notebooklm-skill) + +> Use this skill to query your Google NotebookLM notebooks directly from Claude Code for source-grounded, citation-backed answers from Gemini. Browser automation, library management, persistent auth. Drastically reduced hallucinations - answers only from your uploaded documents. + +[Installation](#installation) • [Quick Start](#quick-start) • [Why NotebookLM](#why-notebooklm-not-local-rag) • [How It Works](#how-it-works) • [MCP Alternative](https://github.com/PleasePrompto/notebooklm-mcp) + +
+ +--- + +## ⚠️ Important: Local Claude Code Only + +**This skill works ONLY with local [Claude Code](https://github.com/anthropics/claude-code) installations, NOT in the web UI.** + +The web UI runs skills in a sandbox without network access, which this skill requires for browser automation. You must use [Claude Code](https://github.com/anthropics/claude-code) locally on your machine. + +--- + +## The Problem + +When you tell [Claude Code](https://github.com/anthropics/claude-code) to "search through my local documentation", here's what happens: +- **Massive token consumption**: Searching through documentation means reading multiple files repeatedly +- **Inaccurate retrieval**: Searches for keywords, misses context and connections between docs +- **Hallucinations**: When it can't find something, it invents plausible-sounding APIs +- **Manual copy-paste**: Switching between NotebookLM browser and your editor constantly + +## The Solution + +This Claude Code Skill lets [Claude Code](https://github.com/anthropics/claude-code) chat directly with [**NotebookLM**](https://notebooklm.google/) — Google's **source-grounded knowledge base** powered by Gemini 2.5 that provides intelligent, synthesized answers exclusively from your uploaded documents. + +``` +Your Task → Claude asks NotebookLM → Gemini synthesizes answer → Claude writes correct code +``` + +**No more copy-paste dance**: Claude asks questions directly and gets answers straight back in the CLI. It builds deep understanding through automatic follow-ups, getting specific implementation details, edge cases, and best practices. + +--- + +## Why NotebookLM, Not Local RAG? + +| Approach | Token Cost | Setup Time | Hallucinations | Answer Quality | +|----------|------------|------------|----------------|----------------| +| **Feed docs to Claude** | 🔴 Very high (multiple file reads) | Instant | Yes - fills gaps | Variable retrieval | +| **Web search** | 🟡 Medium | Instant | High - unreliable sources | Hit or miss | +| **Local RAG** | 🟡 Medium-High | Hours (embeddings, chunking) | Medium - retrieval gaps | Depends on setup | +| **NotebookLM Skill** | 🟢 Minimal | 5 minutes | **Minimal** - source-grounded only | Expert synthesis | + +### What Makes NotebookLM Superior? + +1. **Pre-processed by Gemini**: Upload docs once, get instant expert knowledge +2. **Natural language Q&A**: Not just retrieval — actual understanding and synthesis +3. **Multi-source correlation**: Connects information across 50+ documents +4. **Citation-backed**: Every answer includes source references +5. **No infrastructure**: No vector DBs, embeddings, or chunking strategies needed + +--- + +## Installation + +### The simplest installation ever: + +```bash +# 1. Create skills directory (if it doesn't exist) +mkdir -p ~/.claude/skills + +# 2. Clone this repository +cd ~/.claude/skills +git clone https://github.com/PleasePrompto/notebooklm-skill notebooklm + +# 3. That's it! Open Claude Code and say: +"What are my skills?" +``` + +When you first use the skill, it automatically: +- Creates an isolated Python environment (`.venv`) +- Installs all dependencies including **Google Chrome** +- Sets up browser automation with Chrome (not Chromium) for maximum reliability +- Everything stays contained in the skill folder + +**Note:** The setup uses real Chrome instead of Chromium for cross-platform reliability, consistent browser fingerprinting, and better anti-detection with Google services + +--- + +## Quick Start + +### 1. Check your skills + +Say in Claude Code: +``` +"What skills do I have?" +``` + +Claude will list your available skills including NotebookLM. + +### 2. Authenticate with Google (one-time) + +``` +"Set up NotebookLM authentication" +``` +*A Chrome window opens → log in with your Google account* + +### 3. Create your knowledge base + +Go to [notebooklm.google.com](https://notebooklm.google.com) → Create notebook → Upload your docs: +- 📄 PDFs, Google Docs, markdown files +- 🔗 Websites, GitHub repos +- 🎥 YouTube videos +- 📚 Multiple sources per notebook + +Share: **⚙️ Share → Anyone with link → Copy** + +### 4. Add to your library + +**Option A: Let Claude figure it out (Smart Add)** +``` +"Query this notebook about its content and add it to my library: [your-link]" +``` +Claude will automatically query the notebook to discover its content, then add it with appropriate metadata. + +**Option B: Manual add** +``` +"Add this NotebookLM to my library: [your-link]" +``` +Claude will ask for a name and topics, then save it for future use. + +### 5. Start researching + +``` +"What does my React docs say about hooks?" +``` + +Claude automatically selects the right notebook and gets the answer directly from NotebookLM. + +--- + +## How It Works + +This is a **Claude Code Skill** - a local folder containing instructions and scripts that Claude Code can use when needed. Unlike the [MCP server version](https://github.com/PleasePrompto/notebooklm-mcp), this runs directly in Claude Code without needing a separate server. + +### Key Differences from MCP Server + +| Feature | This Skill | MCP Server | +|---------|------------|------------| +| **Protocol** | Claude Skills | Model Context Protocol | +| **Installation** | Clone to `~/.claude/skills` | `claude mcp add ...` | +| **Sessions** | Fresh browser each question | Persistent chat sessions | +| **Compatibility** | Claude Code only (local) | Claude Code, Codex, Cursor, etc. | +| **Language** | Python | TypeScript | +| **Distribution** | Git clone | npm package | + +### Architecture + +``` +~/.claude/skills/notebooklm/ +├── SKILL.md # Instructions for Claude +├── scripts/ # Python automation scripts +│ ├── ask_question.py # Query NotebookLM +│ ├── notebook_manager.py # Library management +│ └── auth_manager.py # Google authentication +├── .venv/ # Isolated Python environment (auto-created) +└── data/ # Local notebook library +``` + +When you mention NotebookLM or send a notebook URL, Claude: +1. Loads the skill instructions +2. Runs the appropriate Python script +3. Opens a browser, asks your question +4. Returns the answer directly to you +5. Uses that knowledge to help with your task + +--- + +## Core Features + +### **Source-Grounded Responses** +NotebookLM significantly reduces hallucinations by answering exclusively from your uploaded documents. If information isn't available, it indicates uncertainty rather than inventing content. + +### **Direct Integration** +No copy-paste between browser and editor. Claude asks and receives answers programmatically. + +### **Smart Library Management** +Save NotebookLM links with tags and descriptions. Claude auto-selects the right notebook for your task. + +### **Automatic Authentication** +One-time Google login, then authentication persists across sessions. + +### **Self-Contained** +Everything runs in the skill folder with an isolated Python environment. No global installations. + +### **Human-Like Automation** +Uses realistic typing speeds and interaction patterns to avoid detection. + +--- + +## Common Commands + +| What you say | What happens | +|--------------|--------------| +| *"Set up NotebookLM authentication"* | Opens Chrome for Google login | +| *"Add [link] to my NotebookLM library"* | Saves notebook with metadata | +| *"Show my NotebookLM notebooks"* | Lists all saved notebooks | +| *"Ask my API docs about [topic]"* | Queries the relevant notebook | +| *"Use the React notebook"* | Sets active notebook | +| *"Clear NotebookLM data"* | Fresh start (keeps library) | + +--- + +## Real-World Examples + +### Example 1: Workshop Manual Query + +**User asks**: "Check my Suzuki GSR 600 workshop manual for brake fluid type, engine oil specs, and rear axle torque." + +**Claude automatically**: +- Authenticates with NotebookLM +- Asks comprehensive questions about each specification +- Follows up when prompted "Is that ALL you need to know?" +- Provides accurate specifications: DOT 4 brake fluid, SAE 10W-40 oil, 100 N·m rear axle torque + +![NotebookLM Chat Example](images/example_notebookchat.png) + +### Example 2: Building Without Hallucinations + +**You**: "I need to build an n8n workflow for Gmail spam filtering. Use my n8n notebook." + +**Claude's internal process:** +``` +→ Loads NotebookLM skill +→ Activates n8n notebook +→ Asks comprehensive questions with follow-ups +→ Synthesizes complete answer from multiple queries +``` + +**Result**: Working workflow on first try, no debugging hallucinated APIs. + +--- + +## Technical Details + +### Core Technology +- **Patchright**: Browser automation library (Playwright-based) +- **Python**: Implementation language for this skill +- **Stealth techniques**: Human-like typing and interaction patterns + +Note: The MCP server uses the same Patchright library but via TypeScript/npm ecosystem. + +### Dependencies +- **patchright==1.55.2**: Browser automation +- **python-dotenv==1.0.0**: Environment configuration +- Automatically installed in `.venv` on first use + +### Data Storage + +All data is stored locally within the skill directory: + +``` +~/.claude/skills/notebooklm/data/ +├── library.json - Your notebook library with metadata +├── auth_info.json - Authentication status info +└── browser_state/ - Browser cookies and session data +``` + +**Important Security Note:** +- The `data/` directory contains sensitive authentication data and personal notebooks +- It's automatically excluded from git via `.gitignore` +- NEVER manually commit or share the contents of the `data/` directory + +### Session Model + +Unlike the MCP server, this skill uses a **stateless model**: +- Each question opens a fresh browser +- Asks the question, gets the answer +- Adds a follow-up prompt to encourage Claude to ask more questions +- Closes the browser immediately + +This means: +- No persistent chat context +- Each question is independent +- But your notebook library persists +- **Follow-up mechanism**: Each answer includes "Is that ALL you need to know?" to prompt Claude to ask comprehensive follow-ups + +For multi-step research, Claude automatically asks follow-up questions when needed. + +--- + +## Limitations + +### Skill-Specific +- **Local Claude Code only** - Does not work in web UI (sandbox restrictions) +- **No session persistence** - Each question is independent +- **No follow-up context** - Can't reference "the previous answer" + +### NotebookLM +- **Rate limits** - Free tier has daily query limits +- **Manual upload** - You must upload docs to NotebookLM first +- **Share requirement** - Notebooks must be shared publicly + +--- + +## FAQ + +**Why doesn't this work in the Claude web UI?** +The web UI runs skills in a sandbox without network access. Browser automation requires network access to reach NotebookLM. + +**How is this different from the MCP server?** +This is a simpler, Python-based implementation that runs directly as a Claude Skill. The MCP server is more feature-rich with persistent sessions and works with multiple tools (Codex, Cursor, etc.). + +**Can I use both this skill and the MCP server?** +Yes! They serve different purposes. Use the skill for quick Claude Code integration, use the MCP server for persistent sessions and multi-tool support. + +**What if Chrome crashes?** +Run: `"Clear NotebookLM browser data"` and try again. + +**Is my Google account secure?** +Chrome runs locally on your machine. Your credentials never leave your computer. Use a dedicated Google account if you're concerned. + +--- + +## Troubleshooting + +### Skill not found +```bash +# Make sure it's in the right location +ls ~/.claude/skills/notebooklm/ +# Should show: SKILL.md, scripts/, etc. +``` + +### Authentication issues +Say: `"Reset NotebookLM authentication"` + +### Browser crashes +Say: `"Clear NotebookLM browser data"` + +### Dependencies issues +```bash +# Manual reinstall if needed +cd ~/.claude/skills/notebooklm +rm -rf .venv +python -m venv .venv +source .venv/bin/activate # or .venv\Scripts\activate on Windows +pip install -r requirements.txt +``` + +--- + +## Disclaimer + +This tool automates browser interactions with NotebookLM to make your workflow more efficient. However, a few friendly reminders: + +**About browser automation:** +While I've built in humanization features (realistic typing speeds, natural delays, mouse movements) to make the automation behave more naturally, I can't guarantee Google won't detect or flag automated usage. I recommend using a dedicated Google account for automation rather than your primary account—think of it like web scraping: probably fine, but better safe than sorry! + +**About CLI tools and AI agents:** +CLI tools like Claude Code, Codex, and similar AI-powered assistants are incredibly powerful, but they can make mistakes. Please use them with care and awareness: +- Always review changes before committing or deploying +- Test in safe environments first +- Keep backups of important work +- Remember: AI agents are assistants, not infallible oracles + +I built this tool for myself because I was tired of the copy-paste dance between NotebookLM and my editor. I'm sharing it in the hope it helps others too, but I can't take responsibility for any issues, data loss, or account problems that might occur. Use at your own discretion and judgment. + +That said, if you run into problems or have questions, feel free to open an issue on GitHub. I'm happy to help troubleshoot! + +--- + +## Credits + +This skill is inspired by my [**NotebookLM MCP Server**](https://github.com/PleasePrompto/notebooklm-mcp) and provides an alternative implementation as a Claude Code Skill: +- Both use Patchright for browser automation (TypeScript for MCP, Python for Skill) +- Skill version runs directly in Claude Code without MCP protocol +- Stateless design optimized for skill architecture + +If you need: +- **Persistent sessions** → Use the [MCP Server](https://github.com/PleasePrompto/notebooklm-mcp) +- **Multiple tool support** (Codex, Cursor) → Use the [MCP Server](https://github.com/PleasePrompto/notebooklm-mcp) +- **Quick Claude Code integration** → Use this skill + +--- + +## The Bottom Line + +**Without this skill**: NotebookLM in browser → Copy answer → Paste in Claude → Copy next question → Back to browser... + +**With this skill**: Claude researches directly → Gets answers instantly → Writes correct code + +Stop the copy-paste dance. Start getting accurate, grounded answers directly in Claude Code. + +```bash +# Get started in 30 seconds +cd ~/.claude/skills +git clone https://github.com/PleasePrompto/notebooklm-skill notebooklm +# Open Claude Code: "What are my skills?" +``` + +--- + +
+ +Built as a Claude Code Skill adaptation of my [NotebookLM MCP Server](https://github.com/PleasePrompto/notebooklm-mcp) + +For source-grounded, document-based research directly in Claude Code + +
diff --git a/personas/_shared/community-skills/notebooklm/SKILL.md b/personas/_shared/community-skills/notebooklm/SKILL.md new file mode 100755 index 0000000..2be7e16 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/SKILL.md @@ -0,0 +1,269 @@ +--- +name: notebooklm +description: Use this skill to query your Google NotebookLM notebooks directly from Claude Code for source-grounded, citation-backed answers from Gemini. Browser automation, library management, persistent auth. Drastically reduced hallucinations through document-only responses. +--- + +# NotebookLM Research Assistant Skill + +Interact with Google NotebookLM to query documentation with Gemini's source-grounded answers. Each question opens a fresh browser session, retrieves the answer exclusively from your uploaded documents, and closes. + +## When to Use This Skill + +Trigger when user: +- Mentions NotebookLM explicitly +- Shares NotebookLM URL (`https://notebooklm.google.com/notebook/...`) +- Asks to query their notebooks/documentation +- Wants to add documentation to NotebookLM library +- Uses phrases like "ask my NotebookLM", "check my docs", "query my notebook" + +## ⚠️ CRITICAL: Add Command - Smart Discovery + +When user wants to add a notebook without providing details: + +**SMART ADD (Recommended)**: Query the notebook first to discover its content: +```bash +# Step 1: Query the notebook about its content +python scripts/run.py ask_question.py --question "What is the content of this notebook? What topics are covered? Provide a complete overview briefly and concisely" --notebook-url "[URL]" + +# Step 2: Use the discovered information to add it +python scripts/run.py notebook_manager.py add --url "[URL]" --name "[Based on content]" --description "[Based on content]" --topics "[Based on content]" +``` + +**MANUAL ADD**: If user provides all details: +- `--url` - The NotebookLM URL +- `--name` - A descriptive name +- `--description` - What the notebook contains (REQUIRED!) +- `--topics` - Comma-separated topics (REQUIRED!) + +NEVER guess or use generic descriptions! If details missing, use Smart Add to discover them. + +## Critical: Always Use run.py Wrapper + +**NEVER call scripts directly. ALWAYS use `python scripts/run.py [script]`:** + +```bash +# ✅ CORRECT - Always use run.py: +python scripts/run.py auth_manager.py status +python scripts/run.py notebook_manager.py list +python scripts/run.py ask_question.py --question "..." + +# ❌ WRONG - Never call directly: +python scripts/auth_manager.py status # Fails without venv! +``` + +The `run.py` wrapper automatically: +1. Creates `.venv` if needed +2. Installs all dependencies +3. Activates environment +4. Executes script properly + +## Core Workflow + +### Step 1: Check Authentication Status +```bash +python scripts/run.py auth_manager.py status +``` + +If not authenticated, proceed to setup. + +### Step 2: Authenticate (One-Time Setup) +```bash +# Browser MUST be visible for manual Google login +python scripts/run.py auth_manager.py setup +``` + +**Important:** +- Browser is VISIBLE for authentication +- Browser window opens automatically +- User must manually log in to Google +- Tell user: "A browser window will open for Google login" + +### Step 3: Manage Notebook Library + +```bash +# List all notebooks +python scripts/run.py notebook_manager.py list + +# BEFORE ADDING: Ask user for metadata if unknown! +# "What does this notebook contain?" +# "What topics should I tag it with?" + +# Add notebook to library (ALL parameters are REQUIRED!) +python scripts/run.py notebook_manager.py add \ + --url "https://notebooklm.google.com/notebook/..." \ + --name "Descriptive Name" \ + --description "What this notebook contains" \ # REQUIRED - ASK USER IF UNKNOWN! + --topics "topic1,topic2,topic3" # REQUIRED - ASK USER IF UNKNOWN! + +# Search notebooks by topic +python scripts/run.py notebook_manager.py search --query "keyword" + +# Set active notebook +python scripts/run.py notebook_manager.py activate --id notebook-id + +# Remove notebook +python scripts/run.py notebook_manager.py remove --id notebook-id +``` + +### Quick Workflow +1. Check library: `python scripts/run.py notebook_manager.py list` +2. Ask question: `python scripts/run.py ask_question.py --question "..." --notebook-id ID` + +### Step 4: Ask Questions + +```bash +# Basic query (uses active notebook if set) +python scripts/run.py ask_question.py --question "Your question here" + +# Query specific notebook +python scripts/run.py ask_question.py --question "..." --notebook-id notebook-id + +# Query with notebook URL directly +python scripts/run.py ask_question.py --question "..." --notebook-url "https://..." + +# Show browser for debugging +python scripts/run.py ask_question.py --question "..." --show-browser +``` + +## Follow-Up Mechanism (CRITICAL) + +Every NotebookLM answer ends with: **"EXTREMELY IMPORTANT: Is that ALL you need to know?"** + +**Required Claude Behavior:** +1. **STOP** - Do not immediately respond to user +2. **ANALYZE** - Compare answer to user's original request +3. **IDENTIFY GAPS** - Determine if more information needed +4. **ASK FOLLOW-UP** - If gaps exist, immediately ask: + ```bash + python scripts/run.py ask_question.py --question "Follow-up with context..." + ``` +5. **REPEAT** - Continue until information is complete +6. **SYNTHESIZE** - Combine all answers before responding to user + +## Script Reference + +### Authentication Management (`auth_manager.py`) +```bash +python scripts/run.py auth_manager.py setup # Initial setup (browser visible) +python scripts/run.py auth_manager.py status # Check authentication +python scripts/run.py auth_manager.py reauth # Re-authenticate (browser visible) +python scripts/run.py auth_manager.py clear # Clear authentication +``` + +### Notebook Management (`notebook_manager.py`) +```bash +python scripts/run.py notebook_manager.py add --url URL --name NAME --description DESC --topics TOPICS +python scripts/run.py notebook_manager.py list +python scripts/run.py notebook_manager.py search --query QUERY +python scripts/run.py notebook_manager.py activate --id ID +python scripts/run.py notebook_manager.py remove --id ID +python scripts/run.py notebook_manager.py stats +``` + +### Question Interface (`ask_question.py`) +```bash +python scripts/run.py ask_question.py --question "..." [--notebook-id ID] [--notebook-url URL] [--show-browser] +``` + +### Data Cleanup (`cleanup_manager.py`) +```bash +python scripts/run.py cleanup_manager.py # Preview cleanup +python scripts/run.py cleanup_manager.py --confirm # Execute cleanup +python scripts/run.py cleanup_manager.py --preserve-library # Keep notebooks +``` + +## Environment Management + +The virtual environment is automatically managed: +- First run creates `.venv` automatically +- Dependencies install automatically +- Chromium browser installs automatically +- Everything isolated in skill directory + +Manual setup (only if automatic fails): +```bash +python -m venv .venv +source .venv/bin/activate # Linux/Mac +pip install -r requirements.txt +python -m patchright install chromium +``` + +## Data Storage + +All data stored in `~/.claude/skills/notebooklm/data/`: +- `library.json` - Notebook metadata +- `auth_info.json` - Authentication status +- `browser_state/` - Browser cookies and session + +**Security:** Protected by `.gitignore`, never commit to git. + +## Configuration + +Optional `.env` file in skill directory: +```env +HEADLESS=false # Browser visibility +SHOW_BROWSER=false # Default browser display +STEALTH_ENABLED=true # Human-like behavior +TYPING_WPM_MIN=160 # Typing speed +TYPING_WPM_MAX=240 +DEFAULT_NOTEBOOK_ID= # Default notebook +``` + +## Decision Flow + +``` +User mentions NotebookLM + ↓ +Check auth → python scripts/run.py auth_manager.py status + ↓ +If not authenticated → python scripts/run.py auth_manager.py setup + ↓ +Check/Add notebook → python scripts/run.py notebook_manager.py list/add (with --description) + ↓ +Activate notebook → python scripts/run.py notebook_manager.py activate --id ID + ↓ +Ask question → python scripts/run.py ask_question.py --question "..." + ↓ +See "Is that ALL you need?" → Ask follow-ups until complete + ↓ +Synthesize and respond to user +``` + +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| ModuleNotFoundError | Use `run.py` wrapper | +| Authentication fails | Browser must be visible for setup! --show-browser | +| Rate limit (50/day) | Wait or switch Google account | +| Browser crashes | `python scripts/run.py cleanup_manager.py --preserve-library` | +| Notebook not found | Check with `notebook_manager.py list` | + +## Best Practices + +1. **Always use run.py** - Handles environment automatically +2. **Check auth first** - Before any operations +3. **Follow-up questions** - Don't stop at first answer +4. **Browser visible for auth** - Required for manual login +5. **Include context** - Each question is independent +6. **Synthesize answers** - Combine multiple responses + +## Limitations + +- No session persistence (each question = new browser) +- Rate limits on free Google accounts (50 queries/day) +- Manual upload required (user must add docs to NotebookLM) +- Browser overhead (few seconds per question) + +## Resources (Skill Structure) + +**Important directories and files:** + +- `scripts/` - All automation scripts (ask_question.py, notebook_manager.py, etc.) +- `data/` - Local storage for authentication and notebook library +- `references/` - Extended documentation: + - `api_reference.md` - Detailed API documentation for all scripts + - `troubleshooting.md` - Common issues and solutions + - `usage_patterns.md` - Best practices and workflow examples +- `.venv/` - Isolated Python environment (auto-created on first run) +- `.gitignore` - Protects sensitive data from being committed diff --git a/personas/_shared/community-skills/notebooklm/references/api_reference.md b/personas/_shared/community-skills/notebooklm/references/api_reference.md new file mode 100755 index 0000000..a0ce65e --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/references/api_reference.md @@ -0,0 +1,309 @@ +# NotebookLM Skill API Reference + +Complete API documentation for all NotebookLM skill modules. + +## Important: Always Use run.py Wrapper + +**All commands must use the `run.py` wrapper to ensure proper environment:** + +```bash +# ✅ CORRECT: +python scripts/run.py [script_name].py [arguments] + +# ❌ WRONG: +python scripts/[script_name].py [arguments] # Will fail without venv! +``` + +## Core Scripts + +### ask_question.py +Query NotebookLM with automated browser interaction. + +```bash +# Basic usage +python scripts/run.py ask_question.py --question "Your question" + +# With specific notebook +python scripts/run.py ask_question.py --question "..." --notebook-id notebook-id + +# With direct URL +python scripts/run.py ask_question.py --question "..." --notebook-url "https://..." + +# Show browser (debugging) +python scripts/run.py ask_question.py --question "..." --show-browser +``` + +**Parameters:** +- `--question` (required): Question to ask +- `--notebook-id`: Use notebook from library +- `--notebook-url`: Use URL directly +- `--show-browser`: Make browser visible + +**Returns:** Answer text with follow-up prompt appended + +### notebook_manager.py +Manage notebook library with CRUD operations. + +```bash +# Smart Add (discover content first) +python scripts/run.py ask_question.py --question "What is the content of this notebook? What topics are covered? Provide a complete overview briefly and concisely" --notebook-url "[URL]" +# Then add with discovered info +python scripts/run.py notebook_manager.py add \ + --url "https://notebooklm.google.com/notebook/..." \ + --name "Name" \ + --description "Description" \ + --topics "topic1,topic2" + +# Direct add (when you know the content) +python scripts/run.py notebook_manager.py add \ + --url "https://notebooklm.google.com/notebook/..." \ + --name "Name" \ + --description "What it contains" \ + --topics "topic1,topic2" + +# List notebooks +python scripts/run.py notebook_manager.py list + +# Search notebooks +python scripts/run.py notebook_manager.py search --query "keyword" + +# Activate notebook +python scripts/run.py notebook_manager.py activate --id notebook-id + +# Remove notebook +python scripts/run.py notebook_manager.py remove --id notebook-id + +# Show statistics +python scripts/run.py notebook_manager.py stats +``` + +**Commands:** +- `add`: Add notebook (requires --url, --name, --topics) +- `list`: Show all notebooks +- `search`: Find notebooks by keyword +- `activate`: Set default notebook +- `remove`: Delete from library +- `stats`: Display library statistics + +### auth_manager.py +Handle Google authentication and browser state. + +```bash +# Setup (browser visible for login) +python scripts/run.py auth_manager.py setup + +# Check status +python scripts/run.py auth_manager.py status + +# Re-authenticate +python scripts/run.py auth_manager.py reauth + +# Clear authentication +python scripts/run.py auth_manager.py clear +``` + +**Commands:** +- `setup`: Initial authentication (browser MUST be visible) +- `status`: Check if authenticated +- `reauth`: Clear and re-setup +- `clear`: Remove all auth data + +### cleanup_manager.py +Clean skill data with preservation options. + +```bash +# Preview cleanup +python scripts/run.py cleanup_manager.py + +# Execute cleanup +python scripts/run.py cleanup_manager.py --confirm + +# Keep library +python scripts/run.py cleanup_manager.py --confirm --preserve-library + +# Force without prompt +python scripts/run.py cleanup_manager.py --confirm --force +``` + +**Options:** +- `--confirm`: Actually perform cleanup +- `--preserve-library`: Keep notebook library +- `--force`: Skip confirmation prompt + +### run.py +Script wrapper that handles environment setup. + +```bash +# Usage +python scripts/run.py [script_name].py [arguments] + +# Examples +python scripts/run.py auth_manager.py status +python scripts/run.py ask_question.py --question "..." +``` + +**Automatic actions:** +1. Creates `.venv` if missing +2. Installs dependencies +3. Activates environment +4. Executes target script + +## Python API Usage + +### Using subprocess with run.py + +```python +import subprocess +import json + +# Always use run.py wrapper +result = subprocess.run([ + "python", "scripts/run.py", "ask_question.py", + "--question", "Your question", + "--notebook-id", "notebook-id" +], capture_output=True, text=True) + +answer = result.stdout +``` + +### Direct imports (after venv exists) + +```python +# Only works if venv is already created and activated +from notebook_manager import NotebookLibrary +from auth_manager import AuthManager + +library = NotebookLibrary() +notebooks = library.list_notebooks() + +auth = AuthManager() +is_auth = auth.is_authenticated() +``` + +## Data Storage + +Location: `~/.claude/skills/notebooklm/data/` + +``` +data/ +├── library.json # Notebook metadata +├── auth_info.json # Auth status +└── browser_state/ # Browser cookies + └── state.json +``` + +**Security:** Protected by `.gitignore`, never commit. + +## Environment Variables + +Optional `.env` file configuration: + +```env +HEADLESS=false # Browser visibility +SHOW_BROWSER=false # Default display +STEALTH_ENABLED=true # Human behavior +TYPING_WPM_MIN=160 # Typing speed +TYPING_WPM_MAX=240 +DEFAULT_NOTEBOOK_ID= # Default notebook +``` + +## Error Handling + +Common patterns: + +```python +# Using run.py prevents most errors +result = subprocess.run([ + "python", "scripts/run.py", "ask_question.py", + "--question", "Question" +], capture_output=True, text=True) + +if result.returncode != 0: + error = result.stderr + if "rate limit" in error.lower(): + # Wait or switch accounts + pass + elif "not authenticated" in error.lower(): + # Run auth setup + subprocess.run(["python", "scripts/run.py", "auth_manager.py", "setup"]) +``` + +## Rate Limits + +Free Google accounts: 50 queries/day + +Solutions: +1. Wait for reset (midnight PST) +2. Switch accounts with `reauth` +3. Use multiple Google accounts + +## Advanced Patterns + +### Parallel Queries + +```python +import concurrent.futures +import subprocess + +def query(question, notebook_id): + result = subprocess.run([ + "python", "scripts/run.py", "ask_question.py", + "--question", question, + "--notebook-id", notebook_id + ], capture_output=True, text=True) + return result.stdout + +# Run multiple queries simultaneously +with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: + futures = [ + executor.submit(query, q, nb) + for q, nb in zip(questions, notebooks) + ] + results = [f.result() for f in futures] +``` + +### Batch Processing + +```python +def batch_research(questions, notebook_id): + results = [] + for question in questions: + result = subprocess.run([ + "python", "scripts/run.py", "ask_question.py", + "--question", question, + "--notebook-id", notebook_id + ], capture_output=True, text=True) + results.append(result.stdout) + time.sleep(2) # Avoid rate limits + return results +``` + +## Module Classes + +### NotebookLibrary +- `add_notebook(url, name, topics)` +- `list_notebooks()` +- `search_notebooks(query)` +- `get_notebook(notebook_id)` +- `activate_notebook(notebook_id)` +- `remove_notebook(notebook_id)` + +### AuthManager +- `is_authenticated()` +- `setup_auth(headless=False)` +- `get_auth_info()` +- `clear_auth()` +- `validate_auth()` + +### BrowserSession (internal) +- Handles browser automation +- Manages stealth behavior +- Not intended for direct use + +## Best Practices + +1. **Always use run.py** - Ensures environment +2. **Check auth first** - Before operations +3. **Handle rate limits** - Implement retries +4. **Include context** - Questions are independent +5. **Clean sessions** - Use cleanup_manager \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/references/troubleshooting.md b/personas/_shared/community-skills/notebooklm/references/troubleshooting.md new file mode 100755 index 0000000..992aeb7 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/references/troubleshooting.md @@ -0,0 +1,376 @@ +# NotebookLM Skill Troubleshooting Guide + +## Quick Fix Table + +| Error | Solution | +|-------|----------| +| ModuleNotFoundError | Use `python scripts/run.py [script].py` | +| Authentication failed | Browser must be visible for setup | +| Browser crash | `python scripts/run.py cleanup_manager.py --preserve-library` | +| Rate limit hit | Wait 1 hour or switch accounts | +| Notebook not found | `python scripts/run.py notebook_manager.py list` | +| Script not working | Always use run.py wrapper | + +## Critical: Always Use run.py + +Most issues are solved by using the run.py wrapper: + +```bash +# ✅ CORRECT - Always: +python scripts/run.py auth_manager.py status +python scripts/run.py ask_question.py --question "..." + +# ❌ WRONG - Never: +python scripts/auth_manager.py status # ModuleNotFoundError! +``` + +## Common Issues and Solutions + +### Authentication Issues + +#### Not authenticated error +``` +Error: Not authenticated. Please run auth setup first. +``` + +**Solution:** +```bash +# Check status +python scripts/run.py auth_manager.py status + +# Setup authentication (browser MUST be visible!) +python scripts/run.py auth_manager.py setup +# User must manually log in to Google + +# If setup fails, try re-authentication +python scripts/run.py auth_manager.py reauth +``` + +#### Authentication expires frequently +**Solution:** +```bash +# Clear old authentication +python scripts/run.py cleanup_manager.py --preserve-library + +# Fresh authentication setup +python scripts/run.py auth_manager.py setup --timeout 15 + +# Use persistent browser profile +export PERSIST_AUTH=true +``` + +#### Google blocks automated login +**Solution:** +1. Use dedicated Google account for automation +2. Enable "Less secure app access" if available +3. ALWAYS use visible browser: +```bash +python scripts/run.py auth_manager.py setup +# Browser MUST be visible - user logs in manually +# NO headless parameter exists - use --show-browser for debugging +``` + +### Browser Issues + +#### Browser crashes or hangs +``` +TimeoutError: Waiting for selector failed +``` + +**Solution:** +```bash +# Kill hanging processes +pkill -f chromium +pkill -f chrome + +# Clean browser state +python scripts/run.py cleanup_manager.py --confirm --preserve-library + +# Re-authenticate +python scripts/run.py auth_manager.py reauth +``` + +#### Browser not found error +**Solution:** +```bash +# Install Chromium via run.py (automatic) +python scripts/run.py auth_manager.py status +# run.py will install Chromium automatically + +# Or manual install if needed +cd ~/.claude/skills/notebooklm +source .venv/bin/activate +python -m patchright install chromium +``` + +### Rate Limiting + +#### Rate limit exceeded (50 queries/day) +**Solutions:** + +**Option 1: Wait** +```bash +# Check when limit resets (usually midnight PST) +date -d "tomorrow 00:00 PST" +``` + +**Option 2: Switch accounts** +```bash +# Clear current auth +python scripts/run.py auth_manager.py clear + +# Login with different account +python scripts/run.py auth_manager.py setup +``` + +**Option 3: Rotate accounts** +```python +# Use multiple accounts +accounts = ["account1", "account2"] +for account in accounts: + # Switch account on rate limit + subprocess.run(["python", "scripts/run.py", "auth_manager.py", "reauth"]) +``` + +### Notebook Access Issues + +#### Notebook not found +**Solution:** +```bash +# List all notebooks +python scripts/run.py notebook_manager.py list + +# Search for notebook +python scripts/run.py notebook_manager.py search --query "keyword" + +# Add notebook if missing +python scripts/run.py notebook_manager.py add \ + --url "https://notebooklm.google.com/..." \ + --name "Name" \ + --topics "topics" +``` + +#### Access denied to notebook +**Solution:** +1. Check if notebook is still shared publicly +2. Re-add notebook with updated URL +3. Verify correct Google account is used + +#### Wrong notebook being used +**Solution:** +```bash +# Check active notebook +python scripts/run.py notebook_manager.py list | grep "active" + +# Activate correct notebook +python scripts/run.py notebook_manager.py activate --id correct-id +``` + +### Virtual Environment Issues + +#### ModuleNotFoundError +``` +ModuleNotFoundError: No module named 'patchright' +``` + +**Solution:** +```bash +# ALWAYS use run.py - it handles venv automatically! +python scripts/run.py [any_script].py + +# run.py will: +# 1. Create .venv if missing +# 2. Install dependencies +# 3. Run the script +``` + +#### Wrong Python version +**Solution:** +```bash +# Check Python version (needs 3.8+) +python --version + +# If wrong version, specify correct Python +python3.8 scripts/run.py auth_manager.py status +``` + +### Network Issues + +#### Connection timeouts +**Solution:** +```bash +# Increase timeout +export TIMEOUT_SECONDS=60 + +# Check connectivity +ping notebooklm.google.com + +# Use proxy if needed +export HTTP_PROXY=http://proxy:port +export HTTPS_PROXY=http://proxy:port +``` + +### Data Issues + +#### Corrupted notebook library +``` +JSON decode error when listing notebooks +``` + +**Solution:** +```bash +# Backup current library +cp ~/.claude/skills/notebooklm/data/library.json library.backup.json + +# Reset library +rm ~/.claude/skills/notebooklm/data/library.json + +# Re-add notebooks +python scripts/run.py notebook_manager.py add --url ... --name ... +``` + +#### Disk space full +**Solution:** +```bash +# Check disk usage +df -h ~/.claude/skills/notebooklm/data/ + +# Clean up +python scripts/run.py cleanup_manager.py --confirm --preserve-library +``` + +## Debugging Techniques + +### Enable verbose logging +```bash +export DEBUG=true +export LOG_LEVEL=DEBUG +python scripts/run.py ask_question.py --question "Test" --show-browser +``` + +### Test individual components +```bash +# Test authentication +python scripts/run.py auth_manager.py status + +# Test notebook access +python scripts/run.py notebook_manager.py list + +# Test browser launch +python scripts/run.py ask_question.py --question "test" --show-browser +``` + +### Save screenshots on error +Add to scripts for debugging: +```python +try: + # Your code +except Exception as e: + page.screenshot(path=f"error_{timestamp}.png") + raise e +``` + +## Recovery Procedures + +### Complete reset +```bash +#!/bin/bash +# Kill processes +pkill -f chromium + +# Backup library if exists +if [ -f ~/.claude/skills/notebooklm/data/library.json ]; then + cp ~/.claude/skills/notebooklm/data/library.json ~/library.backup.json +fi + +# Clean everything +cd ~/.claude/skills/notebooklm +python scripts/run.py cleanup_manager.py --confirm --force + +# Remove venv +rm -rf .venv + +# Reinstall (run.py will handle this) +python scripts/run.py auth_manager.py setup + +# Restore library if backup exists +if [ -f ~/library.backup.json ]; then + mkdir -p ~/.claude/skills/notebooklm/data/ + cp ~/library.backup.json ~/.claude/skills/notebooklm/data/library.json +fi +``` + +### Partial recovery (keep data) +```bash +# Keep auth and library, fix execution +cd ~/.claude/skills/notebooklm +rm -rf .venv + +# run.py will recreate venv automatically +python scripts/run.py auth_manager.py status +``` + +## Error Messages Reference + +### Authentication Errors +| Error | Cause | Solution | +|-------|-------|----------| +| Not authenticated | No valid auth | `run.py auth_manager.py setup` | +| Authentication expired | Session old | `run.py auth_manager.py reauth` | +| Invalid credentials | Wrong account | Check Google account | +| 2FA required | Security challenge | Complete in visible browser | + +### Browser Errors +| Error | Cause | Solution | +|-------|-------|----------| +| Browser not found | Chromium missing | Use run.py (auto-installs) | +| Connection refused | Browser crashed | Kill processes, restart | +| Timeout waiting | Page slow | Increase timeout | +| Context closed | Browser terminated | Check logs for crashes | + +### Notebook Errors +| Error | Cause | Solution | +|-------|-------|----------| +| Notebook not found | Invalid ID | `run.py notebook_manager.py list` | +| Access denied | Not shared | Re-share in NotebookLM | +| Invalid URL | Wrong format | Use full NotebookLM URL | +| No active notebook | None selected | `run.py notebook_manager.py activate` | + +## Prevention Tips + +1. **Always use run.py** - Prevents 90% of issues +2. **Regular maintenance** - Clear browser state weekly +3. **Monitor queries** - Track daily count to avoid limits +4. **Backup library** - Export notebook list regularly +5. **Use dedicated account** - Separate Google account for automation + +## Getting Help + +### Diagnostic information to collect +```bash +# System info +python --version +cd ~/.claude/skills/notebooklm +ls -la + +# Skill status +python scripts/run.py auth_manager.py status +python scripts/run.py notebook_manager.py list | head -5 + +# Check data directory +ls -la ~/.claude/skills/notebooklm/data/ +``` + +### Common questions + +**Q: Why doesn't this work in Claude web UI?** +A: Web UI has no network access. Use local Claude Code. + +**Q: Can I use multiple Google accounts?** +A: Yes, use `run.py auth_manager.py reauth` to switch. + +**Q: How to increase rate limit?** +A: Use multiple accounts or upgrade to Google Workspace. + +**Q: Is this safe for my Google account?** +A: Use dedicated account for automation. Only accesses NotebookLM. \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/references/usage_patterns.md b/personas/_shared/community-skills/notebooklm/references/usage_patterns.md new file mode 100755 index 0000000..ad517e9 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/references/usage_patterns.md @@ -0,0 +1,338 @@ +# NotebookLM Skill Usage Patterns + +Advanced patterns for using the NotebookLM skill effectively. + +## Critical: Always Use run.py + +**Every command must use the run.py wrapper:** +```bash +# ✅ CORRECT: +python scripts/run.py auth_manager.py status +python scripts/run.py ask_question.py --question "..." + +# ❌ WRONG: +python scripts/auth_manager.py status # Will fail! +``` + +## Pattern 1: Initial Setup + +```bash +# 1. Check authentication (using run.py!) +python scripts/run.py auth_manager.py status + +# 2. If not authenticated, setup (Browser MUST be visible!) +python scripts/run.py auth_manager.py setup +# Tell user: "Please log in to Google in the browser window" + +# 3. Add first notebook - ASK USER FOR DETAILS FIRST! +# Ask: "What does this notebook contain?" +# Ask: "What topics should I tag it with?" +python scripts/run.py notebook_manager.py add \ + --url "https://notebooklm.google.com/notebook/..." \ + --name "User provided name" \ + --description "User provided description" \ # NEVER GUESS! + --topics "user,provided,topics" # NEVER GUESS! +``` + +**Critical Notes:** +- Virtual environment created automatically by run.py +- Browser MUST be visible for authentication +- ALWAYS discover content via query OR ask user for notebook metadata + +## Pattern 2: Adding Notebooks (Smart Discovery!) + +**When user shares a NotebookLM URL:** + +**OPTION A: Smart Discovery (Recommended)** +```bash +# 1. Query the notebook to discover its content +python scripts/run.py ask_question.py \ + --question "What is the content of this notebook? What topics are covered? Provide a complete overview briefly and concisely" \ + --notebook-url "[URL]" + +# 2. Use discovered info to add it +python scripts/run.py notebook_manager.py add \ + --url "[URL]" \ + --name "[Based on content]" \ + --description "[From discovery]" \ + --topics "[Extracted topics]" +``` + +**OPTION B: Ask User (Fallback)** +```bash +# If discovery fails, ask user: +"What does this notebook contain?" +"What topics does it cover?" + +# Then add with user-provided info: +python scripts/run.py notebook_manager.py add \ + --url "[URL]" \ + --name "[User's answer]" \ + --description "[User's description]" \ + --topics "[User's topics]" +``` + +**NEVER:** +- Guess what's in a notebook +- Use generic descriptions +- Skip discovering content + +## Pattern 3: Daily Research Workflow + +```bash +# Check library +python scripts/run.py notebook_manager.py list + +# Research with comprehensive questions +python scripts/run.py ask_question.py \ + --question "Detailed question with all context" \ + --notebook-id notebook-id + +# Follow-up when you see "Is that ALL you need to know?" +python scripts/run.py ask_question.py \ + --question "Follow-up question with previous context" +``` + +## Pattern 4: Follow-Up Questions (CRITICAL!) + +When NotebookLM responds with "EXTREMELY IMPORTANT: Is that ALL you need to know?": + +```python +# 1. STOP - Don't respond to user yet +# 2. ANALYZE - Is answer complete? +# 3. If gaps exist, ask follow-up: +python scripts/run.py ask_question.py \ + --question "Specific follow-up with context from previous answer" + +# 4. Repeat until complete +# 5. Only then synthesize and respond to user +``` + +## Pattern 5: Multi-Notebook Research + +```python +# Query different notebooks for comparison +python scripts/run.py notebook_manager.py activate --id notebook-1 +python scripts/run.py ask_question.py --question "Question" + +python scripts/run.py notebook_manager.py activate --id notebook-2 +python scripts/run.py ask_question.py --question "Same question" + +# Compare and synthesize answers +``` + +## Pattern 6: Error Recovery + +```bash +# If authentication fails +python scripts/run.py auth_manager.py status +python scripts/run.py auth_manager.py reauth # Browser visible! + +# If browser crashes +python scripts/run.py cleanup_manager.py --preserve-library +python scripts/run.py auth_manager.py setup # Browser visible! + +# If rate limited +# Wait or switch accounts +python scripts/run.py auth_manager.py reauth # Login with different account +``` + +## Pattern 7: Batch Processing + +```bash +#!/bin/bash +NOTEBOOK_ID="notebook-id" +QUESTIONS=( + "First comprehensive question" + "Second comprehensive question" + "Third comprehensive question" +) + +for question in "${QUESTIONS[@]}"; do + echo "Asking: $question" + python scripts/run.py ask_question.py \ + --question "$question" \ + --notebook-id "$NOTEBOOK_ID" + sleep 2 # Avoid rate limits +done +``` + +## Pattern 8: Automated Research Script + +```python +#!/usr/bin/env python +import subprocess + +def research_topic(topic, notebook_id): + # Comprehensive question + question = f""" + Explain {topic} in detail: + 1. Core concepts + 2. Implementation details + 3. Best practices + 4. Common pitfalls + 5. Examples + """ + + result = subprocess.run([ + "python", "scripts/run.py", "ask_question.py", + "--question", question, + "--notebook-id", notebook_id + ], capture_output=True, text=True) + + return result.stdout +``` + +## Pattern 9: Notebook Organization + +```python +# Organize by domain - with proper metadata +# ALWAYS ask user for descriptions! + +# Backend notebooks +add_notebook("Backend API", "Complete API documentation", "api,rest,backend") +add_notebook("Database", "Schema and queries", "database,sql,backend") + +# Frontend notebooks +add_notebook("React Docs", "React framework documentation", "react,frontend") +add_notebook("CSS Framework", "Styling documentation", "css,styling,frontend") + +# Search by domain +python scripts/run.py notebook_manager.py search --query "backend" +python scripts/run.py notebook_manager.py search --query "frontend" +``` + +## Pattern 10: Integration with Development + +```python +# Query documentation during development +def check_api_usage(api_endpoint): + result = subprocess.run([ + "python", "scripts/run.py", "ask_question.py", + "--question", f"Parameters and response format for {api_endpoint}", + "--notebook-id", "api-docs" + ], capture_output=True, text=True) + + # If follow-up needed + if "Is that ALL you need" in result.stdout: + # Ask for examples + follow_up = subprocess.run([ + "python", "scripts/run.py", "ask_question.py", + "--question", f"Show code examples for {api_endpoint}", + "--notebook-id", "api-docs" + ], capture_output=True, text=True) + + return combine_answers(result.stdout, follow_up.stdout) +``` + +## Best Practices + +### 1. Question Formulation +- Be specific and comprehensive +- Include all context in each question +- Request structured responses +- Ask for examples when needed + +### 2. Notebook Management +- **ALWAYS ask user for metadata** +- Use descriptive names +- Add comprehensive topics +- Keep URLs current + +### 3. Performance +- Batch related questions +- Use parallel processing for different notebooks +- Monitor rate limits (50/day) +- Switch accounts if needed + +### 4. Error Handling +- Always use run.py to prevent venv issues +- Check auth before operations +- Implement retry logic +- Have fallback notebooks ready + +### 5. Security +- Use dedicated Google account +- Never commit data/ directory +- Regularly refresh auth +- Track all access + +## Common Workflows for Claude + +### Workflow 1: User Sends NotebookLM URL + +```python +# 1. Detect URL in message +if "notebooklm.google.com" in user_message: + url = extract_url(user_message) + + # 2. Check if in library + notebooks = run("notebook_manager.py list") + + if url not in notebooks: + # 3. ASK USER FOR METADATA (CRITICAL!) + name = ask_user("What should I call this notebook?") + description = ask_user("What does this notebook contain?") + topics = ask_user("What topics does it cover?") + + # 4. Add with user-provided info + run(f"notebook_manager.py add --url {url} --name '{name}' --description '{description}' --topics '{topics}'") + + # 5. Use the notebook + answer = run(f"ask_question.py --question '{user_question}'") +``` + +### Workflow 2: Research Task + +```python +# 1. Understand task +task = "Implement feature X" + +# 2. Formulate comprehensive questions +questions = [ + "Complete implementation guide for X", + "Error handling for X", + "Performance considerations for X" +] + +# 3. Query with follow-ups +for q in questions: + answer = run(f"ask_question.py --question '{q}'") + + # Check if follow-up needed + if "Is that ALL you need" in answer: + # Ask more specific question + follow_up = run(f"ask_question.py --question 'Specific detail about {q}'") + +# 4. Synthesize and implement +``` + +## Tips and Tricks + +1. **Always use run.py** - Prevents all venv issues +2. **Ask for metadata** - Never guess notebook contents +3. **Use verbose questions** - Include all context +4. **Follow up automatically** - When you see the prompt +5. **Monitor rate limits** - 50 queries per day +6. **Batch operations** - Group related queries +7. **Export important answers** - Save locally +8. **Version control notebooks** - Track changes +9. **Test auth regularly** - Before important tasks +10. **Document everything** - Keep notes on notebooks + +## Quick Reference + +```bash +# Always use run.py! +python scripts/run.py [script].py [args] + +# Common operations +run.py auth_manager.py status # Check auth +run.py auth_manager.py setup # Login (browser visible!) +run.py notebook_manager.py list # List notebooks +run.py notebook_manager.py add ... # Add (ask user for metadata!) +run.py ask_question.py --question ... # Query +run.py cleanup_manager.py ... # Clean up +``` + +**Remember:** When in doubt, use run.py and ask the user for notebook details! \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/requirements.txt b/personas/_shared/community-skills/notebooklm/requirements.txt new file mode 100755 index 0000000..6e38008 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/requirements.txt @@ -0,0 +1,10 @@ +# NotebookLM Skill Dependencies +# These will be installed in the skill's local .venv + +# Core browser automation with anti-detection +# Note: After installation, run: patchright install chrome +# (Chrome is required, not Chromium, for cross-platform reliability) +patchright==1.55.2 + +# Environment management +python-dotenv==1.0.0 \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/scripts/__init__.py b/personas/_shared/community-skills/notebooklm/scripts/__init__.py new file mode 100755 index 0000000..e77fffc --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/__init__.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +NotebookLM Skill Scripts Package +Provides automatic environment management for all scripts +""" + +import os +import sys +import subprocess +from pathlib import Path + + +def ensure_venv_and_run(): + """ + Ensure virtual environment exists and run the requested script. + This is called when any script is imported or run directly. + """ + # Only do this if we're not already in the skill's venv + skill_dir = Path(__file__).parent.parent + venv_dir = skill_dir / ".venv" + + # Check if we're in a venv + in_venv = hasattr(sys, 'real_prefix') or ( + hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix + ) + + # Check if it's OUR venv + if in_venv: + venv_path = Path(sys.prefix) + if venv_path == venv_dir: + # We're already in the correct venv + return + + # We need to set up or switch to our venv + if not venv_dir.exists(): + print("🔧 First-time setup detected...") + print(" Creating isolated environment for NotebookLM skill...") + print(" This ensures clean dependency management...") + + # Create venv + import venv + venv.create(venv_dir, with_pip=True) + + # Install requirements + requirements_file = skill_dir / "requirements.txt" + if requirements_file.exists(): + if os.name == 'nt': # Windows + pip_exe = venv_dir / "Scripts" / "pip.exe" + else: + pip_exe = venv_dir / "bin" / "pip" + + print(" Installing dependencies in isolated environment...") + subprocess.run( + [str(pip_exe), "install", "-q", "-r", str(requirements_file)], + check=True + ) + + # Also install patchright's chromium + print(" Setting up browser automation...") + if os.name == 'nt': + python_exe = venv_dir / "Scripts" / "python.exe" + else: + python_exe = venv_dir / "bin" / "python" + + subprocess.run( + [str(python_exe), "-m", "patchright", "install", "chromium"], + check=True, + capture_output=True + ) + + print("✅ Environment ready! All dependencies isolated in .venv/") + + # If we're here and not in the venv, we should recommend using the venv + if not in_venv: + print("\n⚠️ Running outside virtual environment") + print(" Recommended: Use scripts/run.py to ensure clean execution") + print(" Or activate: source .venv/bin/activate") + + +# Check environment when module is imported +ensure_venv_and_run() \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/scripts/ask_question.py b/personas/_shared/community-skills/notebooklm/scripts/ask_question.py new file mode 100755 index 0000000..aa47e4b --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/ask_question.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +""" +Simple NotebookLM Question Interface +Based on MCP server implementation - simplified without sessions + +Implements hybrid auth approach: +- Persistent browser profile (user_data_dir) for fingerprint consistency +- Manual cookie injection from state.json for session cookies (Playwright bug workaround) +See: https://github.com/microsoft/playwright/issues/36139 +""" + +import argparse +import sys +import time +import re +from pathlib import Path + +from patchright.sync_api import sync_playwright + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent)) + +from auth_manager import AuthManager +from notebook_manager import NotebookLibrary +from config import QUERY_INPUT_SELECTORS, RESPONSE_SELECTORS +from browser_utils import BrowserFactory, StealthUtils + + +# Follow-up reminder (adapted from MCP server for stateless operation) +# Since we don't have persistent sessions, we encourage comprehensive questions +FOLLOW_UP_REMINDER = ( + "\n\nEXTREMELY IMPORTANT: Is that ALL you need to know? " + "You can always ask another question! Think about it carefully: " + "before you reply to the user, review their original request and this answer. " + "If anything is still unclear or missing, ask me another comprehensive question " + "that includes all necessary context (since each question opens a new browser session)." +) + + +def ask_notebooklm(question: str, notebook_url: str, headless: bool = True) -> str: + """ + Ask a question to NotebookLM + + Args: + question: Question to ask + notebook_url: NotebookLM notebook URL + headless: Run browser in headless mode + + Returns: + Answer text from NotebookLM + """ + auth = AuthManager() + + if not auth.is_authenticated(): + print("⚠️ Not authenticated. Run: python auth_manager.py setup") + return None + + print(f"💬 Asking: {question}") + print(f"📚 Notebook: {notebook_url}") + + playwright = None + context = None + + try: + # Start playwright + playwright = sync_playwright().start() + + # Launch persistent browser context using factory + context = BrowserFactory.launch_persistent_context( + playwright, + headless=headless + ) + + # Navigate to notebook + page = context.new_page() + print(" 🌐 Opening notebook...") + page.goto(notebook_url, wait_until="domcontentloaded") + + # Wait for NotebookLM + page.wait_for_url(re.compile(r"^https://notebooklm\.google\.com/"), timeout=10000) + + # Wait for query input (MCP approach) + print(" ⏳ Waiting for query input...") + query_element = None + + for selector in QUERY_INPUT_SELECTORS: + try: + query_element = page.wait_for_selector( + selector, + timeout=10000, + state="visible" # Only check visibility, not disabled! + ) + if query_element: + print(f" ✓ Found input: {selector}") + break + except: + continue + + if not query_element: + print(" ❌ Could not find query input") + return None + + # Type question (human-like, fast) + print(" ⏳ Typing question...") + + # Use primary selector for typing + input_selector = QUERY_INPUT_SELECTORS[0] + StealthUtils.human_type(page, input_selector, question) + + # Submit + print(" 📤 Submitting...") + page.keyboard.press("Enter") + + # Small pause + StealthUtils.random_delay(500, 1500) + + # Wait for response (MCP approach: poll for stable text) + print(" ⏳ Waiting for answer...") + + answer = None + stable_count = 0 + last_text = None + deadline = time.time() + 120 # 2 minutes timeout + + while time.time() < deadline: + # Check if NotebookLM is still thinking (most reliable indicator) + try: + thinking_element = page.query_selector('div.thinking-message') + if thinking_element and thinking_element.is_visible(): + time.sleep(1) + continue + except: + pass + + # Try to find response with MCP selectors + for selector in RESPONSE_SELECTORS: + try: + elements = page.query_selector_all(selector) + if elements: + # Get last (newest) response + latest = elements[-1] + text = latest.inner_text().strip() + + if text: + if text == last_text: + stable_count += 1 + if stable_count >= 3: # Stable for 3 polls + answer = text + break + else: + stable_count = 0 + last_text = text + except: + continue + + if answer: + break + + time.sleep(1) + + if not answer: + print(" ❌ Timeout waiting for answer") + return None + + print(" ✅ Got answer!") + # Add follow-up reminder to encourage Claude to ask more questions + return answer + FOLLOW_UP_REMINDER + + except Exception as e: + print(f" ❌ Error: {e}") + import traceback + traceback.print_exc() + return None + + finally: + # Always clean up + if context: + try: + context.close() + except: + pass + + if playwright: + try: + playwright.stop() + except: + pass + + +def main(): + parser = argparse.ArgumentParser(description='Ask NotebookLM a question') + + parser.add_argument('--question', required=True, help='Question to ask') + parser.add_argument('--notebook-url', help='NotebookLM notebook URL') + parser.add_argument('--notebook-id', help='Notebook ID from library') + parser.add_argument('--show-browser', action='store_true', help='Show browser') + + args = parser.parse_args() + + # Resolve notebook URL + notebook_url = args.notebook_url + + if not notebook_url and args.notebook_id: + library = NotebookLibrary() + notebook = library.get_notebook(args.notebook_id) + if notebook: + notebook_url = notebook['url'] + else: + print(f"❌ Notebook '{args.notebook_id}' not found") + return 1 + + if not notebook_url: + # Check for active notebook first + library = NotebookLibrary() + active = library.get_active_notebook() + if active: + notebook_url = active['url'] + print(f"📚 Using active notebook: {active['name']}") + else: + # Show available notebooks + notebooks = library.list_notebooks() + if notebooks: + print("\n📚 Available notebooks:") + for nb in notebooks: + mark = " [ACTIVE]" if nb.get('id') == library.active_notebook_id else "" + print(f" {nb['id']}: {nb['name']}{mark}") + print("\nSpecify with --notebook-id or set active:") + print("python scripts/run.py notebook_manager.py activate --id ID") + else: + print("❌ No notebooks in library. Add one first:") + print("python scripts/run.py notebook_manager.py add --url URL --name NAME --description DESC --topics TOPICS") + return 1 + + # Ask the question + answer = ask_notebooklm( + question=args.question, + notebook_url=notebook_url, + headless=not args.show_browser + ) + + if answer: + print("\n" + "=" * 60) + print(f"Question: {args.question}") + print("=" * 60) + print() + print(answer) + print() + print("=" * 60) + return 0 + else: + print("\n❌ Failed to get answer") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/personas/_shared/community-skills/notebooklm/scripts/auth_manager.py b/personas/_shared/community-skills/notebooklm/scripts/auth_manager.py new file mode 100755 index 0000000..54c8b3b --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/auth_manager.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +""" +Authentication Manager for NotebookLM +Handles Google login and browser state persistence +Based on the MCP server implementation + +Implements hybrid auth approach: +- Persistent browser profile (user_data_dir) for fingerprint consistency +- Manual cookie injection from state.json for session cookies (Playwright bug workaround) +See: https://github.com/microsoft/playwright/issues/36139 +""" + +import json +import time +import argparse +import shutil +import re +import sys +from pathlib import Path +from typing import Optional, Dict, Any + +from patchright.sync_api import sync_playwright, BrowserContext + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent)) + +from config import BROWSER_STATE_DIR, STATE_FILE, AUTH_INFO_FILE, DATA_DIR +from browser_utils import BrowserFactory + + +class AuthManager: + """ + Manages authentication and browser state for NotebookLM + + Features: + - Interactive Google login + - Browser state persistence + - Session restoration + - Account switching + """ + + def __init__(self): + """Initialize the authentication manager""" + # Ensure directories exist + DATA_DIR.mkdir(parents=True, exist_ok=True) + BROWSER_STATE_DIR.mkdir(parents=True, exist_ok=True) + + self.state_file = STATE_FILE + self.auth_info_file = AUTH_INFO_FILE + self.browser_state_dir = BROWSER_STATE_DIR + + def is_authenticated(self) -> bool: + """Check if valid authentication exists""" + if not self.state_file.exists(): + return False + + # Check if state file is not too old (7 days) + age_days = (time.time() - self.state_file.stat().st_mtime) / 86400 + if age_days > 7: + print(f"⚠️ Browser state is {age_days:.1f} days old, may need re-authentication") + + return True + + def get_auth_info(self) -> Dict[str, Any]: + """Get authentication information""" + info = { + 'authenticated': self.is_authenticated(), + 'state_file': str(self.state_file), + 'state_exists': self.state_file.exists() + } + + if self.auth_info_file.exists(): + try: + with open(self.auth_info_file, 'r') as f: + saved_info = json.load(f) + info.update(saved_info) + except Exception: + pass + + if info['state_exists']: + age_hours = (time.time() - self.state_file.stat().st_mtime) / 3600 + info['state_age_hours'] = age_hours + + return info + + def setup_auth(self, headless: bool = False, timeout_minutes: int = 10) -> bool: + """ + Perform interactive authentication setup + + Args: + headless: Run browser in headless mode (False for login) + timeout_minutes: Maximum time to wait for login + + Returns: + True if authentication successful + """ + print("🔐 Starting authentication setup...") + print(f" Timeout: {timeout_minutes} minutes") + + playwright = None + context = None + + try: + playwright = sync_playwright().start() + + # Launch using factory + context = BrowserFactory.launch_persistent_context( + playwright, + headless=headless + ) + + # Navigate to NotebookLM + page = context.new_page() + page.goto("https://notebooklm.google.com", wait_until="domcontentloaded") + + # Check if already authenticated + if "notebooklm.google.com" in page.url and "accounts.google.com" not in page.url: + print(" ✅ Already authenticated!") + self._save_browser_state(context) + return True + + # Wait for manual login + print("\n ⏳ Please log in to your Google account...") + print(f" ⏱️ Waiting up to {timeout_minutes} minutes for login...") + + try: + # Wait for URL to change to NotebookLM (regex ensures it's the actual domain, not a parameter) + timeout_ms = int(timeout_minutes * 60 * 1000) + page.wait_for_url(re.compile(r"^https://notebooklm\.google\.com/"), timeout=timeout_ms) + + print(f" ✅ Login successful!") + + # Save authentication state + self._save_browser_state(context) + self._save_auth_info() + return True + + except Exception as e: + print(f" ❌ Authentication timeout: {e}") + return False + + except Exception as e: + print(f" ❌ Error: {e}") + return False + + finally: + # Clean up browser resources + if context: + try: + context.close() + except Exception: + pass + + if playwright: + try: + playwright.stop() + except Exception: + pass + + def _save_browser_state(self, context: BrowserContext): + """Save browser state to disk""" + try: + # Save storage state (cookies, localStorage) + context.storage_state(path=str(self.state_file)) + print(f" 💾 Saved browser state to: {self.state_file}") + except Exception as e: + print(f" ❌ Failed to save browser state: {e}") + raise + + def _save_auth_info(self): + """Save authentication metadata""" + try: + info = { + 'authenticated_at': time.time(), + 'authenticated_at_iso': time.strftime('%Y-%m-%d %H:%M:%S') + } + with open(self.auth_info_file, 'w') as f: + json.dump(info, f, indent=2) + except Exception: + pass # Non-critical + + def clear_auth(self) -> bool: + """ + Clear all authentication data + + Returns: + True if cleared successfully + """ + print("🗑️ Clearing authentication data...") + + try: + # Remove browser state + if self.state_file.exists(): + self.state_file.unlink() + print(" ✅ Removed browser state") + + # Remove auth info + if self.auth_info_file.exists(): + self.auth_info_file.unlink() + print(" ✅ Removed auth info") + + # Clear entire browser state directory + if self.browser_state_dir.exists(): + shutil.rmtree(self.browser_state_dir) + self.browser_state_dir.mkdir(parents=True, exist_ok=True) + print(" ✅ Cleared browser data") + + return True + + except Exception as e: + print(f" ❌ Error clearing auth: {e}") + return False + + def re_auth(self, headless: bool = False, timeout_minutes: int = 10) -> bool: + """ + Perform re-authentication (clear and setup) + + Args: + headless: Run browser in headless mode + timeout_minutes: Login timeout in minutes + + Returns: + True if successful + """ + print("🔄 Starting re-authentication...") + + # Clear existing auth + self.clear_auth() + + # Setup new auth + return self.setup_auth(headless, timeout_minutes) + + def validate_auth(self) -> bool: + """ + Validate that stored authentication works + Uses persistent context to match actual usage pattern + + Returns: + True if authentication is valid + """ + if not self.is_authenticated(): + return False + + print("🔍 Validating authentication...") + + playwright = None + context = None + + try: + playwright = sync_playwright().start() + + # Launch using factory + context = BrowserFactory.launch_persistent_context( + playwright, + headless=True + ) + + # Try to access NotebookLM + page = context.new_page() + page.goto("https://notebooklm.google.com", wait_until="domcontentloaded", timeout=30000) + + # Check if we can access NotebookLM + if "notebooklm.google.com" in page.url and "accounts.google.com" not in page.url: + print(" ✅ Authentication is valid") + return True + else: + print(" ❌ Authentication is invalid (redirected to login)") + return False + + except Exception as e: + print(f" ❌ Validation failed: {e}") + return False + + finally: + if context: + try: + context.close() + except Exception: + pass + if playwright: + try: + playwright.stop() + except Exception: + pass + + +def main(): + """Command-line interface for authentication management""" + parser = argparse.ArgumentParser(description='Manage NotebookLM authentication') + + subparsers = parser.add_subparsers(dest='command', help='Commands') + + # Setup command + setup_parser = subparsers.add_parser('setup', help='Setup authentication') + setup_parser.add_argument('--headless', action='store_true', help='Run in headless mode') + setup_parser.add_argument('--timeout', type=float, default=10, help='Login timeout in minutes (default: 10)') + + # Status command + subparsers.add_parser('status', help='Check authentication status') + + # Validate command + subparsers.add_parser('validate', help='Validate authentication') + + # Clear command + subparsers.add_parser('clear', help='Clear authentication') + + # Re-auth command + reauth_parser = subparsers.add_parser('reauth', help='Re-authenticate (clear + setup)') + reauth_parser.add_argument('--timeout', type=float, default=10, help='Login timeout in minutes (default: 10)') + + args = parser.parse_args() + + # Initialize manager + auth = AuthManager() + + # Execute command + if args.command == 'setup': + if auth.setup_auth(headless=args.headless, timeout_minutes=args.timeout): + print("\n✅ Authentication setup complete!") + print("You can now use ask_question.py to query NotebookLM") + else: + print("\n❌ Authentication setup failed") + exit(1) + + elif args.command == 'status': + info = auth.get_auth_info() + print("\n🔐 Authentication Status:") + print(f" Authenticated: {'Yes' if info['authenticated'] else 'No'}") + if info.get('state_age_hours'): + print(f" State age: {info['state_age_hours']:.1f} hours") + if info.get('authenticated_at_iso'): + print(f" Last auth: {info['authenticated_at_iso']}") + print(f" State file: {info['state_file']}") + + elif args.command == 'validate': + if auth.validate_auth(): + print("Authentication is valid and working") + else: + print("Authentication is invalid or expired") + print("Run: auth_manager.py setup") + + elif args.command == 'clear': + if auth.clear_auth(): + print("Authentication cleared") + + elif args.command == 'reauth': + if auth.re_auth(timeout_minutes=args.timeout): + print("\n✅ Re-authentication complete!") + else: + print("\n❌ Re-authentication failed") + exit(1) + + else: + parser.print_help() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/scripts/browser_session.py b/personas/_shared/community-skills/notebooklm/scripts/browser_session.py new file mode 100755 index 0000000..b121af8 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/browser_session.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +""" +Browser Session Management for NotebookLM +Individual browser session for persistent NotebookLM conversations +Based on the original NotebookLM API implementation +""" + +import time +import sys +from typing import Any, Dict, Optional +from pathlib import Path + +from patchright.sync_api import BrowserContext, Page + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent)) + +from browser_utils import StealthUtils + + +class BrowserSession: + """ + Represents a single persistent browser session for NotebookLM + + Each session gets its own Page (tab) within a shared BrowserContext, + allowing for contextual conversations where NotebookLM remembers + previous messages. + """ + + def __init__(self, session_id: str, context: BrowserContext, notebook_url: str): + """ + Initialize a new browser session + + Args: + session_id: Unique identifier for this session + context: Browser context (shared or dedicated) + notebook_url: Target NotebookLM URL for this session + """ + self.id = session_id + self.created_at = time.time() + self.last_activity = time.time() + self.message_count = 0 + self.notebook_url = notebook_url + self.context = context + self.page = None + self.stealth = StealthUtils() + + # Initialize the session + self._initialize() + + def _initialize(self): + """Initialize the browser session and navigate to NotebookLM""" + print(f"🚀 Creating session {self.id}...") + + # Create new page (tab) in context + self.page = self.context.new_page() + print(f" 🌐 Navigating to NotebookLM...") + + try: + # Navigate to notebook + self.page.goto(self.notebook_url, wait_until="domcontentloaded", timeout=30000) + + # Check if login is needed + if "accounts.google.com" in self.page.url: + raise RuntimeError("Authentication required. Please run auth_manager.py setup first.") + + # Wait for page to be ready + self._wait_for_ready() + + # Simulate human inspection + self.stealth.random_mouse_movement(self.page) + self.stealth.random_delay(300, 600) + + print(f"✅ Session {self.id} ready!") + + except Exception as e: + print(f"❌ Failed to initialize session: {e}") + if self.page: + self.page.close() + raise + + def _wait_for_ready(self): + """Wait for NotebookLM page to be ready""" + try: + # Wait for chat input + self.page.wait_for_selector("textarea.query-box-input", timeout=10000, state="visible") + except Exception: + # Try alternative selector + self.page.wait_for_selector('textarea[aria-label="Feld für Anfragen"]', timeout=5000, state="visible") + + def ask(self, question: str) -> Dict[str, Any]: + """ + Ask a question in this session + + Args: + question: The question to ask + + Returns: + Dict with status, question, answer, session_id + """ + try: + self.last_activity = time.time() + self.message_count += 1 + + print(f"💬 [{self.id}] Asking: {question}") + + # Snapshot current answer to detect new response + previous_answer = self._snapshot_latest_response() + + # Find chat input + chat_input_selector = "textarea.query-box-input" + try: + self.page.wait_for_selector(chat_input_selector, timeout=5000, state="visible") + except Exception: + chat_input_selector = 'textarea[aria-label="Feld für Anfragen"]' + self.page.wait_for_selector(chat_input_selector, timeout=5000, state="visible") + + # Click and type with human-like behavior + self.stealth.realistic_click(self.page, chat_input_selector) + self.stealth.human_type(self.page, chat_input_selector, question) + + # Small pause before submit + self.stealth.random_delay(300, 800) + + # Submit + self.page.keyboard.press("Enter") + + # Wait for response + print(" ⏳ Waiting for response...") + self.stealth.random_delay(1500, 3000) + + # Get new answer + answer = self._wait_for_latest_answer(previous_answer) + + if not answer: + raise Exception("Empty response from NotebookLM") + + print(f" ✅ Got response ({len(answer)} chars)") + + return { + "status": "success", + "question": question, + "answer": answer, + "session_id": self.id, + "notebook_url": self.notebook_url + } + + except Exception as e: + print(f" ❌ Error: {e}") + return { + "status": "error", + "question": question, + "error": str(e), + "session_id": self.id + } + + def _snapshot_latest_response(self) -> Optional[str]: + """Get the current latest response text""" + try: + # Use correct NotebookLM selector + responses = self.page.query_selector_all(".to-user-container .message-text-content") + if responses: + return responses[-1].inner_text() + except Exception: + pass + return None + + def _wait_for_latest_answer(self, previous_answer: Optional[str], timeout: int = 120) -> str: + """Wait for and extract the new answer""" + start_time = time.time() + last_candidate = None + stable_count = 0 + + while time.time() - start_time < timeout: + # Check if NotebookLM is still thinking (most reliable indicator) + try: + thinking_element = self.page.query_selector('div.thinking-message') + if thinking_element and thinking_element.is_visible(): + time.sleep(0.5) + continue + except Exception: + pass + + try: + # Use correct NotebookLM selector + responses = self.page.query_selector_all(".to-user-container .message-text-content") + + if responses: + latest_text = responses[-1].inner_text().strip() + + # Check if it's a new response + if latest_text and latest_text != previous_answer: + # Check if text is stable (3 consecutive polls) + if latest_text == last_candidate: + stable_count += 1 + if stable_count >= 3: + return latest_text + else: + stable_count = 1 + last_candidate = latest_text + + except Exception: + pass + + time.sleep(0.5) + + raise TimeoutError(f"No response received within {timeout} seconds") + + def reset(self): + """Reset the chat by reloading the page""" + print(f"🔄 Resetting session {self.id}...") + + self.page.reload(wait_until="domcontentloaded") + self._wait_for_ready() + + previous_count = self.message_count + self.message_count = 0 + self.last_activity = time.time() + + print(f"✅ Session reset (cleared {previous_count} messages)") + return previous_count + + def close(self): + """Close this session and clean up resources""" + print(f"🛑 Closing session {self.id}...") + + if self.page: + try: + self.page.close() + except Exception as e: + print(f" ⚠️ Error closing page: {e}") + + print(f"✅ Session {self.id} closed") + + def get_info(self) -> Dict[str, Any]: + """Get information about this session""" + return { + "id": self.id, + "created_at": self.created_at, + "last_activity": self.last_activity, + "age_seconds": time.time() - self.created_at, + "inactive_seconds": time.time() - self.last_activity, + "message_count": self.message_count, + "notebook_url": self.notebook_url + } + + def is_expired(self, timeout_seconds: int = 900) -> bool: + """Check if session has expired (default: 15 minutes)""" + return (time.time() - self.last_activity) > timeout_seconds + + +if __name__ == "__main__": + # Example usage + print("Browser Session Module - Use ask_question.py for main interface") + print("This module provides low-level browser session management.") \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/scripts/browser_utils.py b/personas/_shared/community-skills/notebooklm/scripts/browser_utils.py new file mode 100755 index 0000000..60a1210 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/browser_utils.py @@ -0,0 +1,107 @@ +""" +Browser Utilities for NotebookLM Skill +Handles browser launching, stealth features, and common interactions +""" + +import json +import time +import random +from typing import Optional, List + +from patchright.sync_api import Playwright, BrowserContext, Page +from config import BROWSER_PROFILE_DIR, STATE_FILE, BROWSER_ARGS, USER_AGENT + + +class BrowserFactory: + """Factory for creating configured browser contexts""" + + @staticmethod + def launch_persistent_context( + playwright: Playwright, + headless: bool = True, + user_data_dir: str = str(BROWSER_PROFILE_DIR) + ) -> BrowserContext: + """ + Launch a persistent browser context with anti-detection features + and cookie workaround. + """ + # Launch persistent context + context = playwright.chromium.launch_persistent_context( + user_data_dir=user_data_dir, + channel="chrome", # Use real Chrome + headless=headless, + no_viewport=True, + ignore_default_args=["--enable-automation"], + user_agent=USER_AGENT, + args=BROWSER_ARGS + ) + + # Cookie Workaround for Playwright bug #36139 + # Session cookies (expires=-1) don't persist in user_data_dir automatically + BrowserFactory._inject_cookies(context) + + return context + + @staticmethod + def _inject_cookies(context: BrowserContext): + """Inject cookies from state.json if available""" + if STATE_FILE.exists(): + try: + with open(STATE_FILE, 'r') as f: + state = json.load(f) + if 'cookies' in state and len(state['cookies']) > 0: + context.add_cookies(state['cookies']) + # print(f" 🔧 Injected {len(state['cookies'])} cookies from state.json") + except Exception as e: + print(f" ⚠️ Could not load state.json: {e}") + + +class StealthUtils: + """Human-like interaction utilities""" + + @staticmethod + def random_delay(min_ms: int = 100, max_ms: int = 500): + """Add random delay""" + time.sleep(random.uniform(min_ms / 1000, max_ms / 1000)) + + @staticmethod + def human_type(page: Page, selector: str, text: str, wpm_min: int = 320, wpm_max: int = 480): + """Type with human-like speed""" + element = page.query_selector(selector) + if not element: + # Try waiting if not immediately found + try: + element = page.wait_for_selector(selector, timeout=2000) + except: + pass + + if not element: + print(f"⚠️ Element not found for typing: {selector}") + return + + # Click to focus + element.click() + + # Type + for char in text: + element.type(char, delay=random.uniform(25, 75)) + if random.random() < 0.05: + time.sleep(random.uniform(0.15, 0.4)) + + @staticmethod + def realistic_click(page: Page, selector: str): + """Click with realistic movement""" + element = page.query_selector(selector) + if not element: + return + + # Optional: Move mouse to element (simplified) + box = element.bounding_box() + if box: + x = box['x'] + box['width'] / 2 + y = box['y'] + box['height'] / 2 + page.mouse.move(x, y, steps=5) + + StealthUtils.random_delay(100, 300) + element.click() + StealthUtils.random_delay(100, 300) diff --git a/personas/_shared/community-skills/notebooklm/scripts/cleanup_manager.py b/personas/_shared/community-skills/notebooklm/scripts/cleanup_manager.py new file mode 100755 index 0000000..c4a8fc2 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/cleanup_manager.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +""" +Cleanup Manager for NotebookLM Skill +Manages cleanup of skill data and browser state +""" + +import shutil +import argparse +from pathlib import Path +from typing import Dict, List, Any + + +class CleanupManager: + """ + Manages cleanup of NotebookLM skill data + + Features: + - Preview what will be deleted + - Selective cleanup options + - Library preservation + - Safe deletion with confirmation + """ + + def __init__(self): + """Initialize the cleanup manager""" + # Skill directory paths + self.skill_dir = Path(__file__).parent.parent + self.data_dir = self.skill_dir / "data" + + def get_cleanup_paths(self, preserve_library: bool = False) -> Dict[str, Any]: + """ + Get paths that would be cleaned up + + Args: + preserve_library: Keep library.json if True + + Returns: + Dict with paths and sizes + + Note: .venv is NEVER deleted - it's part of the skill infrastructure + """ + paths = { + 'browser_state': [], + 'sessions': [], + 'library': [], + 'auth': [], + 'other': [] + } + + total_size = 0 + + if self.data_dir.exists(): + # Browser state + browser_state_dir = self.data_dir / "browser_state" + if browser_state_dir.exists(): + for item in browser_state_dir.iterdir(): + size = self._get_size(item) + paths['browser_state'].append({ + 'path': str(item), + 'size': size, + 'type': 'dir' if item.is_dir() else 'file' + }) + total_size += size + + # Sessions + sessions_file = self.data_dir / "sessions.json" + if sessions_file.exists(): + size = sessions_file.stat().st_size + paths['sessions'].append({ + 'path': str(sessions_file), + 'size': size, + 'type': 'file' + }) + total_size += size + + # Library (unless preserved) + if not preserve_library: + library_file = self.data_dir / "library.json" + if library_file.exists(): + size = library_file.stat().st_size + paths['library'].append({ + 'path': str(library_file), + 'size': size, + 'type': 'file' + }) + total_size += size + + # Auth info + auth_info = self.data_dir / "auth_info.json" + if auth_info.exists(): + size = auth_info.stat().st_size + paths['auth'].append({ + 'path': str(auth_info), + 'size': size, + 'type': 'file' + }) + total_size += size + + # Other files in data dir (but NEVER .venv!) + for item in self.data_dir.iterdir(): + if item.name not in ['browser_state', 'sessions.json', 'library.json', 'auth_info.json']: + size = self._get_size(item) + paths['other'].append({ + 'path': str(item), + 'size': size, + 'type': 'dir' if item.is_dir() else 'file' + }) + total_size += size + + return { + 'categories': paths, + 'total_size': total_size, + 'total_items': sum(len(items) for items in paths.values()) + } + + def _get_size(self, path: Path) -> int: + """Get size of file or directory in bytes""" + if path.is_file(): + return path.stat().st_size + elif path.is_dir(): + total = 0 + try: + for item in path.rglob('*'): + if item.is_file(): + total += item.stat().st_size + except Exception: + pass + return total + return 0 + + def _format_size(self, size: int) -> str: + """Format size in human-readable form""" + for unit in ['B', 'KB', 'MB', 'GB']: + if size < 1024: + return f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} TB" + + def perform_cleanup( + self, + preserve_library: bool = False, + dry_run: bool = False + ) -> Dict[str, Any]: + """ + Perform the actual cleanup + + Args: + preserve_library: Keep library.json if True + dry_run: Preview only, don't delete + + Returns: + Dict with cleanup results + """ + cleanup_data = self.get_cleanup_paths(preserve_library) + deleted_items = [] + failed_items = [] + deleted_size = 0 + + if dry_run: + return { + 'dry_run': True, + 'would_delete': cleanup_data['total_items'], + 'would_free': cleanup_data['total_size'] + } + + # Perform deletion + for category, items in cleanup_data['categories'].items(): + for item_info in items: + path = Path(item_info['path']) + try: + if path.exists(): + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + deleted_items.append(str(path)) + deleted_size += item_info['size'] + print(f" ✅ Deleted: {path.name}") + except Exception as e: + failed_items.append({ + 'path': str(path), + 'error': str(e) + }) + print(f" ❌ Failed: {path.name} ({e})") + + # Recreate browser_state dir if everything was deleted + if not preserve_library and not failed_items: + browser_state_dir = self.data_dir / "browser_state" + browser_state_dir.mkdir(parents=True, exist_ok=True) + + return { + 'deleted_items': deleted_items, + 'failed_items': failed_items, + 'deleted_size': deleted_size, + 'deleted_count': len(deleted_items), + 'failed_count': len(failed_items) + } + + def print_cleanup_preview(self, preserve_library: bool = False): + """Print a preview of what will be cleaned""" + data = self.get_cleanup_paths(preserve_library) + + print("\n🔍 Cleanup Preview") + print("=" * 60) + + for category, items in data['categories'].items(): + if items: + print(f"\n📁 {category.replace('_', ' ').title()}:") + for item in items: + path = Path(item['path']) + size_str = self._format_size(item['size']) + type_icon = "📂" if item['type'] == 'dir' else "📄" + print(f" {type_icon} {path.name:<30} {size_str:>10}") + + print("\n" + "=" * 60) + print(f"Total items: {data['total_items']}") + print(f"Total size: {self._format_size(data['total_size'])}") + + if preserve_library: + print("\n📚 Library will be preserved") + + print("\nThis preview shows what would be deleted.") + print("Use --confirm to actually perform the cleanup.") + + +def main(): + """Command-line interface for cleanup management""" + parser = argparse.ArgumentParser( + description='Clean up NotebookLM skill data', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Preview what will be deleted + python cleanup_manager.py + + # Perform cleanup (delete everything) + python cleanup_manager.py --confirm + + # Cleanup but keep library + python cleanup_manager.py --confirm --preserve-library + + # Force cleanup without preview + python cleanup_manager.py --confirm --force + """ + ) + + parser.add_argument( + '--confirm', + action='store_true', + help='Actually perform the cleanup (without this, only preview)' + ) + + parser.add_argument( + '--preserve-library', + action='store_true', + help='Keep the notebook library (library.json)' + ) + + parser.add_argument( + '--force', + action='store_true', + help='Skip confirmation prompt' + ) + + args = parser.parse_args() + + # Initialize manager + manager = CleanupManager() + + if args.confirm: + # Show preview first unless forced + if not args.force: + manager.print_cleanup_preview(args.preserve_library) + + print("\n⚠️ WARNING: This will delete the files shown above!") + print(" Note: .venv is preserved (part of skill infrastructure)") + response = input("Are you sure? (yes/no): ") + + if response.lower() != 'yes': + print("Cleanup cancelled.") + return + + # Perform cleanup + print("\n🗑️ Performing cleanup...") + result = manager.perform_cleanup(args.preserve_library, dry_run=False) + + print(f"\n✅ Cleanup complete!") + print(f" Deleted: {result['deleted_count']} items") + print(f" Freed: {manager._format_size(result['deleted_size'])}") + + if result['failed_count'] > 0: + print(f" ⚠️ Failed: {result['failed_count']} items") + + else: + # Just show preview + manager.print_cleanup_preview(args.preserve_library) + print("\n💡 Note: Virtual environment (.venv) is never deleted") + print(" It's part of the skill infrastructure, not user data") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/scripts/config.py b/personas/_shared/community-skills/notebooklm/scripts/config.py new file mode 100755 index 0000000..4486b55 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/config.py @@ -0,0 +1,44 @@ +""" +Configuration for NotebookLM Skill +Centralizes constants, selectors, and paths +""" + +from pathlib import Path + +# Paths +SKILL_DIR = Path(__file__).parent.parent +DATA_DIR = SKILL_DIR / "data" +BROWSER_STATE_DIR = DATA_DIR / "browser_state" +BROWSER_PROFILE_DIR = BROWSER_STATE_DIR / "browser_profile" +STATE_FILE = BROWSER_STATE_DIR / "state.json" +AUTH_INFO_FILE = DATA_DIR / "auth_info.json" +LIBRARY_FILE = DATA_DIR / "library.json" + +# NotebookLM Selectors +QUERY_INPUT_SELECTORS = [ + "textarea.query-box-input", # Primary + 'textarea[aria-label="Feld für Anfragen"]', # Fallback German + 'textarea[aria-label="Input for queries"]', # Fallback English +] + +RESPONSE_SELECTORS = [ + ".to-user-container .message-text-content", # Primary + "[data-message-author='bot']", + "[data-message-author='assistant']", +] + +# Browser Configuration +BROWSER_ARGS = [ + '--disable-blink-features=AutomationControlled', # Patches navigator.webdriver + '--disable-dev-shm-usage', + '--no-sandbox', + '--no-first-run', + '--no-default-browser-check' +] + +USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' + +# Timeouts +LOGIN_TIMEOUT_MINUTES = 10 +QUERY_TIMEOUT_SECONDS = 120 +PAGE_LOAD_TIMEOUT = 30000 diff --git a/personas/_shared/community-skills/notebooklm/scripts/debug_homepage.py b/personas/_shared/community-skills/notebooklm/scripts/debug_homepage.py new file mode 100644 index 0000000..3e858ab --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/debug_homepage.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Dump NotebookLM homepage HTML + screenshot to find correct selectors.""" +import sys, time +from pathlib import Path +from patchright.sync_api import sync_playwright +sys.path.insert(0, str(Path(__file__).parent)) +from browser_utils import BrowserFactory + +with sync_playwright() as p: + ctx = BrowserFactory.launch_persistent_context(p, headless=True) + page = ctx.new_page() + page.goto("https://notebooklm.google.com", wait_until="networkidle", timeout=60000) + time.sleep(5) + # Try to force lazy load + for _ in range(8): + page.mouse.wheel(0, 2000); time.sleep(0.3) + page.mouse.wheel(0, -20000); time.sleep(1) + + html = page.content() + with open("/tmp/nb_home.html", "w") as f: f.write(html) + page.screenshot(path="/tmp/nb_home.png", full_page=True) + + # Extract probable notebook info + candidates = page.evaluate("""() => { + // Try many approaches + const out = { anchors: [], cards: [], data_ids: [], tiles: [] }; + document.querySelectorAll("a").forEach(a => { + if (a.href.includes("/notebook/") || a.href.includes("/project/")) { + out.anchors.push({href: a.href, text: (a.innerText||"").trim().slice(0,100), aria: a.getAttribute("aria-label")||""}); + } + }); + document.querySelectorAll("[data-id],[data-notebook-id],[data-project-id]").forEach(el => { + out.data_ids.push({id: el.dataset.id||el.dataset.notebookId||el.dataset.projectId, tag: el.tagName, text: (el.innerText||"").trim().slice(0,80)}); + }); + document.querySelectorAll("project-card, notebook-card, mat-card, [class*='card'], [class*='tile'], [class*='project']").forEach(c => { + const title = c.querySelector("[class*='title'], h3, h4, [class*='name'], [class*='heading']"); + out.cards.push({tag: c.tagName, class: c.className.slice(0,100), title: title ? title.innerText.trim().slice(0,80) : (c.innerText||"").trim().slice(0,80)}); + }); + return out; + }""") + import json + print(json.dumps(candidates, indent=2, ensure_ascii=False)) + ctx.close() diff --git a/personas/_shared/community-skills/notebooklm/scripts/list_notebooks.py b/personas/_shared/community-skills/notebooklm/scripts/list_notebooks.py new file mode 100644 index 0000000..e775165 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/list_notebooks.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +""" +List all NotebookLM notebooks by scraping the homepage. +Uses the authenticated browser state from auth_manager. + +Output: JSON array of {url, title} to stdout and /tmp/notebooklm_inventory.json +""" +import json +import re +import sys +import time +from pathlib import Path + +from patchright.sync_api import sync_playwright + +sys.path.insert(0, str(Path(__file__).parent)) +from auth_manager import AuthManager +from browser_utils import BrowserFactory + + +def main(): + auth = AuthManager() + if not auth.is_authenticated(): + print("Not authenticated. Run: auth_manager.py setup") + sys.exit(1) + + playwright = None + context = None + try: + playwright = sync_playwright().start() + context = BrowserFactory.launch_persistent_context(playwright, headless=True) + page = context.new_page() + page.goto("https://notebooklm.google.com", wait_until="domcontentloaded", timeout=60000) + + # Wait for notebook grid to load + print("Waiting for notebooks to load...", file=sys.stderr) + try: + page.wait_for_selector("a[href*='/notebook/']", timeout=20000) + except Exception: + print("Timeout waiting for notebook grid — dumping page anyway", file=sys.stderr) + + # Let lazy-loaded items render + time.sleep(3) + # Scroll to force lazy rendering + for _ in range(5): + page.mouse.wheel(0, 1500) + time.sleep(0.5) + + notebooks = page.evaluate(""" + () => { + const results = []; + const seen = new Set(); + document.querySelectorAll("a[href*='/notebook/']").forEach(a => { + const href = a.href; + const m = href.match(/\\/notebook\\/([a-f0-9-]+)/); + if (!m) return; + if (seen.has(m[1])) return; + seen.add(m[1]); + // Try multiple selectors for title + let title = ''; + const titleEl = a.querySelector('[class*="title"], [class*="name"], [class*="heading"]'); + if (titleEl) title = titleEl.innerText.trim(); + if (!title) title = a.innerText.trim().split('\\n')[0]; + if (!title) title = a.getAttribute('aria-label') || ''; + results.push({ id: m[1], url: 'https://notebooklm.google.com/notebook/' + m[1], title: title }); + }); + return results; + } + """) + + print(json.dumps(notebooks, ensure_ascii=False, indent=2)) + with open("/tmp/notebooklm_inventory.json", "w") as f: + json.dump(notebooks, f, ensure_ascii=False, indent=2) + print(f"\n{len(notebooks)} notebooks saved to /tmp/notebooklm_inventory.json", file=sys.stderr) + + finally: + if context: + try: context.close() + except: pass + if playwright: + try: playwright.stop() + except: pass + + +if __name__ == "__main__": + main() diff --git a/personas/_shared/community-skills/notebooklm/scripts/notebook_manager.py b/personas/_shared/community-skills/notebooklm/scripts/notebook_manager.py new file mode 100755 index 0000000..e10e156 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/notebook_manager.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +""" +Notebook Library Management for NotebookLM +Manages a library of NotebookLM notebooks with metadata +Based on the MCP server implementation +""" + +import json +import argparse +import uuid +import os +from pathlib import Path +from typing import Dict, List, Optional, Any +from datetime import datetime + + +class NotebookLibrary: + """Manages a collection of NotebookLM notebooks with metadata""" + + def __init__(self): + """Initialize the notebook library""" + # Store data within the skill directory + skill_dir = Path(__file__).parent.parent + self.data_dir = skill_dir / "data" + self.data_dir.mkdir(parents=True, exist_ok=True) + + self.library_file = self.data_dir / "library.json" + self.notebooks: Dict[str, Dict[str, Any]] = {} + self.active_notebook_id: Optional[str] = None + + # Load existing library + self._load_library() + + def _load_library(self): + """Load library from disk""" + if self.library_file.exists(): + try: + with open(self.library_file, 'r') as f: + data = json.load(f) + self.notebooks = data.get('notebooks', {}) + self.active_notebook_id = data.get('active_notebook_id') + print(f"📚 Loaded library with {len(self.notebooks)} notebooks") + except Exception as e: + print(f"⚠️ Error loading library: {e}") + self.notebooks = {} + self.active_notebook_id = None + else: + self._save_library() + + def _save_library(self): + """Save library to disk""" + try: + data = { + 'notebooks': self.notebooks, + 'active_notebook_id': self.active_notebook_id, + 'updated_at': datetime.now().isoformat() + } + with open(self.library_file, 'w') as f: + json.dump(data, f, indent=2) + except Exception as e: + print(f"❌ Error saving library: {e}") + + def add_notebook( + self, + url: str, + name: str, + description: str, + topics: List[str], + content_types: Optional[List[str]] = None, + use_cases: Optional[List[str]] = None, + tags: Optional[List[str]] = None + ) -> Dict[str, Any]: + """ + Add a new notebook to the library + + Args: + url: NotebookLM notebook URL + name: Display name for the notebook + description: What's in this notebook + topics: Topics covered + content_types: Types of content (optional) + use_cases: When to use this notebook (optional) + tags: Additional tags for organization (optional) + + Returns: + The created notebook object + """ + # Generate ID from name + notebook_id = name.lower().replace(' ', '-').replace('_', '-') + + # Check for duplicates + if notebook_id in self.notebooks: + raise ValueError(f"Notebook with ID '{notebook_id}' already exists") + + # Create notebook object + notebook = { + 'id': notebook_id, + 'url': url, + 'name': name, + 'description': description, + 'topics': topics, + 'content_types': content_types or [], + 'use_cases': use_cases or [], + 'tags': tags or [], + 'created_at': datetime.now().isoformat(), + 'updated_at': datetime.now().isoformat(), + 'use_count': 0, + 'last_used': None + } + + # Add to library + self.notebooks[notebook_id] = notebook + + # Set as active if it's the first notebook + if len(self.notebooks) == 1: + self.active_notebook_id = notebook_id + + self._save_library() + + print(f"✅ Added notebook: {name} ({notebook_id})") + return notebook + + def remove_notebook(self, notebook_id: str) -> bool: + """ + Remove a notebook from the library + + Args: + notebook_id: ID of notebook to remove + + Returns: + True if removed, False if not found + """ + if notebook_id in self.notebooks: + del self.notebooks[notebook_id] + + # Clear active if it was removed + if self.active_notebook_id == notebook_id: + self.active_notebook_id = None + # Set new active if there are other notebooks + if self.notebooks: + self.active_notebook_id = list(self.notebooks.keys())[0] + + self._save_library() + print(f"✅ Removed notebook: {notebook_id}") + return True + + print(f"⚠️ Notebook not found: {notebook_id}") + return False + + def update_notebook( + self, + notebook_id: str, + name: Optional[str] = None, + description: Optional[str] = None, + topics: Optional[List[str]] = None, + content_types: Optional[List[str]] = None, + use_cases: Optional[List[str]] = None, + tags: Optional[List[str]] = None, + url: Optional[str] = None + ) -> Dict[str, Any]: + """ + Update notebook metadata + + Args: + notebook_id: ID of notebook to update + Other args: Fields to update (None = keep existing) + + Returns: + Updated notebook object + """ + if notebook_id not in self.notebooks: + raise ValueError(f"Notebook not found: {notebook_id}") + + notebook = self.notebooks[notebook_id] + + # Update fields if provided + if name is not None: + notebook['name'] = name + if description is not None: + notebook['description'] = description + if topics is not None: + notebook['topics'] = topics + if content_types is not None: + notebook['content_types'] = content_types + if use_cases is not None: + notebook['use_cases'] = use_cases + if tags is not None: + notebook['tags'] = tags + if url is not None: + notebook['url'] = url + + notebook['updated_at'] = datetime.now().isoformat() + + self._save_library() + print(f"✅ Updated notebook: {notebook['name']}") + return notebook + + def get_notebook(self, notebook_id: str) -> Optional[Dict[str, Any]]: + """Get a specific notebook by ID""" + return self.notebooks.get(notebook_id) + + def list_notebooks(self) -> List[Dict[str, Any]]: + """List all notebooks in the library""" + return list(self.notebooks.values()) + + def search_notebooks(self, query: str) -> List[Dict[str, Any]]: + """ + Search notebooks by query + + Args: + query: Search query (searches name, description, topics, tags) + + Returns: + List of matching notebooks + """ + query_lower = query.lower() + results = [] + + for notebook in self.notebooks.values(): + # Search in various fields + searchable = [ + notebook['name'].lower(), + notebook['description'].lower(), + ' '.join(notebook['topics']).lower(), + ' '.join(notebook['tags']).lower(), + ' '.join(notebook.get('use_cases', [])).lower() + ] + + if any(query_lower in field for field in searchable): + results.append(notebook) + + return results + + def select_notebook(self, notebook_id: str) -> Dict[str, Any]: + """ + Set a notebook as active + + Args: + notebook_id: ID of notebook to activate + + Returns: + The activated notebook + """ + if notebook_id not in self.notebooks: + raise ValueError(f"Notebook not found: {notebook_id}") + + self.active_notebook_id = notebook_id + self._save_library() + + notebook = self.notebooks[notebook_id] + print(f"✅ Activated notebook: {notebook['name']}") + return notebook + + def get_active_notebook(self) -> Optional[Dict[str, Any]]: + """Get the currently active notebook""" + if self.active_notebook_id: + return self.notebooks.get(self.active_notebook_id) + return None + + def increment_use_count(self, notebook_id: str) -> Dict[str, Any]: + """ + Increment usage counter for a notebook + + Args: + notebook_id: ID of notebook that was used + + Returns: + Updated notebook + """ + if notebook_id not in self.notebooks: + raise ValueError(f"Notebook not found: {notebook_id}") + + notebook = self.notebooks[notebook_id] + notebook['use_count'] += 1 + notebook['last_used'] = datetime.now().isoformat() + + self._save_library() + return notebook + + def get_stats(self) -> Dict[str, Any]: + """Get library statistics""" + total_notebooks = len(self.notebooks) + total_topics = set() + total_use_count = 0 + + for notebook in self.notebooks.values(): + total_topics.update(notebook['topics']) + total_use_count += notebook['use_count'] + + # Find most used + most_used = None + if self.notebooks: + most_used = max( + self.notebooks.values(), + key=lambda n: n['use_count'] + ) + + return { + 'total_notebooks': total_notebooks, + 'total_topics': len(total_topics), + 'total_use_count': total_use_count, + 'active_notebook': self.get_active_notebook(), + 'most_used_notebook': most_used, + 'library_path': str(self.library_file) + } + + +def main(): + """Command-line interface for notebook management""" + parser = argparse.ArgumentParser(description='Manage NotebookLM library') + + subparsers = parser.add_subparsers(dest='command', help='Commands') + + # Add command + add_parser = subparsers.add_parser('add', help='Add a notebook') + add_parser.add_argument('--url', required=True, help='NotebookLM URL') + add_parser.add_argument('--name', required=True, help='Display name') + add_parser.add_argument('--description', required=True, help='Description') + add_parser.add_argument('--topics', required=True, help='Comma-separated topics') + add_parser.add_argument('--use-cases', help='Comma-separated use cases') + add_parser.add_argument('--tags', help='Comma-separated tags') + + # List command + subparsers.add_parser('list', help='List all notebooks') + + # Search command + search_parser = subparsers.add_parser('search', help='Search notebooks') + search_parser.add_argument('--query', required=True, help='Search query') + + # Activate command + activate_parser = subparsers.add_parser('activate', help='Set active notebook') + activate_parser.add_argument('--id', required=True, help='Notebook ID') + + # Remove command + remove_parser = subparsers.add_parser('remove', help='Remove a notebook') + remove_parser.add_argument('--id', required=True, help='Notebook ID') + + # Stats command + subparsers.add_parser('stats', help='Show library statistics') + + args = parser.parse_args() + + # Initialize library + library = NotebookLibrary() + + # Execute command + if args.command == 'add': + topics = [t.strip() for t in args.topics.split(',')] + use_cases = [u.strip() for u in args.use_cases.split(',')] if args.use_cases else None + tags = [t.strip() for t in args.tags.split(',')] if args.tags else None + + notebook = library.add_notebook( + url=args.url, + name=args.name, + description=args.description, + topics=topics, + use_cases=use_cases, + tags=tags + ) + print(json.dumps(notebook, indent=2)) + + elif args.command == 'list': + notebooks = library.list_notebooks() + if notebooks: + print("\n📚 Notebook Library:") + for notebook in notebooks: + active = " [ACTIVE]" if notebook['id'] == library.active_notebook_id else "" + print(f"\n 📓 {notebook['name']}{active}") + print(f" ID: {notebook['id']}") + print(f" Topics: {', '.join(notebook['topics'])}") + print(f" Uses: {notebook['use_count']}") + else: + print("📚 Library is empty. Add notebooks with: notebook_manager.py add") + + elif args.command == 'search': + results = library.search_notebooks(args.query) + if results: + print(f"\n🔍 Found {len(results)} notebooks:") + for notebook in results: + print(f"\n 📓 {notebook['name']} ({notebook['id']})") + print(f" {notebook['description']}") + else: + print(f"🔍 No notebooks found for: {args.query}") + + elif args.command == 'activate': + notebook = library.select_notebook(args.id) + print(f"Now using: {notebook['name']}") + + elif args.command == 'remove': + if library.remove_notebook(args.id): + print("Notebook removed from library") + + elif args.command == 'stats': + stats = library.get_stats() + print("\n📊 Library Statistics:") + print(f" Total notebooks: {stats['total_notebooks']}") + print(f" Total topics: {stats['total_topics']}") + print(f" Total uses: {stats['total_use_count']}") + if stats['active_notebook']: + print(f" Active: {stats['active_notebook']['name']}") + if stats['most_used_notebook']: + print(f" Most used: {stats['most_used_notebook']['name']} ({stats['most_used_notebook']['use_count']} uses)") + print(f" Library path: {stats['library_path']}") + + else: + parser.print_help() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/scripts/run.py b/personas/_shared/community-skills/notebooklm/scripts/run.py new file mode 100755 index 0000000..7c47a92 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/run.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Universal runner for NotebookLM skill scripts +Ensures all scripts run with the correct virtual environment +""" + +import os +import sys +import subprocess +from pathlib import Path + + +def get_venv_python(): + """Get the virtual environment Python executable""" + skill_dir = Path(__file__).parent.parent + venv_dir = skill_dir / ".venv" + + if os.name == 'nt': # Windows + venv_python = venv_dir / "Scripts" / "python.exe" + else: # Unix/Linux/Mac + venv_python = venv_dir / "bin" / "python" + + return venv_python + + +def ensure_venv(): + """Ensure virtual environment exists""" + skill_dir = Path(__file__).parent.parent + venv_dir = skill_dir / ".venv" + setup_script = skill_dir / "scripts" / "setup_environment.py" + + # Check if venv exists + if not venv_dir.exists(): + print("🔧 First-time setup: Creating virtual environment...") + print(" This may take a minute...") + + # Run setup with system Python + result = subprocess.run([sys.executable, str(setup_script)]) + if result.returncode != 0: + print("❌ Failed to set up environment") + sys.exit(1) + + print("✅ Environment ready!") + + return get_venv_python() + + +def main(): + """Main runner""" + if len(sys.argv) < 2: + print("Usage: python run.py [args...]") + print("\nAvailable scripts:") + print(" ask_question.py - Query NotebookLM") + print(" notebook_manager.py - Manage notebook library") + print(" session_manager.py - Manage sessions") + print(" auth_manager.py - Handle authentication") + print(" cleanup_manager.py - Clean up skill data") + sys.exit(1) + + script_name = sys.argv[1] + script_args = sys.argv[2:] + + # Handle both "scripts/script.py" and "script.py" formats + if script_name.startswith('scripts/'): + # Remove the scripts/ prefix if provided + script_name = script_name[8:] # len('scripts/') = 8 + + # Ensure .py extension + if not script_name.endswith('.py'): + script_name += '.py' + + # Get script path + skill_dir = Path(__file__).parent.parent + script_path = skill_dir / "scripts" / script_name + + if not script_path.exists(): + print(f"❌ Script not found: {script_name}") + print(f" Working directory: {Path.cwd()}") + print(f" Skill directory: {skill_dir}") + print(f" Looked for: {script_path}") + sys.exit(1) + + # Ensure venv exists and get Python executable + venv_python = ensure_venv() + + # Build command + cmd = [str(venv_python), str(script_path)] + script_args + + # Run the script + try: + result = subprocess.run(cmd) + sys.exit(result.returncode) + except KeyboardInterrupt: + print("\n⚠️ Interrupted by user") + sys.exit(130) + except Exception as e: + print(f"❌ Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/personas/_shared/community-skills/notebooklm/scripts/setup_environment.py b/personas/_shared/community-skills/notebooklm/scripts/setup_environment.py new file mode 100755 index 0000000..a4167d0 --- /dev/null +++ b/personas/_shared/community-skills/notebooklm/scripts/setup_environment.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +Environment Setup for NotebookLM Skill +Manages virtual environment and dependencies automatically +""" + +import os +import sys +import subprocess +import venv +from pathlib import Path + + +class SkillEnvironment: + """Manages skill-specific virtual environment""" + + def __init__(self): + # Skill directory paths + self.skill_dir = Path(__file__).parent.parent + self.venv_dir = self.skill_dir / ".venv" + self.requirements_file = self.skill_dir / "requirements.txt" + + # Python executable in venv + if os.name == 'nt': # Windows + self.venv_python = self.venv_dir / "Scripts" / "python.exe" + self.venv_pip = self.venv_dir / "Scripts" / "pip.exe" + else: # Unix/Linux/Mac + self.venv_python = self.venv_dir / "bin" / "python" + self.venv_pip = self.venv_dir / "bin" / "pip" + + def ensure_venv(self) -> bool: + """Ensure virtual environment exists and is set up""" + + # Check if we're already in the correct venv + if self.is_in_skill_venv(): + print("✅ Already running in skill virtual environment") + return True + + # Create venv if it doesn't exist + if not self.venv_dir.exists(): + print(f"🔧 Creating virtual environment in {self.venv_dir.name}/") + try: + venv.create(self.venv_dir, with_pip=True) + print("✅ Virtual environment created") + except Exception as e: + print(f"❌ Failed to create venv: {e}") + return False + + # Install/update dependencies + if self.requirements_file.exists(): + print("📦 Installing dependencies...") + try: + # Upgrade pip first + subprocess.run( + [str(self.venv_pip), "install", "--upgrade", "pip"], + check=True, + capture_output=True, + text=True + ) + + # Install requirements + result = subprocess.run( + [str(self.venv_pip), "install", "-r", str(self.requirements_file)], + check=True, + capture_output=True, + text=True + ) + print("✅ Dependencies installed") + + # Install Chrome for Patchright (not Chromium!) + # Using real Chrome ensures cross-platform reliability and consistent browser fingerprinting + # See: https://github.com/Kaliiiiiiiiii-Vinyzu/patchright-python#anti-detection + print("🌐 Installing Google Chrome for Patchright...") + try: + subprocess.run( + [str(self.venv_python), "-m", "patchright", "install", "chrome"], + check=True, + capture_output=True, + text=True + ) + print("✅ Chrome installed") + except subprocess.CalledProcessError as e: + print(f"⚠️ Warning: Failed to install Chrome: {e}") + print(" You may need to run manually: python -m patchright install chrome") + print(" Chrome is required (not Chromium) for reliability!") + + return True + except subprocess.CalledProcessError as e: + print(f"❌ Failed to install dependencies: {e}") + print(f" Output: {e.output if hasattr(e, 'output') else 'No output'}") + return False + else: + print("⚠️ No requirements.txt found, skipping dependency installation") + return True + + def is_in_skill_venv(self) -> bool: + """Check if we're already running in the skill's venv""" + if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix): + # We're in a venv, check if it's ours + venv_path = Path(sys.prefix) + return venv_path == self.venv_dir + return False + + def get_python_executable(self) -> str: + """Get the correct Python executable to use""" + if self.venv_python.exists(): + return str(self.venv_python) + return sys.executable + + def run_script(self, script_name: str, args: list = None) -> int: + """Run a script with the virtual environment""" + script_path = self.skill_dir / "scripts" / script_name + + if not script_path.exists(): + print(f"❌ Script not found: {script_path}") + return 1 + + # Ensure venv is set up + if not self.ensure_venv(): + print("❌ Failed to set up environment") + return 1 + + # Build command + cmd = [str(self.venv_python), str(script_path)] + if args: + cmd.extend(args) + + print(f"🚀 Running: {script_name} with venv Python") + + try: + # Run the script with venv Python + result = subprocess.run(cmd) + return result.returncode + except Exception as e: + print(f"❌ Failed to run script: {e}") + return 1 + + def activate_instructions(self) -> str: + """Get instructions for manual activation""" + if os.name == 'nt': + activate = self.venv_dir / "Scripts" / "activate.bat" + return f"Run: {activate}" + else: + activate = self.venv_dir / "bin" / "activate" + return f"Run: source {activate}" + + +def main(): + """Main entry point for environment setup""" + import argparse + + parser = argparse.ArgumentParser( + description='Setup NotebookLM skill environment' + ) + + parser.add_argument( + '--check', + action='store_true', + help='Check if environment is set up' + ) + + parser.add_argument( + '--run', + help='Run a script with the venv (e.g., --run ask_question.py)' + ) + + parser.add_argument( + 'args', + nargs='*', + help='Arguments to pass to the script' + ) + + args = parser.parse_args() + + env = SkillEnvironment() + + if args.check: + if env.venv_dir.exists(): + print(f"✅ Virtual environment exists: {env.venv_dir}") + print(f" Python: {env.get_python_executable()}") + print(f" To activate manually: {env.activate_instructions()}") + else: + print(f"❌ No virtual environment found") + print(f" Run setup_environment.py to create it") + return + + if args.run: + # Run a script with venv + return env.run_script(args.run, args.args) + + # Default: ensure environment is set up + if env.ensure_venv(): + print("\n✅ Environment ready!") + print(f" Virtual env: {env.venv_dir}") + print(f" Python: {env.get_python_executable()}") + print(f"\nTo activate manually: {env.activate_instructions()}") + print(f"Or run scripts directly: python setup_environment.py --run script_name.py") + else: + print("\n❌ Environment setup failed") + return 1 + + +if __name__ == "__main__": + sys.exit(main() or 0) \ No newline at end of file diff --git a/personas/_shared/community-skills/obsidian-tasks/SKILL.md b/personas/_shared/community-skills/obsidian-tasks/SKILL.md new file mode 100644 index 0000000..c42eb7c --- /dev/null +++ b/personas/_shared/community-skills/obsidian-tasks/SKILL.md @@ -0,0 +1,248 @@ +--- +name: obsidian-tasks +description: Use when managing tasks, daily notes, reading lists, or project tracking in Obsidian vault. Triggers on task creation, task status updates, daily journal updates, PDF reading assignments, Google Tasks sync, language learning progress, or any "add task / update daily / check progress" request. Also triggers when user mentions Google Tasks lists, Obsidian dashboard, weekly program, or task organization. +--- + +# Obsidian Task Manager + +## Overview + +Manage the user's integrated task system spanning Google Tasks (18 lists), Obsidian vault task files, daily notes with git activity, and two dashboards. The system is bi-directionally synced via `obsidian-gcal-sync` plugin. + +## Architecture + +``` +Google Tasks (18 lists) <──obsidian-gcal-sync──> Obsidian Task Files + │ + Dashboard.md (Command Center) + Daily Dashboard.md (Daily view) + │ + 8-Daily Notes/YYYY-MM-DD.md +``` + +## Google Tasks ↔ Obsidian Mapping (18 lists) + +| Google Tasks List | Google List ID | Obsidian File | Tag | Category | +|---|---|---|---|---| +| Do It Now | `MTY4NDM5Mjk5OTU0MjIxNzE5MzQ6MDow` | `0x534C56/Tasks/Do It Now.md` | #acil | Urgent | +| BAM | `TGRRMm1OYzhheTJZQXh6Mg` | `6-Geopolitics/BAM/Tasks/BAM.md` | #bam | Academic | +| Development | `UHA4LTk5alB4TXdPUHVYMA` | `0x534C56/Tasks/Development.md` | #Development | Tech | +| Cybersec & Bug Bounty | `MV9aRmFybVd5Q09hN2VQMw` | `0x534C56/Tasks/Cybersec & Bug Bounty.md` | #Cybersec | Tech | +| İstihbarat Doktrin | `cjZ2Z2ctUXNTVF9ncjhteg` | `0x534C56/Tasks/İstihbarat Doktrin.md` | #doktrin | Intelligence | +| İstihbarat Araştırma | `MFoxNTd5WXpDVWlRWUdUMQ` | `0x534C56/Tasks/İstihbarat Araştırma.md` | #Intelligence | Intelligence | +| Siber & Hibrit | `c0ZSRkJxTkd6bWhDWEU2cg` | `0x534C56/Tasks/Siber & Hibrit.md` | #cyber | Intelligence | +| Russian & TORFL | `aEVSS3RCbFVkTkRTS3NUNA` | `0x534C56/Tasks/Russian & TORFL.md` | #Russian | Language | +| Swahili | `bEh2bzRNOEdFMnFDaTFzbw` | `0x534C56/Tasks/Swahili.md` | #swahili | Language | +| Kariyer & Altyapı | `NWo0VUt0RklrSkloM0FyTg` | `0x534C56/Tasks/Career Roadmap.md` | #Career | Personal | +| Freelance İşler | `RGhRRktUOGc3TDlJNDB2eQ` | `0x534C56/Tasks/Freelance İşler.md` | #Freelance | Work | +| English & IELTS | `b0NDWkZsTWlhMURuaTNsQQ` | `0x534C56/Tasks/English & IELTS.md` | #IELTS | Language | +| French | `aW0yd0pDXzlxdjdUYUFHbA` | `0x534C56/Tasks/French.md` | #French | Language | +| Persian | `T1Z5SFlTbFlJVkVNR3Z3NQ` | `0x534C56/Tasks/Persian.md` | #Persian | Language | +| Arabic | `WXM1N3pySkFCMlBRX0Q3Wg` | `0x534C56/Tasks/Arabic.md` | #arabic | Language | +| Hindi & Urdu | `VGZSblhEbkIyNUJSOGVMVA` | `0x534C56/Tasks/Hindi & Urdu.md` | #Hindi | Language | +| Spanish | `V1dnTWNnWGZWNDlVeUtJbg` | `0x534C56/Tasks/Spanish.md` | #Spanish | Language | +| Turkish | `bnl0S0hUZ3hZQU1ZT2g5SA` | `0x534C56/Tasks/Turkish.md` | #Turkish | Language | + +### Accessing Google Tasks Lists via API + +The `google-workspace` MCP has no `list_task_lists` endpoint. To get list IDs: + +```bash +TOKEN=$(python3 -c "import json; d=json.load(open('~/.google_workspace_mcp/credentials/token_.json')); print(d['token'])") +curl -s "https://tasks.googleapis.com/tasks/v1/users/@me/lists" -H "Authorization: Bearer $TOKEN" +``` + +Once you have the list ID, use `mcp__google-workspace__list_tasks` with `task_list_id` set to the ID above. Use `@default` as alias for "Do It Now" (the first/default list). + +**Additional task-related files (not synced to Google Tasks):** +- `0x534C56/Tasks/Geopolitics Research.md` — PDF reading assignments, country research gaps +- `0x534C56/Tasks/Afrika Ödevler.md` — Yunus Hoca weekly assignments, UNESCO series +- `0x534C56/Tasks/Intel.md`, `Tools.md`, `Needs.md`, `Hursit Hoca.md`, `Language.md` + +## Task Format (Obsidian ↔ Google Tasks compatible) + +```markdown +- [ ] 📖 Task title #tag 📅 2026-04-15 + Optional notes or description + 📎 Vault: [[path/to/file.pdf|Display Name]] + 📎 Books: `/mnt/storage/Common/Books/Category/file.pdf` +- [x] Completed task #tag ✅ 2026-04-03 +``` + +**Rules:** +- `- [ ]` = open task (Google Tasks: needsAction) +- `- [x]` = completed (Google Tasks: completed) +- `📅 YYYY-MM-DD` = due date +- `✅ YYYY-MM-DD` = completion date +- `#tag` = category tag (must match the file's assigned tag from mapping above) +- `📎 Vault:` = Obsidian wikilink to PDF in vault +- `📎 Books:` = path to PDF in `/mnt/storage/Common/Books/` +- Indented lines = notes/subtasks (sync as Google Tasks notes field) + +## Daily Notes Format + +Path: `8-Daily Notes/YYYY-MM-DD.md` + +```markdown +--- +up:: [[8-Daily Notes]] +tag:: [[Daily Planning]] +created:: YYYY-MM-DD +--- + +# YYYY-MM-DD (Gün) + +## Day planner + +%% gcal-sync-start %% +(AUTO-GENERATED by obsidian-gcal-sync — DO NOT EDIT) +%% gcal-sync-end %% + +--- + +## Aktiviteler + +- [x] Russian: Rosetta Stone — Business Unite 1 +- [x] ASM: 18 commit — worker auto-restart, pipeline watchdog +- [x] Pico-Ducky: 1 commit — webhook security + +## Notlar + +### Dil Calismalari +- Details here... + +--- + +## Proje Aktivitesi + +### Project Name (X commit) +- Bullet point summaries... + +--- + +## Gunun Ozeti + +- Yapilan: ... +- Yapilamayan: +- Yarina: ... +``` + +**Critical:** NEVER touch content between `%% gcal-sync-start %%` and `%% gcal-sync-end %%` — plugin manages that. + +**Dashboard reads:** `file.tasks` (checkboxes) from daily notes. The `## Aktiviteler` section items MUST be `- [x]`/`- [ ]` format for Dashboard to count them. Dashboard also keyword-matches: `russian|ielts|swahili|french|persian|arabic|hindi` for language stats, `meet|toplant` for meetings. + +## Dashboard Files + +### Command Center (`0x534C56/Tasks/Dashboard.md`) +- Donut chart: open/completed/overdue tasks across ALL task files +- File progress grid: per-file completion bars +- Category distribution: tag-based horizontal bars +- Language progress: 9-language donut grid +- Overdue, This Week, Last Completed sections +- Uses `dataviewjs` — reads from `"0x534C56/Tasks" OR "6-Geopolitics/BAM/Tasks"` + +### Daily Dashboard (`8-Daily Notes/Daily Dashboard.md`) +- Today's tasks from daily note +- Week bar chart (7 days) +- Weekly summary metrics (notes, events, completed, meetings, language) +- 30-day activity heatmap with streak counter +- Language study bar chart (30 days, keyword-based) +- Upcoming 7 days +- Task timeline (this week / next week / 2-4 weeks) + +## PDF Libraries + +Two source locations for reading materials: + +### 1. Vault PDFs (`/`) +- `6-Geopolitics/assets/` — 20+ categorized subfolders (İSTİHBARAT, SAVAŞ VE STRATEJİ, HİBRİT TEHDİTLER, NÜKLEER VE KBRN, etc.) +- `6-Geopolitics/BAM/Afrika/assets/` — Africa studies by topic +- `6-Geopolitics/Russia/`, `China/`, `Suriye/`, `USA/` — Country-specific +- `6-Geopolitics/Intelligence and National Security/` — NATO/UK doctrine PDFs + +### 2. Books Library (`/mnt/storage/Common/Books/`) +- 37 top-level categories (managed by `librarian` skill) +- Key folders: `Istihbarat/`, `GuvenlikStratejileri/`, `AskeriDoktrin/`, `NATO/`, `SiberGuvenlik/`, `UluslararasiIliskiler/` +- Naming: `Author - Title (Year).pdf` + +### Linking PDFs to Tasks +When adding reading tasks, search BOTH locations and add links: +```markdown +- [ ] 📖 Read **Book Title** #tag 📅 2026-04-15 + 📎 Vault: [[6-Geopolitics/assets/04. GÜVENLİK TEORİLERİ/filename.pdf]] + 📎 Books: `/mnt/storage/Common/Books/Category/Author - Title (Year).pdf` +``` + +## Plugin Configuration + +**Config file:** `/.obsidian/plugins/obsidian-gcal-sync/data.json` +**Credentials:** `~/.google_workspace_mcp/credentials/token_.json` + +Settings: +- Auto sync: 60 minutes +- Reverse sync: enabled (Obsidian → Google Calendar) +- Lookahead: 14 days +- Daily notes template: `11-Templates/Daily Template & Checklist.md` +- Calendar output: `0x534C56/Tasks/Haftalik Program.md` +- Calendars: Takvimim (primary), Coursera, HackerOne + +**Note:** `obsidian-google-tasks` plugin is also installed but has empty refresh token — NOT active. All sync goes through `obsidian-gcal-sync`. + +## MCP Tools Available + +### Google Workspace MCP +- `mcp__google-workspace__list_tasks` — List tasks in a specific Google Tasks list (requires `task_list_id`, use `@default` for main list) +- `mcp__google-workspace__manage_task` — Create/update/delete/move tasks +- `mcp__google-workspace__get_task` — Get task details + +**Limitation:** No `list_task_lists` endpoint — cannot enumerate all 18 lists via MCP. Work through Obsidian files instead (they're the source of truth after sync). + +### Google Calendar MCP (claude.ai) +- `gcal_create_event`, `gcal_list_events`, `gcal_list_calendars`, etc. + +## Common Operations + +### Add a new task +1. Determine which list/file it belongs to (use mapping table above) +2. Read the target Obsidian file +3. Add task in correct format with tag and due date +4. If reading task: search both PDF libraries, add 📎 links + +### Update daily note with git activity +1. Run `git log --format="%ad | %s" --date=short --since=DATE --until=NEXT_DATE` for each project in `~/Documents/` +2. Active projects: `asm`, `pico-ducky`, `killer-claude`, `Reporter-ollama`, `Frodo-code`, `Telegram-Crawler`, `claude-code`, `orsam`, `twitterintel`, `bamdergi` +3. Add `## Aktiviteler` section with `- [x]` checkboxes (Dashboard-compatible) +4. Add `## Proje Aktivitesi` with detailed per-project breakdowns +5. Fill `## Gunun Ozeti` + +### Track language learning +1. Update the language task file (e.g., `Russian & TORFL.md`) with progress +2. Add `- [x] Russian: description` to daily note `## Aktiviteler` (keyword must match Dashboard regex) +3. Supported keywords: `russian|ielts|swahili|french|persian|arabic|hindi|spanish|turkish` + +### Add PDF reading assignment +1. Create task in relevant file with `📖` prefix +2. Search Vault: `find -iname "*keyword*" -name "*.pdf"` +3. Search Books: `find /mnt/storage/Common/Books -iname "*keyword*" -name "*.pdf"` +4. Add `📎 Vault:` and/or `📎 Books:` links under the task +5. If PDF not found locally, add URL if available online + +### Check progress +1. Read the target task file +2. Count `- [x]` vs `- [ ]` items +3. For language overview: read all 9 language files +4. For overall: reference Dashboard.md logic (reads all task files) + +## Important Files Quick Reference + +| Purpose | Path | +|---|---| +| Task Dashboard | `0x534C56/Tasks/Dashboard.md` | +| Daily Dashboard | `8-Daily Notes/Daily Dashboard.md` | +| Weekly Program | `0x534C56/Tasks/Haftalik Program.md` | +| Daily Note Template | `11-Templates/Daily Template & Checklist.md` | +| Plugin Config | `.obsidian/plugins/obsidian-gcal-sync/data.json` | +| Google Token | `~/.google_workspace_mcp/credentials/token_.json` | +| Vault Root | `/` | +| Books Library | `/mnt/storage/Common/Books/` | +| Projects Dir | `~/Documents/` | diff --git a/personas/_shared/community-skills/opencode-cli/SKILL.md b/personas/_shared/community-skills/opencode-cli/SKILL.md new file mode 100644 index 0000000..64fd741 --- /dev/null +++ b/personas/_shared/community-skills/opencode-cli/SKILL.md @@ -0,0 +1,411 @@ +--- +name: opencode-cli +description: Configure and operate the OpenCode CLI agent. Covers AGENTS.md rules, custom agents, models & providers, OpenCode Zen, custom slash commands, formatters, permissions, SKILL.md authoring, MCP servers, plugins, custom tools, LSP, themes, keybinds, TUI, server/SDK, GitHub Actions integration, IDE extensions, proxy/network setup, and troubleshooting. Use when the user asks to set up opencode, write a custom opencode agent, add a /command, restrict permissions, change the default model, configure formatters or LSP, port a Claude Code project, register an MCP server, write a plugin or custom tool, run opencode headless or in CI/GitHub Actions, troubleshoot startup/auth/cache issues, or any opencode.json/AGENTS.md/tui.json question. Triggers include "opencode", "opencode.json", "AGENTS.md", "tui.json", "@subagent", "/connect", "/init", "opencode permissions", "opencode skill", "opencode mcp", "opencode serve", "opencode github install". +license: MIT +compatibility: opencode, claude-code +metadata: + audience: developers, devops, security-engineers + source: https://opencode.ai/docs +--- + +# OpenCode CLI — Configure & Operate + +Reference for **using** opencode (not building it). Jump to the right section for +the task; deep details live in `references/`. + +> Skill spec rule: `name` matches `^[a-z0-9]+(-[a-z0-9]+)*$` (≤64 chars). +> `description` ≤1024 chars and must be specific enough for autoselection. + +## References (load on demand) + +- `references/configuration.md` — full `opencode.json` + `tui.json` schema, precedence, env var/file substitution, themes, keybinds, LSP +- `references/agents-skills-commands.md` — custom agent MD format, SKILL.md authoring, slash commands, custom tools +- `references/permissions-tools.md` — every permission state, built-in tool catalog, per-agent overrides +- `references/providers-models.md` — providers, `/connect`, OpenCode Zen, OpenAI-compatible custom endpoints, MCP servers +- `references/server-api.md` — `opencode serve` full HTTP API: every endpoint, auth, CORS, mDNS, SSE event stream, curl recipes, security checklist +- `references/operate.md` — CLI flags, SDK (`@opencode-ai/sdk`), GitHub Actions, IDE, proxy, troubleshooting + +## Configuration locations (precedence) + +| Layer | Path | Use for | +| ------------ | ---------------------------------------------------------------------- | ----------------------------- | +| Project | `./opencode.json`, `./AGENTS.md`, `.opencode/{agents,commands,skills,plugins,tools,themes}` | repo conventions, commit it | +| User-global | `~/.config/opencode/{AGENTS.md,opencode.json,agents,commands,skills,plugins,tools,themes}` | personal preferences | +| TUI | `~/.config/opencode/tui.json` | theme + keybinds (TUI only) | +| CC-fallback | `~/.claude/{CLAUDE.md,skills}`, `.claude/skills` | reused from Claude Code | +| Agents-spec | `~/.agents/skills`, `.agents/skills` | shared with other agent tools | + +Full precedence chain (low→high override): remote org defaults → user-global → +custom (`OPENCODE_CONFIG` env) → project → directory (`OPENCODE_CONFIG_DIR`) → +inline (`OPENCODE_CONFIG_CONTENT`) → managed. + +Disable Claude Code fallback: +``` +OPENCODE_DISABLE_CLAUDE_CODE=1 +OPENCODE_DISABLE_CLAUDE_CODE_PROMPT=1 +OPENCODE_DISABLE_CLAUDE_CODE_SKILLS=1 +``` + +Data & cache locations: +- App data: `~/.local/share/opencode/` (auth, sessions) +- Logs: `~/.local/share/opencode/log/` +- Cache: `~/.cache/opencode/` + +## 1. Rules — AGENTS.md + +Project: `./AGENTS.md` (commit). Global: `~/.config/opencode/AGENTS.md`. + +``` +opencode +> /init +``` +`/init` analyzes the repo and creates/improves AGENTS.md without overwriting. + +Reference external rule files via `opencode.json`: +```json +{ + "$schema": "https://opencode.ai/config.json", + "instructions": [ + "docs/development-standards.md", + "packages/*/AGENTS.md", + "https://raw.githubusercontent.com/org/rules/main/style.md" + ] +} +``` +Glob patterns scale to monorepos. Remote URLs have a 5s timeout (silent drop on +failure). + +Resolution order: project `AGENTS.md` (walks up tree) → project `CLAUDE.md` → +global `~/.config/opencode/AGENTS.md` → `~/.claude/CLAUDE.md`. + +## 2. Agents (built-ins) + +Primary (Tab-cycle): `build` (default, all tools), `plan` (analysis; +edits/bash → `ask`). +Subagents (`@`-mention or `task` tool): `general` (full tools), `explore` +(read-only). +Hidden system agents: `compaction`, `title`, `summary`. + +Custom agents → see `references/agents-skills-commands.md`. + +## 3. Models + +Default in `opencode.json`: +```json +{ "$schema": "https://opencode.ai/config.json", "model": "anthropic/claude-sonnet-4-5" } +``` +Format: `provider_id/model_id`. Set `small_model` for compaction/title. + +Resolution order: `--model`/`-m` → `opencode.json` → last used → first by priority. + +Switch interactively: `/models`. Add provider creds: `/connect`. List: `opencode models`. + +Reasoning-effort variants: +- Anthropic: `high` (default), `max` +- OpenAI: `none|minimal|low|medium|high|xhigh` +- Google: `low|high` + +Append to id: `openai/gpt-5/high`. Override under `provider.models..variants`. + +OpenCode Zen (`opencode/`): pre-tested gateway, pay-as-you-go, includes +GPT-5/Codex, Claude Opus/Sonnet/Haiku, Qwen, Gemini, MiniMax, GLM, free-tier +betas. Auto-reload $20 when balance < $5. + +Full provider config (Ollama, LM Studio, OpenRouter, Bedrock, Vertex, +custom OpenAI-compatible) → see `references/providers-models.md`. + +## 4. Custom commands (slash) + +`~/.config/opencode/commands/.md` or `.opencode/commands/.md`. + +```yaml +--- +description: Run tests with coverage and summarize failures. +agent: build +model: anthropic/claude-sonnet-4-5 +--- +Run `!`pnpm test --coverage`` then read @coverage/lcov-report/index.html. +Summarize failing tests for $ARGUMENTS (default: all). +``` + +Templating in body: +- `$ARGUMENTS` (full arg string), `$1`, `$2`, … (positional) +- `` !`` `` — embeds stdout at prompt-build time +- `@` — inlines file contents + +Custom commands shadow built-ins (`/init`, `/help`, `/undo`, `/share`, `/models`). + +Built-in TUI commands: `/new`, `/sessions`, `/models`, `/themes`, `/share`, +`/unshare`, `/connect`, `/init`, `/undo`, `/redo`, `/editor`, `/export`, +`/thinking`, `/help`. + +## 5. Formatters + +Auto-runs after every successful write/edit. Built-ins cover JS/TS, Python, +Go, Rust, Ruby, HTML/CSS, MD/JSON/YAML, C/C++, PHP, Kotlin, Dart, Haskell, +Clojure, Elixir. + +Disable all: `{ "formatter": false }`. Disable one: +```json +{ "formatter": { "prettier": { "disabled": true } } } +``` + +Override (`$FILE` is substituted per write): +```json +{ + "formatter": { + "ruff": { + "command": ["uv", "run", "ruff", "format", "$FILE"], + "extensions": [".py", ".pyi"] + } + } +} +``` + +## 6. Permissions + +States: `allow` (silent), `ask` (prompt), `deny` (blocked & hidden where +applicable). + +Tools: `read`, `edit`, `glob`, `bash`, `grep`, `webfetch`, `websearch`, `task`, +`skill`, `lsp`, `question`, `external_directory`, `doom_loop`, `list`, `todowrite`. + +Defaults: most → `allow`. `doom_loop` & `external_directory` → `ask`. `.env` +denied to `read` by default. + +Pattern matching: `*` (any), `?` (one), `~`/`$HOME` expansion. **Last match wins.** + +Recommended baseline: +```json +{ + "permission": { + "*": "allow", + "edit": "allow", + "bash": { + "*": "ask", + "git diff*": "allow", + "git log*": "allow", + "git status": "allow", + "ls *": "allow", + "rm *": "deny", + "rm -rf *": "deny", + "git push*": "ask", + "git push --force*": "deny" + }, + "webfetch": "ask", + "external_directory": { "~/projects/*": "allow", "*": "ask" } + } +} +``` + +Per-agent override → `references/permissions-tools.md`. + +## 7. Skills (SKILL.md authoring) + +Skills are **on-demand** (agent calls `skill({ name })` after seeing them +in tool docs). Discovery: walk-up from CWD to git worktree. + +Project: `.opencode/skills/`, `.claude/skills/`, `.agents/skills/`. +Global: `~/.config/opencode/skills/`, `~/.claude/skills/`, `~/.agents/skills/`. + +Frontmatter: +```yaml +--- +name: git-release # required, kebab, ≤64 +description: | # required, ≤1024, drives autoselection + Cut a release: bump version, generate changelog, tag, draft GH release. + Use when user says "release", "tag a version", "publish". +license: MIT # optional +compatibility: opencode # optional +metadata: { audience: maintainers } # optional, string-to-string only +--- +``` + +Permission filter: +```json +{ "permission": { "skill": { "*": "allow", "internal-*": "deny", "experimental-*": "ask" } } } +``` + +Disable entirely for an agent: `tools: { skill: false }`. + +## 8. MCP servers (quick) + +Local stdio: +```json +{ "mcp": { "fs": { "type": "local", + "command": ["npx", "-y", "@modelcontextprotocol/server-filesystem", "/tmp"], + "enabled": true } } } +``` + +Remote HTTP: +```json +{ "mcp": { "context7": { "type": "remote", + "url": "https://mcp.context7.com/mcp", + "headers": { "CONTEXT7_API_KEY": "{env:CONTEXT7_API_KEY}" } } } } +``` + +OAuth flow: `opencode mcp auth `, list: `opencode mcp list`, +debug: `opencode mcp debug `. Disable per-server with `enabled: false`, +disable globally with `tools: { my-mcp*: false }`. Full schema → +`references/providers-models.md`. + +## 9. Plugins, custom tools, LSP, themes + +- **Plugins** (lifecycle hooks, env injection, shell, custom tools): + `.opencode/plugins/` or NPM via config `plugin: ["pkg-name"]`. TS/JS, + imports `@opencode-ai/plugin`. Loads global → project, plugin order matters. +- **Custom tools**: `.opencode/tools/.ts` exporting `tool({...})`. Filename + becomes tool name. Override built-ins with same name. Multi-language wrappers + via shell-out. Args validated via Zod. +- **LSP**: 30+ built-in language servers auto-activate. Disable all + with `"lsp": false`. Custom: `lsp.` with `command/extensions/env`. +- **Themes**: built-ins (`system`, `tokyonight`, `everforest`, `ayu`, + `catppuccin`, `gruvbox`, `kanagawa`, `nord`, `matrix`, `one-dark`). + Custom JSON in `~/.config/opencode/themes/` or `.opencode/themes/`. + Switch: `/themes` or `tui.json` → `theme: "..."`. + +Details: `references/configuration.md`, `references/agents-skills-commands.md`. + +## 10. Keybinds (TUI) + +Leader = `ctrl+x` (configurable). Defaults: + +| Action | Binding | +| --------------- | -------------------- | +| New session | `n` | +| Session list | `l` | +| Model list | `m` | +| Theme switch | `t` | +| Open editor | `e` | +| Export markdown | `x` | +| Exit | `ctrl+c` / `ctrl+d` / `q` | +| Command list | `ctrl+p` | +| Agent cycle | `tab` / `shift+tab` | + +Customize in `~/.config/opencode/tui.json`: +```json +{ "$schema": "https://opencode.ai/tui.json", + "theme": "tokyonight", + "keybinds": { "leader": "ctrl+x", "app_exit": "ctrl+c,ctrl+d,q" } } +``` +Set any binding to `"none"` to disable. + +Built-in readline-style input edit (non-configurable): `ctrl+a/e/w`, `alt+f`. + +## 11. Operate (cheatsheet) + +- TUI: `opencode` (cwd) or `opencode /path/to/project` +- One-shot: `opencode run "Explain closures"` +- Headless server: `opencode serve` → REST API on `127.0.0.1:4096`, + OpenAPI at `/doc`. Set `OPENCODE_SERVER_PASSWORD` (required) and + `OPENCODE_SERVER_USERNAME` (default `opencode`). +- Auth: `opencode auth login` (writes `~/.local/share/opencode/auth.json`) +- GH Actions: `opencode github install` → comment `/opencode` or `/oc` on + PRs/issues. See `references/operate.md`. +- IDE (VS Code/Cursor/Windsurf/Codium): auto-installs from terminal, + shortcut `Cmd/Ctrl+Esc` (open), `Cmd/Ctrl+Shift+Esc` (new session), + `Cmd+Option+K` / `Alt+Ctrl+K` (insert file ref). +- SDK: `npm i @opencode-ai/sdk`, `createOpencode({ config: {...} })` for + programmatic control. +- Proxy: `HTTPS_PROXY`, `NO_PROXY=localhost,127.0.0.1` (must bypass localhost + for TUI). Custom CA: `NODE_EXTRA_CA_CERTS=/path/ca.pem`. +- Debug: `opencode --log-level DEBUG --print-logs`. Logs at + `~/.local/share/opencode/log/`. +- Reset: `rm -rf ~/.cache/opencode` (clears provider cache). + +## 12. Quick recipes + +**Port Claude Code project to opencode** (no migration needed): +1. Don't delete `.claude/`. Opencode reads it natively. +2. Create `.opencode/` siblings only for opencode-specific tweaks. +3. Optional: rename `CLAUDE.md` → `AGENTS.md` (or keep both — AGENTS.md wins). + +**Lock project to one model + provider**: +```json +{ "$schema": "https://opencode.ai/config.json", + "model": "anthropic/claude-sonnet-4-5", + "enabled_providers": ["anthropic"] } +``` + +**Make `plan` agent read-only globally**: +```json +{ "agent": { "plan": { "permission": { "edit": "deny", "bash": "deny" } } } } +``` + +**Add `/refactor` for a specific subagent**: +`.opencode/commands/refactor.md`: +```yaml +--- +description: Hand a refactor brief to the refactor-bot subagent. +agent: refactor-bot +--- +@$1 +$ARGUMENTS +``` + +**Headless CI run (one-shot, JSON output)**: +```bash +OPENCODE_API_KEY=$KEY opencode run \ + --model anthropic/claude-sonnet-4-5 \ + --print "Audit src/ for SQL injection. Output as JSON." +``` + +**GitHub Action workflow stub** (`.github/workflows/opencode.yml`): +```yaml +name: opencode +on: + issue_comment: { types: [created] } + pull_request_review_comment: { types: [created] } +jobs: + run: + if: contains(github.event.comment.body, '/opencode') || contains(github.event.comment.body, '/oc') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: opencode-ai/opencode-action@v1 + with: + model: anthropic/claude-sonnet-4-5 + token: ${{ secrets.GITHUB_TOKEN }} + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +## 13. Common pitfalls + +- Skill `name` must match `^[a-z0-9]+(-[a-z0-9]+)*$` — underscores break loading. +- `description` is the **only** signal opencode uses to autoselect a skill — + vague descriptions cause wrong-skill calls or no-call. +- `bash: { "git push*": "allow" }` includes `git push --force` — guard with a + more specific deny rule, last-match-wins. +- Per-agent permission in MD frontmatter overrides `opencode.json` for that + agent only — they don't merge field-by-field. +- `mode: primary` agents are Tab-cycle slots; too many makes the cycle painful. + Default to `subagent` unless the agent is a daily driver. +- `hidden: true` only hides from `@` autocomplete — orchestrators can still + invoke via the `task` tool. +- Remote `instructions:` URLs have a 5s timeout; an unreachable URL silently + drops that rule layer. +- MCP servers consume context tokens — selectively enable. +- `NO_PROXY` must include `localhost,127.0.0.1` or the TUI's local server + loops through the corporate proxy and hangs. +- `share: "auto"` plus public-repo `.git` config = unintended public links. + Lock to `share: "manual"` or `"disabled"` in the project config. +- `OPENCODE_DISABLE_CLAUDE_CODE_SKILLS=1` is the only way to stop opencode from + auto-loading `~/.claude/skills/` — useful when CC and OC have skills with + conflicting names. +- Custom tools with the same name as built-ins **silently override** them. + Audit `.opencode/tools/` for shadowing of `bash`/`edit`/`read`. + +## 14. Operational checklist + +When asked to "set up opencode" in a fresh repo: + +- [ ] `opencode` → `/init` (creates `AGENTS.md`) +- [ ] Add `opencode.json` with `$schema`, `model`, baseline `permission` +- [ ] Define `bash` allowlist for project's hot commands (test, build, lint) +- [ ] Lock `share` to `manual` or `disabled` for private repos +- [ ] If repo has `.claude/skills/`, leave them — they auto-load +- [ ] If team needs custom workflow agents, drop them in `.opencode/agents/` +- [ ] If team needs slash commands, drop them in `.opencode/commands/` +- [ ] Add `.opencode/`, `AGENTS.md`, `opencode.json` to git; ignore + `~/.config/opencode/` content (per-user) diff --git a/personas/_shared/community-skills/opencode-cli/references/agents-skills-commands.md b/personas/_shared/community-skills/opencode-cli/references/agents-skills-commands.md new file mode 100644 index 0000000..531c6c9 --- /dev/null +++ b/personas/_shared/community-skills/opencode-cli/references/agents-skills-commands.md @@ -0,0 +1,257 @@ +# Custom Agents, Skills, Commands, and Tools + +## Custom agents + +Locations: `~/.config/opencode/agents/.md`, `.opencode/agents/.md`, +or inline under `agent.` in `opencode.json`. + +Name regex: `^[a-z0-9]+(-[a-z0-9]+)*$`. Filename = agent name. + +### Frontmatter (full) + +```yaml +--- +description: Reviews PR diffs against repo conventions before merge. # required +mode: subagent # primary | subagent | all +model: anthropic/claude-sonnet-4-5 +temperature: 0.2 # 0.0-1.0 +top_p: 1.0 +steps: 12 # max iterations before forced text reply +permission: + read: allow + edit: deny + glob: allow + grep: allow + list: allow + bash: + "*": ask + "git diff*": allow + "git log*": allow + "git status": allow + webfetch: allow + websearch: allow + task: allow + external_directory: ask + todowrite: allow + lsp: allow + skill: allow + question: allow + doom_loop: ask +color: "#7c3aed" # or: primary|secondary|accent|success|warning|error|info +hidden: false # hide from @ autocomplete (still callable via task tool) +disable: false +prompt: "{file:./prompts/pr-reviewer.md}" # load body from external file +tools: + skill: false # disable a tool entirely for this agent + my-mcp-tool: true +--- + +You are a PR reviewer. Read the diff, flag regressions, suggest fixes… +``` + +### JSON-style equivalent + +```json +{ + "agent": { + "pr-reviewer": { + "description": "Reviews PR diffs", + "mode": "subagent", + "model": "anthropic/claude-sonnet-4-5", + "temperature": 0.2, + "permission": { + "edit": "deny", + "bash": { "*": "ask", "git diff*": "allow" } + }, + "tools": { "skill": false } + } + } +} +``` + +### Mode semantics + +- `primary` — Tab-cycle slot, full chat surface. Examples: `build`, `plan`. +- `subagent` — invoked via `@` in chat or via the `task` tool from another + agent. Hidden agents are still callable via `task`. +- `all` — both primary and subagent. Rare; use only for daily-driver agents + that also act as orchestrator helpers. + +### Scaffolder + +`opencode agent create` opens an interactive wizard that picks tools, +permissions, and writes the MD file. + +### Permission merging + +- `opencode.json → permission` is the global base. +- `opencode.json → agent..permission` overrides per agent. +- Agent MD frontmatter `permission:` overrides everything for that agent. + +Object-style permissions (e.g. `bash: { "git *": "allow" }`) replace the entire +parent value when overridden — they do **not** deep-merge. Spell out every +pattern when you override. + +## Skills (SKILL.md spec) + +Discovery (walk-up from CWD to git worktree, all matches loaded): + +- `.opencode/skills//SKILL.md` +- `.claude/skills//SKILL.md` +- `.agents/skills//SKILL.md` +- `~/.config/opencode/skills//SKILL.md` +- `~/.claude/skills//SKILL.md` +- `~/.agents/skills//SKILL.md` + +### Frontmatter + +```yaml +--- +name: git-release # required, kebab-case, ≤64 chars +description: | # required, ≤1024 chars, drives autoselection + Cut a release: bump version, generate changelog from merged PRs, tag, draft + GH release notes. Use when user says "release", "tag a version", "publish". +license: MIT # optional +compatibility: opencode # optional +metadata: # optional, string-to-string only + audience: maintainers + source: https://example.com/release-runbook +--- +``` + +### Body conventions + +- Short, headed reference text the agent can grep. +- Absolute paths inside the skill dir for scripts: `scripts/foo.py`. +- Examples in `examples/`, deeper docs in `references/`. +- Skill itself does **not** load reference files automatically — let the agent + decide based on user task. + +### Loading model + +Agents see available skills' name+description in the `skill` tool's docs and +call `skill({ name: "" })` on demand. The body of `SKILL.md` is +returned verbatim. **No auto-injection** — vague descriptions fail silently. + +### Permission filter + +```json +{ + "permission": { + "skill": { + "*": "allow", + "internal-*": "deny", + "experimental-*": "ask" + } + } +} +``` + +Disable entirely for an agent: `tools: { skill: false }` in agent frontmatter. + +### Common authoring mistakes + +- Description too generic (`"helps with deployment"`) → never selected. +- Description too keyword-stuffed → spurious triggers, wrong-context calls. +- Hyphens vs underscores: only hyphens valid in `name`. +- Forgetting `description` in body — only the frontmatter field counts for selection. +- Auto-loading huge `references/*.md` content into the body — bloats every + invocation. Move depth into separate files; reference them by relative path. + +## Custom commands (slash) + +Locations: `~/.config/opencode/commands/.md`, `.opencode/commands/.md`, +or inline under `command.` in `opencode.json`. + +```yaml +--- +description: Run tests with coverage and summarize failures. +agent: build +model: anthropic/claude-sonnet-4-5 +--- +Run `!`pnpm test --coverage`` then read @coverage/lcov-report/index.html. +Summarize failing tests for $ARGUMENTS (default: all). +``` + +Body templating: +- `$ARGUMENTS` (full arg string), `$1`, `$2`, … (positional) +- `` !`` `` — embeds stdout at prompt-build time (runs once per invocation) +- `@` — inlines file contents + +Invocation: +- `/test` +- `/component Button` — `$1=Button`, `$ARGUMENTS=Button` +- `/create-file config.json src "content"` — `$1=config.json`, `$2=src`, `$3="content"` + +Custom commands shadow built-ins (`/init`, `/help`, `/undo`, `/share`, +`/models`, `/themes`, `/connect`). + +## Custom tools + +Location: `.opencode/tools/.ts` (project) or `~/.config/opencode/tools/` +(global). Filename = tool name. + +```typescript +import { tool } from "@opencode-ai/plugin" + +export default tool({ + description: "Query the project database", + args: { + query: tool.schema.string().describe("SQL query to execute"), + limit: tool.schema.number().optional().describe("Max rows"), + }, + async execute(args, context) { + // context: { agent, sessionID, messageID, directory, worktree } + return `Executed: ${args.query} (limit=${args.limit ?? 100})` + }, +}) +``` + +Notes: +- Custom tool with same name as built-in (`bash`, `edit`, `read`, …) **silently + overrides** the built-in. Audit before deploying. +- Multi-language: TS wrapper that shells out — Python, Go, anything. +- Multiple tools per file: export named functions → tool name becomes + `_`. +- Args validated with Zod (provided by `tool.schema`). +- Permission: respects `tools: { my-tool: false }` and `permission: { ... }` + rules just like built-ins. + +## Plugins (lifecycle hooks) + +Location: `.opencode/plugins/.ts` or NPM packages listed in +`opencode.json` → `plugin: ["pkg-a", "pkg-b"]`. + +Load order: global config → project config → global plugins dir → project +plugins dir. Order within an array is preserved. + +```typescript +// .opencode/plugins/notify.ts +import type { Plugin } from "@opencode-ai/plugin" + +export default (async ({ project, client, $, directory, worktree }) => { + return { + "session.created": async (event) => { + await $`notify-send "opencode" "New session in ${project.name}"` + }, + "tool.before": async (event) => { + // mutate args, log, gate execution + client.app.log({ level: "info", message: `tool=${event.name}` }) + }, + "session.compaction": async (event) => { + // override compaction strategy + }, + } +}) satisfies Plugin +``` + +Capabilities: +- Hook session/file/message/tool lifecycle events +- Add custom tools from inside the plugin +- Modify tool execution (gate, mutate args, replay) +- Inject env vars before commands run +- Run shell via `$` (Bun shell) +- Structured logging via `client.app.log()` + +Local NPM deps: drop a `package.json` next to the plugin. Imports resolve from +`.opencode/package.json` (project) or `~/.config/opencode/package.json` (global). diff --git a/personas/_shared/community-skills/opencode-cli/references/configuration.md b/personas/_shared/community-skills/opencode-cli/references/configuration.md new file mode 100644 index 0000000..302ae2d --- /dev/null +++ b/personas/_shared/community-skills/opencode-cli/references/configuration.md @@ -0,0 +1,175 @@ +# OpenCode Configuration Deep Dive + +`opencode.json` (project + user-global) and `tui.json` (TUI only) are the two +schema-driven config files. Source of truth: `https://opencode.ai/config.json`, +`https://opencode.ai/tui.json`. + +## Precedence chain (low → high override) + +1. Remote (organizational defaults pulled by URL) +2. User-global: `~/.config/opencode/opencode.json` +3. Custom path: `OPENCODE_CONFIG=/path/to/cfg.json` +4. Project: `./opencode.json` +5. Directory: `OPENCODE_CONFIG_DIR=/dir/with/cfg.json` +6. Inline: `OPENCODE_CONFIG_CONTENT='{"model":"..."}'` +7. Managed settings (highest, MDM-style) + +Higher items win on conflict. Arrays and maps merge field-by-field where +sensible. + +## Variable substitution inside JSON + +```jsonc +{ + "model": "{env:OPENCODE_MODEL}", // env var + "provider": { + "openai": { "apiKey": "{file:~/.secrets/openai}" } // file contents (trimmed) + } +} +``` + +Both forms expand at load time. Useful for keeping secrets out of the file. + +## Top-level fields (full) + +| Field | Purpose | +| -------------------- | -------------------------------------------------------------------- | +| `$schema` | URL pin for editor autocomplete | +| `model` | Primary LLM, e.g. `anthropic/claude-sonnet-4-5` | +| `small_model` | Lightweight tasks (compaction, title, summary) | +| `default_agent` | Fallback when none specified | +| `provider` | Per-provider opts (timeout, caching, baseURL, models) | +| `enabled_providers` | Allowlist | +| `disabled_providers` | Blocklist | +| `shell` | `pwsh`, `/bin/bash`, etc. — interactive bash tool target | +| `autoupdate` | `true` / `false` / `"notify"` | +| `snapshot` | Track file changes (default `true`) — powers `/undo` | +| `share` | `"manual"` (default) / `"auto"` / `"disabled"` | +| `server` | `port`, `hostname`, `mdns`, `cors` | +| `tools` | Toggle tools per name (`bash: false`, `my-mcp*: false`, …) | +| `permission` | Tool permissions (see permissions-tools.md) | +| `formatter` | Per-formatter override | +| `watcher` | `ignore: []` patterns | +| `compaction` | Context-window compaction tuning | +| `mcp` | MCP server map (see providers-models.md) | +| `plugin` | Array of NPM plugin package names or paths | +| `instructions` | Array of paths/globs/URLs for rule files | +| `command` | Inline command templates (alternative to `.opencode/commands/`) | +| `agent` | Inline agent defs (alternative to `.opencode/agents/`) | +| `lsp` | Per-server LSP config | +| `experimental` | Unstable feature flags | + +## Worked example + +```json +{ + "$schema": "https://opencode.ai/config.json", + "model": "anthropic/claude-sonnet-4-5", + "small_model": "anthropic/claude-haiku-4-5", + "default_agent": "build", + "autoupdate": "notify", + "share": "manual", + "snapshot": true, + "shell": "/bin/bash", + "server": { "port": 4096, "hostname": "127.0.0.1", "mdns": false }, + "tools": { "write": true, "bash": true, "internal-*": false }, + "permission": { + "*": "allow", + "bash": { "*": "ask", "git status": "allow", "rm -rf *": "deny" }, + "external_directory": { "~/projects/*": "allow", "*": "ask" }, + "skill": { "*": "allow", "internal-*": "deny" } + }, + "instructions": ["CONTRIBUTING.md", "docs/standards/*.md"], + "command": { + "test": { "description": "Run full test suite", "template": "Run !`pnpm test`" } + }, + "formatter": { "prettier": { "disabled": false } }, + "watcher": { "ignore": ["node_modules/**", "dist/**", ".turbo/**"] }, + "mcp": { + "context7": { + "type": "remote", + "url": "https://mcp.context7.com/mcp", + "headers": { "CONTEXT7_API_KEY": "{env:CONTEXT7_API_KEY}" } + } + }, + "plugin": ["opencode-helicone-session"], + "agent": { + "plan": { "permission": { "edit": "deny", "bash": "deny" } } + } +} +``` + +## tui.json (TUI only) + +```json +{ + "$schema": "https://opencode.ai/tui.json", + "theme": "tokyonight", + "keybinds": { + "leader": "ctrl+x", + "app_exit": "ctrl+c,ctrl+d,q", + "session_new": "n", + "session_list": "l", + "model_list": "m", + "theme_select": "t", + "command_list": "ctrl+p", + "terminal_suspend": "none" + }, + "scroll": { "speed": 3 }, + "diff": { "style": "split" }, + "mouse": true +} +``` + +Set any keybind to `"none"` to disable. Built-in readline edits +(`ctrl+a/e/w`, `alt+f`) are not configurable. + +## Themes + +Built-ins: `system` (terminal-adaptive), `tokyonight`, `everforest`, `ayu`, +`catppuccin`, `gruvbox`, `kanagawa`, `nord`, `matrix`, `one-dark`. + +Custom: `~/.config/opencode/themes/.json` or `.opencode/themes/.json`. +Supports hex (`#1e1e2e`), ANSI codes, color references, dark/light variants. +Special value `"none"` inherits terminal default. + +24-bit truecolor recommended. Verify: `echo $COLORTERM` should print `truecolor` +or `24bit`. + +Activate: `/themes` (TUI) or `tui.json` → `theme: "..."`. + +## LSP + +30+ built-in language servers auto-activate per file extension. + +Disable all: `{ "lsp": false }`. Disable one: +```json +{ "lsp": { "intelephense": { "disabled": true } } } +``` + +Custom server: +```json +{ "lsp": { "myls": { + "command": ["my-lang-server", "--stdio"], + "extensions": [".mylang"], + "env": { "MY_VAR": "v" }, + "initialization": { "settings": { "lint": true } } +} } } +``` + +PHP Intelephense premium: place license at `$HOME/intelephense/license.txt` +(Unix) or `%USERPROFILE%/intelephense/license.txt` (Windows). + +## Watcher / ignore + +```json +{ "watcher": { "ignore": ["node_modules/**", ".git/**", "dist/**", "*.lock"] } } +``` + +Limits which files trigger snapshot/refresh and which are visible to +`glob`/`grep` walks. + +## Experimental flags + +Treat as throwaway. Document any flag you ship with `# why we set this` +comments next to it because they break between releases. diff --git a/personas/_shared/community-skills/opencode-cli/references/operate.md b/personas/_shared/community-skills/opencode-cli/references/operate.md new file mode 100644 index 0000000..ce1bb2e --- /dev/null +++ b/personas/_shared/community-skills/opencode-cli/references/operate.md @@ -0,0 +1,267 @@ +# Operate: CLI, Server, SDK, GitHub, IDE, Network, Troubleshooting + +## CLI surface + +``` +opencode # launch TUI in cwd +opencode /path/to/project # launch TUI in another dir +opencode run "" # one-shot, non-interactive +opencode serve # headless HTTP API server +opencode auth login # add provider creds +opencode models [provider] # list available models +opencode mcp +opencode agent create # interactive agent wizard +opencode github install # GitHub Action setup +opencode upgrade # self-update binary +``` + +### Global flags + +| Flag | Effect | +| ------------------------ | ---------------------------------------- | +| `--model`, `-m ` | Override default model for this run | +| `--agent ` | Use a specific agent | +| `--print` | Print final output to stdout (good for CI) | +| `--print-logs` | Stream logs to stdout | +| `--log-level ` | `DEBUG` / `INFO` / `WARN` / `ERROR` | +| `--help` | Help | +| `--version` | Version | + +### Environment variables (selected) + +``` +OPENCODE_CONFIG=/path/to/opencode.json # custom config path +OPENCODE_CONFIG_DIR=/path/to/dir # custom config dir +OPENCODE_CONFIG_CONTENT='{"model":"..."}' # inline config +OPENCODE_MODEL=anthropic/claude-sonnet-4-5 +OPENCODE_API_KEY=... # default API key +OPENCODE_LOG_LEVEL=DEBUG +OPENCODE_ENABLE_EXA=1 # enable websearch +OPENCODE_DISABLE_CLAUDE_CODE=1 +OPENCODE_DISABLE_CLAUDE_CODE_PROMPT=1 +OPENCODE_DISABLE_CLAUDE_CODE_SKILLS=1 +OPENCODE_SERVER_PASSWORD=... # required for `opencode serve` +OPENCODE_SERVER_USERNAME=opencode # default +HTTPS_PROXY=... +HTTP_PROXY=... +NO_PROXY=localhost,127.0.0.1 +NODE_EXTRA_CA_CERTS=/path/to/ca.pem +``` + +### One-shot CI usage + +```bash +ANTHROPIC_API_KEY=$KEY \ +opencode run \ + --model anthropic/claude-sonnet-4-5 \ + --agent build \ + --print \ + --log-level WARN \ + "Audit src/ for SQL injection. Output JSON: { \"findings\": [...] }." +``` + +Permissions in headless: any `ask` rule effectively becomes `deny` — no UI to +approve. Configure permission sets that are explicitly `allow` or `deny`. + +## Server mode (`opencode serve`) + +Headless HTTP API. Default `127.0.0.1:4096`. **Always set +`OPENCODE_SERVER_PASSWORD`** — without it the server starts unauthenticated. + +```bash +export OPENCODE_SERVER_PASSWORD="$(openssl rand -hex 24)" +opencode serve --port 4096 --hostname 127.0.0.1 + +# Smoke test: +curl -u "opencode:$OPENCODE_SERVER_PASSWORD" http://127.0.0.1:4096/global/health +``` + +OpenAPI 3.1 lives at `http://:/doc`. Endpoint categories: global +health/events, projects, config (live `PATCH /config`), providers + OAuth, +sessions (create / fork / share / revert / abort), messages (sync, async, +slash command, raw shell), files & search (`/find`, `/find/file`, +`/find/symbol`, `/file/content`), LSP/formatter/MCP status, agents, logging, +**TUI control** (used by IDE extensions to drive a running TUI), instance +dispose. + +Multiple clients attach to one server simultaneously (TUI, IDE extension, SDK, +curl scripts). Live MCP-add (`POST /mcp`) and config patching make the server +hot-reconfigurable. + +> **Full endpoint catalog, curl recipes, SSE pattern, security checklist, and +> pitfalls → `references/server-api.md`.** + +## SDK (`@opencode-ai/sdk`) + +Programmatic control from Node/Bun/Deno. + +```typescript +import { createOpencode } from "@opencode-ai/sdk" + +const opencode = await createOpencode({ + hostname: "127.0.0.1", + port: 4096, + config: { model: "anthropic/claude-sonnet-4-5" } +}) + +const session = await opencode.client.session.create({ projectID: "default" }) +const reply = await opencode.client.session.prompt({ + sessionID: session.id, + prompt: "Summarize README.md", +}) + +// Structured output: +const summary = await opencode.client.session.prompt({ + sessionID: session.id, + prompt: "Extract repo metadata", + format: { type: "json_schema", schema: { /* zod-style */ } } +}) + +// Stream events: +opencode.client.events.subscribe(session.id, (e) => console.log(e)) + +// TUI control: +await opencode.client.tui.notify({ message: "Build done" }) +``` + +Connect to existing instance: +```typescript +import { createOpencodeClient } from "@opencode-ai/sdk" +const client = createOpencodeClient({ baseURL: "http://127.0.0.1:4096", auth: { user: "opencode", pass: PW } }) +``` + +Types are generated from the server's OpenAPI — fully type-safe. + +## GitHub integration + +Install: +```bash +opencode github install +``` + +This: +1. Adds the GitHub App at `github.com/apps/opencode-agent`. +2. Writes `.github/workflows/opencode.yml`. +3. Reminds you to add `ANTHROPIC_API_KEY` (or other provider key) to repo secrets. + +Workflow stub: +```yaml +name: opencode +on: + issue_comment: { types: [created] } + pull_request_review_comment: { types: [created] } + issues: { types: [opened] } + pull_request: { types: [opened, synchronize] } + schedule: + - cron: "0 6 * * 1" # weekly Monday triage +jobs: + run: + if: | + github.event_name == 'schedule' || + github.event_name == 'issues' || + github.event_name == 'pull_request' || + contains(github.event.comment.body, '/opencode') || + contains(github.event.comment.body, '/oc') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: opencode-ai/opencode-action@v1 + with: + model: anthropic/claude-sonnet-4-5 + agent: build + share: false # default true for public repos + token: ${{ secrets.GITHUB_TOKEN }} + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +Triggers: +- `/opencode ` or `/oc ` in issue/PR comments +- `/opencode` in PR review comments (gets file path + line + diff context) +- `pull_request` events → auto-review +- `issues` events → auto-triage +- `schedule` cron → bulk runs + +Common asks: +- `/oc explain this issue` +- `/oc fix this` +- `/oc add error handling here` +- `/oc review this PR for security` + +## IDE integration + +Supported: VS Code, Cursor, Windsurf, VSCodium (auto-installs from terminal). + +Shortcuts: +- `Cmd+Esc` (mac) / `Ctrl+Esc` (win/linux) → toggle opencode +- `Cmd+Shift+Esc` / `Ctrl+Shift+Esc` → new session +- `Cmd+Option+K` / `Alt+Ctrl+K` → insert file ref into prompt + +Manual install: marketplace search "OpenCode". Verify the appropriate CLI +(`code`, `cursor`, `windsurf`, `codium`) is on `PATH`. + +ACP (Agent Client Protocol) is the spec used by IDE extensions to talk to a +running opencode server — relevant for building integrations. + +## Network / proxy + +Standard proxy env vars respected: +```bash +export HTTPS_PROXY="https://user:pass@proxy.corp:8080" +export HTTP_PROXY="http://proxy.corp:8080" +export NO_PROXY="localhost,127.0.0.1" # MUST include — TUI uses local server +``` + +Custom CA: +```bash +export NODE_EXTRA_CA_CERTS=/etc/ssl/certs/corp-ca.pem +``` + +NTLM/Kerberos: not natively supported. Front the proxy with an LLM gateway +that handles enterprise auth. + +## Troubleshooting + +| Symptom | Fix | +| ------------------------------------ | -------------------------------------------------------- | +| Won't start | `opencode --print-logs --log-level DEBUG`, then `opencode upgrade` | +| Auth failing | `/connect` again; remove `~/.local/share/opencode/auth.json` to reset | +| Model not in `/models` | `opencode models` to confirm; check provider's + `enabled_providers` allowlist | +| API errors / stale cache | `rm -rf ~/.cache/opencode` | +| Plugin breaks startup | `mv .opencode/plugins .opencode/plugins.disabled` | +| Linux clipboard broken | `apt install xclip` (X11) or `wl-clipboard` (Wayland) | +| TUI hangs through corp proxy | Add `localhost,127.0.0.1` to `NO_PROXY` | +| `Reload Webview` (mac desktop app) | Cmd+Q then relaunch | + +Logs: +- macOS/Linux: `~/.local/share/opencode/log/` +- Windows: `%USERPROFILE%\.local\share\opencode\log\` + +## Share feature + +Modes via `opencode.json`: +```json +{ "share": "manual" } // default — explicit /share +{ "share": "auto" } // every new session shared +{ "share": "disabled" } // hard-off (commit to repo to enforce team-wide) +``` + +URLs: `opncd.ai/s/`. Unshare: `/unshare`. + +Privacy: opencode warns to review before sharing. Don't share sensitive +proprietary code. Enterprise can self-host or restrict to SSO users. + +## Snapshot / undo + +`snapshot: true` (default) tracks file changes per session. `/undo` and `/redo` +walk through them with Git-style integration. Disable for noisy projects: +`{ "snapshot": false }`. + +## Compaction + +Long sessions get auto-compacted when context approaches model limits. Tune: +```json +{ "compaction": { "threshold": 0.85, "model": "anthropic/claude-haiku-4-5" } } +``` +The compaction agent uses `small_model` if not overridden. diff --git a/personas/_shared/community-skills/opencode-cli/references/permissions-tools.md b/personas/_shared/community-skills/opencode-cli/references/permissions-tools.md new file mode 100644 index 0000000..1ccc7dc --- /dev/null +++ b/personas/_shared/community-skills/opencode-cli/references/permissions-tools.md @@ -0,0 +1,144 @@ +# Permissions & Built-in Tools + +## States + +| State | Behavior | +| ------- | ------------------------------------- | +| `allow` | Run silently | +| `ask` | Prompt user, wait for approval | +| `deny` | Block; tool typically hidden from LLM | + +Defaults: most → `allow`; `doom_loop` and `external_directory` → `ask`. `.env` +denied to `read` by default — opt in explicitly if you really need it. + +## Pattern matching + +- `*` zero-or-more chars +- `?` exactly one char +- `~`, `$HOME` expand to home dir +- **Last matching rule wins** — order matters + +## Configuration shapes + +Top-level uniform: +```json +{ "permission": { "*": "ask", "bash": "allow", "edit": "deny" } } +``` + +Pattern map per tool: +```json +{ "permission": { "bash": { "*": "ask", "git *": "allow", "rm *": "deny" } } } +``` + +Per-agent override (deep override, not merge): +```json +{ "agent": { "build": { "permission": { + "bash": { "*": "ask", "git push *": "deny" } +} } } } +``` + +Or in agent MD frontmatter under `permission:` — wins over `opencode.json`. + +## Tool catalog + +| Tool | Purpose | Default | Notes | +| -------------------- | ------------------------------------------------ | ------- | ---------------------------------------------- | +| `read` | Read file contents (line-range supported) | allow | `.env` blocked by default | +| `edit` | In-place exact-string replacement | allow | Triggers formatter | +| `write` | Create or overwrite files | allow | Governed by `edit` permission | +| `apply_patch` | Apply unified-diff patches | allow | Governed by `edit` permission | +| `bash` | Execute shell commands | allow | Pattern map highly recommended | +| `glob` | Pattern-match filenames (`**/*.ts`) | allow | Honors `watcher.ignore` | +| `grep` | Regex content search | allow | | +| `list` | Directory listing | allow | | +| `webfetch` | Fetch & read HTTP(S) page | allow | Consider `ask` for hardened envs | +| `websearch` | Web search via Exa | allow | Requires `OPENCODE_ENABLE_EXA=1` | +| `task` | Spawn a subagent | allow | The orchestrator's gateway to subagents | +| `lsp` | Query language server (definitions, refs, hover) | allow | Experimental | +| `skill` | Load `SKILL.md` content | allow | Filter via `permission.skill: { ... }` | +| `todowrite` | Manage in-session todo list | allow | | +| `question` | Prompt the user with options mid-run | allow | | +| `external_directory` | Access paths outside project root | ask | Whitelist `~/projects/*` etc. | +| `doom_loop` | Long-running iteration mode | ask | Off by default for safety | + +## Recommended baselines + +### Standard dev project + +```json +{ + "permission": { + "*": "allow", + "edit": "allow", + "bash": { + "*": "ask", + "git status": "allow", + "git diff*": "allow", + "git log*": "allow", + "ls *": "allow", + "cat *": "allow", + "pnpm *": "allow", + "rm *": "deny", + "rm -rf *": "deny", + "git push*": "ask", + "git push --force*": "deny", + "git push --force-with-lease*": "ask", + "sudo*": "deny" + }, + "external_directory": { "~/projects/*": "allow", "*": "ask" }, + "webfetch": "ask", + "skill": { "*": "allow", "internal-*": "deny" } + } +} +``` + +### Locked-down audit / pentest project + +```json +{ + "permission": { + "*": "ask", + "read": "allow", + "glob": "allow", + "grep": "allow", + "list": "allow", + "edit": "deny", + "bash": { + "*": "deny", + "ls *": "allow", + "cat *": "allow", + "grep *": "allow", + "find *": "allow" + }, + "external_directory": "deny", + "skill": { "*": "ask" } + } +} +``` + +### CI / headless + +```json +{ + "permission": { + "*": "allow", + "external_directory": "deny", + "doom_loop": "deny" + } +} +``` + +`ask` becomes effectively `deny` in headless mode (no UI to prompt). + +## Common gotchas + +- `bash: { "git push*": "allow" }` covers `git push --force` because `*` is + greedy. Always pair with a stricter `"git push --force*": "deny"` rule. +- Object-style permission overrides **replace**, not merge. If global allows + `git diff*` and per-agent says `{ "*": "ask" }`, the `git diff*` rule is gone. +- `ask` outside a TUI → blocks indefinitely or fails depending on caller. + Never use `ask` for CI runs. +- Custom tools live alongside built-ins in the same permission namespace — + control with `tools: { my-tool: false }` or `permission: { my-tool: "ask" }`. +- A skill that calls `bash` is gated by **both** `permission.skill` and + `permission.bash` — the stricter wins for that operation. diff --git a/personas/_shared/community-skills/opencode-cli/references/providers-models.md b/personas/_shared/community-skills/opencode-cli/references/providers-models.md new file mode 100644 index 0000000..7f0443e --- /dev/null +++ b/personas/_shared/community-skills/opencode-cli/references/providers-models.md @@ -0,0 +1,223 @@ +# Providers, Models, OpenCode Zen & MCP + +## Provider auth flow + +1. `/connect` (in TUI) — picks provider, prompts for key, writes + `~/.local/share/opencode/auth.json`. Or: `opencode auth login`. +2. Or set provider-specific env var (e.g. `ANTHROPIC_API_KEY`, + `OPENAI_API_KEY`). +3. Or set in `opencode.json` → `provider..apiKey` (use + `{file:~/.secrets/...}` to avoid plaintext keys). + +> Config-file keys take precedence over env vars. + +## Setting models + +```json +{ "$schema": "https://opencode.ai/config.json", + "model": "anthropic/claude-sonnet-4-5", + "small_model": "anthropic/claude-haiku-4-5" } +``` + +Resolution: `--model`/`-m` CLI flag → `opencode.json` → last used → first by +internal priority. + +Switch interactively: `/models`. List available: `opencode models`. Optional +provider filter: `opencode models anthropic`. + +## Reasoning-effort variants + +Append to id (e.g. `openai/gpt-5/high`): + +| Provider | Variants | +| --------- | -------------------------------------- | +| Anthropic | `high` (default), `max` | +| OpenAI | `none`, `minimal`, `low`, `medium`, `high`, `xhigh` | +| Google | `low`, `high` | + +Override or add via `provider..models..variants`. + +## Major providers (preloaded) + +- Anthropic (`anthropic/...`) +- OpenAI (`openai/...`) +- Google Vertex AI (`google/...`) +- Amazon Bedrock +- GitHub Copilot +- GitLab Duo +- OpenCode Zen (`opencode/...`) +- Local: Ollama, LM Studio, llama.cpp + +75+ supported via Vercel AI SDK + Models.dev. + +## OpenCode Zen + +Curated gateway. Models prefixed `opencode/`. Setup: `/connect` → "OpenCode +Zen" → paste API key from `opencode.ai/auth`. + +40+ models incl. Claude Opus/Sonnet/Haiku, GPT-5/Codex variants, Qwen, Gemini, +MiniMax, GLM, free-tier betas. + +Pricing: pay-as-you-go per 1M tokens. Claude Opus ~$5–25 per 1M (in/out). +Cached reads/writes cheaper. Auto-reload $20 when balance < $5 (adjustable, +disable-able). + +Team features: roles (Admin/Member), per-member spend limits, model +allow/deny workspace-wide, BYOK for OpenAI/Anthropic. + +Privacy: US-hosted. Most paid models zero-retention. Free-tier betas may +retain. Some NVIDIA endpoints log prompts. + +## Custom OpenAI-compatible provider + +For Ollama, LM Studio, OpenRouter, Groq, Together, vLLM, or anything +OpenAI-API-compatible: + +```json +{ + "$schema": "https://opencode.ai/config.json", + "provider": { + "ollama-local": { + "npm": "@ai-sdk/openai-compatible", + "name": "Ollama", + "baseURL": "http://localhost:11434/v1", + "models": { + "qwen2.5-coder:32b": { "name": "Qwen 2.5 Coder 32B" }, + "llama3.3:70b": { "name": "Llama 3.3 70B" } + } + }, + "lmstudio": { + "npm": "@ai-sdk/openai-compatible", + "baseURL": "http://localhost:1234/v1", + "models": { "google/gemma-3n-e4b": {} } + }, + "openrouter": { + "npm": "@ai-sdk/openai-compatible", + "baseURL": "https://openrouter.ai/api/v1", + "apiKey": "{env:OPENROUTER_API_KEY}", + "models": { + "anthropic/claude-3.5-sonnet": {}, + "deepseek/deepseek-r1": {} + } + } + }, + "model": "ollama-local/qwen2.5-coder:32b" +} +``` + +Then run `/connect` → "Other" if you want the `auth.json` flow to remember the +provider id, or just rely on the inline config. + +## Provider-level options + +```json +{ "provider": { + "anthropic": { + "timeout": 600000, + "options": { "cacheControl": true }, + "models": { "claude-sonnet-4-5": { "name": "Sonnet 4.5" } } + } +} } +``` + +`disabled_providers` / `enabled_providers` at top level allow/blocklist +entire providers (useful in restricted enterprise environments). + +## MCP servers + +Schema: +```json +{ "mcp": { "": { + "type": "local" | "remote", + "enabled": true, + "timeout": 5000, + /* local: */ "command": ["..."], "environment": {}, + /* remote: */ "url": "https://...", "headers": {}, "oauth": {} +} } } +``` + +### Local (stdio) + +```json +{ "mcp": { + "filesystem": { + "type": "local", + "command": ["npx", "-y", "@modelcontextprotocol/server-filesystem", "/tmp", "/srv/data"], + "environment": {}, + "enabled": true, + "timeout": 5000 + } +} } +``` + +### Remote (HTTP / SSE) + +```json +{ "mcp": { + "context7": { + "type": "remote", + "url": "https://mcp.context7.com/mcp", + "headers": { "CONTEXT7_API_KEY": "{env:CONTEXT7_API_KEY}" }, + "enabled": true + }, + "sentry": { + "type": "remote", + "url": "https://mcp.sentry.dev/mcp", + "oauth": {} + }, + "gh_grep": { + "type": "remote", + "url": "https://mcp.grep.app" + } +} } +``` + +### OAuth flow + +```bash +opencode mcp auth # complete browser auth +opencode mcp list # see auth status +opencode mcp logout +opencode mcp debug # diagnose +``` + +To bypass OAuth on a server that uses raw API keys: `"oauth": false`. +With env-injected creds: +```json +{ "oauth": { + "clientId": "{env:CLIENT_ID}", + "clientSecret": "{env:CLIENT_SECRET}", + "scope": "tools:read tools:execute" +} } +``` + +### Tool gating + +MCP tools register under their server name. Gate them like normal tools: + +Globally disable a server's tools: +```json +{ "tools": { "filesystem*": false } } +``` + +Enable only for one agent: +```json +{ "agent": { "explorer": { "tools": { "filesystem*": true } } } } +``` + +Glob: `*` (any), `?` (one). MCP tools cost context tokens — only enable per-agent +where actually needed. + +## Pitfalls + +- Setting `apiKey` inline in `opencode.json` then committing the file leaks + the key. Always use `{env:...}` or `{file:...}`. +- `enabled_providers` is an **allowlist**: setting it disables every provider + not listed, including ones the user has authed via `/connect`. +- OAuth-secured MCP servers fail silently if the cached token expires — + re-run `opencode mcp auth` rather than restarting. +- Local MCP servers that block on `npx -y` first-run can blow past the 5000ms + default `timeout` — bump it to 15000 for first launch. +- Mixing `model: "openai/gpt-5/high"` with a `provider.openai.models.gpt-5` + override that doesn't define `variants.high` resolves to the bare model + (silently drops `high`). diff --git a/personas/_shared/community-skills/opencode-cli/references/server-api.md b/personas/_shared/community-skills/opencode-cli/references/server-api.md new file mode 100644 index 0000000..01c48ec --- /dev/null +++ b/personas/_shared/community-skills/opencode-cli/references/server-api.md @@ -0,0 +1,426 @@ +# `opencode serve` — HTTP API Reference + +`opencode serve` runs the same backend the TUI uses, headless, with an OpenAPI +3.1 surface. Multiple clients (TUI, IDE extension, SDK, curl) can attach to one +instance simultaneously. + +> Source: . The OpenAPI spec is the source of +> truth for shapes — fetch it from `/doc` on a running instance to see field +> types. + +## 1. Startup + +``` +opencode serve [--port ] [--hostname ] [--cors ] [--mdns] [--mdns-domain ] +``` + +| Flag | Default | Notes | +| ---------------- | -------------- | ---------------------------------------- | +| `--port` | `4096` | Listen port | +| `--hostname` | `127.0.0.1` | Bind address. Set `0.0.0.0` for LAN. | +| `--cors` | none | Repeatable; allowlists browser origins | +| `--mdns` | off | Advertise on local network | +| `--mdns-domain` | `opencode.local` | mDNS service name | + +> When you run plain `opencode`, the TUI launches and a server starts in-process. +> The TUI is just one of its clients. `opencode serve` is "TUI-less mode." + +## 2. Authentication + +HTTP Basic Auth. **Always set a password — without one the server is open.** + +```bash +export OPENCODE_SERVER_PASSWORD="$(openssl rand -hex 24)" +export OPENCODE_SERVER_USERNAME="opencode" # default; optional +opencode serve --port 4096 +``` + +Applies to both `serve` and `web` commands. All endpoints require auth except +the static OpenAPI viewer at `/doc` (the spec itself is fetchable for SDK +generation). + +curl pattern: +```bash +curl -u "$OPENCODE_SERVER_USERNAME:$OPENCODE_SERVER_PASSWORD" \ + http://127.0.0.1:4096/global/health +``` + +## 3. CORS + +Repeat `--cors` for each browser origin. Anything not listed is rejected. + +```bash +opencode serve \ + --cors http://localhost:5173 \ + --cors https://app.example.com +``` + +Same-origin clients (other CLIs, the TUI, the SDK) don't need this — CORS only +gates browsers. + +## 4. mDNS + +``` +opencode serve --mdns --mdns-domain my-team-opencode.local +``` + +Useful for IDE extensions/SDKs that auto-discover a running instance on the +LAN without a hardcoded URL. Don't enable in untrusted networks. + +## 5. OpenAPI + +Live spec: `http://:/doc`. Open in Swagger UI or feed to +`openapi-typescript-codegen` to regenerate clients. The official TS SDK +(`@opencode-ai/sdk`) is auto-generated from this spec. + +## 6. Endpoint catalog + +All paths relative to the server's root URL. + +### 6.1. Global + +| Method | Path | Purpose | +| ------ | ---------------- | ----------------------------------------- | +| GET | `/global/health` | `{ healthy: true, version: string }` | +| GET | `/global/event` | SSE stream; first event `server.connected` | + +### 6.2. Project / environment + +| Method | Path | Purpose | +| ------ | ----------------- | ---------------------------------- | +| GET | `/project` | List all projects | +| GET | `/project/current`| Active project detail | +| GET | `/path` | Working directory of the server | +| GET | `/vcs` | VCS info for the active project | + +### 6.3. Configuration + +| Method | Path | Purpose | +| ------ | ------------------- | -------------------------------------------- | +| GET | `/config` | Effective merged config | +| PATCH | `/config` | Patch settings live | +| GET | `/config/providers` | `{ providers: Provider[], default: {…} }` | + +### 6.4. Providers & auth + +| Method | Path | Purpose | +| ------ | ------------------------------------- | -------------------------------------- | +| GET | `/provider` | All / default / connected providers | +| GET | `/provider/auth` | Auth methods per provider | +| POST | `/provider/{id}/oauth/authorize` | Start OAuth — returns authorize URL | +| POST | `/provider/{id}/oauth/callback` | Body `{ code, state }` → `boolean` | +| PUT | `/auth/:id` | Set raw credentials (provider schema) | + +### 6.5. Sessions (core) + +| Method | Path | Purpose | +| ------ | ----------------------------------------------- | ------- | +| GET | `/session` | List sessions | +| POST | `/session` | Create. Body `{ parentID?, title? }` | +| GET | `/session/status` | Status map of all sessions | +| GET | `/session/:id` | Fetch session | +| PATCH | `/session/:id` | Update title (`{ title? }`) | +| DELETE | `/session/:id` | Delete session + data | +| GET | `/session/:id/children` | Forks of this session | +| GET | `/session/:id/todo` | Session todo list | +| POST | `/session/:id/init` | Run `/init`. Body `{ messageID, providerID, modelID }` | +| POST | `/session/:id/fork` | Branch at message. Body `{ messageID? }` | +| POST | `/session/:id/abort` | Stop a running session | +| POST | `/session/:id/share` | Enable share — returns share token | +| DELETE | `/session/:id/share` | Disable share | +| GET | `/session/:id/diff?messageID=` | File diffs | +| POST | `/session/:id/summarize` | `{ providerID, modelID }` | +| POST | `/session/:id/revert` | `{ messageID, partID? }` | +| POST | `/session/:id/unrevert` | Restore reverts | +| POST | `/session/:id/permissions/:permissionID` | `{ response, remember? }` — answer permission prompts | + +### 6.6. Messages / prompts + +| Method | Path | Purpose | +| ------ | ------------------------------------------------- | ------- | +| GET | `/session/:id/message?limit=` | Paginated messages with parts | +| POST | `/session/:id/message` | Send and **wait** for reply | +| GET | `/session/:id/message/:messageID` | `{ info, parts }` | +| POST | `/session/:id/prompt_async` | Send and return `204` (consume reply via SSE) | +| POST | `/session/:id/command` | Run slash command. Body `{ messageID?, agent?, model?, command, arguments }` | +| POST | `/session/:id/shell` | Run a raw shell. Body `{ agent, model?, command }` | + +POST `/message` body fields (the canonical prompt envelope): +```jsonc +{ + "messageID": "optional-client-side-id", + "model": "anthropic/claude-sonnet-4-5", // optional override + "agent": "build", // optional + "system": "extra system text", // optional + "tools": ["read", "edit", "bash"], // optional allowlist + "noReply": false, // true ⇒ no model call, just append + "parts": [ + { "type": "text", "text": "summarize README.md" }, + { "type": "file", "path": "README.md" } + ] +} +``` + +### 6.7. Commands + +| Method | Path | Purpose | +| ------ | ----------- | ------- | +| GET | `/command` | List slash commands available in this project | + +### 6.8. Files & search + +| Method | Path | Purpose | +| ------ | ------------------------------------ | ------- | +| GET | `/find?pattern=` | Regex content search; returns paths + lines + offsets + submatches | +| GET | `/find/file?query=&type=&directory=&limit=<1-200>` | Fuzzy file/dir search | +| GET | `/find/symbol?query=` | Workspace symbol search (LSP-backed) | +| GET | `/file?path=` | Directory listing as `FileNode[]` | +| GET | `/file/content?path=

` | Read file content | +| GET | `/file/status` | VCS status of tracked files | + +### 6.9. Tools (experimental) + +| Method | Path | Purpose | +| ------ | ------------------------------------------- | ------- | +| GET | `/experimental/tool/ids` | All tool IDs | +| GET | `/experimental/tool?provider=

&model=` | Tools + JSON schemas for a model | + +### 6.10. LSP / formatters / MCP + +| Method | Path | Purpose | +| ------ | ------------ | ------- | +| GET | `/lsp` | Status of every language server | +| GET | `/formatter` | Status of every formatter | +| GET | `/mcp` | Per-server MCP status | +| POST | `/mcp` | Add an MCP server live: `{ name, config }` | + +### 6.11. Agents + +| Method | Path | Purpose | +| ------ | -------- | ------------------------ | +| GET | `/agent` | List available agents | + +### 6.12. Logging + +| Method | Path | Purpose | +| ------ | ------ | ------- | +| POST | `/log` | `{ service, level, message, extra? }` — push entry into server log | + +### 6.13. TUI control (used by IDE extensions) + +| Method | Path | Purpose | +| ------ | -------------------------- | ------- | +| POST | `/tui/append-prompt` | Append text to prompt buffer | +| POST | `/tui/clear-prompt` | Clear prompt buffer | +| POST | `/tui/submit-prompt` | Send the buffer as a prompt | +| POST | `/tui/open-help` | Show help dialog | +| POST | `/tui/open-sessions` | Show session selector | +| POST | `/tui/open-themes` | Show theme selector | +| POST | `/tui/open-models` | Show model selector | +| POST | `/tui/execute-command` | `{ command }` | +| POST | `/tui/show-toast` | `{ title?, message, variant }` | +| GET | `/tui/control/next` | Long-poll for next control request | +| POST | `/tui/control/response` | `{ body }` — answer the control request | + +### 6.14. Instance lifecycle + +| Method | Path | Purpose | +| ------ | ------------------- | ----------------------- | +| POST | `/instance/dispose` | Shut the instance down | + +### 6.15. Events (SSE) + +| Method | Path | Purpose | +| ------ | ----------- | ------- | +| GET | `/event` | Typed event bus (sessions, messages, tool calls, permissions). First frame: `server.connected` | +| GET | `/global/event` | Global event bus (instance-wide signals) | + +Subscribe via standard SSE: `text/event-stream`, one JSON-encoded event per +line. Use this with `/session/:id/prompt_async` to drive prompts without +blocking on POST `/message`. + +## 7. Permission flow + +When an agent hits a tool guarded by `ask`, the server raises a permission +event on `/event`. The client must POST to +`/session/:id/permissions/:permissionID`: +```json +{ "response": "allow", "remember": true } +``` +`response` ∈ `allow | deny | ask`. `remember: true` persists the answer for the +session. + +In headless mode without a client answering, the request hangs until aborted. +For CI, set the relevant permissions to `allow`/`deny` upfront — never `ask`. + +## 8. Curl recipes + +All examples assume: +```bash +HOST=127.0.0.1:4096 +AUTH=opencode:$OPENCODE_SERVER_PASSWORD +``` + +Health check: +```bash +curl -u "$AUTH" "http://$HOST/global/health" +``` + +Create a session and send one message synchronously: +```bash +SID=$(curl -s -u "$AUTH" -X POST "http://$HOST/session" \ + -H 'content-type: application/json' \ + -d '{"title":"audit-readme"}' | jq -r .id) + +curl -u "$AUTH" -X POST "http://$HOST/session/$SID/message" \ + -H 'content-type: application/json' \ + -d '{ + "model": "anthropic/claude-sonnet-4-5", + "agent": "build", + "parts": [{"type":"text","text":"Audit README.md for broken links."}] + }' +``` + +Async send + tail SSE: +```bash +# Terminal A — subscribe +curl -u "$AUTH" -N "http://$HOST/event" + +# Terminal B — fire and forget +curl -u "$AUTH" -X POST "http://$HOST/session/$SID/prompt_async" \ + -H 'content-type: application/json' \ + -d '{"agent":"build","parts":[{"type":"text","text":"List repo TODOs."}]}' +``` + +Run a slash command via API: +```bash +curl -u "$AUTH" -X POST "http://$HOST/session/$SID/command" \ + -H 'content-type: application/json' \ + -d '{"command":"test","arguments":["unit"]}' +``` + +Add an MCP server live (no restart): +```bash +curl -u "$AUTH" -X POST "http://$HOST/mcp" \ + -H 'content-type: application/json' \ + -d '{ + "name":"context7", + "config":{ + "type":"remote", + "url":"https://mcp.context7.com/mcp", + "headers":{"CONTEXT7_API_KEY":"'"$CONTEXT7_API_KEY"'"} + } + }' +``` + +Drive a remote TUI (toast + submit a prompt): +```bash +curl -u "$AUTH" -X POST "http://$HOST/tui/show-toast" \ + -H 'content-type: application/json' \ + -d '{"variant":"info","message":"Build kicking off"}' + +curl -u "$AUTH" -X POST "http://$HOST/tui/append-prompt" \ + -H 'content-type: application/json' \ + -d '{"text":"explain the failing test in src/foo.test.ts"}' + +curl -u "$AUTH" -X POST "http://$HOST/tui/submit-prompt" +``` + +Symbol search: +```bash +curl -u "$AUTH" "http://$HOST/find/symbol?query=parseConfig" +``` + +File search (paths only): +```bash +curl -u "$AUTH" "http://$HOST/find/file?query=user&type=file&limit=50" +``` + +Regex content search: +```bash +curl -u "$AUTH" "http://$HOST/find?pattern=TODO%5Cs%2A%3A" +``` + +Read file content: +```bash +curl -u "$AUTH" "http://$HOST/file/content?path=src/index.ts" +``` + +Shut the server down cleanly: +```bash +curl -u "$AUTH" -X POST "http://$HOST/instance/dispose" +``` + +## 9. SDK shortcut + +```typescript +import { createOpencodeClient } from "@opencode-ai/sdk" + +const client = createOpencodeClient({ + baseURL: "http://127.0.0.1:4096", + auth: { user: "opencode", pass: process.env.OPENCODE_SERVER_PASSWORD! }, +}) + +const session = await client.session.create({ title: "ci-run" }) +const reply = await client.session.message({ + sessionID: session.id, + agent: "build", + parts: [{ type: "text", text: "summarize README.md" }], +}) + +client.events.subscribe(session.id, (e) => console.log(e.type, e.payload)) +``` + +Types are generated from `/doc`, so the client's surface mirrors the endpoint +catalog above 1:1. + +## 10. Architecture & integration patterns + +- **TUI-as-client**: `opencode` (no `serve`) starts both the server and the + TUI client; the TUI talks to the local server over the same HTTP API. +- **IDE extensions**: VS Code/Cursor/Windsurf/Codium attach to a running + instance and drive the running TUI via `/tui/*` endpoints (prefill prompt, + submit, watch results via `/event`). +- **CI / batch**: spawn `opencode serve` once at job start, fire off prompts + with `/session` + `/prompt_async`, drain `/event`, dispose at end. +- **Multi-user**: don't. Sessions are per-user state with no built-in tenant + isolation. For team servers use a per-user gateway / proxy that pins each + user to their own server instance. +- **Live config edit**: `PATCH /config` lets you flip `model`, `permission`, + `mcp`, etc. without restart — handy for ops dashboards. + +## 11. Security checklist + +- [ ] `OPENCODE_SERVER_PASSWORD` is set and not a default +- [ ] `--hostname 127.0.0.1` unless intentionally LAN-exposed +- [ ] `--cors` lists only specific origins (no `*`) +- [ ] `--mdns` off in shared/untrusted networks +- [ ] `permission.bash`, `permission.edit`, `permission.external_directory` + explicitly set — `ask` doesn't work headless +- [ ] `share: "disabled"` for projects with sensitive code +- [ ] `OPENCODE_DISABLE_CLAUDE_CODE_SKILLS=1` if you don't want + `~/.claude/skills/` auto-loaded into server-mode sessions +- [ ] Front the server with TLS (nginx/caddy) if remoting it; the built-in + listener does plain HTTP +- [ ] `/instance/dispose` is reachable from any authed client — anyone with + the password can shut your CI server down + +## 12. Pitfalls + +- Setting `OPENCODE_SERVER_PASSWORD` after start has no effect — set it before + `serve`. +- Without auth env vars, the server still starts but is unauthenticated. + **Bug-shaped, not a feature.** Always set the password. +- `POST /session/:id/message` blocks until the model finishes. For long runs + use `prompt_async` + `/event`. +- `POST /tui/submit-prompt` triggers the **currently running TUI**. If no TUI + is attached (pure `serve`), it no-ops silently. +- `PATCH /config` partial-merges top-level keys but **replaces** nested objects + (`permission.bash`, etc.) — same gotcha as the file-based config. +- The `/find` regex is server-side; URL-encode special chars (`%5C` for `\`, + `%2A` for `*`, `%2B` for `+`). Forgetting to encode silently drops chars. +- `POST /mcp` adds a server but doesn't auto-`opencode mcp auth` it — you + still need to complete OAuth out-of-band. +- `OPENCODE_API_KEY` and provider-specific keys must be set in the **server's** + environment, not the client's. The client just talks to the server; the + server makes outbound model calls. diff --git a/personas/_shared/community-skills/pentest-reporter/SKILL.md b/personas/_shared/community-skills/pentest-reporter/SKILL.md new file mode 100644 index 0000000..10cf4b0 --- /dev/null +++ b/personas/_shared/community-skills/pentest-reporter/SKILL.md @@ -0,0 +1,225 @@ +--- +name: pentest-reporter +description: Generate professional -format pentest reports following the canonical "Sızma Testi Raporu" template (Turkish defense-client schema). Triggers on pentest report, sızma testi raporu, write report, scan results, findings, executive summary, müşteri raporu, final pentest, rapor hazırla. +--- + +# Pentest Reporter — Canonical Schema + +**Canonical template** (must be followed exactly): +`//Sizma_Testi_Örnek.docx` +(Master archived in any client engagement folder; backup with date suffix before edit.) + +This is the **single source of truth** for all deliverables. Schema established 2026-04-20 (Aras EDAŞ engagement). Every Turkish-defense-client report MUST mirror this structure 1:1 — sections, table layouts, finding-card format, fonts (Roboto + RobotoMono), colour bands. + +## Report Schema (21 tables, 9 sections) + +``` +COVER PAGE + ├── "Sızma Testi Raporu" (title) + └── "GİZLİ" + dağıtım uyarısı (red banner) + +SECTION 1 — Gizlilik Beyanı (Confidentiality Declaration) +SECTION 2 — Önemli Uyarı (Important Notice) +SECTION 3 — Yasal Sorumluluklar (Legal Responsibilities) +SECTION 4 — İletişim Bilgileri ve Test Ekibi + └── TABLE 0: Kişi Adı | Ünvan | Sertifikalar (2 rows) +SECTION 5 — Doküman ve Sürüm Kontrolü + ├── TABLE 1: Doküman Kontrolü (11 rows × 2 cols) + │ [Doküman Başlığı, Doküman No, İlk Yayın Tarihi, Revizyon No, + │ Son Revizyon Tarihi, Toplam Sayfa Sayısı, Hazırlayan, Onaylayan, + │ Sınıflandırma, Geçerlilik, Dağıtım] + └── TABLE 2: Sürüm Kontrolü (Sürüm | Tarih | Hazırlayan | Açıklama) +SECTION 6 — İçindekiler (auto TOC) +SECTION 7 — Çalışmaların Özeti (3-paragraph intro: scope, 3-phase methodology, value) +SECTION 8 — Test Metodolojisi + ├── 8.1 Genel Süreç (Planlama→Keşif→Tarama→Zafiyet Analizi→İstismar→Raporlama→Doğrulama) + ├── 8.2 Dış Ağ Testleri + ├── 8.3 Yerel Ağ Testleri + ├── 8.4 Kablosuz Ağ Testleri + ├── 8.5 Sosyal Mühendislik Testleri + └── 8.6 Hizmet Reddi (DDoS) Testleri +SECTION 9 — Risk Ölçeği + ├── TABLE 3: Risk Seviyesi → Açıklama (6 rows: Acil/Kritik/Yüksek/Orta/Düşük/Bilgi) + ├── 9.1 Risk Değerlendirmesi + ├── 9.2 İstismar Edilme İhtimali (4 faktör) + ├── 9.3 Etki (Gizlilik/Bütünlük/Erişilebilirlik) + ├── TABLE 4: Kapsam (Sistem Adı | Erişim Bilgisi | Kullanıcı | Nokta) + └── TABLE 5: Sızma Testi Ekibi IP Adresleri (Sistem | Erişim Bilgisi | Erişim Noktası) +SECTION 10 — Kuruluş Desteği (teşekkür paragrafı) +SECTION 11 — Yönetici Özeti + ├── Hitap paragrafı ("[Müşteri] Yetkilisinin Dikkatine;") + ├── Tarih + bulgu sayı özeti ("X kritik, Y yüksek, Z orta, W düşük") + ├── 11.1 Güvenlik Açıkları Kategorisel Dağılımı + │ └── TABLE 6: Risk Dağılımı (3 rows × 6 cols: Risk|Kritik|Yüksek|Orta|Düşük|Toplam) + ├── 11.2 Tavsiye Özeti + ├── 11.3 Acil Eylem Planı (5 madde: Önceliklendirme, Web/Sistem İyileştirme, Politika+Eğitim, İzleme, Yedekleme) + └── 11.4 Güvenlik Açıkları Listelenmesi + └── TABLE 7: Bulgu Listesi (Bulgu Adı | Risk Seviyesi | Bulgu Kategorisi) +SECTION 12 — Keşif Çalışmaları + ├── DNS/IP/Kullanıcı keşfi paragrafı + ├── Tespit Edilen Aktif Host Listesi + │ ├── 12.1 Dahili / AD Altyapısı + │ │ └── TABLE 8 (IP | Rol/Hizmet Notları | Kaynak) + │ ├── 12.2 İş İstasyonları ve SMB Sunucuları + │ │ └── TABLE 9 (IP | Belirlenen Host Adı / Rol | Kaynak) + │ ├── 12.3 VoIP / Güvenlik Cihazları + │ │ └── TABLE 10 (IP Adresleri | Ortak Hizmetler/Rol | Kaynak) + │ └── 12.4 Diğer Alt Ağlar + │ └── TABLE 11 (IP | Açıklama | Kaynak) +SECTION 13 — İç Ağ Testleri (Findings I1..In) + ├── 13.0 Test Çalışmalarının Özeti (Black-Box, OWASP+NIST, araç güncelliği, Acil Eylem) + └── For each finding I: + ├── Heading 2: "I – " + ├── Heading 3: "Bulgu Kanıtı:" + ├── Komut/çıktı bloğu (kod biçemi, RobotoMono) + └── TABLE 12+: Bulgu Kartı (5 rows × 2 cols) + R0: Seviye | KRİTİK / YÜKSEK / ORTA / DÜŞÜK / BİLGİ + R1: Açıklama Ve Etki| Tanım: ... | Etki: ... + R2: Referans | URL/CVE/CWE listesi + R3: Çözüm Önerisi | Birincil + İkincil çözüm + R4: Etkilenen Sistemler | IP listesi +SECTION 14 — Dış Ağ Testleri (varsa, aynı I-card formatı, prefix "E") +SECTION 15 — Web Uygulama Testleri (varsa, "W") +SECTION 16 — Raporda Geçen Teknik Terimler + └── TABLE 18: Terim | Açıklama (22 rows std) +SECTION 17 — Raporda Geçen Kısaltmalar + └── TABLE 19: Kısaltma | Açılım | Türkçesi (57 rows std) +SECTION 18 — Testler Esnasında Kullanılan Araçlar + └── TABLE 20: Aracın İsmi | Kategori | Web Adresi (81 rows std) +``` + +## Severity Color Bands (cell fill in finding cards) + +| Seviye | Hex | Use | +|--------|-----|-----| +| ACİL | `#7B1F1F` (dark red) | Reserved for active malware/C2 | +| KRİTİK | `#C0392B` (red) | RCE, full domain compromise, DA on EOL DC | +| YÜKSEK | `#E67E22` (orange) | Privesc, kerberoast DA, unconstrained delegation | +| ORTA | `#F1C40F` (yellow) | Info disclosure, weak crypto, missing headers | +| DÜŞÜK | `#3498DB` (blue) | RPC info, SSL pref, expired-cert non-prod | +| BİLGİ | `#7F8C8D` (gray) | Version banner, open port, OS guess | + +## CVSS Mapping + +| CVSS 3.1 | Seviye | +|----------|--------| +| 9.0–10.0 | KRİTİK | +| 7.0–8.9 | YÜKSEK | +| 4.0–6.9 | ORTA | +| 0.1–3.9 | DÜŞÜK | +| 0.0 | BİLGİ | + +(ACİL is a qualitative escalation — used only for **active intrusion evidence** like C2 beacons / live worms detected during the scan.) + +## Document Control Defaults + +| Alan | Varsayılan | +|------|-----------| +| Doküman Başlığı | " Sızma Testi Raporu" | +| Doküman No | "ST-" (engagement start date) | +| İlk Yayın Tarihi | engagement closing date (DD.MM.YYYY) | +| Revizyon No | "-" first issue, "1.1" / "1.2" sonrası | +| Hazırlayan | "" | +| Onaylayan | "< proje yöneticisi>" | +| Sınıflandırma | "GİZLİ" | +| Geçerlilik | "12 ay" (yeniden test gerektirir) | +| Dağıtım | "Müşteri ve sınırlı dağıtım" | + +## Sürüm Kontrolü Defaults + +``` +| Sürüm | Tarih | Hazırlayan | Açıklama | +| V1.0 | DD.MM.YYYY | | İlk yayın | +| V1.1 | DD.MM.YYYY | | | +``` + +## Workflow + +```dot +digraph reporter { + start [label="Müşteri raporu hazırla isteği"]; + read_template [label="1. Sizma_Testi_Örnek.docx oku\n (python-docx ile)"]; + collect_data [label="2. Tüm scan/log/rapor verileri topla\n (10+ paralel agent dispatch)"]; + classify [label="3. Bulguları KRİTİK/YÜKSEK/ORTA/DÜŞÜK/BİLGİ\n sınıflandır + CVSS ata"]; + build [label="4. Python jeneratör çalıştır\n (template klon + tablo doldur)"]; + manual [label="5. Word'de elle rötuş\n (renk bantları, başlık formatı, sayfa kırma)"]; + deliver [label="6. PDF export + GİZLİ damga + dağıtım"]; + + start -> read_template -> collect_data -> classify -> build -> manual -> deliver; +} +``` + +## Python Generator Pattern + +Use `python-docx` to clone the template and replace text + table cells, **never** rebuild from scratch. This preserves Roboto/RobotoMono fonts, theme colours, header/footer, and page numbering. + +```python +from docx import Document +from copy import deepcopy + +doc = Document('Sizma_Testi_Örnek.docx') + +# 1. Replace cover page client name +for p in doc.paragraphs: + if 'Soler Palau' in p.text or '[CLIENT]' in p.text: + for run in p.runs: + run.text = run.text.replace('Soler Palau', client_name) + +# 2. Update document control table (Table 1) +table_doc = doc.tables[1] +table_doc.cell(0, 1).text = f"{client_name} Sızma Testi Raporu" +table_doc.cell(1, 1).text = f"ST-{engagement_id}" +# ... per template default + +# 3. Replace risk distribution table (Table 6) +table_risk = doc.tables[6] +table_risk.cell(1, 1).text = str(critical_count) +# ... + +# 4. For each finding, deepcopy the I1 finding card table and append +template_card = deepcopy(doc.tables[12]) # I1 card as template +for finding in findings: + new_card = deepcopy(template_card) + new_card.cell(0, 1).text = finding['severity'] + new_card.cell(1, 1).text = f"Tanım: {finding['description']} | Etki: {finding['impact']}" + new_card.cell(2, 1).text = '\n'.join(finding['references']) + new_card.cell(3, 1).text = finding['remediation'] + new_card.cell(4, 1).text = '\n'.join(finding['affected_hosts']) + # Insert after I_N heading paragraph + insert_table_after(doc, new_card, anchor_para) + +doc.save(f'{client_name}_Sizma_Testi_Raporu_{date}.docx') +``` + +## Checklist (must-pass before delivery) + +- [ ] Template original yedeklendi (`_BACKUP_.docx`) +- [ ] Cover page müşteri ismi değiştirildi (Soler Palau → ) +- [ ] "GİZLİ" banner korundu, dağıtım uyarısı korundu +- [ ] Sections 1-3 (Gizlilik/Uyarı/Yasal) DEĞİŞMEDEN korundu — hukuki standart metni +- [ ] Doküman Kontrolü tablosu güncel (yeni doküman no, tarih, sayfa sayısı) +- [ ] Sürüm Kontrolü tablosu V1.0 satırı eklendi +- [ ] Yönetici Özeti hitabı doğru müşteri yetkilisine yazıldı +- [ ] Risk Dağılımı tablosu doğru sayılarla dolduruldu +- [ ] Tavsiye Özeti + Acil Eylem Planı standart 5-madde + müşteri-özel cümle +- [ ] Bulgu Listesi tablosu (TABLE 7) tüm I-card sayısı kadar satır +- [ ] Her finding için bulgu kartı (5x2) doldurulmuş +- [ ] Bulgu Kanıtı bölümü RAW komut+çıktı (RobotoMono kod bloğu) +- [ ] IP adresleri / hostnames / hash'ler verbatim — değiştirilmemiş +- [ ] Renk bantları severity'e göre uygulanmış +- [ ] İçindekiler güncellenmiş (Word: F9 ile refresh) +- [ ] Header/footer logosu + sayfa no çalışıyor +- [ ] PDF export "GİZLİ" damgası + dağıtım kuyruğu hazır + +## Reference Engagements + +- **Aras EDAŞ (2026-04)** — first engagement using this canonical schema. 4 KRİTİK + 10 YÜKSEK + 8 ORTA + 8 DÜŞÜK/BİLGİ. Final report path: `scans/aras/05_reports/ARAS_FINAL_PENTEST_REPORT_.docx`. Generator: `scans/aras/06_raw/scripts/generate_aras_report.py`. +- **Soler Palau (2025-10)** — original template source. 3 KRİTİK + 0 YÜKSEK + 2 ORTA + 1 DÜŞÜK = 6 bulgu örneği. + +## Integration Pointers + +- **Kill Chain Scanner DB** (MariaDB `killchain-mariadb`) — bulgular `findings` tablosundan çekilir, severity + cvss + cwe + remediation alanları doğrudan map edilir +- **Burp Suite Pro v2026.2.1** — XML export `--burp-import` ile parse, her issue bir I-card olur +- **Obsidian** — markdown ön-taslak `pentest reports/` vault'una yazılır, sonra DOCX üretilir +- **PDF export** — Word "Save As PDF" (PDF/A-3 önerilir; arşiv için) +- **Authorization log** — `db_log_authorization()` ile rapor üretim kaydı tutulur (PoC paylaşımı, dağıtım listesi) diff --git a/personas/_shared/community-skills/ui-ux-pro-max/SKILL.md b/personas/_shared/community-skills/ui-ux-pro-max/SKILL.md new file mode 100644 index 0000000..7079108 --- /dev/null +++ b/personas/_shared/community-skills/ui-ux-pro-max/SKILL.md @@ -0,0 +1,134 @@ +--- +name: ui-ux-pro-max +description: Use when building UI, choosing styles/colors/fonts, creating components, reviewing UX, fixing UI bugs, or making any frontend design decision. Triggers on new project UI, landing page, dashboard, component creation, style selection, dark mode, charts, responsive design, or any "make this look good" request. Searchable database of 67 UI styles, 161 product types, 57 font pairings, 161 color palettes, 99 UX guidelines. +--- + +# UI/UX Pro Max — AI Design Intelligence + +## Overview + +Searchable design knowledge base with BM25-ranked results across 10 domains. Use the Python search engine to get data-driven design recommendations before writing any frontend code. + +**Script location:** `~/.claude/skills/ui-ux-pro-max/scripts/search.py` +**Data location:** `~/.claude/skills/ui-ux-pro-max/data/` + +## When to Use + +| Scenario | Action | +|----------|--------| +| New project / page | `--design-system` for full recommendations | +| New component | `--domain style` or `--domain ux` | +| Style / color / font selection | `--design-system` or `--domain style/color/typography` | +| Review existing UI | `--domain ux` for best practices checklist | +| Fix a UI bug | `--domain ux` or `--domain web` | +| Dark mode | `--domain style "dark mode"` | +| Charts / data viz | `--domain chart` | +| Landing page | `--domain landing` | + +## Core Workflow + +### Step 1: Analyze Requirements + +Extract from user request: +- **Product type**: SaaS, fintech, creative tool, e-commerce, portfolio, enterprise, consumer +- **Target audience**: developers, consumers, enterprise, mixed +- **Style keywords**: minimal, dark, playful, professional, brutalist, glassmorphism +- **Stack**: React/Next.js, vanilla, Tailwind, etc. + +### Step 2: Generate Design System (REQUIRED for new projects) + +```bash +python3 ~/.claude/skills/ui-ux-pro-max/scripts/search.py " " --design-system [-p "Project Name"] +``` + +This runs 5 parallel domain searches (product, style, color, landing, typography), applies reasoning rules from `ui-reasoning.csv`, and returns a complete design system recommendation with anti-patterns. + +**Persist across sessions:** +```bash +python3 ~/.claude/skills/ui-ux-pro-max/scripts/search.py "" --design-system --persist -p "Project Name" +``` + +Creates `design-system/MASTER.md` + optional `design-system/pages/.md` overrides. + +### Step 3: Supplement with Domain Searches + +```bash +python3 ~/.claude/skills/ui-ux-pro-max/scripts/search.py "" --domain [-n ] +``` + +| Domain | Use For | Example | +|--------|---------|---------| +| `product` | Product type recommendations | `"SaaS dashboard analytics"` | +| `style` | UI styles, effects, aesthetics | `"glassmorphism dark mode"` | +| `color` | Color palettes by product type | `"fintech trust professional"` | +| `typography` | Font pairings, Google Fonts | `"elegant modern"` | +| `landing` | Page structure, CTA strategies | `"hero social-proof pricing"` | +| `chart` | Chart types, library recommendations | `"real-time dashboard trend"` | +| `ux` | Best practices, anti-patterns | `"animation accessibility loading"` | +| `react` | React/Next.js performance | `"suspense memo rerender cache"` | +| `web` | Web accessibility, touch targets | `"a11y focus-management"` | +| `prompt` | AI prompt / CSS keywords | `"minimalism"` | + +### Step 4: Output Formats + +```bash +# ASCII box (default) +python3 ~/.claude/skills/ui-ux-pro-max/scripts/search.py "fintech crypto" --design-system + +# Markdown (for documentation) +python3 ~/.claude/skills/ui-ux-pro-max/scripts/search.py "fintech crypto" --design-system -f markdown +``` + +## Data Inventory + +| File | Content | +|------|---------| +| `styles.csv` | 67 UI styles with colors, effects, best-for, anti-patterns | +| `products.csv` | 161 product types with recommended patterns | +| `ui-reasoning.csv` | 161 reasoning rules with JSON conditionals | +| `colors.csv` | 161 color palettes aligned to product types | +| `typography.csv` | 57 font pairings with Google Fonts imports | +| `design.csv` | Comprehensive design reference | +| `ux-guidelines.csv` | 99 UX best practices and anti-patterns | +| `charts.csv` | 25 chart types with library recommendations | +| `landing.csv` | 24 landing page patterns and CTA strategies | +| `icons.csv` | Icon library recommendations | +| `google-fonts.csv` | Full Google Fonts catalog | + +## Quick Reference Priorities + +### CRITICAL (Always Check) +- Contrast ratio >= 4.5:1 (text), >= 3:1 (large text/UI) +- Touch targets >= 44px, spacing >= 8px between +- `prefers-reduced-motion` respected for all animations +- Focus indicators visible, keyboard navigation complete + +### HIGH +- Max 4 font sizes, 2 weights per page +- Spacing from 8-point grid (multiples of 4px) +- All states designed: default, hover, active, focus, disabled, loading, error, empty +- Dark mode tested independently (not assumed from light mode) + +### MEDIUM +- Animation budget: 16ms/frame, spring physics preferred +- Z-index scale defined (no arbitrary values) +- Form validation inline, not after submit +- Navigation hierarchy max 2 levels + +## Integration with DESIGN.md + +This skill pairs with the awesome-design-md brand reference library (58 brands). Workflow: + +1. Use `--design-system` to determine style direction +2. Reference a similar brand's DESIGN.md for exact token values +3. Create project DESIGN.md with tokens from both sources +4. Build components using the token system + +## Pre-Delivery Checklist + +- [ ] `--domain ux "animation accessibility z-index loading"` validation pass +- [ ] Critical + High priority rules verified +- [ ] Test at 375px width and landscape orientation +- [ ] `prefers-reduced-motion` behavior verified +- [ ] Dark mode contrast checked independently +- [ ] All touch targets >= 44px, no content behind safe areas diff --git a/personas/_shared/community-skills/ui-ux-pro-max/data/_sync_all.py b/personas/_shared/community-skills/ui-ux-pro-max/data/_sync_all.py new file mode 100644 index 0000000..37f7c3a --- /dev/null +++ b/personas/_shared/community-skills/ui-ux-pro-max/data/_sync_all.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 +""" +Sync colors.csv and ui-reasoning.csv with the updated products.csv (161 entries). +- Remove deleted product types +- Rename mismatched entries +- Add new entries for missing product types +- Keep colors.csv aligned 1:1 with products.csv +- Renumber everything +""" +import csv, os, json + +BASE = os.path.dirname(os.path.abspath(__file__)) + +# ─── Color derivation helpers ──────────────────────────────────────────────── +def h2r(h): + h = h.lstrip("#") + return tuple(int(h[i:i+2], 16) for i in (0, 2, 4)) + +def r2h(r, g, b): + return f"#{max(0,min(255,int(r))):02X}{max(0,min(255,int(g))):02X}{max(0,min(255,int(b))):02X}" + +def lum(h): + r, g, b = [x/255.0 for x in h2r(h)] + r, g, b = [(x/12.92 if x<=0.03928 else ((x+0.055)/1.055)**2.4) for x in (r, g, b)] + return 0.2126*r + 0.7152*g + 0.0722*b + +def is_dark(bg): + return lum(bg) < 0.18 + +def on_color(bg): + return "#FFFFFF" if lum(bg) < 0.4 else "#0F172A" + +def blend(a, b, f=0.15): + ra, ga, ba = h2r(a) + rb, gb, bb = h2r(b) + return r2h(ra+(rb-ra)*f, ga+(gb-ga)*f, ba+(bb-ba)*f) + +def shift(h, n): + r, g, b = h2r(h) + return r2h(r+n, g+n, b+n) + +def derive_row(pt, pri, sec, acc, bg, notes=""): + """Generate full 16-token color row from 4 base colors.""" + dark = is_dark(bg) + fg = "#FFFFFF" if dark else "#0F172A" + on_pri = on_color(pri) + on_sec = on_color(sec) + on_acc = on_color(acc) + card = shift(bg, 10) if dark else "#FFFFFF" + card_fg = "#FFFFFF" if dark else "#0F172A" + muted = blend(bg, pri, 0.08) if dark else blend("#FFFFFF", pri, 0.06) + muted_fg = "#94A3B8" if dark else "#64748B" + border = f"rgba(255,255,255,0.08)" if dark else blend("#FFFFFF", pri, 0.12) + destr = "#DC2626" + on_destr = "#FFFFFF" + ring = pri + return [pt, pri, on_pri, sec, on_sec, acc, on_acc, bg, fg, card, card_fg, muted, muted_fg, border, destr, on_destr, ring, notes] + +# ─── Rename maps ───────────────────────────────────────────────────────────── +COLOR_RENAMES = { + "Quantum Computing": "Quantum Computing Interface", + "Biohacking / Longevity": "Biohacking / Longevity App", + "Autonomous Systems": "Autonomous Drone Fleet Manager", + "Generative AI Art": "Generative Art Platform", + "Spatial / Vision OS": "Spatial Computing OS / App", + "Climate Tech": "Sustainable Energy / Climate Tech", +} +UI_RENAMES = { + "Architecture/Interior": "Architecture / Interior", + "Autonomous Drone Fleet": "Autonomous Drone Fleet Manager", + "B2B SaaS Enterprise": "B2B Service", + "Biohacking/Longevity App": "Biohacking / Longevity App", + "Biotech/Life Sciences": "Biotech / Life Sciences", + "Developer Tool/IDE": "Developer Tool / IDE", + "Education": "Educational App", + "Fintech (Banking)": "Fintech/Crypto", + "Government/Public": "Government/Public Service", + "Home Services": "Home Services (Plumber/Electrician)", + "Micro-Credentials/Badges": "Micro-Credentials/Badges Platform", + "Music/Entertainment": "Music Streaming", + "Quantum Computing": "Quantum Computing Interface", + "Real Estate": "Real Estate/Property", + "Remote Work/Collaboration": "Remote Work/Collaboration Tool", + "Restaurant/Food": "Restaurant/Food Service", + "SaaS Dashboard": "Analytics Dashboard", + "Space Tech/Aerospace": "Space Tech / Aerospace", + "Spatial Computing OS": "Spatial Computing OS / App", + "Startup Landing": "Micro SaaS", + "Sustainable Energy/Climate": "Sustainable Energy / Climate Tech", + "Travel/Tourism": "Travel/Tourism Agency", + "Wellness/Mental Health": "Mental Health App", +} + +REMOVE_TYPES = { + "Service Landing Page", "Sustainability/ESG Platform", + "Cleaning Service", "Coffee Shop", + "Consulting Firm", "Conference/Webinar Platform", +} + +# ─── New color definitions: (primary, secondary, accent, bg, notes) ────────── +# Grouped by category for clarity. Each tuple generates a full 16-token row. +NEW_COLORS = { + # ── Old #97-#116 that never got colors ── + "Todo & Task Manager": ("#2563EB","#3B82F6","#059669","#F8FAFC","Functional blue + progress green"), + "Personal Finance Tracker": ("#1E40AF","#3B82F6","#059669","#0F172A","Trust blue + profit green on dark"), + "Chat & Messaging App": ("#2563EB","#6366F1","#059669","#FFFFFF","Messenger blue + online green"), + "Notes & Writing App": ("#78716C","#A8A29E","#D97706","#FFFBEB","Warm ink + amber accent on cream"), + "Habit Tracker": ("#D97706","#F59E0B","#059669","#FFFBEB","Streak amber + habit green"), + "Food Delivery / On-Demand": ("#EA580C","#F97316","#2563EB","#FFF7ED","Appetizing orange + trust blue"), + "Ride Hailing / Transportation":("#1E293B","#334155","#2563EB","#0F172A","Map dark + route blue"), + "Recipe & Cooking App": ("#9A3412","#C2410C","#059669","#FFFBEB","Warm terracotta + fresh green"), + "Meditation & Mindfulness": ("#7C3AED","#8B5CF6","#059669","#FAF5FF","Calm lavender + mindful green"), + "Weather App": ("#0284C7","#0EA5E9","#F59E0B","#F0F9FF","Sky blue + sun amber"), + "Diary & Journal App": ("#92400E","#A16207","#6366F1","#FFFBEB","Warm journal brown + ink violet"), + "CRM & Client Management": ("#2563EB","#3B82F6","#059669","#F8FAFC","Professional blue + deal green"), + "Inventory & Stock Management":("#334155","#475569","#059669","#F8FAFC","Industrial slate + stock green"), + "Flashcard & Study Tool": ("#7C3AED","#8B5CF6","#059669","#FAF5FF","Study purple + correct green"), + "Booking & Appointment App": ("#0284C7","#0EA5E9","#059669","#F0F9FF","Calendar blue + available green"), + "Invoice & Billing Tool": ("#1E3A5F","#2563EB","#059669","#F8FAFC","Navy professional + paid green"), + "Grocery & Shopping List": ("#059669","#10B981","#D97706","#ECFDF5","Fresh green + food amber"), + "Timer & Pomodoro": ("#DC2626","#EF4444","#059669","#0F172A","Focus red on dark + break green"), + "Parenting & Baby Tracker": ("#EC4899","#F472B6","#0284C7","#FDF2F8","Soft pink + trust blue"), + "Scanner & Document Manager": ("#1E293B","#334155","#2563EB","#F8FAFC","Document grey + scan blue"), + # ── A. Utility / Productivity ── + "Calendar & Scheduling App": ("#2563EB","#3B82F6","#059669","#F8FAFC","Calendar blue + event green"), + "Password Manager": ("#1E3A5F","#334155","#059669","#0F172A","Vault dark blue + secure green"), + "Expense Splitter / Bill Split":("#059669","#10B981","#DC2626","#F8FAFC","Balance green + owe red"), + "Voice Recorder & Memo": ("#DC2626","#EF4444","#2563EB","#FFFFFF","Recording red + waveform blue"), + "Bookmark & Read-Later": ("#D97706","#F59E0B","#2563EB","#FFFBEB","Warm amber + link blue"), + "Translator App": ("#2563EB","#0891B2","#EA580C","#F8FAFC","Global blue + teal + accent orange"), + "Calculator & Unit Converter": ("#EA580C","#F97316","#2563EB","#1C1917","Operation orange on dark"), + "Alarm & World Clock": ("#D97706","#F59E0B","#6366F1","#0F172A","Time amber + night indigo on dark"), + "File Manager & Transfer": ("#2563EB","#3B82F6","#D97706","#F8FAFC","Folder blue + file amber"), + "Email Client": ("#2563EB","#3B82F6","#DC2626","#FFFFFF","Inbox blue + priority red"), + # ── B. Games ── + "Casual Puzzle Game": ("#EC4899","#8B5CF6","#F59E0B","#FDF2F8","Cheerful pink + reward gold"), + "Trivia & Quiz Game": ("#2563EB","#7C3AED","#F59E0B","#EFF6FF","Quiz blue + gold leaderboard"), + "Card & Board Game": ("#15803D","#166534","#D97706","#0F172A","Felt green + gold on dark"), + "Idle & Clicker Game": ("#D97706","#F59E0B","#7C3AED","#FFFBEB","Coin gold + prestige purple"), + "Word & Crossword Game": ("#15803D","#059669","#D97706","#FFFFFF","Word green + letter amber"), + "Arcade & Retro Game": ("#DC2626","#2563EB","#22C55E","#0F172A","Neon red+blue on dark + score green"), + # ── C. Creator Tools ── + "Photo Editor & Filters": ("#7C3AED","#6366F1","#0891B2","#0F172A","Editor violet + filter cyan on dark"), + "Short Video Editor": ("#EC4899","#DB2777","#2563EB","#0F172A","Video pink on dark + timeline blue"), + "Drawing & Sketching Canvas": ("#7C3AED","#8B5CF6","#0891B2","#1C1917","Canvas purple + tool teal on dark"), + "Music Creation & Beat Maker": ("#7C3AED","#6366F1","#22C55E","#0F172A","Studio purple + waveform green on dark"), + "Meme & Sticker Maker": ("#EC4899","#F59E0B","#2563EB","#FFFFFF","Viral pink + comedy yellow + share blue"), + "AI Photo & Avatar Generator": ("#7C3AED","#6366F1","#EC4899","#FAF5FF","AI purple + generation pink"), + "Link-in-Bio Page Builder": ("#2563EB","#7C3AED","#EC4899","#FFFFFF","Brand blue + creator purple"), + # ── D. Personal Life ── + "Wardrobe & Outfit Planner": ("#BE185D","#EC4899","#D97706","#FDF2F8","Fashion rose + gold accent"), + "Plant Care Tracker": ("#15803D","#059669","#D97706","#F0FDF4","Nature green + sun yellow"), + "Book & Reading Tracker": ("#78716C","#92400E","#D97706","#FFFBEB","Book brown + page amber"), + "Couple & Relationship App": ("#BE185D","#EC4899","#DC2626","#FDF2F8","Romance rose + love red"), + "Family Calendar & Chores": ("#2563EB","#059669","#D97706","#F8FAFC","Family blue + chore green"), + "Mood Tracker": ("#7C3AED","#6366F1","#D97706","#FAF5FF","Mood purple + insight amber"), + "Gift & Wishlist": ("#DC2626","#D97706","#EC4899","#FFF1F2","Gift red + gold + surprise pink"), + # ── E. Health ── + "Running & Cycling GPS": ("#EA580C","#F97316","#059669","#0F172A","Energetic orange + pace green on dark"), + "Yoga & Stretching Guide": ("#6B7280","#78716C","#0891B2","#F5F5F0","Sage neutral + calm teal"), + "Sleep Tracker": ("#4338CA","#6366F1","#7C3AED","#0F172A","Night indigo + dream violet on dark"), + "Calorie & Nutrition Counter": ("#059669","#10B981","#EA580C","#ECFDF5","Healthy green + macro orange"), + "Period & Cycle Tracker": ("#BE185D","#EC4899","#7C3AED","#FDF2F8","Blush rose + fertility lavender"), + "Medication & Pill Reminder": ("#0284C7","#0891B2","#DC2626","#F0F9FF","Medical blue + alert red"), + "Water & Hydration Reminder": ("#0284C7","#06B6D4","#0891B2","#F0F9FF","Refreshing blue + water cyan"), + "Fasting & Intermittent Timer":("#6366F1","#4338CA","#059669","#0F172A","Fasting indigo on dark + eating green"), + # ── F. Social ── + "Anonymous Community / Confession":("#475569","#334155","#0891B2","#0F172A","Protective grey + subtle teal on dark"), + "Local Events & Discovery": ("#EA580C","#F97316","#2563EB","#FFF7ED","Event orange + map blue"), + "Study Together / Virtual Coworking":("#2563EB","#3B82F6","#059669","#F8FAFC","Focus blue + session green"), + # ── G. Education ── + "Coding Challenge & Practice": ("#22C55E","#059669","#D97706","#0F172A","Code green + difficulty amber on dark"), + "Kids Learning (ABC & Math)": ("#2563EB","#F59E0B","#EC4899","#EFF6FF","Learning blue + play yellow + fun pink"), + "Music Instrument Learning": ("#DC2626","#9A3412","#D97706","#FFFBEB","Musical red + warm amber"), + # ── H. Transport ── + "Parking Finder": ("#2563EB","#059669","#DC2626","#F0F9FF","Available blue/green + occupied red"), + "Public Transit Guide": ("#2563EB","#0891B2","#EA580C","#F8FAFC","Transit blue + line colors"), + "Road Trip Planner": ("#EA580C","#0891B2","#D97706","#FFF7ED","Adventure orange + map teal"), + # ── I. Safety & Lifestyle ── + "VPN & Privacy Tool": ("#1E3A5F","#334155","#22C55E","#0F172A","Shield dark + connected green"), + "Emergency SOS & Safety": ("#DC2626","#EF4444","#2563EB","#FFF1F2","Alert red + safety blue"), + "Wallpaper & Theme App": ("#7C3AED","#EC4899","#2563EB","#FAF5FF","Aesthetic purple + trending pink"), + "White Noise & Ambient Sound": ("#475569","#334155","#4338CA","#0F172A","Ambient grey + deep indigo on dark"), + "Home Decoration & Interior Design":("#78716C","#A8A29E","#D97706","#FAF5F2","Interior warm grey + gold accent"), +} + +# ─── 1. REBUILD colors.csv ─────────────────────────────────────────────────── +def rebuild_colors(): + src = os.path.join(BASE, "colors.csv") + with open(src, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + headers = reader.fieldnames + existing = list(reader) + + # Build lookup: Product Type -> row data + color_map = {} + for row in existing: + pt = row.get("Product Type", "").strip() + if not pt: + continue + # Remove deleted types + if pt in REMOVE_TYPES: + print(f" [colors] REMOVE: {pt}") + continue + # Rename mismatched types + if pt in COLOR_RENAMES: + new_name = COLOR_RENAMES[pt] + print(f" [colors] RENAME: {pt} → {new_name}") + row["Product Type"] = new_name + pt = new_name + color_map[pt] = row + + # Read products.csv to get the correct order + with open(os.path.join(BASE, "products.csv"), newline="", encoding="utf-8") as f: + products = list(csv.DictReader(f)) + + # Build final rows in products.csv order + final_rows = [] + added = 0 + for i, prod in enumerate(products, 1): + pt = prod["Product Type"] + if pt in color_map: + row = color_map[pt] + row["No"] = str(i) + final_rows.append(row) + elif pt in NEW_COLORS: + pri, sec, acc, bg, notes = NEW_COLORS[pt] + new_row = derive_row(pt, pri, sec, acc, bg, notes) + d = dict(zip(headers, [str(i)] + new_row)) + final_rows.append(d) + added += 1 + else: + print(f" [colors] WARNING: No color data for '{pt}' - using defaults") + new_row = derive_row(pt, "#2563EB", "#3B82F6", "#059669", "#F8FAFC", "Auto-generated default") + d = dict(zip(headers, [str(i)] + new_row)) + final_rows.append(d) + added += 1 + + # Write + with open(src, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=headers) + writer.writeheader() + writer.writerows(final_rows) + + product_count = len(products) + print(f"\n ✅ colors.csv: {len(final_rows)} rows ({product_count} products)") + print(f" Added: {added} new color rows") + +# ─── 2. REBUILD ui-reasoning.csv ───────────────────────────────────────────── +def derive_ui_reasoning(prod): + """Generate ui-reasoning row from products.csv row.""" + pt = prod["Product Type"] + style = prod.get("Primary Style Recommendation", "") + landing = prod.get("Landing Page Pattern", "") + color_focus = prod.get("Color Palette Focus", "") + considerations = prod.get("Key Considerations", "") + keywords = prod.get("Keywords", "") + + # Typography mood derived from style + typo_map = { + "Minimalism": "Professional + Clean hierarchy", + "Glassmorphism": "Modern + Clear hierarchy", + "Brutalism": "Bold + Oversized + Monospace", + "Claymorphism": "Playful + Rounded + Friendly", + "Dark Mode": "High contrast + Light on dark", + "Neumorphism": "Subtle + Soft + Monochromatic", + "Flat Design": "Bold + Clean + Sans-serif", + "Vibrant": "Energetic + Bold + Large", + "Aurora": "Elegant + Gradient-friendly", + "AI-Native": "Conversational + Minimal chrome", + "Organic": "Warm + Humanist + Natural", + "Motion": "Dynamic + Hierarchy-shifting", + "Accessible": "Large + High contrast + Clear", + "Soft UI": "Modern + Accessible + Balanced", + "Trust": "Professional + Serif accents", + "Swiss": "Grid-based + Mathematical + Helvetica", + "3D": "Immersive + Spatial + Variable", + "Retro": "Nostalgic + Monospace + Neon", + "Cyberpunk": "Terminal + Monospace + Neon", + "Pixel": "Retro + Blocky + 8-bit", + } + typo_mood = "Professional + Clear hierarchy" + for key, val in typo_map.items(): + if key.lower() in style.lower(): + typo_mood = val + break + + # Key effects from style + eff_map = { + "Glassmorphism": "Backdrop blur (10-20px) + Translucent overlays", + "Neumorphism": "Dual shadows (light+dark) + Soft press 150ms", + "Claymorphism": "Multi-layer shadows + Spring bounce + Soft press 200ms", + "Brutalism": "No transitions + Hard borders + Instant feedback", + "Dark Mode": "Subtle glow + Neon accents + High contrast", + "Flat Design": "Color shift hover + Fast 150ms transitions + No shadows", + "Minimalism": "Subtle hover 200ms + Smooth transitions + Clean", + "Motion-Driven": "Scroll animations + Parallax + Page transitions", + "Micro-interactions": "Haptic feedback + Small 50-100ms animations", + "Vibrant": "Large section gaps 48px+ + Color shift hover + Scroll-snap", + "Aurora": "Flowing gradients 8-12s + Color morphing", + "AI-Native": "Typing indicator + Streaming text + Context reveal", + "Organic": "Rounded 16-24px + Natural shadows + Flowing SVG", + "Soft UI": "Improved shadows + Modern 200-300ms + Focus visible", + "3D": "WebGL/Three.js + Parallax 3-5 layers + Physics 300-400ms", + "Trust": "Clear focus rings + Badge hover + Metric pulse", + "Accessible": "Focus rings 3-4px + ARIA + Reduced motion", + } + key_effects = "Subtle hover (200ms) + Smooth transitions" + for key, val in eff_map.items(): + if key.lower() in style.lower(): + key_effects = val + break + + # Decision rules + rules = {} + if "dark" in style.lower() or "oled" in style.lower(): + rules["if_light_mode_needed"] = "provide-theme-toggle" + if "glass" in style.lower(): + rules["if_low_performance"] = "fallback-to-flat" + if "conversion" in landing.lower(): + rules["if_conversion_focused"] = "add-urgency-colors" + if "social" in landing.lower(): + rules["if_trust_needed"] = "add-testimonials" + if "data" in keywords.lower() or "dashboard" in keywords.lower(): + rules["if_data_heavy"] = "prioritize-data-density" + if not rules: + rules["if_ux_focused"] = "prioritize-clarity" + rules["if_mobile"] = "optimize-touch-targets" + + # Anti-patterns + anti_patterns = [] + if "minimalism" in style.lower() or "minimal" in style.lower(): + anti_patterns.append("Excessive decoration") + if "dark" in style.lower(): + anti_patterns.append("Pure white backgrounds") + if "flat" in style.lower(): + anti_patterns.append("Complex shadows + 3D effects") + if "vibrant" in style.lower(): + anti_patterns.append("Muted colors + Low energy") + if "accessible" in style.lower(): + anti_patterns.append("Color-only indicators") + if not anti_patterns: + anti_patterns = ["Inconsistent styling", "Poor contrast ratios"] + anti_str = " + ".join(anti_patterns[:2]) + + return { + "UI_Category": pt, + "Recommended_Pattern": landing, + "Style_Priority": style, + "Color_Mood": color_focus, + "Typography_Mood": typo_mood, + "Key_Effects": key_effects, + "Decision_Rules": json.dumps(rules), + "Anti_Patterns": anti_str, + "Severity": "HIGH" + } + + +def rebuild_ui_reasoning(): + src = os.path.join(BASE, "ui-reasoning.csv") + with open(src, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + headers = reader.fieldnames + existing = list(reader) + + # Build lookup + ui_map = {} + for row in existing: + cat = row.get("UI_Category", "").strip() + if not cat: + continue + if cat in REMOVE_TYPES: + print(f" [ui-reason] REMOVE: {cat}") + continue + if cat in UI_RENAMES: + new_name = UI_RENAMES[cat] + print(f" [ui-reason] RENAME: {cat} → {new_name}") + row["UI_Category"] = new_name + cat = new_name + ui_map[cat] = row + + with open(os.path.join(BASE, "products.csv"), newline="", encoding="utf-8") as f: + products = list(csv.DictReader(f)) + + final_rows = [] + added = 0 + for i, prod in enumerate(products, 1): + pt = prod["Product Type"] + if pt in ui_map: + row = ui_map[pt] + row["No"] = str(i) + final_rows.append(row) + else: + row = derive_ui_reasoning(prod) + row["No"] = str(i) + final_rows.append(row) + added += 1 + + with open(src, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=headers) + writer.writeheader() + writer.writerows(final_rows) + + print(f"\n ✅ ui-reasoning.csv: {len(final_rows)} rows") + print(f" Added: {added} new reasoning rows") + + +# ─── MAIN ──────────────────────────────────────────────────────────────────── +if __name__ == "__main__": + print("=== Rebuilding colors.csv ===") + rebuild_colors() + print("\n=== Rebuilding ui-reasoning.csv ===") + rebuild_ui_reasoning() + print("\n🎉 Done!") diff --git a/personas/_shared/community-skills/ui-ux-pro-max/data/app-interface.csv b/personas/_shared/community-skills/ui-ux-pro-max/data/app-interface.csv new file mode 100644 index 0000000..f34c3cd --- /dev/null +++ b/personas/_shared/community-skills/ui-ux-pro-max/data/app-interface.csv @@ -0,0 +1,31 @@ +No,Category,Issue,Keywords,Platform,Description,Do,Don't,Code Example Good,Code Example Bad,Severity +1,Accessibility,Icon Button Labels,icon button accessibilityLabel,iOS/Android/React Native,Icon-only buttons must expose an accessible label,Set accessibilityLabel or label prop on icon buttons,Icon buttons without accessible names,"","",Critical +2,Accessibility,Form Control Labels,form input label accessibilityLabel,iOS/Android/React Native,All inputs must have a visible label and an accessibility label,Pair Text label with input and set accessibilityLabel,Inputs with placeholder only,"Email","",Critical +3,Accessibility,Role & Traits,accessibilityRole accessibilityTraits,iOS/Android/React Native,Interactive elements must expose correct roles/traits,Use accessibilityRole/button/link/checkbox etc.,Rely on generic views with no roles,"Submit","Submit",High +4,Accessibility,Dynamic Updates,accessibilityLiveRegion announce,iOS/Android/React Native,Async status updates should be announced to screen readers,Use accessibilityLiveRegion or announceForAccessibility,Update text silently with no announcement,"{status}","{status}",Medium +5,Accessibility,Decorative Icons,accessible={false} importantForAccessibility,iOS/Android/React Native,Decorative icons should be hidden from screen readers,Mark decorative icons as not accessible,Have screen reader read every icon,"","",Medium +6,Touch,Touch Target Size,touch 44x44 hitSlop,iOS/Android/React Native,Primary touch targets must be at least 44x44pt,Increase hitSlop or padding to meet minimum,Small icons with tiny touch area,"","",Critical +7,Touch,Touch Spacing,touch spacing gap 8px,iOS/Android/React Native,Adjacent touch targets need enough spacing,Keep at least 8dp spacing between touchables,Cluster many buttons with no gap,"