Claude CodeでLLM論文の自動収集・要約・公開パイプラインを構築した

[launchd: 毎日7:00]
       |
       v
[run_pipeline.sh]
       |
       v
[pipeline.py]
  |
  |-- 1. 状態ファイル読み込み (state.py)
  |-- 2. 論文収集 (fetchers.py)
  |       |-- arXiv API
  |       |-- Semantic Scholar API
  |       |-- HuggingFace Daily Papers API
  |       |-- 重複排除 + 人気度ソート
  |-- 3. 処理済み論文の除外
  |-- 4. 日本語要約生成 (summarizer.py) ... claude -p
  |-- 5. スクリーンショット取得 (screenshot.py) ... Playwright
  |-- 6. Hugo記事生成 (publisher.py)
  |-- 7. 状態ファイル保存
  |-- 8. git commit → hugo build → git push
  |-- 9. Slack通知 (notifier.py) ... claude -p + Slack MCP

scripts/llm_papers/
├── config.py               # 設定値の一元管理
├── fetchers.py             # 3ソースからの論文取得
├── summarizer.py           # Claude CLIによる要約生成
├── screenshot.py           # Playwrightによるスクリーンショット
├── publisher.py            # Hugo記事の生成
├── notifier.py             # Slack通知
├── state.py                # 冪等性の状態管理
├── pipeline.py             # メインオーケストレーター
├── run_pipeline.sh         # launchdから呼ばれるシェルラッパー
├── requirements.txt        # Python依存パッケージ
├── processed_papers.json   # 処理済み論文のID記録
├── pipeline.log            # アプリケーションログ
└── venv/                   # Python仮想環境

@dataclass
class Paper:
    paper_id: str
    title: str
    authors: list[str]
    abstract: str
    published: str          # ISO date string
    url: str
    source: str             # "arxiv", "semantic_scholar", "huggingface"
    arxiv_id: str | None = None
    doi: str | None = None
    categories: list[str] = field(default_factory=list)
    popularity_score: float = 0.0  # 人気度スコア

cat_query = " OR ".join(f"cat:{cat}" for cat in ["cs.CL", "cs.AI"])
keyword_query = " OR ".join(f'abs:"{kw}"' for kw in ARXIV_KEYWORDS[:5])
query = f"({cat_query}) AND ({keyword_query})"

LLM_FILTER_KEYWORDS = [
    "language model", "LLM", "transformer", "GPT", "BERT",
    "instruction tuning", "RLHF", "prompt", "chain of thought",
    "retrieval augmented", "RAG", "fine-tuning", "alignment",
    "reasoning", "tokeniz", "attention mechanism", ...
]

def _is_llm_related(title: str, abstract: str) -> bool:
    text = (title + " " + abstract).lower()
    return any(kw.lower() in text for kw in LLM_FILTER_KEYWORDS)

def _enrich_popularity_from_s2(papers: list[Paper]) -> None:
    papers_to_enrich = [p for p in papers if p.popularity_score < 0.1 and p.arxiv_id]
    arxiv_ids = [f"ArXiv:{p.arxiv_id}" for p in papers_to_enrich]

    resp = requests.post(
        "https://api.semanticscholar.org/graph/v1/paper/batch",
        json={"ids": arxiv_ids[:500]},
        params={"fields": "citationCount,externalIds"},
    )
    # 結果をpopularity_scoreに反映

def generate_summary(paper: Paper) -> str:
    prompt = (
        f"以下の論文のAbstractを日本語で要約してください。要約は3〜5文程度で、"
        f"技術的な内容を正確に伝えてください。要約のテキストのみを出力し、"
        f"前置きや説明は不要です。\n\n"
        f"タイトル: {paper.title}\n"
        f"Abstract:\n{paper.abstract}"
    )

    result = subprocess.run(
        ["claude", "-p", prompt],
        capture_output=True, text=True, timeout=120,
        env=_get_claude_env(),
    )

def _get_claude_env() -> dict:
    env = os.environ.copy()
    env.pop("CLAUDECODE", None)
    return env

def notify_slack(papers, summaries, date) -> bool:
    message = _format_slack_message(papers, summaries, date, blog_url)

    prompt = (
        f"以下のメッセージをSlackのnotifyチャンネル(ID: {SLACK_CHANNEL_ID})に"
        f"mcp__slack__slack_post_messageツールを使って投稿してください。"
        f"メッセージの内容をそのまま投稿し、それ以外の出力は不要です。\n\n"
        f"メッセージ:\n{message}"
    )

    result = subprocess.run(
        ["claude", "-p", "--allowedTools", "mcp__slack__slack_post_message"],
        input=prompt,
        capture_output=True, text=True, timeout=60,
        env=_get_claude_env(),
    )

{
  "processed_ids": {
    "arxiv:2602.10693": {
      "title": "VESPO: Variational Sequence-Level Soft Policy Optimization...",
      "processed_at": "2026-02-23T21:52:36.202101"
    },
    "arxiv:2602.08354": {
      "title": "Does Your Reasoning Model Implicitly Know When to Stop Thinking?",
      "processed_at": "2026-02-23T21:52:36.202114"
    }
  },
  "last_run": "2026-02-24T09:16:33.952783"
}

new_papers = [p for p in all_papers if not is_processed(state, p.paper_id)]
featured_papers = new_papers[:TOP_N_PAPERS]

---
title: "LLM論文サーベイ（2026-02-24）"
tags: ["LLM", "AI", "論文"]
url: llm-papers-2026-02-24
date: 2026-02-24
---

## 1. 論文タイトル
- **著者**: ...
- **ソース**: [huggingface](https://arxiv.org/abs/...)

### 要約
（Claude CLIが生成した日本語要約）

{{< details "原文Abstract" >}}
（英語の原文）
{{< /details >}}

def git_commit_and_push(post_dir: Path) -> None:
    subprocess.run(["git", "add", str(post_dir)], cwd=PROJECT_ROOT)
    subprocess.run(["git", "commit", "-m", f"Add LLM papers survey ({date})"], cwd=PROJECT_ROOT)
    subprocess.run(["hugo", "--config", "hugo.toml", "-d", "docs"], cwd=PROJECT_ROOT)
    subprocess.run(["git", "add", "docs/"], cwd=PROJECT_ROOT)
    subprocess.run(["git", "commit", "-m", "Build website"], cwd=PROJECT_ROOT)
    subprocess.run(["git", "push", "origin", "main"], cwd=PROJECT_ROOT)

<!-- ~/Library/LaunchAgents/com.zatoima.llm-papers-pipeline.plist -->
<dict>
    <key>Label</key>
    <string>com.zatoima.llm-papers-pipeline</string>
    <key>ProgramArguments</key>
    <array>
        <string>/bin/bash</string>
        <string>/.../scripts/llm_papers/run_pipeline.sh</string>
    </array>
    <key>StartCalendarInterval</key>
    <dict>
        <key>Hour</key>
        <integer>7</integer>
        <key>Minute</key>
        <integer>0</integer>
    </dict>
</dict>

# activate ではなく直接パス指定（launchdとの互換性のため）
./venv/bin/python3 pipeline.py --verbose

def _retry_request(func, *args, **kwargs):
    for attempt in range(MAX_RETRIES):
        try:
            return func(*args, **kwargs)
        except Exception as e:
            if attempt < MAX_RETRIES - 1:
                wait = RETRY_DELAY * (2 ** attempt)  # 2s, 4s, 8s
                time.sleep(wait)
            else:
                raise