diff --git a/.gitignore b/.gitignore index 34bf8571..2d38dcaf 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,10 @@ Thumbs.db *.log *.bak .codewiki/ +!tests/updater_toy.py +!tests/test_updater_graph_diff.py +!tests/test_updater_tree_repair.py +!tests/test_updater_reference_index.py +!tests/test_updater_change_report.py +!tests/test_updater_write_guard.py +!tests/test_updater_orchestrator.py diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py index 54b94da6..92c066dd 100644 --- a/codewiki/cli/adapters/doc_generator.py +++ b/codewiki/cli/adapters/doc_generator.py @@ -186,6 +186,17 @@ async def _run_backend_generation(self, backend_config: BackendConfig): # Create documentation generator doc_generator = DocumentationGenerator(backend_config, commit_id=self.commit_id) + # Incremental update (component-level): keep the previous graph before + # the builder overwrites it, so the updater can diff old against new. + update_opts = self._update_options() + prev_graph_path = None + if update_opts is not None: + from codewiki.src.be.updater.graph_store import snapshot_old_graph + + prev_graph_path = snapshot_old_graph( + backend_config.dependency_graph_dir, str(self.repo_path) + ) + if self.verbose: self.progress_tracker.update_stage(0.5, "Parsing source files...") @@ -211,6 +222,27 @@ async def _run_backend_generation(self, backend_config: BackendConfig): self.progress_tracker.complete_stage() + if update_opts is not None: + outcome = await self._run_incremental_update( + backend_config, doc_generator, update_opts, prev_graph_path, components, leaf_nodes + ) + if outcome in ("incremental", "no_change"): + # The builder wrote the new graph under this checkout's name. Drop graphs + # left by earlier checkouts so the next update has exactly one to pick. + from codewiki.src.be.updater.graph_store import ( + graph_file_path, + prune_superseded_graphs, + ) + + prune_superseded_graphs( + backend_config.dependency_graph_dir, + keep=graph_file_path(backend_config.dependency_graph_dir, str(self.repo_path)), + ) + return + # full_fallback / detector_failure: preserve the old docs, rebuild from scratch + self._move_docs_aside() + components, leaf_nodes = doc_generator.graph_builder.build_dependency_graph() + # Stage 2: Module Clustering self.progress_tracker.start_stage(2, "Module Clustering") if self.verbose: @@ -334,6 +366,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig): # Create metadata doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes)) + self._merge_update_summary(working_dir) # Collect generated files for file_path in os.listdir(working_dir): @@ -356,6 +389,111 @@ async def _run_backend_generation(self, backend_config: BackendConfig): self.progress_tracker.complete_stage() + # ------------------------------------------------------------------ + # Incremental update helpers + # ------------------------------------------------------------------ + + def _update_options(self): + """Return ``UpdateOptions`` when this run is a component-level update, else None.""" + if not self.config.get("update"): + return None + raw = dict(self.config.get("update_options") or {}) + rung = str(raw.pop("rung", "3") or "3") + if rung == "0": + return None # legacy path handled in generate.py + if not (self.output_dir / "module_tree.json").exists(): + return None # nothing to update against: normal generation + from codewiki.src.be.updater.options import UpdateOptions + + return UpdateOptions.from_rung(rung, **raw) + + async def _run_incremental_update( + self, backend_config, doc_generator, update_opts, prev_graph_path, components, leaf_nodes + ) -> str: + import json + + from codewiki.src.be.updater.orchestrator import IncrementalUpdater + + self.progress_tracker.start_stage(2, "Incremental Update") + prev_commit = None + try: + with open(self.output_dir / "metadata.json", encoding="utf-8") as f: + prev_commit = (json.load(f).get("generation_info") or {}).get("commit_id") + except (OSError, json.JSONDecodeError, AttributeError): + pass + revision = { + "old_commit": prev_commit, + "new_commit": self.commit_id, + "repo_path": str(self.repo_path), + } + updater = IncrementalUpdater( + backend_config, doc_generator.backend, doc_generator, update_opts + ) + record = await updater.run(prev_graph_path, components, leaf_nodes, revision) + self._last_update_record = record + summary = record.summary() + if self.verbose: + self.progress_tracker.update_stage( + 0.9, + f"Update outcome: {record.outcome} (diff {summary['diff_counts']}, " + f"{summary['n_active']} active leaves, {summary['n_calls']} LLM calls)", + ) + working_dir = str(self.output_dir.absolute()) + if record.outcome in ("incremental", "no_change"): + doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes)) + self._merge_update_summary(working_dir) + for file_path in os.listdir(working_dir): + if file_path.endswith((".md", ".json")): + self.job.files_generated.append(file_path) + tree_path = os.path.join(working_dir, "module_tree.json") + if os.path.exists(tree_path): + with open(tree_path, encoding="utf-8") as f: + self.job.module_count = len(json.load(f)) + missing_docs = doc_generator.validate_generated_docs(working_dir) + if missing_docs: + raise IncompleteGenerationError( + "Incremental update finished but these module docs are missing: " + + ", ".join(f"{name}.md" for name in missing_docs), + missing_modules=missing_docs, + ) + self.progress_tracker.complete_stage() + return record.outcome + + def _move_docs_aside(self) -> None: + """Keep the previous docs (and the failed attempt's record) next to the output dir.""" + import shutil + + suffix = (self.commit_id or "unknown")[:8] + target = self.output_dir.parent / f"{self.output_dir.name}.prev-{suffix}" + n = 2 + while target.exists(): + target = self.output_dir.parent / f"{self.output_dir.name}.prev-{suffix}-{n}" + n += 1 + shutil.move(str(self.output_dir), str(target)) + self.output_dir.mkdir(parents=True, exist_ok=True) + self._preserved_docs_dir = str(target) + record = getattr(self, "_last_update_record", None) + if record is not None: + record.detector_notes.append(f"previous docs moved to {target}") + if self.verbose: + self.progress_tracker.update_stage(0.1, f"Previous docs preserved at {target}") + + def _merge_update_summary(self, working_dir: str) -> None: + record = getattr(self, "_last_update_record", None) + if record is None: + return + from codewiki.src.be.updater.record import merge_into_metadata + + summary = record.summary() + preserved = getattr(self, "_preserved_docs_dir", None) + if preserved: + summary["previous_docs"] = preserved + merge_into_metadata(working_dir, summary) + try: + record.save(working_dir) + except OSError: + pass + def _run_html_generation(self): """Run HTML generation stage.""" self.progress_tracker.start_stage(4, "HTML Generation") diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py index 6e4c7079..34bb8ba2 100644 --- a/codewiki/cli/commands/generate.py +++ b/codewiki/cli/commands/generate.py @@ -330,6 +330,49 @@ def _find_affected(tree, parent_names=None): is_flag=True, help="Incremental update: only regenerate modules affected by changes since last generation", ) +@click.option( + "--update-rung", + type=click.Choice(["0", "1", "2", "3", "3b"]), + default="3", + show_default=True, + help=( + "Incremental updater variant used with --update. 0 = legacy file-level " + "invalidation; 1-3 = component-level updater ablation rungs; 3b = rung 3 with 2 hops." + ), +) +@click.option( + "--tau-ren", type=float, default=None, help="Rename similarity threshold (default 0.95)." +) +@click.option( + "--tau-nb", type=float, default=None, help="Neighbour-majority routing share (default 0.5)." +) +@click.option( + "--tau-grow", + type=float, + default=None, + help="Leaf growth share that re-clusters (default 0.33).", +) +@click.option( + "--tau-full", + type=float, + default=None, + help="Active-leaf share that forces a full build (default 0.5).", +) +@click.option( + "--tau-tree", + type=float, + default=None, + help="Structural-change share that forces a full build (default 0.3).", +) +@click.option( + "--k-hop", type=int, default=None, help="Dependency hops followed for Up (default 1)." +) +@click.option( + "--max-diff-tokens", + type=int, + default=None, + help="Cap on one component diff in a report (default 8000).", +) @click.option( "--compare-to", type=str, @@ -361,6 +404,14 @@ def generate_command( artifact_exclude: str | None = None, update: bool = False, compare_to: str | None = None, + update_rung: str = "3", + tau_ren: float | None = None, + tau_nb: float | None = None, + tau_grow: float | None = None, + tau_full: float | None = None, + tau_tree: float | None = None, + k_hop: int | None = None, + max_diff_tokens: int | None = None, ): """ Generate comprehensive documentation for a code repository. @@ -485,12 +536,23 @@ def generate_command( "No changes detected since last generation. Documentation is up to date." ) sys.exit(EXIT_SUCCESS) - if changed_files is not None: + if changed_files is not None and update_rung == "0": logger.info( f" Detected {len(changed_files)} changed files — regenerating affected modules." ) # Remove cached module docs for affected files so they get regenerated _invalidate_affected_modules(output_dir, changed_files, logger, verbose) + elif update_rung != "0": + if changed_files is not None: + logger.info( + f" Detected {len(changed_files)} changed files — running the " + f"component-level updater (rung {update_rung})." + ) + else: + logger.info( + f" Git diff unavailable — the component-level updater (rung {update_rung}) " + f"will compare the saved dependency graph instead." + ) # Check for existing documentation if ( @@ -665,6 +727,18 @@ def generate_command( "artifacts_enabled": artifacts, "artifact_token_budget": artifact_token_budget, "with_prose": with_prose, + # Incremental updater (runtime-only) + "update": update, + "update_options": { + "rung": update_rung, + "tau_ren": tau_ren, + "tau_nb": tau_nb, + "tau_grow": tau_grow, + "tau_full": tau_full, + "tau_tree": tau_tree, + "k_hop": k_hop, + "max_diff_tokens": max_diff_tokens, + }, }, verbose=verbose, generate_html=github_pages, diff --git a/codewiki/src/be/agent_tools/deps.py b/codewiki/src/be/agent_tools/deps.py index 6f2c469a..56180845 100644 --- a/codewiki/src/be/agent_tools/deps.py +++ b/codewiki/src/be/agent_tools/deps.py @@ -2,6 +2,7 @@ from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.config import Config + @dataclass class CodeWikiDeps: absolute_docs_path: str @@ -14,4 +15,8 @@ class CodeWikiDeps: max_depth: int current_depth: int config: Config # LLM configuration - custom_instructions: str = None \ No newline at end of file + custom_instructions: str = None + # Incremental updates: when set, ``str_replace_editor`` refuses any write + # (create / str_replace / insert / undo_edit) to a docs file whose + # absolute, resolved path is not in this set. ``None`` = unrestricted. + allowed_write_paths: set[str] | None = None diff --git a/codewiki/src/be/agent_tools/str_replace_editor.py b/codewiki/src/be/agent_tools/str_replace_editor.py index 49a84db1..257630d3 100644 --- a/codewiki/src/be/agent_tools/str_replace_editor.py +++ b/codewiki/src/be/agent_tools/str_replace_editor.py @@ -5,6 +5,7 @@ import io import json import logging +import os import re import shlex import subprocess @@ -13,16 +14,16 @@ from pathlib import Path from typing import Annotated, Literal -# Configure logging and monitoring - -logger = logging.getLogger(__name__) - from pydantic import BeforeValidator from pydantic_ai import RunContext, Tool from ..utils import validate_mermaid_diagrams from .deps import CodeWikiDeps +# Configure logging and monitoring + +logger = logging.getLogger(__name__) + def _coerce_json_string(value): """Coerce a JSON encoded string to its parsed Python value before pydantic @@ -50,7 +51,13 @@ def _coerce_json_string(value): # There are some super strange "ascii can't decode x" errors, # that can be solved with setting the default encoding for stdout # (note that python3.6 doesn't have the reconfigure method) -sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8") +# Only rewrap when stdout is not already UTF-8. Replacing sys.stdout +# unconditionally drops the previous wrapper, which closes the underlying +# buffer when garbage-collected; under pytest that kills output capture. +if (getattr(sys.stdout, "encoding", "") or "").lower().replace("-", "") != "utf8" and hasattr( + sys.stdout, "buffer" +): + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8") TRUNCATED_MESSAGE: str = "To save on context only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for." MAX_RESPONSE_LEN: int = 16000 @@ -803,6 +810,35 @@ def _make_output( ) +def check_write_allowed(absolute_path: str, allowed: "set[str] | None") -> str | None: + """Return an error message when ``absolute_path`` is outside the write set. + + ``allowed`` holds absolute paths; both sides are resolved so symlinks and + ``..`` segments cannot slip a write past the guard. ``None`` means the + agent may write anywhere under its working dir (normal generation). + """ + if allowed is None: + return None + try: + resolved = str(Path(absolute_path).resolve()) + except OSError: + resolved = os.path.abspath(absolute_path) + allowed_resolved = set() + for a in allowed: + try: + allowed_resolved.add(str(Path(a).resolve())) + except OSError: + allowed_resolved.add(os.path.abspath(a)) + if resolved in allowed_resolved: + return None + names = sorted(os.path.basename(a) for a in allowed_resolved) + return ( + f"Error: {os.path.basename(absolute_path)!r} is not in this agent's write set. " + f"You may only edit these pages: {names}. Everything else must stay untouched; " + f"if it needs a change, say so in your final verdict instead." + ) + + async def str_replace_editor( ctx: RunContext[CodeWikiDeps], working_dir: Literal["repo", "docs"], @@ -842,14 +878,30 @@ async def str_replace_editor( path = file tool = EditTool(ctx.deps.registry, ctx.deps.absolute_docs_path) - if working_dir == "docs": - absolute_path = str(Path(ctx.deps.absolute_docs_path) / path) - else: - absolute_path = str(Path(ctx.deps.absolute_repo_path) / path) + # Absolute paths would resolve *outside* the chosen working dir + # (``Path(base) / "/abs"`` == ``/abs``); force relative paths like the caw path does. + if os.path.isabs(path): + return ( + f"Error: `path` must be relative to `working_dir` ({working_dir!r}), " + f"got absolute path {path!r}." + ) + base_dir = ctx.deps.absolute_docs_path if working_dir == "docs" else ctx.deps.absolute_repo_path + absolute_path = str(Path(base_dir) / path) + try: + Path(absolute_path).resolve().relative_to(Path(base_dir).resolve()) + except ValueError: + return ( + f"Error: resolved path {absolute_path!r} escapes working_dir={working_dir!r} " + f"root {base_dir!r}." + ) # validate command if command != "view" and working_dir == "repo": return "The `view` command is the only allowed command when `working_dir` is `repo`." + if command != "view": + denied = check_write_allowed(absolute_path, getattr(ctx.deps, "allowed_write_paths", None)) + if denied: + return denied tool( command=command, diff --git a/codewiki/src/be/backend.py b/codewiki/src/be/backend.py index f8706773..294a3fc8 100644 --- a/codewiki/src/be/backend.py +++ b/codewiki/src/be/backend.py @@ -18,12 +18,53 @@ from __future__ import annotations import abc +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Dict, List if TYPE_CHECKING: + from codewiki.src.be.agent_tools.deps import CodeWikiDeps from codewiki.src.be.dependency_analyzer.models.core import Node +@dataclass +class AgentReply: + """Result of one agentic run whose final message matters (the updater).""" + + text: str + usage: dict[str, Any] | None = None + seconds: float = 0.0 + meta: dict[str, Any] = field(default_factory=dict) + + +def usage_to_dict(usage: Any) -> dict[str, Any] | None: + """Best-effort conversion of a provider usage object to plain numbers.""" + if usage is None: + return None + if isinstance(usage, dict): + return {k: v for k, v in usage.items() if isinstance(v, (int, float))} or None + out: dict[str, Any] = {} + for key in ( + "input_tokens", + "output_tokens", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "cache_read_tokens", + "cache_write_tokens", + "requests", + "cost_usd", + ): + value = getattr(usage, key, None) + if isinstance(value, (int, float)) and not isinstance(value, bool): + out[key] = value + if not out and hasattr(usage, "model_dump"): + try: + return {k: v for k, v in usage.model_dump().items() if isinstance(v, (int, float))} + except Exception: # noqa: BLE001 — usage is optional telemetry + return None + return out or None + + CAW_PROVIDERS = frozenset({"claude-code", "codex"}) @@ -33,7 +74,14 @@ def is_caw_provider(provider: str) -> bool: class LLMBackend(abc.ABC): - """Abstract LLM backend used by the documentation generator.""" + """Abstract LLM backend used by the documentation generator. + + ``last_usage`` holds the token usage of the most recent ``complete`` / + ``run_module_agent`` / ``run_update_agent`` call when the provider exposes + it (``None`` otherwise). The incremental updater reads it for its record. + """ + + last_usage: dict[str, Any] | None = None @abc.abstractmethod def complete( @@ -55,12 +103,25 @@ async def run_module_agent( ) -> Dict[str, Any]: """Run the per-module agent loop. Returns the updated module_tree dict.""" + async def run_update_agent( + self, + system_prompt: str, + user_prompt: str, + deps: "CodeWikiDeps", + ) -> AgentReply: + """Run an editing agent (read + str_replace_editor tools, no delegation) + and return its final message. Used by the incremental updater; writes + are limited by ``deps.allowed_write_paths``.""" + raise NotImplementedError(f"{type(self).__name__} does not support update agents") + def get_backend(config) -> "LLMBackend": """Return the backend instance matching ``config.provider``.""" provider = getattr(config, "provider", "openai-compatible") if is_caw_provider(provider): from codewiki.src.be.caw_backend import CawBackend + return CawBackend(config) from codewiki.src.be.pydantic_ai_backend import PydanticAIBackend + return PydanticAIBackend(config) diff --git a/codewiki/src/be/caw_backend.py b/codewiki/src/be/caw_backend.py index 649aaa07..6a2c6970 100644 --- a/codewiki/src/be/caw_backend.py +++ b/codewiki/src/be/caw_backend.py @@ -22,6 +22,7 @@ import asyncio import logging import os +import time import shutil from typing import Any @@ -29,7 +30,7 @@ from caw import ToolGroup from codewiki.src.be.agent_tools.deps import CodeWikiDeps -from codewiki.src.be.backend import LLMBackend +from codewiki.src.be.backend import AgentReply, LLMBackend, usage_to_dict from codewiki.src.be.cluster_modules import format_potential_core_components from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.be.prompt_template import ( @@ -257,8 +258,59 @@ def complete( tools=ToolGroup.READER, ) traj = agent.completion(prompt) + self.last_usage = usage_to_dict(getattr(traj, "total_usage", None)) return traj.result + # ------------------------------------------------------------------ + # Update agent (incremental updater): read tools + str_replace_editor, + # no delegation, final message returned. + # ------------------------------------------------------------------ + + async def run_update_agent( + self, + system_prompt: str, + user_prompt: str, + deps: CodeWikiDeps, + ) -> AgentReply: + set_main_loop(asyncio.get_running_loop()) + return await asyncio.to_thread( + self._run_update_agent_sync, system_prompt, user_prompt, deps + ) + + def _run_update_agent_sync( + self, system_prompt: str, user_prompt: str, deps: CodeWikiDeps + ) -> AgentReply: + from codewiki.src.be.caw_toolkit import CawToolKit # local import to avoid cycles + + toolkit = CawToolKit(deps=deps, backend=self, allow_subagent=False) + agent = CawAgent( + provider=self._caw_provider, + model=self._model, + system_prompt=system_prompt, + tools=_agent_tool_group_for_provider(self._caw_provider), + tool_servers=[toolkit], + ) + original_cwd = os.getcwd() + run_cwd = deps.absolute_docs_path if self._caw_provider == "codex" else self._repo_root + started = time.time() + try: + os.chdir(run_cwd) + try: + traj = agent.completion(user_prompt) + finally: + os.chdir(original_cwd) + except Exception as e: + logger.error("Update agent for %s failed via caw: %s", deps.current_module_name, e) + raise + usage = usage_to_dict(getattr(traj, "total_usage", None)) + self.last_usage = usage + return AgentReply( + text=traj.result or "", + usage=usage, + seconds=time.time() - started, + meta={"turns": traj.num_turns, "tool_calls": traj.total_tool_calls}, + ) + # ------------------------------------------------------------------ # Per-module agent loop # ------------------------------------------------------------------ @@ -434,6 +486,7 @@ def _run_module_agent_sync( traj.num_turns, traj.total_tool_calls, ) + self.last_usage = usage_to_dict(getattr(traj, "total_usage", None)) file_manager.save_json(deps.module_tree, module_tree_path) return deps.module_tree except Exception as e: diff --git a/codewiki/src/be/caw_toolkit.py b/codewiki/src/be/caw_toolkit.py index cdb51832..ba296f06 100644 --- a/codewiki/src/be/caw_toolkit.py +++ b/codewiki/src/be/caw_toolkit.py @@ -204,6 +204,15 @@ async def str_replace_editor( f"Pass a path that stays inside the working directory." ) + if command != "view": + from codewiki.src.be.agent_tools.str_replace_editor import check_write_allowed + + denied = check_write_allowed( + absolute_path, getattr(self._deps, "allowed_write_paths", None) + ) + if denied: + return denied + edit_tool( command=command, path=absolute_path, diff --git a/codewiki/src/be/llm_services.py b/codewiki/src/be/llm_services.py index 8a2ff5f6..38fca92b 100644 --- a/codewiki/src/be/llm_services.py +++ b/codewiki/src/be/llm_services.py @@ -6,6 +6,7 @@ Supports multiple providers: openai-compatible, anthropic, bedrock, azure-openai. """ + import inspect import logging from typing import Optional @@ -80,12 +81,8 @@ def _build_model_settings(config: Config, model_name: str) -> OpenAIChatModelSet provider default. """ if _should_use_max_completion_tokens(model_name, config.llm_base_url): - return OpenAIChatModelSettings( - max_completion_tokens=config.max_tokens - ) - return OpenAIChatModelSettings( - max_tokens=config.max_tokens - ) + return OpenAIChatModelSettings(max_completion_tokens=config.max_tokens) + return OpenAIChatModelSettings(max_tokens=config.max_tokens) def _get_litellm_model_name(model_name: str, provider: str) -> str: @@ -218,6 +215,7 @@ def _create_litellm_openai_client(config: Config) -> OpenAI: # Configure litellm for the provider if config.provider == "bedrock": import os + os.environ.setdefault("AWS_DEFAULT_REGION", config.aws_region) os.environ.setdefault("AWS_REGION_NAME", config.aws_region) @@ -236,11 +234,8 @@ def create_main_model(config: Config) -> CachingOpenAIModel: model_name=config.main_model, prompt_caching=config.prompt_caching, cache_registry_key=config.llm_base_url or "", - provider=OpenAIProvider( - base_url=config.llm_base_url, - api_key=config.llm_api_key - ), - settings=_build_model_settings(config, config.main_model) + provider=OpenAIProvider(base_url=config.llm_base_url, api_key=config.llm_api_key), + settings=_build_model_settings(config, config.main_model), ) @@ -250,11 +245,8 @@ def create_fallback_model(config: Config) -> CachingOpenAIModel: model_name=config.fallback_model, prompt_caching=config.prompt_caching, cache_registry_key=config.llm_base_url or "", - provider=OpenAIProvider( - base_url=config.llm_base_url, - api_key=config.llm_api_key - ), - settings=_build_model_settings(config, config.fallback_model) + provider=OpenAIProvider(base_url=config.llm_base_url, api_key=config.llm_api_key), + settings=_build_model_settings(config, config.fallback_model), ) @@ -267,10 +259,26 @@ def create_fallback_models(config: Config) -> FallbackModel: def create_openai_client(config: Config) -> OpenAI: """Create OpenAI client from configuration.""" - return OpenAI( - base_url=config.llm_base_url, - api_key=config.llm_api_key - ) + return OpenAI(base_url=config.llm_base_url, api_key=config.llm_api_key) + + +# Usage of the most recent completion, read by LLMBackend.complete implementations. +_LAST_USAGE: dict = {"usage": None} + + +def pop_last_usage() -> Optional[dict]: + usage = _LAST_USAGE["usage"] + _LAST_USAGE["usage"] = None + return usage + + +def _remember_usage(response) -> None: + from codewiki.src.be.backend import usage_to_dict + + try: + _LAST_USAGE["usage"] = usage_to_dict(getattr(response, "usage", None)) + except Exception: # noqa: BLE001 — usage is optional telemetry + _LAST_USAGE["usage"] = None def _extract_content(response, model: str) -> Optional[str]: @@ -280,6 +288,7 @@ def _extract_content(response, model: str) -> Optional[str]: which some proxies pair with ``content: null``) is visible in the logs instead of surfacing later as an opaque NoneType error. """ + _remember_usage(response) choice = response.choices[0] content = choice.message.content finish_reason = getattr(choice, "finish_reason", None) @@ -299,11 +308,7 @@ def _extract_content(response, model: str) -> Optional[str]: return content -def call_llm( - prompt: str, - config: Config, - model: str = None -) -> Optional[str]: +def call_llm(prompt: str, config: Config, model: str = None) -> Optional[str]: """ Call LLM with the given prompt. @@ -356,7 +361,9 @@ def call_llm( if _is_unsupported_token_param_error(e, primary_key): logger.info( "Provider rejected %s for model %s; retrying with %s.", - primary_key, model, fallback_key, + primary_key, + model, + fallback_key, ) response = client.chat.completions.create( **base_kwargs, @@ -380,11 +387,7 @@ def _is_unsupported_token_param_error(err: BadRequestError, param: str) -> bool: return "unsupported parameter" in msg and param in msg -def _call_llm_via_litellm( - prompt: str, - config: Config, - model: str -) -> Optional[str]: +def _call_llm_via_litellm(prompt: str, config: Config, model: str) -> Optional[str]: """ Call LLM via litellm for Bedrock/Anthropic providers. @@ -411,11 +414,7 @@ def _call_llm_via_litellm( return _extract_content(response, litellm_model) -def _call_llm_via_azure( - prompt: str, - config: Config, - model: str -) -> Optional[str]: +def _call_llm_via_azure(prompt: str, config: Config, model: str) -> Optional[str]: """ Call LLM via Azure OpenAI. @@ -431,7 +430,9 @@ def _call_llm_via_azure( ) deployment = config.azure_deployment or model - logger.debug("Calling Azure OpenAI deployment %s (api_version=%s)", deployment, config.api_version) + logger.debug( + "Calling Azure OpenAI deployment %s (api_version=%s)", deployment, config.api_version + ) response = client.chat.completions.create( model=deployment, diff --git a/codewiki/src/be/pydantic_ai_backend.py b/codewiki/src/be/pydantic_ai_backend.py index 3b3ddea9..23ad08e9 100644 --- a/codewiki/src/be/pydantic_ai_backend.py +++ b/codewiki/src/be/pydantic_ai_backend.py @@ -10,6 +10,7 @@ import logging import os +import time import traceback from typing import Any @@ -21,9 +22,9 @@ ) from codewiki.src.be.agent_tools.read_code_components import read_code_components_tool from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor_tool -from codewiki.src.be.backend import LLMBackend +from codewiki.src.be.backend import AgentReply, LLMBackend, usage_to_dict from codewiki.src.be.dependency_analyzer.models.core import Node -from codewiki.src.be.llm_services import call_llm, create_fallback_models +from codewiki.src.be.llm_services import call_llm, create_fallback_models, pop_last_usage from codewiki.src.be.prompt_template import ( format_leaf_system_prompt, format_system_prompt, @@ -36,6 +37,18 @@ logger = logging.getLogger(__name__) +def _run_usage(result: Any) -> dict[str, Any] | None: + """Token usage of a pydantic-ai run; ``usage`` is a property in pydantic-ai + 2.x and a method in earlier releases.""" + try: + usage = getattr(result, "usage", None) + if callable(usage): + usage = usage() + return usage_to_dict(usage) + except Exception: # noqa: BLE001 — usage is optional telemetry + return None + + class PydanticAIBackend(LLMBackend): """API-key based backend using pydantic-ai + openai/litellm clients.""" @@ -43,6 +56,7 @@ def __init__(self, config: Config) -> None: self._config = config self._fallback_models = create_fallback_models(config) self._custom_instructions = config.get_prompt_addition() + self.last_usage: dict[str, Any] | None = None def complete( self, @@ -50,7 +64,31 @@ def complete( *, model: str | None = None, ) -> str: - return call_llm(prompt, self._config, model=model) + pop_last_usage() + result = call_llm(prompt, self._config, model=model) + self.last_usage = pop_last_usage() + return result + + async def run_update_agent( + self, + system_prompt: str, + user_prompt: str, + deps: CodeWikiDeps, + ) -> AgentReply: + agent = Agent( + self._fallback_models, + name=f"update:{deps.current_module_name}", + deps_type=CodeWikiDeps, + tools=[read_code_components_tool, str_replace_editor_tool], + system_prompt=system_prompt, + ) + started = time.time() + result = await agent.run(user_prompt, deps=deps) + seconds = time.time() - started + usage = _run_usage(result) + self.last_usage = usage + text = result.output if isinstance(result.output, str) else str(result.output) + return AgentReply(text=text, usage=usage, seconds=seconds) async def run_module_agent( self, @@ -114,7 +152,7 @@ async def run_module_agent( ) try: - await agent.run( + result = await agent.run( format_user_prompt( module_name=module_name, core_component_ids=core_component_ids, @@ -123,6 +161,7 @@ async def run_module_agent( ), deps=deps, ) + self.last_usage = _run_usage(result) file_manager.save_json(deps.module_tree, module_tree_path) return deps.module_tree except Exception as e: diff --git a/codewiki/src/be/updater/__init__.py b/codewiki/src/be/updater/__init__.py new file mode 100644 index 00000000..06a132cc --- /dev/null +++ b/codewiki/src/be/updater/__init__.py @@ -0,0 +1,21 @@ +"""Component-level incremental updater (Contribution 3). + +Given the graph, module tree, and pages left behind by a previous build, +and the fresh code graph of the new revision, this package: + +1. diffs the two graphs component by component (``graph_diff``), +2. repairs the module tree (``tree_repair``), +3. builds a change report per leaf module (``change_report``), +4. decides whether an incremental update is worth doing (``change_report.fallback_ratios``), +5. runs one editing agent per active leaf, restricted to a write set (``leaf_agent``), +6. generates any page still missing, scans for stale names, and records + every decision (``orchestrator``, ``stale_scan``, ``record``). + +Rung 0 (the file-level cache invalidation that predates this package) stays +available through ``UpdateOptions(rung=0)`` and is implemented in +``codewiki/cli/commands/generate.py``. +""" + +from codewiki.src.be.updater.options import UpdateOptions + +__all__ = ["UpdateOptions"] diff --git a/codewiki/src/be/updater/change_report.py b/codewiki/src/be/updater/change_report.py new file mode 100644 index 00000000..d00ed630 --- /dev/null +++ b/codewiki/src/be/updater/change_report.py @@ -0,0 +1,271 @@ +"""Step 3: one change report per update unit (leaf), the active set, and the +fallback ratios of Step 4.""" + +from __future__ import annotations + +import logging +from dataclasses import asdict, dataclass, field +from typing import Any + +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.updater import tree as T +from codewiki.src.be.updater.graph_diff import GraphDiff +from codewiki.src.be.updater.options import UpdateOptions +from codewiki.src.be.updater.reference_index import unique_names_of +from codewiki.src.be.updater.tree_repair import RepairResult + +logger = logging.getLogger(__name__) + +MODE_EDIT = "edit" +MODE_CREATE = "create" +MODE_DELETE = "delete" + + +@dataclass +class LeafReport: + leaf_path: tuple[str, ...] + mode: str = MODE_EDIT + own: list[str] = field(default_factory=list) + up: list[str] = field(default_factory=list) + context: list[str] = field(default_factory=list) # untracked changes next to this leaf + refch: list[str] = field(default_factory=list) + entered: list[str] = field(default_factory=list) + left: list[str] = field(default_factory=list) + children_added: list[str] = field(default_factory=list) + children_removed: list[str] = field(default_factory=list) + reclustered: bool = False + + @property + def page(self) -> str: + return T.page_stem(self.leaf_path) + + @property + def is_empty(self) -> bool: + # ``context`` (untracked changes next to this leaf) is information for + # an agent that runs anyway; on its own it never activates a leaf. + return not ( + self.own + or self.up + or self.refch + or self.entered + or self.left + or self.children_added + or self.children_removed + or self.reclustered + ) + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["leaf_path"] = list(self.leaf_path) + d["page"] = self.page + return d + + +def _out_edges(cid: str, old: dict[str, Node], new: dict[str, Node]) -> set[str]: + out: set[str] = set() + if cid in old: + out |= set(old[cid].depends_on or ()) + if cid in new: + out |= set(new[cid].depends_on or ()) + return out + + +def _children_names(tree: dict[str, Any], path: tuple[str, ...]) -> set[str]: + info = T.node_at(tree, path) + if not info: + return set() + ch = info.get("children") + return set(ch.keys()) if isinstance(ch, dict) else set() + + +def build_reports( + diff: GraphDiff, + old_tree: dict[str, Any], + new_tree: dict[str, Any], + old_graph: dict[str, Node], + new_graph: dict[str, Node], + ref_index: dict[str, dict[str, list[str]]] | None, + repair: RepairResult, + opts: UpdateOptions, + reclustered: set[tuple[str, ...]] | None = None, +) -> dict[tuple[str, ...], LeafReport]: + """Build ``Report(l)`` for every unit of the new tree plus deleted leaves.""" + reclustered = reclustered or set() + old_owner = T.owner_map(old_tree) + new_owner = T.owner_map(new_tree) + old_units = set(T.unit_paths(old_tree)) + new_units = T.unit_paths(new_tree) + reports: dict[tuple[str, ...], LeafReport] = {p: LeafReport(leaf_path=p) for p in new_units} + + # Own: changed components in comp_t(l) ∪ comp_{t+1}(l). An untracked + # method counts as its class's (see tree.resolve_owner). + for cid in diff.changed_ids | set(diff.deleted): + owners = set() + for o in (T.resolve_owner(new_owner, cid), T.resolve_owner(old_owner, cid)): + if o is not None: + owners.add(o) + rec = diff.record_for(cid) + if rec and rec.old_id: + o = T.resolve_owner(old_owner, rec.old_id) + if o is not None: + owners.add(o) + for p in owners: + if p in reports and cid not in reports[p].own: + reports[p].own.append(cid) + + # Up: interface / deleted / renamed components used by this leaf's code. + contract_moved = set(diff.interface) | set(diff.deleted) | set(diff.renamed.values()) + if opts.use_up and contract_moved: + rev_old = T.reverse_edges(old_graph) + rev_new = T.reverse_edges(new_graph) + old_ids_of_renames = {v: k for k, v in diff.renamed.items()} + for cid in contract_moved: + frontier = {cid} + if cid in old_ids_of_renames: + frontier.add(old_ids_of_renames[cid]) + seen: set[str] = set() + for _ in range(max(1, opts.k_hop)): + nxt: set[str] = set() + for x in frontier: + nxt |= rev_old.get(x, set()) | rev_new.get(x, set()) + nxt -= seen + seen |= nxt + frontier = nxt + for user in seen: + p = T.resolve_owner(new_owner, user) or T.resolve_owner(old_owner, user) + if p is None or p not in reports: + continue + own_leaf = ( + T.resolve_owner(new_owner, cid) + or T.resolve_owner(old_owner, cid) + or T.resolve_owner(old_owner, old_ids_of_renames.get(cid, "")) + ) + if p == own_leaf: + continue + if cid not in reports[p].up: + reports[p].up.append(cid) + + # Untracked changed components (no class to attach to): context for the + # leaves owning their neighbours. + for cid in diff.changed_ids: + if T.resolve_owner(new_owner, cid) or T.resolve_owner(old_owner, cid): + continue + neighbours = _out_edges(cid, old_graph, new_graph) + for other, node in new_graph.items(): + if cid in (node.depends_on or ()): + neighbours.add(other) + for other, node in old_graph.items(): + if cid in (node.depends_on or ()): + neighbours.add(other) + for n in neighbours: + p = T.resolve_owner(new_owner, n) or T.resolve_owner(old_owner, n) + if p in reports and cid not in reports[p].context: + reports[p].context.append(cid) + + # RefCh: things this leaf's page refers to that changed or vanished. + if ref_index: + deleted_pages = {p[-1] for p in repair.deleted_nodes} + # Bare-name mentions count only when the contract moved (interface, + # delete, rename); a body-only change does not make a mention stale. + gone_names = unique_names_of(old_graph, set(diff.deleted) | set(diff.renamed.keys())) + gone_names |= unique_names_of(new_graph, set(diff.interface) | set(diff.renamed.values())) + changed_ids = diff.changed_ids | set(diff.deleted) | set(diff.renamed.keys()) + for p, rep in reports.items(): + refs = ref_index.get(rep.page) + if not refs: + continue + hits: list[str] = [] + hits += [x for x in refs.get("ids", []) if x in changed_ids] + hits += [f"{x}.md" for x in refs.get("links", []) if x in deleted_pages] + hits += [x for x in refs.get("names", []) if x in gone_names] + own_set = set(rep.own) + rep.refch = sorted({h for h in hits if h not in own_set}) + + # Tree changes. + for p, rep in reports.items(): + rep.entered = list(repair.entered.get(p, [])) + rep.left = list(repair.left.get(p, [])) + old_children = _children_names(old_tree, p) + new_children = _children_names(new_tree, p) + rep.children_added = sorted(new_children - old_children) + rep.children_removed = sorted(old_children - new_children) + rep.reclustered = p in reclustered + if p in repair.created_leaves: + rep.mode = MODE_CREATE + elif p not in old_units and p not in reclustered: + # A parent that became a unit (all children removed) or a brand new node. + rep.mode = MODE_CREATE if T.node_at(old_tree, p) is None else MODE_EDIT + + # Deleted leaves: report with mode delete, keyed by the old path. + for p in repair.deleted_nodes: + if p in reports: + continue + rep = LeafReport(leaf_path=p, mode=MODE_DELETE) + rep.left = list(repair.left.get(p, [])) + old_info = T.node_at(old_tree, p) or {} + rep.own = [c for c in T.components_of(old_info) if c in diff.deleted] + reports[p] = rep + + active = [p for p, r in reports.items() if not r.is_empty or r.mode != MODE_EDIT] + logger.info("Change reports: %d units, %d active", len(reports), len(active)) + return reports + + +def active_set(reports: dict[tuple[str, ...], LeafReport]) -> list[tuple[str, ...]]: + return [p for p, r in reports.items() if not r.is_empty or r.mode != MODE_EDIT] + + +def fallback_ratios( + reports: dict[tuple[str, ...], LeafReport], + new_tree: dict[str, Any], + repair: RepairResult, + reclustered: set[tuple[str, ...]] | None = None, +) -> dict[str, float]: + n_leaves = max(1, len(T.unit_paths(new_tree))) + active = active_set(reports) + structural = len(repair.created_leaves) + len(repair.deleted_nodes) + len(reclustered or ()) + return { + "r_leaf": len(active) / n_leaves, + "r_tree": structural / n_leaves, + "n_active": len(active), + "n_leaves": n_leaves, + "n_structural": structural, + } + + +def order_active( + active: list[tuple[str, ...]], + new_tree: dict[str, Any], + new_graph: dict[str, Node], +) -> list[tuple[str, ...]]: + """Topological order under the lifted dependency: if A uses B, B runs first. + Cycles and ties are broken by tree pre-order; deleted leaves (not in the + new tree) go last.""" + pre = {p: i for i, p in enumerate(T.preorder_paths(new_tree))} + dep = T.leaf_dependents(new_tree, new_graph) # leaf -> set of leaves that use it + active_set_ = set(active) + # edges: used -> user (used first) + indeg = {p: 0 for p in active} + users_of: dict[tuple[str, ...], set[tuple[str, ...]]] = {p: set() for p in active} + for used, users in dep.items(): + if used not in active_set_: + continue + for user in users: + if user in active_set_ and user != used: + users_of[used].add(user) + for used, users in users_of.items(): + for user in users: + indeg[user] += 1 + order: list[tuple[str, ...]] = [] + remaining = set(active) + while remaining: + ready = sorted((p for p in remaining if indeg[p] == 0), key=lambda p: pre.get(p, 10**9)) + if not ready: # cycle: pick by pre-order + ready = [min(remaining, key=lambda p: pre.get(p, 10**9))] + p = ready[0] + order.append(p) + remaining.discard(p) + for user in users_of.get(p, ()): + if user in remaining: + indeg[user] -= 1 + return order diff --git a/codewiki/src/be/updater/graph_diff.py b/codewiki/src/be/updater/graph_diff.py new file mode 100644 index 00000000..4b8b4a2d --- /dev/null +++ b/codewiki/src/be/updater/graph_diff.py @@ -0,0 +1,300 @@ +"""Step 1: component-level diff between two saved code graphs. + +Components are joined on id (``path::name``). For each pair we compare a +whitespace-normalised hash of the body and the signature +``(name, parameters, base_classes)``. Deleted/added pairs whose bodies are +near-identical are paired as renames. +""" + +from __future__ import annotations + +import difflib +import hashlib +import logging +import re +from dataclasses import asdict, dataclass, field +from typing import Any + +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.updater.options import UpdateOptions + +logger = logging.getLogger(__name__) + +CLASS_ADDED = "added" +CLASS_DELETED = "deleted" +CLASS_IFACE = "interface" +CLASS_BODY = "body" +CLASS_EDGE = "edge" +CLASS_RENAMED = "renamed" + +DIFF_TRUNCATED_MARKER = "... [diff truncated] ..." + +_WS = re.compile(r"\s+") +_TOKEN = re.compile(r"\w+|[^\w\s]") + + +def normalize_body(text: str | None) -> str: + return _WS.sub(" ", (text or "").strip()) + + +def body_hash(node: Node) -> str: + return hashlib.sha1(normalize_body(node.source_code).encode("utf-8")).hexdigest() + + +def signature(node: Node) -> dict[str, Any]: + return { + "name": node.name, + "parameters": list(node.parameters or []), + "base_classes": list(node.base_classes or []), + } + + +def _tokens(text: str | None) -> list[str]: + return _TOKEN.findall(text or "") + + +def body_similarity(a: str | None, b: str | None) -> float: + """Token-level similarity in [0, 1] (``difflib`` ratio over token lists).""" + ta, tb = _tokens(a), _tokens(b) + if not ta and not tb: + return 1.0 + if not ta or not tb: + return 0.0 + return difflib.SequenceMatcher(None, ta, tb, autojunk=False).ratio() + + +def _approx_tokens(text: str) -> int: + # Cheap estimate; good enough to cap a diff. ~4 chars per token. + return len(text) // 4 + 1 + + +def truncate_to_tokens(text: str, max_tokens: int) -> str: + if max_tokens <= 0 or _approx_tokens(text) <= max_tokens: + return text + budget = max_tokens * 4 + head = text[: budget // 2] + tail = text[-(budget // 2) :] + return f"{head}\n{DIFF_TRUNCATED_MARKER}\n{tail}" + + +def unified_body_diff(old: Node | None, new: Node | None, label: str) -> str: + old_lines = (old.source_code or "").splitlines(keepends=True) if old else [] + new_lines = (new.source_code or "").splitlines(keepends=True) if new else [] + diff = difflib.unified_diff( + old_lines, new_lines, fromfile=f"a/{label}", tofile=f"b/{label}", n=2 + ) + return "".join(diff) + + +@dataclass +class ChangeRecord: + component_id: str # new id (or old id for deletions) + change_class: str + old_id: str | None = None + new_id: str | None = None + relative_path: str | None = None + component_type: str | None = None + old_signature: dict[str, Any] | None = None + new_signature: dict[str, Any] | None = None + diff: str = "" + similarity: float | None = None + signature_changed: bool = False + edges_added: list[str] = field(default_factory=list) + edges_removed: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass +class GraphDiff: + added: set[str] = field(default_factory=set) + deleted: set[str] = field(default_factory=set) + interface: set[str] = field(default_factory=set) + body: set[str] = field(default_factory=set) + edge: set[str] = field(default_factory=set) + renamed: dict[str, str] = field(default_factory=dict) # old id -> new id + records: dict[str, ChangeRecord] = field(default_factory=dict) + + @property + def changed_ids(self) -> set[str]: + """Every id that appears in the change set (new ids for renames).""" + return ( + set(self.added) + | set(self.deleted) + | set(self.interface) + | set(self.body) + | set(self.edge) + | set(self.renamed.values()) + ) + + @property + def is_empty(self) -> bool: + return not self.changed_ids + + def counts(self) -> dict[str, int]: + return { + CLASS_ADDED: len(self.added), + CLASS_DELETED: len(self.deleted), + CLASS_IFACE: len(self.interface), + CLASS_BODY: len(self.body), + CLASS_EDGE: len(self.edge), + CLASS_RENAMED: len(self.renamed), + } + + def record_for(self, cid: str) -> ChangeRecord | None: + return self.records.get(cid) + + def to_dict(self) -> dict[str, Any]: + return { + "counts": self.counts(), + "added": sorted(self.added), + "deleted": sorted(self.deleted), + "interface": sorted(self.interface), + "body": sorted(self.body), + "edge": sorted(self.edge), + "renamed": dict(sorted(self.renamed.items())), + } + + +def _pair_renames( + deleted: set[str], + added: set[str], + old: dict[str, Node], + new: dict[str, Node], + tau_ren: float, +) -> list[tuple[str, str, float]]: + """Greedy best-match pairing of deleted x added by body similarity.""" + if not deleted or not added: + return [] + candidates: list[tuple[float, str, str]] = [] + added_by_type: dict[str, list[str]] = {} + for cid in added: + added_by_type.setdefault(new[cid].component_type, []).append(cid) + for old_id in deleted: + o = old[old_id] + o_body = normalize_body(o.source_code) + if not o_body: + continue + for new_id in added_by_type.get(o.component_type, []): + n_body = normalize_body(new[new_id].source_code) + if not n_body: + continue + ratio = ( + len(o_body) / len(n_body) + if len(n_body) >= len(o_body) + else len(n_body) / len(o_body) + ) + if ratio < tau_ren * 0.9: + continue # lengths too different to reach tau_ren + sim = body_similarity(o.source_code, new[new_id].source_code) + if sim >= tau_ren: + candidates.append((sim, old_id, new_id)) + candidates.sort(reverse=True) + used_old: set[str] = set() + used_new: set[str] = set() + pairs: list[tuple[str, str, float]] = [] + for sim, old_id, new_id in candidates: + if old_id in used_old or new_id in used_new: + continue + used_old.add(old_id) + used_new.add(new_id) + pairs.append((old_id, new_id, sim)) + return pairs + + +def diff_graphs( + old: dict[str, Node], new: dict[str, Node], opts: UpdateOptions | None = None +) -> GraphDiff: + """Compute the component-level change set between two graphs.""" + opts = opts or UpdateOptions() + result = GraphDiff() + old_ids, new_ids = set(old), set(new) + + result.added = new_ids - old_ids + result.deleted = old_ids - new_ids + + for cid in old_ids & new_ids: + o, n = old[cid], new[cid] + sig_changed = signature(o) != signature(n) + hash_changed = body_hash(o) != body_hash(n) + edges_changed = set(o.depends_on or ()) != set(n.depends_on or ()) + if not (sig_changed or hash_changed or edges_changed): + continue + if sig_changed: + klass = CLASS_IFACE + result.interface.add(cid) + elif hash_changed: + klass = CLASS_BODY + result.body.add(cid) + else: + klass = CLASS_EDGE + result.edge.add(cid) + rec = ChangeRecord( + component_id=cid, + change_class=klass, + old_id=cid, + new_id=cid, + relative_path=n.relative_path, + component_type=n.component_type, + old_signature=signature(o), + new_signature=signature(n), + signature_changed=sig_changed, + edges_added=sorted(set(n.depends_on or ()) - set(o.depends_on or ())), + edges_removed=sorted(set(o.depends_on or ()) - set(n.depends_on or ())), + ) + if hash_changed: + rec.diff = truncate_to_tokens(unified_body_diff(o, n, cid), opts.max_diff_tokens) + result.records[cid] = rec + + for old_id, new_id, sim in _pair_renames(result.deleted, result.added, old, new, opts.tau_ren): + result.deleted.discard(old_id) + result.added.discard(new_id) + result.renamed[old_id] = new_id + o, n = old[old_id], new[new_id] + sig_changed = signature(o) != signature(n) + rec = ChangeRecord( + component_id=new_id, + change_class=CLASS_RENAMED, + old_id=old_id, + new_id=new_id, + relative_path=n.relative_path, + component_type=n.component_type, + old_signature=signature(o), + new_signature=signature(n), + similarity=round(sim, 4), + signature_changed=sig_changed, + ) + if body_hash(o) != body_hash(n): + rec.diff = truncate_to_tokens(unified_body_diff(o, n, new_id), opts.max_diff_tokens) + result.records[new_id] = rec + if sig_changed: + # A renamed component whose contract also moved counts as an + # interface change for its dependents. + result.interface.add(new_id) + + for cid in result.added: + n = new[cid] + result.records[cid] = ChangeRecord( + component_id=cid, + change_class=CLASS_ADDED, + new_id=cid, + relative_path=n.relative_path, + component_type=n.component_type, + new_signature=signature(n), + diff=truncate_to_tokens(unified_body_diff(None, n, cid), opts.max_diff_tokens), + ) + for cid in result.deleted: + o = old[cid] + result.records[cid] = ChangeRecord( + component_id=cid, + change_class=CLASS_DELETED, + old_id=cid, + relative_path=o.relative_path, + component_type=o.component_type, + old_signature=signature(o), + diff=truncate_to_tokens(unified_body_diff(o, None, cid), opts.max_diff_tokens), + ) + + logger.info("Graph diff: %s", result.counts()) + return result diff --git a/codewiki/src/be/updater/graph_store.py b/codewiki/src/be/updater/graph_store.py new file mode 100644 index 00000000..4306a86c --- /dev/null +++ b/codewiki/src/be/updater/graph_store.py @@ -0,0 +1,134 @@ +"""Load and snapshot the saved dependency graph. + +``DependencyGraphBuilder.build_dependency_graph`` writes the graph to +``/temp/dependency_graphs/_dependency_graph.json`` and +overwrites it on every run. The updater therefore copies the previous +graph aside *before* the new graph is built, then loads that copy. +""" + +from __future__ import annotations + +import json +import logging +import os +import shutil + +from codewiki.src.be.dependency_analyzer.models.core import Node + +logger = logging.getLogger(__name__) + +PREV_SUFFIX = ".prev.json" + + +def sanitize_repo_name(repo_path: str) -> str: + repo_name = os.path.basename(os.path.normpath(repo_path)) + return "".join(c if c.isalnum() else "_" for c in repo_name) + + +def graph_file_path(dependency_graph_dir: str, repo_path: str) -> str: + """Path of the graph JSON for ``repo_path`` inside ``dependency_graph_dir``.""" + return os.path.join( + dependency_graph_dir, f"{sanitize_repo_name(repo_path)}_dependency_graph.json" + ) + + +def prev_graph_path(dependency_graph_dir: str, repo_path: str) -> str: + return graph_file_path(dependency_graph_dir, repo_path)[: -len(".json")] + PREV_SUFFIX + + +def list_graph_files(dependency_graph_dir: str) -> list[str]: + """All current ``*_dependency_graph.json`` files in the dir (``.prev.json`` excluded).""" + if not os.path.isdir(dependency_graph_dir): + return [] + return sorted( + os.path.join(dependency_graph_dir, f) + for f in os.listdir(dependency_graph_dir) + if f.endswith("_dependency_graph.json") + ) + + +def find_any_graph_file(dependency_graph_dir: str) -> str | None: + """Return the previous build's graph when it is not under the current repo name. + + The repo may have been analysed from a differently named checkout (a git + worktree per revision, for example), so the sanitized name is not always + the same. With exactly one candidate that is the answer. With several + (each earlier worktree left its own file) the newest by mtime is taken and + the choice is logged; ``prune_superseded_graphs`` keeps this case rare. + """ + candidates = list_graph_files(dependency_graph_dir) + if not candidates: + return None + if len(candidates) == 1: + return candidates[0] + newest = max(candidates, key=os.path.getmtime) + logger.warning( + "Several dependency graphs found in %s; using the newest (%s) as the previous graph. " + "Candidates: %s", + dependency_graph_dir, + os.path.basename(newest), + ", ".join(os.path.basename(c) for c in candidates), + ) + return newest + + +def prune_superseded_graphs(dependency_graph_dir: str, keep: str) -> list[str]: + """Delete current-graph files other than ``keep`` so the next update finds one graph. + + The previous graph survives as ``*.prev.json``; only stale copies left by + earlier checkouts under other names are removed. Returns the removed paths. + """ + removed = [] + keep_abs = os.path.abspath(keep) + for path in list_graph_files(dependency_graph_dir): + if os.path.abspath(path) != keep_abs: + os.remove(path) + removed.append(path) + if removed: + logger.info( + "Removed superseded dependency graph(s): %s", + ", ".join(os.path.basename(r) for r in removed), + ) + return removed + + +def snapshot_old_graph(dependency_graph_dir: str, repo_path: str) -> str | None: + """Copy the previous build's graph to ``*.prev.json``; return that path or None.""" + src = graph_file_path(dependency_graph_dir, repo_path) + if not os.path.exists(src): + src = find_any_graph_file(dependency_graph_dir) + if src is None or not os.path.exists(src): + return None + dst = prev_graph_path(dependency_graph_dir, repo_path) + shutil.copyfile(src, dst) + logger.info("Saved previous dependency graph to %s", dst) + return dst + + +def load_graph(path: str) -> dict[str, Node]: + """Read a saved graph JSON back into ``Node`` objects keyed by component id.""" + with open(path, encoding="utf-8") as f: + raw = json.load(f) + graph: dict[str, Node] = {} + for cid, data in raw.items(): + if not isinstance(data, dict): + continue + data = dict(data) + deps = data.get("depends_on") or [] + data["depends_on"] = set(deps) + data.setdefault("id", cid) + graph[cid] = Node(**data) + return graph + + +def save_graph(graph: dict[str, Node], path: str) -> None: + """Write ``graph`` in the same shape ``DependencyParser.save_dependency_graph`` uses.""" + result = {} + for cid, node in graph.items(): + d = node.model_dump() + if isinstance(d.get("depends_on"), set): + d["depends_on"] = sorted(d["depends_on"]) + result[cid] = d + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2, ensure_ascii=False) diff --git a/codewiki/src/be/updater/leaf_agent.py b/codewiki/src/be/updater/leaf_agent.py new file mode 100644 index 00000000..feaa20ac --- /dev/null +++ b/codewiki/src/be/updater/leaf_agent.py @@ -0,0 +1,245 @@ +"""Step 5: one editing agent per active leaf, restricted to a write set.""" + +from __future__ import annotations + +import logging +import os +import time +from typing import Any + +from codewiki.src.be.agent_tools.deps import CodeWikiDeps +from codewiki.src.be.backend import LLMBackend +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.updater import pages as P +from codewiki.src.be.updater.change_report import MODE_CREATE, MODE_DELETE, MODE_EDIT, LeafReport +from codewiki.src.be.updater.graph_diff import GraphDiff +from codewiki.src.be.updater.options import UpdateOptions +from codewiki.src.be.updater.prompts import format_update_system_prompt, format_update_user_prompt +from codewiki.src.be.updater.record import CallCost, PageVerdict, UpdateRecord +from codewiki.src.be.updater.verdicts import parse_verdicts +from codewiki.src.config import Config + +logger = logging.getLogger(__name__) + + +class LeafAgentRunner: + def __init__( + self, + config: Config, + backend: LLMBackend, + docs_dir: str, + graph: dict[str, Node], + tree: dict[str, Any], + diff: GraphDiff, + opts: UpdateOptions, + record: UpdateRecord, + ) -> None: + self.config = config + self.backend = backend + self.docs_dir = docs_dir + self.graph = graph + self.tree = tree + self.diff = diff + self.opts = opts + self.record = record + self.custom_instructions = config.get_prompt_addition() + + # ------------------------------------------------------------------ utils + def _deps( + self, leaf_name: str, module_path: list[str], allowed: set[str] | None + ) -> CodeWikiDeps: + return CodeWikiDeps( + absolute_docs_path=self.docs_dir, + absolute_repo_path=str(os.path.abspath(self.config.repo_path)), + registry={}, + components=self.graph, + path_to_current_module=module_path, + current_module_name=leaf_name, + module_tree=self.tree, + max_depth=self.config.max_depth, + current_depth=1, + config=self.config, + custom_instructions=self.custom_instructions, + allowed_write_paths=allowed, + ) + + def _remove_page(self, stem: str, by_leaf: str, reason: str) -> None: + path = P.page_path(self.docs_dir, stem) + if os.path.exists(path): + os.remove(path) + self.record.pages_removed.append(stem) + self.record.add_verdict(PageVerdict(stem, "delete", reason, by_leaf, True)) + + async def _regenerate_leaf( + self, leaf_name: str, module_path: list[str], component_ids: list[str], why: str + ) -> None: + """Delete the page (if any) and let the normal module agent write it anew.""" + path = P.page_path(self.docs_dir, leaf_name) + existed = os.path.exists(path) + if existed: + os.remove(path) + started = time.time() + err = None + try: + await self.backend.run_module_agent( + module_name=leaf_name, + components=self.graph, + core_component_ids=component_ids, + module_path=module_path, + working_dir=self.docs_dir, + ) + except Exception as e: # noqa: BLE001 — recorded, the run continues + err = f"{type(e).__name__}: {e}" + logger.error("Regenerating %s failed: %s", leaf_name, e) + self.record.add_call( + CallCost( + "rewrite" if existed else "create", + leaf_name, + time.time() - started, + getattr(self.backend, "last_usage", None), + err, + ) + ) + self.record.add_verdict( + PageVerdict( + leaf_name, "rewrite" if existed else "create", why, leaf_name, os.path.exists(path) + ) + ) + if os.path.exists(path): + self.record.pages_written.append(leaf_name) + + async def _run_editing_agent( + self, + leaf_name: str, + module_path: list[str], + mode: str, + roles: dict[str, list[str]], + report: LeafReport, + component_ids: list[str], + kind: str = "leaf_agent", + ) -> dict[str, dict[str, str]]: + """Run the update agent over ``roles`` (page -> roles). Returns verdicts.""" + if not roles: + return {} + allowed = {P.page_path(self.docs_dir, stem) for stem in roles} + deps = self._deps(leaf_name, module_path, allowed) + system_prompt = format_update_system_prompt(leaf_name, self.custom_instructions) + user_prompt = format_update_user_prompt( + leaf_name=leaf_name, + mode=mode, + roles=roles, + report=report, + diff=self.diff, + tree=self.tree, + component_ids=component_ids, + graph=self.graph, + leaf_page_text=P.read_page(self.docs_dir, leaf_name) if leaf_name in roles else None, + ) + before = P.page_hashes(self.docs_dir) + started = time.time() + err = None + text = "" + usage = None + try: + reply = await self.backend.run_update_agent(system_prompt, user_prompt, deps) + text, usage = reply.text, reply.usage + except Exception as e: # noqa: BLE001 — recorded, the run continues + err = f"{type(e).__name__}: {e}" + logger.error("Update agent for %s failed: %s", leaf_name, e) + seconds = time.time() - started + self.record.add_call(CallCost(kind, leaf_name, seconds, usage, err)) + after = P.page_hashes(self.docs_dir) + changed = P.changed_pages(before, after) + verdicts, notes = parse_verdicts(text) + if notes: + self.record.detector_notes.append(f"[{leaf_name}] agent notes: {notes[:500]}") + for stem in roles: + v = verdicts.get(stem) + on_disk = stem in changed + if v is None: + verdict = "patch" if on_disk else "no-op" + reason = "no verdict returned by agent" + ( + " (page changed on disk)" if on_disk else "" + ) + else: + verdict, reason = v["verdict"], v["reason"] + if verdict not in ("no-op", "patch", "rewrite"): + verdict = "patch" if on_disk else "no-op" + if verdict == "no-op" and on_disk: + verdict, reason = "patch", (reason + " [page changed on disk]").strip() + if verdict == "patch" and not on_disk: + reason = (reason + " [no change on disk]").strip() + self.record.add_verdict(PageVerdict(stem, verdict, reason, leaf_name, on_disk)) + if on_disk: + self.record.pages_written.append(stem) + verdicts[stem] = {"verdict": verdict, "reason": reason} + for stem in changed - set(roles): + self.record.write_set_violations.append( + {"leaf": leaf_name, "page": stem, "note": "changed outside the write set"} + ) + logger.error("Write-set violation: %s changed while updating %s", stem, leaf_name) + return verdicts + + # ------------------------------------------------------------------- main + async def run( + self, + report: LeafReport, + write_roles: dict[str, list[str]], + component_ids: list[str], + ) -> None: + """``write_roles``: page stem -> roles, restricted to pages that exist + (plus the leaf page itself in edit mode).""" + leaf_name = report.page + module_path = list(report.leaf_path) + related = {p: r for p, r in write_roles.items() if p != leaf_name} + self.record.write_sets[leaf_name] = sorted(write_roles) + + if report.mode == MODE_DELETE: + self._remove_page(leaf_name, leaf_name, "module removed from the tree") + if related and self.opts.agent_patches_related: + await self._run_editing_agent( + leaf_name, module_path, MODE_DELETE, related, report, component_ids + ) + elif related: + self._legacy_invalidate(related, leaf_name) + return + + if report.mode == MODE_CREATE or not P.page_exists(self.docs_dir, leaf_name): + await self._regenerate_leaf(leaf_name, module_path, component_ids, "new module page") + if related and self.opts.agent_patches_related: + await self._run_editing_agent( + leaf_name, module_path, MODE_CREATE, related, report, component_ids + ) + elif related: + self._legacy_invalidate(related, leaf_name) + return + + # edit mode + if not self.opts.agent_may_patch_leaf: + await self._regenerate_leaf( + leaf_name, module_path, component_ids, "rewrite_always (ablation rung)" + ) + if related and self.opts.agent_patches_related: + await self._run_editing_agent( + leaf_name, module_path, "related_only", related, report, component_ids + ) + elif related: + self._legacy_invalidate(related, leaf_name) + return + + roles = {leaf_name: ["leaf"], **related} + verdicts = await self._run_editing_agent( + leaf_name, module_path, MODE_EDIT, roles, report, component_ids + ) + own = verdicts.get(leaf_name, {}) + if own.get("verdict") == "rewrite": + await self._regenerate_leaf( + leaf_name, module_path, component_ids, own.get("reason", "agent chose rewrite") + ) + + def _legacy_invalidate(self, related: dict[str, list[str]], by_leaf: str) -> None: + """Rung 1 behaviour: ancestors are deleted and regenerated later; other + related pages are left alone.""" + for stem, roles in related.items(): + if "ancestor" in roles: + self._remove_page(stem, by_leaf, "ancestor invalidated (rung 1)") diff --git a/codewiki/src/be/updater/options.py b/codewiki/src/be/updater/options.py new file mode 100644 index 00000000..b6b8e446 --- /dev/null +++ b/codewiki/src/be/updater/options.py @@ -0,0 +1,77 @@ +"""Hyperparameters and ablation rungs for the incremental updater. + +Every threshold the method depends on lives here with its default. The +ablation ladder (rungs 0..3 and 3b) is expressed as a preset over the same +fields so a run is fully described by one ``UpdateOptions`` value. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + +VALID_RUNGS = ("0", "1", "2", "3", "3b") + + +@dataclass +class UpdateOptions: + # Ablation rung. "0" = legacy file-level invalidation (not handled by this + # package), "3" = the proposed method, "3b" = rung 3 with k_hop = 2. + rung: str = "3" + + # Step 1: graph diff + tau_ren: float = 0.95 # body similarity to call a delete+add pair a rename + max_diff_tokens: int = 8000 # cap on one component diff inside a report + + # Step 2: tree repair + tau_nb: float = 0.5 # share of graph neighbours in one leaf to route there + tau_grow: float = 0.33 # share of new components that triggers re-clustering + + # Step 3: change report + k_hop: int = 1 # dependency hops followed for Up + + # Step 4: fallback + tau_full: float = 0.5 # active leaves / all leaves + tau_tree: float = 0.3 # (created + deleted + re-clustered) / all leaves + + # Derived toggles, set by ``from_rung`` (kept as fields so a record shows them). + use_routing_agent: bool = True # rule 4 orphans go to an agent + use_growth_recluster: bool = True + use_up: bool = True # dependents receive Up + agent_may_patch_leaf: bool = True # False = rewrite the leaf page always + agent_patches_related: bool = True # False = delete ancestors and regenerate + use_stale_scan: bool = True + + extra: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_rung(cls, rung: str | int, **overrides: Any) -> "UpdateOptions": + rung = str(rung) + if rung not in VALID_RUNGS: + raise ValueError(f"unknown update rung {rung!r}; expected one of {VALID_RUNGS}") + opts = cls(rung=rung) + if rung == "1": + opts.use_routing_agent = False + opts.use_growth_recluster = False + opts.use_up = False + opts.agent_may_patch_leaf = False + opts.agent_patches_related = False + opts.use_stale_scan = False + elif rung == "2": + opts.agent_may_patch_leaf = False + elif rung == "3b": + opts.k_hop = 2 + for key, value in overrides.items(): + if value is None: + continue + if not hasattr(opts, key): + raise ValueError(f"unknown update option {key!r}") + setattr(opts, key, value) + return opts + + @property + def is_legacy(self) -> bool: + return self.rung == "0" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) diff --git a/codewiki/src/be/updater/orchestrator.py b/codewiki/src/be/updater/orchestrator.py new file mode 100644 index 00000000..83884b6c --- /dev/null +++ b/codewiki/src/be/updater/orchestrator.py @@ -0,0 +1,416 @@ +"""The incremental updater: Steps 1-6 in order, one record for everything.""" + +from __future__ import annotations + +import logging +import os +import time +import traceback +from typing import Any + +from codewiki.src.be.backend import LLMBackend +from codewiki.src.be.cluster_modules import cluster_modules +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.updater import pages as P +from codewiki.src.be.updater import tree as T +from codewiki.src.be.updater.change_report import ( + MODE_DELETE, + LeafReport, + active_set, + build_reports, + fallback_ratios, + order_active, +) +from codewiki.src.be.updater.graph_diff import GraphDiff, diff_graphs +from codewiki.src.be.updater.graph_store import load_graph +from codewiki.src.be.updater.leaf_agent import LeafAgentRunner +from codewiki.src.be.updater.options import UpdateOptions +from codewiki.src.be.updater.record import ( + OUTCOME_DETECTOR_FAILURE, + OUTCOME_FULL_FALLBACK, + OUTCOME_INCREMENTAL, + OUTCOME_NO_CHANGE, + CallCost, + UpdateRecord, +) +from codewiki.src.be.updater.reference_index import ( + build_reference_index, + inverse, + load_reference_index, + save_reference_index, +) +from codewiki.src.be.updater.routing import RoutingAgent +from codewiki.src.be.updater.stale_scan import StaleScanner +from codewiki.src.be.updater.tree_repair import RepairResult, repair_tree +from codewiki.src.config import MODULE_TREE_FILENAME, Config +from codewiki.src.utils import file_manager + +logger = logging.getLogger(__name__) + + +class IncrementalUpdater: + """Runs one incremental step over an existing docs directory. + + ``doc_generator`` is the normal ``DocumentationGenerator``; its + ``generate_module_documentation`` regenerates whatever page is missing + after the leaf agents ran (Step 6.1). + """ + + def __init__( + self, + config: Config, + backend: LLMBackend, + doc_generator: Any, + opts: UpdateOptions, + ) -> None: + self.config = config + self.backend = backend + self.doc_generator = doc_generator + self.opts = opts + self.docs_dir = os.path.abspath(config.docs_dir) + self.repo_name = os.path.basename(os.path.normpath(config.repo_path)) + self.whole_repo = False + self._deleted_nodes: list[tuple[str, ...]] = [] + self.record = UpdateRecord(options=opts.to_dict()) + + # ------------------------------------------------------------------ steps + def _load_state(self, old_graph_path: str) -> tuple[dict[str, Node], dict[str, Any]]: + old_graph = load_graph(old_graph_path) + tree = file_manager.load_json(os.path.join(self.docs_dir, MODULE_TREE_FILENAME)) + if tree is None: + raise FileNotFoundError(f"{MODULE_TREE_FILENAME} missing in {self.docs_dir}") + if len(tree) == 0: + self.whole_repo = True + tree = T.virtual_whole_repo_tree(P.OVERVIEW_STEM, sorted(old_graph)) + self.record.detector_notes.append("whole-repository mode: one virtual leaf (overview)") + return old_graph, tree + + def _module_path(self, path: tuple[str, ...]) -> list[str]: + if self.whole_repo and path == (P.OVERVIEW_STEM,): + return [] + return list(path) + + def _recluster( + self, + tree: dict[str, Any], + flagged: list[tuple[str, ...]], + new_graph: dict[str, Node], + tracked_new: set[str], + ) -> tuple[set[tuple[str, ...]], set[str]]: + """Re-cluster the parent subtree of every growth-flagged leaf. + + Returns the set of unit paths whose pages must be regenerated and the + set of page stems removed from disk.""" + reclustered: set[tuple[str, ...]] = set() + removed_pages: set[str] = set() + parents = sorted({p[:-1] for p in flagged if len(p) > 1}, key=len) + done: set[tuple[str, ...]] = set() + for parent in parents: + if any(parent[: len(d)] == d for d in done): + continue # an ancestor was already re-clustered + info = T.node_at(tree, parent) + if info is None: + continue + comps = [c for c in T.components_of(info) if c in new_graph and c in tracked_new] + if not comps: + continue + old_units = [p for p, _ in T.iter_nodes(info.get("children", {}), parent)] + cluster_model = self.config.cluster_model or None + started = time.time() + err = None + try: + info["children"] = {} + sub = cluster_modules( + comps, + new_graph, + self.config, + current_module_tree=tree, + current_module_name=parent[-1], + current_module_path=list(parent), + completer=lambda p, m=cluster_model: self.backend.complete(p, model=m), + ) + if not sub: + info["children"] = {} + info["components"] = comps + except Exception as e: # noqa: BLE001 — recorded; subtree left as is + err = f"{type(e).__name__}: {e}" + logger.error("Re-clustering %s failed: %s", "/".join(parent), e) + self.record.errors.append(f"recluster {'/'.join(parent)}: {err}") + self.record.add_call( + CallCost( + "recluster", + "/".join(parent), + time.time() - started, + getattr(self.backend, "last_usage", None), + err, + ) + ) + if err: + continue + done.add(parent) + for p in old_units: + stem = p[-1] + if P.page_exists(self.docs_dir, stem): + os.remove(P.page_path(self.docs_dir, stem)) + removed_pages.add(stem) + self.record.pages_removed.append(stem) + if P.page_exists(self.docs_dir, parent[-1]): + os.remove(P.page_path(self.docs_dir, parent[-1])) + removed_pages.add(parent[-1]) + self.record.pages_removed.append(parent[-1]) + new_info = T.node_at(tree, parent) or {} + for p, _ in T.iter_nodes(new_info.get("children", {}), parent): + reclustered.add(p) + if T.is_leaf(new_info): + reclustered.add(parent) + self.record.reclustered.append(list(parent)) + return reclustered, removed_pages + + def _write_roles( + self, + report: LeafReport, + new_tree: dict[str, Any], + dep: dict[tuple[str, ...], set[tuple[str, ...]]], + inv: dict[str, set[str]], + ) -> dict[str, list[str]]: + stem = report.page + roles: dict[str, list[str]] = {} + if report.mode != MODE_DELETE: + roles[stem] = ["leaf"] + for anc in T.ancestors(report.leaf_path): + roles.setdefault(anc[-1], []).append("ancestor") + if stem != P.OVERVIEW_STEM: + roles.setdefault(P.OVERVIEW_STEM, []).append("ancestor") + for d in sorted(dep.get(report.leaf_path, ())): + roles.setdefault(d[-1], []).append("dependent") + for page in sorted(inv.get(stem, ())): + roles.setdefault(page, []).append("referrer") + existing = set(P.list_pages(self.docs_dir)) + doomed = {p[-1] for p in self._deleted_nodes} - {stem} + return { + p: r + for p, r in roles.items() + if (p in existing or (p == stem and report.mode != MODE_DELETE)) and p not in doomed + } + + # ------------------------------------------------------------------- main + async def run( + self, + old_graph_path: str | None, + new_graph: dict[str, Node], + leaf_nodes: list[str], + revision: dict[str, Any], + ) -> UpdateRecord: + rec = self.record + rec.revision = dict(revision) + t0 = time.time() + try: + return await self._run(old_graph_path, new_graph, leaf_nodes) + finally: + rec.wall_seconds = round(time.time() - t0, 2) + rec.finished_at = time.strftime("%Y-%m-%dT%H:%M:%S%z") + try: + rec.save(self.docs_dir) + except OSError as e: + logger.error("Could not save update record: %s", e) + + async def _run( + self, old_graph_path: str | None, new_graph: dict[str, Node], leaf_nodes: list[str] + ) -> UpdateRecord: + rec = self.record + # ---- Step 0: load what the previous build left behind + try: + if not old_graph_path or not os.path.exists(old_graph_path): + raise FileNotFoundError("previous dependency graph not found") + old_graph, old_tree = self._load_state(old_graph_path) + except Exception as e: # noqa: BLE001 — a detector failure is an outcome, not a crash + rec.outcome = OUTCOME_DETECTOR_FAILURE + rec.errors.append(f"load previous state: {type(e).__name__}: {e}") + logger.warning("Incremental update impossible (%s); falling back to a full build", e) + return rec + + ref_index = load_reference_index(self.docs_dir) + if ref_index is None: + ref_index = build_reference_index( + self.docs_dir, old_graph, None if self.whole_repo else old_tree + ) + rec.detector_notes.append( + "reference index rebuilt from pages (none saved by previous build)" + ) + + # ---- Step 1: diff + diff: GraphDiff = diff_graphs(old_graph, new_graph, self.opts) + rec.diff = diff.to_dict() + if diff.is_empty: + rec.outcome = OUTCOME_NO_CHANGE + logger.info("No component-level change detected; documentation is up to date") + return rec + + # ---- Step 2: repair + tracked_new = set(leaf_nodes) | (T.tracked_ids(old_tree) & set(new_graph)) + router = ( + RoutingAgent(self.backend, self.docs_dir, rec, self.config.cluster_model or None) + if self.opts.use_routing_agent + else None + ) + repair: RepairResult = repair_tree( + old_tree, diff, new_graph, tracked_new, self.opts, route_orphans=router + ) + new_tree = repair.tree + reclustered: set[tuple[str, ...]] = set() + removed_pages: set[str] = set() + if self.opts.use_growth_recluster and repair.growth_flagged and not self.whole_repo: + reclustered, removed_pages = self._recluster( + new_tree, repair.growth_flagged, new_graph, tracked_new + ) + rec.repair = repair.to_dict() + self._deleted_nodes = list(repair.deleted_nodes) + + # ---- Step 3: reports + reports = build_reports( + diff, + old_tree, + new_tree, + old_graph, + new_graph, + ref_index, + repair, + self.opts, + reclustered, + ) + active = active_set(reports) + rec.reports = {"/".join(p): r.to_dict() for p, r in reports.items() if p in active} + + # ---- Step 4: fallback check + ratios = fallback_ratios(reports, new_tree, repair, reclustered) + ratios["tau_full"] = self.opts.tau_full + ratios["tau_tree"] = self.opts.tau_tree + ratios["fired"] = ( + ratios["r_leaf"] >= self.opts.tau_full or ratios["r_tree"] >= self.opts.tau_tree + ) + if self.whole_repo and ratios["fired"]: + # One virtual leaf: any change is 100% active by construction, and + # patching that single page is exactly the incremental step. + ratios["fired"] = False + ratios["note"] = "whole-repository mode: fallback rule not applied" + rec.fallback = ratios + if ratios["fired"]: + rec.outcome = OUTCOME_FULL_FALLBACK + logger.warning( + "Fallback to full build: r_leaf=%.2f (tau %.2f), r_tree=%.2f (tau %.2f)", + ratios["r_leaf"], + self.opts.tau_full, + ratios["r_tree"], + self.opts.tau_tree, + ) + return rec + + # Persist the repaired tree so the normal pipeline and the agents see it. + if not self.whole_repo: + file_manager.save_json(new_tree, os.path.join(self.docs_dir, MODULE_TREE_FILENAME)) + + # ---- Step 5: sequential leaf agents + order = order_active(active, new_tree, new_graph) + dep = T.leaf_dependents(new_tree, new_graph) + inv = inverse(ref_index) + rec.active = [ + {"leaf": "/".join(p), "page": reports[p].page, "mode": reports[p].mode, "order": i} + for i, p in enumerate(order) + ] + runner = LeafAgentRunner( + self.config, self.backend, self.docs_dir, new_graph, new_tree, diff, self.opts, rec + ) + for path in order: + report = reports[path] + roles = self._write_roles(report, new_tree, dep, inv) + info = T.node_at(new_tree, path) or {} + component_ids = [c for c in T.components_of(info) if c in new_graph] + if self.whole_repo: + component_ids = [c for c in leaf_nodes if c in new_graph] + report.leaf_path = tuple(path) + try: + logger.info( + "Updating leaf %s (mode=%s, write set=%s)", + "/".join(path), + report.mode, + sorted(roles), + ) + if self.whole_repo: + # The virtual leaf is the overview page with an empty module path. + await runner.run( + _WholeRepoReport(report, P.OVERVIEW_STEM), roles, component_ids + ) + else: + await runner.run(report, roles, component_ids) + except Exception as e: # noqa: BLE001 — one failed leaf must not abort the update + rec.errors.append(f"leaf {'/'.join(path)}: {type(e).__name__}: {e}") + logger.error("Leaf %s failed: %s\n%s", "/".join(path), e, traceback.format_exc()) + + # ---- Step 6.1: generate any page still missing (new parents, re-clustered subtrees, root) + before = P.page_hashes(self.docs_dir) + started = time.time() + err = None + try: + await self.doc_generator.generate_module_documentation(new_graph, leaf_nodes) + except Exception as e: # noqa: BLE001 — recorded + err = f"{type(e).__name__}: {e}" + rec.errors.append(f"missing pages: {err}") + logger.error("Generating missing pages failed: %s", e) + created = sorted(P.changed_pages(before, P.page_hashes(self.docs_dir))) + rec.add_call( + CallCost("missing_pages", ",".join(created) or "-", time.time() - started, None, err) + ) + rec.pages_written.extend(created) + + # ---- Step 6.2: stale-name scan over pages not written this round + removed_all = set(rec.pages_removed) | removed_pages + if self.opts.use_stale_scan: + replacements = {stem: self._nearest_page(stem, new_tree) for stem in removed_all} + scanner = StaleScanner( + self.config, self.backend, self.docs_dir, new_graph, new_tree, rec + ) + rec.stale_scan = await scanner.run( + diff, old_graph, set(rec.pages_written), removed_all, replacements + ) + + # ---- Step 6.3: rebuild the reference index + new_index = build_reference_index( + self.docs_dir, new_graph, None if self.whole_repo else new_tree + ) + save_reference_index(new_index, self.docs_dir) + rec.outcome = OUTCOME_INCREMENTAL + return rec + + def _nearest_page(self, removed_stem: str, new_tree: dict[str, Any]) -> str | None: + """Best existing page to redirect a dangling link to: the removed module's + nearest surviving ancestor, else the overview.""" + for path, _ in T.iter_nodes(new_tree): + if path[-1] == removed_stem: + return None + # Look the removed module up in the record's deleted nodes to find its parent. + for p in self.record.repair.get("deleted_nodes", []): + if p and p[-1] == removed_stem: + for anc in reversed(p[:-1]): + if P.page_exists(self.docs_dir, anc): + return anc + return P.OVERVIEW_STEM if P.page_exists(self.docs_dir, P.OVERVIEW_STEM) else None + + +class _WholeRepoReport: + """Proxy so the leaf agent treats the whole-repo virtual leaf as the + overview page with an empty module path.""" + + def __init__(self, report: LeafReport, page: str) -> None: + self._r = report + self._page = page + + def __getattr__(self, item: str) -> Any: + return getattr(self._r, item) + + @property + def page(self) -> str: + return self._page + + @property + def leaf_path(self) -> tuple[str, ...]: + return tuple() diff --git a/codewiki/src/be/updater/pages.py b/codewiki/src/be/updater/pages.py new file mode 100644 index 00000000..bf4f543e --- /dev/null +++ b/codewiki/src/be/updater/pages.py @@ -0,0 +1,51 @@ +"""Small helpers over the flat docs directory (page stems <-> files, hashes).""" + +from __future__ import annotations + +import hashlib +import os + +from codewiki.src.config import OVERVIEW_FILENAME + +OVERVIEW_STEM = OVERVIEW_FILENAME[: -len(".md")] + + +def page_path(docs_dir: str, stem: str) -> str: + return os.path.join(docs_dir, f"{stem}.md") + + +def page_exists(docs_dir: str, stem: str) -> bool: + return os.path.isfile(page_path(docs_dir, stem)) + + +def read_page(docs_dir: str, stem: str) -> str | None: + try: + with open(page_path(docs_dir, stem), encoding="utf-8") as f: + return f.read() + except OSError: + return None + + +def list_pages(docs_dir: str) -> list[str]: + try: + return sorted( + f[:-3] for f in os.listdir(docs_dir) if f.endswith(".md") and not f.startswith(".") + ) + except OSError: + return [] + + +def page_hashes(docs_dir: str) -> dict[str, str]: + out = {} + for stem in list_pages(docs_dir): + try: + with open(page_path(docs_dir, stem), "rb") as f: + out[stem] = hashlib.sha1(f.read()).hexdigest() + except OSError: + continue + return out + + +def changed_pages(before: dict[str, str], after: dict[str, str]) -> set[str]: + """Pages created, removed, or whose bytes changed.""" + return {s for s in set(before) | set(after) if before.get(s) != after.get(s)} diff --git a/codewiki/src/be/updater/prompts.py b/codewiki/src/be/updater/prompts.py new file mode 100644 index 00000000..828cc672 --- /dev/null +++ b/codewiki/src/be/updater/prompts.py @@ -0,0 +1,343 @@ +"""Prompts for the incremental updater's agents. + +Three agents: the per-leaf editing agent (Step 5), the orphan routing agent +(Step 2, rule 4) and the stale-name fixer (Step 6). Every agent that edits +pages must end its answer with a fenced JSON verdict block. +""" + +from __future__ import annotations + +import json +from typing import Any + +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.prompt_template import _fence_language, _format_module_tree_str +from codewiki.src.be.updater.change_report import LeafReport +from codewiki.src.be.updater.graph_diff import GraphDiff + +VERDICT_VALUES = ("no-op", "patch", "rewrite") + +UPDATE_LEAF_SYSTEM_PROMPT = """ + +You maintain an existing documentation wiki for a code repository. The code moved to a new +revision. You receive a precise report of what changed for ONE module ("the leaf") and you +bring the affected documentation pages up to date with the smallest correct edits. + + + +1. You may edit ONLY the pages in the WRITE SET below. The editor tool refuses anything else. + If another page needs a change, say so in your final verdict instead of trying. +2. Prefer surgical edits with `str_replace_editor` (`str_replace` / `insert`). Keep every + sentence that is still true word for word. Do not reflow, restyle, or "improve" prose that + the report does not touch. +3. Per page role: + - the LEAF PAGE ({leaf_name}.md): update sections, tables and diagrams that describe changed + components; add new components; remove deleted ones. If the change is so large that the + page is better rebuilt from scratch, do NOT rebuild it yourself: return verdict "rewrite" + for this page and leave it untouched (the normal module agent will regenerate it). + - an ANCESTOR page: change only where it summarizes this leaf or lists its children. + - a DEPENDENT page: change only where it describes the contract of a component listed under + UP (a signature that moved, a component that was deleted or renamed) or a call that no + longer exists. + - a REFERRER page: change only where it names a component or page listed under REFCH. +4. Use `read_code_components` to read the fresh code of any component id when the diff alone is + not enough. Use `str_replace_editor` with `working_dir="docs"` and `command="view"` to read a + page before editing it. +5. Mermaid diagrams must stay valid. Links between pages are relative: `[text](page.md)`. + + + +When you are done, end your answer with exactly one fenced JSON block: +```json +{{"verdicts": {{".md": {{"verdict": "no-op|patch|rewrite", "reason": ""}}, ...}}, + "notes": ""}} +``` +Give a verdict for EVERY page in the write set. "patch" means you edited it; "no-op" means it is +already correct; "rewrite" is allowed only for the leaf page. + +{custom_instructions} +""".strip() + +UPDATE_LEAF_USER_PROMPT = """ +Update the documentation for the module `{leaf_name}` (mode: {mode}). + +{mode_note} + + +{write_set} + + + +{report} + + + +{module_tree} + + + +{leaf_components} + + + +{leaf_page} + + +Work through the write set page by page, then end with the JSON verdict block. +""".strip() + +MODE_NOTES = { + "edit": ( + "The leaf page exists. Decide per page: patch in place, no-op, or (leaf page only) " + "'rewrite' if a fresh page would be better than patching." + ), + "create": ( + "The leaf page was just generated from scratch and must NOT be changed here. Your job is " + "the related pages: make ancestors list and summarize the new module, and fix any " + "referrer that should now point at it." + ), + "delete": ( + "This module no longer exists and its page has been removed. Update the related pages: " + "drop it from ancestor summaries and child lists, and remove or redirect every link or " + "mention of it on the referrer pages." + ), + "related_only": ( + "The leaf page was regenerated from scratch by the normal module agent and must NOT be " + "changed here. Update the related pages so they match the regenerated leaf page." + ), +} + +ROUTING_SYSTEM_PROMPT = """ +You place newly added code components into an existing module tree of a documentation wiki. +Answer with JSON only. +""".strip() + +ROUTING_USER_PROMPT = """ +The repository changed and these new components could not be placed by rules (same file, +same directory, or majority of graph neighbours). Place each one. + + +{module_tree} + + + +{orphans} + + +For each orphan choose exactly one action: +- "place": add it to an existing leaf module ("leaf": exact module name from the tree). +- "create": create a new leaf module ("new_leaf": short snake_case name, "parent": exact name of an + existing module that has children, or null for top level). Use the same new_leaf name for + orphans that belong together; a new subsystem landing in one commit should become one new leaf. +- "untracked": leave it out of the wiki (tests, throwaway scripts, trivial helpers). + +Return exactly one fenced JSON block: +```json +{{"decisions": [{{"component_id": "...", "action": "place|create|untracked", "leaf": "...", + "new_leaf": "...", "parent": "...", "reason": ""}}]}} +``` +""".strip() + +STALE_FIX_SYSTEM_PROMPT = """ +You fix stale references in ONE documentation page after a code change. Make the smallest edits +that remove or correct the stale items; everything else on the page stays word for word. +End your answer with a fenced JSON block: {"verdicts": {".md": {"verdict": "patch|no-op", +"reason": "..."}}}. +""".strip() + +STALE_FIX_USER_PROMPT = """ +Page to fix: `{page}.md` (view it with str_replace_editor, working_dir="docs"). + + +{items} + + +Rules: a renamed component gets its new id/name; a deleted component or page is removed from +prose, tables, lists and diagrams (or the sentence is rephrased so it stays true); a link to a +page that no longer exists is removed or repointed to the page listed as its replacement. +""".strip() + + +def _clip(text: str, limit: int) -> str: + if len(text) <= limit: + return text + return text[: limit // 2] + "\n... [clipped] ...\n" + text[-(limit // 2) :] + + +def render_record(diff: GraphDiff, cid: str, max_chars: int = 40_000) -> str: + rec = diff.record_for(cid) + if rec is None: + return f"- {cid}" + lines = [f"### {cid} [{rec.change_class}]"] + if rec.change_class == "renamed": + lines.append(f"renamed from `{rec.old_id}` to `{rec.new_id}`") + if rec.signature_changed or rec.change_class == "interface": + lines.append(f"signature before: {json.dumps(rec.old_signature)}") + lines.append(f"signature after: {json.dumps(rec.new_signature)}") + if rec.edges_added or rec.edges_removed: + lines.append(f"now uses: {rec.edges_added}; no longer uses: {rec.edges_removed}") + if rec.diff: + lines.append("```diff\n" + _clip(rec.diff, max_chars) + "\n```") + return "\n".join(lines) + + +def render_report(report: LeafReport, diff: GraphDiff) -> str: + parts: list[str] = [] + if report.own: + parts.append("## OWN — components of this leaf that changed") + parts += [render_record(diff, c) for c in report.own] + if report.up: + parts.append( + "## UP — components outside this leaf that its code uses and whose contract moved" + ) + parts += [render_record(diff, c) for c in report.up] + if report.context: + parts.append("## CONTEXT — changed components not tracked by any module, next to this leaf") + parts += [render_record(diff, c) for c in report.context] + if report.refch: + parts.append("## REFCH — things this leaf's page refers to that changed or vanished") + parts += [f"- {x}" for x in report.refch] + tree_lines = [] + if report.entered: + tree_lines.append(f"- components that entered this module: {report.entered}") + if report.left: + tree_lines.append(f"- components that left this module: {report.left}") + if report.children_added: + tree_lines.append(f"- child modules added: {report.children_added}") + if report.children_removed: + tree_lines.append(f"- child modules removed: {report.children_removed}") + if report.reclustered: + tree_lines.append("- this subtree was re-clustered") + if tree_lines: + parts.append("## TREE — structural changes") + parts += tree_lines + return ( + "\n".join(parts) + if parts + else "(no direct changes; this leaf is active for structural reasons)" + ) + + +def render_write_set(roles: dict[str, list[str]]) -> str: + """``roles``: page stem -> list of roles (leaf/ancestor/dependent/referrer).""" + lines = [] + for page, rs in roles.items(): + lines.append(f"- {page}.md ({', '.join(rs)})") + return "\n".join(lines) if lines else "(empty)" + + +def render_leaf_components( + component_ids: list[str], + graph: dict[str, Node], + changed: set[str], + max_code_chars: int = 60_000, +) -> str: + """List every component of the leaf; inline fresh code only for changed ones.""" + lines = ["Components of this module (fresh revision):"] + for cid in component_ids: + node = graph.get(cid) + if node is None: + lines.append(f"- {cid} (no longer in the code graph)") + continue + sig = ", ".join(node.parameters or []) + lines.append( + f"- {cid} [{node.component_type}] ({sig}) lines {node.start_line}-{node.end_line}" + ) + budget = max_code_chars + shown = 0 + for cid in component_ids: + node = graph.get(cid) + if node is None or cid not in changed or not node.source_code: + continue + code = node.source_code + if budget <= 0: + lines.append(f"\n(code of {cid} omitted for length; use read_code_components)") + continue + code = _clip(code, budget) + budget -= len(code) + shown += 1 + lines.append( + f'\n\n```{_fence_language(node.relative_path)}\n{code}\n```\n' + ) + if shown == 0: + lines.append("\n(no changed component code inlined; use read_code_components as needed)") + return "\n".join(lines) + + +def render_tree_outline(tree: dict[str, Any], current: str | None) -> str: + return _format_module_tree_str(tree, current, include_components=False) + + +def format_update_system_prompt(leaf_name: str, custom_instructions: str | None) -> str: + extra = ( + f"\n\n{custom_instructions}\n" + if custom_instructions + else "" + ) + return UPDATE_LEAF_SYSTEM_PROMPT.format(leaf_name=leaf_name, custom_instructions=extra) + + +def format_update_user_prompt( + *, + leaf_name: str, + mode: str, + roles: dict[str, list[str]], + report: LeafReport, + diff: GraphDiff, + tree: dict[str, Any], + component_ids: list[str], + graph: dict[str, Node], + leaf_page_text: str | None, +) -> str: + changed = set(report.own) + return UPDATE_LEAF_USER_PROMPT.format( + leaf_name=leaf_name, + mode=mode, + mode_note=MODE_NOTES.get(mode, MODE_NOTES["edit"]), + write_set=render_write_set(roles), + report=render_report(report, diff), + module_tree=render_tree_outline(tree, leaf_name), + leaf_components=render_leaf_components(component_ids, graph, changed), + leaf_page=( + leaf_page_text if leaf_page_text is not None else "(page does not exist / was removed)" + ), + ) + + +def format_routing_prompt( + tree_outline: str, + orphans: list[str], + graph: dict[str, Node], + neighbours: dict[str, list[str]], + max_code_chars: int = 6_000, +) -> str: + blocks = [] + for cid in orphans: + node = graph[cid] + code = _clip(node.source_code or "", max_code_chars) + nb = neighbours.get(cid) or [] + blocks.append( + f'\n' + f"neighbour modules in the code graph: {nb if nb else 'none'}\n" + f"```{_fence_language(node.relative_path)}\n{code}\n```\n" + ) + return ROUTING_USER_PROMPT.format(module_tree=tree_outline, orphans="\n".join(blocks)) + + +def format_stale_prompt(page: str, items: list[dict[str, Any]]) -> str: + lines = [] + for it in items: + kind = it.get("kind") + if kind == "renamed": + lines.append(f"- renamed component: `{it['old']}` is now `{it['new']}`") + elif kind == "deleted": + lines.append(f"- deleted component: `{it['old']}` no longer exists") + elif kind == "deleted_page": + repl = it.get("replacement") + lines.append( + f"- dangling link: `{it['old']}.md` no longer exists" + + (f"; nearest existing page: `{repl}.md`" if repl else "") + ) + else: + lines.append(f"- {json.dumps(it)}") + return STALE_FIX_USER_PROMPT.format(page=page, items="\n".join(lines)) diff --git a/codewiki/src/be/updater/record.py b/codewiki/src/be/updater/record.py new file mode 100644 index 00000000..5c0344ad --- /dev/null +++ b/codewiki/src/be/updater/record.py @@ -0,0 +1,116 @@ +"""The update record: every decision of one incremental step, written to +``update_record.json`` in the docs dir and summarised in ``metadata.json``.""" + +from __future__ import annotations + +import json +import os +import time +from dataclasses import asdict, dataclass, field +from typing import Any + +RECORD_FILENAME = "update_record.json" + +OUTCOME_NO_CHANGE = "no_change" +OUTCOME_INCREMENTAL = "incremental" +OUTCOME_FULL_FALLBACK = "full_fallback" +OUTCOME_DETECTOR_FAILURE = "detector_failure" + + +@dataclass +class CallCost: + kind: str # leaf_agent | rewrite | routing | stale_fix | missing_page | recluster + target: str + seconds: float + usage: dict[str, Any] | None = None + error: str | None = None + + +@dataclass +class PageVerdict: + page: str + verdict: str # no-op | patch | rewrite | create | delete + reason: str = "" + by_leaf: str = "" + changed_on_disk: bool | None = None + + +@dataclass +class UpdateRecord: + started_at: str = field(default_factory=lambda: time.strftime("%Y-%m-%dT%H:%M:%S%z")) + finished_at: str | None = None + outcome: str = OUTCOME_INCREMENTAL + options: dict[str, Any] = field(default_factory=dict) + revision: dict[str, Any] = field(default_factory=dict) # old/new commit, repo path + diff: dict[str, Any] = field(default_factory=dict) + repair: dict[str, Any] = field(default_factory=dict) + reclustered: list[list[str]] = field(default_factory=list) + reports: dict[str, Any] = field(default_factory=dict) + active: list[dict[str, Any]] = field(default_factory=list) # {leaf, mode, order} + write_sets: dict[str, list[str]] = field(default_factory=dict) + fallback: dict[str, Any] = field(default_factory=dict) + verdicts: list[dict[str, Any]] = field(default_factory=list) + pages_written: list[str] = field(default_factory=list) + pages_removed: list[str] = field(default_factory=list) + write_set_violations: list[dict[str, Any]] = field(default_factory=list) + stale_scan: dict[str, Any] = field(default_factory=dict) + calls: list[dict[str, Any]] = field(default_factory=list) + detector_notes: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + wall_seconds: float | None = None + + def add_call(self, cost: CallCost) -> None: + self.calls.append(asdict(cost)) + + def add_verdict(self, v: PageVerdict) -> None: + self.verdicts.append(asdict(v)) + + def summary(self) -> dict[str, Any]: + usage_total: dict[str, float] = {} + for c in self.calls: + for k, v in (c.get("usage") or {}).items(): + if isinstance(v, (int, float)): + usage_total[k] = usage_total.get(k, 0) + v + return { + "outcome": self.outcome, + "finished_at": self.finished_at, + "rung": self.options.get("rung"), + "revision": self.revision, + "diff_counts": self.diff.get("counts", {}), + "n_active": len(self.active), + "fallback": self.fallback, + "n_calls": len(self.calls), + "usage_total": usage_total, + "pages_written": sorted(set(self.pages_written)), + "pages_removed": sorted(set(self.pages_removed)), + "wall_seconds": self.wall_seconds, + } + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + def save(self, docs_dir: str) -> str: + path = os.path.join(docs_dir, RECORD_FILENAME) + with open(path, "w", encoding="utf-8") as f: + json.dump(self.to_dict(), f, indent=2, ensure_ascii=False, default=str) + return path + + +def merge_into_metadata(docs_dir: str, summary: dict[str, Any]) -> None: + """Append ``summary`` under ``last_update`` (and an ``update_history`` list).""" + path = os.path.join(docs_dir, "metadata.json") + meta: dict[str, Any] = {} + if os.path.exists(path): + try: + with open(path, encoding="utf-8") as f: + meta = json.load(f) or {} + except (OSError, json.JSONDecodeError): + meta = {} + meta["last_update"] = summary + history = meta.get("update_history") + if not isinstance(history, list): + history = [] + history.append(summary) + meta["update_history"] = history + with open(path, "w", encoding="utf-8") as f: + json.dump(meta, f, indent=4, ensure_ascii=False, default=str) diff --git a/codewiki/src/be/updater/reference_index.py b/codewiki/src/be/updater/reference_index.py new file mode 100644 index 00000000..4b3d8543 --- /dev/null +++ b/codewiki/src/be/updater/reference_index.py @@ -0,0 +1,138 @@ +"""Reference index: which pages and component ids each page links to or names. + +Built by parsing the markdown after each write. ``inverse`` answers "who +refers to x" for pages, component ids, and bare component names. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Any + +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.updater import tree as T + +INDEX_FILENAME = "reference_index.json" + +_LINK_RE = re.compile(r"\[[^\]]*\]\(\s*#]+\.md)(?:#[^)]*)?>?\s*\)") +_BARE_MD_RE = re.compile(r"(?-]+") +_CODE_RE = re.compile(r"`([^`\n]{2,120})`") +_WORD_RE = re.compile(r"[A-Za-z_][\w]*") + + +def index_path(docs_dir: str) -> str: + return os.path.join(docs_dir, "temp", INDEX_FILENAME) + + +def _page_stems(docs_dir: str) -> list[str]: + try: + return sorted( + os.path.splitext(f)[0] + for f in os.listdir(docs_dir) + if f.endswith(".md") and not f.startswith(".") + ) + except OSError: + return [] + + +def extract_references( + text: str, + known_ids: set[str], + known_names: dict[str, set[str]], + known_pages: set[str], +) -> dict[str, list[str]]: + """Return ``links`` (page stems), ``ids`` (component ids), ``names`` (bare names).""" + links: set[str] = set() + for m in _LINK_RE.finditer(text): + stem = os.path.splitext(os.path.basename(m.group(1)))[0] + links.add(stem) + for m in _BARE_MD_RE.finditer(text): + if m.group(1) in known_pages: + links.add(m.group(1)) + ids = {m.group(0) for m in _ID_RE.finditer(text) if m.group(0) in known_ids} + names: set[str] = set() + for m in _CODE_RE.finditer(text): + for w in _WORD_RE.findall(m.group(1)): + if w in known_names: + names.add(w) + return {"links": sorted(links), "ids": sorted(ids), "names": sorted(names)} + + +def build_reference_index( + docs_dir: str, graph: dict[str, Node], tree: dict[str, Any] | None = None +) -> dict[str, dict[str, list[str]]]: + known_ids = set(graph) + known_names: dict[str, set[str]] = {} + for cid, node in graph.items(): + name = node.name + if name and len(name) >= 3: + known_names.setdefault(name, set()).add(cid) + pages = _page_stems(docs_dir) + known_pages = set(pages) + if tree is not None: + known_pages |= {p[-1] for p, _ in T.iter_nodes(tree)} + index: dict[str, dict[str, list[str]]] = {} + for stem in pages: + try: + with open(os.path.join(docs_dir, f"{stem}.md"), encoding="utf-8") as f: + text = f.read() + except OSError: + continue + index[stem] = extract_references(text, known_ids, known_names, known_pages) + return index + + +def save_reference_index(index: dict[str, Any], docs_dir: str) -> str: + path = index_path(docs_dir) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(index, f, indent=2, ensure_ascii=False, sort_keys=True) + return path + + +def load_reference_index(docs_dir: str) -> dict[str, dict[str, list[str]]] | None: + path = index_path(docs_dir) + if not os.path.exists(path): + return None + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def inverse(index: dict[str, dict[str, list[str]]]) -> dict[str, set[str]]: + """Map every referenced page stem / id / name to the set of pages that mention it.""" + inv: dict[str, set[str]] = {} + for page, refs in index.items(): + for kind in ("links", "ids", "names"): + for x in refs.get(kind, []): + inv.setdefault(x, set()).add(page) + return inv + + +MIN_BARE_NAME_LEN = 5 + + +def unique_names_of(graph: dict[str, Node], ids: set[str]) -> set[str]: + """Bare names of ``ids`` that identify exactly one component in ``graph``. + + A mention like `update` could be any of several functions, so it is + ignored; `update_all_packages` names one thing and counts. + """ + counts: dict[str, int] = {} + for node in graph.values(): + if node.name: + counts[node.name] = counts.get(node.name, 0) + 1 + return { + graph[c].name + for c in ids + if c in graph + and graph[c].name + and len(graph[c].name) >= MIN_BARE_NAME_LEN + and counts.get(graph[c].name, 0) == 1 + } + + +def names_of(graph: dict[str, Node], ids: set[str]) -> set[str]: + return {graph[c].name for c in ids if c in graph and graph[c].name and len(graph[c].name) >= 3} diff --git a/codewiki/src/be/updater/routing.py b/codewiki/src/be/updater/routing.py new file mode 100644 index 00000000..4bea412a --- /dev/null +++ b/codewiki/src/be/updater/routing.py @@ -0,0 +1,129 @@ +"""Rule 4 of tree repair: the orphan routing agent (one single-shot call).""" + +from __future__ import annotations + +import logging +import time +from typing import Any + +from codewiki.src.be.backend import LLMBackend +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.updater import pages as P +from codewiki.src.be.updater import tree as T +from codewiki.src.be.updater.prompts import ROUTING_SYSTEM_PROMPT, format_routing_prompt +from codewiki.src.be.updater.record import CallCost, UpdateRecord +from codewiki.src.be.updater.tree_repair import RULE_AGENT, RoutingDecision +from codewiki.src.be.updater.verdicts import parse_json_block + +logger = logging.getLogger(__name__) + + +def _first_sentence(text: str | None) -> str: + if not text: + return "" + for line in text.splitlines(): + s = line.strip() + if not s or s.startswith("#") or s.startswith("```") or s.startswith("|"): + continue + return s[:160] + return "" + + +def tree_outline_with_summaries(tree: dict[str, Any], docs_dir: str) -> str: + lines = [] + for path, info in T.iter_nodes(tree): + name = path[-1] + kind = "leaf" if T.is_leaf(info) else "parent" + summary = _first_sentence(P.read_page(docs_dir, name)) + n = len(T.components_of(info)) + lines.append(f"{' ' * (len(path) - 1)}- {name} [{kind}, {n} components] {summary}") + return "\n".join(lines) + + +class RoutingAgent: + def __init__( + self, backend: LLMBackend, docs_dir: str, record: UpdateRecord, model: str | None = None + ): + self.backend = backend + self.docs_dir = docs_dir + self.record = record + self.model = model + + def __call__(self, orphans: list[str], context: dict[str, Any]) -> list[RoutingDecision]: + tree: dict[str, Any] = context["tree"] + owner: dict[str, tuple[str, ...]] = context["owner"] + graph: dict[str, Node] = context["graph"] + rev = T.reverse_edges(graph) + neighbours = {} + for cid in orphans: + nb = set(graph[cid].depends_on or ()) | rev.get(cid, set()) + resolved = {T.resolve_owner(owner, c) for c in nb} + neighbours[cid] = sorted({p[-1] for p in resolved if p is not None}) + prompt = ( + ROUTING_SYSTEM_PROMPT + + "\n\n" + + format_routing_prompt( + tree_outline_with_summaries(tree, self.docs_dir), orphans, graph, neighbours + ) + ) + started = time.time() + err = None + text = "" + try: + text = self.backend.complete(prompt, model=self.model) or "" + except Exception as e: # noqa: BLE001 — recorded; orphans stay untracked + err = f"{type(e).__name__}: {e}" + logger.error("Routing agent failed: %s", e) + self.record.add_call( + CallCost( + "routing", + f"{len(orphans)} orphans", + time.time() - started, + getattr(self.backend, "last_usage", None), + err, + ) + ) + data = parse_json_block(text) or {} + by_name = {path[-1]: path for path, _ in T.iter_nodes(tree)} + parents = {path[-1]: path for path, info in T.iter_nodes(tree) if not T.is_leaf(info)} + decisions: list[RoutingDecision] = [] + for d in data.get("decisions", []) or []: + if not isinstance(d, dict): + continue + cid = d.get("component_id") + if cid not in orphans: + continue + action = str(d.get("action", "")).lower() + reason = str(d.get("reason", ""))[:200] + if action == "place" and d.get("leaf") in by_name: + path = by_name[d["leaf"]] + if not T.is_leaf(T.node_at(tree, path) or {}): + decisions.append( + RoutingDecision(cid, RULE_AGENT, None, detail=f"{d['leaf']} is not a leaf") + ) + continue + decisions.append(RoutingDecision(cid, RULE_AGENT, path, detail=reason)) + elif action == "create" and d.get("new_leaf"): + parent_name = d.get("parent") + parent: tuple[str, ...] = () + if parent_name: + if parent_name in parents: + parent = parents[parent_name] + elif parent_name in by_name: + parent = by_name[parent_name][:-1] # sibling of a leaf + else: + parent = () + decisions.append( + RoutingDecision( + cid, + RULE_AGENT, + parent + (str(d["new_leaf"]),), + new_leaf=True, + detail=reason, + ) + ) + else: + decisions.append( + RoutingDecision(cid, RULE_AGENT, None, detail=reason or "untracked by agent") + ) + return decisions diff --git a/codewiki/src/be/updater/stale_scan.py b/codewiki/src/be/updater/stale_scan.py new file mode 100644 index 00000000..8bc2afc1 --- /dev/null +++ b/codewiki/src/be/updater/stale_scan.py @@ -0,0 +1,153 @@ +"""Step 6.2: scan pages not written this round for stale ids, names and links.""" + +from __future__ import annotations + +import logging +import os +import re +import time +from typing import Any + +from codewiki.src.be.agent_tools.deps import CodeWikiDeps +from codewiki.src.be.backend import LLMBackend +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.updater import pages as P +from codewiki.src.be.updater.graph_diff import GraphDiff +from codewiki.src.be.updater.prompts import STALE_FIX_SYSTEM_PROMPT, format_stale_prompt +from codewiki.src.be.updater.reference_index import unique_names_of +from codewiki.src.be.updater.record import CallCost, PageVerdict, UpdateRecord +from codewiki.src.be.updater.verdicts import parse_verdicts +from codewiki.src.config import Config + +logger = logging.getLogger(__name__) + +_LINK_RE = re.compile(r"\]\(\s*#]+\.md)(?:#[^)]*)?>?\s*\)") + + +def find_stale_items( + text: str, + diff: GraphDiff, + old_graph: dict[str, Node], + existing_pages: set[str], + removed_pages: set[str], + page_replacements: dict[str, str | None], +) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + unique_gone = unique_names_of(old_graph, set(diff.deleted) | set(diff.renamed.keys())) + for old_id, new_id in diff.renamed.items(): + if old_id in text: + items.append({"kind": "renamed", "old": old_id, "new": new_id}) + else: + old_name = old_graph[old_id].name if old_id in old_graph else None + new_name = new_id.split("::", 1)[-1] + if ( + old_name + and old_name in unique_gone + and old_name != new_name + and re.search(rf"`{re.escape(old_name)}`", text) + ): + items.append({"kind": "renamed", "old": old_name, "new": new_name}) + for cid in diff.deleted: + if cid in text: + items.append({"kind": "deleted", "old": cid}) + else: + name = old_graph[cid].name if cid in old_graph else None + if name and name in unique_gone and re.search(rf"`{re.escape(name)}`", text): + items.append({"kind": "deleted", "old": name}) + for m in _LINK_RE.finditer(text): + stem = os.path.splitext(os.path.basename(m.group(1)))[0] + if stem in removed_pages or (stem not in existing_pages and "/" not in m.group(1)): + items.append( + {"kind": "deleted_page", "old": stem, "replacement": page_replacements.get(stem)} + ) + # dedupe + seen = set() + out = [] + for it in items: + key = (it["kind"], it["old"]) + if key not in seen: + seen.add(key) + out.append(it) + return out + + +class StaleScanner: + def __init__( + self, + config: Config, + backend: LLMBackend, + docs_dir: str, + graph: dict[str, Node], + tree: dict[str, Any], + record: UpdateRecord, + ): + self.config = config + self.backend = backend + self.docs_dir = docs_dir + self.graph = graph + self.tree = tree + self.record = record + + async def run( + self, + diff: GraphDiff, + old_graph: dict[str, Node], + skip_pages: set[str], + removed_pages: set[str], + page_replacements: dict[str, str | None], + ) -> dict[str, Any]: + existing = set(P.list_pages(self.docs_dir)) + scanned, hits, fixed = 0, {}, [] + for stem in sorted(existing - skip_pages): + text = P.read_page(self.docs_dir, stem) or "" + scanned += 1 + items = find_stale_items( + text, diff, old_graph, existing, removed_pages, page_replacements + ) + if not items: + continue + hits[stem] = items + await self._fix(stem, items) + fixed.append(stem) + summary = {"scanned": scanned, "pages_with_hits": hits, "fixed": fixed} + logger.info("Stale scan: %d pages scanned, %d with hits", scanned, len(hits)) + return summary + + async def _fix(self, stem: str, items: list[dict[str, Any]]) -> None: + deps = CodeWikiDeps( + absolute_docs_path=self.docs_dir, + absolute_repo_path=str(os.path.abspath(self.config.repo_path)), + registry={}, + components=self.graph, + path_to_current_module=[], + current_module_name=stem, + module_tree=self.tree, + max_depth=self.config.max_depth, + current_depth=1, + config=self.config, + custom_instructions=self.config.get_prompt_addition(), + allowed_write_paths={P.page_path(self.docs_dir, stem)}, + ) + before = P.page_hashes(self.docs_dir) + started = time.time() + err = None + text = "" + usage = None + try: + reply = await self.backend.run_update_agent( + STALE_FIX_SYSTEM_PROMPT, format_stale_prompt(stem, items), deps + ) + text, usage = reply.text, reply.usage + except Exception as e: # noqa: BLE001 — recorded; page stays as is + err = f"{type(e).__name__}: {e}" + logger.error("Stale fix for %s failed: %s", stem, e) + self.record.add_call(CallCost("stale_fix", stem, time.time() - started, usage, err)) + changed = stem in P.changed_pages(before, P.page_hashes(self.docs_dir)) + verdicts, _ = parse_verdicts(text) + v = verdicts.get(stem, {}) + verdict = "patch" if changed else v.get("verdict", "no-op") + self.record.add_verdict( + PageVerdict(stem, verdict, v.get("reason", "stale-name scan"), "stale_scan", changed) + ) + if changed: + self.record.pages_written.append(stem) diff --git a/codewiki/src/be/updater/tree.py b/codewiki/src/be/updater/tree.py new file mode 100644 index 00000000..1da90e0e --- /dev/null +++ b/codewiki/src/be/updater/tree.py @@ -0,0 +1,258 @@ +"""Helpers over ``module_tree.json``. + +Shape: ``{name: {"path"?: str, "components": [ids], "children": {...}}}``. +A node is a leaf when ``children`` is missing or empty. Parents usually carry +the union of their children's components (super-grouping) or the set that +was later subdivided (clustering), so the *owner* of a component is the +deepest node that lists it. Agent-inserted sub-modules have no ``path`` key. + +Paths are tuples of names from the root, e.g. ``("core", "auth")``. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from copy import deepcopy +from typing import Any + +from codewiki.src.be.dependency_analyzer.models.core import Node + +Path = tuple[str, ...] + + +def is_leaf(info: dict[str, Any]) -> bool: + children = info.get("children") + return not children or (isinstance(children, dict) and len(children) == 0) + + +def components_of(info: dict[str, Any]) -> list[str]: + comps = info.get("components") + return list(comps) if isinstance(comps, list) else [] + + +def iter_nodes(tree: dict[str, Any], prefix: Path = ()) -> Iterator[tuple[Path, dict[str, Any]]]: + """Pre-order walk yielding ``(path, info)`` for every node.""" + for name, info in tree.items(): + if not isinstance(info, dict): + continue + path = prefix + (name,) + yield path, info + children = info.get("children") + if isinstance(children, dict) and children: + yield from iter_nodes(children, path) + + +def iter_leaves(tree: dict[str, Any]) -> Iterator[tuple[Path, dict[str, Any]]]: + for path, info in iter_nodes(tree): + if is_leaf(info): + yield path, info + + +def leaf_paths(tree: dict[str, Any]) -> list[Path]: + return [p for p, _ in iter_leaves(tree)] + + +def preorder_paths(tree: dict[str, Any]) -> list[Path]: + return [p for p, _ in iter_nodes(tree)] + + +def node_at(tree: dict[str, Any], path: Path) -> dict[str, Any] | None: + level = tree + info: dict[str, Any] | None = None + for i, name in enumerate(path): + if not isinstance(level, dict) or name not in level: + return None + info = level[name] + if i < len(path) - 1: + level = info.get("children", {}) + return info + + +def ancestors(path: Path) -> list[Path]: + """Proper ancestors, nearest first: ``("a","b","c") -> [("a","b"), ("a",)]``.""" + return [path[:i] for i in range(len(path) - 1, 0, -1)] + + +def owner_map(tree: dict[str, Any]) -> dict[str, Path]: + """Map each component id to the deepest node that lists it.""" + owner: dict[str, Path] = {} + for path, info in iter_nodes(tree): + for cid in components_of(info): + prev = owner.get(cid) + if prev is None or len(path) > len(prev): + owner[cid] = path + return owner + + +def resolve_owner(owner: dict[str, Path], cid: str) -> Path | None: + """Owner of ``cid``, or of its enclosing class when ``cid`` itself is untracked. + + Clustering only places selected leaf nodes (mostly classes). A method + ``path::Cls.m`` therefore has no owner of its own; it belongs to the leaf + of ``path::Cls``. Nested classes resolve the same way, one dot at a time. + """ + if cid in owner: + return owner[cid] + if "::" not in cid: + return None + path, name = cid.split("::", 1) + while "." in name: + name = name.rsplit(".", 1)[0] + parent = f"{path}::{name}" + if parent in owner: + return owner[parent] + return None + + +def owned_directly(tree: dict[str, Any]) -> dict[Path, list[str]]: + """Components per node that no descendant lists (what a node really owns). + + For leaves this is their whole component list. A parent normally owns + nothing directly; when it does, it must be treated as a leaf for those + components (Part 13 of the redesign note). + """ + owner = owner_map(tree) + result: dict[Path, list[str]] = {} + for path, info in iter_nodes(tree): + mine = [cid for cid in components_of(info) if owner.get(cid) == path] + if mine or is_leaf(info): + result[path] = mine + return result + + +def unit_paths(tree: dict[str, Any]) -> list[Path]: + """Leaves plus parents that own components directly: the update units.""" + return list(owned_directly(tree).keys()) + + +def tracked_ids(tree: dict[str, Any]) -> set[str]: + return set(owner_map(tree).keys()) + + +def add_component(tree: dict[str, Any], path: Path, cid: str) -> None: + """Add ``cid`` to the node at ``path`` and to every ancestor that keeps a list.""" + info = node_at(tree, path) + if info is None: + raise KeyError(f"no module at {path}") + comps = info.setdefault("components", []) + if cid not in comps: + comps.append(cid) + for anc in ancestors(path): + anc_info = node_at(tree, anc) + if anc_info is None or "components" not in anc_info: + continue + if cid not in anc_info["components"]: + anc_info["components"].append(cid) + + +def remove_component(tree: dict[str, Any], cid: str) -> list[Path]: + """Remove ``cid`` from every node that lists it; return the touched paths.""" + touched: list[Path] = [] + for path, info in iter_nodes(tree): + comps = info.get("components") + if isinstance(comps, list) and cid in comps: + info["components"] = [c for c in comps if c != cid] + touched.append(path) + return touched + + +def rename_component(tree: dict[str, Any], old_id: str, new_id: str) -> list[Path]: + touched: list[Path] = [] + for path, info in iter_nodes(tree): + comps = info.get("components") + if isinstance(comps, list) and old_id in comps: + info["components"] = [new_id if c == old_id else c for c in comps] + touched.append(path) + return touched + + +def prune_empty(tree: dict[str, Any]) -> list[Path]: + """Drop leaves with no components, then parents left with no children and + no components, recursively. Returns the removed paths (deepest first).""" + removed: list[Path] = [] + + def _prune(level: dict[str, Any], prefix: Path) -> None: + for name in list(level.keys()): + info = level[name] + if not isinstance(info, dict): + continue + path = prefix + (name,) + children = info.get("children") + if isinstance(children, dict) and children: + _prune(children, path) + children = info.get("children") + has_children = isinstance(children, dict) and len(children) > 0 + if not has_children and not components_of(info): + del level[name] + removed.append(path) + + _prune(tree, ()) + return removed + + +def insert_leaf( + tree: dict[str, Any], + parent: Path, + name: str, + components: list[str], + path_hint: str | None = None, +) -> Path: + """Create a new leaf ``name`` under ``parent`` (``()`` = top level).""" + if parent: + parent_info = node_at(tree, parent) + if parent_info is None: + raise KeyError(f"no module at {parent}") + level = parent_info.setdefault("children", {}) + else: + level = tree + if name in level: + raise KeyError(f"module {name!r} already exists under {parent}") + info: dict[str, Any] = {"components": list(components), "children": {}} + if path_hint: + info["path"] = path_hint + level[name] = info + new_path = parent + (name,) + for cid in components: + add_component(tree, new_path, cid) + return new_path + + +def reverse_edges(graph: dict[str, Node]) -> dict[str, set[str]]: + """``in(c)``: the components whose ``depends_on`` contains ``c``.""" + rev: dict[str, set[str]] = {} + for cid, node in graph.items(): + for dep in node.depends_on or (): + rev.setdefault(dep, set()).add(cid) + return rev + + +def leaf_dependents( + tree: dict[str, Any], graph: dict[str, Node], owner: dict[str, Path] | None = None +) -> dict[Path, set[Path]]: + """Lifted dependency: ``Dep(l)`` = units whose code *uses* code in ``l``.""" + owner = owner if owner is not None else owner_map(tree) + dep: dict[Path, set[Path]] = {p: set() for p in unit_paths(tree)} + for user, node in graph.items(): + user_leaf = resolve_owner(owner, user) + if user_leaf is None: + continue + for used in node.depends_on or (): + used_leaf = resolve_owner(owner, used) + if used_leaf is not None and used_leaf != user_leaf: + dep.setdefault(used_leaf, set()).add(user_leaf) + return dep + + +def copy_tree(tree: dict[str, Any]) -> dict[str, Any]: + return deepcopy(tree) + + +def page_stem(path: Path) -> str: + """Module page file stem (docs are flat: ``.md``).""" + return path[-1] + + +def virtual_whole_repo_tree(repo_name: str, tracked: list[str]) -> dict[str, Any]: + """Whole-repository mode has an empty tree and one page. Model it as a + single leaf holding every tracked component so any change activates it.""" + return {repo_name: {"components": list(tracked), "children": {}}} diff --git a/codewiki/src/be/updater/tree_repair.py b/codewiki/src/be/updater/tree_repair.py new file mode 100644 index 00000000..f38c8e9e --- /dev/null +++ b/codewiki/src/be/updater/tree_repair.py @@ -0,0 +1,246 @@ +"""Step 2: repair the module tree after a component-level diff. + +Order: renames are rewritten in place, deleted components are removed +(empty leaves disappear), added components are routed by rules 1-3, and +what the rules cannot place is handed to a routing callable (rule 4). +The growth check only *flags* leaves; re-clustering is the orchestrator's +job because it needs the clustering step and an LLM. +""" + +from __future__ import annotations + +import logging +import os +from collections import Counter +from collections.abc import Callable +from dataclasses import asdict, dataclass, field +from typing import Any + +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.module_naming import ( + collect_module_tree_names, + resolve_unique_name, + sanitize_module_name, +) +from codewiki.src.be.updater import tree as T +from codewiki.src.be.updater.graph_diff import GraphDiff +from codewiki.src.be.updater.options import UpdateOptions + +logger = logging.getLogger(__name__) + +RULE_SAME_FILE = "1:same_file" +RULE_SAME_DIR = "2:same_dir" +RULE_NEIGHBOR = "3:neighbor_majority" +RULE_AGENT = "4:routing_agent" +RULE_UNTRACKED = "untracked" + + +@dataclass +class RoutingDecision: + component_id: str + rule: str + leaf_path: tuple[str, ...] | None = None # None = left untracked + new_leaf: bool = False + detail: str = "" + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["leaf_path"] = list(self.leaf_path) if self.leaf_path else None + return d + + +@dataclass +class RepairResult: + tree: dict[str, Any] + renamed: dict[str, str] = field(default_factory=dict) + removed: list[tuple[str, tuple[str, ...]]] = field(default_factory=list) + routing: list[RoutingDecision] = field(default_factory=list) + orphans: list[str] = field(default_factory=list) + deleted_nodes: list[tuple[str, ...]] = field(default_factory=list) + created_leaves: list[tuple[str, ...]] = field(default_factory=list) + growth: dict[tuple[str, ...], float] = field(default_factory=dict) + growth_flagged: list[tuple[str, ...]] = field(default_factory=list) + entered: dict[tuple[str, ...], list[str]] = field(default_factory=dict) + left: dict[tuple[str, ...], list[str]] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "renamed": dict(sorted(self.renamed.items())), + "removed": [{"component_id": c, "leaf_path": list(p)} for c, p in self.removed], + "routing": [r.to_dict() for r in self.routing], + "orphans": sorted(self.orphans), + "deleted_nodes": [list(p) for p in self.deleted_nodes], + "created_leaves": [list(p) for p in self.created_leaves], + "growth": {"/".join(p): round(g, 4) for p, g in self.growth.items()}, + "growth_flagged": [list(p) for p in self.growth_flagged], + } + + +# Signature of the rule-4 callable: (orphan ids, context) -> decisions. +OrphanRouter = Callable[[list[str], dict[str, Any]], list[RoutingDecision]] + + +def _dirname(rel: str) -> str: + return os.path.dirname(rel.replace("\\", "/")) + + +def _route_by_rules( + cid: str, + node: Node, + owner: dict[str, tuple[str, ...]], + new_graph: dict[str, Node], + opts: UpdateOptions, +) -> RoutingDecision | None: + # Rule 1: same file already tracked. + same_file = Counter( + owner[c] + for c in owner + if c in new_graph and new_graph[c].relative_path == node.relative_path + ) + if same_file: + leaf, n = same_file.most_common(1)[0] + return RoutingDecision(cid, RULE_SAME_FILE, leaf, detail=f"{n} tracked components in file") + # Rule 2: same directory, single leaf. + d = _dirname(node.relative_path) + same_dir = { + owner[c] for c in owner if c in new_graph and _dirname(new_graph[c].relative_path) == d + } + if len(same_dir) == 1: + return RoutingDecision(cid, RULE_SAME_DIR, next(iter(same_dir)), detail=f"dir {d!r}") + # Rule 3: neighbour majority. + rev_users = {c for c, n in new_graph.items() if cid in (n.depends_on or ())} + neighbours = { + c for c in (set(node.depends_on or ()) | rev_users) if T.resolve_owner(owner, c) is not None + } + if neighbours: + votes = Counter(T.resolve_owner(owner, c) for c in neighbours) + leaf, n = votes.most_common(1)[0] + share = n / len(neighbours) + if share >= opts.tau_nb: + return RoutingDecision( + cid, RULE_NEIGHBOR, leaf, detail=f"{n}/{len(neighbours)} neighbours ({share:.2f})" + ) + return None + + +def unique_leaf_name(tree: dict[str, Any], requested: str, parent: tuple[str, ...]) -> str: + taken = collect_module_tree_names(tree) + parent_name = parent[-1] if parent else None + return resolve_unique_name(sanitize_module_name(requested), parent_name, taken) + + +def repair_tree( + tree: dict[str, Any], + diff: GraphDiff, + new_graph: dict[str, Node], + tracked_new: set[str], + opts: UpdateOptions, + route_orphans: OrphanRouter | None = None, +) -> RepairResult: + """Return a repaired *copy* of ``tree`` plus every decision taken.""" + tree = T.copy_tree(tree) + before = {p: set(T.components_of(i)) for p, i in T.iter_nodes(tree)} + result = RepairResult(tree=tree) + + # Rename: rewrite ids in place. + for old_id, new_id in diff.renamed.items(): + if T.rename_component(tree, old_id, new_id): + result.renamed[old_id] = new_id + + # Delete: remove, then drop empty nodes. + owner_before = T.owner_map(tree) + for cid in sorted(diff.deleted): + if cid in owner_before: + T.remove_component(tree, cid) + result.removed.append((cid, owner_before[cid])) + result.deleted_nodes = T.prune_empty(tree) + + # Add: route every added component that the new build selected as tracked. + owner = T.owner_map(tree) + to_route = sorted(c for c in diff.added if c in tracked_new and c in new_graph) + orphans: list[str] = [] + for cid in to_route: + decision = _route_by_rules(cid, new_graph[cid], owner, new_graph, opts) + if decision is None: + orphans.append(cid) + continue + T.add_component(tree, decision.leaf_path, cid) + owner[cid] = decision.leaf_path + result.routing.append(decision) + for cid in sorted(c for c in diff.added if c not in tracked_new): + result.routing.append( + RoutingDecision(cid, RULE_UNTRACKED, None, detail="not a selected leaf node") + ) + + # Rule 4: orphans. + if orphans and route_orphans is not None and opts.use_routing_agent: + context = {"tree": tree, "owner": owner, "graph": new_graph} + decisions = route_orphans(orphans, context) + decided = {d.component_id: d for d in decisions} + for cid in orphans: + d = decided.get(cid) + if d is None or d.leaf_path is None: + result.routing.append( + d or RoutingDecision(cid, RULE_AGENT, None, detail="agent left untracked") + ) + result.orphans.append(cid) + continue + if d.new_leaf and T.node_at(tree, d.leaf_path) is None: + parent, name = d.leaf_path[:-1], d.leaf_path[-1] + if parent and T.node_at(tree, parent) is None: + result.routing.append( + RoutingDecision(cid, RULE_AGENT, None, detail=f"unknown parent {parent}") + ) + result.orphans.append(cid) + continue + name = unique_leaf_name(tree, name, parent) + rel = new_graph[cid].relative_path + new_path = T.insert_leaf(tree, parent, name, [], path_hint=_dirname(rel) or ".") + d.leaf_path = new_path + result.created_leaves.append(new_path) + elif T.node_at(tree, d.leaf_path) is None: + result.routing.append( + RoutingDecision(cid, RULE_AGENT, None, detail=f"unknown leaf {d.leaf_path}") + ) + result.orphans.append(cid) + continue + T.add_component(tree, d.leaf_path, cid) + owner[cid] = d.leaf_path + d.rule = RULE_AGENT + result.routing.append(d) + else: + for cid in orphans: + result.routing.append(RoutingDecision(cid, RULE_AGENT, None, detail="no routing agent")) + result.orphans.extend(orphans) + + # Tree deltas and growth check per unit. + after = {p: set(T.components_of(i)) for p, i in T.iter_nodes(tree)} + for path, now in after.items(): + was = before.get(path, set()) + entered = sorted(now - was) + left = sorted(was - now) + if entered: + result.entered[path] = entered + if left: + result.left[path] = left + if T.is_leaf(T.node_at(tree, path) or {}) and entered and now: + g = len(set(entered)) / len(now) + result.growth[path] = g + if path not in result.created_leaves and g >= opts.tau_grow: + result.growth_flagged.append(path) + for path in before: + if path not in after and before[path]: + result.left[path] = sorted(before[path]) + + logger.info( + "Tree repair: %d renamed, %d removed, %d routed, %d orphans, %d nodes deleted, " + "%d leaves created, %d growth-flagged", + len(result.renamed), + len(result.removed), + sum(1 for r in result.routing if r.leaf_path is not None), + len(result.orphans), + len(result.deleted_nodes), + len(result.created_leaves), + len(result.growth_flagged), + ) + return result diff --git a/codewiki/src/be/updater/verdicts.py b/codewiki/src/be/updater/verdicts.py new file mode 100644 index 00000000..9f3e2695 --- /dev/null +++ b/codewiki/src/be/updater/verdicts.py @@ -0,0 +1,70 @@ +"""Parse the JSON verdict block an editing agent returns.""" + +from __future__ import annotations + +import json +import re +from typing import Any + +_FENCE_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.S) + + +def _candidates(text: str) -> list[str]: + out = [m.group(1) for m in _FENCE_RE.finditer(text or "")] + # Fallback: the last top-level {...} that mentions "verdicts" or "decisions". + for key in ("verdicts", "decisions"): + idx = (text or "").rfind(f'"{key}"') + if idx == -1: + continue + start = (text or "").rfind("{", 0, idx) + depth = 0 + for i in range(start, len(text)): + if text[i] == "{": + depth += 1 + elif text[i] == "}": + depth -= 1 + if depth == 0: + out.append(text[start : i + 1]) + break + return out + + +def parse_json_block(text: str) -> dict[str, Any] | None: + for cand in reversed(_candidates(text)): + try: + data = json.loads(cand) + except json.JSONDecodeError: + continue + if isinstance(data, dict): + return data + return None + + +def parse_verdicts(text: str) -> tuple[dict[str, dict[str, str]], str]: + """Return ``{page_stem: {"verdict": ..., "reason": ...}}`` and free-text notes.""" + data = parse_json_block(text) or {} + raw = data.get("verdicts") or {} + verdicts: dict[str, dict[str, str]] = {} + if isinstance(raw, dict): + for page, v in raw.items(): + stem = str(page) + if stem.endswith(".md"): + stem = stem[:-3] + if isinstance(v, str): + verdicts[stem] = {"verdict": v.strip().lower(), "reason": ""} + elif isinstance(v, dict): + verdicts[stem] = { + "verdict": str(v.get("verdict", "")).strip().lower(), + "reason": str(v.get("reason", "")).strip(), + } + elif isinstance(raw, list): + for v in raw: + if isinstance(v, dict) and "page" in v: + stem = str(v["page"]) + stem = stem[:-3] if stem.endswith(".md") else stem + verdicts[stem] = { + "verdict": str(v.get("verdict", "")).strip().lower(), + "reason": str(v.get("reason", "")).strip(), + } + notes = str(data.get("notes", "") or "") + return verdicts, notes diff --git a/tests/test_updater_change_report.py b/tests/test_updater_change_report.py new file mode 100644 index 00000000..e61f1ae6 --- /dev/null +++ b/tests/test_updater_change_report.py @@ -0,0 +1,181 @@ +"""Step 3/4: per-leaf change reports, active set, fallback ratios, ordering.""" + +from __future__ import annotations + +from updater_toy import ( + HANDLE, + OAUTH, + REFRESH, + USER, + VALIDATE, + graph_r1, + graph_r2, + tracked_r2, + tree_r1, + write_pages, +) + +from codewiki.src.be.updater.change_report import ( + MODE_DELETE, + MODE_EDIT, + active_set, + build_reports, + fallback_ratios, + order_active, +) +from codewiki.src.be.updater.graph_diff import diff_graphs +from codewiki.src.be.updater.options import UpdateOptions +from codewiki.src.be.updater.reference_index import build_reference_index +from codewiki.src.be.updater.tree_repair import repair_tree + + +def _toy(tmp_path, opts=None): + opts = opts or UpdateOptions() + old_g, new_g = graph_r1(), graph_r2() + old_t = tree_r1() + d = diff_graphs(old_g, new_g, opts) + r = repair_tree(old_t, d, new_g, tracked_r2(), opts) + write_pages(tmp_path) + idx = build_reference_index(str(tmp_path), old_g, old_t) + reports = build_reports(d, old_t, r.tree, old_g, new_g, idx, r, opts) + return d, r, reports, new_g + + +def test_toy_reports_match_part_7_table(tmp_path): + d, r, reports, _ = _toy(tmp_path) + auth = reports[("core", "auth")] + assert set(auth.own) == {VALIDATE, REFRESH, OAUTH} + assert auth.up == [] and auth.entered == [OAUTH] and auth.mode == MODE_EDIT + api = reports[("core", "api")] + assert api.own == [] and api.up == [REFRESH] and api.mode == MODE_EDIT + # api's page names refresh by id and by name; its own Up already covers it, + # but RefCh is about references, so it lists both forms. + assert REFRESH in api.refch + pipeline = reports[("pipeline",)] + assert pipeline.own == [] and pipeline.up == [] + assert pipeline.refch == ["refresh"] # names a component whose contract moved + storage = reports[("storage",)] + assert storage.mode == MODE_DELETE and storage.own == [USER] + # parents have no report + assert ("core",) not in reports + active = active_set(reports) + assert set(active) == {("core", "auth"), ("core", "api"), ("pipeline",), ("storage",)} + + +def test_body_only_change_does_not_reach_dependents(tmp_path): + # Only validate's body changes: login (same leaf) uses it; api does not. + old_g, new_g = graph_r1(), graph_r1() + new_g[VALIDATE] = new_g[VALIDATE].model_copy( + update={"source_code": "def validate(user, pw):\n return True\n"} + ) + opts = UpdateOptions() + d = diff_graphs(old_g, new_g, opts) + r = repair_tree(tree_r1(), d, new_g, tracked_r2() - {OAUTH}, opts) + write_pages(tmp_path) + idx = build_reference_index(str(tmp_path), old_g, tree_r1()) + reports = build_reports(d, tree_r1(), r.tree, old_g, new_g, idx, r, opts) + assert active_set(reports) == [("core", "auth")] + assert reports[("core", "auth")].own == [VALIDATE] + assert reports[("core", "api")].is_empty + + +def test_up_disabled_at_rung_1(tmp_path): + _, _, reports, _ = _toy(tmp_path, UpdateOptions.from_rung(1)) + assert reports[("core", "api")].up == [] + + +def test_fallback_ratios_and_order(tmp_path): + d, r, reports, new_g = _toy(tmp_path) + ratios = fallback_ratios(reports, r.tree, r) + assert ratios["n_leaves"] == 3 and ratios["n_active"] == 4 + assert ratios["r_leaf"] >= UpdateOptions().tau_full + assert ratios["n_structural"] == 1 # storage deleted + order = order_active(active_set(reports), r.tree, new_g) + # api uses auth -> auth first; deleted storage (not in new tree) last + assert order.index(("core", "auth")) < order.index(("core", "api")) + assert order[-1] == ("storage",) + + +def test_whole_repo_mode_is_one_virtual_leaf(tmp_path): + from codewiki.src.be.updater import tree as T + + old_g, new_g = graph_r1(), graph_r2() + tracked_old = set(old_g) + old_t = T.virtual_whole_repo_tree("repo", sorted(tracked_old)) + opts = UpdateOptions() + d = diff_graphs(old_g, new_g, opts) + r = repair_tree(old_t, d, new_g, tracked_r2(), opts) + reports = build_reports(d, old_t, r.tree, old_g, new_g, None, r, opts) + assert active_set(reports) == [("repo",)] + assert set(reports[("repo",)].own) == {VALIDATE, REFRESH, OAUTH, USER} + + +def test_untracked_method_attaches_to_its_class_leaf(tmp_path): + """Only classes are tracked in real trees; a method change must reach the + class's leaf as Own, and a method signature change must reach the leaves + whose methods call it as Up.""" + from updater_toy import node + + from codewiki.src.be.updater import tree as T + + store_cls = "src/db/store.py::Store" + store_get = "src/db/store.py::Store.get" + api_cls = "src/api/view.py::View" + api_show = "src/api/view.py::View.show" + old_g = { + store_cls: node(store_cls, "class", "class Store:\n pass\n"), + store_get: node( + store_get, "method", "def get(self, k):\n return self.d[k]\n", ["self", "k"] + ), + api_cls: node(api_cls, "class", "class View:\n pass\n"), + api_show: node( + api_show, + "method", + "def show(self):\n return store.get(1)\n", + ["self"], + deps=[store_get], + ), + } + new_g = {k: v.model_copy(deep=True) for k, v in old_g.items()} + new_g[store_get] = node( + store_get, + "method", + "def get(self, k, default=None):\n return self.d.get(k, default)\n", + ["self", "k", "default"], + ) + tree = { + "db": {"components": [store_cls], "children": {}}, + "api": {"components": [api_cls], "children": {}}, + } + owner = T.owner_map(tree) + assert T.resolve_owner(owner, store_get) == ("db",) + assert T.resolve_owner(owner, "src/x.py::free_function") is None + assert T.leaf_dependents(tree, new_g) == {("db",): {("api",)}, ("api",): set()} + + opts = UpdateOptions() + d = diff_graphs(old_g, new_g, opts) + assert d.interface == {store_get} + r = repair_tree(tree, d, new_g, {store_cls, api_cls}, opts) + reports = build_reports(d, tree, r.tree, old_g, new_g, None, r, opts) + assert reports[("db",)].own == [store_get] + assert reports[("api",)].up == [store_get] + assert reports[("api",)].context == [] + + +def test_context_alone_does_not_activate(tmp_path): + """An untracked free function next to a leaf changed: the leaf gets it as + context but is not active for that reason alone.""" + from updater_toy import node + + helper = "src/api/util.py::helper" + old_g = graph_r1() + old_g[helper] = node(helper, "function", "def helper():\n return 1\n") + old_g[HANDLE] = old_g[HANDLE].model_copy(update={"depends_on": {REFRESH, helper}}) + new_g = {k: v.model_copy(deep=True) for k, v in old_g.items()} + new_g[helper] = node(helper, "function", "def helper():\n return 2\n") + opts = UpdateOptions() + d = diff_graphs(old_g, new_g, opts) + r = repair_tree(tree_r1(), d, new_g, tracked_r2() - {OAUTH}, opts) + reports = build_reports(d, tree_r1(), r.tree, old_g, new_g, None, r, opts) + assert reports[("core", "api")].context == [helper] + assert active_set(reports) == [] diff --git a/tests/test_updater_graph_diff.py b/tests/test_updater_graph_diff.py new file mode 100644 index 00000000..074edc11 --- /dev/null +++ b/tests/test_updater_graph_diff.py @@ -0,0 +1,85 @@ +"""Step 1 of the component-level updater: graph diff and rename pairing.""" + +from __future__ import annotations + +from updater_toy import OAUTH, REFRESH, USER, VALIDATE, graph_r1, graph_r2, node + +from codewiki.src.be.updater.graph_diff import ( + CLASS_ADDED, + CLASS_BODY, + CLASS_DELETED, + CLASS_IFACE, + CLASS_RENAMED, + DIFF_TRUNCATED_MARKER, + body_hash, + diff_graphs, +) +from codewiki.src.be.updater.options import UpdateOptions + + +def test_toy_change_classes(): + d = diff_graphs(graph_r1(), graph_r2(), UpdateOptions()) + assert d.body == {VALIDATE} + assert d.interface == {REFRESH} + assert d.added == {OAUTH} + assert d.deleted == {USER} + assert d.renamed == {} + assert d.records[VALIDATE].change_class == CLASS_BODY + assert "+ log('validate')" in d.records[VALIDATE].diff + assert d.records[REFRESH].change_class == CLASS_IFACE + assert d.records[REFRESH].old_signature["parameters"] == ["self"] + assert d.records[REFRESH].new_signature["parameters"] == ["self", "force"] + assert d.records[OAUTH].change_class == CLASS_ADDED + assert d.records[USER].change_class == CLASS_DELETED + assert d.counts()["body"] == 1 and d.counts()["interface"] == 1 + + +def test_unchanged_component_in_changed_file_is_absent(): + # login.py changed (validate) but login itself kept its hash. + d = diff_graphs(graph_r1(), graph_r2()) + assert "src/auth/login.py::login" not in d.changed_ids + + +def test_whitespace_only_change_is_not_a_change(): + a = node("f.py::g", "function", "def g():\n return 1\n") + b = node("f.py::g", "function", "def g():\n\n return 1\n") + assert body_hash(a) == body_hash(b) + assert diff_graphs({a.id: a}, {b.id: b}).is_empty + + +def test_edge_only_change_is_classified_as_edge(): + a = node("f.py::g", "function", "def g():\n return h()\n") + b = node("f.py::g", "function", "def g():\n return h()\n", deps=["f.py::h"]) + d = diff_graphs({a.id: a}, {b.id: b}) + assert d.edge == {"f.py::g"} + assert d.records["f.py::g"].edges_added == ["f.py::h"] + + +def test_rename_pairing_respects_tau_ren(): + body = "class Store:\n def get(self, k):\n return self.d[k]\n def put(self, k, v):\n self.d[k] = v\n" + old = {"a.py::Store": node("a.py::Store", "class", body)} + new = {"b.py::Store2": node("b.py::Store2", "class", body.replace("Store", "Store2"))} + d = diff_graphs(old, new, UpdateOptions(tau_ren=0.9)) + assert d.renamed == {"a.py::Store": "b.py::Store2"} + assert not d.added and not d.deleted + assert d.records["b.py::Store2"].change_class == CLASS_RENAMED + strict = diff_graphs(old, new, UpdateOptions(tau_ren=0.999)) + assert strict.renamed == {} + assert strict.added == {"b.py::Store2"} and strict.deleted == {"a.py::Store"} + + +def test_rename_with_signature_change_is_also_interface(): + old = {"a.py::f": node("a.py::f", "function", "def f(x):\n return x + 1\n", ["x"])} + new = {"a.py::g": node("a.py::g", "function", "def g(x, y=0):\n return x + 1\n", ["x", "y"])} + d = diff_graphs(old, new, UpdateOptions(tau_ren=0.7)) + assert d.renamed == {"a.py::f": "a.py::g"} + assert "a.py::g" in d.interface + + +def test_diff_truncation(): + big_old = "def f():\n" + "".join(f" x{i} = {i}\n" for i in range(3000)) + big_new = "def f():\n" + "".join(f" x{i} = {i + 1}\n" for i in range(3000)) + old = {"a.py::f": node("a.py::f", "function", big_old)} + new = {"a.py::f": node("a.py::f", "function", big_new)} + d = diff_graphs(old, new, UpdateOptions(max_diff_tokens=50)) + assert DIFF_TRUNCATED_MARKER in d.records["a.py::f"].diff diff --git a/tests/test_updater_graph_store.py b/tests/test_updater_graph_store.py new file mode 100644 index 00000000..c82a74e7 --- /dev/null +++ b/tests/test_updater_graph_store.py @@ -0,0 +1,68 @@ +"""Previous-graph lookup when the repo was analysed from differently named checkouts.""" + +from __future__ import annotations + +import json +import os +import time + +from codewiki.src.be.updater.graph_store import ( + find_any_graph_file, + graph_file_path, + prune_superseded_graphs, + snapshot_old_graph, +) + + +def _write_graph(d, name, payload): + path = os.path.join(d, f"{name}_dependency_graph.json") + with open(path, "w") as f: + json.dump(payload, f) + return path + + +def test_single_candidate_is_used_whatever_its_name(tmp_path): + d = str(tmp_path) + old = _write_graph(d, "rq3_svelte_anchor", {"a": 1}) + assert find_any_graph_file(d) == old + prev = snapshot_old_graph(d, "/repos/rq3-svelte-w1") + assert prev is not None and prev.endswith("rq3_svelte_w1_dependency_graph.prev.json") + assert json.load(open(prev)) == {"a": 1} + + +def test_several_candidates_pick_newest_not_none(tmp_path): + d = str(tmp_path) + older = _write_graph(d, "rq3_svelte_anchor", {"rev": "anchor"}) + time.sleep(0.01) + newer = _write_graph(d, "rq3_svelte_w1", {"rev": "w1"}) + os.utime(older, (1, 1)) # make the order unambiguous + assert find_any_graph_file(d) == newer + prev = snapshot_old_graph(d, "/repos/rq3-svelte-w2") + assert json.load(open(prev)) == {"rev": "w1"} + + +def test_prev_snapshots_are_not_candidates(tmp_path): + d = str(tmp_path) + _write_graph(d, "rq3_svelte_w1", {"rev": "w1"}) + with open(os.path.join(d, "rq3_svelte_w1_dependency_graph.prev.json"), "w") as f: + json.dump({"rev": "anchor"}, f) + assert find_any_graph_file(d).endswith("rq3_svelte_w1_dependency_graph.json") + + +def test_prune_keeps_current_and_prev_only(tmp_path): + d = str(tmp_path) + _write_graph(d, "rq3_svelte_anchor", {}) + _write_graph(d, "rq3_svelte_w1", {}) + current = _write_graph(d, "rq3_svelte_w2", {}) + with open(os.path.join(d, "rq3_svelte_w2_dependency_graph.prev.json"), "w") as f: + json.dump({}, f) + removed = prune_superseded_graphs(d, keep=graph_file_path(d, "/repos/rq3-svelte-w2")) + assert sorted(os.path.basename(r) for r in removed) == [ + "rq3_svelte_anchor_dependency_graph.json", + "rq3_svelte_w1_dependency_graph.json", + ] + assert sorted(os.listdir(d)) == [ + "rq3_svelte_w2_dependency_graph.json", + "rq3_svelte_w2_dependency_graph.prev.json", + ] + assert os.path.exists(current) diff --git a/tests/test_updater_orchestrator.py b/tests/test_updater_orchestrator.py new file mode 100644 index 00000000..6bfb2a3f --- /dev/null +++ b/tests/test_updater_orchestrator.py @@ -0,0 +1,230 @@ +"""End-to-end run of the incremental updater on the toy repository with a fake backend.""" + +from __future__ import annotations + +import asyncio +import json +import os +from pathlib import Path +from types import SimpleNamespace + +from updater_toy import OAUTH, USER, graph_r1, graph_r2, tracked_r2, tree_r1, write_pages + +from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor +from codewiki.src.be.backend import AgentReply +from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.be.updater.graph_store import save_graph +from codewiki.src.be.updater.options import UpdateOptions +from codewiki.src.be.updater.orchestrator import IncrementalUpdater +from codewiki.src.be.updater.record import RECORD_FILENAME + + +class FakeBackend: + """Writes pages the way the real agents do (through the editor tool).""" + + def __init__(self, own_verdict="patch"): + self.own_verdict = own_verdict + self.update_calls = [] + self.module_calls = [] + self.complete_calls = [] + self.last_usage = None + + def complete(self, prompt, *, model=None): + self.complete_calls.append(prompt[:80]) + self.last_usage = {"prompt_tokens": 10, "completion_tokens": 5} + return "regenerated overview" + + async def run_module_agent( + self, module_name, components, core_component_ids, module_path, working_dir + ): + self.module_calls.append(module_name) + page = Path(working_dir) / f"{module_name}.md" + if not page.exists(): + page.write_text(f"# {module_name}\n\nregenerated with {sorted(core_component_ids)}\n") + self.last_usage = {"prompt_tokens": 100, "completion_tokens": 50} + return json.load(open(os.path.join(working_dir, "module_tree.json"))) + + async def run_update_agent(self, system_prompt, user_prompt, deps): + self.update_calls.append((deps.current_module_name, sorted(deps.allowed_write_paths))) + ctx = SimpleNamespace(deps=deps) + verdicts = {} + own = deps.current_module_name + # Try to touch a page outside the write set: the tool must refuse. + outsider = "storage" if own != "storage" else "core" + out = await str_replace_editor( + ctx, "docs", "insert", path=f"{outsider}.md", insert_line=1, new_str="SNEAK\n" + ) + assert "not in this agent's write set" in out or "does not exist" in out + for path in sorted(deps.allowed_write_paths): + stem = Path(path).stem + if stem == own and self.own_verdict == "rewrite": + verdicts[f"{stem}.md"] = {"verdict": "rewrite", "reason": "too much changed"} + continue + if os.path.exists(path): + await str_replace_editor( + ctx, + "docs", + "insert", + path=f"{stem}.md", + insert_line=1, + new_str=f"\n", + ) + verdicts[f"{stem}.md"] = {"verdict": "patch", "reason": f"updated for {own}"} + else: + verdicts[f"{stem}.md"] = {"verdict": "no-op", "reason": "missing"} + text = "done\n```json\n" + json.dumps({"verdicts": verdicts, "notes": ""}) + "\n```" + return AgentReply( + text=text, usage={"prompt_tokens": 30, "completion_tokens": 10}, seconds=0.01 + ) + + +def _setup(tmp_path, tree=None, with_graph=True): + docs = tmp_path / "docs" + repo = tmp_path / "repo" + repo.mkdir() + write_pages(docs) + (docs / "module_tree.json").write_text(json.dumps(tree if tree is not None else tree_r1())) + (docs / "metadata.json").write_text(json.dumps({"generation_info": {"commit_id": "old"}})) + graph_dir = docs / "temp" / "dependency_graphs" + graph_dir.mkdir(parents=True) + prev = graph_dir / "repo_dependency_graph.prev.json" + if with_graph: + save_graph(graph_r1(), str(prev)) + config = SimpleNamespace( + docs_dir=str(docs), + repo_path=str(repo), + dependency_graph_dir=str(graph_dir), + max_depth=5, + cluster_model=None, + main_model="fake", + max_token_per_module=36369, + get_prompt_addition=lambda: None, + ) + return docs, config, str(prev) + + +def _generator(config, backend): + gen = object.__new__(DocumentationGenerator) + gen.config = config + gen.backend = backend + gen.commit_id = "new" + return gen + + +def _run(docs, config, prev, backend, opts): + gen = _generator(config, backend) + upd = IncrementalUpdater(config, backend, gen, opts) + return asyncio.run( + upd.run(prev, graph_r2(), sorted(tracked_r2()), {"old_commit": "old", "new_commit": "new"}) + ) + + +def test_toy_incremental_run_with_rewrite(tmp_path): + docs, config, prev = _setup(tmp_path) + backend = FakeBackend(own_verdict="rewrite") + rec = _run(docs, config, prev, backend, UpdateOptions(tau_full=2.0, tau_tree=2.0)) + + assert rec.outcome == "incremental" + assert rec.diff["counts"] == { + "added": 1, + "deleted": 1, + "interface": 1, + "body": 1, + "edge": 0, + "renamed": 0, + } + order = [a["leaf"] for a in rec.active] + assert order.index("core/auth") < order.index("core/api") + assert order[-1] == "storage" and rec.active[-1]["mode"] == "delete" + # deleted leaf page gone, its tree entry gone, the new class routed into auth + assert not (docs / "storage.md").exists() + tree = json.load(open(docs / "module_tree.json")) + assert "storage" not in tree + assert ( + OAUTH in tree["core"]["children"]["auth"]["components"] + and USER not in tree["core"]["components"] + ) + # auth was rewritten by the normal module agent after the agent's verdict + assert "auth" in backend.module_calls + assert "regenerated with" in (docs / "auth.md").read_text() + verdicts = {(v["page"], v["by_leaf"]): v["verdict"] for v in rec.verdicts} + assert verdicts[("auth", "auth")] == "rewrite" + assert verdicts[("api", "auth")] == "patch" # dependent, patched by auth's agent + assert verdicts[("pipeline", "auth")] == "patch" # referrer + assert verdicts[("core", "auth")] == "patch" and verdicts[("overview", "auth")] == "patch" + assert verdicts[("overview", "storage")] == "patch" + # write sets recorded and respected + assert set(rec.write_sets["auth"]) == {"auth", "core", "overview", "api", "pipeline"} + assert set(rec.write_sets["storage"]) == {"overview"} + assert rec.write_set_violations == [] + written = set(rec.pages_written) + allowed = set().union(*[set(v) for v in rec.write_sets.values()]) | {"auth"} + assert written <= allowed + # every page in the tree exists; record, index and metadata summary present + assert (docs / RECORD_FILENAME).exists() + assert (docs / "temp" / "reference_index.json").exists() + assert all((docs / f"{s}.md").exists() for s in ("auth", "api", "core", "pipeline", "overview")) + kinds = [c["kind"] for c in rec.calls] + assert "leaf_agent" in kinds and "rewrite" in kinds and "missing_pages" in kinds + assert all(c["usage"] for c in rec.calls if c["kind"] in ("leaf_agent", "rewrite")) + assert rec.stale_scan["scanned"] >= 0 + + +def test_no_change_short_circuits(tmp_path): + docs, config, prev = _setup(tmp_path) + save_graph(graph_r1(), prev) + backend = FakeBackend() + gen = _generator(config, backend) + upd = IncrementalUpdater(config, backend, gen, UpdateOptions()) + rec = asyncio.run(upd.run(prev, graph_r1(), sorted(graph_r1()), {})) + assert rec.outcome == "no_change" + assert backend.update_calls == [] and backend.module_calls == [] + assert (docs / "storage.md").exists() + + +def test_missing_old_graph_is_a_detector_failure(tmp_path): + docs, config, prev = _setup(tmp_path, with_graph=False) + backend = FakeBackend() + rec = _run(docs, config, prev, backend, UpdateOptions()) + assert rec.outcome == "detector_failure" + assert rec.errors and backend.update_calls == [] + assert (docs / "storage.md").exists() # nothing touched + + +def test_fallback_fires_with_default_thresholds(tmp_path): + docs, config, prev = _setup(tmp_path) + backend = FakeBackend() + rec = _run(docs, config, prev, backend, UpdateOptions()) + assert rec.outcome == "full_fallback" + assert rec.fallback["fired"] and rec.fallback["r_leaf"] >= 0.5 + assert backend.update_calls == [] and backend.module_calls == [] + # tree on disk untouched, pages untouched + assert "storage" in json.load(open(docs / "module_tree.json")) + assert (docs / "storage.md").exists() + + +def test_rung_1_regenerates_instead_of_patching(tmp_path): + docs, config, prev = _setup(tmp_path) + backend = FakeBackend() + opts = UpdateOptions.from_rung(1, tau_full=2.0, tau_tree=2.0) + rec = _run(docs, config, prev, backend, opts) + assert rec.outcome == "incremental" + assert backend.update_calls == [] # no editing agent at rung 1 + assert "auth" in backend.module_calls # rewritten + # ancestors were invalidated and regenerated through complete() + assert "regenerated overview" in (docs / "core.md").read_text() + assert "regenerated overview" in (docs / "overview.md").read_text() + # api's Up is empty at rung 1: page untouched + assert (docs / "api.md").read_text().startswith("# api") + + +def test_whole_repo_mode(tmp_path): + docs, config, prev = _setup(tmp_path, tree={}) + for stem in ("auth", "api", "core", "storage", "pipeline"): + (docs / f"{stem}.md").unlink() + backend = FakeBackend() + rec = _run(docs, config, prev, backend, UpdateOptions()) + assert rec.outcome == "incremental" + assert [a["page"] for a in rec.active] == ["overview"] + assert json.load(open(docs / "module_tree.json")) == {} + assert backend.update_calls and backend.update_calls[0][0] == "overview" diff --git a/tests/test_updater_reference_index.py b/tests/test_updater_reference_index.py new file mode 100644 index 00000000..2aaedae5 --- /dev/null +++ b/tests/test_updater_reference_index.py @@ -0,0 +1,34 @@ +"""Reference index extraction and inverse lookup.""" + +from __future__ import annotations + +from updater_toy import REFRESH, graph_r1, tree_r1, write_pages + +from codewiki.src.be.updater.reference_index import ( + build_reference_index, + inverse, + load_reference_index, + save_reference_index, +) + + +def test_links_ids_names(tmp_path): + write_pages(tmp_path) + idx = build_reference_index(str(tmp_path), graph_r1(), tree_r1()) + assert idx["api"]["links"] == ["auth"] + assert idx["api"]["ids"] == [REFRESH] + assert "refresh" in idx["api"]["names"] + assert idx["pipeline"]["links"] == ["auth"] + assert "refresh" in idx["pipeline"]["names"] + assert idx["overview"]["links"] == ["core", "pipeline", "storage"] + inv = inverse(idx) + assert inv["auth"] == {"api", "core", "pipeline"} + assert inv[REFRESH] == {"api"} + assert "User" in inv and inv["User"] == {"storage"} + + +def test_roundtrip(tmp_path): + write_pages(tmp_path) + idx = build_reference_index(str(tmp_path), graph_r1()) + save_reference_index(idx, str(tmp_path)) + assert load_reference_index(str(tmp_path)) == idx diff --git a/tests/test_updater_tree_repair.py b/tests/test_updater_tree_repair.py new file mode 100644 index 00000000..4edec55a --- /dev/null +++ b/tests/test_updater_tree_repair.py @@ -0,0 +1,169 @@ +"""Step 2 of the component-level updater: tree repair rules.""" + +from __future__ import annotations + +from updater_toy import ( + HANDLE, + OAUTH, + PIPELINE, + REFRESH, + USER, + graph_r1, + graph_r2, + node, + tracked_r2, + tree_r1, +) + +from codewiki.src.be.updater import tree as T +from codewiki.src.be.updater.graph_diff import diff_graphs +from codewiki.src.be.updater.options import UpdateOptions +from codewiki.src.be.updater.tree_repair import ( + RULE_AGENT, + RULE_NEIGHBOR, + RULE_SAME_DIR, + RULE_SAME_FILE, + RoutingDecision, + repair_tree, +) + + +def test_owner_map_uses_deepest_node_and_parents_keep_unions(): + owner = T.owner_map(tree_r1()) + assert owner[HANDLE] == ("core", "api") + assert owner[USER] == ("storage",) + assert T.unit_paths(tree_r1()) == [ + ("core", "auth"), + ("core", "api"), + ("storage",), + ("pipeline",), + ] + + +def test_toy_repair(): + old, new = graph_r1(), graph_r2() + d = diff_graphs(old, new) + r = repair_tree(tree_r1(), d, new, tracked_r2(), UpdateOptions()) + # storage lost its only component and is gone + assert ("storage",) in r.deleted_nodes + assert "storage" not in r.tree + assert r.removed == [(USER, ("storage",))] + # OAuthClient routed by rule 2 (same directory, one leaf) into auth and its ancestors + dec = {x.component_id: x for x in r.routing} + assert dec[OAUTH].rule == RULE_SAME_DIR and dec[OAUTH].leaf_path == ("core", "auth") + assert OAUTH in r.tree["core"]["children"]["auth"]["components"] + assert OAUTH in r.tree["core"]["components"] + assert r.entered[("core", "auth")] == [OAUTH] + # 1 of 4 is under the default third: not flagged + assert r.growth[("core", "auth")] == 0.25 + assert r.growth_flagged == [] + # source tree untouched + assert "storage" in tree_r1() + + +def test_rule_1_same_file_wins(): + tree = tree_r1() + new = graph_r2() + new["src/api/routes.py::other"] = node( + "src/api/routes.py::other", "function", "def other(): pass" + ) + d = diff_graphs(graph_r1(), new) + r = repair_tree(tree, d, new, tracked_r2() | {"src/api/routes.py::other"}, UpdateOptions()) + dec = {x.component_id: x for x in r.routing} + assert dec["src/api/routes.py::other"].rule == RULE_SAME_FILE + assert dec["src/api/routes.py::other"].leaf_path == ("core", "api") + + +def test_rule_3_neighbor_majority_and_orphan(): + tree = tree_r1() + new = graph_r2() + nb = "src/util/helper.py::Helper" + new[nb] = node(nb, "class", "class Helper: pass", deps=[REFRESH, HANDLE, PIPELINE]) + orphan = "src/misc/lonely.py::Lonely" + new[orphan] = node(orphan, "class", "class Lonely: pass") + d = diff_graphs(graph_r1(), new) + tracked = tracked_r2() | {nb, orphan} + # neighbours: refresh (auth), handle (api), Pipeline (pipeline) -> 1/3 each < 0.5 -> orphan + r = repair_tree(tree, d, new, tracked, UpdateOptions(tau_nb=0.5, use_routing_agent=False)) + assert set(r.orphans) == {nb, orphan} + r2 = repair_tree(tree, d, new, tracked, UpdateOptions(tau_nb=0.3, use_routing_agent=False)) + dec = {x.component_id: x for x in r2.routing} + assert dec[nb].rule == RULE_NEIGHBOR + assert r2.orphans == [orphan] + + +def test_routing_agent_can_create_a_leaf(): + tree = tree_r1() + new = graph_r2() + a, b = "src/misc/lonely.py::Lonely", "src/misc/lonely.py::Lonelier" + new[a] = node(a, "class", "class Lonely: pass") + new[b] = node(b, "class", "class Lonelier: pass") + d = diff_graphs(graph_r1(), new) + + def router(orphans, ctx): + return [ + RoutingDecision( + cid, RULE_AGENT, ("core", "misc"), new_leaf=True, detail="new subsystem" + ) + for cid in orphans + ] + + r = repair_tree(tree, d, new, tracked_r2() | {a, b}, UpdateOptions(), route_orphans=router) + assert r.created_leaves == [("core", "misc")] + assert set(r.tree["core"]["children"]["misc"]["components"]) == {a, b} + assert a in r.tree["core"]["components"] + assert r.orphans == [] + assert ("core", "misc") not in r.growth_flagged # created leaves are not growth-flagged + + +def test_untracked_added_components_are_not_routed(): + new = graph_r2() + d = diff_graphs(graph_r1(), new) + r = repair_tree(tree_r1(), d, new, tracked_r2() - {OAUTH}, UpdateOptions()) + dec = {x.component_id: x for x in r.routing} + assert dec[OAUTH].leaf_path is None and dec[OAUTH].rule == "untracked" + assert OAUTH not in T.tracked_ids(r.tree) + + +def test_rename_rewrites_all_levels_and_agent_inserted_nodes_survive(): + tree = tree_r1() + tree["core"]["children"]["auth"]["children"] = { + "auth_tokens": {"components": [REFRESH], "children": {}} # agent-inserted: no path + } + old = graph_r1() + new = graph_r2() + moved = "src/auth/tokens.py::refresh" + new[moved] = node(moved, "function", old[REFRESH].source_code, ["self"]) # pure move + del new[REFRESH] + d = diff_graphs(old, new, UpdateOptions(tau_ren=0.9)) + assert d.renamed == {REFRESH: moved} + r = repair_tree(tree, d, new, (tracked_r2() - {REFRESH}) | {moved}, UpdateOptions()) + assert moved in r.tree["core"]["components"] + assert moved in r.tree["core"]["children"]["auth"]["components"] + assert r.tree["core"]["children"]["auth"]["children"]["auth_tokens"]["components"] == [moved] + assert "path" not in r.tree["core"]["children"]["auth"]["children"]["auth_tokens"] + + +def test_prune_removes_parent_when_all_children_go(): + tree = { + "p": { + "components": ["a.py::A"], + "children": {"c": {"components": ["a.py::A"], "children": {}}}, + } + } + T.remove_component(tree, "a.py::A") + removed = T.prune_empty(tree) + assert removed == [("p", "c"), ("p",)] + assert tree == {} + + +def test_growth_flag(): + tree = tree_r1() + new = graph_r2() + extra = [f"src/api/more{i}.py::M{i}" for i in range(3)] + for cid in extra: + new[cid] = node(cid, "class", f"class {cid.split('::')[1]}: pass", deps=[HANDLE]) + d = diff_graphs(graph_r1(), new) + r = repair_tree(tree, d, new, tracked_r2() | set(extra), UpdateOptions(tau_grow=0.33)) + assert ("core", "api") in r.growth_flagged + assert r.growth[("core", "api")] == 0.75 diff --git a/tests/test_updater_write_guard.py b/tests/test_updater_write_guard.py new file mode 100644 index 00000000..b712a4ee --- /dev/null +++ b/tests/test_updater_write_guard.py @@ -0,0 +1,106 @@ +"""The str_replace_editor write-set guard used by the incremental updater.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +from codewiki.src.be.agent_tools.str_replace_editor import check_write_allowed, str_replace_editor + + +def _ctx(docs_dir, allowed): + deps = SimpleNamespace( + registry={}, + absolute_docs_path=str(docs_dir), + absolute_repo_path=str(docs_dir / "repo"), + allowed_write_paths=allowed, + ) + return SimpleNamespace(deps=deps) + + +def test_check_write_allowed_none_is_unrestricted(tmp_path): + assert check_write_allowed(str(tmp_path / "x.md"), None) is None + + +def test_check_write_allowed_resolves_paths(tmp_path): + page = tmp_path / "a.md" + page.write_text("x") + allowed = {str(tmp_path / "." / "a.md")} + assert check_write_allowed(str(page), allowed) is None + denied = check_write_allowed(str(tmp_path / "b.md"), allowed) + assert denied and "not in this agent's write set" in denied and "a.md" in denied + + +def test_editor_refuses_pages_outside_write_set(tmp_path): + (tmp_path / "repo").mkdir() + (tmp_path / "a.md").write_text("hello a\n") + (tmp_path / "b.md").write_text("hello b\n") + allowed = {str(tmp_path / "a.md")} + + out = asyncio.run( + str_replace_editor( + _ctx(tmp_path, allowed), + "docs", + "str_replace", + path="b.md", + old_str="hello b", + new_str="bye b", + ) + ) + assert "not in this agent's write set" in out + assert (tmp_path / "b.md").read_text() == "hello b\n" + + out = asyncio.run( + str_replace_editor( + _ctx(tmp_path, allowed), + "docs", + "str_replace", + path="a.md", + old_str="hello a", + new_str="bye a", + ) + ) + assert "not in this agent's write set" not in out + assert (tmp_path / "a.md").read_text() == "bye a\n" + + # view stays allowed everywhere + out = asyncio.run(str_replace_editor(_ctx(tmp_path, allowed), "docs", "view", path="b.md")) + assert "hello b" in out + + # create of a page outside the set is refused too + out = asyncio.run( + str_replace_editor(_ctx(tmp_path, allowed), "docs", "create", path="c.md", file_text="new") + ) + assert "not in this agent's write set" in out + assert not (tmp_path / "c.md").exists() + + +def test_editor_rejects_absolute_and_escaping_paths(tmp_path): + (tmp_path / "repo").mkdir() + out = asyncio.run( + str_replace_editor(_ctx(tmp_path, None), "docs", "view", path=str(tmp_path / "a.md")) + ) + assert "must be relative" in out + out = asyncio.run(str_replace_editor(_ctx(tmp_path, None), "docs", "view", path="../x.md")) + assert "escapes" in out + + +def test_usage_to_dict_handles_property_and_method(): + from types import SimpleNamespace + + from codewiki.src.be.backend import usage_to_dict + from codewiki.src.be.pydantic_ai_backend import _run_usage + + usage = SimpleNamespace(input_tokens=10, output_tokens=3, requests=1, cost=None, details={}) + assert usage_to_dict(usage) == {"input_tokens": 10, "output_tokens": 3, "requests": 1} + assert _run_usage(SimpleNamespace(usage=usage)) == { + "input_tokens": 10, + "output_tokens": 3, + "requests": 1, + } + assert _run_usage(SimpleNamespace(usage=lambda: usage)) == { + "input_tokens": 10, + "output_tokens": 3, + "requests": 1, + } + assert _run_usage(SimpleNamespace()) is None diff --git a/tests/updater_toy.py b/tests/updater_toy.py new file mode 100644 index 00000000..4b1a718c --- /dev/null +++ b/tests/updater_toy.py @@ -0,0 +1,124 @@ +"""Toy repository from Part 4 of the C3 redesign note, shared by the updater tests.""" + +from __future__ import annotations + +from copy import deepcopy + +from codewiki.src.be.dependency_analyzer.models.core import Node + + +def node(cid: str, ctype: str, body: str, params=None, bases=None, deps=()) -> Node: + rel, name = cid.split("::", 1) + return Node( + id=cid, + name=name, + component_type=ctype, + file_path=f"/repo/{rel}", + relative_path=rel, + depends_on=set(deps), + source_code=body, + parameters=list(params or []), + base_classes=list(bases or []) or None, + node_type=ctype, + component_id=cid, + ) + + +LOGIN = "src/auth/login.py::login" +VALIDATE = "src/auth/login.py::validate" +REFRESH = "src/auth/token.py::refresh" +HANDLE = "src/api/routes.py::handle" +USER = "src/db/models.py::User" +PIPELINE = "src/pipeline/run.py::Pipeline" +OAUTH = "src/auth/oauth.py::OAuthClient" + + +def graph_r1() -> dict[str, Node]: + return { + LOGIN: node( + LOGIN, + "function", + "def login(user, pw):\n return validate(user, pw)\n", + ["user", "pw"], + deps=[VALIDATE], + ), + VALIDATE: node( + VALIDATE, "function", "def validate(user, pw):\n return pw == 'x'\n", ["user", "pw"] + ), + REFRESH: node( + REFRESH, "function", "def refresh(self):\n return new_token()\n", ["self"] + ), + HANDLE: node( + HANDLE, "function", "def handle(req):\n return refresh()\n", ["req"], deps=[REFRESH] + ), + USER: node(USER, "class", "class User(Base):\n id = Column()\n", bases=["Base"]), + PIPELINE: node(PIPELINE, "class", "class Pipeline:\n def run(self):\n pass\n"), + } + + +def graph_r2() -> dict[str, Node]: + g = graph_r1() + g[VALIDATE] = node( + VALIDATE, + "function", + "def validate(user, pw):\n log('validate')\n return pw == 'x'\n", + ["user", "pw"], + ) + g[REFRESH] = node( + REFRESH, + "function", + "def refresh(self, force=False):\n return new_token(force)\n", + ["self", "force"], + ) + g[OAUTH] = node( + OAUTH, + "class", + "class OAuthClient:\n def token(self):\n return refresh(self)\n", + deps=[REFRESH], + ) + del g[USER] + return g + + +def tree_r1() -> dict: + return { + "core": { + "path": "src", + "components": [LOGIN, VALIDATE, REFRESH, HANDLE], + "children": { + "auth": { + "path": "src/auth", + "components": [LOGIN, VALIDATE, REFRESH], + "children": {}, + }, + "api": {"path": "src/api", "components": [HANDLE], "children": {}}, + }, + }, + "storage": {"path": "src/db", "components": [USER], "children": {}}, + "pipeline": {"path": "src/pipeline", "components": [PIPELINE], "children": {}}, + } + + +PAGES_R1 = { + "auth": "# auth\n\nHandles `login` and `refresh` of tokens. See [api](api.md).\n", + "api": "# api\n\nRoutes call `refresh` from [auth](auth.md) (`src/auth/token.py::refresh`).\n", + "core": "# core\n\nChildren: [auth](auth.md), [api](api.md).\n", + "storage": "# storage\n\nThe `User` model.\n", + "pipeline": "# pipeline\n\nTokens are refreshed by [auth](auth.md) every five minutes via `refresh`.\n", + "overview": "# overview\n\n- [core](core.md)\n- [storage](storage.md)\n- [pipeline](pipeline.md)\n", +} + + +def write_pages(docs_dir, pages=None) -> None: + pages = pages if pages is not None else PAGES_R1 + docs_dir.mkdir(parents=True, exist_ok=True) + for stem, text in pages.items(): + (docs_dir / f"{stem}.md").write_text(text, encoding="utf-8") + + +def tracked_r2() -> set[str]: + return {LOGIN, VALIDATE, REFRESH, HANDLE, PIPELINE, OAUTH} + + +def copy(x): + return deepcopy(x)