From aed1c19d04b94189451ecc045a8ce703a1efea72 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Thu, 10 Sep 2026 10:49:36 +0700 Subject: [PATCH 1/2] Add artifact-aware documentation generation CodeWiki only documented what reached the dependency graph, and the graph was built from files whose extension is in CODE_EXTENSIONS. Dockerfiles, CI workflows, Makefiles, package manifests, config, schema and script files were dropped before a single Node existed, so no wiki ever described how a system is built, packaged, shipped, configured or tested. This adds a second analysis pass over the same file tree (dependency_analyzer/analyzers/artifact.py) that emits `artifact` nodes: - one node per artifact file (id `::`, source = capped 16 KB head) plus unit nodes for CI jobs, Dockerfile stages, Makefile targets, package.json scripts and pyproject/setup.cfg entry points; - edges from artifacts to the code they reference (COPY/ENTRYPOINT paths, `run:` lines, `python -m`, `make `, `npm run`, `module:function` entry points, compose build contexts), emitted only for fully resolved ids; - caps: per-file head, 40 files per class, total token budget filled in class priority order; lock files and binaries are never read; README/docs stay out unless --with-prose. Pipeline changes so the new nodes flow through unchanged machinery: - repo_analyzer: an ARTIFACT_WHITELIST lets `.github/workflows`, `*.ini`, `bin/*.sh`, `*.gradle` survive the default ignore list (user excludes and .gitignore still win); artifact names added to DEFAULT_INCLUDE_PATTERNS. - leaf_selection: `artifact` is always a valid leaf type; topo_sort no longer lets artifact->code edges prune code leaves. - cluster prompts mark artifact files `(artifact: )` and tell the LLM they are essential; `ensure_artifact_module` inserts a top-level "Build, Deployment and Configuration" module when clustering placed fewer than 80% of the artifact nodes (both the CLI adapter and run() paths). - prompt_template: fence language falls back to `text` instead of raising KeyError for Dockerfile/Makefile/.yml/.toml; artifact groups inline their capped source instead of re-reading the whole file; a index is appended to module prompts and to the repository overview prompt (USER_PROMPT itself is unchanged for the MCP prompt server). - str_replace_editor `view` on a directory no longer hides `.github` and quotes the path. - Node gains `artifact_class`; the graph builder writes temp/artifact_index.json. Flags: --artifacts/--no-artifacts (default on), --artifact-token-budget (200000), --with-prose (off), --artifact-exclude, and an agent_instructions.artifact_exclude key. Tests: tests/test_artifact_analyzer.py (classifier, whitelist, nodes and units, edge resolution, caps, leaf selection, prompt, fallback module). --- .gitignore | 1 + codewiki/cli/adapters/doc_generator.py | 8 + codewiki/cli/commands/generate.py | 39 +- codewiki/cli/models/config.py | 5 + .../src/be/agent_tools/str_replace_editor.py | 14 +- codewiki/src/be/caw_toolkit.py | 2 +- codewiki/src/be/cluster_modules.py | 89 +- .../analysis/analysis_service.py | 34 +- .../analysis/repo_analyzer.py | 46 +- .../dependency_analyzer/analyzers/artifact.py | 964 ++++++++++++++++++ .../src/be/dependency_analyzer/ast_parser.py | 14 +- .../dependency_graphs_builder.py | 26 + .../be/dependency_analyzer/leaf_selection.py | 5 + .../src/be/dependency_analyzer/models/core.py | 4 + .../src/be/dependency_analyzer/topo_sort.py | 10 +- .../be/dependency_analyzer/utils/patterns.py | 85 ++ .../be/dependency_analyzer/utils/security.py | 25 + codewiki/src/be/documentation_generator.py | 26 +- codewiki/src/be/prompt_template.py | 144 ++- codewiki/src/config.py | 27 + tests/test_artifact_analyzer.py | 414 ++++++++ 21 files changed, 1925 insertions(+), 57 deletions(-) create mode 100644 codewiki/src/be/dependency_analyzer/analyzers/artifact.py create mode 100644 tests/test_artifact_analyzer.py diff --git a/.gitignore b/.gitignore index 30891645..b6cd8465 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ tests/* !tests/test_ruby_analyzer.py !tests/test_processing_order_update.py !tests/test_leaf_selection.py +!tests/test_artifact_analyzer.py # Jupyter *.ipynb diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py index 27afc06a..26d5aef0 100644 --- a/codewiki/cli/adapters/doc_generator.py +++ b/codewiki/cli/adapters/doc_generator.py @@ -150,6 +150,9 @@ def generate(self) -> DocumentationJob: agent_instructions=self.config.get('agent_instructions'), use_gitignore=self.config.get('use_gitignore', True), prompt_caching=self.config.get('prompt_caching', True), + artifacts_enabled=self.config.get('artifacts_enabled', True), + artifact_token_budget=self.config.get('artifact_token_budget', 200_000), + with_prose=self.config.get('with_prose', False), ) # Run backend documentation generation @@ -215,6 +218,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig): # Import clustering function from codewiki.src.be.cluster_modules import ( cluster_modules, + ensure_artifact_module, get_clustering_input_token_count, super_group_modules, ) @@ -273,6 +277,10 @@ async def _run_backend_generation(self, backend_config: BackendConfig): backend_config, completer=lambda p: doc_generator.backend.complete(p, model=cluster_model), ) + # Artifact nodes the clustering LLM dropped get a fixed module + # so build/CI/config coverage does not depend on the LLM. + if getattr(backend_config, "artifacts_enabled", True): + module_tree = ensure_artifact_module(module_tree, leaf_nodes, components) # Only freshly clustered trees are deduped: renaming a cached # key whose .md already exists would orphan the doc. from codewiki.src.be.module_naming import dedupe_module_tree_names diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py index be41f41e..924bb35e 100644 --- a/codewiki/cli/commands/generate.py +++ b/codewiki/cli/commands/generate.py @@ -307,6 +307,30 @@ def _find_affected(tree, parent_names=None): help="Add prompt-cache breakpoints to agentic LLM calls; auto-falls back to " "normal calls if the provider rejects them (default: enabled)", ) +@click.option( + "--artifacts/--no-artifacts", + default=True, + help="Document build, CI, container, packaging, manifest, config, schema and " + "script files as part of the dependency graph (default: enabled)", +) +@click.option( + "--artifact-token-budget", + type=int, + default=200_000, + show_default=True, + help="Total token budget for artifact file contents added to the graph", +) +@click.option( + "--with-prose", + is_flag=True, + help="Also read the root README and docs/ as a `prose` artifact class (off by default)", +) +@click.option( + "--artifact-exclude", + type=str, + default=None, + help="Comma-separated patterns skipped by artifact analysis (e.g. 'docker/data/*,config/generated/*')", +) @click.option( "--update", is_flag=True, @@ -337,6 +361,10 @@ def generate_command( max_token_per_leaf_module: Optional[int], max_depth: Optional[int], prompt_caching: Optional[bool], + artifacts: bool = True, + artifact_token_budget: int = 200_000, + with_prose: bool = False, + artifact_exclude: Optional[str] = None, update: bool = False, compare_to: Optional[str] = None ): @@ -511,13 +539,14 @@ def generate_command( # Create runtime agent instructions from CLI options runtime_instructions = None - if any([include, exclude, focus, doc_type, instructions]): + if any([include, exclude, focus, doc_type, instructions, artifact_exclude]): runtime_instructions = AgentInstructions( include_patterns=parse_patterns(include) if include else None, exclude_patterns=parse_patterns(exclude) if exclude else None, focus_modules=parse_patterns(focus) if focus else None, doc_type=doc_type, custom_instructions=instructions, + artifact_exclude=parse_patterns(artifact_exclude) if artifact_exclude else None, ) if verbose: @@ -531,6 +560,8 @@ def generate_command( logger.debug(f"Doc type: {doc_type}") if instructions: logger.debug(f"Custom instructions: {instructions}") + if artifact_exclude: + logger.debug(f"Artifact exclude patterns: {parse_patterns(artifact_exclude)}") # Log max token settings if verbose if verbose: @@ -546,6 +577,7 @@ def generate_command( logger.debug(f"Max depth: {effective_max_depth}") logger.debug(f"Use gitignore: {effective_use_gitignore}") logger.debug(f"Prompt caching: {effective_prompt_caching}") + logger.debug(f"Artifacts: {artifacts} (token budget {artifact_token_budget}, prose {with_prose})") # Get agent instructions (merge runtime with persistent) agent_instructions_dict = None @@ -557,6 +589,7 @@ def generate_command( focus_modules=runtime_instructions.focus_modules or (config.agent_instructions.focus_modules if config.agent_instructions else None), doc_type=runtime_instructions.doc_type or (config.agent_instructions.doc_type if config.agent_instructions else None), custom_instructions=runtime_instructions.custom_instructions or (config.agent_instructions.custom_instructions if config.agent_instructions else None), + artifact_exclude=runtime_instructions.artifact_exclude or (config.agent_instructions.artifact_exclude if config.agent_instructions else None), ) agent_instructions_dict = merged.to_dict() elif config.agent_instructions and not config.agent_instructions.is_empty(): @@ -587,6 +620,10 @@ def generate_command( 'use_gitignore': use_gitignore if use_gitignore is not None else config.use_gitignore, # Prompt caching setting (runtime override takes precedence) 'prompt_caching': prompt_caching if prompt_caching is not None else config.prompt_caching, + # Artifact-aware generation (runtime-only flags) + 'artifacts_enabled': artifacts, + 'artifact_token_budget': artifact_token_budget, + 'with_prose': with_prose, }, verbose=verbose, generate_html=github_pages, diff --git a/codewiki/cli/models/config.py b/codewiki/cli/models/config.py index 93081d7a..5f55351b 100644 --- a/codewiki/cli/models/config.py +++ b/codewiki/cli/models/config.py @@ -40,6 +40,7 @@ class AgentInstructions: focus_modules: Optional[List[str]] = None # e.g., ["src/core", "src/api"] doc_type: Optional[str] = None # e.g., "api", "architecture", "user-guide" custom_instructions: Optional[str] = None # Free-form instructions + artifact_exclude: Optional[List[str]] = None # e.g., ["docker/data/*"] skipped by artifact analysis def to_dict(self) -> dict: """Convert to dictionary, excluding None values.""" @@ -48,6 +49,8 @@ def to_dict(self) -> dict: result['include_patterns'] = self.include_patterns if self.exclude_patterns: result['exclude_patterns'] = self.exclude_patterns + if self.artifact_exclude: + result['artifact_exclude'] = self.artifact_exclude if self.focus_modules: result['focus_modules'] = self.focus_modules if self.doc_type: @@ -65,6 +68,7 @@ def from_dict(cls, data: dict) -> 'AgentInstructions': focus_modules=data.get('focus_modules'), doc_type=data.get('doc_type'), custom_instructions=data.get('custom_instructions'), + artifact_exclude=data.get('artifact_exclude'), ) def is_empty(self) -> bool: @@ -72,6 +76,7 @@ def is_empty(self) -> bool: return not any([ self.include_patterns, self.exclude_patterns, + self.artifact_exclude, self.focus_modules, self.doc_type, self.custom_instructions, diff --git a/codewiki/src/be/agent_tools/str_replace_editor.py b/codewiki/src/be/agent_tools/str_replace_editor.py index 3d71e68d..3ee8b9c4 100644 --- a/codewiki/src/be/agent_tools/str_replace_editor.py +++ b/codewiki/src/be/agent_tools/str_replace_editor.py @@ -6,6 +6,7 @@ import json import re +import shlex import subprocess import sys from collections import defaultdict @@ -487,8 +488,11 @@ def view(self, path: Path, view_range: Optional[List[int]] = None): self.logs.append("The `view_range` parameter is not allowed when `path` points to a directory.") return + # Hidden entries are skipped except `.github` (CI workflows are + # documentation-relevant artifacts the agent must be able to find). out = subprocess.run( - rf"find {path} -maxdepth 2 -not -path '*/\.*'", + rf"find {shlex.quote(str(path))} -maxdepth 2 " + r"\( -not -path '*/.*' -o -name .github -o -path '*/.github/*' \)", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -499,7 +503,7 @@ def view(self, path: Path, view_range: Optional[List[int]] = None): if not stderr: stdout = stdout.replace(str(path), self._get_display_path(path)) - stdout = f"Here's the files and directories up to 2 levels deep in {self._get_display_path(path)}, excluding hidden items:\n{stdout}\n" + stdout = f"Here's the files and directories up to 2 levels deep in {self._get_display_path(path)}, excluding hidden items (except .github):\n{stdout}\n" self.logs.append(stdout) return @@ -747,14 +751,14 @@ async def str_replace_editor( """ Custom editing tool for viewing, creating and editing files * State is persistent across command calls and discussions with the user - * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep. + * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories (plus `.github`) up to 2 levels deep. * The `create` command cannot be used if the specified `path` already exists as a file * If a `command` generates a long output, it will be truncated and marked with `` * The `undo_edit` command will revert the last edit made to the file at `path` * Only `view` command is allowed when `working_dir` is `repo`. Args: - working_dir: The working directory to use. Choose `repo` to work with the repository files, or `docs` to work with the generated documentation files. + working_dir: The working directory to use. Choose `repo` to view repository files (source code, and build/CI/container/manifest/config artifacts such as Dockerfile, Makefile, .github/workflows/*.yml, pyproject.toml), or `docs` to work with the generated documentation files. command: The command to run. Allowed options are: `view`, `create`, `str_replace`, `insert`, `undo_edit`. path: Path to file or directory, e.g. `./chat_core.md` or `./agents/` file: Alias for `path` parameter (for compatibility with some models) @@ -805,7 +809,7 @@ async def str_replace_editor( description=""" Custom editing tool for viewing, creating and editing files * State is persistent across command calls and discussions with the user - * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep. + * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories (plus `.github`) up to 2 levels deep. * The `create` command cannot be used if the specified `path` already exists as a file * If a `command` generates a long output, it will be truncated and marked with `` * The `undo_edit` command will revert the last edit made to the file at `path` diff --git a/codewiki/src/be/caw_toolkit.py b/codewiki/src/be/caw_toolkit.py index e412b09f..f409b6e5 100644 --- a/codewiki/src/be/caw_toolkit.py +++ b/codewiki/src/be/caw_toolkit.py @@ -123,7 +123,7 @@ async def read_code_components(self, component_ids: list[str]) -> str: description=( "Custom editing tool for viewing, creating and editing files.\n" "* If `path` is a file, `view` displays the result of applying `cat -n`. " - "If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep.\n" + "If `path` is a directory, `view` lists non-hidden files and directories (plus `.github`) up to 2 levels deep.\n" "* The `create` command cannot be used if the specified `path` already exists as a file.\n" "* If a `command` generates a long output, it will be truncated and marked with ``.\n" "* The `undo_edit` command will revert the last edit made to the file at `path`.\n" diff --git a/codewiki/src/be/cluster_modules.py b/codewiki/src/be/cluster_modules.py index 54c6ed32..080e6bd1 100644 --- a/codewiki/src/be/cluster_modules.py +++ b/codewiki/src/be/cluster_modules.py @@ -43,8 +43,11 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str potential_core_components = "" potential_core_components_with_code = "" for file, leaf_nodes in dict(sorted(leaf_nodes_by_file.items())).items(): - potential_core_components += f"# {file}\n" - potential_core_components_with_code += f"# {file}\n" + header = f"# {file}" + if all(components[n].component_type == "artifact" for n in leaf_nodes): + header += f" (artifact: {components[leaf_nodes[0]].artifact_class or 'config'})" + potential_core_components += f"{header}\n" + potential_core_components_with_code += f"{header}\n" for leaf_node in leaf_nodes: potential_core_components += f"\t{leaf_node}\n" potential_core_components_with_code += f"\t{leaf_node}\n" @@ -603,3 +606,85 @@ def super_group_modules( len(subsystems), ) return result + + +# --------------------------------------------------------------------------- # +# Guaranteed artifact module +# --------------------------------------------------------------------------- # + +ARTIFACT_MODULE_NAME = "Build, Deployment and Configuration" +# Insert the fallback module when clustering kept less than this share of the +# artifact leaf nodes. +ARTIFACT_MIN_SHARE = 0.8 + + +def collect_module_tree_component_ids(module_tree: Dict[str, Any]) -> set: + """Return every component id referenced anywhere in ``module_tree``.""" + ids: set = set() + + def _walk(tree: Dict[str, Any]) -> None: + for module_info in tree.values(): + if not isinstance(module_info, dict): + continue + ids.update(module_info.get("components", []) or []) + children = module_info.get("children", {}) + if isinstance(children, dict): + _walk(children) + + _walk(module_tree) + return ids + + +def ensure_artifact_module( + module_tree: Dict[str, Any], + leaf_nodes: List[str], + components: Dict[str, Node], + min_share: float = ARTIFACT_MIN_SHARE, +) -> Dict[str, Any]: + """Guarantee that artifact leaf nodes are documented. + + Clustering is an LLM call and may drop or scatter artifact nodes despite + the prompt. If fewer than ``min_share`` of the artifact leaf nodes landed + in ``module_tree``, add a fixed top-level module holding every unassigned + artifact node. Returns ``module_tree`` unchanged in whole-repository mode + (empty tree: one agent documents all leaf nodes anyway). + """ + if not module_tree: + return module_tree + artifact_leaves = [ + n for n in leaf_nodes + if n in components and components[n].component_type == "artifact" + ] + if not artifact_leaves: + return module_tree + assigned = collect_module_tree_component_ids(module_tree) + unassigned = [n for n in artifact_leaves if n not in assigned] + share = 1.0 - len(unassigned) / len(artifact_leaves) + logger.info( + "Artifact coverage after clustering: %d/%d artifact leaf nodes assigned (%.0f%%)", + len(artifact_leaves) - len(unassigned), len(artifact_leaves), share * 100, + ) + if not unassigned or share >= min_share: + return module_tree + + from codewiki.src.be.dependency_analyzer.analyzers.artifact import CLASS_PRIORITY + from codewiki.src.be.module_naming import collect_module_tree_names + + def _order(node_id: str): + node = components[node_id] + cls = node.artifact_class or "config" + rank = CLASS_PRIORITY.index(cls) if cls in CLASS_PRIORITY else len(CLASS_PRIORITY) + return (rank, node.relative_path, node_id) + + unassigned.sort(key=_order) + name = resolve_unique_name(ARTIFACT_MODULE_NAME, None, collect_module_tree_names(module_tree)) + module_tree[name] = { + "path": _common_path_prefix([components[n].relative_path for n in unassigned]) or ".", + "components": unassigned, + "children": {}, + } + logger.info( + "Artifact coverage %.0f%% < %.0f%%; inserted top-level module '%s' with %d components", + share * 100, min_share * 100, name, len(unassigned), + ) + return module_tree diff --git a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py index b3aaf625..f0cb468d 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py +++ b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py @@ -9,7 +9,7 @@ import logging import traceback from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import CallGraphAnalyzer from codewiki.src.be.dependency_analyzer.analysis.cloning import ( @@ -22,6 +22,9 @@ from codewiki.src.be.dependency_analyzer.models.core import Repository from codewiki.src.be.dependency_analyzer.utils.security import assert_safe_path, safe_open_text +if TYPE_CHECKING: # pragma: no cover + from codewiki.src.be.dependency_analyzer.analyzers.artifact import ArtifactOptions + logger = logging.getLogger(__name__) @@ -288,14 +291,20 @@ def _read_readme_file(self, repo_dir: str) -> str | None: logger.debug("No README file found in repository root.") return None - def _analyze_call_graph(self, file_tree: dict[str, Any], repo_dir: str) -> dict[str, Any]: + def _analyze_call_graph( + self, + file_tree: dict[str, Any], + repo_dir: str, + artifact_options: "ArtifactOptions | None" = None, + ) -> dict[str, Any]: """ Perform multi-language call graph analysis. - This method will be expanded to handle: - - Python AST analysis (current) - - JavaScript/TypeScript AST analysis (planned) - - Additional language support (future) + When ``artifact_options`` is given and enabled, a second pass turns + build/CI/container/manifest/config files in the same file tree into + ``artifact`` nodes (see ``analyzers/artifact.py``) and appends them to + the ``functions``/``relationships`` lists so the rest of the pipeline + treats them like any other component. """ logger.debug("Extracting code files from file tree...") code_files = self.call_graph_analyzer.extract_code_files(file_tree) @@ -311,6 +320,19 @@ def _analyze_call_graph(self, file_tree: dict[str, Any], repo_dir: str) -> dict[ result["call_graph"]["supported_languages"] = self._get_supported_languages() result["call_graph"]["unsupported_files"] = len(code_files) - len(supported_files) + if artifact_options is not None and artifact_options.enabled: + from codewiki.src.be.dependency_analyzer.analyzers.artifact import analyze_artifacts + + artifacts = analyze_artifacts( + file_tree, repo_dir, result.get("functions", []), artifact_options + ) + result.setdefault("functions", []).extend(n.model_dump() for n in artifacts.nodes) + result.setdefault("relationships", []).extend( + r.model_dump() for r in artifacts.relationships + ) + result["artifact_index"] = artifacts.index + result["call_graph"]["artifact_nodes"] = len(artifacts.nodes) + return result def _filter_supported_languages(self, code_files: list[dict]) -> list[dict]: diff --git a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py index aa13c9c4..c30b3a7c 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py @@ -6,6 +6,7 @@ """ import fnmatch +import os import logging import shutil import subprocess @@ -15,6 +16,7 @@ from pathspec import GitIgnoreSpec from codewiki.src.be.dependency_analyzer.utils.patterns import ( + ARTIFACT_WHITELIST, DEFAULT_IGNORE_PATTERNS, DEFAULT_INCLUDE_PATTERNS, ) @@ -184,12 +186,12 @@ def __init__( self.include_patterns = ( include_patterns if include_patterns is not None else DEFAULT_INCLUDE_PATTERNS ) - # Exclude patterns: if specified, MERGE with default ignore patterns - self.exclude_patterns = ( - list(DEFAULT_IGNORE_PATTERNS) + exclude_patterns - if exclude_patterns is not None - else list(DEFAULT_IGNORE_PATTERNS) - ) + # Exclude patterns: if specified, MERGE with default ignore patterns. + # The two sets are also kept apart: user excludes always win, while + # the defaults yield to ARTIFACT_WHITELIST (e.g. `.github/workflows`). + self.default_exclude_patterns = list(DEFAULT_IGNORE_PATTERNS) + self.user_exclude_patterns = list(exclude_patterns) if exclude_patterns is not None else [] + self.exclude_patterns = self.default_exclude_patterns + self.user_exclude_patterns self.use_gitignore = use_gitignore self._gitignore_filter: Optional[GitIgnoreFilter] = None @@ -261,8 +263,9 @@ def build_tree(path: Path, base_path: Path) -> Optional[Dict]: return build_tree(Path(repo_dir), Path(repo_dir)) - def _should_exclude_path(self, path: str, filename: str, is_dir: bool = False) -> bool: - for pattern in self.exclude_patterns: + @staticmethod + def _matches_any(path: str, filename: str, patterns: List[str]) -> bool: + for pattern in patterns: if fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(filename, pattern): return True if pattern.endswith("/") and path.startswith(pattern.rstrip("/")): @@ -271,6 +274,33 @@ def _should_exclude_path(self, path: str, filename: str, is_dir: bool = False) - return True if pattern in path.split("/"): return True + return False + + @staticmethod + def _is_artifact_whitelisted(path: str, filename: str, is_dir: bool) -> bool: + """True when ``path`` is an artifact the default ignore list must not drop. + + Directories count as whitelisted when a whitelist pattern lives below + them, so ``.github`` survives long enough for ``.github/workflows/*`` + to be visited. + """ + norm = path.replace(os.sep, "/") + for pattern in ARTIFACT_WHITELIST: + if fnmatch.fnmatch(norm, pattern) or fnmatch.fnmatch(filename, pattern): + return True + if is_dir and "/" in pattern and pattern.startswith(norm.rstrip("/") + "/"): + return True + return False + + def _should_exclude_path(self, path: str, filename: str, is_dir: bool = False) -> bool: + # User-provided excludes always win. + if self._matches_any(path, filename, self.user_exclude_patterns): + return True + # Built-in ignores yield to the artifact whitelist. + if not self._is_artifact_whitelisted(path, filename, is_dir) and self._matches_any( + path, filename, self.default_exclude_patterns + ): + return True if self._gitignore_filter and self._gitignore_filter.is_ignored(path, is_dir): return True return False diff --git a/codewiki/src/be/dependency_analyzer/analyzers/artifact.py b/codewiki/src/be/dependency_analyzer/analyzers/artifact.py new file mode 100644 index 00000000..7bc6b118 --- /dev/null +++ b/codewiki/src/be/dependency_analyzer/analyzers/artifact.py @@ -0,0 +1,964 @@ +"""Artifact analyzer: build, CI, container, packaging, manifest, config, +schema and script files as first-class dependency-graph nodes. + +The language analyzers only see files whose extension is in +``CODE_EXTENSIONS``. Everything that describes how the system is built, +packaged, shipped, configured and tested (Dockerfiles, GitHub workflows, +Makefiles, ``pyproject.toml``, ``package.json``, ``*.proto`` ...) never +became a ``Node`` and was therefore never documented. + +This module runs *after* the language analyzers over the same file tree and +emits: + +* one ``Node`` per artifact file (``component_type="artifact"``, + id ``::``, ``source_code`` = capped file head), +* one child ``Node`` per unit where a cheap parser exists (CI job, + Dockerfile stage, Makefile target, ``package.json`` script, + ``pyproject``/``setup.cfg`` entry point), id ``::``, +* ``CallRelationship`` edges from artifacts to the code components (or other + artifacts) they reference. Only fully resolved ids are emitted, because + ``ast_parser`` falls back to *name* matching for unresolved callees. + +Caps keep huge repositories bounded: per-file head, per-class file count and +a total token budget filled in ``CLASS_PRIORITY`` order. + +The module deliberately imports nothing from config or prompt code so that +``prompt_template`` can import it without a cycle. +""" + +from __future__ import annotations + +import fnmatch +import json +import logging +import os +import posixpath +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Optional + +from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node +from codewiki.src.be.dependency_analyzer.utils.patterns import ARTIFACT_LOCKFILES +from codewiki.src.be.dependency_analyzer.utils.security import safe_read_head +from codewiki.src.be.utils import count_tokens + +try: # Python >= 3.11 + import tomllib +except ImportError: # pragma: no cover + tomllib = None # type: ignore[assignment] + +logger = logging.getLogger(__name__) + +ARTIFACT_TYPE = "artifact" +ARTIFACT_FILE_NODE_TYPE = "artifact_file" +ARTIFACT_UNIT_NODE_TYPE = "artifact_unit" + +# Order in which classes consume the total token budget (and are rendered). +CLASS_PRIORITY = [ + "manifest", + "build", + "container", + "ci", + "packaging", + "test_infra", + "schema", + "config", + "script", + "prose", +] + +TRUNCATION_MARKER = "\n# [codewiki: truncated - showing first {shown} of {total} bytes]\n" + +# Unit names that should be kept first when a file has more units than the cap. +PRIORITY_UNITS = { + "all", "build", "test", "tests", "install", "lint", "release", "dev", + "start", "ci", "docker", "publish", "check", "format", "deploy", +} + +_PROSE_EXTS = {".md", ".mdx", ".rst", ".txt"} +_CODE_OR_SCRIPT_EXTS = ( + "py", "sh", "bash", "js", "mjs", "cjs", "ts", "tsx", "jsx", "rb", "ps1", + "mk", "toml", "yaml", "yml", "json", "cfg", "ini", "java", "kt", "go", + "rs", "c", "cc", "cpp", "h", "hpp", "cs", "php", "proto", "fbs", +) +PATH_TOKEN_RE = re.compile( + r"(? str: + rel_path = rel_path.replace(os.sep, "/") + return f"{rel_path}::{posixpath.basename(rel_path)}" + + +def is_artifact_node(node: Any) -> bool: + return getattr(node, "component_type", None) == ARTIFACT_TYPE + + +def is_artifact_file_node(node: Any) -> bool: + return is_artifact_node(node) and getattr(node, "node_type", None) == ARTIFACT_FILE_NODE_TYPE + + +# --------------------------------------------------------------------------- # +# Classification +# --------------------------------------------------------------------------- # + +_DROP_SEGMENTS = { + "docs", "doc", "node_modules", "vendor", "third_party", "dist", ".git", + "fixtures", "fixture", "testdata", "test_data", "__snapshots__", +} +_CI_NAMES = { + ".gitlab-ci.yml", "Jenkinsfile", ".travis.yml", "azure-pipelines.yml", + "appveyor.yml", ".appveyor.yml", "bitbucket-pipelines.yml", "cloudbuild.yaml", + "cloudbuild.yml", ".drone.yml", +} +_MANIFEST_NAMES = { + "package.json", "pyproject.toml", "setup.py", "setup.cfg", "Cargo.toml", + "go.mod", "Gemfile", "pnpm-workspace.yaml", "lerna.json", "nx.json", + "turbo.json", "composer.json", "pom.xml", "build.gradle", "build.gradle.kts", + "settings.gradle", "settings.gradle.kts", "Package.swift", "pubspec.yaml", + "Pipfile", "environment.yml", "environment.yaml", "MANIFEST.in", "Procfile", + "conda.yaml", "conda.yml", +} +_MANIFEST_EXTS = {".gemspec", ".csproj", ".fsproj", ".vbproj", ".sln", ".podspec"} +_MANIFEST_RES = [re.compile(r"^requirements[\w.-]*\.txt$"), re.compile(r"^tsconfig[\w.-]*\.json$")] +_PACKAGING_EXTS = {".spec", ".service", ".socket", ".timer", ".plist", ".nuspec", ".wxs", ".desktop"} +_PACKAGING_TOPS = {"debian", "rpm", "installer", "pkg", "packaging"} +_PACKAGING_PACKAGES_NAMES = { + "control", "rules", "changelog", "postinst", "prerm", "postrm", "preinst", "copyright", +} +_BUILD_NAMES = { + "Makefile", "GNUmakefile", "makefile", "CMakeLists.txt", "Rakefile", "BUILD", + "BUILD.gn", "BUILD.bazel", "WORKSPACE", "DEPS", "meson.build", "SConstruct", + "SConscript", "build.xml", "gulpfile.js", "Gruntfile.js", "Herebyfile.mjs", +} +_BUILD_EXTS = {".gn", ".gni", ".gradle", ".rake", ".mk", ".cmake", ".bzl", ".ninja"} +_BUILD_CONFIG_RE = re.compile(r"^(webpack|rollup|vite|esbuild|tsup|babel)\.config\.[cm]?[jt]s$") +_BUILD_TOPS = {"build", "rakelib", "cmake"} +_TEST_INFRA_NAMES = { + "pytest.ini", "tox.ini", "conftest.py", ".coveragerc", "codecov.yml", + "karma.conf.js", ".nycrc", "noxfile.py", +} +_TEST_INFRA_RE = re.compile(r"^(jest|vitest|playwright|cypress|wdio|mocha)\.(config|workspace)\.[\w.]+$") +_SCHEMA_EXTS = {".proto", ".fbs", ".avsc", ".thrift", ".graphql", ".gql", ".capnp", ".xsd", ".wsdl"} +_SCHEMA_RE = re.compile(r"^(openapi|swagger)[\w.-]*\.(ya?ml|json)$") +_CONFIG_EXTS = { + ".toml", ".yml", ".yaml", ".ini", ".cfg", ".conf", ".options", ".properties", + ".tf", ".nix", ".editorconfig", ".env", +} +_CONFIG_TOPS = {"config", "configs", "conf", "etc", "settings", ".github"} +_SCRIPT_EXTS = {".sh", ".bash", ".zsh", ".ps1", ".bat", ".cmd"} +_SCRIPT_TOPS = {"bin", "scripts", "script", "tools", "tool", "hack", "ci"} + + +def classify_artifact( + rel_path: str, + name: str, + size: int, + opts: ArtifactOptions, + first_line: str | None = None, +) -> Optional[str]: + """Return the artifact class of ``rel_path`` or ``None`` when it is not one. + + ``first_line`` is only needed for extension-less files under ``bin/`` or + ``scripts/`` (shebang check); callers may pass ``None`` elsewhere. + """ + rel = rel_path.replace(os.sep, "/") + while rel.startswith("./"): + rel = rel[2:] + segs = rel.split("/") + depth = len(segs) - 1 + top = segs[0] if depth > 0 else "" + ext = Path(name).suffix.lower() + lower = name.lower() + + # ---- hard drops ------------------------------------------------------- + if size <= 0 or name in ARTIFACT_LOCKFILES: + return None + if any(seg in _DROP_SEGMENTS for seg in segs[:-1]) and not (opts.with_prose and top in {"docs", "doc"}): + return None + for pat in opts.exclude_patterns or []: + if fnmatch.fnmatch(rel, pat) or fnmatch.fnmatch(name, pat): + return None + if pat and (rel.startswith(pat.rstrip("/") + "/") or pat.rstrip("/") in segs[:-1]): + return None + if rel.startswith(".github/ISSUE_TEMPLATE/") or lower.startswith("pull_request_template"): + return None + if name in {"CODEOWNERS", "FUNDING.yml", "funding.yml"}: + return None + + # ---- prose (opt-in) ---------------------------------------------------- + if ext in _PROSE_EXTS and not any(r.match(name) for r in _MANIFEST_RES): + if not opts.with_prose: + return None + if depth == 0 and (re.match(r"^readme", lower) or re.match(r"^contributing", lower)): + return "prose" + if top in {"docs", "doc"} and ext in {".md", ".mdx", ".rst"}: + return "prose" + return None + + # ---- ci ------------------------------------------------------------------ + if rel.startswith(".github/workflows/") and ext in {".yml", ".yaml"}: + return "ci" + if rel.startswith(".github/actions/") and lower in {"action.yml", "action.yaml"}: + return "ci" + if name in _CI_NAMES: + return "ci" + if top in {"ci", ".circleci", ".buildkite", ".yamato"} and ext in {".yml", ".yaml", ".sh"}: + return "ci" + + # ---- container ----------------------------------------------------------- + if fnmatch.fnmatch(name, "Dockerfile*") or lower.endswith(".dockerfile") or lower == "containerfile": + return "container" + if re.match(r"^(docker-)?compose[.\w-]*\.ya?ml$", lower): + return "container" + if top == "docker" and ( + ext in {".yml", ".yaml", ".sh", ".erb", ".conf", ".env", ".mk"} + or "makefile" in lower + or lower.endswith(".conf.py") # e.g. gunicorn.conf.py; other .py files are code + ): + return "container" + + # ---- manifest (before packaging: packages/*/package.json is a manifest) -- + if name in _MANIFEST_NAMES or ext in _MANIFEST_EXTS or any(r.match(name) for r in _MANIFEST_RES): + return "manifest" + + # Source files are code, not artifacts, unless an explicit name rule above + # (setup.py, conftest.py, noxfile.py, *.conf.py) already claimed them. + if ext in {".py", ".js", ".ts", ".java", ".rb", ".go", ".rs", ".c", ".cpp", ".cs", ".php", ".kt"}: + if name in _TEST_INFRA_NAMES: + return "test_infra" + if name in _BUILD_NAMES or _BUILD_CONFIG_RE.match(name) or _TEST_INFRA_RE.match(name): + return "build" if name in _BUILD_NAMES or _BUILD_CONFIG_RE.match(name) else "test_infra" + return None + + # ---- packaging ------------------------------------------------------------- + if ext in _PACKAGING_EXTS: + return "packaging" + if top in _PACKAGING_TOPS: + return "packaging" + if top == "packages" and (ext in {".sh", ".conf", ".spec", ".service"} or name in _PACKAGING_PACKAGES_NAMES): + return "packaging" + + # ---- build ------------------------------------------------------------------- + if name in _BUILD_NAMES or ext in _BUILD_EXTS or _BUILD_CONFIG_RE.match(name) or lower.startswith(".babelrc"): + return "build" + if top in _BUILD_TOPS and ext in _BUILD_EXTS: + return "build" + + # ---- test infrastructure --------------------------------------------------- + if name in _TEST_INFRA_NAMES or _TEST_INFRA_RE.match(name) or lower.startswith(".mocharc"): + return "test_infra" + + # ---- schema ------------------------------------------------------------------ + if ext in _SCHEMA_EXTS or _SCHEMA_RE.match(lower) or lower.endswith(".schema.json"): + return "schema" + if top in {"schema", "schemas"} and ext in {".json", ".yml", ".yaml"}: + return "schema" + + # ---- config ------------------------------------------------------------------ + if ext in _CONFIG_EXTS or ( + name.startswith(".") and ext in {"", ".json", ".yml", ".yaml", ".js", ".cjs"} + ): + if depth <= 1 or top in _CONFIG_TOPS: + return "config" + if ext in {".json", ".xml"} and (depth == 0 or top in {"config", "configs", "conf", "etc"}): + return "config" + + # ---- script ------------------------------------------------------------------ + if ext in _SCRIPT_EXTS and (depth <= 2 or top in _SCRIPT_TOPS): + return "script" + if ext == "" and top in {"bin", "scripts"} and first_line is not None and first_line.startswith("#!"): + return "script" + + return None + + +# --------------------------------------------------------------------------- # +# Unit parsers +# --------------------------------------------------------------------------- # + + +def _line_of(text: str, offset: int) -> int: + return text.count("\n", 0, offset) + 1 + + +def _yaml_children(text: str, top_key: str) -> list[tuple[str, int, int]]: + """Return ``(name, start, end)`` offsets of the direct children of a top-level + YAML mapping such as ``jobs:`` or ``services:`` without a YAML library.""" + m = re.search(rf"^{re.escape(top_key)}:[ \t]*(?:#.*)?$", text, re.M) + if not m: + return [] + body_start = m.end() + nxt = re.compile(r"^\S", re.M).search(text, body_start + 1) + body_end = nxt.start() if nxt else len(text) + body = text[body_start:body_end] + indent = None + for line in body.splitlines(): + if line.strip() and not line.lstrip().startswith("#"): + indent = len(line) - len(line.lstrip(" ")) + break + if not indent: + return [] + child_re = re.compile(rf"^ {{{indent}}}([A-Za-z_\"'][\w.\"'-]*):[ \t]*(?:#.*)?$", re.M) + matches = list(child_re.finditer(body)) + units: list[tuple[str, int, int]] = [] + for i, cm in enumerate(matches): + start = body_start + cm.start() + end = body_start + (matches[i + 1].start() if i + 1 < len(matches) else len(body)) + units.append((cm.group(1).strip("\"'"), start, end)) + return units + + +def _dockerfile_units(text: str) -> list[tuple[str, int, int]]: + from_re = re.compile(r"^FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", re.I | re.M) + matches = list(from_re.finditer(text)) + if not matches or (len(matches) == 1 and not matches[0].group(2)): + return [] + units = [] + for i, m in enumerate(matches): + name = m.group(2) or f"stage_{i}" + end = matches[i + 1].start() if i + 1 < len(matches) else len(text) + units.append((name, m.start(), end)) + return units + + +def _makefile_units(text: str) -> list[tuple[str, int, int]]: + """Return ``(target, start, end)`` for explicit Makefile targets. + + Pattern rules (``%``), variable expansions in names, dot-targets such as + ``.PHONY`` and assignments (``VAR := x``) are skipped. A unit spans the + header line plus the tab-indented recipe lines that follow it. + """ + target_re = re.compile(r"^([A-Za-z0-9][\w./-]*)\s*:(?![:=])(?P.*)$") + units: list[tuple[str, int, int]] = [] + seen: set[str] = set() + lines = text.split("\n") + offsets: list[int] = [] + pos = 0 + for line in lines: + offsets.append(pos) + pos += len(line) + 1 + i = 0 + while i < len(lines): + m = target_re.match(lines[i]) + if not m or "%" in m.group(1) or "$" in m.group(1) or m.group(1).startswith(".") \ + or re.match(r"\s*[?+!]?=", m.group("rest")): + i += 1 + continue + name = m.group(1) + j = i + 1 + while j < len(lines) and (lines[j].startswith("\t") or (lines[j].strip() == "" and j + 1 < len(lines) and lines[j + 1].startswith("\t"))): + j += 1 + end = offsets[j] - 1 if j < len(lines) else len(text) + if name not in seen: + seen.add(name) + units.append((name, offsets[i], end)) + i = j + return units + + +def _package_json_units(text: str) -> tuple[list[tuple[str, int, int, str]], dict | None]: + try: + data = json.loads(text) + except (json.JSONDecodeError, ValueError): + return [], None + if not isinstance(data, dict): + return [], None + scripts = data.get("scripts") + units: list[tuple[str, int, int, str]] = [] + if isinstance(scripts, dict): + for k, v in scripts.items(): + if not isinstance(v, str): + continue + unit_text = json.dumps({k: v}) + m = re.search(rf"\"{re.escape(k)}\"\s*:", text) + start = m.start() if m else 0 + units.append((str(k), start, start + len(unit_text), unit_text)) + return units, data + + +def _entry_point_units(text: str, kind: str) -> list[tuple[str, int, int, str, str]]: + """Return ``(name, start, end, module, func)`` for console-script entry points. + + ``kind`` is ``"pyproject"`` or ``"setup_cfg"``. + """ + results: list[tuple[str, int, int, str, str]] = [] + if kind == "pyproject": + tables: dict[str, Any] = {} + if tomllib is not None: + try: + data = tomllib.loads(text) + proj = data.get("project", {}) if isinstance(data, dict) else {} + for key in ("scripts", "gui-scripts"): + val = proj.get(key) if isinstance(proj, dict) else None + if isinstance(val, dict): + tables.update(val) + poetry = data.get("tool", {}).get("poetry", {}) if isinstance(data, dict) else {} + if isinstance(poetry, dict) and isinstance(poetry.get("scripts"), dict): + tables.update(poetry["scripts"]) + except Exception: # noqa: BLE001 - fall back to regex below + tables = {} + if not tables: + for sec in re.finditer(r"^\[(?:project\.(?:gui-)?scripts|tool\.poetry\.scripts)\]\s*$(.*?)(?=^\[|\Z)", text, re.M | re.S): + for line in sec.group(1).splitlines(): + lm = re.match(r"^\s*([\w.-]+)\s*=\s*[\"']([\w.]+):([\w.]+)[\"']", line) + if lm: + tables[lm.group(1)] = f"{lm.group(2)}:{lm.group(3)}" + for name, target in tables.items(): + if not isinstance(target, str) or ":" not in target: + continue + module, func = target.split(":", 1) + m = re.search(rf"^\s*{re.escape(name)}\s*=", text, re.M) + start = m.start() if m else 0 + snippet = f'{name} = "{target}"' + results.append((str(name), start, start + len(snippet), module.strip(), func.strip())) + else: # setup.cfg + m = re.search(r"^console_scripts\s*=\s*$(.*?)(?=^\S|\Z)", text, re.M | re.S) + if m: + for line in m.group(1).splitlines(): + lm = re.match(r"^\s*([\w.-]+)\s*=\s*([\w.]+):([\w.]+)", line) + if lm: + pos = text.find(line) + results.append((lm.group(1), pos, pos + len(line), lm.group(2), lm.group(3))) + return results + + +def _prioritise_units(units: list, cap: int) -> list: + if len(units) <= cap: + return units + prio = [u for u in units if u[0].lower() in PRIORITY_UNITS] + rest = [u for u in units if u[0].lower() not in PRIORITY_UNITS] + return (prio + rest)[:cap] + + +# --------------------------------------------------------------------------- # +# Reference resolution +# --------------------------------------------------------------------------- # + + +class _Resolver: + """Map textual references (paths, ``module:function``) to known node ids.""" + + def __init__(self, code_functions: Iterable[dict], tree_files: set[str], tree_dirs: set[str]): + self.code_ids: set[str] = set() + self.by_path: dict[str, list[str]] = {} + for f in code_functions: + cid = f.get("id") or "" + if not cid: + continue + self.code_ids.add(cid) + if f.get("class_name"): + continue # only top-level definitions represent a file + rel = (f.get("relative_path") or "").replace(os.sep, "/") + if rel: + self.by_path.setdefault(rel, []).append(cid) + self.tree_files = tree_files + self.tree_dirs = tree_dirs + self.artifact_ids: set[str] = set() + self.artifact_files: set[str] = set() + + def known(self, node_id: str) -> bool: + return node_id in self.code_ids or node_id in self.artifact_ids + + def resolve_path(self, ref: str, base_dir: str = "") -> list[str]: + ref = ref.strip().strip("\"'`") + if not ref or any(ch in ref for ch in "*?$[{}<>|"): + return [] + if ref.startswith(("http://", "https://", "/", "..", "~")) or "://" in ref: + return [] + ref = ref[2:] if ref.startswith("./") else ref + cand = posixpath.normpath(posixpath.join(base_dir, ref)) if base_dir else posixpath.normpath(ref) + if cand in (".", "") or cand.startswith("../"): + return [] + if cand in self.tree_dirs: + return [] + if cand in self.by_path: + return self.by_path[cand][:10] + if cand in self.artifact_files: + return [artifact_file_node_id(cand)] + # dist/lib path -> source fallback: unique match on the last two segments + parts = cand.split("/") + if len(parts) >= 2: + tail = "/".join(parts[-2:]) + stem_tail = re.sub(r"\.(js|mjs|cjs)$", "", tail) + hits = [ + p + for p in self.by_path + if p == tail + or p.endswith("/" + tail) + or re.sub(r"\.(ts|tsx)$", "", p).endswith("/" + stem_tail) + ] + if len(hits) == 1: + return self.by_path[hits[0]][:10] + return [] + + def resolve_module(self, module: str, func: str | None = None) -> list[str]: + module = module.strip() + if not module or not re.match(r"^[A-Za-z_][\w.]*$", module): + return [] + mod_path = module.replace(".", "/") + candidates = [f"{mod_path}.py", f"{mod_path}/__init__.py", f"src/{mod_path}.py", f"{mod_path}/__main__.py"] + for p in candidates: + if func: + fid = f"{p}::{func}" + if fid in self.code_ids: + return [fid] + for p in candidates: + if p in self.by_path: + return self.by_path[p][:10] + return [] + + +def _refs_from_shell_text(text: str, resolver: _Resolver, base_dir: str, artifact_units: dict[str, set[str]]) -> set[str]: + """Collect ids referenced by shell-ish text (CI ``run:`` blocks, RUN lines, + Makefile recipes, npm script values).""" + found: set[str] = set() + for m in PATH_TOKEN_RE.finditer(text): + found.update(resolver.resolve_path(m.group(1), base_dir)) + for m in PYTHON_MODULE_RE.finditer(text): + found.update(resolver.resolve_module(m.group(1))) + for m in MAKE_TARGET_RE.finditer(text): + target = m.group(1) + for mk in ("Makefile", "GNUmakefile", "makefile"): + mk_path = posixpath.normpath(posixpath.join(base_dir, mk)) if base_dir else mk + if target in artifact_units.get(mk_path, set()): + found.add(f"{mk_path}::{target}") + for m in NPM_SCRIPT_RE.finditer(text): + script = m.group(1) + if script in _NPM_RESERVED: + continue + pj = posixpath.normpath(posixpath.join(base_dir, "package.json")) if base_dir else "package.json" + if script in artifact_units.get(pj, set()): + found.add(f"{pj}::{script}") + for m in DOCKER_BUILD_FILE_RE.finditer(text): + found.update(resolver.resolve_path(m.group(1), base_dir)) + return found + + +# --------------------------------------------------------------------------- # +# Main entry +# --------------------------------------------------------------------------- # + + +def _walk_tree(tree: dict | None) -> tuple[list[dict], set[str], set[str]]: + files: list[dict] = [] + file_paths: set[str] = set() + dir_paths: set[str] = set() + + def _walk(node: dict | None) -> None: + if not node: + return + if node.get("type") == "file": + path = (node.get("path") or "").replace(os.sep, "/") + files.append({**node, "path": path}) + file_paths.add(path) + elif node.get("type") == "directory": + path = (node.get("path") or "").replace(os.sep, "/") + if path not in ("", "."): + dir_paths.add(path) + for child in node.get("children", []) or []: + _walk(child) + + _walk(tree) + return files, file_paths, dir_paths + + +def _first_line(base: Path, rel_path: str) -> str | None: + try: + text, _, is_binary = safe_read_head(base, base / rel_path, 256) + except (OSError, PermissionError): + return None + if is_binary: + return None + return text.split("\n", 1)[0] + + +def analyze_artifacts( + file_tree: dict | None, + repo_dir: str, + code_functions: list[dict], + opts: ArtifactOptions | None = None, +) -> ArtifactAnalysis: + """Turn artifact files in ``file_tree`` into nodes, unit nodes and edges.""" + opts = opts or ArtifactOptions() + base = Path(repo_dir) + files, tree_files, tree_dirs = _walk_tree(file_tree) + + # 1. classify (no reads except a shebang sniff) + candidates: dict[str, list[dict]] = {cls: [] for cls in CLASS_PRIORITY} + for f in files: + rel = f["path"] + name = f.get("name") or posixpath.basename(rel) + size = int(f.get("_size_bytes") or 0) + first_line = None + if Path(name).suffix == "" and rel.split("/")[0] in {"bin", "scripts"} and "/" in rel: + first_line = _first_line(base, rel) + cls = classify_artifact(rel, name, size, opts, first_line) + if cls: + candidates[cls].append({"path": rel, "name": name, "size": size}) + + # 2. per-class cap + index_classes: dict[str, Any] = {} + selected: list[tuple[str, dict]] = [] + for cls in CLASS_PRIORITY: + items = sorted(candidates[cls], key=lambda d: (d["path"].count("/"), d["path"])) + keep, omitted = items[: opts.per_class_files], items[opts.per_class_files :] + index_classes[cls] = { + "files": [], + "omitted_by_class_cap": [d["path"] for d in omitted[:50]], + "not_loaded_budget": [], + } + selected.extend((cls, d) for d in keep) + + # 3./4. read heads within the budget, in class priority order + resolver = _Resolver(code_functions, tree_files, tree_dirs) + loaded: list[tuple[str, dict, str, bool]] = [] # (cls, info, text, truncated) + tokens_used = 0 + for cls, info in selected: + rel = info["path"] + try: + text, total, is_binary = safe_read_head(base, base / rel, opts.per_file_bytes) + except (OSError, PermissionError) as e: + logger.debug("Skipping artifact %s: %s", rel, e) + continue + if is_binary or not text.strip(): + continue + truncated = total > opts.per_file_bytes + if truncated: + text = text + TRUNCATION_MARKER.format(shown=len(text.encode("utf-8", "replace")), total=total) + n_tokens = count_tokens(text) + if tokens_used + n_tokens > opts.token_budget: + index_classes[cls]["not_loaded_budget"].append(rel) + continue + tokens_used += n_tokens + loaded.append((cls, info, text, truncated)) + resolver.artifact_files.add(rel) + resolver.artifact_ids.add(artifact_file_node_id(rel)) + + # 5. nodes, units and edges + nodes: list[Node] = [] + relationships: list[CallRelationship] = [] + artifact_units: dict[str, set[str]] = {} + unit_specs: list[tuple[str, str, str, str, int, int, dict]] = [] # (cls, rel, unit, text, start_line, end_line, extra) + + def _make_node(rel: str, name: str, cls: str, text: str, node_type: str, start: int, end: int) -> Node: + return Node( + id=f"{rel}::{name}", + name=name, + component_type=ARTIFACT_TYPE, + file_path=str(base / rel), + relative_path=rel, + source_code=text, + start_line=start, + end_line=end, + has_docstring=False, + docstring="", + parameters=[], + node_type=node_type, + display_name=rel if node_type == ARTIFACT_FILE_NODE_TYPE else f"{rel}::{name}", + component_id=f"{rel}::{name}", + artifact_class=cls, + ) + + # First pass: file nodes + unit discovery (so cross-file unit refs resolve). + for cls, info, text, truncated in loaded: + rel, name = info["path"], info["name"] + line_count = text.count("\n") + 1 + nodes.append(_make_node(rel, name, cls, text, ARTIFACT_FILE_NODE_TYPE, 1, line_count)) + lower = name.lower() + units: list[tuple[str, int, int]] = [] + extra: dict[str, Any] = {} + if cls == "ci" and rel.startswith(".github/workflows/"): + units = _yaml_children(text, "jobs") + elif cls == "container" and re.match(r"^(docker-)?compose[.\w-]*\.ya?ml$", lower): + units = _yaml_children(text, "services") + extra["compose"] = True + elif cls == "container" and (fnmatch.fnmatch(name, "Dockerfile*") or lower.endswith(".dockerfile")): + units = _dockerfile_units(text) + extra["dockerfile"] = True + elif lower in {"makefile", "gnumakefile"} or lower.endswith(".mk"): + units = _makefile_units(text) + extra["makefile"] = True + elif name == "package.json": + pj_units, data = _package_json_units(text) + extra["package_json"] = data + for uname, start, end, utext in _prioritise_units(pj_units, opts.per_file_units): + unit_specs.append((cls, rel, uname, utext, _line_of(text, start), _line_of(text, end), {"script": True})) + artifact_units.setdefault(rel, set()).add(uname) + units = [] + elif name == "pyproject.toml" or name == "setup.cfg": + kind = "pyproject" if name == "pyproject.toml" else "setup_cfg" + for uname, start, end, module, func in _prioritise_units(_entry_point_units(text, kind), opts.per_file_units): + snippet = f"{uname} = {module}:{func}" + unit_specs.append((cls, rel, uname, snippet, _line_of(text, start), _line_of(text, end), {"entry": (module, func)})) + artifact_units.setdefault(rel, set()).add(uname) + units = [] + for uname, start, end in _prioritise_units(units, opts.per_file_units): + unit_specs.append((cls, rel, uname, text[start:end].rstrip() + "\n", _line_of(text, start), _line_of(text, max(start, end - 1)), extra)) + artifact_units.setdefault(rel, set()).add(uname) + index_classes[cls]["files"].append( + { + "path": rel, + "bytes": info["size"], + "truncated": truncated, + "loaded": True, + "units": sorted(artifact_units.get(rel, set())), + } + ) + + for cls, rel, uname, utext, s_line, e_line, extra in unit_specs: + uid = f"{rel}::{uname}" + if resolver.known(uid): + continue # unit name collides with the file node name; keep the file node + nodes.append(_make_node(rel, uname, cls, utext, ARTIFACT_UNIT_NODE_TYPE, s_line, e_line)) + resolver.artifact_ids.add(uid) + + edges: set[tuple[str, str]] = set() + + def _add_edges(caller: str, callees: Iterable[str]) -> None: + for callee in callees: + if callee and callee != caller and resolver.known(callee): + edges.add((caller, callee)) + + # Second pass: edges. + for cls, info, text, _ in loaded: + rel, name = info["path"], info["name"] + lower = name.lower() + file_id = artifact_file_node_id(rel) + base_dir = posixpath.dirname(rel) + for uname in artifact_units.get(rel, set()): + _add_edges(file_id, [f"{rel}::{uname}"]) + + if cls == "ci" and rel.startswith(".github/workflows/"): + for uname, start, end in _yaml_children(text, "jobs"): + job_text = text[start:end] + refs: set[str] = set() + for m in re.finditer(r"uses:\s*\./(\S+)", job_text): + for action in ("action.yml", "action.yaml"): + refs.update(resolver.resolve_path(posixpath.join(m.group(1), action))) + wd = re.search(r"working-directory:\s*(\S+)", job_text) + job_base = posixpath.normpath(wd.group(1).strip("\"'")) if wd else "" + if job_base in (".", "/"): + job_base = "" + refs.update(_refs_from_shell_text(job_text, resolver, job_base, artifact_units)) + _add_edges(f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id, refs) + elif cls == "container" and re.match(r"^(docker-)?compose[.\w-]*\.ya?ml$", lower): + for uname, start, end in _yaml_children(text, "services"): + svc = text[start:end] + refs = set() + ctx = re.search(r"^\s+context:\s*(\S+)", svc, re.M) + dfile = re.search(r"^\s+dockerfile:\s*(\S+)", svc, re.M) + build_str = re.search(r"^\s+build:\s*(\S+)\s*$", svc, re.M) + context = (ctx.group(1) if ctx else (build_str.group(1) if build_str else "")).strip("\"'") + dockerfile = (dfile.group(1) if dfile else "Dockerfile").strip("\"'") + if ctx or build_str or dfile: + refs.update(resolver.resolve_path(posixpath.join(context, dockerfile), base_dir)) + _add_edges(f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id, refs) + elif cls == "container" and (fnmatch.fnmatch(name, "Dockerfile*") or lower.endswith(".dockerfile")): + units = _dockerfile_units(text) or [(None, 0, len(text))] + for uname, start, end in units: + stage = text[start:end] + refs = set() + caller = f"{rel}::{uname}" if uname and resolver.known(f"{rel}::{uname}") else file_id + for m in re.finditer(r"^(?:COPY|ADD)\s+(.*)$", stage, re.I | re.M): + args = [a for a in m.group(1).split() if not a.startswith("--")] + if m.group(0).find("--from=") != -1: + alias = re.search(r"--from=(\S+)", m.group(0)).group(1) + _add_edges(caller, [f"{rel}::{alias}"]) + continue + for a in args[:-1]: + refs.update(resolver.resolve_path(a, base_dir)) + for m in re.finditer(r"^(?:ENTRYPOINT|CMD|RUN)\s+(.*)$", stage, re.I | re.M): + refs.update(_refs_from_shell_text(m.group(1), resolver, base_dir, artifact_units)) + _add_edges(caller, refs) + elif lower in {"makefile", "gnumakefile"} or lower.endswith(".mk"): + for m in re.finditer(r"^(?:-?include|sinclude)\s+(\S+)", text, re.M): + _add_edges(file_id, resolver.resolve_path(m.group(1), base_dir)) + for uname, start, end in _makefile_units(text): + recipe = text[start:end] + refs = _refs_from_shell_text(recipe, resolver, base_dir, artifact_units) + # prerequisites on the header line + header = recipe.split("\n", 1)[0] + if ":" in header: + for prereq in header.split(":", 1)[1].split(): + if prereq in artifact_units.get(rel, set()): + refs.add(f"{rel}::{prereq}") + else: + refs.update(resolver.resolve_path(prereq, base_dir)) + _add_edges(f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id, refs) + elif name == "package.json": + _, data = _package_json_units(text) + if isinstance(data, dict): + refs = set() + for key in ("main", "module", "types", "browser"): + if isinstance(data.get(key), str): + refs.update(resolver.resolve_path(data[key], base_dir)) + bin_field = data.get("bin") + for v in ([bin_field] if isinstance(bin_field, str) else list(bin_field.values()) if isinstance(bin_field, dict) else []): + if isinstance(v, str): + refs.update(resolver.resolve_path(v, base_dir)) + + def _walk_exports(val: Any) -> None: + if isinstance(val, str): + refs.update(resolver.resolve_path(val, base_dir)) + elif isinstance(val, dict): + for v in val.values(): + _walk_exports(v) + elif isinstance(val, list): + for v in val: + _walk_exports(v) + + _walk_exports(data.get("exports")) + _add_edges(file_id, refs) + scripts = data.get("scripts") if isinstance(data.get("scripts"), dict) else {} + for sname, sval in scripts.items(): + if not isinstance(sval, str): + continue + caller = f"{rel}::{sname}" if resolver.known(f"{rel}::{sname}") else file_id + _add_edges(caller, _refs_from_shell_text(sval, resolver, base_dir, artifact_units)) + elif name in {"pyproject.toml", "setup.cfg"}: + kind = "pyproject" if name == "pyproject.toml" else "setup_cfg" + for uname, _s, _e, module, func in _entry_point_units(text, kind): + caller = f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id + _add_edges(caller, resolver.resolve_module(module, func)) + elif cls in {"script", "ci", "packaging", "test_infra"}: + _add_edges(file_id, _refs_from_shell_text(text, resolver, base_dir, artifact_units)) + + for caller, callee in sorted(edges): + relationships.append(CallRelationship(caller=caller, callee=callee, is_resolved=True)) + + index = { + "token_budget": opts.token_budget, + "tokens_used": tokens_used, + "caps": { + "per_file_bytes": opts.per_file_bytes, + "per_class_files": opts.per_class_files, + "per_file_units": opts.per_file_units, + }, + "with_prose": opts.with_prose, + "classes": {cls: v for cls, v in index_classes.items() if v["files"] or v["omitted_by_class_cap"] or v["not_loaded_budget"]}, + "nodes": len(nodes), + "edges": len(relationships), + } + logger.info( + "Artifact analysis: %d files, %d nodes, %d edges, %d tokens (budget %d)", + len(loaded), len(nodes), len(relationships), tokens_used, opts.token_budget, + ) + return ArtifactAnalysis(nodes=nodes, relationships=relationships, index=index) + + +# --------------------------------------------------------------------------- # +# Index rendering (from Node objects, so prompts never read the JSON) +# --------------------------------------------------------------------------- # + + +def _human_size(n: int) -> str: + if n < 1024: + return f"{n} B" + if n < 1024 * 1024: + return f"{n / 1024:.1f} KB" + return f"{n / (1024 * 1024):.1f} MB" + + +_UNIT_LABEL = { + "ci": "jobs", + "container": "stages/services", + "build": "targets", + "manifest": "scripts/entry points", +} + + +def build_artifact_index(components: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + """Group artifact file nodes by class with their unit names.""" + units_by_file: dict[str, list[str]] = {} + files: dict[str, Any] = {} + for node in components.values(): + if not is_artifact_node(node): + continue + if is_artifact_file_node(node): + files[node.relative_path] = node + else: + units_by_file.setdefault(node.relative_path, []).append(node.name) + index: dict[str, list[dict[str, Any]]] = {} + for rel, node in sorted(files.items()): + src = node.source_code or "" + index.setdefault(node.artifact_class or "config", []).append( + { + "path": rel, + "id": node.id, + "size": len(src.encode("utf-8", "replace")), + "truncated": "[codewiki: truncated" in src, + "units": sorted(units_by_file.get(rel, [])), + } + ) + return index + + +def render_artifact_index(components: dict[str, Any], max_files_per_class: int = 40) -> str: + """Render the artifact index as a compact text block, or ``""`` if none.""" + index = build_artifact_index(components) + if not index: + return "" + lines = [ + "", + "Build, CI, container, packaging, manifest, config, schema and script files in this " + "repository, grouped by class. Component ids are `::`; read a file with " + "`str_replace_editor view` (working_dir=`repo`).", + ] + for cls in CLASS_PRIORITY: + entries = index.get(cls) + if not entries: + continue + n_trunc = sum(1 for e in entries if e["truncated"]) + header = f"## {cls} ({len(entries)} files)" + if n_trunc: + header += f" [{n_trunc} truncated]" + lines.append(header) + for e in entries[:max_files_per_class]: + line = f"- {e['path']} ({_human_size(e['size'])})" + if e["units"]: + label = _UNIT_LABEL.get(cls, "units") + shown = e["units"][:8] + more = len(e["units"]) - len(shown) + line += f" - {label}: {', '.join(shown)}" + (f", +{more} more" if more > 0 else "") + lines.append(line) + if len(entries) > max_files_per_class: + lines.append(f"- ... {len(entries) - max_files_per_class} more {cls} files") + lines.append("") + return "\n".join(lines) diff --git a/codewiki/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py index a0aaffa6..4554acb9 100644 --- a/codewiki/src/be/dependency_analyzer/ast_parser.py +++ b/codewiki/src/be/dependency_analyzer/ast_parser.py @@ -1,10 +1,14 @@ import json import logging import os +from typing import TYPE_CHECKING from codewiki.src.be.dependency_analyzer.analysis.analysis_service import AnalysisService from codewiki.src.be.dependency_analyzer.models.core import Node +if TYPE_CHECKING: # pragma: no cover + from codewiki.src.be.dependency_analyzer.analyzers.artifact import ArtifactOptions + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -18,6 +22,7 @@ def __init__( include_patterns: list[str] | None = None, exclude_patterns: list[str] | None = None, use_gitignore: bool = True, + artifact_options: "ArtifactOptions | None" = None, ): """ Initialize the dependency parser. @@ -27,7 +32,12 @@ def __init__( include_patterns: File patterns to include (e.g., ["*.cs", "*.py"]) exclude_patterns: File/directory patterns to exclude (e.g., ["*Tests*"]) use_gitignore: Whether to apply Git ignore rules + artifact_options: When given and enabled, also emit ``artifact`` + nodes for build/CI/container/manifest/config files + (``analyzers/artifact.py``). ``None`` keeps the code-only graph. """ + self.artifact_options = artifact_options + self.artifact_index: dict | None = None self.repo_path = os.path.abspath(repo_path) self.components: dict[str, Node] = {} self.modules: set[str] = set() @@ -54,8 +64,9 @@ def parse_repository(self, filtered_folders: list[str] | None = None) -> dict[st ) call_graph_result = self.analysis_service._analyze_call_graph( - structure_result["file_tree"], self.repo_path + structure_result["file_tree"], self.repo_path, artifact_options=self.artifact_options ) + self.artifact_index = call_graph_result.get("artifact_index") self._build_components_from_analysis(call_graph_result) @@ -92,6 +103,7 @@ def _build_components_from_analysis(self, call_graph_result: dict): class_name=func_dict.get("class_name"), display_name=func_dict.get("display_name", ""), component_id=component_id, + artifact_class=func_dict.get("artifact_class"), ) self.components[component_id] = node diff --git a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py index a1186e6e..5e39085f 100644 --- a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py +++ b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py @@ -2,6 +2,7 @@ import os from codewiki.src.config import Config from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser +from codewiki.src.be.dependency_analyzer.analyzers.artifact import ArtifactOptions from codewiki.src.be.dependency_analyzer.topo_sort import build_graph_from_components, get_leaf_nodes from codewiki.src.be.dependency_analyzer.leaf_selection import compute_valid_leaf_types, filter_leaf_nodes from codewiki.src.utils import file_manager @@ -42,11 +43,19 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: include_patterns = self.config.include_patterns if self.config.include_patterns else None exclude_patterns = self.config.exclude_patterns if self.config.exclude_patterns else None + artifact_options = ArtifactOptions( + enabled=getattr(self.config, "artifacts_enabled", True), + token_budget=getattr(self.config, "artifact_token_budget", 200_000), + with_prose=getattr(self.config, "with_prose", False), + exclude_patterns=list(getattr(self.config, "artifact_exclude", None) or []), + ) + parser = DependencyParser( self.config.repo_path, include_patterns=include_patterns, exclude_patterns=exclude_patterns, use_gitignore=self.config.use_gitignore, + artifact_options=artifact_options, ) filtered_folders = None @@ -64,6 +73,23 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: # Save dependency graph parser.save_dependency_graph(dependency_graph_path) + + # Save the artifact index next to the graph (/temp/artifact_index.json) + if artifact_options.enabled: + if parser.artifact_index is not None: + file_manager.save_json( + parser.artifact_index, + os.path.join(self.config.output_dir, "artifact_index.json"), + ) + n_artifacts = sum(1 for c in components.values() if c.component_type == "artifact") + if n_artifacts == 0: + logger.warning( + "Artifact analysis is enabled but found no artifact files. " + "If you passed --include, add artifact names (Dockerfile, Makefile, " + "*.yml, pyproject.toml, ...) to the include patterns." + ) + else: + logger.info("Artifact nodes in dependency graph: %d", n_artifacts) # Build graph for traversal graph = build_graph_from_components(components) diff --git a/codewiki/src/be/dependency_analyzer/leaf_selection.py b/codewiki/src/be/dependency_analyzer/leaf_selection.py index cd972d3e..3218441f 100644 --- a/codewiki/src/be/dependency_analyzer/leaf_selection.py +++ b/codewiki/src/be/dependency_analyzer/leaf_selection.py @@ -14,6 +14,7 @@ LEAF_REDUCTION_THRESHOLD = 400 OOP_TYPES = {"class", "interface", "struct"} +ARTIFACT_TYPE = "artifact" def compute_valid_leaf_types(components: dict[str, Node]) -> set[str]: @@ -34,6 +35,10 @@ def compute_valid_leaf_types(components: dict[str, Node]) -> set[str]: n_func += 1 valid_types = set(OOP_TYPES) + # Artifact nodes (build, CI, container, manifest, config files) are always + # leaf candidates: nothing in the code graph depends on them, and they are + # the only route to documenting how the system is built and shipped. + valid_types.add(ARTIFACT_TYPE) include_functions = ( n_oop == 0 or (n_oop < MIN_OOP_COMPONENTS and n_func > n_oop) diff --git a/codewiki/src/be/dependency_analyzer/models/core.py b/codewiki/src/be/dependency_analyzer/models/core.py index 6174e057..8a4f8382 100644 --- a/codewiki/src/be/dependency_analyzer/models/core.py +++ b/codewiki/src/be/dependency_analyzer/models/core.py @@ -43,6 +43,10 @@ class Node(BaseModel): qualified_name: Optional[str] = None + # Set only on artifact nodes (component_type == "artifact"): one of the + # classes in analyzers/artifact.py CLASS_PRIORITY (build, ci, container, ...). + artifact_class: Optional[str] = None + def get_display_name(self) -> str: return self.display_name or self.name diff --git a/codewiki/src/be/dependency_analyzer/topo_sort.py b/codewiki/src/be/dependency_analyzer/topo_sort.py index 633b5e38..ed7ce9b3 100644 --- a/codewiki/src/be/dependency_analyzer/topo_sort.py +++ b/codewiki/src/be/dependency_analyzer/topo_sort.py @@ -321,8 +321,14 @@ def concise_node(leaf_nodes: set[str]) -> set[str]: count_before, LEAF_REDUCTION_THRESHOLD, ) - # Remove nodes that are dependencies of other nodes - for deps in acyclic_graph.values(): + # Remove nodes that are dependencies of other nodes. Edges that start + # at an artifact node (a Dockerfile COPY, a CI `run:` line, a manifest + # entry point) are references, not calls: they must not demote the + # code component they point at. + for node, deps in acyclic_graph.items(): + owner = components.get(node) + if owner is not None and owner.component_type == "artifact": + continue for dep in deps: leaf_nodes.discard(dep) diff --git a/codewiki/src/be/dependency_analyzer/utils/patterns.py b/codewiki/src/be/dependency_analyzer/utils/patterns.py index 32472dd0..c7c3f0b0 100644 --- a/codewiki/src/be/dependency_analyzer/utils/patterns.py +++ b/codewiki/src/be/dependency_analyzer/utils/patterns.py @@ -177,8 +177,93 @@ "*.toml", "*.cfg", "*.ini", + # Artifacts (build, container, CI, packaging, manifest, config, schema, + # script files). These reach the file tree so analyzers/artifact.py can + # turn them into `artifact` nodes; the language analyzers ignore them. + "Dockerfile*", + "*.dockerfile", + "docker-compose*", + "compose.yml", + "compose.yaml", + "Makefile", + "GNUmakefile", + "makefile", + "*.mk", + "CMakeLists.txt", + "*.cmake", + "*.gradle", + "*.gn", + "*.gni", + "DEPS", + "BUILD", + "BUILD.*", + "WORKSPACE", + "*.bzl", + "*.rake", + "Rakefile", + "Gemfile", + "*.gemspec", + "Jenkinsfile", + "*.sh", + "*.bash", + "*.ps1", + "*.spec", + "*.service", + "*.conf", + "*.options", + "*.properties", + "*.proto", + "*.fbs", + "*.avsc", + "*.thrift", + "*.graphql", + "*.tf", + "Procfile", + "go.mod", + ".editorconfig", + ".pre-commit-config.yaml", + ".eslintrc*", + ".prettierrc*", + ".nvmrc", ] +# Paths that survive DEFAULT_IGNORE_PATTERNS (but never user --exclude or +# .gitignore) because they are artifacts the documentation should cover. +# Matched with fnmatch against the repo-relative path and the basename; a +# directory is kept when some pattern lives underneath it (e.g. `.github` +# for `.github/workflows/*`). +ARTIFACT_WHITELIST = [ + ".github/workflows/*", + ".github/actions/*", + ".github/actions/*/*", + ".github/dependabot.yml", + "*.ini", + "bin/*.sh", + "bin/*.bash", + "*.gradle", + "settings.gradle*", + "gradle.properties", +] + +# Lock files are never read as artifacts: huge, generated, no design content. +ARTIFACT_LOCKFILES = { + "package-lock.json", + "npm-shrinkwrap.json", + "yarn.lock", + "pnpm-lock.yaml", + "poetry.lock", + "Pipfile.lock", + "Cargo.lock", + "go.sum", + "Gemfile.lock", + "composer.lock", + "uv.lock", + "bun.lock", + "bun.lockb", + "flake.lock", + "pdm.lock", +} + CODE_EXTENSIONS = { ".py": "python", ".js": "javascript", diff --git a/codewiki/src/be/dependency_analyzer/utils/security.py b/codewiki/src/be/dependency_analyzer/utils/security.py index 724421e6..b8a1a95f 100644 --- a/codewiki/src/be/dependency_analyzer/utils/security.py +++ b/codewiki/src/be/dependency_analyzer/utils/security.py @@ -31,3 +31,28 @@ def safe_open_text(base_dir: Path, target: Path, encoding="utf-8"): os.close(fd) except OSError: pass + + +def safe_read_head(base_dir: Path, target: Path, max_bytes: int, encoding="utf-8") -> tuple[str, int, bool]: + """Read at most ``max_bytes`` of ``target`` with the same symlink/escape checks + as :func:`safe_open_text`. + + Returns ``(text, total_size, is_binary)``. ``is_binary`` is True when the + head contains a NUL byte; callers should then drop the file. + """ + assert_safe_path(base_dir, target) + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(str(target), flags) + try: + total_size = os.fstat(fd).st_size + data = os.read(fd, max(0, int(max_bytes))) + finally: + try: + os.close(fd) + except OSError: + pass + if b"\0" in data: + return "", total_size, True + return data.decode(encoding, errors="replace"), total_size, False diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index b93f3516..4790c6c6 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -12,9 +12,11 @@ from codewiki.src.be.backend import LLMBackend, get_backend from codewiki.src.be.cluster_modules import ( cluster_modules, + ensure_artifact_module, get_clustering_input_token_count, super_group_modules, ) +from codewiki.src.be.dependency_analyzer.analyzers.artifact import render_artifact_index from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder from codewiki.src.be.module_naming import ( dedupe_module_tree_names, @@ -23,6 +25,7 @@ ) from codewiki.src.be.prompt_template import ( MODULE_OVERVIEW_PROMPT, + REPO_OVERVIEW_ARTIFACT_ADDENDUM, REPO_OVERVIEW_PROMPT, ) from codewiki.src.config import ( @@ -269,7 +272,9 @@ async def generate_module_documentation( # Generate repo overview logger.info("📚 Generating repository overview") - final_module_tree = await self.generate_parent_module_docs([], working_dir) + final_module_tree = await self.generate_parent_module_docs( + [], working_dir, components=components + ) else: logger.info("Processing whole repo because repo can fit in the context window") repo_name = os.path.basename(os.path.normpath(self.config.repo_path)) @@ -294,9 +299,16 @@ async def generate_module_documentation( return working_dir async def generate_parent_module_docs( - self, module_path: list[str], working_dir: str + self, + module_path: list[str], + working_dir: str, + components: dict[str, Any] | None = None, ) -> dict[str, Any]: - """Generate documentation for a parent module based on its children's documentation.""" + """Generate documentation for a parent module based on its children's documentation. + + For the repository overview (``module_path == []``) pass ``components`` + so the artifact index can be appended to the prompt. + """ module_name = ( module_path[-1] if len(module_path) >= 1 @@ -333,6 +345,10 @@ async def generate_parent_module_docs( repo_name=module_name, repo_structure=json.dumps(repo_structure, indent=2) ) ) + if len(module_path) == 0 and components: + artifact_index = render_artifact_index(components) + if artifact_index: + prompt += "\n\n" + REPO_OVERVIEW_ARTIFACT_ADDENDUM.format(artifact_index=artifact_index) logger.debug(f"Overview prompt for {module_name}: {len(prompt)} chars") try: @@ -417,6 +433,10 @@ async def run(self) -> None: self.config, completer=lambda p: self.backend.complete(p, model=cluster_model), ) + # Artifact nodes the clustering LLM dropped get a fixed module + # so build/CI/config coverage does not depend on the LLM. + if getattr(self.config, "artifacts_enabled", True): + module_tree = ensure_artifact_module(module_tree, leaf_nodes, components) # Only freshly clustered trees are deduped: renaming a cached # key whose .md already exists would orphan the doc. module_tree = dedupe_module_tree_names(module_tree) diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py index 10f70bcb..b0fb88a7 100644 --- a/codewiki/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -38,7 +38,7 @@ -- `str_replace_editor`: File system operations for creating and editing documentation files +- `str_replace_editor`: File system operations for creating and editing documentation files, and (with `working_dir="repo"`, `view` only) for reading build, CI, container, packaging, manifest and config files such as Dockerfile, Makefile, .github/workflows/*.yml, pyproject.toml or package.json - `read_code_components`: Explore additional code dependencies not included in the provided components - `generate_sub_module_documentation`: Generate detailed documentation for individual sub-modules via sub-agents @@ -71,7 +71,7 @@ -- `str_replace_editor`: File system operations for creating and editing documentation files +- `str_replace_editor`: File system operations for creating and editing documentation files, and (with `working_dir="repo"`, `view` only) for reading build, CI, container, packaging, manifest and config files such as Dockerfile, Makefile, .github/workflows/*.yml, pyproject.toml or package.json - `read_code_components`: Explore additional code dependencies not included in the provided components {custom_instructions} @@ -140,6 +140,8 @@ Please group the components into groups such that each group is a set of components that are closely related to each other and together they form a module. DO NOT include components that are not essential to the repository. +Files marked `(artifact: )` are build, CI, container, packaging, manifest, configuration, schema or script files. Their components describe how the system is built, packaged, shipped, configured and tested. They ARE essential: group them into a dedicated build/deployment/configuration module, or attach them to the module they configure. Never drop them. + Each component ID has the form `::`. Return the IDs EXACTLY as given — do NOT strip the `::` prefix or shorten the ID to the bare name. Firstly reason about the components and then group them and return the result in the following format: @@ -180,6 +182,8 @@ Please group the components into groups such that each group is a set of components that are closely related to each other and together they form a smaller module. DO NOT include components that are not essential to the module. +Files marked `(artifact: )` are build, CI, container, packaging, manifest, configuration, schema or script files. Their components describe how the system is built, packaged, shipped, configured and tested. They ARE essential: group them into a dedicated build/deployment/configuration module, or attach them to the module they configure. Never drop them. + Each component ID has the form `::`. Return the IDs EXACTLY as given — do NOT strip the `::` prefix or shorten the ID to the bare name. Firstly reason based on given context and then group them and return the result in the following format: @@ -283,12 +287,53 @@ "file-reading tools to read the full files]" ) +# Appended to the user prompt (after USER_PROMPT) when the dependency graph +# contains artifact nodes. Kept out of USER_PROMPT itself so callers that +# format the template directly (MCP prompt server) keep working. +ARTIFACT_USAGE_NOTE = ( + "* NOTE: when this module's behaviour depends on how the system is built, " + "configured, packaged, deployed or tested, read the relevant artifact file " + "with `str_replace_editor` (`command=\"view\"`, `working_dir=\"repo\"`, path as " + "listed above) and cite the file path in the documentation." +) + +REPO_OVERVIEW_ARTIFACT_ADDENDUM = """ +The repository also contains the following build, CI, container, packaging, manifest and configuration artifacts: +{artifact_index} + +Include a short section titled "How it is built and run" that summarises how the project is built, tested, packaged and deployed, and links to the module documentation that covers these artifacts (for example a `Build, Deployment and Configuration` module) instead of repeating its content. +""".strip() + EXTENSION_TO_LANGUAGE = { ".py": "python", ".md": "markdown", ".sh": "bash", + ".bash": "bash", ".json": "json", ".yaml": "yaml", + ".yml": "yaml", + ".toml": "toml", + ".ini": "ini", + ".cfg": "ini", + ".conf": "text", + ".mk": "makefile", + ".gradle": "groovy", + ".proto": "protobuf", + ".gn": "text", + ".gni": "text", + ".rake": "ruby", + ".ps1": "powershell", + ".xml": "xml", + ".html": "html", + ".css": "css", + # extension-less artifact files are looked up by basename + "Dockerfile": "dockerfile", + "Containerfile": "dockerfile", + "Makefile": "makefile", + "GNUmakefile": "makefile", + "Jenkinsfile": "groovy", + "Rakefile": "ruby", + "Gemfile": "ruby", ".java": "java", ".js": "javascript", ".ts": "typescript", @@ -360,6 +405,40 @@ def _walk(tree: dict[str, Any], indent: int = 0) -> None: return "\n".join(lines) +def _fence_language(path: str) -> str: + """Markdown fence language for ``path`` (falls back to ``text``).""" + base = path.replace("\\", "/").rsplit("/", 1)[-1] + if base in EXTENSION_TO_LANGUAGE: + return EXTENSION_TO_LANGUAGE[base] + if "." in base: + ext = "." + base.rsplit(".", 1)[-1].lower() + if ext in EXTENSION_TO_LANGUAGE: + return EXTENSION_TO_LANGUAGE[ext] + stem = base.split(".", 1)[0] # Dockerfile.dev -> Dockerfile + if stem in EXTENSION_TO_LANGUAGE: + return EXTENSION_TO_LANGUAGE[stem] + return "text" + + +def _artifact_group_source(component_ids: list[str], components: dict[str, Any]) -> str | None: + """Return the capped artifact text for a file group made only of artifact + nodes, or ``None`` when the group contains code components.""" + nodes = [components[c] for c in component_ids if c in components] + if not nodes or any(getattr(n, "component_type", None) != "artifact" for n in nodes): + return None + file_nodes = [n for n in nodes if getattr(n, "node_type", None) == "artifact_file"] + if file_nodes: + return file_nodes[0].source_code or "" + # Only unit nodes were selected: the file node carries the full head, use + # it when present in the graph, otherwise join the unit slices. + rel = nodes[0].relative_path + file_id = f"{rel}::{rel.replace(chr(92), '/').rsplit('/', 1)[-1]}" + file_node = components.get(file_id) + if file_node is not None and file_node.source_code: + return file_node.source_code + return "\n\n".join((n.source_code or "") for n in nodes) + + def format_user_prompt( module_name: str, core_component_ids: list[str], @@ -377,6 +456,8 @@ def format_user_prompt( Returns: Formatted user prompt string """ + from codewiki.src.be.dependency_analyzer.analyzers.artifact import render_artifact_index + formatted_module_tree = _format_module_tree_str(module_tree, module_name) # Group core component IDs by their file path @@ -398,25 +479,40 @@ def format_user_prompt( for component_id in component_ids_in_file: core_component_codes += f"- {component_id}\n" - core_component_codes += ( - f"\n## File Content:\n```{EXTENSION_TO_LANGUAGE['.' + path.split('.')[-1]]}\n" - ) + core_component_codes += f"\n## File Content:\n```{_fence_language(path)}\n" + + artifact_source = _artifact_group_source(component_ids_in_file, components) + if artifact_source is not None: + # Artifact files are inlined from their capped head, never re-read + # in full (a 150 KB YAML must not blow up the prompt). + core_component_codes += artifact_source + else: + # Read content of the file using the first component's file path + try: + core_component_codes += file_manager.load_text( + components[component_ids_in_file[0]].file_path + ) + except (OSError, FileNotFoundError) as e: + core_component_codes += f"# Error reading file: {e}\n" + + core_component_codes += "\n```\n\n" + + artifact_index = render_artifact_index(components) + artifact_section = ( + f"\n\n{artifact_index}\n{ARTIFACT_USAGE_NOTE}" if artifact_index else "" + ) - # Read content of the file using the first component's file path - try: - core_component_codes += file_manager.load_text( - components[component_ids_in_file[0]].file_path + def _assemble(codes: str, tree: str) -> str: + return ( + USER_PROMPT.format( + module_name=module_name, + formatted_core_component_codes=codes, + module_tree=tree, ) - except (OSError, FileNotFoundError) as e: - core_component_codes += f"# Error reading file: {e}\n" - - core_component_codes += "```\n\n" + + artifact_section + ) - prompt = USER_PROMPT.format( - module_name=module_name, - formatted_core_component_codes=core_component_codes, - module_tree=formatted_module_tree, - ) + prompt = _assemble(core_component_codes, formatted_module_tree) if len(prompt) > MAX_USER_PROMPT_CHARS: full_len = len(prompt) @@ -425,11 +521,7 @@ def format_user_prompt( + "\n\n" + _format_module_tree_str(module_tree, module_name, include_components=False) ) - prompt = USER_PROMPT.format( - module_name=module_name, - formatted_core_component_codes=core_component_codes, - module_tree=formatted_module_tree, - ) + prompt = _assemble(core_component_codes, formatted_module_tree) logger.warning( "Module %s: user prompt (%d chars) exceeds %d; " "module tree trimmed to names only (%d chars)", @@ -447,11 +539,7 @@ def format_user_prompt( core_component_codes = ( core_component_codes[: max(0, len(core_component_codes) - excess)] + CODE_TRUNCATED_NOTE ) - prompt = USER_PROMPT.format( - module_name=module_name, - formatted_core_component_codes=core_component_codes, - module_tree=formatted_module_tree, - ) + prompt = _assemble(core_component_codes, formatted_module_tree) logger.warning( "Module %s: user prompt still over %d chars after tree trim; " "truncated inlined file contents (now %d chars)", diff --git a/codewiki/src/config.py b/codewiki/src/config.py index 97bb9dcf..c20848ce 100644 --- a/codewiki/src/config.py +++ b/codewiki/src/config.py @@ -26,6 +26,9 @@ # this many leaf nodes (and further bounded by an output-token budget derived # from max_tokens). DEFAULT_MAX_LEAF_NODES_PER_CLUSTER = 600 +# Artifact-aware generation: total token budget for build/CI/container/ +# manifest/config file contents added to the dependency graph. +DEFAULT_ARTIFACT_TOKEN_BUDGET = 200_000 # Legacy constants (for backward compatibility) MAX_TOKEN_PER_MODULE = DEFAULT_MAX_TOKEN_PER_MODULE MAX_TOKEN_PER_LEAF_MODULE = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE @@ -87,7 +90,21 @@ class Config: agent_instructions: Optional[Dict[str, Any]] = None # Apply Git ignore rules before dependency analysis use_gitignore: bool = True + # Artifact-aware generation (Dockerfiles, CI workflows, Makefiles, + # manifests, config, schemas, scripts become `artifact` graph nodes) + artifacts_enabled: bool = True + artifact_token_budget: int = DEFAULT_ARTIFACT_TOKEN_BUDGET + # Also read the root README and docs/ as a `prose` artifact class (off by + # default: documentation without existing prose is the benchmark setting) + with_prose: bool = False + @property + def artifact_exclude(self) -> Optional[List[str]]: + """Extra patterns excluded from artifact analysis (from agent instructions).""" + if self.agent_instructions: + return self.agent_instructions.get('artifact_exclude') + return None + @property def include_patterns(self) -> Optional[List[str]]: """Get file include patterns from agent instructions.""" @@ -193,6 +210,9 @@ def from_cli( agent_instructions: Optional[Dict[str, Any]] = None, use_gitignore: bool = True, prompt_caching: bool = True, + artifacts_enabled: bool = True, + artifact_token_budget: int = DEFAULT_ARTIFACT_TOKEN_BUDGET, + with_prose: bool = False, ) -> 'Config': """ Create configuration for CLI context. @@ -221,6 +241,10 @@ def from_cli( agent_instructions: Custom agent instructions dict use_gitignore: Whether to apply Git ignore rules prompt_caching: Whether to add prompt-cache breakpoints to agentic calls + artifacts_enabled: Add build/CI/container/manifest/config files to + the dependency graph and document them + artifact_token_budget: Total token budget for artifact file contents + with_prose: Also read README and docs/ as a `prose` artifact class Returns: Config instance @@ -251,4 +275,7 @@ def from_cli( agent_instructions=agent_instructions, use_gitignore=use_gitignore, prompt_caching=prompt_caching, + artifacts_enabled=artifacts_enabled, + artifact_token_budget=artifact_token_budget, + with_prose=with_prose, ) diff --git a/tests/test_artifact_analyzer.py b/tests/test_artifact_analyzer.py new file mode 100644 index 00000000..0019a084 --- /dev/null +++ b/tests/test_artifact_analyzer.py @@ -0,0 +1,414 @@ +"""Tests for artifact-aware generation. + +Build, CI, container, packaging, manifest, config, schema and script files +("artifacts") used to be dropped before any Node was created, so CodeWiki +never documented how a system is built and shipped. These tests cover the +classifier, the file-walk whitelist, node/unit/edge emission, caps, leaf +selection, the user prompt and the guaranteed fallback module. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from codewiki.src.be.cluster_modules import ( + ARTIFACT_MODULE_NAME, + ensure_artifact_module, + format_potential_core_components, +) +from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer +from codewiki.src.be.dependency_analyzer.analyzers.artifact import ( + ArtifactOptions, + TRUNCATION_MARKER, + classify_artifact, + render_artifact_index, +) +from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser +from codewiki.src.be.dependency_analyzer.leaf_selection import compute_valid_leaf_types +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.dependency_analyzer.topo_sort import build_graph_from_components, get_leaf_nodes +from codewiki.src.be.prompt_template import USER_PROMPT, format_user_prompt + + +# --------------------------------------------------------------------------- # +# fixture +# --------------------------------------------------------------------------- # + + +def _write(root: Path, rel: str, text: str) -> None: + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +@pytest.fixture +def mini_repo(tmp_path: Path) -> Path: + _write(tmp_path, "pkg/__init__.py", "") + _write(tmp_path, "pkg/cli.py", "def main():\n return 0\n\n\ndef helper():\n return 1\n") + _write(tmp_path, "pkg/core.py", "class Engine:\n def run(self):\n return 1\n") + _write( + tmp_path, + "pyproject.toml", + '[project]\nname = "mini"\nversion = "0.1"\n\n[project.scripts]\nmytool = "pkg.cli:main"\n', + ) + _write( + tmp_path, + "package.json", + '{\n "name": "mini",\n "main": "pkg/index.js",\n "scripts": {\n "build": "node scripts/build.js",\n "test": "npm run build && node test.js"\n }\n}\n', + ) + _write(tmp_path, "scripts/build.js", "function build() { return 1; }\nmodule.exports = { build };\n") + _write( + tmp_path, + "Makefile", + "VAR := x\n\n.PHONY: build test\n\nbuild:\n\tpython -m pkg.cli\n\ntest: build\n\tpython pkg/cli.py\n\n%.o: %.c\n\t$(CC) -c $<\n", + ) + _write( + tmp_path, + "Dockerfile", + "FROM python:3.12 AS builder\nCOPY pkg/cli.py /app/cli.py\nRUN make build\n\nFROM python:3.12-slim\nCOPY --from=builder /app /app\nENTRYPOINT [\"python\", \"pkg/cli.py\"]\n", + ) + _write( + tmp_path, + ".github/workflows/ci.yml", + "name: CI\non: [push]\njobs:\n lint:\n runs-on: ubuntu-latest\n steps:\n - run: ruff check .\n test:\n runs-on: ubuntu-latest\n steps:\n - run: make test\n", + ) + _write(tmp_path, ".github/ISSUE_TEMPLATE/bug.md", "# bug\n") + _write(tmp_path, "README.md", "# mini\n") + _write(tmp_path, "docs/guide.md", "# guide\n") + _write(tmp_path, "config/big.yaml", "key: value\n" * 3000) # > 16 KB + _write(tmp_path, "conftest.py", "import pytest\n") + _write(tmp_path, "tests/conftest.py", "import pytest\n") + _write(tmp_path, "package-lock.json", '{"lockfileVersion": 3}\n') + _write(tmp_path, "pytest.ini", "[pytest]\naddopts = -q\n") + return tmp_path + + +def _artifact_nodes(components: dict[str, Node]) -> dict[str, Node]: + return {k: v for k, v in components.items() if v.component_type == "artifact"} + + +# --------------------------------------------------------------------------- # +# 1. classifier +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "rel_path, expected", + [ + (".github/workflows/ci.yml", "ci"), + (".gitlab-ci.yml", "ci"), + ("Dockerfile", "container"), + ("docker/Dockerfile.erb", "container"), + ("docker-compose.yml", "container"), + ("docker/entrypoint.sh", "container"), + ("package.json", "manifest"), + ("packages/foo/package.json", "manifest"), + ("pyproject.toml", "manifest"), + ("setup.cfg", "manifest"), + ("requirements-dev.txt", "manifest"), + ("packages/deb/control", "packaging"), + ("pkg/logstash.service", "packaging"), + ("Makefile", "build"), + ("BUILD.gn", "build"), + ("rakelib/artifacts.rake", "build"), + ("pytest.ini", "test_infra"), + ("conftest.py", "test_infra"), + ("proto/msg.proto", "schema"), + ("config/jvm.options", "config"), + ("ruff.toml", "config"), + ("deep/nested/dir/settings.yml", None), + ("bin/run.sh", "script"), + ("README.md", None), + ("docs/guide.md", None), + ("package-lock.json", None), + (".github/ISSUE_TEMPLATE/bug.md", None), + ("CODEOWNERS", None), + ("src/main.py", None), + ], +) +def test_classify_artifact_table(rel_path: str, expected: str | None) -> None: + name = rel_path.rsplit("/", 1)[-1] + assert classify_artifact(rel_path, name, 100, ArtifactOptions()) == expected + + +def test_classify_prose_and_exclude() -> None: + prose = ArtifactOptions(with_prose=True) + assert classify_artifact("README.md", "README.md", 10, prose) == "prose" + assert classify_artifact("docs/guide.md", "guide.md", 10, prose) == "prose" + assert classify_artifact("pkg/notes.md", "notes.md", 10, prose) is None + excl = ArtifactOptions(exclude_patterns=["docker/data/*"]) + assert classify_artifact("docker/data/huge.yml", "huge.yml", 10, excl) is None + assert classify_artifact("docker/data/huge.yml", "huge.yml", 10, ArtifactOptions()) == "container" + assert classify_artifact("Dockerfile", "Dockerfile", 0, ArtifactOptions()) is None + + +# --------------------------------------------------------------------------- # +# 2. file walk whitelist +# --------------------------------------------------------------------------- # + + +def _tree_paths(tree: dict) -> set[str]: + out: set[str] = set() + + def _walk(node: dict | None) -> None: + if not node: + return + if node["type"] == "file": + out.add(node["path"].replace(os.sep, "/")) + for child in node.get("children", []) or []: + _walk(child) + + _walk(tree) + return out + + +def test_repo_analyzer_whitelist(mini_repo: Path) -> None: + paths = _tree_paths(RepoAnalyzer(use_gitignore=False).analyze_repository_structure(str(mini_repo))["file_tree"]) + assert {".github/workflows/ci.yml", "pytest.ini", "Dockerfile", "Makefile", "pyproject.toml"} <= paths + assert "tests/conftest.py" not in paths + assert ".github/ISSUE_TEMPLATE/bug.md" not in paths + # user excludes still win over the whitelist + paths_user = _tree_paths( + RepoAnalyzer(exclude_patterns=[".github"], use_gitignore=False) + .analyze_repository_structure(str(mini_repo))["file_tree"] + ) + assert ".github/workflows/ci.yml" not in paths_user + + +# --------------------------------------------------------------------------- # +# 3. nodes and units +# --------------------------------------------------------------------------- # + + +def test_parse_repository_emits_artifact_nodes_and_units(mini_repo: Path) -> None: + components = DependencyParser( + str(mini_repo), use_gitignore=False, artifact_options=ArtifactOptions() + ).parse_repository() + artifacts = _artifact_nodes(components) + ids = set(artifacts) + expected = { + "Dockerfile::Dockerfile", + "Dockerfile::builder", + "Makefile::Makefile", + "Makefile::build", + "Makefile::test", + "package.json::package.json", + "package.json::build", + "pyproject.toml::mytool", + ".github/workflows/ci.yml::ci.yml", + ".github/workflows/ci.yml::lint", + ".github/workflows/ci.yml::test", + "pytest.ini::pytest.ini", + "conftest.py::conftest.py", + } + missing = expected - ids + assert not missing, f"missing artifact ids: {sorted(missing)}" + assert "Makefile::.PHONY" not in ids and "Makefile::VAR" not in ids + assert not any(i.startswith(("README", "docs/")) for i in ids) + assert artifacts["Dockerfile::Dockerfile"].artifact_class == "container" + assert artifacts["Makefile::build"].artifact_class == "build" + assert artifacts[".github/workflows/ci.yml::lint"].artifact_class == "ci" + assert artifacts["Makefile::test"].source_code.startswith("test: build\n\tpython pkg/cli.py") + # code side is untouched + assert "pkg/cli.py::main" in components and components["pkg/cli.py::main"].component_type == "function" + # a parser without options keeps the code-only graph + plain = DependencyParser(str(mini_repo), use_gitignore=False).parse_repository() + assert not _artifact_nodes(plain) + + +# --------------------------------------------------------------------------- # +# 4. edges +# --------------------------------------------------------------------------- # + + +def test_edges_resolve_only_to_known_ids(mini_repo: Path) -> None: + components = DependencyParser( + str(mini_repo), use_gitignore=False, artifact_options=ArtifactOptions() + ).parse_repository() + assert "pkg/cli.py::main" in components["pyproject.toml::mytool"].depends_on + assert "pkg/cli.py::main" in components["Makefile::build"].depends_on # python -m pkg.cli + assert "pkg/cli.py::main" in components["Makefile::test"].depends_on # pkg/cli.py path + assert "Makefile::build" in components["Makefile::test"].depends_on # prerequisite + assert "Makefile::test" in components[".github/workflows/ci.yml::test"].depends_on + assert "Makefile::build" in components["Dockerfile::builder"].depends_on # RUN make build + assert "pkg/cli.py::main" in components["Dockerfile::builder"].depends_on # COPY pkg/cli.py + assert "Dockerfile::builder" in components["Dockerfile::stage_1"].depends_on # --from=builder + assert "scripts/build.js::build" in components["package.json::build"].depends_on + assert "package.json::build" in components["package.json::test"].depends_on # npm run build + # file nodes point at their units + assert "Makefile::build" in components["Makefile::Makefile"].depends_on + # every edge target exists, and no code node depends on an artifact + for node in components.values(): + for dep in node.depends_on: + assert dep in components, (node.id, dep) + if node.component_type != "artifact": + assert components[dep].component_type != "artifact", (node.id, dep) + + +# --------------------------------------------------------------------------- # +# 5. caps +# --------------------------------------------------------------------------- # + + +def test_caps(mini_repo: Path) -> None: + for i in range(45): + _write(mini_repo, f"config/c{i:02d}.yaml", f"n: {i}\n") + parser = DependencyParser(str(mini_repo), use_gitignore=False, artifact_options=ArtifactOptions()) + components = parser.parse_repository() + big = components["config/big.yaml::big.yaml"] + marker_prefix = TRUNCATION_MARKER.split("{")[0] + assert marker_prefix in big.source_code + assert len(big.source_code) < 16_384 + len(TRUNCATION_MARKER) + 32 + config_files = [n for n in _artifact_nodes(components).values() if n.artifact_class == "config" and n.node_type == "artifact_file"] + assert len(config_files) == 40 + assert len(parser.artifact_index["classes"]["config"]["omitted_by_class_cap"]) == 6 # 46 config files - 40 + # a tiny budget keeps manifests (highest priority) and records what was skipped + tight = DependencyParser( + str(mini_repo), use_gitignore=False, artifact_options=ArtifactOptions(token_budget=120) + ) + tight_components = tight.parse_repository() + tight_artifacts = _artifact_nodes(tight_components) + # manifests are loaded first; the big config file no longer fits (small + # later files may still slip into the leftover budget, by design) + assert any(n.artifact_class == "manifest" for n in tight_artifacts.values()) + assert "config/big.yaml::big.yaml" not in tight_artifacts + assert "config/big.yaml" in tight.artifact_index["classes"]["config"]["not_loaded_budget"] + assert tight.artifact_index["tokens_used"] <= 120 + + +# --------------------------------------------------------------------------- # +# 6. leaf selection +# --------------------------------------------------------------------------- # + + +def _node(node_id: str, component_type: str, deps: set[str] | None = None) -> Node: + path = node_id.split("::")[0] + return Node( + id=node_id, + name=node_id.split("::")[-1], + component_type=component_type, + file_path=path, + relative_path=path, + depends_on=set(deps or set()), + source_code="x", + ) + + +def test_leaf_types_and_pruning() -> None: + assert "artifact" in compute_valid_leaf_types({}) + components: dict[str, Node] = {} + for i in range(420): + components[f"src/m{i}.py::C{i}"] = _node(f"src/m{i}.py::C{i}", "class") + components["src/x.py::X"] = _node("src/x.py::X", "class") + components["src/z.py::Z"] = _node("src/z.py::Z", "class") + components["src/y.py::Y"] = _node("src/y.py::Y", "class", {"src/z.py::Z"}) + components["Dockerfile::Dockerfile"] = _node("Dockerfile::Dockerfile", "artifact", {"src/x.py::X"}) + leaves = set(get_leaf_nodes(build_graph_from_components(components), components)) + assert "src/x.py::X" in leaves # referenced only by an artifact: kept + assert "Dockerfile::Dockerfile" in leaves + assert "src/z.py::Z" not in leaves # referenced by code: pruned + assert "src/y.py::Y" in leaves + + +# --------------------------------------------------------------------------- # +# 7. user prompt +# --------------------------------------------------------------------------- # + + +def test_format_user_prompt_with_artifacts() -> None: + dockerfile = Node( + id="Dockerfile::Dockerfile", + name="Dockerfile", + component_type="artifact", + file_path="/nonexistent/Dockerfile", + relative_path="Dockerfile", + source_code="FROM python:3.12\nCMD [\"python\"]\n", + node_type="artifact_file", + artifact_class="container", + ) + stage = Node( + id="Dockerfile::runtime", + name="runtime", + component_type="artifact", + file_path="/nonexistent/Dockerfile", + relative_path="Dockerfile", + source_code="FROM python:3.12\n", + node_type="artifact_unit", + artifact_class="container", + ) + components = {dockerfile.id: dockerfile, stage.id: stage} + tree = {"Build": {"path": ".", "components": list(components), "children": {}}} + prompt = format_user_prompt("Build", list(components), components, tree) + assert "```dockerfile\nFROM python:3.12\nCMD" in prompt + assert "" in prompt and "## container (1 files)" in prompt + assert "stages/services: runtime" in prompt + assert "# Error reading file" not in prompt # capped source used, file never re-read + + # code-only components: no artifact section, unknown extension does not raise + code = Node( + id="cfg/app.yml::app", + name="app", + component_type="class", + file_path="/nonexistent/app.yml", + relative_path="cfg/app.yml", + source_code="a: 1", + ) + prompt2 = format_user_prompt("M", [code.id], {code.id: code}, {"M": {"path": "", "components": [code.id], "children": {}}}) + assert "" not in prompt2 + assert "```yaml" in prompt2 + # MCP contract: USER_PROMPT still has exactly the three original placeholders + USER_PROMPT.format(module_name="m", module_tree="t", formatted_core_component_codes="c") + + +def test_cluster_input_tags_artifact_files() -> None: + art = _node("Makefile::Makefile", "artifact") + art.artifact_class = "build" + code = _node("src/a.py::A", "class") + ids_only, _ = format_potential_core_components([art.id, code.id], {art.id: art, code.id: code}) + assert "# Makefile (artifact: build)\n\tMakefile::Makefile" in ids_only + assert "# src/a.py\n\tsrc/a.py::A" in ids_only + + +# --------------------------------------------------------------------------- # +# 8. guaranteed module +# --------------------------------------------------------------------------- # + + +def _artifact_components(n: int) -> dict[str, Node]: + comps: dict[str, Node] = {} + for i in range(n): + node = _node(f"ci/w{i}.yml::w{i}.yml", "artifact") + node.artifact_class = "ci" + comps[node.id] = node + comps["src/a.py::A"] = _node("src/a.py::A", "class") + return comps + + +def test_ensure_artifact_module() -> None: + comps = _artifact_components(5) + leaves = list(comps) + art_ids = [i for i in leaves if i.startswith("ci/")] + # 1/5 assigned -> module inserted with the other 4 + tree = {"Core": {"path": "src", "components": ["src/a.py::A", art_ids[0]], "children": {}}} + out = ensure_artifact_module(tree, leaves, comps) + assert ARTIFACT_MODULE_NAME in out + mod = out[ARTIFACT_MODULE_NAME] + assert set(mod["components"]) == set(art_ids[1:]) + assert mod["children"] == {} and mod["path"] + # 4/5 assigned -> unchanged + tree = {"Core": {"path": "src", "components": ["src/a.py::A", *art_ids[:4]], "children": {}}} + assert ARTIFACT_MODULE_NAME not in ensure_artifact_module(tree, leaves, comps) + # whole-repo mode -> unchanged + assert ensure_artifact_module({}, leaves, comps) == {} + # name collision -> unique variant + tree = {ARTIFACT_MODULE_NAME: {"path": "", "components": ["src/a.py::A"], "children": {}}} + out = ensure_artifact_module(tree, leaves, comps) + inserted = [k for k in out if k != ARTIFACT_MODULE_NAME] + assert len(inserted) == 1 and inserted[0].startswith(ARTIFACT_MODULE_NAME) + + +def test_render_artifact_index_empty_for_code_only() -> None: + assert render_artifact_index({"src/a.py::A": _node("src/a.py::A", "class")}) == "" From 621da80cf72d7cd631c5a0a7b71f94b30a676527 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Thu, 10 Sep 2026 10:57:39 +0700 Subject: [PATCH 2/2] Fix ruff findings in files touched by artifact-aware generation CI lints every changed Python file in full, so pre-existing findings in the files this branch touches also fail the check. Apply ruff's safe fixes and formatting (pep585/pep604 annotations, import ordering, regex flag aliases) and resolve the rest by hand without behaviour changes: - blind `except Exception` sites get `# noqa: BLE001` with the reason, as the codebase already does elsewhere; bare `except:` becomes `except Exception` - subprocess.run calls pass `check=False` and use `capture_output=True` - dead assignments removed (`repo_name`, `filtered_folders_path`, `generation_time`, `start_time`, `generation_options`, `current_branch`) along with the imports they kept alive - mutable default arguments in `cluster_modules()` become `None` and are normalised at the top of the function - nested ifs merged, `endswith` tuple, needless-bool return, loop-variable closure in the package.json exports walker takes explicit parameters - shebang removed from the non-executable str_replace_editor module --- codewiki/cli/adapters/doc_generator.py | 210 ++++---- codewiki/cli/commands/generate.py | 333 +++++++------ codewiki/cli/models/config.py | 220 +++++---- .../src/be/agent_tools/str_replace_editor.py | 203 +++++--- codewiki/src/be/caw_toolkit.py | 19 +- codewiki/src/be/cluster_modules.py | 179 +++---- .../analysis/repo_analyzer.py | 30 +- .../dependency_analyzer/analyzers/artifact.py | 466 ++++++++++++++---- .../dependency_graphs_builder.py | 45 +- .../src/be/dependency_analyzer/models/core.py | 47 +- .../be/dependency_analyzer/utils/security.py | 9 +- codewiki/src/be/documentation_generator.py | 6 +- codewiki/src/be/prompt_template.py | 6 +- codewiki/src/config.py | 113 +++-- tests/test_artifact_analyzer.py | 69 ++- 15 files changed, 1208 insertions(+), 747 deletions(-) diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py index 26d5aef0..54b94da6 100644 --- a/codewiki/cli/adapters/doc_generator.py +++ b/codewiki/cli/adapters/doc_generator.py @@ -5,44 +5,43 @@ and provides CLI-specific functionality like progress reporting. """ -from pathlib import Path -from typing import Dict, Any -import time import asyncio -import os import logging +import os import sys +from pathlib import Path +from typing import Any - -from codewiki.cli.utils.progress import ProgressTracker from codewiki.cli.models.job import DocumentationJob, LLMConfig from codewiki.cli.utils.errors import APIError, IncompleteGenerationError +from codewiki.cli.utils.progress import ProgressTracker # Import backend modules from codewiki.src.be.documentation_generator import DocumentationGenerator -from codewiki.src.config import Config as BackendConfig, set_cli_context +from codewiki.src.config import Config as BackendConfig +from codewiki.src.config import set_cli_context class CLIDocumentationGenerator: """ CLI adapter for documentation generation with progress reporting. - + This class wraps the backend documentation generator and adds CLI-specific features like progress tracking and error handling. """ - + def __init__( self, repo_path: Path, output_dir: Path, - config: Dict[str, Any], + config: dict[str, Any], verbose: bool = False, generate_html: bool = False, - commit_id: str = None, + commit_id: str | None = None, ): """ Initialize the CLI documentation generator. - + Args: repo_path: Repository path output_dir: Output directory @@ -59,139 +58,137 @@ def __init__( self.commit_id = commit_id self.progress_tracker = ProgressTracker(total_stages=5, verbose=verbose) self.job = DocumentationJob() - + # Setup job metadata self.job.repository_path = str(repo_path) self.job.repository_name = repo_path.name self.job.output_directory = str(output_dir) self.job.llm_config = LLMConfig( - main_model=config.get('main_model', ''), - cluster_model=config.get('cluster_model', ''), - base_url=config.get('base_url', '') + main_model=config.get("main_model", ""), + cluster_model=config.get("cluster_model", ""), + base_url=config.get("base_url", ""), ) - + # Configure backend logging self._configure_backend_logging() - + def _configure_backend_logging(self): """Configure backend logger for CLI use with colored output.""" from codewiki.src.be.dependency_analyzer.utils.logging_config import ColoredFormatter - + # Get backend logger (parent of all backend modules) - backend_logger = logging.getLogger('codewiki.src.be') - + backend_logger = logging.getLogger("codewiki.src.be") + # Remove existing handlers to avoid duplicates backend_logger.handlers.clear() - + if self.verbose: # In verbose mode, show INFO and above backend_logger.setLevel(logging.INFO) - + # Create console handler with formatting console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(logging.INFO) - + # Use colored formatter for better readability colored_formatter = ColoredFormatter() console_handler.setFormatter(colored_formatter) - + # Add handler to logger backend_logger.addHandler(console_handler) else: # In non-verbose mode, suppress backend logs (use WARNING level to hide INFO/DEBUG) backend_logger.setLevel(logging.WARNING) - + # Create console handler for warnings and errors only console_handler = logging.StreamHandler(sys.stderr) console_handler.setLevel(logging.WARNING) - + # Use colored formatter even for warnings/errors colored_formatter = ColoredFormatter() console_handler.setFormatter(colored_formatter) - + backend_logger.addHandler(console_handler) - + # Prevent propagation to root logger to avoid duplicate messages backend_logger.propagate = False - + def generate(self) -> DocumentationJob: """ Generate documentation with progress tracking. - + Returns: Completed DocumentationJob - + Raises: APIError: If LLM API call fails """ self.job.start() - start_time = time.time() - + try: # Set CLI context for backend set_cli_context(True) - + # Create backend config with CLI settings backend_config = BackendConfig.from_cli( repo_path=str(self.repo_path), output_dir=str(self.output_dir), - llm_base_url=self.config.get('base_url'), - llm_api_key=self.config.get('api_key'), - main_model=self.config.get('main_model'), - cluster_model=self.config.get('cluster_model'), - fallback_model=self.config.get('fallback_model'), - provider=self.config.get('provider', 'openai-compatible'), - aws_region=self.config.get('aws_region', 'us-east-1'), - max_tokens=self.config.get('max_tokens', 32768), - max_token_per_module=self.config.get('max_token_per_module', 36369), - max_token_per_leaf_module=self.config.get('max_token_per_leaf_module', 16000), - max_leaf_nodes_per_cluster=self.config.get('max_leaf_nodes_per_cluster', 600), - max_depth=self.config.get('max_depth', 2), - agent_instructions=self.config.get('agent_instructions'), - use_gitignore=self.config.get('use_gitignore', True), - prompt_caching=self.config.get('prompt_caching', True), - artifacts_enabled=self.config.get('artifacts_enabled', True), - artifact_token_budget=self.config.get('artifact_token_budget', 200_000), - with_prose=self.config.get('with_prose', False), + llm_base_url=self.config.get("base_url"), + llm_api_key=self.config.get("api_key"), + main_model=self.config.get("main_model"), + cluster_model=self.config.get("cluster_model"), + fallback_model=self.config.get("fallback_model"), + provider=self.config.get("provider", "openai-compatible"), + aws_region=self.config.get("aws_region", "us-east-1"), + max_tokens=self.config.get("max_tokens", 32768), + max_token_per_module=self.config.get("max_token_per_module", 36369), + max_token_per_leaf_module=self.config.get("max_token_per_leaf_module", 16000), + max_leaf_nodes_per_cluster=self.config.get("max_leaf_nodes_per_cluster", 600), + max_depth=self.config.get("max_depth", 2), + agent_instructions=self.config.get("agent_instructions"), + use_gitignore=self.config.get("use_gitignore", True), + prompt_caching=self.config.get("prompt_caching", True), + artifacts_enabled=self.config.get("artifacts_enabled", True), + artifact_token_budget=self.config.get("artifact_token_budget", 200_000), + with_prose=self.config.get("with_prose", False), ) - + # Run backend documentation generation asyncio.run(self._run_backend_generation(backend_config)) - + # Stage 4: HTML Generation (optional) if self.generate_html: self._run_html_generation() - + # Stage 5: Finalization (metadata already created by backend) self._finalize_job() - + # Complete job - generation_time = time.time() - start_time self.job.complete() - + return self.job - + except APIError as e: self.job.fail(str(e)) raise except Exception as e: self.job.fail(str(e)) raise - + async def _run_backend_generation(self, backend_config: BackendConfig): """Run the backend documentation generation with progress tracking.""" - + # Stage 1: Dependency Analysis self.progress_tracker.start_stage(1, "Dependency Analysis") if self.verbose: self.progress_tracker.update_stage(0.2, "Initializing dependency analyzer...") - + # Create documentation generator doc_generator = DocumentationGenerator(backend_config, commit_id=self.commit_id) - + if self.verbose: self.progress_tracker.update_stage(0.5, "Parsing source files...") - + # Build dependency graph try: components, leaf_nodes = doc_generator.graph_builder.build_dependency_graph() @@ -199,22 +196,26 @@ async def _run_backend_generation(self, backend_config: BackendConfig): self.job.statistics.leaf_nodes = len(leaf_nodes) if self.verbose: - self.progress_tracker.update_stage(0.8, f"Analyzed {len(components)} files, found {len(leaf_nodes)} leaf nodes") + self.progress_tracker.update_stage( + 0.8, f"Analyzed {len(components)} files, found {len(leaf_nodes)} leaf nodes" + ) # Log individual files analyzed for comp_name in sorted(components.keys())[:20]: self.progress_tracker.update_stage(0.9, f" File: {comp_name}") if len(components) > 20: - self.progress_tracker.update_stage(0.9, f" ... and {len(components) - 20} more files") - except Exception as e: + self.progress_tracker.update_stage( + 0.9, f" ... and {len(components) - 20} more files" + ) + except Exception as e: # noqa: BLE001 — surfaced to the user as an APIError raise APIError(f"Dependency analysis failed: {e}") - + self.progress_tracker.complete_stage() - + # Stage 2: Module Clustering self.progress_tracker.start_stage(2, "Module Clustering") if self.verbose: self.progress_tracker.update_stage(0.5, "Clustering modules with LLM...") - + # Import clustering function from codewiki.src.be.cluster_modules import ( cluster_modules, @@ -222,8 +223,8 @@ async def _run_backend_generation(self, backend_config: BackendConfig): get_clustering_input_token_count, super_group_modules, ) - from codewiki.src.utils import file_manager from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME + from codewiki.src.utils import file_manager working_dir = str(self.output_dir.absolute()) file_manager.ensure_directory(working_dir) @@ -243,9 +244,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig): self.progress_tracker.update_stage(0.5, "Loaded cached module tree") else: if self.verbose: - clustering_tokens = get_clustering_input_token_count( - leaf_nodes, components - ) + clustering_tokens = get_clustering_input_token_count(leaf_nodes, components) self.progress_tracker.update_stage( 0.3, ( @@ -284,6 +283,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig): # Only freshly clustered trees are deduped: renaming a cached # key whose .md already exists would orphan the doc. from codewiki.src.be.module_naming import dedupe_module_tree_names + module_tree = dedupe_module_tree_names(module_tree) file_manager.save_json(module_tree, first_module_tree_path) file_manager.save_json(module_tree, module_tree_path) @@ -302,37 +302,45 @@ async def _run_backend_generation(self, backend_config: BackendConfig): f"Created {len(module_tree)} modules", ) for mod_name in sorted(module_tree.keys()): - file_count = len(module_tree[mod_name]) if isinstance(module_tree[mod_name], list) else "?" - self.progress_tracker.update_stage(1.0, f" Module: {mod_name} ({file_count} files)") - except Exception as e: + file_count = ( + len(module_tree[mod_name]) + if isinstance(module_tree[mod_name], list) + else "?" + ) + self.progress_tracker.update_stage( + 1.0, f" Module: {mod_name} ({file_count} files)" + ) + except Exception as e: # noqa: BLE001 — surfaced to the user as an APIError raise APIError(f"Module clustering failed: {e}") - + self.progress_tracker.complete_stage() - + # Stage 3: Documentation Generation self.progress_tracker.start_stage(3, "Documentation Generation") if self.verbose: self.progress_tracker.update_stage(0.1, "Generating module documentation...") - + try: if self.verbose: - self.progress_tracker.update_stage(0.2, f"Generating documentation for {self.job.module_count} modules...") + self.progress_tracker.update_stage( + 0.2, f"Generating documentation for {self.job.module_count} modules..." + ) # Run the actual documentation generation await doc_generator.generate_module_documentation(components, leaf_nodes) if self.verbose: self.progress_tracker.update_stage(0.9, "Creating repository overview...") - + # Create metadata doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes)) - + # Collect generated files for file_path in os.listdir(working_dir): - if file_path.endswith('.md') or file_path.endswith('.json'): + if file_path.endswith((".md", ".json")): self.job.files_generated.append(file_path) - except Exception as e: + except Exception as e: # noqa: BLE001 — surfaced to the user as an APIError raise APIError(f"Documentation generation failed: {e}") # Reconcile the final module tree against the docs on disk so name @@ -347,43 +355,43 @@ async def _run_backend_generation(self, backend_config: BackendConfig): ) self.progress_tracker.complete_stage() - + def _run_html_generation(self): """Run HTML generation stage.""" self.progress_tracker.start_stage(4, "HTML Generation") - + from codewiki.cli.html_generator import HTMLGenerator - + # Generate HTML html_generator = HTMLGenerator() - + if self.verbose: self.progress_tracker.update_stage(0.3, "Loading module tree and metadata...") - + repo_info = html_generator.detect_repository_info(self.repo_path) - + # Generate HTML with auto-loading of module_tree and metadata from docs_dir output_path = self.output_dir / "index.html" html_generator.generate( output_path=output_path, - title=repo_info['name'], - repository_url=repo_info['url'], - github_pages_url=repo_info['github_pages_url'], - docs_dir=self.output_dir # Auto-load module_tree and metadata from here + title=repo_info["name"], + repository_url=repo_info["url"], + github_pages_url=repo_info["github_pages_url"], + docs_dir=self.output_dir, # Auto-load module_tree and metadata from here ) - + self.job.files_generated.append("index.html") - + if self.verbose: self.progress_tracker.update_stage(1.0, "Generated index.html") - + self.progress_tracker.complete_stage() - + def _finalize_job(self): """Finalize the job (metadata already created by backend).""" # Just verify metadata exists metadata_path = self.output_dir / "metadata.json" if not metadata_path.exists(): # Create our own if backend didn't - with open(metadata_path, 'w') as f: + with open(metadata_path, "w") as f: f.write(self.job.to_json()) diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py index 924bb35e..6e4c7079 100644 --- a/codewiki/cli/commands/generate.py +++ b/codewiki/cli/commands/generate.py @@ -2,51 +2,45 @@ Generate command for documentation generation. """ -import sys import logging +import sys +import time import traceback from pathlib import Path -from typing import Optional, List, Tuple + import click -import time +from codewiki.cli.adapters.doc_generator import CLIDocumentationGenerator from codewiki.cli.config_manager import ConfigManager +from codewiki.cli.models.config import AgentInstructions from codewiki.cli.utils.errors import ( - ConfigurationError, - RepositoryError, + EXIT_SUCCESS, APIError, + ConfigurationError, IncompleteGenerationError, + RepositoryError, handle_error, - EXIT_SUCCESS, ) +from codewiki.cli.utils.instructions import display_post_generation_instructions +from codewiki.cli.utils.logging import create_logger from codewiki.cli.utils.repo_validator import ( - validate_repository, check_writable_output, - is_git_repository, get_git_commit_hash, - get_git_branch, + is_git_repository, + validate_repository, ) -from codewiki.cli.utils.logging import create_logger -from codewiki.cli.adapters.doc_generator import CLIDocumentationGenerator -from codewiki.cli.utils.instructions import display_post_generation_instructions -from codewiki.cli.models.job import GenerationOptions -from codewiki.cli.models.config import AgentInstructions -def parse_patterns(patterns_str: str) -> List[str]: +def parse_patterns(patterns_str: str) -> list[str]: """Parse comma-separated patterns into a list.""" if not patterns_str: return [] - return [p.strip() for p in patterns_str.split(',') if p.strip()] + return [p.strip() for p in patterns_str.split(",") if p.strip()] def _detect_changed_files( - repo_path: Path, - output_dir: Path, - logger, - verbose: bool, - compare_to: Optional[str] = None -) -> Optional[List[str]]: + repo_path: Path, output_dir: Path, logger, verbose: bool, compare_to: str | None = None +) -> list[str] | None: """ Detect files changed since the last documentation generation. @@ -66,7 +60,9 @@ def _detect_changed_files( metadata_path = output_dir / "metadata.json" if not metadata_path.exists(): if verbose: - logger.debug("No metadata.json found — cannot detect changes, running full generation.") + logger.debug( + "No metadata.json found — cannot detect changes, running full generation." + ) return None try: @@ -82,9 +78,10 @@ def _detect_changed_files( # Get current HEAD commit try: import git + repo = git.Repo(repo_path, search_parent_directories=True) current_commit = repo.head.commit.hexsha - except Exception: + except Exception: # noqa: BLE001 — any git failure means "no incremental update" if verbose: logger.debug("Cannot access git repo — running full generation.") return None @@ -128,7 +125,7 @@ def _detect_changed_files( prefix = subpath_prefix + "/" for path in changed: if path.startswith(prefix): - filtered.append(path[len(prefix):]) + filtered.append(path[len(prefix) :]) if verbose: logger.debug(f"Changes between {prev_commit[:8]} and {current_commit[:8]}:") @@ -140,18 +137,13 @@ def _detect_changed_files( logger.debug(f" ... and {len(filtered) - 10} more") return filtered - except Exception as e: + except Exception as e: # noqa: BLE001 — fall back to regenerating everything if verbose: logger.debug(f"Git diff failed: {e} — running full generation.") return None -def _invalidate_affected_modules( - output_dir: Path, - changed_files: List[str], - logger, - verbose: bool -): +def _invalidate_affected_modules(output_dir: Path, changed_files: list[str], logger, verbose: bool): """ Remove cached module documentation for modules that contain changed files. @@ -180,7 +172,9 @@ def _find_affected(tree, parent_names=None): # Check if any component path overlaps with changed files for comp in components: # Component IDs may be class names, check if they match any changed file path - if any(changed_file in comp or comp in changed_file for changed_file in changed_set): + if any( + changed_file in comp or comp in changed_file for changed_file in changed_set + ): modules_to_invalidate.add(mod_name) # Also invalidate parent modules for parent in parent_names: @@ -256,7 +250,7 @@ def _find_affected(tree, parent_names=None): @click.option( "--doc-type", "-t", - type=click.Choice(['api', 'architecture', 'user-guide', 'developer'], case_sensitive=False), + type=click.Choice(["api", "architecture", "user-guide", "developer"], case_sensitive=False), default=None, help="Type of documentation to generate", ) @@ -305,13 +299,13 @@ def _find_affected(tree, parent_names=None): "--prompt-caching/--no-prompt-caching", default=None, help="Add prompt-cache breakpoints to agentic LLM calls; auto-falls back to " - "normal calls if the provider rejects them (default: enabled)", + "normal calls if the provider rejects them (default: enabled)", ) @click.option( "--artifacts/--no-artifacts", default=True, help="Document build, CI, container, packaging, manifest, config, schema and " - "script files as part of the dependency graph (default: enabled)", + "script files as part of the dependency graph (default: enabled)", ) @click.option( "--artifact-token-budget", @@ -349,41 +343,41 @@ def generate_command( create_branch: bool, github_pages: bool, no_cache: bool, - include: Optional[str], - exclude: Optional[str], - focus: Optional[str], - doc_type: Optional[str], - instructions: Optional[str], - use_gitignore: Optional[bool], + include: str | None, + exclude: str | None, + focus: str | None, + doc_type: str | None, + instructions: str | None, + use_gitignore: bool | None, verbose: bool, - max_tokens: Optional[int], - max_token_per_module: Optional[int], - max_token_per_leaf_module: Optional[int], - max_depth: Optional[int], - prompt_caching: Optional[bool], + max_tokens: int | None, + max_token_per_module: int | None, + max_token_per_leaf_module: int | None, + max_depth: int | None, + prompt_caching: bool | None, artifacts: bool = True, artifact_token_budget: int = 200_000, with_prose: bool = False, - artifact_exclude: Optional[str] = None, + artifact_exclude: str | None = None, update: bool = False, - compare_to: Optional[str] = None + compare_to: str | None = None, ): """ Generate comprehensive documentation for a code repository. - + Analyzes the current repository and generates documentation using LLM-powered analysis. Documentation is output to ./docs/ by default. - + Examples: - + \b # Basic generation $ codewiki generate - + \b # With git branch creation and GitHub Pages $ codewiki generate --create-branch --github-pages - + \b # Force full regeneration $ codewiki generate --no-cache @@ -391,41 +385,41 @@ def generate_command( \b # Analyze ignored files as well $ codewiki generate --no-gitignore - + \b # C# project: only .cs files, exclude tests $ codewiki generate --include "*.cs" --exclude "*Tests*,*Specs*" - + \b # Focus on specific modules with architecture docs $ codewiki generate --focus "src/core,src/api" --doc-type architecture - + \b # Custom instructions $ codewiki generate --instructions "Focus on public APIs and include usage examples" - + \b # Override max tokens for this generation $ codewiki generate --max-tokens 16384 - + \b # Set all max token limits $ codewiki generate --max-tokens 32768 --max-token-per-module 40000 --max-token-per-leaf-module 20000 - + \b # Override max depth for hierarchical decomposition $ codewiki generate --max-depth 3 """ logger = create_logger(verbose=verbose) start_time = time.time() - + # Suppress httpx INFO logs logging.getLogger("httpx").setLevel(logging.WARNING) - + try: # Pre-generation checks logger.step("Validating configuration...", 1, 4) - + # Load configuration config_manager = ConfigManager() if not config_manager.load(): @@ -436,27 +430,29 @@ def generate_command( " --main-model --cluster-model \n\n" "For more help: codewiki config --help" ) - + if not config_manager.is_configured(): raise ConfigurationError( "Configuration is incomplete. Please run 'codewiki config validate'" ) - + config = config_manager.get_config() api_key = config_manager.get_api_key() - + logger.success("Configuration valid") - + # Validate repository logger.step("Validating repository...", 2, 4) - + repo_path = Path.cwd() repo_path, languages = validate_repository(repo_path) - + logger.success(f"Repository valid: {repo_path.name}") if verbose: - logger.debug(f"Detected languages: {', '.join(f'{lang} ({count} files)' for lang, count in languages)}") - + logger.debug( + f"Detected languages: {', '.join(f'{lang} ({count} files)' for lang, count in languages)}" + ) + # Check git repository if not is_git_repository(repo_path): if create_branch: @@ -467,13 +463,13 @@ def generate_command( ) else: logger.warning("Not a git repository. Git features unavailable.") - + # Validate output directory output_dir = Path(output).expanduser().resolve() check_writable_output(output_dir.parent) - + logger.success(f"Output directory: {output_dir}") - + # If a base commit is specified to compare against, implicitly enable update if compare_to: update = True @@ -481,33 +477,42 @@ def generate_command( # Incremental update: detect changed files and selectively regenerate changed_files = None if update and output_dir.exists(): - changed_files = _detect_changed_files(repo_path, output_dir, logger, verbose, compare_to=compare_to) + changed_files = _detect_changed_files( + repo_path, output_dir, logger, verbose, compare_to=compare_to + ) if changed_files is not None and len(changed_files) == 0: - logger.success("No changes detected since last generation. Documentation is up to date.") + logger.success( + "No changes detected since last generation. Documentation is up to date." + ) sys.exit(EXIT_SUCCESS) if changed_files is not None: - logger.info(f" Detected {len(changed_files)} changed files — regenerating affected modules.") + logger.info( + f" Detected {len(changed_files)} changed files — regenerating affected modules." + ) # Remove cached module docs for affected files so they get regenerated _invalidate_affected_modules(output_dir, changed_files, logger, verbose) # Check for existing documentation - if not update and output_dir.exists() and list(output_dir.glob("*.md")): - if not click.confirm( - f"\n{output_dir} already contains documentation. Overwrite?", - default=True - ): - logger.info("Generation cancelled by user.") - sys.exit(EXIT_SUCCESS) - + if ( + not update + and output_dir.exists() + and list(output_dir.glob("*.md")) + and not click.confirm( + f"\n{output_dir} already contains documentation. Overwrite?", default=True + ) + ): + logger.info("Generation cancelled by user.") + sys.exit(EXIT_SUCCESS) + # Git branch creation (if requested) branch_name = None if create_branch: logger.step("Creating git branch...", 3, 4) - + from codewiki.cli.git_manager import GitManager - + git_manager = GitManager(repo_path) - + # Check clean working directory is_clean, status_msg = git_manager.check_clean_working_directory() if not is_clean: @@ -516,27 +521,19 @@ def generate_command( f"{status_msg}\n\n" "Cannot create documentation branch with uncommitted changes.\n" "Please commit or stash your changes first:\n" - " git add -A && git commit -m \"Your message\"\n" + ' git add -A && git commit -m "Your message"\n' " # or\n" " git stash" ) - + # Create branch branch_name = git_manager.create_documentation_branch() logger.success(f"Created branch: {branch_name}") - + # Generate documentation logger.step("Generating documentation...", 4, 4) click.echo() - - # Create generation options - generation_options = GenerationOptions( - create_branch=create_branch, - github_pages=github_pages, - no_cache=no_cache, - custom_output=output if output != "docs" else None - ) - + # Create runtime agent instructions from CLI options runtime_instructions = None if any([include, exclude, focus, doc_type, instructions, artifact_exclude]): @@ -548,7 +545,7 @@ def generate_command( custom_instructions=instructions, artifact_exclude=parse_patterns(artifact_exclude) if artifact_exclude else None, ) - + if verbose: if include: logger.debug(f"Include patterns: {parse_patterns(include)}") @@ -562,39 +559,75 @@ def generate_command( logger.debug(f"Custom instructions: {instructions}") if artifact_exclude: logger.debug(f"Artifact exclude patterns: {parse_patterns(artifact_exclude)}") - + # Log max token settings if verbose if verbose: effective_max_tokens = max_tokens if max_tokens is not None else config.max_tokens - effective_max_token_per_module = max_token_per_module if max_token_per_module is not None else config.max_token_per_module - effective_max_token_per_leaf = max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module + effective_max_token_per_module = ( + max_token_per_module + if max_token_per_module is not None + else config.max_token_per_module + ) + effective_max_token_per_leaf = ( + max_token_per_leaf_module + if max_token_per_leaf_module is not None + else config.max_token_per_leaf_module + ) effective_max_depth = max_depth if max_depth is not None else config.max_depth - effective_use_gitignore = use_gitignore if use_gitignore is not None else config.use_gitignore - effective_prompt_caching = prompt_caching if prompt_caching is not None else config.prompt_caching + effective_use_gitignore = ( + use_gitignore if use_gitignore is not None else config.use_gitignore + ) + effective_prompt_caching = ( + prompt_caching if prompt_caching is not None else config.prompt_caching + ) logger.debug(f"Max tokens: {effective_max_tokens}") logger.debug(f"Max token/module: {effective_max_token_per_module}") logger.debug(f"Max token/leaf module: {effective_max_token_per_leaf}") logger.debug(f"Max depth: {effective_max_depth}") logger.debug(f"Use gitignore: {effective_use_gitignore}") logger.debug(f"Prompt caching: {effective_prompt_caching}") - logger.debug(f"Artifacts: {artifacts} (token budget {artifact_token_budget}, prose {with_prose})") - + logger.debug( + f"Artifacts: {artifacts} (token budget {artifact_token_budget}, prose {with_prose})" + ) + # Get agent instructions (merge runtime with persistent) agent_instructions_dict = None if runtime_instructions and not runtime_instructions.is_empty(): # Merge with persistent settings merged = AgentInstructions( - include_patterns=runtime_instructions.include_patterns or (config.agent_instructions.include_patterns if config.agent_instructions else None), - exclude_patterns=runtime_instructions.exclude_patterns or (config.agent_instructions.exclude_patterns if config.agent_instructions else None), - focus_modules=runtime_instructions.focus_modules or (config.agent_instructions.focus_modules if config.agent_instructions else None), - doc_type=runtime_instructions.doc_type or (config.agent_instructions.doc_type if config.agent_instructions else None), - custom_instructions=runtime_instructions.custom_instructions or (config.agent_instructions.custom_instructions if config.agent_instructions else None), - artifact_exclude=runtime_instructions.artifact_exclude or (config.agent_instructions.artifact_exclude if config.agent_instructions else None), + include_patterns=runtime_instructions.include_patterns + or ( + config.agent_instructions.include_patterns + if config.agent_instructions + else None + ), + exclude_patterns=runtime_instructions.exclude_patterns + or ( + config.agent_instructions.exclude_patterns + if config.agent_instructions + else None + ), + focus_modules=runtime_instructions.focus_modules + or (config.agent_instructions.focus_modules if config.agent_instructions else None), + doc_type=runtime_instructions.doc_type + or (config.agent_instructions.doc_type if config.agent_instructions else None), + custom_instructions=runtime_instructions.custom_instructions + or ( + config.agent_instructions.custom_instructions + if config.agent_instructions + else None + ), + artifact_exclude=runtime_instructions.artifact_exclude + or ( + config.agent_instructions.artifact_exclude + if config.agent_instructions + else None + ), ) agent_instructions_dict = merged.to_dict() elif config.agent_instructions and not config.agent_instructions.is_empty(): agent_instructions_dict = config.agent_instructions.to_dict() - + # Create generator # Get commit_id early so it can be stored in metadata.json for --update support commit_id = get_git_commit_hash(repo_path) @@ -602,53 +635,61 @@ def generate_command( repo_path=repo_path, output_dir=output_dir, config={ - 'main_model': config.main_model, - 'cluster_model': config.cluster_model, - 'fallback_model': config.fallback_model, - 'base_url': config.base_url, - 'api_key': api_key, - 'provider': getattr(config, 'provider', 'openai-compatible'), - 'aws_region': getattr(config, 'aws_region', 'us-east-1'), - 'agent_instructions': agent_instructions_dict, + "main_model": config.main_model, + "cluster_model": config.cluster_model, + "fallback_model": config.fallback_model, + "base_url": config.base_url, + "api_key": api_key, + "provider": getattr(config, "provider", "openai-compatible"), + "aws_region": getattr(config, "aws_region", "us-east-1"), + "agent_instructions": agent_instructions_dict, # Max token settings (runtime overrides take precedence) - 'max_tokens': max_tokens if max_tokens is not None else config.max_tokens, - 'max_token_per_module': max_token_per_module if max_token_per_module is not None else config.max_token_per_module, - 'max_token_per_leaf_module': max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module, + "max_tokens": max_tokens if max_tokens is not None else config.max_tokens, + "max_token_per_module": max_token_per_module + if max_token_per_module is not None + else config.max_token_per_module, + "max_token_per_leaf_module": max_token_per_leaf_module + if max_token_per_leaf_module is not None + else config.max_token_per_leaf_module, # Max depth setting (runtime override takes precedence) - 'max_depth': max_depth if max_depth is not None else config.max_depth, + "max_depth": max_depth if max_depth is not None else config.max_depth, # Gitignore setting (runtime override takes precedence) - 'use_gitignore': use_gitignore if use_gitignore is not None else config.use_gitignore, + "use_gitignore": use_gitignore + if use_gitignore is not None + else config.use_gitignore, # Prompt caching setting (runtime override takes precedence) - 'prompt_caching': prompt_caching if prompt_caching is not None else config.prompt_caching, + "prompt_caching": prompt_caching + if prompt_caching is not None + else config.prompt_caching, # Artifact-aware generation (runtime-only flags) - 'artifacts_enabled': artifacts, - 'artifact_token_budget': artifact_token_budget, - 'with_prose': with_prose, + "artifacts_enabled": artifacts, + "artifact_token_budget": artifact_token_budget, + "with_prose": with_prose, }, verbose=verbose, generate_html=github_pages, commit_id=commit_id, ) - + # Run generation job = generator.generate() - + # Post-generation generation_time = time.time() - start_time - + # Get repository info repo_url = None - current_branch = get_git_branch(repo_path) - + if is_git_repository(repo_path): try: import git + repo = git.Repo(repo_path) if repo.remotes: repo_url = repo.remotes.origin.url - except: + except Exception: # noqa: BLE001, S110 — the remote URL is optional pass - + # Display instructions display_post_generation_instructions( output_dir=output_dir, @@ -658,13 +699,13 @@ def generate_command( github_pages=github_pages, files_generated=job.files_generated, statistics={ - 'module_count': job.module_count, - 'total_files_analyzed': job.statistics.total_files_analyzed, - 'generation_time': generation_time, - 'total_tokens_used': job.statistics.total_tokens_used, - } + "module_count": job.module_count, + "total_files_analyzed": job.statistics.total_files_analyzed, + "generation_time": generation_time, + "total_tokens_used": job.statistics.total_tokens_used, + }, ) - + except ConfigurationError as e: logger.error(e.message) logger.error(f"Traceback: {traceback.format_exc()}") @@ -689,5 +730,5 @@ def generate_command( except KeyboardInterrupt: click.echo("\n\nInterrupted by user") sys.exit(130) - except Exception as e: + except Exception as e: # noqa: BLE001 — top-level CLI error handler sys.exit(handle_error(e, verbose=verbose)) diff --git a/codewiki/cli/models/config.py b/codewiki/cli/models/config.py index 5f55351b..c1cabe07 100644 --- a/codewiki/cli/models/config.py +++ b/codewiki/cli/models/config.py @@ -6,14 +6,11 @@ to the backend Config class when running documentation generation. """ -from dataclasses import dataclass, asdict, field -from typing import Optional, List -from pathlib import Path +from dataclasses import dataclass, field from codewiki.cli.utils.validation import ( - validate_url, - validate_api_key, validate_model_name, + validate_url, ) @@ -21,13 +18,13 @@ class AgentInstructions: """ Custom instructions for the documentation agent. - + Allows users to customize: - File filtering (include/exclude patterns) - Module focus (prioritize certain modules) - Documentation type (API docs, architecture docs, etc.) - Custom instructions for the LLM - + Attributes: include_patterns: File patterns to include (e.g., ["*.cs", "*.py"]) exclude_patterns: File/directory patterns to exclude (e.g., ["*Tests*", "*test*"]) @@ -35,75 +32,82 @@ class AgentInstructions: doc_type: Type of documentation to generate custom_instructions: Additional instructions for the documentation agent """ - include_patterns: Optional[List[str]] = None # e.g., ["*.cs"] for C# projects - exclude_patterns: Optional[List[str]] = None # e.g., ["*Tests*", "*Specs*"] - focus_modules: Optional[List[str]] = None # e.g., ["src/core", "src/api"] - doc_type: Optional[str] = None # e.g., "api", "architecture", "user-guide" - custom_instructions: Optional[str] = None # Free-form instructions - artifact_exclude: Optional[List[str]] = None # e.g., ["docker/data/*"] skipped by artifact analysis - + + include_patterns: list[str] | None = None # e.g., ["*.cs"] for C# projects + exclude_patterns: list[str] | None = None # e.g., ["*Tests*", "*Specs*"] + focus_modules: list[str] | None = None # e.g., ["src/core", "src/api"] + doc_type: str | None = None # e.g., "api", "architecture", "user-guide" + custom_instructions: str | None = None # Free-form instructions + artifact_exclude: list[str] | None = ( + None # e.g., ["docker/data/*"] skipped by artifact analysis + ) + def to_dict(self) -> dict: """Convert to dictionary, excluding None values.""" result = {} if self.include_patterns: - result['include_patterns'] = self.include_patterns + result["include_patterns"] = self.include_patterns if self.exclude_patterns: - result['exclude_patterns'] = self.exclude_patterns + result["exclude_patterns"] = self.exclude_patterns if self.artifact_exclude: - result['artifact_exclude'] = self.artifact_exclude + result["artifact_exclude"] = self.artifact_exclude if self.focus_modules: - result['focus_modules'] = self.focus_modules + result["focus_modules"] = self.focus_modules if self.doc_type: - result['doc_type'] = self.doc_type + result["doc_type"] = self.doc_type if self.custom_instructions: - result['custom_instructions'] = self.custom_instructions + result["custom_instructions"] = self.custom_instructions return result - + @classmethod - def from_dict(cls, data: dict) -> 'AgentInstructions': + def from_dict(cls, data: dict) -> "AgentInstructions": """Create AgentInstructions from dictionary.""" return cls( - include_patterns=data.get('include_patterns'), - exclude_patterns=data.get('exclude_patterns'), - focus_modules=data.get('focus_modules'), - doc_type=data.get('doc_type'), - custom_instructions=data.get('custom_instructions'), - artifact_exclude=data.get('artifact_exclude'), + include_patterns=data.get("include_patterns"), + exclude_patterns=data.get("exclude_patterns"), + focus_modules=data.get("focus_modules"), + doc_type=data.get("doc_type"), + custom_instructions=data.get("custom_instructions"), + artifact_exclude=data.get("artifact_exclude"), ) - + def is_empty(self) -> bool: """Check if all fields are empty/None.""" - return not any([ - self.include_patterns, - self.exclude_patterns, - self.artifact_exclude, - self.focus_modules, - self.doc_type, - self.custom_instructions, - ]) - + return not any( + [ + self.include_patterns, + self.exclude_patterns, + self.artifact_exclude, + self.focus_modules, + self.doc_type, + self.custom_instructions, + ] + ) + def get_prompt_addition(self) -> str: """Generate prompt additions based on instructions.""" additions = [] - + if self.doc_type: doc_type_instructions = { - 'api': "Focus on API documentation: endpoints, parameters, return types, and usage examples.", - 'architecture': "Focus on architecture documentation: system design, component relationships, and data flow.", - 'user-guide': "Focus on user guide documentation: how to use features, step-by-step tutorials.", - 'developer': "Focus on developer documentation: code structure, contribution guidelines, and implementation details.", + "api": "Focus on API documentation: endpoints, parameters, return types, and usage examples.", + "architecture": "Focus on architecture documentation: system design, component relationships, and data flow.", + "user-guide": "Focus on user guide documentation: how to use features, step-by-step tutorials.", + "developer": "Focus on developer documentation: code structure, contribution guidelines, and implementation details.", } if self.doc_type.lower() in doc_type_instructions: additions.append(doc_type_instructions[self.doc_type.lower()]) else: additions.append(f"Focus on generating {self.doc_type} documentation.") - + if self.focus_modules: - additions.append(f"Pay special attention to and provide more detailed documentation for these modules: {', '.join(self.focus_modules)}") - + additions.append( + f"Pay special attention to and provide more detailed documentation for these modules: {', '.join(self.focus_modules)}" + ) + if self.custom_instructions: additions.append(f"Additional instructions: {self.custom_instructions}") - + return "\n".join(additions) if additions else "" @@ -130,6 +134,7 @@ class Configuration: prompt_caching: Add prompt-cache breakpoints to agentic LLM calls (default: True) agent_instructions: Custom agent instructions for documentation generation """ + base_url: str main_model: str cluster_model: str @@ -146,7 +151,7 @@ class Configuration: use_gitignore: bool = True prompt_caching: bool = True agent_instructions: AgentInstructions = field(default_factory=AgentInstructions) - + def validate(self): """ Validate all configuration fields. @@ -158,6 +163,7 @@ def validate(self): ConfigurationError: If validation fails """ from codewiki.src.be.backend import is_caw_provider + if is_caw_provider(self.provider): validate_model_name(self.main_model) return @@ -165,64 +171,64 @@ def validate(self): validate_model_name(self.main_model) validate_model_name(self.cluster_model) validate_model_name(self.fallback_model) - + def to_dict(self) -> dict: """Convert to dictionary.""" result = { - 'base_url': self.base_url, - 'main_model': self.main_model, - 'cluster_model': self.cluster_model, - 'default_output': self.default_output, - 'provider': self.provider, - 'aws_region': self.aws_region, - 'api_version': self.api_version, - 'azure_deployment': self.azure_deployment, - 'max_tokens': self.max_tokens, - 'max_token_per_module': self.max_token_per_module, - 'max_token_per_leaf_module': self.max_token_per_leaf_module, - 'max_depth': self.max_depth, - 'use_gitignore': self.use_gitignore, - 'prompt_caching': self.prompt_caching, - 'fallback_model': self.fallback_model, + "base_url": self.base_url, + "main_model": self.main_model, + "cluster_model": self.cluster_model, + "default_output": self.default_output, + "provider": self.provider, + "aws_region": self.aws_region, + "api_version": self.api_version, + "azure_deployment": self.azure_deployment, + "max_tokens": self.max_tokens, + "max_token_per_module": self.max_token_per_module, + "max_token_per_leaf_module": self.max_token_per_leaf_module, + "max_depth": self.max_depth, + "use_gitignore": self.use_gitignore, + "prompt_caching": self.prompt_caching, + "fallback_model": self.fallback_model, } if self.agent_instructions and not self.agent_instructions.is_empty(): - result['agent_instructions'] = self.agent_instructions.to_dict() + result["agent_instructions"] = self.agent_instructions.to_dict() return result - + @classmethod - def from_dict(cls, data: dict) -> 'Configuration': + def from_dict(cls, data: dict) -> "Configuration": """ Create Configuration from dictionary. - + Args: data: Configuration dictionary - + Returns: Configuration instance """ agent_instructions = AgentInstructions() - if 'agent_instructions' in data: - agent_instructions = AgentInstructions.from_dict(data['agent_instructions']) - + if "agent_instructions" in data: + agent_instructions = AgentInstructions.from_dict(data["agent_instructions"]) + return cls( - base_url=data.get('base_url', ''), - main_model=data.get('main_model', ''), - cluster_model=data.get('cluster_model', ''), - fallback_model=data.get('fallback_model', 'glm-4p5'), - default_output=data.get('default_output', 'docs'), - provider=data.get('provider', 'openai-compatible'), - aws_region=data.get('aws_region', 'us-east-1'), - api_version=data.get('api_version', '2024-12-01-preview'), - azure_deployment=data.get('azure_deployment', ''), - max_tokens=data.get('max_tokens', 32768), - max_token_per_module=data.get('max_token_per_module', 36369), - max_token_per_leaf_module=data.get('max_token_per_leaf_module', 16000), - max_depth=data.get('max_depth', 2), - use_gitignore=data.get('use_gitignore', True), - prompt_caching=data.get('prompt_caching', True), + base_url=data.get("base_url", ""), + main_model=data.get("main_model", ""), + cluster_model=data.get("cluster_model", ""), + fallback_model=data.get("fallback_model", "glm-4p5"), + default_output=data.get("default_output", "docs"), + provider=data.get("provider", "openai-compatible"), + aws_region=data.get("aws_region", "us-east-1"), + api_version=data.get("api_version", "2024-12-01-preview"), + azure_deployment=data.get("azure_deployment", ""), + max_tokens=data.get("max_tokens", 32768), + max_token_per_module=data.get("max_token_per_module", 36369), + max_token_per_leaf_module=data.get("max_token_per_leaf_module", 16000), + max_depth=data.get("max_depth", 2), + use_gitignore=data.get("use_gitignore", True), + prompt_caching=data.get("prompt_caching", True), agent_instructions=agent_instructions, ) - + def is_complete(self) -> bool: """Check if all required fields are set. @@ -231,45 +237,53 @@ def is_complete(self) -> bool: are unused. """ from codewiki.src.be.backend import is_caw_provider + if is_caw_provider(self.provider): return bool(self.main_model) return bool( - self.base_url and - self.main_model and - self.cluster_model and - self.fallback_model + self.base_url and self.main_model and self.cluster_model and self.fallback_model ) - - def to_backend_config(self, repo_path: str, output_dir: str, api_key: str, runtime_instructions: AgentInstructions = None): + + def to_backend_config( + self, + repo_path: str, + output_dir: str, + api_key: str, + runtime_instructions: AgentInstructions = None, + ): """ Convert CLI Configuration to Backend Config. - + This method bridges the gap between persistent user settings (CLI Configuration) and runtime job configuration (Backend Config). - + Args: repo_path: Path to the repository to document output_dir: Output directory for generated documentation api_key: LLM API key (from keyring) runtime_instructions: Runtime agent instructions (override persistent settings) - + Returns: Backend Config instance ready for documentation generation """ from codewiki.src.config import Config - + # Merge runtime instructions with persistent settings # Runtime instructions take precedence final_instructions = self.agent_instructions if runtime_instructions and not runtime_instructions.is_empty(): final_instructions = AgentInstructions( - include_patterns=runtime_instructions.include_patterns or self.agent_instructions.include_patterns, - exclude_patterns=runtime_instructions.exclude_patterns or self.agent_instructions.exclude_patterns, - focus_modules=runtime_instructions.focus_modules or self.agent_instructions.focus_modules, + include_patterns=runtime_instructions.include_patterns + or self.agent_instructions.include_patterns, + exclude_patterns=runtime_instructions.exclude_patterns + or self.agent_instructions.exclude_patterns, + focus_modules=runtime_instructions.focus_modules + or self.agent_instructions.focus_modules, doc_type=runtime_instructions.doc_type or self.agent_instructions.doc_type, - custom_instructions=runtime_instructions.custom_instructions or self.agent_instructions.custom_instructions, + custom_instructions=runtime_instructions.custom_instructions + or self.agent_instructions.custom_instructions, ) - + return Config.from_cli( repo_path=repo_path, output_dir=output_dir, diff --git a/codewiki/src/be/agent_tools/str_replace_editor.py b/codewiki/src/be/agent_tools/str_replace_editor.py index 3ee8b9c4..49a84db1 100644 --- a/codewiki/src/be/agent_tools/str_replace_editor.py +++ b/codewiki/src/be/agent_tools/str_replace_editor.py @@ -1,20 +1,17 @@ -#!/usr/bin/env python3 - """Source: https://github.com/SWE-agent/SWE-agent/blob/main/tools/edit_anthropic/bin/str_replace_editor This tool is used to view the given source code and view/edit the documentation files in the separate docs directory. """ +import io import json +import logging import re import shlex import subprocess import sys from collections import defaultdict from pathlib import Path -from typing import Annotated, List, Literal, Optional, Tuple -import io - -import logging +from typing import Annotated, Literal # Configure logging and monitoring @@ -23,8 +20,8 @@ from pydantic import BeforeValidator from pydantic_ai import RunContext, Tool -from .deps import CodeWikiDeps from ..utils import validate_mermaid_diagrams +from .deps import CodeWikiDeps def _coerce_json_string(value): @@ -46,8 +43,8 @@ def _coerce_json_string(value): return value -ViewRange = Annotated[Optional[List[int]], BeforeValidator(_coerce_json_string)] -InsertLine = Annotated[Optional[int], BeforeValidator(_coerce_json_string)] +ViewRange = Annotated[list[int] | None, BeforeValidator(_coerce_json_string)] +InsertLine = Annotated[int | None, BeforeValidator(_coerce_json_string)] # There are some super strange "ascii can't decode x" errors, @@ -85,7 +82,7 @@ def _coerce_json_string(value): """ -def maybe_truncate(content: str, truncate_after: Optional[int] = MAX_RESPONSE_LEN): +def maybe_truncate(content: str, truncate_after: int | None = MAX_RESPONSE_LEN): """Truncate content and append a notice if content exceeds the specified length.""" return ( content @@ -128,8 +125,10 @@ def __repr__(self): def _update_previous_errors( - previous_errors: List[Flake8Error], replacement_window: Tuple[int, int], replacement_n_lines: int -) -> List[Flake8Error]: + previous_errors: list[Flake8Error], + replacement_window: tuple[int, int], + replacement_n_lines: int, +) -> list[Flake8Error]: """Update the line numbers of the previous errors to what they would be after the edit window. This is a helper function for `_filter_previous_errors`. @@ -156,7 +155,11 @@ def _update_previous_errors( # either way (we wouldn't know how to adjust the line number anyway) continue # We're out of the edit window, so we need to adjust the line number - updated.append(Flake8Error(error.filename, error.line_number + lines_added, error.col_number, error.problem)) + updated.append( + Flake8Error( + error.filename, error.line_number + lines_added, error.col_number, error.problem + ) + ) return updated @@ -165,8 +168,8 @@ def format_flake8_output( show_line_numbers: bool = False, *, previous_errors_string: str = "", - replacement_window: Optional[Tuple[int, int]] = None, - replacement_n_lines: Optional[int] = None, + replacement_window: tuple[int, int] | None = None, + replacement_n_lines: int | None = None, ) -> str: """Filter flake8 output for previous errors and print it for a given file. @@ -184,17 +187,23 @@ def format_flake8_output( # print("Replacement n lines:", replacement_n_lines) # print("Previous errors string:", previous_errors_string) # print("Input string:", input_string) - errors = [Flake8Error.from_line(line.strip()) for line in input_string.split("\n") if line.strip()] + errors = [ + Flake8Error.from_line(line.strip()) for line in input_string.split("\n") if line.strip() + ] # print(f"New errors before filtering: {errors=}") lines = [] if previous_errors_string: assert replacement_window is not None assert replacement_n_lines is not None previous_errors = [ - Flake8Error.from_line(line.strip()) for line in previous_errors_string.split("\n") if line.strip() + Flake8Error.from_line(line.strip()) + for line in previous_errors_string.split("\n") + if line.strip() ] # print(f"Previous errors before updating: {previous_errors=}") - previous_errors = _update_previous_errors(previous_errors, replacement_window, replacement_n_lines) + previous_errors = _update_previous_errors( + previous_errors, replacement_window, replacement_n_lines + ) # print(f"Previous errors after updating: {previous_errors=}") errors = [error for error in errors if error not in previous_errors] # Sometimes new errors appear above the replacement window that were 'shadowed' by the previous errors @@ -214,8 +223,9 @@ def flake8(file_path: str) -> str: if Path(file_path).suffix != ".py": return "" cmd = "flake8 --isolated --select=F821,F822,F831,E111,E112,E113,E999,E902 {file_path}" - # don't use capture_output because it's not compatible with python3.6 - out = subprocess.run(cmd.format(file_path=file_path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out = subprocess.run( + cmd.format(file_path=file_path), shell=True, check=False, capture_output=True + ) # Use errors="replace" so non-UTF-8 bytes (e.g. GBK-encoded paths on Windows) don't crash decoding. return out.stdout.decode("utf-8", errors="replace") @@ -223,6 +233,7 @@ def flake8(file_path: str) -> str: class Filemap: def show_filemap(self, file_contents: str, encoding: str = "utf8"): import warnings + from tree_sitter_languages import get_language, get_parser warnings.simplefilter("ignore", category=FutureWarning) @@ -249,12 +260,20 @@ def show_filemap(self, file_contents: str, encoding: str = "utf8"): ] # Note that tree-sitter line numbers are 0-indexed, but we display 1-indexed. elide_lines = {line for start, end in elide_line_ranges for line in range(start, end + 1)} - elide_messages = [(start, f"... eliding lines {start+1}-{end+1} ...") for start, end in elide_line_ranges] + elide_messages = [ + (start, f"... eliding lines {start + 1}-{end + 1} ...") + for start, end in elide_line_ranges + ] out = [] for i, line in sorted( - elide_messages + [(i, line) for i, line in enumerate(file_contents.splitlines()) if i not in elide_lines] + elide_messages + + [ + (i, line) + for i, line in enumerate(file_contents.splitlines()) + if i not in elide_lines + ] ): - out.append(f"{i+1:6d} {line}") + out.append(f"{i + 1:6d} {line}") return "\n".join(out) @@ -270,7 +289,9 @@ def __init__(self, suffix: str = ""): if self.suffix: assert self.suffix.startswith(".") - def _find_breakpoints(self, lines: List[str], current_line: int, direction=1, max_added_lines: int = 30) -> int: + def _find_breakpoints( + self, lines: list[str], current_line: int, direction=1, max_added_lines: int = 30 + ) -> int: """Returns 1-based line number of breakpoint. This line is meant to still be included in the viewport. Args: @@ -337,13 +358,20 @@ def _find_breakpoints(self, lines: List[str], current_line: int, direction=1, ma # print(f"Score {score} for line {i_line} ({line})") # print(f"Best score {best_score} for line {best_breakpoint} ({lines[best_breakpoint-1]})") - if direction == 1 and best_breakpoint < current_line or direction == -1 and best_breakpoint > current_line: + if ( + direction == 1 + and best_breakpoint < current_line + or direction == -1 + and best_breakpoint > current_line + ): # We don't want to shrink the view port, so we return the current line return current_line return best_breakpoint - def expand_window(self, lines: List[str], start: int, stop: int, max_added_lines: int) -> Tuple[int, int]: + def expand_window( + self, lines: list[str], start: int, stop: int, max_added_lines: int + ) -> tuple[int, int]: """ Args: @@ -361,7 +389,9 @@ def expand_window(self, lines: List[str], start: int, stop: int, max_added_lines if max_added_lines <= 0: # Already at max range, no expansion return start, stop - new_start = self._find_breakpoints(lines, start, direction=-1, max_added_lines=max_added_lines) + new_start = self._find_breakpoints( + lines, start, direction=-1, max_added_lines=max_added_lines + ) new_stop = self._find_breakpoints(lines, stop, direction=1, max_added_lines=max_added_lines) # print(f"Expanded window is {new_start} to {new_stop}") assert new_start <= new_stop, (new_start, new_stop) @@ -410,11 +440,11 @@ def __call__( *, command: Command, path: str, - file_text: Optional[str] = None, - view_range: Optional[List[int]] = None, - old_str: Optional[str] = None, - new_str: Optional[str] = None, - insert_line: Optional[int] = None, + file_text: str | None = None, + view_range: list[int] | None = None, + old_str: str | None = None, + new_str: str | None = None, + insert_line: int | None = None, **kwargs, ): _path = Path(path) @@ -461,31 +491,40 @@ def validate_path(self, command: str, path: Path): return False # Check if path exists if not path.exists() and command != "create": - self.logs.append(f"The path {self._get_display_path(path)} does not exist. Please provide a valid path.") + self.logs.append( + f"The path {self._get_display_path(path)} does not exist. Please provide a valid path." + ) return False if path.exists() and command == "create": - self.logs.append(f"File already exists at: {self._get_display_path(path)}. Cannot overwrite files using command `create`.") + self.logs.append( + f"File already exists at: {self._get_display_path(path)}. Cannot overwrite files using command `create`." + ) return False # Check if the path points to a directory - if path.is_dir(): - if command != "view": - self.logs.append(f"The path {self._get_display_path(path)} is a directory and only the `view` command can be used on directories") - return False + if path.is_dir() and command != "view": + self.logs.append( + f"The path {self._get_display_path(path)} is a directory and only the `view` command can be used on directories" + ) + return False return True def create_file(self, path: Path, file_text: str): if not path.parent.exists(): - self.logs.append(f"The parent directory {self._get_display_path(path.parent)} does not exist. Please create it first.") + self.logs.append( + f"The parent directory {self._get_display_path(path.parent)} does not exist. Please create it first." + ) return self.write_file(path, file_text) self._file_history[path].append(file_text) self.logs.append(f"File created successfully at: {self._get_display_path(path)}") - def view(self, path: Path, view_range: Optional[List[int]] = None): + def view(self, path: Path, view_range: list[int] | None = None): """Implement the view command""" if path.is_dir(): if view_range: - self.logs.append("The `view_range` parameter is not allowed when `path` points to a directory.") + self.logs.append( + "The `view_range` parameter is not allowed when `path` points to a directory." + ) return # Hidden entries are skipped except `.github` (CI workflows are @@ -494,8 +533,8 @@ def view(self, path: Path, view_range: Optional[List[int]] = None): rf"find {shlex.quote(str(path))} -maxdepth 2 " r"\( -not -path '*/.*' -o -name .github -o -path '*/.github/*' \)", shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + check=False, + capture_output=True, ) # Use errors="replace" so non-UTF-8 bytes (e.g. GBK-encoded filenames on Windows) don't crash decoding. stdout = out.stdout.decode("utf-8", errors="replace") @@ -543,8 +582,10 @@ def view(self, path: Path, view_range: Optional[List[int]] = None): else: if path.suffix == ".py" and len(file_content) > MAX_RESPONSE_LEN and USE_FILEMAP: try: - filemap = Filemap().show_filemap(file_content, encoding=self._encoding or "utf-8") - except Exception: + filemap = Filemap().show_filemap( + file_content, encoding=self._encoding or "utf-8" + ) + except Exception: # noqa: BLE001, S110 — filemap is optional # If we fail to show the filemap, just show the truncated file content pass else: @@ -562,9 +603,11 @@ def view(self, path: Path, view_range: Optional[List[int]] = None): init_line = 1 # init_line is 1-based - self.logs.append(self._make_output(file_content, self._get_display_path(path), init_line=init_line)) + self.logs.append( + self._make_output(file_content, self._get_display_path(path), init_line=init_line) + ) - def str_replace(self, path: Path, old_str: str, new_str: Optional[str]): + def str_replace(self, path: Path, old_str: str, new_str: str | None): """Implement the str_replace command, which replaces old_str with new_str in the file content""" # Read the file content file_content = self.read_file(path).expandtabs() @@ -574,7 +617,9 @@ def str_replace(self, path: Path, old_str: str, new_str: Optional[str]): # Check if old_str is unique in the file occurrences = file_content.count(old_str) if occurrences == 0: - self.logs.append(f"No replacement was performed, old_str `{old_str}` did not appear verbatim in {self._get_display_path(path)}.") + self.logs.append( + f"No replacement was performed, old_str `{old_str}` did not appear verbatim in {self._get_display_path(path)}." + ) return elif occurrences > 1: file_content_lines = file_content.split("\n") @@ -585,14 +630,16 @@ def str_replace(self, path: Path, old_str: str, new_str: Optional[str]): return if new_str == old_str: - self.logs.append(f"No replacement was performed, old_str `{old_str}` is the same as new_str `{new_str}`.") + self.logs.append( + f"No replacement was performed, old_str `{old_str}` is the same as new_str `{new_str}`." + ) return pre_edit_lint = "" if USE_LINTER: try: pre_edit_lint = flake8(str(path)) - except Exception as e: + except Exception as e: # noqa: BLE001 — linting must never block an edit self.logs.append(f"Warning: Failed to run pre-edit linter on {path}: {e}") # Replace old_str with new_str @@ -605,12 +652,11 @@ def str_replace(self, path: Path, old_str: str, new_str: Optional[str]): if USE_LINTER: try: post_edit_lint = flake8(str(path)) - except Exception as e: + except Exception as e: # noqa: BLE001 — linting must never block an edit self.logs.append(f"Warning: Failed to run post-edit linter on {path}: {e}") epilogue = "" if post_edit_lint: - ... replacement_window_start_line = file_content.split(old_str)[0].count("\n") + 1 replacement_lines = len(new_str.split("\n")) replacement_window_end_line = replacement_window_start_line + replacement_lines - 1 @@ -630,15 +676,23 @@ def str_replace(self, path: Path, old_str: str, new_str: Optional[str]): # Create a snippet of the edited section replacement_line = file_content.split(old_str)[0].count("\n") start_line = max(1, replacement_line - SNIPPET_LINES) - end_line = min(replacement_line + SNIPPET_LINES + new_str.count("\n"), len(new_file_content.splitlines())) + end_line = min( + replacement_line + SNIPPET_LINES + new_str.count("\n"), + len(new_file_content.splitlines()), + ) start_line, end_line = WindowExpander(suffix=path.suffix).expand_window( - new_file_content.split("\n"), start_line, end_line, max_added_lines=MAX_WINDOW_EXPANSION_EDIT_CONFIRM + new_file_content.split("\n"), + start_line, + end_line, + max_added_lines=MAX_WINDOW_EXPANSION_EDIT_CONFIRM, ) snippet = "\n".join(new_file_content.split("\n")[start_line - 1 : end_line]) # Prepare the success message success_msg = f"The file {self._get_display_path(path)} has been edited. " - success_msg += self._make_output(snippet, f"a snippet of {self._get_display_path(path)}", start_line) + success_msg += self._make_output( + snippet, f"a snippet of {self._get_display_path(path)}", start_line + ) success_msg += "Review the changes and make sure they are as expected. Edit the file again if necessary." success_msg += epilogue @@ -658,7 +712,9 @@ def insert(self, path: Path, insert_line: int, new_str: str): return new_str_lines = new_str.split("\n") - new_file_text_lines = file_text_lines[:insert_line] + new_str_lines + file_text_lines[insert_line:] + new_file_text_lines = ( + file_text_lines[:insert_line] + new_str_lines + file_text_lines[insert_line:] + ) snippet_lines = ( file_text_lines[max(0, insert_line - SNIPPET_LINES) : insert_line] + new_str_lines @@ -691,7 +747,9 @@ def undo_edit(self, path: Path): old_text = self._file_history[path].pop() self.write_file(path, old_text) - self.logs.append(f"Last edit to {self._get_display_path(path)} undone successfully. {self._make_output(old_text, self._get_display_path(path))}") + self.logs.append( + f"Last edit to {self._get_display_path(path)} undone successfully. {self._make_output(old_text, self._get_display_path(path))}" + ) def read_file(self, path: Path): """Read the content of a file from a given path; raise a ToolError if an error occurs.""" @@ -710,7 +768,9 @@ def read_file(self, path: Path): else: break else: - self.logs.append(f"Ran into UnicodeDecodeError {exception} while trying to read {self._get_display_path(path)}") + self.logs.append( + f"Ran into UnicodeDecodeError {exception} while trying to read {self._get_display_path(path)}" + ) return return text @@ -718,8 +778,10 @@ def write_file(self, path: Path, file: str): """Write the content of a file to a given path; raise a ToolError if an error occurs.""" try: path.write_text(file, encoding=self._encoding or "utf-8") - except Exception as e: - self.logs.append(f"Ran into {e} while trying to write to {self._get_display_path(path)}") + except Exception as e: # noqa: BLE001 — reported to the agent as a tool message + self.logs.append( + f"Ran into {e} while trying to write to {self._get_display_path(path)}" + ) return def _make_output( @@ -733,19 +795,24 @@ def _make_output( file_content = maybe_truncate(file_content) if expand_tabs: file_content = file_content.expandtabs() - file_content = "\n".join([f"{i + init_line:6}\t{line}" for i, line in enumerate(file_content.split("\n"))]) - return f"Here's the result of running `cat -n` on {file_descriptor}:\n" + file_content + "\n" + file_content = "\n".join( + [f"{i + init_line:6}\t{line}" for i, line in enumerate(file_content.split("\n"))] + ) + return ( + f"Here's the result of running `cat -n` on {file_descriptor}:\n" + file_content + "\n" + ) + async def str_replace_editor( ctx: RunContext[CodeWikiDeps], working_dir: Literal["repo", "docs"], command: Literal["view", "create", "str_replace", "insert", "undo_edit"], - path: Optional[str] = None, - file: Optional[str] = None, - file_text: Optional[str] = None, + path: str | None = None, + file: str | None = None, + file_text: str | None = None, view_range: ViewRange = None, - old_str: Optional[str] = None, - new_str: Optional[str] = None, + old_str: str | None = None, + new_str: str | None = None, insert_line: InsertLine = None, ) -> str: """ @@ -815,5 +882,5 @@ async def str_replace_editor( * The `undo_edit` command will revert the last edit made to the file at `path` * Only `view` command is allowed when `working_dir` is `repo`. """.strip(), - takes_ctx=True + takes_ctx=True, ) diff --git a/codewiki/src/be/caw_toolkit.py b/codewiki/src/be/caw_toolkit.py index f409b6e5..cdb51832 100644 --- a/codewiki/src/be/caw_toolkit.py +++ b/codewiki/src/be/caw_toolkit.py @@ -56,7 +56,7 @@ async def _heartbeat(ctx: Context, work: asyncio.Task) -> None: total=None, message="sub-module generation in progress", ) - except Exception: + except Exception: # noqa: BLE001, S110 — progress reporting is best-effort pass @@ -82,7 +82,7 @@ class CawToolKit( def __init__( self, deps: CodeWikiDeps, - backend: "CawBackend", + backend: CawBackend, allow_subagent: bool, ) -> None: self._deps = deps @@ -108,8 +108,7 @@ async def read_code_components(self, component_ids: list[str]) -> str: results.append(f"# Component {cid} not found") else: results.append( - f"# Component {cid}:\n" - f"{self._deps.components[cid].source_code.strip()}\n\n" + f"# Component {cid}:\n{self._deps.components[cid].source_code.strip()}\n\n" ) return "\n".join(results) @@ -257,9 +256,7 @@ async def generate_sub_module_documentation( # event loop stays responsive while sub-agents run. A heartbeat task # emits MCP progress notifications so the CLI does not treat the long # tool call as a stalled / cancelled invocation. - work = asyncio.create_task( - asyncio.to_thread(self._run_sub_modules, sub_module_specs) - ) + work = asyncio.create_task(asyncio.to_thread(self._run_sub_modules, sub_module_specs)) heartbeat = asyncio.create_task(_heartbeat(ctx, work)) try: return await work @@ -300,7 +297,9 @@ def _run_sub_modules(self, sub_module_specs: dict[str, list[str]]) -> str: for sub_name, core_ids in final_specs.items(): indent = " " * deps.current_depth arrow = "└─" if deps.current_depth > 0 else "→" - logger.info("%s%s Generating documentation for sub-module: %s", indent, arrow, sub_name) + logger.info( + "%s%s Generating documentation for sub-module: %s", indent, arrow, sub_name + ) deps.current_module_name = sub_name deps.path_to_current_module.append(sub_name) @@ -341,5 +340,7 @@ def _run_sub_modules(self, sub_module_specs: dict[str, list[str]]) -> str: report = f"Saved documentations: {', '.join(saved) if saved else 'none'}." if missing: report += f" MISSING (generation did not produce these files): {', '.join(missing)}." - logger.warning("Sub-module documentation missing after generation: %s", ", ".join(missing)) + logger.warning( + "Sub-module documentation missing after generation: %s", ", ".join(missing) + ) return report diff --git a/codewiki/src/be/cluster_modules.py b/codewiki/src/be/cluster_modules.py index 080e6bd1..90263584 100644 --- a/codewiki/src/be/cluster_modules.py +++ b/codewiki/src/be/cluster_modules.py @@ -1,29 +1,33 @@ -from typing import List, Dict, Any, Callable, Optional -from collections import defaultdict import ast import logging import traceback +from collections import defaultdict +from collections.abc import Callable +from typing import Any + logger = logging.getLogger(__name__) from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.be.llm_services import call_llm from codewiki.src.be.module_naming import resolve_unique_name, sanitize_module_name +from codewiki.src.be.prompt_template import format_cluster_prompt, format_super_group_prompt from codewiki.src.be.utils import count_tokens from codewiki.src.config import ( - Config, DEFAULT_MAX_LEAF_NODES_PER_CLUSTER, DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING, + Config, ) -from codewiki.src.be.prompt_template import format_cluster_prompt, format_super_group_prompt -Completer = Callable[[str], Optional[str]] +Completer = Callable[[str], str | None] # When whole-repo mode is chosen but leaf entry points touch fewer than this # fraction of parsed files, warn that coverage depends on agent exploration. LOW_COVERAGE_RATIO = 0.5 -def format_potential_core_components(leaf_nodes: List[str], components: Dict[str, Node]) -> tuple[str, str]: +def format_potential_core_components( + leaf_nodes: list[str], components: dict[str, Node] +) -> tuple[str, str]: """ Format the potential core components into a string that can be used in the prompt. """ @@ -34,21 +38,21 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str valid_leaf_nodes.append(leaf_node) else: logger.warning(f"Skipping invalid leaf node '{leaf_node}' - not found in components") - - #group leaf nodes by file + + # group leaf nodes by file leaf_nodes_by_file = defaultdict(list) for leaf_node in valid_leaf_nodes: leaf_nodes_by_file[components[leaf_node].relative_path].append(leaf_node) potential_core_components = "" potential_core_components_with_code = "" - for file, leaf_nodes in dict(sorted(leaf_nodes_by_file.items())).items(): + for file, file_nodes in dict(sorted(leaf_nodes_by_file.items())).items(): header = f"# {file}" - if all(components[n].component_type == "artifact" for n in leaf_nodes): - header += f" (artifact: {components[leaf_nodes[0]].artifact_class or 'config'})" + if all(components[n].component_type == "artifact" for n in file_nodes): + header += f" (artifact: {components[file_nodes[0]].artifact_class or 'config'})" potential_core_components += f"{header}\n" potential_core_components_with_code += f"{header}\n" - for leaf_node in leaf_nodes: + for leaf_node in file_nodes: potential_core_components += f"\t{leaf_node}\n" potential_core_components_with_code += f"\t{leaf_node}\n" potential_core_components_with_code += f"{components[leaf_node].source_code}\n" @@ -56,9 +60,7 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str return potential_core_components, potential_core_components_with_code -def get_clustering_input_token_count( - leaf_nodes: List[str], components: Dict[str, Node] -) -> int: +def get_clustering_input_token_count(leaf_nodes: list[str], components: dict[str, Node]) -> int: """Count the tokens used to decide whether a module needs clustering.""" _, potential_core_components_with_code = format_potential_core_components( leaf_nodes, components @@ -66,16 +68,14 @@ def get_clustering_input_token_count( return count_tokens(potential_core_components_with_code) -def _cluster_batch_fits(node_ids: List[str], config: Config) -> bool: +def _cluster_batch_fits(node_ids: list[str], config: Config) -> bool: """Whether a single LLM clustering call can handle these nodes. The clustering response must re-emit every component ID verbatim, so the joined ID list is a direct proxy for output size; keep 2x headroom under max_tokens for dict syntax, module names/paths, and preamble. """ - max_nodes = getattr( - config, "max_leaf_nodes_per_cluster", DEFAULT_MAX_LEAF_NODES_PER_CLUSTER - ) + max_nodes = getattr(config, "max_leaf_nodes_per_cluster", DEFAULT_MAX_LEAF_NODES_PER_CLUSTER) if len(node_ids) > max_nodes: return False output_budget = max(2048, config.max_tokens // 2) @@ -83,10 +83,10 @@ def _cluster_batch_fits(node_ids: List[str], config: Config) -> bool: def partition_leaf_nodes_by_structure( - leaf_nodes: List[str], - components: Dict[str, Node], - fits: Callable[[List[str]], bool], -) -> List[List[str]]: + leaf_nodes: list[str], + components: dict[str, Node], + fits: Callable[[list[str]], bool], +) -> list[list[str]]: """Partition leaf nodes into batches that each satisfy ``fits``. Splits along the directory structure of the nodes' relative paths, then @@ -105,10 +105,10 @@ def partition_leaf_nodes_by_structure( if not valid or fits(valid): return [valid] - def path_parts(node: str) -> List[str]: + def path_parts(node: str) -> list[str]: return components[node].relative_path.strip("/").split("/") - def chunk(nodes: List[str]) -> List[List[str]]: + def chunk(nodes: list[str]) -> list[list[str]]: # Nodes that share one directory/file and still don't fit can only be # cut into fixed-size slices. size = len(nodes) @@ -120,13 +120,13 @@ def chunk(nodes: List[str]) -> List[List[str]]: len(nodes), size, ) - return [nodes[i:i + size] for i in range(0, len(nodes), size)] + return [nodes[i : i + size] for i in range(0, len(nodes), size)] - def split(nodes: List[str], depth: int) -> List[List[str]]: + def split(nodes: list[str], depth: int) -> list[list[str]]: by_prefix = defaultdict(list) for node in nodes: by_prefix["/".join(path_parts(node)[:depth])].append(node) - groups: List[List[str]] = [] + groups: list[list[str]] = [] for prefix in sorted(by_prefix): sub = by_prefix[prefix] if fits(sub): @@ -137,8 +137,8 @@ def split(nodes: List[str], depth: int) -> List[List[str]]: groups.extend(chunk(sub)) return groups - batches: List[List[str]] = [] - current: List[str] = [] + batches: list[list[str]] = [] + current: list[str] = [] for group in split(valid, 1): if not current: current = group @@ -153,21 +153,23 @@ def split(nodes: List[str], depth: int) -> List[List[str]]: def _cluster_via_llm( - leaf_nodes: List[str], - components: Dict[str, Node], + leaf_nodes: list[str], + components: dict[str, Node], config: Config, - current_module_tree: Dict[str, Any], - current_module_name: Optional[str], + current_module_tree: dict[str, Any], + current_module_name: str | None, module_label: str, - completer: Optional[Completer], -) -> Dict[str, Any]: + completer: Completer | None, +) -> dict[str, Any]: """Run one clustering LLM call over these nodes. Returns {} for any empty, malformed, or non-dict response instead of raising, so callers can fall back gracefully. """ potential_core_components, _ = format_potential_core_components(leaf_nodes, components) - prompt = format_cluster_prompt(potential_core_components, current_module_tree, current_module_name) + prompt = format_cluster_prompt( + potential_core_components, current_module_tree, current_module_name + ) if completer is not None: response = completer(prompt) else: @@ -191,14 +193,16 @@ def _cluster_via_llm( ) return {} - response_content = response.split("")[1].split("")[0] + response_content = response.split("")[1].split("")[ + 0 + ] module_tree = eval(response_content) if not isinstance(module_tree, dict): logger.error(f"Invalid module tree format - expected dict, got {type(module_tree)}") return {} - except Exception as e: + except Exception as e: # noqa: BLE001 — a failed LLM call must not abort clustering logger.warning( "Failed to parse LLM clustering response for %s; falling back. " "Error: %s. Response preview: %s...", @@ -212,7 +216,7 @@ def _cluster_via_llm( return module_tree -def _merge_module_trees(target: Dict[str, Any], addition: Dict[str, Any]) -> None: +def _merge_module_trees(target: dict[str, Any], addition: dict[str, Any]) -> None: """Merge one batch's module dict into the accumulated tree in place. Batches cluster independently, so two of them may propose the same module @@ -240,15 +244,14 @@ def _merge_module_trees(target: Dict[str, Any], addition: Dict[str, Any]) -> Non paths = [existing.get("path", ""), info.get("path", "")] existing["path"] = _common_path_prefix(paths) if all(paths) else "" logger.info( - "Module '%s' was produced by multiple clustering batches; merged " - "into %d components.", + "Module '%s' was produced by multiple clustering batches; merged into %d components.", name, len(merged), ) def _batch_fallback_name( - batch: List[str], components: Dict[str, Node], existing: Dict[str, Any] + batch: list[str], components: dict[str, Node], existing: dict[str, Any] ) -> str: """Directory-derived module name for a batch whose LLM clustering failed.""" prefix = _common_path_prefix( @@ -259,14 +262,14 @@ def _batch_fallback_name( def cluster_modules( - leaf_nodes: List[str], - components: Dict[str, Node], + leaf_nodes: list[str], + components: dict[str, Node], config: Config, - current_module_tree: dict[str, Any] = {}, - current_module_name: str = None, - current_module_path: List[str] = [], - completer: Optional[Completer] = None, -) -> Dict[str, Any]: + current_module_tree: dict[str, Any] | None = None, + current_module_name: str | None = None, + current_module_path: list[str] | None = None, + completer: Completer | None = None, +) -> dict[str, Any]: """ Cluster the potential core components into modules. @@ -277,8 +280,10 @@ def cluster_modules( subscription-mode (caw) routing. If ``None``, falls back to ``call_llm`` for backward compatibility with direct callers. """ - _, potential_core_components_with_code = ( - format_potential_core_components(leaf_nodes, components) + current_module_tree = {} if current_module_tree is None else current_module_tree + current_module_path = [] if current_module_path is None else current_module_path + _, potential_core_components_with_code = format_potential_core_components( + leaf_nodes, components ) input_tokens = count_tokens(potential_core_components_with_code) threshold = config.max_token_per_module @@ -409,15 +414,17 @@ def cluster_modules( for module_name, module_info in module_tree.items(): sub_leaf_nodes = module_info.get("components", []) - + # Filter sub_leaf_nodes to ensure they exist in components valid_sub_leaf_nodes = [] for node in sub_leaf_nodes: if node in components: valid_sub_leaf_nodes.append(node) else: - logger.warning(f"Skipping invalid sub leaf node '{node}' in module '{module_name}' - not found in components") - + logger.warning( + f"Skipping invalid sub leaf node '{node}' in module '{module_name}' - not found in components" + ) + current_module_path.append(module_name) module_info["children"] = {} module_info["children"] = cluster_modules( @@ -434,7 +441,7 @@ def cluster_modules( return module_tree -def _common_path_prefix(paths: List[str]) -> str: +def _common_path_prefix(paths: list[str]) -> str: """Longest common directory prefix of the given relative paths.""" split_paths = [p.strip("/").split("/") for p in paths if p] if not split_paths: @@ -448,7 +455,7 @@ def _common_path_prefix(paths: List[str]) -> str: return "/".join(common) -def _parse_super_group_response(response: Optional[str]) -> Optional[Dict[str, Any]]: +def _parse_super_group_response(response: str | None) -> dict[str, Any] | None: if not response: logger.warning( "Empty super-grouping response (provider returned no content, " @@ -466,7 +473,7 @@ def _parse_super_group_response(response: Optional[str]) -> Optional[Dict[str, A grouping = ast.literal_eval( response.split("")[1].split("")[0] ) - except Exception as e: + except Exception as e: # noqa: BLE001 — a failed LLM call must not abort clustering logger.warning( "Failed to parse super-grouping response; keeping the flat module " "tree. Error: %s. Response preview: %s...", @@ -476,8 +483,7 @@ def _parse_super_group_response(response: Optional[str]) -> Optional[Dict[str, A return None if not isinstance(grouping, dict): logger.warning( - "Invalid super-grouping format - expected dict, got %s; keeping the " - "flat module tree.", + "Invalid super-grouping format - expected dict, got %s; keeping the flat module tree.", type(grouping), ) return None @@ -485,10 +491,10 @@ def _parse_super_group_response(response: Optional[str]) -> Optional[Dict[str, A def super_group_modules( - module_tree: Dict[str, Any], + module_tree: dict[str, Any], config: Config, - completer: Optional[Completer] = None, -) -> Dict[str, Any]: + completer: Completer | None = None, +) -> dict[str, Any]: """ Group a flat top level of modules into higher-level architectural subsystems. @@ -501,14 +507,11 @@ def super_group_modules( config, "min_modules_for_super_grouping", DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING ) if min_modules <= 0: - logger.info( - "Super-grouping disabled (min_modules_for_super_grouping=%d).", min_modules - ) + logger.info("Super-grouping disabled (min_modules_for_super_grouping=%d).", min_modules) return module_tree if len(module_tree) <= min_modules: logger.info( - "Skipping super-grouping: %d top-level modules fit within the " - "%d-module threshold.", + "Skipping super-grouping: %d top-level modules fit within the %d-module threshold.", len(module_tree), min_modules, ) @@ -516,8 +519,7 @@ def super_group_modules( prompt = format_super_group_prompt(module_tree) logger.info( - "Requesting super-grouping of %d top-level modules into architectural " - "subsystems.", + "Requesting super-grouping of %d top-level modules into architectural subsystems.", len(module_tree), ) if completer is not None: @@ -532,13 +534,12 @@ def super_group_modules( # Validate assignments: unknown modules are dropped, duplicates keep their # first assignment, unassigned modules stay at the top level. assigned = set() - subsystems: Dict[str, List[str]] = {} + subsystems: dict[str, list[str]] = {} for subsystem_name, info in grouping.items(): members = info.get("modules") if isinstance(info, dict) else None if not isinstance(members, list): logger.warning( - "Skipping subsystem '%s' in super-grouping response: no valid " - "'modules' list.", + "Skipping subsystem '%s' in super-grouping response: no valid 'modules' list.", subsystem_name, ) continue @@ -553,8 +554,7 @@ def super_group_modules( continue if member in assigned: logger.warning( - "Module '%s' assigned to multiple subsystems; keeping its " - "first assignment.", + "Module '%s' assigned to multiple subsystems; keeping its first assignment.", member, ) continue @@ -576,7 +576,7 @@ def super_group_modules( ) return module_tree - result: Dict[str, Any] = {} + result: dict[str, Any] = {} for subsystem_name, members in subsystems.items(): if len(members) == 1: result[members[0]] = module_tree[members[0]] @@ -599,8 +599,7 @@ def super_group_modules( result[name] = module_tree[name] logger.info( - "Super-grouping consolidated %d top-level modules into %d entries " - "(%d subsystems).", + "Super-grouping consolidated %d top-level modules into %d entries (%d subsystems).", len(module_tree), len(result), len(subsystems), @@ -618,11 +617,11 @@ def super_group_modules( ARTIFACT_MIN_SHARE = 0.8 -def collect_module_tree_component_ids(module_tree: Dict[str, Any]) -> set: +def collect_module_tree_component_ids(module_tree: dict[str, Any]) -> set: """Return every component id referenced anywhere in ``module_tree``.""" ids: set = set() - def _walk(tree: Dict[str, Any]) -> None: + def _walk(tree: dict[str, Any]) -> None: for module_info in tree.values(): if not isinstance(module_info, dict): continue @@ -636,11 +635,11 @@ def _walk(tree: Dict[str, Any]) -> None: def ensure_artifact_module( - module_tree: Dict[str, Any], - leaf_nodes: List[str], - components: Dict[str, Node], + module_tree: dict[str, Any], + leaf_nodes: list[str], + components: dict[str, Node], min_share: float = ARTIFACT_MIN_SHARE, -) -> Dict[str, Any]: +) -> dict[str, Any]: """Guarantee that artifact leaf nodes are documented. Clustering is an LLM call and may drop or scatter artifact nodes despite @@ -652,8 +651,7 @@ def ensure_artifact_module( if not module_tree: return module_tree artifact_leaves = [ - n for n in leaf_nodes - if n in components and components[n].component_type == "artifact" + n for n in leaf_nodes if n in components and components[n].component_type == "artifact" ] if not artifact_leaves: return module_tree @@ -662,7 +660,9 @@ def ensure_artifact_module( share = 1.0 - len(unassigned) / len(artifact_leaves) logger.info( "Artifact coverage after clustering: %d/%d artifact leaf nodes assigned (%.0f%%)", - len(artifact_leaves) - len(unassigned), len(artifact_leaves), share * 100, + len(artifact_leaves) - len(unassigned), + len(artifact_leaves), + share * 100, ) if not unassigned or share >= min_share: return module_tree @@ -685,6 +685,9 @@ def _order(node_id: str): } logger.info( "Artifact coverage %.0f%% < %.0f%%; inserted top-level module '%s' with %d components", - share * 100, min_share * 100, name, len(unassigned), + share * 100, + min_share * 100, + name, + len(unassigned), ) return module_tree diff --git a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py index c30b3a7c..6731e666 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py @@ -6,12 +6,11 @@ """ import fnmatch -import os import logging +import os import shutil import subprocess from pathlib import Path -from typing import Dict, List, Optional from pathspec import GitIgnoreSpec @@ -21,7 +20,6 @@ DEFAULT_INCLUDE_PATTERNS, ) - logger = logging.getLogger(__name__) @@ -52,6 +50,7 @@ def _load_git_ignored_paths(self) -> bool: try: root_result = subprocess.run( [git_path, "-C", str(self.repo_dir), "rev-parse", "--show-toplevel"], + check=False, capture_output=True, text=True, timeout=10, @@ -139,8 +138,7 @@ def _load_fallback_specs(self) -> None: def is_ignored(self, relative_path: str, is_dir: bool) -> bool: """Return whether a repository-relative path should be ignored.""" normalized = relative_path.replace("\\", "/") - if normalized.startswith("./"): - normalized = normalized[2:] + normalized = normalized.removeprefix("./") if normalized in ("", "."): return False @@ -178,8 +176,8 @@ def is_ignored(self, relative_path: str, is_dir: bool) -> bool: class RepoAnalyzer: def __init__( self, - include_patterns: Optional[List[str]] = None, - exclude_patterns: Optional[List[str]] = None, + include_patterns: list[str] | None = None, + exclude_patterns: list[str] | None = None, use_gitignore: bool = True, ) -> None: # Include patterns: if specified, use ONLY those patterns (replaces defaults) @@ -193,9 +191,9 @@ def __init__( self.user_exclude_patterns = list(exclude_patterns) if exclude_patterns is not None else [] self.exclude_patterns = self.default_exclude_patterns + self.user_exclude_patterns self.use_gitignore = use_gitignore - self._gitignore_filter: Optional[GitIgnoreFilter] = None + self._gitignore_filter: GitIgnoreFilter | None = None - def analyze_repository_structure(self, repo_dir: str) -> Dict: + def analyze_repository_structure(self, repo_dir: str) -> dict: self._gitignore_filter = GitIgnoreFilter(Path(repo_dir)) if self.use_gitignore else None file_tree = self._build_file_tree(repo_dir) return { @@ -206,8 +204,8 @@ def analyze_repository_structure(self, repo_dir: str) -> Dict: }, } - def _build_file_tree(self, repo_dir: str) -> Dict: - def build_tree(path: Path, base_path: Path) -> Optional[Dict]: + def _build_file_tree(self, repo_dir: str) -> dict: + def build_tree(path: Path, base_path: Path) -> dict | None: relative_path = path.relative_to(base_path) relative_path_str = str(relative_path) @@ -264,7 +262,7 @@ def build_tree(path: Path, base_path: Path) -> Optional[Dict]: return build_tree(Path(repo_dir), Path(repo_dir)) @staticmethod - def _matches_any(path: str, filename: str, patterns: List[str]) -> bool: + def _matches_any(path: str, filename: str, patterns: list[str]) -> bool: for pattern in patterns: if fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(filename, pattern): return True @@ -301,9 +299,7 @@ def _should_exclude_path(self, path: str, filename: str, is_dir: bool = False) - path, filename, self.default_exclude_patterns ): return True - if self._gitignore_filter and self._gitignore_filter.is_ignored(path, is_dir): - return True - return False + return bool(self._gitignore_filter and self._gitignore_filter.is_ignored(path, is_dir)) def _should_include_file(self, path: str, filename: str) -> bool: if not self.include_patterns: @@ -313,12 +309,12 @@ def _should_include_file(self, path: str, filename: str) -> bool: return True return False - def _count_files(self, tree: Dict) -> int: + def _count_files(self, tree: dict) -> int: if tree["type"] == "file": return 1 return sum(self._count_files(child) for child in tree.get("children", [])) - def _calculate_size(self, tree: Dict) -> float: + def _calculate_size(self, tree: dict) -> float: if tree["type"] == "file": return tree.get("_size_bytes", 0) / 1024 return sum(self._calculate_size(child) for child in tree.get("children", [])) diff --git a/codewiki/src/be/dependency_analyzer/analyzers/artifact.py b/codewiki/src/be/dependency_analyzer/analyzers/artifact.py index 7bc6b118..7c2b175f 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/artifact.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/artifact.py @@ -34,9 +34,10 @@ import os import posixpath import re +from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Iterable, Optional +from typing import Any from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node from codewiki.src.be.dependency_analyzer.utils.patterns import ARTIFACT_LOCKFILES @@ -72,15 +73,56 @@ # Unit names that should be kept first when a file has more units than the cap. PRIORITY_UNITS = { - "all", "build", "test", "tests", "install", "lint", "release", "dev", - "start", "ci", "docker", "publish", "check", "format", "deploy", + "all", + "build", + "test", + "tests", + "install", + "lint", + "release", + "dev", + "start", + "ci", + "docker", + "publish", + "check", + "format", + "deploy", } _PROSE_EXTS = {".md", ".mdx", ".rst", ".txt"} _CODE_OR_SCRIPT_EXTS = ( - "py", "sh", "bash", "js", "mjs", "cjs", "ts", "tsx", "jsx", "rb", "ps1", - "mk", "toml", "yaml", "yml", "json", "cfg", "ini", "java", "kt", "go", - "rs", "c", "cc", "cpp", "h", "hpp", "cs", "php", "proto", "fbs", + "py", + "sh", + "bash", + "js", + "mjs", + "cjs", + "ts", + "tsx", + "jsx", + "rb", + "ps1", + "mk", + "toml", + "yaml", + "yml", + "json", + "cfg", + "ini", + "java", + "kt", + "go", + "rs", + "c", + "cc", + "cpp", + "h", + "hpp", + "cs", + "php", + "proto", + "fbs", ) PATH_TOKEN_RE = re.compile( r"(? bool: # --------------------------------------------------------------------------- # _DROP_SEGMENTS = { - "docs", "doc", "node_modules", "vendor", "third_party", "dist", ".git", - "fixtures", "fixture", "testdata", "test_data", "__snapshots__", + "docs", + "doc", + "node_modules", + "vendor", + "third_party", + "dist", + ".git", + "fixtures", + "fixture", + "testdata", + "test_data", + "__snapshots__", } _CI_NAMES = { - ".gitlab-ci.yml", "Jenkinsfile", ".travis.yml", "azure-pipelines.yml", - "appveyor.yml", ".appveyor.yml", "bitbucket-pipelines.yml", "cloudbuild.yaml", - "cloudbuild.yml", ".drone.yml", + ".gitlab-ci.yml", + "Jenkinsfile", + ".travis.yml", + "azure-pipelines.yml", + "appveyor.yml", + ".appveyor.yml", + "bitbucket-pipelines.yml", + "cloudbuild.yaml", + "cloudbuild.yml", + ".drone.yml", } _MANIFEST_NAMES = { - "package.json", "pyproject.toml", "setup.py", "setup.cfg", "Cargo.toml", - "go.mod", "Gemfile", "pnpm-workspace.yaml", "lerna.json", "nx.json", - "turbo.json", "composer.json", "pom.xml", "build.gradle", "build.gradle.kts", - "settings.gradle", "settings.gradle.kts", "Package.swift", "pubspec.yaml", - "Pipfile", "environment.yml", "environment.yaml", "MANIFEST.in", "Procfile", - "conda.yaml", "conda.yml", + "package.json", + "pyproject.toml", + "setup.py", + "setup.cfg", + "Cargo.toml", + "go.mod", + "Gemfile", + "pnpm-workspace.yaml", + "lerna.json", + "nx.json", + "turbo.json", + "composer.json", + "pom.xml", + "build.gradle", + "build.gradle.kts", + "settings.gradle", + "settings.gradle.kts", + "Package.swift", + "pubspec.yaml", + "Pipfile", + "environment.yml", + "environment.yaml", + "MANIFEST.in", + "Procfile", + "conda.yaml", + "conda.yml", } _MANIFEST_EXTS = {".gemspec", ".csproj", ".fsproj", ".vbproj", ".sln", ".podspec"} _MANIFEST_RES = [re.compile(r"^requirements[\w.-]*\.txt$"), re.compile(r"^tsconfig[\w.-]*\.json$")] -_PACKAGING_EXTS = {".spec", ".service", ".socket", ".timer", ".plist", ".nuspec", ".wxs", ".desktop"} +_PACKAGING_EXTS = { + ".spec", + ".service", + ".socket", + ".timer", + ".plist", + ".nuspec", + ".wxs", + ".desktop", +} _PACKAGING_TOPS = {"debian", "rpm", "installer", "pkg", "packaging"} _PACKAGING_PACKAGES_NAMES = { - "control", "rules", "changelog", "postinst", "prerm", "postrm", "preinst", "copyright", + "control", + "rules", + "changelog", + "postinst", + "prerm", + "postrm", + "preinst", + "copyright", } _BUILD_NAMES = { - "Makefile", "GNUmakefile", "makefile", "CMakeLists.txt", "Rakefile", "BUILD", - "BUILD.gn", "BUILD.bazel", "WORKSPACE", "DEPS", "meson.build", "SConstruct", - "SConscript", "build.xml", "gulpfile.js", "Gruntfile.js", "Herebyfile.mjs", + "Makefile", + "GNUmakefile", + "makefile", + "CMakeLists.txt", + "Rakefile", + "BUILD", + "BUILD.gn", + "BUILD.bazel", + "WORKSPACE", + "DEPS", + "meson.build", + "SConstruct", + "SConscript", + "build.xml", + "gulpfile.js", + "Gruntfile.js", + "Herebyfile.mjs", } _BUILD_EXTS = {".gn", ".gni", ".gradle", ".rake", ".mk", ".cmake", ".bzl", ".ninja"} _BUILD_CONFIG_RE = re.compile(r"^(webpack|rollup|vite|esbuild|tsup|babel)\.config\.[cm]?[jt]s$") _BUILD_TOPS = {"build", "rakelib", "cmake"} _TEST_INFRA_NAMES = { - "pytest.ini", "tox.ini", "conftest.py", ".coveragerc", "codecov.yml", - "karma.conf.js", ".nycrc", "noxfile.py", + "pytest.ini", + "tox.ini", + "conftest.py", + ".coveragerc", + "codecov.yml", + "karma.conf.js", + ".nycrc", + "noxfile.py", } -_TEST_INFRA_RE = re.compile(r"^(jest|vitest|playwright|cypress|wdio|mocha)\.(config|workspace)\.[\w.]+$") +_TEST_INFRA_RE = re.compile( + r"^(jest|vitest|playwright|cypress|wdio|mocha)\.(config|workspace)\.[\w.]+$" +) _SCHEMA_EXTS = {".proto", ".fbs", ".avsc", ".thrift", ".graphql", ".gql", ".capnp", ".xsd", ".wsdl"} _SCHEMA_RE = re.compile(r"^(openapi|swagger)[\w.-]*\.(ya?ml|json)$") _CONFIG_EXTS = { - ".toml", ".yml", ".yaml", ".ini", ".cfg", ".conf", ".options", ".properties", - ".tf", ".nix", ".editorconfig", ".env", + ".toml", + ".yml", + ".yaml", + ".ini", + ".cfg", + ".conf", + ".options", + ".properties", + ".tf", + ".nix", + ".editorconfig", + ".env", } _CONFIG_TOPS = {"config", "configs", "conf", "etc", "settings", ".github"} _SCRIPT_EXTS = {".sh", ".bash", ".zsh", ".ps1", ".bat", ".cmd"} @@ -192,7 +336,7 @@ def classify_artifact( size: int, opts: ArtifactOptions, first_line: str | None = None, -) -> Optional[str]: +) -> str | None: """Return the artifact class of ``rel_path`` or ``None`` when it is not one. ``first_line`` is only needed for extension-less files under ``bin/`` or @@ -210,7 +354,9 @@ def classify_artifact( # ---- hard drops ------------------------------------------------------- if size <= 0 or name in ARTIFACT_LOCKFILES: return None - if any(seg in _DROP_SEGMENTS for seg in segs[:-1]) and not (opts.with_prose and top in {"docs", "doc"}): + if any(seg in _DROP_SEGMENTS for seg in segs[:-1]) and not ( + opts.with_prose and top in {"docs", "doc"} + ): return None for pat in opts.exclude_patterns or []: if fnmatch.fnmatch(rel, pat) or fnmatch.fnmatch(name, pat): @@ -243,7 +389,11 @@ def classify_artifact( return "ci" # ---- container ----------------------------------------------------------- - if fnmatch.fnmatch(name, "Dockerfile*") or lower.endswith(".dockerfile") or lower == "containerfile": + if ( + fnmatch.fnmatch(name, "Dockerfile*") + or lower.endswith(".dockerfile") + or lower == "containerfile" + ): return "container" if re.match(r"^(docker-)?compose[.\w-]*\.ya?ml$", lower): return "container" @@ -255,12 +405,29 @@ def classify_artifact( return "container" # ---- manifest (before packaging: packages/*/package.json is a manifest) -- - if name in _MANIFEST_NAMES or ext in _MANIFEST_EXTS or any(r.match(name) for r in _MANIFEST_RES): + if ( + name in _MANIFEST_NAMES + or ext in _MANIFEST_EXTS + or any(r.match(name) for r in _MANIFEST_RES) + ): return "manifest" # Source files are code, not artifacts, unless an explicit name rule above # (setup.py, conftest.py, noxfile.py, *.conf.py) already claimed them. - if ext in {".py", ".js", ".ts", ".java", ".rb", ".go", ".rs", ".c", ".cpp", ".cs", ".php", ".kt"}: + if ext in { + ".py", + ".js", + ".ts", + ".java", + ".rb", + ".go", + ".rs", + ".c", + ".cpp", + ".cs", + ".php", + ".kt", + }: if name in _TEST_INFRA_NAMES: return "test_infra" if name in _BUILD_NAMES or _BUILD_CONFIG_RE.match(name) or _TEST_INFRA_RE.match(name): @@ -272,11 +439,18 @@ def classify_artifact( return "packaging" if top in _PACKAGING_TOPS: return "packaging" - if top == "packages" and (ext in {".sh", ".conf", ".spec", ".service"} or name in _PACKAGING_PACKAGES_NAMES): + if top == "packages" and ( + ext in {".sh", ".conf", ".spec", ".service"} or name in _PACKAGING_PACKAGES_NAMES + ): return "packaging" # ---- build ------------------------------------------------------------------- - if name in _BUILD_NAMES or ext in _BUILD_EXTS or _BUILD_CONFIG_RE.match(name) or lower.startswith(".babelrc"): + if ( + name in _BUILD_NAMES + or ext in _BUILD_EXTS + or _BUILD_CONFIG_RE.match(name) + or lower.startswith(".babelrc") + ): return "build" if top in _BUILD_TOPS and ext in _BUILD_EXTS: return "build" @@ -292,18 +466,23 @@ def classify_artifact( return "schema" # ---- config ------------------------------------------------------------------ - if ext in _CONFIG_EXTS or ( + is_config_like = ext in _CONFIG_EXTS or ( name.startswith(".") and ext in {"", ".json", ".yml", ".yaml", ".js", ".cjs"} - ): - if depth <= 1 or top in _CONFIG_TOPS: - return "config" + ) + if is_config_like and (depth <= 1 or top in _CONFIG_TOPS): + return "config" if ext in {".json", ".xml"} and (depth == 0 or top in {"config", "configs", "conf", "etc"}): return "config" # ---- script ------------------------------------------------------------------ if ext in _SCRIPT_EXTS and (depth <= 2 or top in _SCRIPT_TOPS): return "script" - if ext == "" and top in {"bin", "scripts"} and first_line is not None and first_line.startswith("#!"): + if ( + ext == "" + and top in {"bin", "scripts"} + and first_line is not None + and first_line.startswith("#!") + ): return "script" return None @@ -321,11 +500,11 @@ def _line_of(text: str, offset: int) -> int: def _yaml_children(text: str, top_key: str) -> list[tuple[str, int, int]]: """Return ``(name, start, end)`` offsets of the direct children of a top-level YAML mapping such as ``jobs:`` or ``services:`` without a YAML library.""" - m = re.search(rf"^{re.escape(top_key)}:[ \t]*(?:#.*)?$", text, re.M) + m = re.search(rf"^{re.escape(top_key)}:[ \t]*(?:#.*)?$", text, re.MULTILINE) if not m: return [] body_start = m.end() - nxt = re.compile(r"^\S", re.M).search(text, body_start + 1) + nxt = re.compile(r"^\S", re.MULTILINE).search(text, body_start + 1) body_end = nxt.start() if nxt else len(text) body = text[body_start:body_end] indent = None @@ -335,7 +514,7 @@ def _yaml_children(text: str, top_key: str) -> list[tuple[str, int, int]]: break if not indent: return [] - child_re = re.compile(rf"^ {{{indent}}}([A-Za-z_\"'][\w.\"'-]*):[ \t]*(?:#.*)?$", re.M) + child_re = re.compile(rf"^ {{{indent}}}([A-Za-z_\"'][\w.\"'-]*):[ \t]*(?:#.*)?$", re.MULTILINE) matches = list(child_re.finditer(body)) units: list[tuple[str, int, int]] = [] for i, cm in enumerate(matches): @@ -346,7 +525,9 @@ def _yaml_children(text: str, top_key: str) -> list[tuple[str, int, int]]: def _dockerfile_units(text: str) -> list[tuple[str, int, int]]: - from_re = re.compile(r"^FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", re.I | re.M) + from_re = re.compile( + r"^FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", re.IGNORECASE | re.MULTILINE + ) matches = list(from_re.finditer(text)) if not matches or (len(matches) == 1 and not matches[0].group(2)): return [] @@ -377,13 +558,21 @@ def _makefile_units(text: str) -> list[tuple[str, int, int]]: i = 0 while i < len(lines): m = target_re.match(lines[i]) - if not m or "%" in m.group(1) or "$" in m.group(1) or m.group(1).startswith(".") \ - or re.match(r"\s*[?+!]?=", m.group("rest")): + if ( + not m + or "%" in m.group(1) + or "$" in m.group(1) + or m.group(1).startswith(".") + or re.match(r"\s*[?+!]?=", m.group("rest")) + ): i += 1 continue name = m.group(1) j = i + 1 - while j < len(lines) and (lines[j].startswith("\t") or (lines[j].strip() == "" and j + 1 < len(lines) and lines[j + 1].startswith("\t"))): + while j < len(lines) and ( + lines[j].startswith("\t") + or (lines[j].strip() == "" and j + 1 < len(lines) and lines[j + 1].startswith("\t")) + ): j += 1 end = offsets[j] - 1 if j < len(lines) else len(text) if name not in seen: @@ -435,7 +624,11 @@ def _entry_point_units(text: str, kind: str) -> list[tuple[str, int, int, str, s except Exception: # noqa: BLE001 - fall back to regex below tables = {} if not tables: - for sec in re.finditer(r"^\[(?:project\.(?:gui-)?scripts|tool\.poetry\.scripts)\]\s*$(.*?)(?=^\[|\Z)", text, re.M | re.S): + for sec in re.finditer( + r"^\[(?:project\.(?:gui-)?scripts|tool\.poetry\.scripts)\]\s*$(.*?)(?=^\[|\Z)", + text, + re.MULTILINE | re.DOTALL, + ): for line in sec.group(1).splitlines(): lm = re.match(r"^\s*([\w.-]+)\s*=\s*[\"']([\w.]+):([\w.]+)[\"']", line) if lm: @@ -444,12 +637,12 @@ def _entry_point_units(text: str, kind: str) -> list[tuple[str, int, int, str, s if not isinstance(target, str) or ":" not in target: continue module, func = target.split(":", 1) - m = re.search(rf"^\s*{re.escape(name)}\s*=", text, re.M) + m = re.search(rf"^\s*{re.escape(name)}\s*=", text, re.MULTILINE) start = m.start() if m else 0 snippet = f'{name} = "{target}"' results.append((str(name), start, start + len(snippet), module.strip(), func.strip())) else: # setup.cfg - m = re.search(r"^console_scripts\s*=\s*$(.*?)(?=^\S|\Z)", text, re.M | re.S) + m = re.search(r"^console_scripts\s*=\s*$(.*?)(?=^\S|\Z)", text, re.MULTILINE | re.DOTALL) if m: for line in m.group(1).splitlines(): lm = re.match(r"^\s*([\w.-]+)\s*=\s*([\w.]+):([\w.]+)", line) @@ -502,8 +695,12 @@ def resolve_path(self, ref: str, base_dir: str = "") -> list[str]: return [] if ref.startswith(("http://", "https://", "/", "..", "~")) or "://" in ref: return [] - ref = ref[2:] if ref.startswith("./") else ref - cand = posixpath.normpath(posixpath.join(base_dir, ref)) if base_dir else posixpath.normpath(ref) + ref = ref.removeprefix("./") + cand = ( + posixpath.normpath(posixpath.join(base_dir, ref)) + if base_dir + else posixpath.normpath(ref) + ) if cand in (".", "") or cand.startswith("../"): return [] if cand in self.tree_dirs: @@ -533,7 +730,12 @@ def resolve_module(self, module: str, func: str | None = None) -> list[str]: if not module or not re.match(r"^[A-Za-z_][\w.]*$", module): return [] mod_path = module.replace(".", "/") - candidates = [f"{mod_path}.py", f"{mod_path}/__init__.py", f"src/{mod_path}.py", f"{mod_path}/__main__.py"] + candidates = [ + f"{mod_path}.py", + f"{mod_path}/__init__.py", + f"src/{mod_path}.py", + f"{mod_path}/__main__.py", + ] for p in candidates: if func: fid = f"{p}::{func}" @@ -545,7 +747,9 @@ def resolve_module(self, module: str, func: str | None = None) -> list[str]: return [] -def _refs_from_shell_text(text: str, resolver: _Resolver, base_dir: str, artifact_units: dict[str, set[str]]) -> set[str]: +def _refs_from_shell_text( + text: str, resolver: _Resolver, base_dir: str, artifact_units: dict[str, set[str]] +) -> set[str]: """Collect ids referenced by shell-ish text (CI ``run:`` blocks, RUN lines, Makefile recipes, npm script values).""" found: set[str] = set() @@ -563,7 +767,11 @@ def _refs_from_shell_text(text: str, resolver: _Resolver, base_dir: str, artifac script = m.group(1) if script in _NPM_RESERVED: continue - pj = posixpath.normpath(posixpath.join(base_dir, "package.json")) if base_dir else "package.json" + pj = ( + posixpath.normpath(posixpath.join(base_dir, "package.json")) + if base_dir + else "package.json" + ) if script in artifact_units.get(pj, set()): found.add(f"{pj}::{script}") for m in DOCKER_BUILD_FILE_RE.finditer(text): @@ -661,7 +869,9 @@ def analyze_artifacts( continue truncated = total > opts.per_file_bytes if truncated: - text = text + TRUNCATION_MARKER.format(shown=len(text.encode("utf-8", "replace")), total=total) + text = text + TRUNCATION_MARKER.format( + shown=len(text.encode("utf-8", "replace")), total=total + ) n_tokens = count_tokens(text) if tokens_used + n_tokens > opts.token_budget: index_classes[cls]["not_loaded_budget"].append(rel) @@ -675,9 +885,13 @@ def analyze_artifacts( nodes: list[Node] = [] relationships: list[CallRelationship] = [] artifact_units: dict[str, set[str]] = {} - unit_specs: list[tuple[str, str, str, str, int, int, dict]] = [] # (cls, rel, unit, text, start_line, end_line, extra) + unit_specs: list[ + tuple[str, str, str, str, int, int, dict] + ] = [] # (cls, rel, unit, text, start_line, end_line, extra) - def _make_node(rel: str, name: str, cls: str, text: str, node_type: str, start: int, end: int) -> Node: + def _make_node( + rel: str, name: str, cls: str, text: str, node_type: str, start: int, end: int + ) -> Node: return Node( id=f"{rel}::{name}", name=name, @@ -709,7 +923,9 @@ def _make_node(rel: str, name: str, cls: str, text: str, node_type: str, start: elif cls == "container" and re.match(r"^(docker-)?compose[.\w-]*\.ya?ml$", lower): units = _yaml_children(text, "services") extra["compose"] = True - elif cls == "container" and (fnmatch.fnmatch(name, "Dockerfile*") or lower.endswith(".dockerfile")): + elif cls == "container" and ( + fnmatch.fnmatch(name, "Dockerfile*") or lower.endswith(".dockerfile") + ): units = _dockerfile_units(text) extra["dockerfile"] = True elif lower in {"makefile", "gnumakefile"} or lower.endswith(".mk"): @@ -719,18 +935,50 @@ def _make_node(rel: str, name: str, cls: str, text: str, node_type: str, start: pj_units, data = _package_json_units(text) extra["package_json"] = data for uname, start, end, utext in _prioritise_units(pj_units, opts.per_file_units): - unit_specs.append((cls, rel, uname, utext, _line_of(text, start), _line_of(text, end), {"script": True})) + unit_specs.append( + ( + cls, + rel, + uname, + utext, + _line_of(text, start), + _line_of(text, end), + {"script": True}, + ) + ) artifact_units.setdefault(rel, set()).add(uname) units = [] elif name == "pyproject.toml" or name == "setup.cfg": kind = "pyproject" if name == "pyproject.toml" else "setup_cfg" - for uname, start, end, module, func in _prioritise_units(_entry_point_units(text, kind), opts.per_file_units): + for uname, start, end, module, func in _prioritise_units( + _entry_point_units(text, kind), opts.per_file_units + ): snippet = f"{uname} = {module}:{func}" - unit_specs.append((cls, rel, uname, snippet, _line_of(text, start), _line_of(text, end), {"entry": (module, func)})) + unit_specs.append( + ( + cls, + rel, + uname, + snippet, + _line_of(text, start), + _line_of(text, end), + {"entry": (module, func)}, + ) + ) artifact_units.setdefault(rel, set()).add(uname) units = [] for uname, start, end in _prioritise_units(units, opts.per_file_units): - unit_specs.append((cls, rel, uname, text[start:end].rstrip() + "\n", _line_of(text, start), _line_of(text, max(start, end - 1)), extra)) + unit_specs.append( + ( + cls, + rel, + uname, + text[start:end].rstrip() + "\n", + _line_of(text, start), + _line_of(text, max(start, end - 1)), + extra, + ) + ) artifact_units.setdefault(rel, set()).add(uname) index_classes[cls]["files"].append( { @@ -777,26 +1025,38 @@ def _add_edges(caller: str, callees: Iterable[str]) -> None: if job_base in (".", "/"): job_base = "" refs.update(_refs_from_shell_text(job_text, resolver, job_base, artifact_units)) - _add_edges(f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id, refs) + _add_edges( + f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id, refs + ) elif cls == "container" and re.match(r"^(docker-)?compose[.\w-]*\.ya?ml$", lower): for uname, start, end in _yaml_children(text, "services"): svc = text[start:end] refs = set() - ctx = re.search(r"^\s+context:\s*(\S+)", svc, re.M) - dfile = re.search(r"^\s+dockerfile:\s*(\S+)", svc, re.M) - build_str = re.search(r"^\s+build:\s*(\S+)\s*$", svc, re.M) - context = (ctx.group(1) if ctx else (build_str.group(1) if build_str else "")).strip("\"'") + ctx = re.search(r"^\s+context:\s*(\S+)", svc, re.MULTILINE) + dfile = re.search(r"^\s+dockerfile:\s*(\S+)", svc, re.MULTILINE) + build_str = re.search(r"^\s+build:\s*(\S+)\s*$", svc, re.MULTILINE) + context = ( + ctx.group(1) if ctx else (build_str.group(1) if build_str else "") + ).strip("\"'") dockerfile = (dfile.group(1) if dfile else "Dockerfile").strip("\"'") if ctx or build_str or dfile: - refs.update(resolver.resolve_path(posixpath.join(context, dockerfile), base_dir)) - _add_edges(f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id, refs) - elif cls == "container" and (fnmatch.fnmatch(name, "Dockerfile*") or lower.endswith(".dockerfile")): + refs.update( + resolver.resolve_path(posixpath.join(context, dockerfile), base_dir) + ) + _add_edges( + f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id, refs + ) + elif cls == "container" and ( + fnmatch.fnmatch(name, "Dockerfile*") or lower.endswith(".dockerfile") + ): units = _dockerfile_units(text) or [(None, 0, len(text))] for uname, start, end in units: stage = text[start:end] refs = set() - caller = f"{rel}::{uname}" if uname and resolver.known(f"{rel}::{uname}") else file_id - for m in re.finditer(r"^(?:COPY|ADD)\s+(.*)$", stage, re.I | re.M): + caller = ( + f"{rel}::{uname}" if uname and resolver.known(f"{rel}::{uname}") else file_id + ) + for m in re.finditer(r"^(?:COPY|ADD)\s+(.*)$", stage, re.IGNORECASE | re.MULTILINE): args = [a for a in m.group(1).split() if not a.startswith("--")] if m.group(0).find("--from=") != -1: alias = re.search(r"--from=(\S+)", m.group(0)).group(1) @@ -804,11 +1064,15 @@ def _add_edges(caller: str, callees: Iterable[str]) -> None: continue for a in args[:-1]: refs.update(resolver.resolve_path(a, base_dir)) - for m in re.finditer(r"^(?:ENTRYPOINT|CMD|RUN)\s+(.*)$", stage, re.I | re.M): - refs.update(_refs_from_shell_text(m.group(1), resolver, base_dir, artifact_units)) + for m in re.finditer( + r"^(?:ENTRYPOINT|CMD|RUN)\s+(.*)$", stage, re.IGNORECASE | re.MULTILINE + ): + refs.update( + _refs_from_shell_text(m.group(1), resolver, base_dir, artifact_units) + ) _add_edges(caller, refs) elif lower in {"makefile", "gnumakefile"} or lower.endswith(".mk"): - for m in re.finditer(r"^(?:-?include|sinclude)\s+(\S+)", text, re.M): + for m in re.finditer(r"^(?:-?include|sinclude)\s+(\S+)", text, re.MULTILINE): _add_edges(file_id, resolver.resolve_path(m.group(1), base_dir)) for uname, start, end in _makefile_units(text): recipe = text[start:end] @@ -821,7 +1085,9 @@ def _add_edges(caller: str, callees: Iterable[str]) -> None: refs.add(f"{rel}::{prereq}") else: refs.update(resolver.resolve_path(prereq, base_dir)) - _add_edges(f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id, refs) + _add_edges( + f"{rel}::{uname}" if resolver.known(f"{rel}::{uname}") else file_id, refs + ) elif name == "package.json": _, data = _package_json_units(text) if isinstance(data, dict): @@ -830,28 +1096,36 @@ def _add_edges(caller: str, callees: Iterable[str]) -> None: if isinstance(data.get(key), str): refs.update(resolver.resolve_path(data[key], base_dir)) bin_field = data.get("bin") - for v in ([bin_field] if isinstance(bin_field, str) else list(bin_field.values()) if isinstance(bin_field, dict) else []): + for v in ( + [bin_field] + if isinstance(bin_field, str) + else list(bin_field.values()) + if isinstance(bin_field, dict) + else [] + ): if isinstance(v, str): refs.update(resolver.resolve_path(v, base_dir)) - def _walk_exports(val: Any) -> None: + def _walk_exports(val: Any, found: set[str], rel_dir: str) -> None: if isinstance(val, str): - refs.update(resolver.resolve_path(val, base_dir)) + found.update(resolver.resolve_path(val, rel_dir)) elif isinstance(val, dict): for v in val.values(): - _walk_exports(v) + _walk_exports(v, found, rel_dir) elif isinstance(val, list): for v in val: - _walk_exports(v) + _walk_exports(v, found, rel_dir) - _walk_exports(data.get("exports")) + _walk_exports(data.get("exports"), refs, base_dir) _add_edges(file_id, refs) scripts = data.get("scripts") if isinstance(data.get("scripts"), dict) else {} for sname, sval in scripts.items(): if not isinstance(sval, str): continue caller = f"{rel}::{sname}" if resolver.known(f"{rel}::{sname}") else file_id - _add_edges(caller, _refs_from_shell_text(sval, resolver, base_dir, artifact_units)) + _add_edges( + caller, _refs_from_shell_text(sval, resolver, base_dir, artifact_units) + ) elif name in {"pyproject.toml", "setup.cfg"}: kind = "pyproject" if name == "pyproject.toml" else "setup_cfg" for uname, _s, _e, module, func in _entry_point_units(text, kind): @@ -872,13 +1146,21 @@ def _walk_exports(val: Any) -> None: "per_file_units": opts.per_file_units, }, "with_prose": opts.with_prose, - "classes": {cls: v for cls, v in index_classes.items() if v["files"] or v["omitted_by_class_cap"] or v["not_loaded_budget"]}, + "classes": { + cls: v + for cls, v in index_classes.items() + if v["files"] or v["omitted_by_class_cap"] or v["not_loaded_budget"] + }, "nodes": len(nodes), "edges": len(relationships), } logger.info( "Artifact analysis: %d files, %d nodes, %d edges, %d tokens (budget %d)", - len(loaded), len(nodes), len(relationships), tokens_used, opts.token_budget, + len(loaded), + len(nodes), + len(relationships), + tokens_used, + opts.token_budget, ) return ArtifactAnalysis(nodes=nodes, relationships=relationships, index=index) @@ -937,9 +1219,11 @@ def render_artifact_index(components: dict[str, Any], max_files_per_class: int = return "" lines = [ "", - "Build, CI, container, packaging, manifest, config, schema and script files in this " - "repository, grouped by class. Component ids are `::`; read a file with " - "`str_replace_editor view` (working_dir=`repo`).", + ( + "Build, CI, container, packaging, manifest, config, schema and script files in this " + "repository, grouped by class. Component ids are `::`; read a file with " + "`str_replace_editor view` (working_dir=`repo`)." + ), ] for cls in CLASS_PRIORITY: entries = index.get(cls) diff --git a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py index 5e39085f..ccf5fe17 100644 --- a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py +++ b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py @@ -1,26 +1,33 @@ -from typing import Dict, List, Any +import logging import os -from codewiki.src.config import Config -from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser +from typing import Any + from codewiki.src.be.dependency_analyzer.analyzers.artifact import ArtifactOptions -from codewiki.src.be.dependency_analyzer.topo_sort import build_graph_from_components, get_leaf_nodes -from codewiki.src.be.dependency_analyzer.leaf_selection import compute_valid_leaf_types, filter_leaf_nodes +from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser +from codewiki.src.be.dependency_analyzer.leaf_selection import ( + compute_valid_leaf_types, + filter_leaf_nodes, +) +from codewiki.src.be.dependency_analyzer.topo_sort import ( + build_graph_from_components, + get_leaf_nodes, +) +from codewiki.src.config import Config from codewiki.src.utils import file_manager -import logging logger = logging.getLogger(__name__) class DependencyGraphBuilder: """Handles dependency analysis and graph building.""" - + def __init__(self, config: Config): self.config = config - - def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: + + def build_dependency_graph(self) -> tuple[dict[str, Any], list[str]]: """ Build and save dependency graph, returning components and leaf nodes. - + Returns: Tuple of (components, leaf_nodes) """ @@ -29,20 +36,14 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: # Prepare dependency graph path repo_name = os.path.basename(os.path.normpath(self.config.repo_path)) - sanitized_repo_name = ''.join(c if c.isalnum() else '_' for c in repo_name) + sanitized_repo_name = "".join(c if c.isalnum() else "_" for c in repo_name) dependency_graph_path = os.path.join( - self.config.dependency_graph_dir, - f"{sanitized_repo_name}_dependency_graph.json" + self.config.dependency_graph_dir, f"{sanitized_repo_name}_dependency_graph.json" ) - filtered_folders_path = os.path.join( - self.config.dependency_graph_dir, - f"{sanitized_repo_name}_filtered_folders.json" - ) - # Get custom include/exclude patterns from config include_patterns = self.config.include_patterns if self.config.include_patterns else None exclude_patterns = self.config.exclude_patterns if self.config.exclude_patterns else None - + artifact_options = ArtifactOptions( enabled=getattr(self.config, "artifacts_enabled", True), token_budget=getattr(self.config, "artifact_token_budget", 200_000), @@ -70,7 +71,7 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: # Parse repository components = parser.parse_repository(filtered_folders) - + # Save dependency graph parser.save_dependency_graph(dependency_graph_path) @@ -90,10 +91,10 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: ) else: logger.info("Artifact nodes in dependency graph: %d", n_artifacts) - + # Build graph for traversal graph = build_graph_from_components(components) - + # Get leaf nodes leaf_nodes = get_leaf_nodes(graph, components) diff --git a/codewiki/src/be/dependency_analyzer/models/core.py b/codewiki/src/be/dependency_analyzer/models/core.py index 8a4f8382..403fc940 100644 --- a/codewiki/src/be/dependency_analyzer/models/core.py +++ b/codewiki/src/be/dependency_analyzer/models/core.py @@ -1,51 +1,48 @@ from pydantic import BaseModel -from typing import List, Optional, Dict, Any, Set -from datetime import datetime - class Node(BaseModel): id: str name: str - + component_type: str - + file_path: str - + relative_path: str - - depends_on: Set[str] = set() - - source_code: Optional[str] = None - + + depends_on: set[str] = set() + + source_code: str | None = None + start_line: int = 0 end_line: int = 0 - + has_docstring: bool = False - + docstring: str = "" - - parameters: Optional[List[str]] = None - node_type: Optional[str] = None + parameters: list[str] | None = None - base_classes: Optional[List[str]] = None + node_type: str | None = None - class_name: Optional[str] = None + base_classes: list[str] | None = None - display_name: Optional[str] = None + class_name: str | None = None - component_id: Optional[str] = None + display_name: str | None = None - language: Optional[str] = None + component_id: str | None = None - qualified_name: Optional[str] = None + language: str | None = None + + qualified_name: str | None = None # Set only on artifact nodes (component_type == "artifact"): one of the # classes in analyzers/artifact.py CLASS_PRIORITY (build, ci, container, ...). - artifact_class: Optional[str] = None + artifact_class: str | None = None def get_display_name(self) -> str: return self.display_name or self.name @@ -56,7 +53,7 @@ class CallRelationship(BaseModel): callee: str - call_line: Optional[int] = None + call_line: int | None = None is_resolved: bool = False @@ -67,5 +64,5 @@ class Repository(BaseModel): name: str clone_path: str - + analysis_id: str diff --git a/codewiki/src/be/dependency_analyzer/utils/security.py b/codewiki/src/be/dependency_analyzer/utils/security.py index b8a1a95f..54e47d46 100644 --- a/codewiki/src/be/dependency_analyzer/utils/security.py +++ b/codewiki/src/be/dependency_analyzer/utils/security.py @@ -1,5 +1,6 @@ -from pathlib import Path import os +from pathlib import Path + def _inside(base: Path, target: Path) -> bool: base_r = base.resolve() @@ -9,6 +10,7 @@ def _inside(base: Path, target: Path) -> bool: except AttributeError: return str(target.resolve()).startswith(str(base_r)) + def assert_safe_path(base_dir: Path, target: Path): # Block symlinks (file or dir) if target.is_symlink(): @@ -17,6 +19,7 @@ def assert_safe_path(base_dir: Path, target: Path): if not _inside(base_dir, target): raise PermissionError(f"Path escapes repo: {target} -> {target.resolve()}") + def safe_open_text(base_dir: Path, target: Path, encoding="utf-8"): assert_safe_path(base_dir, target) flags = os.O_RDONLY @@ -33,7 +36,9 @@ def safe_open_text(base_dir: Path, target: Path, encoding="utf-8"): pass -def safe_read_head(base_dir: Path, target: Path, max_bytes: int, encoding="utf-8") -> tuple[str, int, bool]: +def safe_read_head( + base_dir: Path, target: Path, max_bytes: int, encoding="utf-8" +) -> tuple[str, int, bool]: """Read at most ``max_bytes`` of ``target`` with the same symlink/escape checks as :func:`safe_open_text`. diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index 4790c6c6..59e924e6 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -16,8 +16,8 @@ get_clustering_input_token_count, super_group_modules, ) -from codewiki.src.be.dependency_analyzer.analyzers.artifact import render_artifact_index from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder +from codewiki.src.be.dependency_analyzer.analyzers.artifact import render_artifact_index from codewiki.src.be.module_naming import ( dedupe_module_tree_names, find_missing_module_docs, @@ -348,7 +348,9 @@ async def generate_parent_module_docs( if len(module_path) == 0 and components: artifact_index = render_artifact_index(components) if artifact_index: - prompt += "\n\n" + REPO_OVERVIEW_ARTIFACT_ADDENDUM.format(artifact_index=artifact_index) + prompt += "\n\n" + REPO_OVERVIEW_ARTIFACT_ADDENDUM.format( + artifact_index=artifact_index + ) logger.debug(f"Overview prompt for {module_name}: {len(prompt)} chars") try: diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py index b0fb88a7..d28944ae 100644 --- a/codewiki/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -293,7 +293,7 @@ ARTIFACT_USAGE_NOTE = ( "* NOTE: when this module's behaviour depends on how the system is built, " "configured, packaged, deployed or tested, read the relevant artifact file " - "with `str_replace_editor` (`command=\"view\"`, `working_dir=\"repo\"`, path as " + 'with `str_replace_editor` (`command="view"`, `working_dir="repo"`, path as ' "listed above) and cite the file path in the documentation." ) @@ -498,9 +498,7 @@ def format_user_prompt( core_component_codes += "\n```\n\n" artifact_index = render_artifact_index(components) - artifact_section = ( - f"\n\n{artifact_index}\n{ARTIFACT_USAGE_NOTE}" if artifact_index else "" - ) + artifact_section = f"\n\n{artifact_index}\n{ARTIFACT_USAGE_NOTE}" if artifact_index else "" def _assemble(codes: str, tree: str) -> str: return ( diff --git a/codewiki/src/config.py b/codewiki/src/config.py index c20848ce..b98208ed 100644 --- a/codewiki/src/config.py +++ b/codewiki/src/config.py @@ -1,18 +1,19 @@ -from dataclasses import dataclass, field -from typing import Optional, List, Dict, Any import argparse import os -import sys +from dataclasses import dataclass +from typing import Any + from dotenv import load_dotenv + load_dotenv() # Constants -OUTPUT_BASE_DIR = 'output' -DEPENDENCY_GRAPHS_DIR = 'dependency_graphs' -DOCS_DIR = 'docs' -FIRST_MODULE_TREE_FILENAME = 'first_module_tree.json' -MODULE_TREE_FILENAME = 'module_tree.json' -OVERVIEW_FILENAME = 'overview.md' +OUTPUT_BASE_DIR = "output" +DEPENDENCY_GRAPHS_DIR = "dependency_graphs" +DOCS_DIR = "docs" +FIRST_MODULE_TREE_FILENAME = "first_module_tree.json" +MODULE_TREE_FILENAME = "module_tree.json" +OVERVIEW_FILENAME = "overview.md" MAX_DEPTH = 2 # Default max token settings DEFAULT_MAX_TOKENS = 32_768 @@ -36,31 +37,36 @@ # CLI context detection _CLI_CONTEXT = False + def set_cli_context(enabled: bool = True): """Set whether we're running in CLI context (vs web app).""" global _CLI_CONTEXT _CLI_CONTEXT = enabled + def is_cli_context() -> bool: """Check if running in CLI context.""" return _CLI_CONTEXT + # LLM services # In CLI mode, these will be loaded from ~/.codewiki/config.json + keyring # In web app mode, use environment variables -MAIN_MODEL = os.getenv('MAIN_MODEL', 'claude-sonnet-4') -FALLBACK_MODEL_1 = os.getenv('FALLBACK_MODEL_1', 'glm-4p5') -CLUSTER_MODEL = os.getenv('CLUSTER_MODEL', MAIN_MODEL) -LLM_BASE_URL = os.getenv('LLM_BASE_URL', 'http://0.0.0.0:4000/') -LLM_API_KEY = os.getenv('LLM_API_KEY', 'sk-1234') +MAIN_MODEL = os.getenv("MAIN_MODEL", "claude-sonnet-4") +FALLBACK_MODEL_1 = os.getenv("FALLBACK_MODEL_1", "glm-4p5") +CLUSTER_MODEL = os.getenv("CLUSTER_MODEL", MAIN_MODEL) +LLM_BASE_URL = os.getenv("LLM_BASE_URL", "http://0.0.0.0:4000/") +LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-1234") # Atlas Cloud default endpoint (OpenAI-compatible). Used to auto-fill the base URL # when the user selects the `atlas-cloud` provider without passing --base-url. ATLAS_CLOUD_BASE_URL = "https://api.atlascloud.ai/v1" + @dataclass class Config: """Configuration class for CodeWiki.""" + repo_path: str output_dir: str dependency_graph_dir: str @@ -73,7 +79,9 @@ class Config: cluster_model: str fallback_model: str = FALLBACK_MODEL_1 # Provider configuration - provider: str = "openai-compatible" # openai-compatible, atlas-cloud, anthropic, bedrock, azure-openai + provider: str = ( + "openai-compatible" # openai-compatible, atlas-cloud, anthropic, bedrock, azure-openai + ) aws_region: str = "us-east-1" api_version: str = "2024-12-01-preview" # Azure OpenAI API version azure_deployment: str = "" # Azure OpenAI deployment name @@ -87,7 +95,7 @@ class Config: # the provider rejects cache_control markers) prompt_caching: bool = True # Agent instructions for customization - agent_instructions: Optional[Dict[str, Any]] = None + agent_instructions: dict[str, Any] | None = None # Apply Git ignore rules before dependency analysis use_gitignore: bool = True # Artifact-aware generation (Dockerfiles, CI workflows, Makefiles, @@ -97,82 +105,84 @@ class Config: # Also read the root README and docs/ as a `prose` artifact class (off by # default: documentation without existing prose is the benchmark setting) with_prose: bool = False - + @property - def artifact_exclude(self) -> Optional[List[str]]: + def artifact_exclude(self) -> list[str] | None: """Extra patterns excluded from artifact analysis (from agent instructions).""" if self.agent_instructions: - return self.agent_instructions.get('artifact_exclude') + return self.agent_instructions.get("artifact_exclude") return None @property - def include_patterns(self) -> Optional[List[str]]: + def include_patterns(self) -> list[str] | None: """Get file include patterns from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('include_patterns') + return self.agent_instructions.get("include_patterns") return None - + @property - def exclude_patterns(self) -> Optional[List[str]]: + def exclude_patterns(self) -> list[str] | None: """Get file exclude patterns from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('exclude_patterns') + return self.agent_instructions.get("exclude_patterns") return None - + @property - def focus_modules(self) -> Optional[List[str]]: + def focus_modules(self) -> list[str] | None: """Get focus modules from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('focus_modules') + return self.agent_instructions.get("focus_modules") return None - + @property - def doc_type(self) -> Optional[str]: + def doc_type(self) -> str | None: """Get documentation type from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('doc_type') + return self.agent_instructions.get("doc_type") return None - + @property - def custom_instructions(self) -> Optional[str]: + def custom_instructions(self) -> str | None: """Get custom instructions from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('custom_instructions') + return self.agent_instructions.get("custom_instructions") return None - + def get_prompt_addition(self) -> str: """Generate prompt additions based on agent instructions.""" if not self.agent_instructions: return "" - + additions = [] - + if self.doc_type: doc_type_instructions = { - 'api': "Focus on API documentation: endpoints, parameters, return types, and usage examples.", - 'architecture': "Focus on architecture documentation: system design, component relationships, and data flow.", - 'user-guide': "Focus on user guide documentation: how to use features, step-by-step tutorials.", - 'developer': "Focus on developer documentation: code structure, contribution guidelines, and implementation details.", + "api": "Focus on API documentation: endpoints, parameters, return types, and usage examples.", + "architecture": "Focus on architecture documentation: system design, component relationships, and data flow.", + "user-guide": "Focus on user guide documentation: how to use features, step-by-step tutorials.", + "developer": "Focus on developer documentation: code structure, contribution guidelines, and implementation details.", } if self.doc_type.lower() in doc_type_instructions: additions.append(doc_type_instructions[self.doc_type.lower()]) else: additions.append(f"Focus on generating {self.doc_type} documentation.") - + if self.focus_modules: - additions.append(f"Pay special attention to and provide more detailed documentation for these modules: {', '.join(self.focus_modules)}") - + additions.append( + f"Pay special attention to and provide more detailed documentation for these modules: {', '.join(self.focus_modules)}" + ) + if self.custom_instructions: additions.append(f"Additional instructions: {self.custom_instructions}") - + return "\n".join(additions) if additions else "" - + @classmethod - def from_args(cls, args: argparse.Namespace) -> 'Config': + def from_args(cls, args: argparse.Namespace) -> "Config": """Create configuration from parsed arguments.""" repo_name = os.path.basename(os.path.normpath(args.repo_path)) - sanitized_repo_name = ''.join(c if c.isalnum() else '_' for c in repo_name) - + sanitized_repo_name = "".join(c if c.isalnum() else "_" for c in repo_name) + return cls( repo_path=args.repo_path, output_dir=OUTPUT_BASE_DIR, @@ -186,7 +196,7 @@ def from_args(cls, args: argparse.Namespace) -> 'Config': fallback_model=FALLBACK_MODEL_1, use_gitignore=getattr(args, "use_gitignore", True), ) - + @classmethod def from_cli( cls, @@ -207,13 +217,13 @@ def from_cli( min_modules_for_super_grouping: int = DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING, max_leaf_nodes_per_cluster: int = DEFAULT_MAX_LEAF_NODES_PER_CLUSTER, max_depth: int = MAX_DEPTH, - agent_instructions: Optional[Dict[str, Any]] = None, + agent_instructions: dict[str, Any] | None = None, use_gitignore: bool = True, prompt_caching: bool = True, artifacts_enabled: bool = True, artifact_token_budget: int = DEFAULT_ARTIFACT_TOKEN_BUDGET, with_prose: bool = False, - ) -> 'Config': + ) -> "Config": """ Create configuration for CLI context. @@ -249,7 +259,6 @@ def from_cli( Returns: Config instance """ - repo_name = os.path.basename(os.path.normpath(repo_path)) base_output_dir = os.path.join(output_dir, "temp") return cls( diff --git a/tests/test_artifact_analyzer.py b/tests/test_artifact_analyzer.py index 0019a084..76ad638d 100644 --- a/tests/test_artifact_analyzer.py +++ b/tests/test_artifact_analyzer.py @@ -21,18 +21,20 @@ ) from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer from codewiki.src.be.dependency_analyzer.analyzers.artifact import ( - ArtifactOptions, TRUNCATION_MARKER, + ArtifactOptions, classify_artifact, render_artifact_index, ) from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser from codewiki.src.be.dependency_analyzer.leaf_selection import compute_valid_leaf_types from codewiki.src.be.dependency_analyzer.models.core import Node -from codewiki.src.be.dependency_analyzer.topo_sort import build_graph_from_components, get_leaf_nodes +from codewiki.src.be.dependency_analyzer.topo_sort import ( + build_graph_from_components, + get_leaf_nodes, +) from codewiki.src.be.prompt_template import USER_PROMPT, format_user_prompt - # --------------------------------------------------------------------------- # # fixture # --------------------------------------------------------------------------- # @@ -59,7 +61,11 @@ def mini_repo(tmp_path: Path) -> Path: "package.json", '{\n "name": "mini",\n "main": "pkg/index.js",\n "scripts": {\n "build": "node scripts/build.js",\n "test": "npm run build && node test.js"\n }\n}\n', ) - _write(tmp_path, "scripts/build.js", "function build() { return 1; }\nmodule.exports = { build };\n") + _write( + tmp_path, + "scripts/build.js", + "function build() { return 1; }\nmodule.exports = { build };\n", + ) _write( tmp_path, "Makefile", @@ -68,7 +74,7 @@ def mini_repo(tmp_path: Path) -> Path: _write( tmp_path, "Dockerfile", - "FROM python:3.12 AS builder\nCOPY pkg/cli.py /app/cli.py\nRUN make build\n\nFROM python:3.12-slim\nCOPY --from=builder /app /app\nENTRYPOINT [\"python\", \"pkg/cli.py\"]\n", + 'FROM python:3.12 AS builder\nCOPY pkg/cli.py /app/cli.py\nRUN make build\n\nFROM python:3.12-slim\nCOPY --from=builder /app /app\nENTRYPOINT ["python", "pkg/cli.py"]\n', ) _write( tmp_path, @@ -141,7 +147,9 @@ def test_classify_prose_and_exclude() -> None: assert classify_artifact("pkg/notes.md", "notes.md", 10, prose) is None excl = ArtifactOptions(exclude_patterns=["docker/data/*"]) assert classify_artifact("docker/data/huge.yml", "huge.yml", 10, excl) is None - assert classify_artifact("docker/data/huge.yml", "huge.yml", 10, ArtifactOptions()) == "container" + assert ( + classify_artifact("docker/data/huge.yml", "huge.yml", 10, ArtifactOptions()) == "container" + ) assert classify_artifact("Dockerfile", "Dockerfile", 0, ArtifactOptions()) is None @@ -166,14 +174,23 @@ def _walk(node: dict | None) -> None: def test_repo_analyzer_whitelist(mini_repo: Path) -> None: - paths = _tree_paths(RepoAnalyzer(use_gitignore=False).analyze_repository_structure(str(mini_repo))["file_tree"]) - assert {".github/workflows/ci.yml", "pytest.ini", "Dockerfile", "Makefile", "pyproject.toml"} <= paths + paths = _tree_paths( + RepoAnalyzer(use_gitignore=False).analyze_repository_structure(str(mini_repo))["file_tree"] + ) + assert { + ".github/workflows/ci.yml", + "pytest.ini", + "Dockerfile", + "Makefile", + "pyproject.toml", + } <= paths assert "tests/conftest.py" not in paths assert ".github/ISSUE_TEMPLATE/bug.md" not in paths # user excludes still win over the whitelist paths_user = _tree_paths( - RepoAnalyzer(exclude_patterns=[".github"], use_gitignore=False) - .analyze_repository_structure(str(mini_repo))["file_tree"] + RepoAnalyzer( + exclude_patterns=[".github"], use_gitignore=False + ).analyze_repository_structure(str(mini_repo))["file_tree"] ) assert ".github/workflows/ci.yml" not in paths_user @@ -213,7 +230,10 @@ def test_parse_repository_emits_artifact_nodes_and_units(mini_repo: Path) -> Non assert artifacts[".github/workflows/ci.yml::lint"].artifact_class == "ci" assert artifacts["Makefile::test"].source_code.startswith("test: build\n\tpython pkg/cli.py") # code side is untouched - assert "pkg/cli.py::main" in components and components["pkg/cli.py::main"].component_type == "function" + assert ( + "pkg/cli.py::main" in components + and components["pkg/cli.py::main"].component_type == "function" + ) # a parser without options keeps the code-only graph plain = DependencyParser(str(mini_repo), use_gitignore=False).parse_repository() assert not _artifact_nodes(plain) @@ -256,15 +276,23 @@ def test_edges_resolve_only_to_known_ids(mini_repo: Path) -> None: def test_caps(mini_repo: Path) -> None: for i in range(45): _write(mini_repo, f"config/c{i:02d}.yaml", f"n: {i}\n") - parser = DependencyParser(str(mini_repo), use_gitignore=False, artifact_options=ArtifactOptions()) + parser = DependencyParser( + str(mini_repo), use_gitignore=False, artifact_options=ArtifactOptions() + ) components = parser.parse_repository() big = components["config/big.yaml::big.yaml"] marker_prefix = TRUNCATION_MARKER.split("{")[0] assert marker_prefix in big.source_code assert len(big.source_code) < 16_384 + len(TRUNCATION_MARKER) + 32 - config_files = [n for n in _artifact_nodes(components).values() if n.artifact_class == "config" and n.node_type == "artifact_file"] + config_files = [ + n + for n in _artifact_nodes(components).values() + if n.artifact_class == "config" and n.node_type == "artifact_file" + ] assert len(config_files) == 40 - assert len(parser.artifact_index["classes"]["config"]["omitted_by_class_cap"]) == 6 # 46 config files - 40 + assert ( + len(parser.artifact_index["classes"]["config"]["omitted_by_class_cap"]) == 6 + ) # 46 config files - 40 # a tiny budget keeps manifests (highest priority) and records what was skipped tight = DependencyParser( str(mini_repo), use_gitignore=False, artifact_options=ArtifactOptions(token_budget=120) @@ -305,7 +333,9 @@ def test_leaf_types_and_pruning() -> None: components["src/x.py::X"] = _node("src/x.py::X", "class") components["src/z.py::Z"] = _node("src/z.py::Z", "class") components["src/y.py::Y"] = _node("src/y.py::Y", "class", {"src/z.py::Z"}) - components["Dockerfile::Dockerfile"] = _node("Dockerfile::Dockerfile", "artifact", {"src/x.py::X"}) + components["Dockerfile::Dockerfile"] = _node( + "Dockerfile::Dockerfile", "artifact", {"src/x.py::X"} + ) leaves = set(get_leaf_nodes(build_graph_from_components(components), components)) assert "src/x.py::X" in leaves # referenced only by an artifact: kept assert "Dockerfile::Dockerfile" in leaves @@ -325,7 +355,7 @@ def test_format_user_prompt_with_artifacts() -> None: component_type="artifact", file_path="/nonexistent/Dockerfile", relative_path="Dockerfile", - source_code="FROM python:3.12\nCMD [\"python\"]\n", + source_code='FROM python:3.12\nCMD ["python"]\n', node_type="artifact_file", artifact_class="container", ) @@ -356,7 +386,12 @@ def test_format_user_prompt_with_artifacts() -> None: relative_path="cfg/app.yml", source_code="a: 1", ) - prompt2 = format_user_prompt("M", [code.id], {code.id: code}, {"M": {"path": "", "components": [code.id], "children": {}}}) + prompt2 = format_user_prompt( + "M", + [code.id], + {code.id: code}, + {"M": {"path": "", "components": [code.id], "children": {}}}, + ) assert "" not in prompt2 assert "```yaml" in prompt2 # MCP contract: USER_PROMPT still has exactly the three original placeholders