diff --git a/codewiki/src/be/utils.py b/codewiki/src/be/utils.py index 1f14082f..ed50d855 100644 --- a/codewiki/src/be/utils.py +++ b/codewiki/src/be/utils.py @@ -7,7 +7,6 @@ from typing import List, Tuple import logging import tiktoken -import traceback logger = logging.getLogger(__name__) @@ -29,10 +28,12 @@ def set_main_loop(loop: asyncio.AbstractEventLoop) -> None: _main_loop = loop _main_loop_thread_ident = threading.get_ident() + # ------------------------------------------------------------ # ---------------------- Complexity Check -------------------- # ------------------------------------------------------------ + def is_complex_module(components: dict[str, any], core_component_ids: list[str]) -> bool: files = set() for component_id in core_component_ids: @@ -50,6 +51,7 @@ def is_complex_module(components: dict[str, any], core_component_ids: list[str]) enc = tiktoken.encoding_for_model("gpt-4") + def count_tokens(text: str) -> int: """ Count the number of tokens in a text. @@ -63,10 +65,11 @@ def count_tokens(text: str) -> int: # ---------------------- Mermaid Validation ----------------- # ------------------------------------------------------------ + async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> str: """ Validate all Mermaid diagrams in a markdown file. - + Args: md_file_path: Path to the markdown file to check relative_path: Relative path to the markdown file @@ -80,15 +83,15 @@ async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> st file_path = Path(md_file_path) if not file_path.exists(): return f"Error: File '{md_file_path}' does not exist" - - content = file_path.read_text(encoding='utf-8') - + + content = file_path.read_text(encoding="utf-8") + # Extract all mermaid code blocks mermaid_blocks = extract_mermaid_blocks(content) - + if not mermaid_blocks: return "No mermaid diagrams found in the file" - + # Validate each mermaid diagram sequentially to avoid segfaults errors = [] for i, (line_start, diagram_content) in enumerate(mermaid_blocks, 1): @@ -96,15 +99,17 @@ async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> st if error_msg: errors.append("\n") errors.append(error_msg) - + # if errors: # logger.debug(f"Mermaid syntax errors found in file: {md_file_path}: {errors}") - + if errors: - return "Mermaid syntax errors found in file: " + relative_path + "\n" + "\n".join(errors) + return ( + "Mermaid syntax errors found in file: " + relative_path + "\n" + "\n".join(errors) + ) else: return "All mermaid diagrams in file: " + relative_path + " are syntax correct" - + except Exception as e: return f"Error processing file: {str(e)}" @@ -112,36 +117,36 @@ async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> st def extract_mermaid_blocks(content: str) -> List[Tuple[int, str]]: """ Extract all mermaid code blocks from markdown content. - + Returns: List of tuples containing (line_number, diagram_content) """ mermaid_blocks = [] - lines = content.split('\n') + lines = content.split("\n") i = 0 - + while i < len(lines): line = lines[i].strip() - + # Look for mermaid code block start - if line == '```mermaid' or line.startswith('```mermaid'): + if line == "```mermaid" or line.startswith("```mermaid"): start_line = i + 1 diagram_lines = [] i += 1 - + # Collect lines until we find the closing ``` while i < len(lines): - if lines[i].strip() == '```': + if lines[i].strip() == "```": break diagram_lines.append(lines[i]) i += 1 - + if diagram_lines: # Only add non-empty diagrams - diagram_content = '\n'.join(diagram_lines) + diagram_content = "\n".join(diagram_lines) mermaid_blocks.append((start_line, diagram_content)) - + i += 1 - + return mermaid_blocks @@ -151,9 +156,13 @@ def extract_mermaid_blocks(content: str) -> List[Tuple[int, str]]: # Skip it proactively so SpiderMonkey is never loaded into the process. _PYTHONMONKEY_BROKEN = sys.version_info >= (3, 12) -# mermaid-py spawns a Node.js subprocess that can hang indefinitely (e.g. when -# Node.js is missing or the mermaid CLI is misconfigured). Enabled by default; -# set MERMAID_VALIDATE=0 to disable. +# mermaid-py validates diagrams by sending them to a remote rendering service +# (https://mermaid.ink by default, overridable via the MERMAID_INK_SERVER env +# var understood by mermaid-py itself) rather than spawning a local Node.js +# subprocess. Note that this sends diagram content to that third-party service. +# It can still hang or fail when the service is unreachable (no network egress, +# DNS/firewall issues, or an outage). Enabled by default; set MERMAID_VALIDATE=0 +# to disable. _MERMAID_PY_BROKEN = os.environ.get("MERMAID_VALIDATE", "1") == "0" _MERMAID_PY_PROBED = True # Skip probing — rely on env var @@ -178,16 +187,14 @@ async def _try_pythonmonkey_parse(diagram_content: str) -> str | None: return None old_stderr = sys.stderr - sys.stderr = open(os.devnull, 'w') + sys.stderr = open(os.devnull, "w") try: if ( _main_loop is not None and _main_loop.is_running() and threading.get_ident() != _main_loop_thread_ident ): - fut = asyncio.run_coroutine_threadsafe( - parse_mermaid_py(diagram_content), _main_loop - ) + fut = asyncio.run_coroutine_threadsafe(parse_mermaid_py(diagram_content), _main_loop) await asyncio.wrap_future(fut) else: await parse_mermaid_py(diagram_content) @@ -218,6 +225,7 @@ def _parse_via_mermaid_py(diagram_content: str) -> str: text, otherwise a successful SVG gets reported as a parse error. """ import mermaid as md + try: md.Mermaid(diagram_content) return "" @@ -244,7 +252,9 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s # Validation disabled/unavailable is not a syntax error — a # non-empty return here would make callers report valid diagrams # as broken and send agents into fix loops. - logger.debug("Diagram %d: validation skipped (mermaid-py disabled or unavailable)", diagram_num) + logger.debug( + "Diagram %d: validation skipped (mermaid-py disabled or unavailable)", diagram_num + ) return "" try: core_error = await asyncio.wait_for( @@ -254,9 +264,12 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s except asyncio.TimeoutError: # Inconclusive, not a parse failure. Latch so the remaining # diagrams (and later calls) don't each block 15s on the same - # broken Node.js setup. + # unreachable rendering service. _MERMAID_PY_BROKEN = True - logger.warning("Diagram %d: mermaid validation timed out (15s); skipping further validation", diagram_num) + logger.warning( + "Diagram %d: mermaid validation timed out (15s); skipping further validation", + diagram_num, + ) return "" except Exception as e: return f" Diagram {diagram_num}: Exception during validation - {str(e)}" @@ -264,11 +277,11 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s if not core_error: return "" - line_match = re.search(r'line (\d+)', core_error) + line_match = re.search(r"line (\d+)", core_error) if line_match: error_line_in_diagram = int(line_match.group(1)) actual_line_in_file = line_start + error_line_in_diagram - newline = '\n' + newline = "\n" return f"Diagram {diagram_num}: Parse error on line {actual_line_in_file}:{newline}{newline.join(core_error.split(newline)[1:])}" return f"Diagram {diagram_num}: {core_error}" @@ -276,6 +289,7 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s if __name__ == "__main__": # Test with the provided file import asyncio + test_file = "output/docs/SWE_agent-docs/agent_hooks.md" result = asyncio.run(validate_mermaid_diagrams(test_file, "agent_hooks.md")) - print(result) \ No newline at end of file + print(result) diff --git a/pyproject.toml b/pyproject.toml index 2675acfe..6f71a9ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,10 @@ dependencies = [ ] [external] -# Node.js is required for mermaid-py which validates mermaid diagrams in generated documentation +# Node.js/npm are required at *install* time by PythonMonkey (pulled in via +# mermaid-parser-py), whose pminit helper shells out to npm to populate its JS +# dependencies. mermaid-py itself does not need Node.js: it validates diagrams +# with an HTTP call to a remote rendering service (mermaid.ink by default). build-requires = [ { name = "nodejs", version = ">=14.0.0" } ] @@ -127,6 +130,11 @@ disallow_incomplete_defs = false line-length = 100 target-version = "py312" +[tool.ruff.lint] +# ruff 0.16 widened its default rule set considerably; pin the previous +# defaults so an unpinned `pip install ruff` in CI stays stable. +select = ["E4", "E7", "E9", "F"] + [tool.pytest.ini_options] testpaths = ["tests"] norecursedirs = [".venv", "atlascloud", "titan-sight"]