Mark PyDABs generated files as auto-generated

python/codegen/codegen/jsonschema.py

@@ -155,13 +155,15 @@ def get_schemas():
         ["$defs", "github.com", "databricks", "cli", "bundle", "config"],
     )
 
-    # we don't need all spec, only get supported types
     flat_spec = {**sdk_types_spec, **resource_types_spec}
-    flat_spec = {
-        key: value for key, value in flat_spec.items() if packages.should_load_ref(key)
-    }
+
+    # Load only types reachable from the root resources, following $refs.
+    reachable = _collect_reachable_refs(packages.RESOURCE_TYPES, flat_spec)
 
     for name, schema in flat_spec.items():
+        if name not in reachable:
+            continue
+
         try:
             output[name] = _parse_schema(schema)
         except Exception as e:
@@ -170,6 +172,39 @@ def get_schemas():
     return output
 
 
+def _refs_in_raw_schema(schema: dict) -> list[str]:
+    schema = _unwrap_variable(schema) or schema
+
+    return [
+        prop["$ref"].split("/")[-1]
+        for prop in schema.get("properties", {}).values()
+        if prop.get("$ref")
+    ]
+
+
+def _collect_reachable_refs(roots: list[str], flat_spec: dict) -> set[str]:
+    reachable: set[str] = set()
+    stack = list(roots)
+
+    while stack:
+        current = stack.pop()
+        if current in reachable:
+            continue
+
+        reachable.add(current)
+
+        schema = flat_spec.get(current)
+        if schema is None:
+            # primitives and types outside the loaded subtrees are terminal
+            continue
+
+        for ref in _refs_in_raw_schema(schema):
+            if ref not in reachable:
+                stack.append(ref)
+
+    return reachable
+
+
 def _get_spec_path(spec: dict, path: list[str]) -> dict:
     for key in path:
         spec = spec[key]

python/codegen/codegen/main.py

@@ -15,6 +15,8 @@
 from codegen.generated_dataclass import GeneratedDataclass, GeneratedType
 from codegen.generated_enum import GeneratedEnum
 
+HEADER = "# Code generated by pydabs-codegen. DO NOT EDIT.\n\n"
+
 
 def main(output: str):
     schemas = openapi.get_schemas()
@@ -176,7 +178,7 @@ def _write_exports(
 
     source_path = Path(output) / package_path / "__init__.py"
     source_path.parent.mkdir(exist_ok=True, parents=True)
-    source_path.write_text(b.build())
+    source_path.write_text(HEADER + b.build())
 
     print(f"Writing exports into {source_path}")
 
@@ -327,7 +329,7 @@ def _write_code(
         source_path = Path(output) / (package_path + ".py")
 
         source_path.parent.mkdir(exist_ok=True, parents=True)
-        source_path.write_text(code)
+        source_path.write_text(HEADER + code)
 
 
 if __name__ == "__main__":

python/codegen/codegen/packages.py

@@ -13,19 +13,6 @@
 
 RESOURCE_TYPES = list(RESOURCE_NAMESPACE.keys())
 
-# Namespaces to load from OpenAPI spec.
-#
-# We can't load all types because of errors while loading some of them.
-LOADED_NAMESPACES = [
-    "compute",
-    "jobs",
-    "pipelines",
-    "resources",
-    "catalog",
-    "sql",
-    "iam",
-]
-
 RENAMES = {
     "string": "str",
     "boolean": "bool",
@@ -58,16 +45,6 @@ def is_resource(ref: str) -> bool:
     return ref in RESOURCE_TYPES
 
 
-def should_load_ref(ref: str) -> bool:
-    name = ref.split("/")[-1]
-
-    for namespace in LOADED_NAMESPACES:
-        if name.startswith(f"{namespace}."):
-            return True
-
-    return name in PRIMITIVES
-
-
 def get_root_package(namespace: str) -> str:
     return f"databricks.bundles.{namespace}"
 

python/databricks/bundles/.gitattributes

@@ -1,5 +1,5 @@
-catalogs/** linguist-generated=true
-jobs/** linguist-generated=true
-pipelines/** linguist-generated=true
-schemas/** linguist-generated=true
-volumes/** linguist-generated=true
+# Generated by pydabs-codegen (see python/codegen). Each generated namespace
+# has a _models/ tree and an __init__.py; core/ is hand-written, so unset it.
+*/_models/** linguist-generated=true
+*/__init__.py linguist-generated=true
+core/__init__.py linguist-generated=false