diff --git a/CHANGELOG.md b/CHANGELOG.md index bb35f49c..0fc224cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ All notable changes to this project will be documented in this file. +## Unreleased + +### Added + +- Hosted auto-label support, matching the Roboflow MCP tools: + - `Workspace.autolabel_models()` — list the foundation-model catalog + (`gpt-6-astra-boxes`, `sam3-rle`, `gemini-boxes`, ...) with availability, + guidance and credits per image. + - `Project.autolabel_preview(model, image, ontology=...)` — free single-image + preview to compare models before starting a job. `image` accepts an HTTPS + URL, a local file path or a base64 string. + - `Project.autolabel(batch_id, model, model_type="foundational" | "roboflow", ...)` + — start a job over a batch; returns `{jobId, annotationJobId}`. The + `ontology` is keyed by prompt (`{"kitten": "cat", "tabby": "cat"}`), so + several prompts can share one output class. `preserve_existing_annotations=True` + keeps annotations already on the images (the server default replaces them). + - `Project.autolabel_job(job_id)` / `Workspace.autolabel_job(job_id)` — poll + per-subjob progress. + - `roboflow autolabel models | preview | start | job` CLI commands. + `start --preserve-existing` mirrors the SDK flag; `job -p ws/project` + resolves the workspace the same way `start` does. + ## 1.4.1 ### Added diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 960a4489..10ec349a 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -239,6 +239,44 @@ project.accept_annotation_job_images( ) ``` +### Auto-label a batch with a foundation model + +```bash +roboflow autolabel models +roboflow autolabel preview -p my-project -m sam3-rle --image https://example.com/sample.jpg \ + --class cat --class dog +roboflow autolabel start -p my-project --batch-id -m gpt-6-astra-boxes \ + --ontology '{"a cat": "cat", "a dog": "dog"}' --confidence 0.5 --reviewer b@co.com +roboflow autolabel start -p my-project --batch-id -m my-project/3 --model-type roboflow +roboflow autolabel start -p my-project --batch-id -m sam3-rle --preserve-existing +roboflow autolabel job +roboflow autolabel job -p other-workspace/my-project +``` + +`models` lists the catalog for the workspace (id, availability, credits per +image, default). `preview` runs one image through a model for free so you can +compare candidates before spending credits. `start` creates the job and prints +`jobId` and `annotationJobId`; poll it with `job`. Pass the ontology either as +repeated `--class` flags or as `--ontology` JSON. The ontology is keyed by +**prompt**, not by class: `'{"kitten": "cat", "tabby": "cat"}'` labels whatever +matches either prompt as class `cat`. That direction is what lets several +prompts share one output class. JSON options also accept a curl-style file +reference (`--ontology @ontology.json`). +`--image` accepts an HTTPS URL or a local file. By default a job replaces the +annotations already on the batch images; `--preserve-existing` keeps them and +only adds new ones. `job` looks the id up in your default workspace, so when +the job was started with a `workspace/project` shorthand pass the same `-p` to +`job`. + +The same operations are available in Python: + +```python +models = workspace.autolabel_models()["models"] +preview = project.autolabel_preview("sam3-rle", "sample.jpg", ontology={"cat": "cat"}) +job = project.autolabel("batch-id", model="gpt-6-astra-boxes", ontology={"a cat": "cat"}) +project.autolabel_job(job["jobId"])["status"] +``` + ### RFDM devices (v2 deployments) Workspace-scoped device management — backed by the external Deployments API @@ -457,6 +495,7 @@ Version numbers are always numeric — that's how `x/y` is disambiguated between | `workflow` | Manage workflows | | `folder` | Manage workspace folders | | `annotation` | Annotation batches and jobs | +| `autolabel` | Auto-label batches with hosted foundation or Roboflow models | | `asynctasks` | Inspect async background tasks (e.g. project forks) | | `trash` | List items in Trash | | `universe` | Search Roboflow Universe | diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index 9e9b4fc2..e29b2b2e 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -1309,6 +1309,11 @@ def _annotation_pagination_params(api_key, *, limit, after=None, show_empty=None def _annotation_administration_response(response): + return _json_response_or_raise(response) + + +def _json_response_or_raise(response): + """Return the JSON body of a 2xx response; raise ``RoboflowError`` with the HTTP status otherwise.""" if not 200 <= response.status_code < 300: message = response.text try: @@ -1323,6 +1328,124 @@ def _annotation_administration_response(response): return response.json() +# --------------------------------------------------------------------------- +# Hosted auto-label endpoints +# --------------------------------------------------------------------------- + + +def _autolabel_response(response): + return _json_response_or_raise(response) + + +def list_autolabel_models(api_key, workspace_url): + """Fetch the foundation-model catalog for hosted auto-labeling. + + Calls ``GET /:workspace/autolabel/models``. Returns ``{models: [...]}`` + where each entry carries ``id``, ``name``, ``guidance``, ``ontologyFormat``, + ``creditsPerImage``, ``isDefault`` and per-workspace ``available`` (with an + ``unavailableReason`` when the plan blocks a model). + """ + response = requests.get( + f"{API_URL}/{workspace_url}/autolabel/models", + params={"api_key": api_key}, + ) + return _autolabel_response(response) + + +def preview_autolabel( + api_key, + workspace_url, + project_url, + *, + model_type, + image, + ontology=None, + confidence_threshold=None, +): + """Preview one image with a foundation model before starting a job. + + Calls ``POST /:workspace/:project/autolabel/preview``. Free: no job is + created and no credits are spent. ``image`` is + ``{"type": "url" | "base64", "value": ...}`` and ``ontology`` is keyed by + prompt: ``{"kitten": "cat"}`` labels prompt matches as class ``cat``. + Returns + ``{model, predictions, summary, blockErrors?}``. + """ + payload = {"modelType": model_type, "image": image} + if ontology is not None: + payload["ontology"] = ontology + if confidence_threshold is not None: + payload["confidenceThreshold"] = confidence_threshold + response = requests.post( + f"{API_URL}/{workspace_url}/{project_url}/autolabel/preview", + params={"api_key": api_key}, + json=payload, + ) + return _autolabel_response(response) + + +def start_autolabel_job( + api_key, + workspace_url, + project_url, + *, + batch_id, + model_type, + ontology=None, + num_images_to_label=None, + default_confidence=None, + confidence_thresholds=None, + run_nms=None, + reviewer_email=None, + model_options=None, + preserve_existing_annotations=None, +): + """Start a hosted auto-label job over a batch. + + Calls ``POST /:workspace/:project/autolabel``. ``model_type`` is sent + as-is: a catalog id from ``list_autolabel_models`` (for example + ``gpt-6-astra-boxes`` or ``sam3-rle``) or ``custom_roboflow`` with the + Roboflow model id in ``model_options["modelId"]``. ``ontology`` is keyed + by prompt: ``{"kitten": "cat"}`` labels prompt matches as class ``cat``. + The backend fans + ``default_confidence`` out across the ontology when + ``confidence_thresholds`` is omitted and defaults ``num_images_to_label`` + to the whole batch. ``preserve_existing_annotations`` keeps annotations + already on the images and only adds new ones; the server default (False) + replaces them. Returns ``{jobId, annotationJobId, message}``. + """ + payload = {"batchId": batch_id, "modelType": model_type} + optional = { + "ontology": ontology, + "numImagesToLabel": num_images_to_label, + "defaultConfidence": default_confidence, + "confidenceThresholds": confidence_thresholds, + "runNMS": run_nms, + "reviewerEmail": reviewer_email, + "modelOptions": model_options, + "preserveExistingAnnotations": preserve_existing_annotations, + } + payload.update({key: value for key, value in optional.items() if value is not None}) + response = requests.post( + f"{API_URL}/{workspace_url}/{project_url}/autolabel", + params={"api_key": api_key}, + json=payload, + ) + return _autolabel_response(response) + + +def get_autolabel_job(api_key, workspace_url, job_id): + """Fetch per-subjob status and progress for a hosted auto-label job. + + Calls ``GET /:workspace/autolabel/jobs/:jobId``. + """ + response = requests.get( + f"{API_URL}/{workspace_url}/autolabel/jobs/{job_id}", + params={"api_key": api_key}, + ) + return _autolabel_response(response) + + # --------------------------------------------------------------------------- # Phase 2: Folder (project group) endpoints # --------------------------------------------------------------------------- diff --git a/roboflow/cli/__init__.py b/roboflow/cli/__init__.py index 3befa965..2c2610be 100644 --- a/roboflow/cli/__init__.py +++ b/roboflow/cli/__init__.py @@ -188,6 +188,7 @@ def _walk(group: Any, prefix: str = "") -> None: from roboflow.cli.handlers.api_key import api_key_app # noqa: E402 from roboflow.cli.handlers.asynctasks import asynctasks_app # noqa: E402 from roboflow.cli.handlers.auth import auth_app # noqa: E402 +from roboflow.cli.handlers.autolabel import autolabel_app # noqa: E402 from roboflow.cli.handlers.batch import batch_app # noqa: E402 from roboflow.cli.handlers.completion import completion_app # noqa: E402 from roboflow.cli.handlers.deployment import deployment_app # noqa: E402 @@ -213,6 +214,7 @@ def _walk(group: Any, prefix: str = "") -> None: app.add_typer(api_key_app, name="api-key") app.add_typer(asynctasks_app, name="asynctasks") app.add_typer(auth_app, name="auth") +app.add_typer(autolabel_app, name="autolabel") app.add_typer(batch_app, name="batch") app.add_typer(completion_app, name="completion") app.add_typer(deployment_app, name="deployment") diff --git a/roboflow/cli/_resolver.py b/roboflow/cli/_resolver.py index 11f5e3c5..25b45d1e 100644 --- a/roboflow/cli/_resolver.py +++ b/roboflow/cli/_resolver.py @@ -135,3 +135,31 @@ def resolve_ws_and_key(args) -> Optional[Tuple[str, str]]: return None return ws, api_key + + +def resolve_project_context(args) -> Optional[Tuple[str, str, str]]: + """Resolve API key, workspace and project from CLI args. + + Parses ``args.project`` (any ``resolve_resource`` shorthand, honouring + ``args.workspace`` as an override) and loads the API key for that + workspace. Returns ``(api_key, workspace_url, project_slug)`` or ``None`` + after calling ``output_error`` on failure; a missing key exits with the + auth code (2), matching ``resolve_ws_and_key``. + """ + from roboflow.cli._output import output_error + from roboflow.config import load_roboflow_api_key + + try: + workspace, project, _version = resolve_resource( + args.project, workspace_override=getattr(args, "workspace", None) + ) + except ValueError as exc: + output_error(args, str(exc)) + return None + + api_key = getattr(args, "api_key", None) or load_roboflow_api_key(workspace) + if not api_key: + output_error(args, "No API key found.", hint="Set ROBOFLOW_API_KEY or run 'roboflow auth login'.", exit_code=2) + return None + + return api_key, workspace, project diff --git a/roboflow/cli/handlers/annotation.py b/roboflow/cli/handlers/annotation.py index 2ceca056..f4da8b92 100644 --- a/roboflow/cli/handlers/annotation.py +++ b/roboflow/cli/handlers/annotation.py @@ -465,20 +465,9 @@ def job_delete_annotations( def _resolve_project_context(args: Any) -> Optional[tuple[str, str, str]]: - from roboflow.cli._output import output_error - from roboflow.cli._resolver import resolve_resource - from roboflow.config import load_roboflow_api_key + from roboflow.cli._resolver import resolve_project_context - try: - workspace, project, _version = resolve_resource(args.project, workspace_override=args.workspace) - except ValueError as exc: - output_error(args, str(exc)) - return None - api_key = args.api_key or load_roboflow_api_key(workspace) - if not api_key: - output_error(args, "No API key found.", hint="Set ROBOFLOW_API_KEY or run 'roboflow auth login'.", exit_code=2) - return None - return api_key, workspace, project + return resolve_project_context(args) def _call(args: Any, operation: Callable[[str, str, str], Any]) -> Any: diff --git a/roboflow/cli/handlers/autolabel.py b/roboflow/cli/handlers/autolabel.py new file mode 100644 index 00000000..0fe3eab2 --- /dev/null +++ b/roboflow/cli/handlers/autolabel.py @@ -0,0 +1,280 @@ +"""Hosted auto-label commands: list models, preview, start and track jobs.""" + +from __future__ import annotations + +from typing import Annotated, Any, Callable, Optional + +import typer + +from roboflow.cli._compat import SortedGroup, ctx_to_args + +autolabel_app = typer.Typer(cls=SortedGroup, help="Hosted auto-label jobs", no_args_is_help=True) + + +@autolabel_app.command("models") +def models(ctx: typer.Context) -> None: + """List the foundation models available for auto-labeling in this workspace.""" + _models(ctx_to_args(ctx)) + + +@autolabel_app.command("preview") +def preview( + ctx: typer.Context, + project: Annotated[str, typer.Option("-p", "--project", help="Project ID")], + model: Annotated[str, typer.Option("-m", "--model", help="Foundation model ID from 'autolabel models'")], + image: Annotated[str, typer.Option("--image", help="Sample image: HTTPS URL or local file path")], + classes: Annotated[ + Optional[list[str]], + typer.Option("--class", help="Class name to detect; repeat for multiple classes"), + ] = None, + ontology: Annotated[ + Optional[str], + typer.Option( + "--ontology", + help="JSON object mapping each text prompt to the class name it labels, e.g. " + '\'{"kitten": "cat", "tabby": "cat"}\'. Note the direction: prompt first. Also accepts @ontology.json', + ), + ] = None, + confidence: Annotated[ + Optional[float], typer.Option("--confidence", help="Detection threshold from 0.0 to 1.0 (sam3 only)") + ] = None, +) -> None: + """Preview one image with a foundation model. Free: no job is created.""" + args = ctx_to_args(ctx, project=project) + resolved_ontology = _parse_ontology(args, ontology, classes) + resolved_image = _parse_image(args, image) + + _project_command( + args, + lambda key, workspace, proj: _rfapi().preview_autolabel( + key, + workspace, + proj, + model_type=model, + image=resolved_image, + ontology=resolved_ontology, + confidence_threshold=confidence, + ), + ) + + +@autolabel_app.command("start") +def start( + ctx: typer.Context, + project: Annotated[str, typer.Option("-p", "--project", help="Project ID")], + batch_id: Annotated[str, typer.Option("--batch-id", help="Source batch ID containing the images to label")], + model: Annotated[ + str, + typer.Option( + "-m", + "--model", + help="Foundation model ID from 'autolabel models', or a Roboflow model ID with --model-type roboflow", + ), + ], + model_type: Annotated[ + str, + typer.Option("--model-type", help="'foundational' (hosted foundation model) or 'roboflow' (trained model)"), + ] = "foundational", + classes: Annotated[ + Optional[list[str]], + typer.Option("--class", help="Class name to label; repeat for multiple classes"), + ] = None, + ontology: Annotated[ + Optional[str], + typer.Option( + "--ontology", + help="JSON object mapping each text prompt to the class name it labels, e.g. " + '\'{"kitten": "cat", "tabby": "cat"}\'. Note the direction: prompt first. Also accepts @ontology.json', + ), + ] = None, + num_images: Annotated[ + Optional[int], typer.Option("--num-images", help="Number of images to label (default: whole batch)") + ] = None, + confidence: Annotated[ + Optional[float], typer.Option("--confidence", help="Confidence threshold applied to every class") + ] = None, + confidence_thresholds: Annotated[ + Optional[str], + typer.Option("--confidence-thresholds", help="JSON per-class thresholds, e.g. '{\"cat\": 0.5}'"), + ] = None, + no_nms: Annotated[bool, typer.Option("--no-nms", help="Disable non-max suppression")] = False, + reviewer: Annotated[ + Optional[str], typer.Option("--reviewer", help="Reviewer email for the resulting annotation job") + ] = None, + model_options: Annotated[ + Optional[str], + typer.Option( + "--model-options", help='JSON model options, e.g. \'{"outputFormat": "polygon"}\', or @options.json' + ), + ] = None, + preserve_existing: Annotated[ + bool, + typer.Option( + "--preserve-existing", + help="Keep annotations already on the batch images and only add new ones " + "(by default the job replaces them)", + ), + ] = False, +) -> None: + """Start a hosted auto-label job over a batch of images.""" + args = ctx_to_args(ctx, project=project) + resolved_ontology = _parse_ontology(args, ontology, classes) + resolved_thresholds = _parse_json_option(args, "--confidence-thresholds", confidence_thresholds) + resolved_options = _parse_json_option(args, "--model-options", model_options) + from roboflow.util.autolabel_utils import resolve_model + + def start_job(key: str, workspace: str, proj: str) -> Any: + wire_model_type, wire_options = resolve_model(model, model_type, resolved_options) + return _rfapi().start_autolabel_job( + key, + workspace, + proj, + batch_id=batch_id, + model_type=wire_model_type, + ontology=resolved_ontology, + num_images_to_label=num_images, + default_confidence=confidence, + confidence_thresholds=resolved_thresholds, + run_nms=False if no_nms else None, + reviewer_email=reviewer, + model_options=wire_options, + preserve_existing_annotations=True if preserve_existing else None, + ) + + _project_command(args, start_job) + + +@autolabel_app.command("job") +def job( + ctx: typer.Context, + job_id: Annotated[str, typer.Argument(help="Auto-label job ID returned by 'autolabel start'")], + project: Annotated[ + Optional[str], + typer.Option( + "-p", + "--project", + help="Project the job was started on (accepts 'workspace/project'); resolves the workspace " + "the same way 'autolabel start' does. Defaults to --workspace or the default workspace.", + ), + ] = None, +) -> None: + """Get status and per-subjob progress for an auto-label job.""" + args = ctx_to_args(ctx, project=project) + + def get_job(key: str, workspace: str, *_project: str) -> Any: + return _rfapi().get_autolabel_job(key, workspace, job_id) + + if project: + _project_command(args, get_job) + else: + _workspace_command(args, get_job) + + +# --------------------------------------------------------------------------- +# Business logic +# --------------------------------------------------------------------------- + + +def _rfapi(): + from roboflow.adapters import rfapi + + return rfapi + + +def _parse_json_option(args: Any, flag: str, raw: Optional[str]) -> Optional[dict]: + """Parse an optional JSON-object flag: inline JSON or ``@path`` to a file. Exits on invalid input.""" + if raw is None: + return None + from roboflow.cli.handlers.train import _parse_json_flag + + return _parse_json_flag(args, raw, flag) + + +def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[str]]) -> Optional[dict]: + """Build the ontology from --ontology (JSON or @file, takes precedence) or repeated --class. + + ``--ontology`` is keyed by prompt, not by class: ``{"kitten": "cat"}`` labels + whatever matches the prompt "kitten" as class ``cat``. + """ + from roboflow.util.autolabel_utils import ontology_payload + + if ontology is not None: + return ontology_payload(_parse_json_option(args, "--ontology", ontology)) + if classes: + return ontology_payload(classes) + return None + + +def _parse_image(args: Any, image: str) -> dict: + """Build the --image payload before any network call, so a bad path fails fast and cleanly.""" + from roboflow.cli._output import output_error + from roboflow.util.autolabel_utils import image_payload + + hint = "Pass --image as an HTTPS URL or the path of a readable local image file." + try: + return image_payload(image) + except OSError as exc: + output_error(args, f"Cannot read image {image}: {exc.strerror or exc}", hint=hint) + except ValueError as exc: + output_error(args, str(exc), hint=hint) + return {} # unreachable: output_error exits + + +def _run(args: Any, operation: Callable[[], Any], text: Optional[Callable[[Any], str]] = None) -> None: + from roboflow.cli._output import output, output_api_error, output_error + + try: + data = operation() + except _rfapi().RoboflowError as exc: + output_api_error(args, exc) + return + except ValueError as exc: + output_error(args, str(exc)) + return + output(args, data, text=text(data) if text else None) + + +def _workspace_command( + args: Any, operation: Callable[[str, str], Any], text: Optional[Callable[[Any], str]] = None +) -> None: + from roboflow.cli._resolver import resolve_ws_and_key + + resolved = resolve_ws_and_key(args) + if resolved is None: + return + workspace_url, api_key = resolved + _run(args, lambda: operation(api_key, workspace_url), text=text) + + +def _project_command(args: Any, operation: Callable[[str, str, str], Any]) -> None: + from roboflow.cli._resolver import resolve_project_context + + resolved = resolve_project_context(args) + if resolved is None: + return + api_key, workspace, project = resolved + _run(args, lambda: operation(api_key, workspace, project)) + + +def _models(args: Any) -> None: + from roboflow.cli._table import format_table + + def table(data: Any) -> str: + rows = [ + { + "id": model.get("id", ""), + "name": model.get("name", ""), + "available": "yes" if model.get("available", True) else "no", + "default": "yes" if model.get("isDefault") else "", + "credits": model.get("creditsPerImage", ""), + "ontology": model.get("ontologyFormat", ""), + } + for model in data.get("models", []) + ] + return format_table( + rows, + columns=["id", "name", "available", "default", "credits", "ontology"], + headers=["ID", "NAME", "AVAILABLE", "DEFAULT", "CREDITS/IMAGE", "ONTOLOGY"], + ) + + _workspace_command(args, lambda key, workspace: _rfapi().list_autolabel_models(key, workspace), text=table) diff --git a/roboflow/core/project.py b/roboflow/core/project.py index dae10928..f9fe8fb5 100644 --- a/roboflow/core/project.py +++ b/roboflow/core/project.py @@ -14,6 +14,10 @@ from roboflow.adapters.rfapi import AnnotationSaveError, ImageUploadError from roboflow.config import API_URL, DEMO_KEYS from roboflow.core.version import Version +from roboflow.util.autolabel_utils import Ontology as _AutolabelOntology +from roboflow.util.autolabel_utils import image_payload as _autolabel_image_payload +from roboflow.util.autolabel_utils import ontology_payload as _autolabel_ontology_payload +from roboflow.util.autolabel_utils import resolve_model as _resolve_autolabel_model from roboflow.util.general import Retry from roboflow.util.image_utils import load_labelmap @@ -1155,6 +1159,127 @@ def delete_annotation_job_annotations(self, job_id: str) -> Dict: """Delete project annotations from every image assigned to a job.""" return rfapi.delete_annotation_job_annotations(self.__api_key, self.__workspace, self.__project_name, job_id) + def autolabel_preview( + self, + model: str, + image: str, + ontology: Optional[_AutolabelOntology] = None, + confidence_threshold: Optional[float] = None, + ) -> Dict: + """Preview one image with a foundation model before starting an auto-label job. + + Previews are free: no job is created and no credits are spent. Use it to + compare the candidates from ``Workspace.autolabel_models()`` on a sample + image and start the real job with the winner. + + Args: + model: Foundation model id from ``Workspace.autolabel_models()`` + (e.g. ``"gpt-6-astra-boxes"``, ``"sam3-rle"``, ``"gemini-boxes"``). + image: HTTPS URL, local file path, or base64-encoded image. + ontology: ``{"text prompt": "class name"}`` -- keyed by prompt, so + several prompts can share one class + (``{"kitten": "cat", "tabby": "cat"}``). A plain list of class + names prompts each class with its own name. Defaults to the + dataset's own classes. + confidence_threshold: Detection threshold between 0.0 and 1.0 + (sam3 only; other models report fixed confidence). + + Returns: + Dict: ``{model, predictions, summary, blockErrors?}``. ``summary.byClass`` + carries per-class counts and max confidence, and + ``summary.classesWithNoDetections`` lists requested classes the model + did not find. + """ + return rfapi.preview_autolabel( + self.__api_key, + self.__workspace, + self.__project_name, + model_type=model, + image=_autolabel_image_payload(image), + ontology=_autolabel_ontology_payload(ontology), + confidence_threshold=confidence_threshold, + ) + + def autolabel( + self, + batch_id: str, + model: str, + model_type: str = "foundational", + ontology: Optional[_AutolabelOntology] = None, + num_images: Optional[int] = None, + confidence: Optional[float] = None, + confidence_thresholds: Optional[Dict[str, float]] = None, + run_nms: Optional[bool] = None, + reviewer_email: Optional[str] = None, + model_options: Optional[Dict] = None, + preserve_existing_annotations: Optional[bool] = None, + ) -> Dict: + """Start a hosted auto-label job over a batch of images. + + Args: + batch_id: Source batch containing the images to auto-label. + model: For ``model_type="foundational"``, a model id from + ``Workspace.autolabel_models()`` (e.g. ``"gpt-6-astra-boxes"``, + ``"sam3-rle"``, ``"sam3-polygon"``, ``"gemini-boxes"``). For + ``model_type="roboflow"``, a Roboflow model id such as + ``"project-slug/3"`` or ``"workspace-slug/model-id"``. + model_type: ``"foundational"`` (hosted foundation model, sent as-is; + the backend resolves catalog ids) or ``"roboflow"`` (a + Roboflow-trained model). + ontology: ``{"text prompt": "class name"}`` -- keyed by prompt, not + by class, so several prompts can collapse onto one output class + (``{"kitten": "cat", "tabby": "cat"}`` labels both as ``cat``). + A plain list of class names prompts each class with its own + name. For models with ``ontologyFormat="promptMap"`` (sam3) the + prompts are sent to the model; for ``ontologyFormat="classes"`` + only the class names are used. Defaults to the dataset's classes + (or the trained model's classes for ``model_type="roboflow"``). + num_images: Number of images from the batch to label. Defaults to + the whole batch. + confidence: Confidence threshold applied to every class (mirrors + the UI slider). Ignored when ``confidence_thresholds`` is set. + confidence_thresholds: Per-class threshold override, e.g. + ``{"cat": 0.5, "dog": 0.6}``. + run_nms: Whether to run non-max suppression (server default: True). + reviewer_email: Reviewer for the resulting annotation job. Must be a + workspace member; defaults to the workspace owner. + model_options: Model-specific options, e.g. + ``{"outputFormat": "polygon"}`` for segmentation output. + preserve_existing_annotations: ``True`` keeps annotations already + on the batch images and only adds new ones. The server default + (``False``) replaces them, so set this when the batch contains + images that were already labeled or reviewed. + + Returns: + Dict: ``{jobId, annotationJobId, message}``. Poll progress with + ``autolabel_job(jobId)``. + + Example: + >>> job = project.autolabel("batch-id", model="gpt-6-astra-boxes", + ... ontology={"a cat": "cat", "a dog": "dog"}) + >>> project.autolabel_job(job["jobId"])["status"] + """ + wire_model_type, model_options = _resolve_autolabel_model(model, model_type, model_options) + return rfapi.start_autolabel_job( + self.__api_key, + self.__workspace, + self.__project_name, + batch_id=batch_id, + model_type=wire_model_type, + ontology=_autolabel_ontology_payload(ontology), + num_images_to_label=num_images, + default_confidence=confidence, + confidence_thresholds=confidence_thresholds, + run_nms=run_nms, + reviewer_email=reviewer_email, + model_options=model_options, + preserve_existing_annotations=preserve_existing_annotations, + ) + + def autolabel_job(self, job_id: str) -> Dict: + """Get status and per-subjob progress for an auto-label job started with ``autolabel``.""" + return rfapi.get_autolabel_job(self.__api_key, self.__workspace, job_id) + def get_batches(self) -> Dict: """ Get a list of all batches in the project. diff --git a/roboflow/core/workspace.py b/roboflow/core/workspace.py index 7a084102..69fd055f 100644 --- a/roboflow/core/workspace.py +++ b/roboflow/core/workspace.py @@ -1669,6 +1669,33 @@ def restore_from_trash(self, item_type: str, item_id: str, parent_id: Optional[s """ return rfapi.restore_trash_item(self.__api_key, self.url, item_type, item_id, parent_id) + def autolabel_models(self) -> dict: + """ + List the foundation models available for hosted auto-labeling in this workspace. + + Returns ``{models: [...]}``. Each entry's ``id`` (e.g. + ``gpt-6-astra-boxes``, ``sam3-rle``, ``gemini-boxes``) is a valid + ``model`` for ``Project.autolabel`` and ``Project.autolabel_preview``. + Entries carry ``guidance`` on when to prefer each model, the + ``projectTypes`` they support, ``ontologyFormat``, ``creditsPerImage``, + ``isDefault``, and ``available`` with an ``unavailableReason`` when the + workspace plan blocks a model. + + Example: + >>> for m in ws.autolabel_models()["models"]: + ... print(m["id"], m["available"]) + """ + return rfapi.list_autolabel_models(self.__api_key, self.url) + + def autolabel_job(self, job_id: str) -> dict: + """ + Get status and per-subjob progress for a hosted auto-label job. + + Args: + job_id: the ``jobId`` returned by ``Project.autolabel``. + """ + return rfapi.get_autolabel_job(self.__api_key, self.url, job_id) + # Permanent-delete actions (empty trash / delete a single trash item # immediately) are intentionally not exposed in the SDK — they destroy # data irrecoverably and are only available through the web UI's Trash diff --git a/roboflow/util/autolabel_utils.py b/roboflow/util/autolabel_utils.py new file mode 100644 index 00000000..097e733d --- /dev/null +++ b/roboflow/util/autolabel_utils.py @@ -0,0 +1,90 @@ +"""Helpers shared by the SDK and CLI for hosted auto-label requests.""" + +from __future__ import annotations + +import base64 +import binascii +import os +from typing import Any, Dict, Iterable, Optional, Tuple, Union + +MODEL_TYPES = ("foundational", "roboflow") + + +def image_payload(image: str) -> Dict[str, str]: + """Build the ``{type, value}`` image payload for the auto-label preview endpoint. + + Accepts an HTTP(S) URL, a local file path (``~`` is expanded; the file is + read and base64-encoded) or an already base64-encoded string. Anything + else is treated as a mistyped path and rejected here, rather than being + sent to the API as "base64" and failing there with a generic inference + error. + + Raises: + ValueError: ``image`` is neither a URL, an existing file nor base64. + OSError: the file exists but cannot be read. + """ + if image.startswith(("http://", "https://")): + return {"type": "url", "value": image} + path = os.path.expanduser(image) + if os.path.isfile(path): + with open(path, "rb") as handle: + return {"type": "base64", "value": base64.b64encode(handle.read()).decode("ascii")} + compact = "".join(image.split()) + if _is_base64(compact): + return {"type": "base64", "value": compact} + raise ValueError(f"Image file not found: {image} (expected an HTTPS URL, an existing file path or base64 data)") + + +def _is_base64(value: str) -> bool: + if not value: + return False + try: + base64.b64decode(value, validate=True) + except (binascii.Error, ValueError): + return False + return True + + +Ontology = Union[Dict[str, str], Iterable[str]] + + +def ontology_payload(ontology: Optional[Ontology]) -> Optional[Dict[str, str]]: + """Normalize an ontology into the API's ``{prompt: class name}`` object. + + Note the direction: the **key is the text prompt** sent to the model and the + **value is the class name** written onto the annotations. It reads backwards + at first, but it is the shape that lets several prompts collapse onto one + output class, which is what the ontology is for:: + + {"kitten": "cat", "tabby": "cat", "puppy": "dog"} + + A class-keyed object could not express that, since its keys would have to be + unique. This is also the ``CaptionOntology`` shape the labeling worker + consumes, so nothing is translated on the way out. + + A plain iterable of class names is expanded to ``{"cat": "cat"}``, each class + prompted with its own name. + """ + if ontology is None: + return None + if isinstance(ontology, str): + raise ValueError(f"ontology must be a mapping or a list of classes, not a bare string {ontology!r}") + if isinstance(ontology, dict): + return dict(ontology) + return {name: name for name in ontology} + + +def resolve_model( + model: str, model_type: str, model_options: Optional[Dict[str, Any]] = None +) -> Tuple[str, Optional[Dict[str, Any]]]: + """Translate the public ``model``/``model_type`` pair into wire values. + + Foundation models are sent as-is (the backend resolves catalog ids such as + ``gpt-6-astra-boxes``). Roboflow-trained models are sent as + ``custom_roboflow`` with the model id in ``modelOptions.modelId``. + """ + if model_type == "roboflow": + return "custom_roboflow", {**(model_options or {}), "modelId": model} + if model_type == "foundational": + return model, model_options + raise ValueError("model_type must be 'foundational' or 'roboflow'") diff --git a/tests/adapters/test_autolabel.py b/tests/adapters/test_autolabel.py new file mode 100644 index 00000000..27947d79 --- /dev/null +++ b/tests/adapters/test_autolabel.py @@ -0,0 +1,109 @@ +"""HTTP contract tests for hosted auto-label adapters.""" + +import unittest +from unittest.mock import MagicMock, patch + +from roboflow.adapters import rfapi + + +def _response(payload=None, status_code=200, text="error"): + return MagicMock(status_code=status_code, text=text, json=lambda: payload or {"success": True}) + + +class TestAutolabelAdapters(unittest.TestCase): + @patch("roboflow.adapters.rfapi.requests.get") + def test_list_models_contract(self, mock_get): + mock_get.return_value = _response({"models": []}) + + self.assertEqual(rfapi.list_autolabel_models("key", "ws"), {"models": []}) + self.assertTrue(mock_get.call_args.args[0].endswith("/ws/autolabel/models")) + self.assertEqual(mock_get.call_args.kwargs["params"], {"api_key": "key"}) + + @patch("roboflow.adapters.rfapi.requests.get") + def test_get_job_contract(self, mock_get): + mock_get.return_value = _response({"status": "running"}) + + self.assertEqual(rfapi.get_autolabel_job("key", "ws", "job-1"), {"status": "running"}) + self.assertTrue(mock_get.call_args.args[0].endswith("/ws/autolabel/jobs/job-1")) + self.assertEqual(mock_get.call_args.kwargs["params"], {"api_key": "key"}) + + @patch("roboflow.adapters.rfapi.requests.post") + def test_preview_contract(self, mock_post): + mock_post.return_value = _response({"predictions": []}) + image = {"type": "url", "value": "https://example.com/cat.jpg"} + + rfapi.preview_autolabel("key", "ws", "proj", model_type="sam3-rle", image=image) + self.assertTrue(mock_post.call_args.args[0].endswith("/ws/proj/autolabel/preview")) + self.assertEqual(mock_post.call_args.kwargs["params"], {"api_key": "key"}) + self.assertEqual(mock_post.call_args.kwargs["json"], {"modelType": "sam3-rle", "image": image}) + + rfapi.preview_autolabel( + "key", + "ws", + "proj", + model_type="sam3-rle", + image=image, + ontology={"cat": "cat"}, + confidence_threshold=0.4, + ) + self.assertEqual( + mock_post.call_args.kwargs["json"], + {"modelType": "sam3-rle", "image": image, "ontology": {"cat": "cat"}, "confidenceThreshold": 0.4}, + ) + + @patch("roboflow.adapters.rfapi.requests.post") + def test_start_job_contract_omits_unset_fields(self, mock_post): + mock_post.return_value = _response({"jobId": "job-1"}) + + result = rfapi.start_autolabel_job("key", "ws", "proj", batch_id="batch-1", model_type="gpt-6-astra-boxes") + self.assertEqual(result, {"jobId": "job-1"}) + self.assertTrue(mock_post.call_args.args[0].endswith("/ws/proj/autolabel")) + self.assertEqual(mock_post.call_args.kwargs["params"], {"api_key": "key"}) + self.assertEqual( + mock_post.call_args.kwargs["json"], + {"batchId": "batch-1", "modelType": "gpt-6-astra-boxes"}, + ) + + @patch("roboflow.adapters.rfapi.requests.post") + def test_start_job_contract_full_payload(self, mock_post): + mock_post.return_value = _response({"jobId": "job-1"}) + + rfapi.start_autolabel_job( + "key", + "ws", + "proj", + batch_id="batch-1", + model_type="custom_roboflow", + ontology={"a cat": "cat"}, + num_images_to_label=10, + default_confidence=0.5, + confidence_thresholds={"cat": 0.6}, + run_nms=False, + reviewer_email="reviewer@example.com", + model_options={"modelId": "proj/3"}, + preserve_existing_annotations=True, + ) + self.assertEqual( + mock_post.call_args.kwargs["json"], + { + "batchId": "batch-1", + "modelType": "custom_roboflow", + "ontology": {"a cat": "cat"}, + "numImagesToLabel": 10, + "defaultConfidence": 0.5, + "confidenceThresholds": {"cat": 0.6}, + "runNMS": False, + "reviewerEmail": "reviewer@example.com", + "modelOptions": {"modelId": "proj/3"}, + "preserveExistingAnnotations": True, + }, + ) + + @patch("roboflow.adapters.rfapi.requests.post") + def test_errors_raise_roboflow_error_with_status(self, mock_post): + mock_post.return_value = _response({"error": {"message": "batch not found"}}, status_code=404) + + with self.assertRaises(rfapi.RoboflowError) as ctx: + rfapi.start_autolabel_job("key", "ws", "proj", batch_id="missing", model_type="sam3-rle") + self.assertEqual(str(ctx.exception), "batch not found") + self.assertEqual(ctx.exception.status_code, 404) diff --git a/tests/cli/test_autolabel_handler.py b/tests/cli/test_autolabel_handler.py new file mode 100644 index 00000000..580032f4 --- /dev/null +++ b/tests/cli/test_autolabel_handler.py @@ -0,0 +1,273 @@ +"""Unit tests for roboflow.cli.handlers.autolabel.""" + +import json +import os +import tempfile +import unittest +from unittest.mock import patch + +from typer.testing import CliRunner + +from roboflow.adapters.rfapi import RoboflowError +from roboflow.cli import app + +runner = CliRunner() + +# The handler resolves credentials through the shared CLI resolvers (imported lazily), +# so patching them at their definition site is what the handler sees. +_RESOLVE_PROJECT = "roboflow.cli._resolver.resolve_project_context" +_RESOLVE_WORKSPACE = "roboflow.cli._resolver.resolve_ws_and_key" +_DEFAULT_WORKSPACE = "roboflow.cli._resolver.resolve_default_workspace" +_IMAGE_PAYLOAD = "roboflow.util.autolabel_utils.image_payload" + + +class TestAutolabelRegistration(unittest.TestCase): + def test_subcommands_have_help(self): + for name in ["models", "preview", "start", "job"]: + with self.subTest(command=name): + result = runner.invoke(app, ["autolabel", name, "--help"]) + self.assertEqual(result.exit_code, 0, result.output) + + +class TestAutolabelModels(unittest.TestCase): + @patch("roboflow.adapters.rfapi.list_autolabel_models") + @patch(_RESOLVE_WORKSPACE, return_value=("ws", "key")) + def test_text_output_is_a_table(self, _resolve, mock_api): + mock_api.return_value = { + "models": [ + {"id": "gpt-6-astra-boxes", "name": "GPT-6 Astra", "available": True, "isDefault": True}, + {"id": "gemini-boxes", "name": "Gemini", "available": False, "unavailableReason": "plan"}, + ] + } + result = runner.invoke(app, ["autolabel", "models"]) + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("gpt-6-astra-boxes", result.output) + self.assertIn("gemini-boxes", result.output) + mock_api.assert_called_once_with("key", "ws") + + @patch("roboflow.adapters.rfapi.list_autolabel_models", return_value={"models": [{"id": "sam3-rle"}]}) + @patch(_RESOLVE_WORKSPACE, return_value=("ws", "key")) + def test_json_output(self, _resolve, _mock_api): + result = runner.invoke(app, ["--json", "autolabel", "models"]) + self.assertEqual(json.loads(result.output), {"models": [{"id": "sam3-rle"}]}) + + @patch("roboflow.adapters.rfapi.list_autolabel_models") + @patch(_DEFAULT_WORKSPACE, return_value=None) + def test_missing_workspace_exits_with_auth_code(self, _default, mock_api): + # CLAUDE.md pins exit code 2 for auth errors; 'workflow list' exits 2 for the same condition. + with patch.dict(os.environ, {"ROBOFLOW_API_KEY": ""}): + result = runner.invoke(app, ["autolabel", "models"]) + self.assertEqual(result.exit_code, 2, result.output) + mock_api.assert_not_called() + + +class TestAutolabelPreview(unittest.TestCase): + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={"summary": {"totalDetections": 2}}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_classes_become_identity_ontology(self, _resolve, mock_api): + result = runner.invoke( + app, + [ + "--json", + "autolabel", + "preview", + "-p", + "ws/proj", + "-m", + "sam3-rle", + "--image", + "https://example.com/cat.jpg", + "--class", + "cat", + "--class", + "dog", + "--confidence", + "0.4", + ], + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(json.loads(result.output), {"summary": {"totalDetections": 2}}) + mock_api.assert_called_once_with( + "key", + "ws", + "proj", + model_type="sam3-rle", + image={"type": "url", "value": "https://example.com/cat.jpg"}, + ontology={"cat": "cat", "dog": "dog"}, + confidence_threshold=0.4, + ) + + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_ontology_json_takes_precedence_over_classes(self, _resolve, mock_api): + runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + + ["--class", "cat", "--ontology", '{"a tabby cat": "cat"}'], + ) + self.assertEqual(mock_api.call_args.kwargs["ontology"], {"a tabby cat": "cat"}) + + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_ontology_can_be_read_from_a_file(self, _resolve, mock_api): + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle: + json.dump({"a tabby cat": "cat"}, handle) + path = handle.name + try: + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + + ["--ontology", f"@{path}"], + ) + finally: + os.unlink(path) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(mock_api.call_args.kwargs["ontology"], {"a tabby cat": "cat"}) + + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_several_prompts_may_share_one_class(self, _resolve, mock_api): + # Keying by prompt is what makes this expressible at all. + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + + ["--ontology", '{"kitten": "cat", "tabby": "cat"}'], + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(mock_api.call_args.kwargs["ontology"], {"kitten": "cat", "tabby": "cat"}) + + @patch("roboflow.adapters.rfapi.preview_autolabel") + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_invalid_ontology_json_fails_without_calling_api(self, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + + ["--ontology", "not-json"], + ) + self.assertNotEqual(result.exit_code, 0) + mock_api.assert_not_called() + + @patch("roboflow.adapters.rfapi.preview_autolabel") + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_mistyped_image_path_fails_before_resolving_credentials(self, mock_resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "smaple.jpg", "--class", "cat"], + ) + self.assertEqual(result.exit_code, 1, result.output) + self.assertIn("smaple.jpg", result.output) + mock_resolve.assert_not_called() + mock_api.assert_not_called() + + @patch("roboflow.adapters.rfapi.preview_autolabel") + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + @patch(_IMAGE_PAYLOAD, side_effect=PermissionError(13, "Permission denied")) + def test_unreadable_image_is_a_structured_error_not_a_traceback(self, _payload, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "locked.jpg", "--class", "cat"], + ) + self.assertEqual(result.exit_code, 1, result.output) + self.assertNotIsInstance(result.exception, OSError) + self.assertIn("Permission denied", result.output) + mock_api.assert_not_called() + + +class TestAutolabelStart(unittest.TestCase): + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_foundational_start(self, _resolve, mock_api): + result = runner.invoke( + app, + ["--json", "autolabel", "start", "-p", "ws/proj", "--batch-id", "batch-1", "-m", "gpt-6-astra-boxes"] + + ["--class", "cat", "--num-images", "10", "--confidence", "0.5", "--no-nms"] + + ["--reviewer", "r@example.com", "--model-options", '{"outputFormat": "polygon"}'], + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(json.loads(result.output), {"jobId": "job-1"}) + mock_api.assert_called_once_with( + "key", + "ws", + "proj", + batch_id="batch-1", + model_type="gpt-6-astra-boxes", + ontology={"cat": "cat"}, + num_images_to_label=10, + default_confidence=0.5, + confidence_thresholds=None, + run_nms=False, + reviewer_email="r@example.com", + model_options={"outputFormat": "polygon"}, + preserve_existing_annotations=None, + ) + + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_preserve_existing_flag(self, _resolve, mock_api): + # The server default replaces annotations already on the batch images. + result = runner.invoke( + app, + ["autolabel", "start", "-p", "ws/proj", "--batch-id", "batch-1", "-m", "sam3-rle", "--preserve-existing"], + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertIs(mock_api.call_args.kwargs["preserve_existing_annotations"], True) + + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_roboflow_model_type(self, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "start", "-p", "ws/proj", "--batch-id", "batch-1", "-m", "proj/3"] + + ["--model-type", "roboflow", "--confidence-thresholds", '{"cat": 0.6}'], + ) + self.assertEqual(result.exit_code, 0, result.output) + kwargs = mock_api.call_args.kwargs + self.assertEqual(kwargs["model_type"], "custom_roboflow") + self.assertEqual(kwargs["model_options"], {"modelId": "proj/3"}) + self.assertEqual(kwargs["confidence_thresholds"], {"cat": 0.6}) + self.assertIsNone(kwargs["run_nms"]) + + @patch("roboflow.adapters.rfapi.start_autolabel_job") + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_unknown_model_type_is_an_error(self, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "start", "-p", "ws/proj", "--batch-id", "batch-1", "-m", "x", "--model-type", "hosted"], + ) + self.assertNotEqual(result.exit_code, 0) + mock_api.assert_not_called() + + @patch("roboflow.adapters.rfapi.start_autolabel_job", side_effect=RoboflowError("batch not found", status_code=404)) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_api_error_maps_to_not_found_exit_code(self, _resolve, _mock_api): + result = runner.invoke(app, ["autolabel", "start", "-p", "ws/proj", "--batch-id", "missing", "-m", "sam3-rle"]) + self.assertEqual(result.exit_code, 3, result.output) + + +class TestAutolabelJob(unittest.TestCase): + @patch("roboflow.adapters.rfapi.get_autolabel_job", return_value={"status": "running", "progress": 0.5}) + @patch(_RESOLVE_WORKSPACE, return_value=("ws", "key")) + def test_json_output(self, _resolve, mock_api): + result = runner.invoke(app, ["--json", "autolabel", "job", "job-1"]) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(json.loads(result.output)["status"], "running") + mock_api.assert_called_once_with("key", "ws", "job-1") + + @patch("roboflow.adapters.rfapi.get_autolabel_job", return_value={"status": "running"}) + @patch(_RESOLVE_WORKSPACE) + @patch(_RESOLVE_PROJECT, return_value=("key", "other-ws", "proj")) + def test_project_shorthand_resolves_the_same_workspace_as_start(self, _resolve, mock_ws_resolve, mock_api): + # 'start -p other-ws/proj' creates the job in other-ws; 'job -p other-ws/proj' must look there too, + # since the API 404s on a job from another workspace. + result = runner.invoke(app, ["autolabel", "job", "job-1", "-p", "other-ws/proj"]) + self.assertEqual(result.exit_code, 0, result.output) + mock_api.assert_called_once_with("key", "other-ws", "job-1") + mock_ws_resolve.assert_not_called() + + @patch("roboflow.adapters.rfapi.get_autolabel_job") + @patch(_DEFAULT_WORKSPACE, return_value=None) + def test_missing_workspace_exits_with_auth_code(self, _default, mock_api): + with patch.dict(os.environ, {"ROBOFLOW_API_KEY": ""}): + result = runner.invoke(app, ["autolabel", "job", "job-1"]) + self.assertEqual(result.exit_code, 2, result.output) + mock_api.assert_not_called() diff --git a/tests/test_project_autolabel.py b/tests/test_project_autolabel.py new file mode 100644 index 00000000..d6f080cb --- /dev/null +++ b/tests/test_project_autolabel.py @@ -0,0 +1,86 @@ +"""Public Project and Workspace wrapper coverage for hosted auto-label.""" + +from unittest.mock import patch + +from tests import PROJECT_NAME, WORKSPACE_NAME, RoboflowTest + + +class TestProjectAutolabel(RoboflowTest): + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + def test_autolabel_foundational_passes_model_as_is(self, mock_start): + result = self.project.autolabel( + "batch-1", + "gpt-6-astra-boxes", + ontology={"a cat": "cat"}, + num_images=5, + confidence=0.4, + reviewer_email="reviewer@example.com", + ) + + self.assertEqual(result, {"jobId": "job-1"}) + mock_start.assert_called_once_with( + self.rf.api_key, + WORKSPACE_NAME, + PROJECT_NAME, + batch_id="batch-1", + model_type="gpt-6-astra-boxes", + ontology={"a cat": "cat"}, + num_images_to_label=5, + default_confidence=0.4, + confidence_thresholds=None, + run_nms=None, + reviewer_email="reviewer@example.com", + model_options=None, + preserve_existing_annotations=None, + ) + + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + def test_autolabel_forwards_preserve_existing_annotations(self, mock_start): + self.project.autolabel("batch-1", "sam3-rle", preserve_existing_annotations=True) + + self.assertIs(mock_start.call_args.kwargs["preserve_existing_annotations"], True) + + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + def test_autolabel_roboflow_model_is_sent_as_custom_roboflow(self, mock_start): + self.project.autolabel("batch-1", "my-project/3", model_type="roboflow", model_options={"outputFormat": "rle"}) + + kwargs = mock_start.call_args.kwargs + self.assertEqual(kwargs["model_type"], "custom_roboflow") + self.assertEqual(kwargs["model_options"], {"outputFormat": "rle", "modelId": "my-project/3"}) + + def test_autolabel_rejects_unknown_model_type(self): + with self.assertRaises(ValueError): + self.project.autolabel("batch-1", "sam3-rle", model_type="hosted") + + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={"predictions": []}) + def test_autolabel_preview_builds_image_payload(self, mock_preview): + result = self.project.autolabel_preview( + "sam3-rle", + "https://example.com/cat.jpg", + ontology=["cat"], + confidence_threshold=0.3, + ) + + self.assertEqual(result, {"predictions": []}) + mock_preview.assert_called_once_with( + self.rf.api_key, + WORKSPACE_NAME, + PROJECT_NAME, + model_type="sam3-rle", + image={"type": "url", "value": "https://example.com/cat.jpg"}, + ontology={"cat": "cat"}, + confidence_threshold=0.3, + ) + + @patch("roboflow.adapters.rfapi.get_autolabel_job", return_value={"status": "done"}) + def test_autolabel_job_wrappers_delegate(self, mock_get): + self.assertEqual(self.project.autolabel_job("job-1"), {"status": "done"}) + mock_get.assert_called_with(self.rf.api_key, WORKSPACE_NAME, "job-1") + + self.assertEqual(self.workspace.autolabel_job("job-2"), {"status": "done"}) + mock_get.assert_called_with(self.rf.api_key, WORKSPACE_NAME, "job-2") + + @patch("roboflow.adapters.rfapi.list_autolabel_models", return_value={"models": []}) + def test_workspace_autolabel_models_delegates(self, mock_list): + self.assertEqual(self.workspace.autolabel_models(), {"models": []}) + mock_list.assert_called_once_with(self.rf.api_key, WORKSPACE_NAME) diff --git a/tests/util/test_autolabel_utils.py b/tests/util/test_autolabel_utils.py new file mode 100644 index 00000000..59d0b44d --- /dev/null +++ b/tests/util/test_autolabel_utils.py @@ -0,0 +1,115 @@ +"""Unit tests for roboflow.util.autolabel_utils.""" + +import base64 +import os +import tempfile +import unittest +from unittest.mock import patch + +from roboflow.util.autolabel_utils import image_payload, ontology_payload, resolve_model + + +class TestImagePayload(unittest.TestCase): + def test_url(self): + self.assertEqual( + image_payload("https://example.com/cat.jpg"), + {"type": "url", "value": "https://example.com/cat.jpg"}, + ) + + def test_local_file_is_base64_encoded(self): + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as handle: + handle.write(b"fake-image-bytes") + path = handle.name + try: + payload = image_payload(path) + finally: + os.unlink(path) + self.assertEqual(payload["type"], "base64") + self.assertEqual(base64.b64decode(payload["value"]), b"fake-image-bytes") + + def test_other_strings_are_treated_as_base64(self): + encoded = base64.b64encode(b"bytes").decode("ascii") + self.assertEqual(image_payload(encoded), {"type": "base64", "value": encoded}) + + def test_line_wrapped_base64_is_compacted(self): + encoded = base64.b64encode(b"some longer image bytes").decode("ascii") + wrapped = encoded[:8] + "\n" + encoded[8:] + "\n" + self.assertEqual(image_payload(wrapped), {"type": "base64", "value": encoded}) + + def test_home_directory_is_expanded(self): + with tempfile.TemporaryDirectory() as home: + with open(os.path.join(home, "cat.jpg"), "wb") as handle: + handle.write(b"fake-image-bytes") + # expanduser reads HOME on POSIX and USERPROFILE on Windows. + with patch.dict(os.environ, {"HOME": home, "USERPROFILE": home}): + payload = image_payload("~/cat.jpg") + self.assertEqual(base64.b64decode(payload["value"]), b"fake-image-bytes") + + def test_mistyped_path_is_rejected_instead_of_sent_as_base64(self): + with self.assertRaises(ValueError) as ctx: + image_payload("smaple.jpg") + self.assertIn("smaple.jpg", str(ctx.exception)) + + def test_missing_home_path_is_rejected(self): + with self.assertRaises(ValueError): + image_payload("~/definitely-missing-image.png") + + def test_unreadable_file_raises_oserror(self): + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as handle: + path = handle.name + try: + with patch("roboflow.util.autolabel_utils.open", side_effect=PermissionError(13, "denied"), create=True): + with self.assertRaises(OSError): + image_payload(path) + finally: + os.unlink(path) + + +class TestOntologyPayload(unittest.TestCase): + def test_none_stays_none(self): + self.assertIsNone(ontology_payload(None)) + + def test_prompt_keyed_mapping_is_passed_through(self): + self.assertEqual(ontology_payload({"a cat": "cat"}), {"a cat": "cat"}) + + def test_several_prompts_may_share_one_class(self): + # The reason the object is keyed by prompt rather than by class: a + # class-keyed object has room for exactly one prompt per class. + self.assertEqual( + ontology_payload({"kitten": "cat", "tabby": "cat", "puppy": "dog"}), + {"kitten": "cat", "tabby": "cat", "puppy": "dog"}, + ) + + def test_list_of_classes_becomes_identity_prompts(self): + self.assertEqual(ontology_payload(["cat", "dog"]), {"cat": "cat", "dog": "dog"}) + + def test_the_result_is_a_copy(self): + source = {"a cat": "cat"} + self.assertIsNot(ontology_payload(source), source) + + def test_bare_string_is_rejected_rather_than_iterated_per_character(self): + with self.assertRaises(ValueError): + ontology_payload("cat") + + def test_empty_is_preserved_as_empty(self): + self.assertEqual(ontology_payload({}), {}) + + +class TestResolveModel(unittest.TestCase): + def test_foundational_is_pass_through(self): + self.assertEqual(resolve_model("gpt-6-astra-boxes", "foundational"), ("gpt-6-astra-boxes", None)) + self.assertEqual( + resolve_model("sam3-rle", "foundational", {"outputFormat": "rle"}), + ("sam3-rle", {"outputFormat": "rle"}), + ) + + def test_roboflow_model_rides_in_model_options(self): + self.assertEqual( + resolve_model("proj/3", "roboflow", {"outputFormat": "polygon"}), + ("custom_roboflow", {"outputFormat": "polygon", "modelId": "proj/3"}), + ) + self.assertEqual(resolve_model("proj/3", "roboflow"), ("custom_roboflow", {"modelId": "proj/3"})) + + def test_unknown_model_type_raises(self): + with self.assertRaises(ValueError): + resolve_model("x", "hosted")