From f751f2a1a504fc94891510e56ba171f7249675fe Mon Sep 17 00:00:00 2001 From: "user.mail" Date: Thu, 27 Aug 2026 12:00:18 +0300 Subject: [PATCH] feat(marketplace): query Bright Data's pre-collected datasets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI could only ever collect NEW data: pipelines wraps /datasets/v3/*, which scrapes what you point it at and bills accordingly. Bright Data also sells ~1,750 pre-collected datasets queried through a separate, unversioned API (/datasets/list, /datasets/{id}/metadata, POST /datasets/filter, /datasets/snapshots/{id}[/download]) that the CLI did not touch at all — so users had to re-scrape data Bright Data already held, and had no way to discover that these datasets existed. Adds a 'marketplace' command family mapping onto those five endpoints: - list [--featured|--search] discover datasets (the catalogue is ~1,750 rows, so a bare list is filtered rather than dumped) - fields per-field types and descriptions from the metadata endpoint — the CLI's first self-documenting data command; you can see what is filterable before spending anything - filter the query itself; the filter tree is sent verbatim and the API validates it - status status plus records, file size and cost - download [--wait]; exits 3 when not ready yet, so scripts can tell 'retry later' from 'failed' Notes on two decisions: --records-limit is REQUIRED, though the API treats it as optional. Omitting it means no cap on datasets holding hundreds of millions of records, and cost is only observable after the query is committed, so there is no way to preview or undo an accidentally huge query. Making it explicit costs one flag and removes the whole unbounded case. Datasets are addressed by 48 curated aliases or by raw gd_ id, not by the catalogue's own names. A live probe showed why: 'name' is a display string ('Instagram - Profiles', 'Facebook - Comments' with a double space, 'Manta businesses ' with a trailing one), 1630 of 1749 contain spaces, and 43 names are shared across different ids — one name maps to 8 datasets. So names are for reading; --search finds an id to paste. All 48 alias ids were validated against the live catalogue. Also adds src/utils/exit-codes.ts (0 ok / 1 error / 3 not ready). 28 new tests (397 total). Verified live against the API: the full filter -> poll -> download lifecycle, all three formats, and every guard firing before any network call. All 48 alias ids validated against the live catalogue. --- README.md | 78 +++ src/__tests__/commands/marketplace.test.ts | 367 ++++++++++ src/commands/marketplace.ts | 749 +++++++++++++++++++++ src/index.ts | 2 + src/types/marketplace.ts | 87 +++ src/utils/exit-codes.ts | 9 + 6 files changed, 1292 insertions(+) create mode 100644 src/__tests__/commands/marketplace.test.ts create mode 100644 src/commands/marketplace.ts create mode 100644 src/types/marketplace.ts create mode 100644 src/utils/exit-codes.ts diff --git a/README.md b/README.md index e6ea0e0..25c4a03 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ | `brightdata scraper heal` | Fix an existing scraper in place via AI self-healing (stops at an approval gate) | | `brightdata scraper approve` | Approve (or reject) a self-healing fix that is awaiting approval | | `brightdata pipelines` | Extract structured data from 40+ platforms (Amazon, LinkedIn, TikTok…) | +| `brightdata marketplace` | Query Bright Data's pre-collected datasets — filter records that already exist | | `brightdata browser` | Control a real browser via Bright Data's Scraping Browser — navigate, snapshot, click, type, and more | | `brightdata zones` | List and inspect your Bright Data proxy zones | | `brightdata budget` | View account balance and per-zone cost & bandwidth | @@ -56,6 +57,7 @@ - [scraper heal](#scraper-heal) - [scraper approve](#scraper-approve) - [pipelines](#pipelines) + - [marketplace](#marketplace) - [browser](#browser) - [status](#status) - [zones](#zones) @@ -653,6 +655,82 @@ See [Dataset Types Reference](#dataset-types-reference) for the full list. --- +### `marketplace` + +Query Bright Data's **Dataset Marketplace** — data that has already been collected. This is the counterpart to [`pipelines`](#pipelines), and the distinction matters for both speed and cost: + +| | `pipelines` | `marketplace` | +|---|---|---| +| What it does | **Collects new** data from URLs/seeds you supply | **Queries** records Bright Data already holds | +| Billing | Per record scraped | Per record returned by the query | +| Latency | A crawl runs | A filter runs over stored data | + +```bash +brightdata marketplace list [--featured] [--search ] +brightdata marketplace fields +brightdata marketplace filter --dataset --filter '' --records-limit +brightdata marketplace status +brightdata marketplace download [--wait] +``` + +**Finding a dataset.** The catalogue holds ~1,750 datasets, so `list` has two filters. `--featured` shows the 48 with short names you can type; `--search` finds anything else, and you pass its `gd_` id to the other subcommands with `--dataset-id`. + +```bash +brightdata marketplace list --featured +brightdata marketplace list --search zillow +``` + +**Seeing what you can filter on** — before spending anything: + +```bash +brightdata marketplace fields linkedin_people_profiles +# field type required description +# about text A concise profile summary... +# city text Geographical location of the user +``` + +**Querying.** The filter is a JSON tree, sent to the API as-is: + +```bash +brightdata marketplace filter --dataset linkedin_company_profiles \ + --filter '{"name":"industry","operator":"=","value":"Technology"}' \ + --records-limit 1000 --format jsonl -o tech.jsonl + +# combine conditions with and/or +--filter '{"operator":"and","filters":[ + {"name":"industry","operator":"=","value":"Technology"}, + {"name":"followers","operator":">","value":10000}]}' +``` + +> **`--records-limit` is required.** Queries are billed by records returned, some datasets hold hundreds of millions of rows, and the cost is only known once the query has run — so the CLI makes you state a cap rather than letting an omission become an expensive one. + +**Submit now, collect later:** + +```bash +brightdata marketplace filter --dataset x_twitter_posts \ + --filter '{"name":"likes","operator":">","value":10000}' \ + --records-limit 500 --async +# → Snapshot ID: s_abc123 + +brightdata marketplace status s_abc123 # status, records, size, cost +brightdata marketplace download s_abc123 --wait +``` + +`download` exit codes: `0` data returned · `1` error · `3` snapshot not ready yet (retry later, or use `--wait`). + +| Flag | Applies to | Description | +|---|---|---| +| `--featured` / `--search ` | `list` | Filter the catalogue | +| `--dataset ` | `filter` | One of the 48 named datasets | +| `--dataset-id ` | `filter` | Any dataset id from `list` | +| `--filter ` / `--filter-file ` | `filter` | The filter tree | +| `--records-limit ` | `filter` | **Required** — cap on records returned | +| `--async` | `filter` | Submit only; print the snapshot ID | +| `--wait` | `download` | Poll until ready, then download | +| `--format ` | `filter`, `download` | `json` · `jsonl` · `csv` (no `ndjson`) | +| `-o, --output ` / `--json` / `--pretty` | all | Output handling | + + ### `browser` Control a real browser session powered by [Bright Data's Scraping Browser](https://brightdata.com/products/scraping-browser). A lightweight local daemon holds the browser connection open between commands, giving you persistent state without reconnecting on every call. diff --git a/src/__tests__/commands/marketplace.test.ts b/src/__tests__/commands/marketplace.test.ts new file mode 100644 index 0000000..63d2a0f --- /dev/null +++ b/src/__tests__/commands/marketplace.test.ts @@ -0,0 +1,367 @@ +import {describe, it, expect, beforeEach, afterEach, vi} from 'vitest'; + +const mocks = vi.hoisted(()=>({ + get: vi.fn(), + post: vi.fn(), + ensure_authenticated: vi.fn(), + start: vi.fn(), + stop: vi.fn(), + print: vi.fn(), + print_table: vi.fn(), + info: vi.fn(), + success: vi.fn(), + warn: vi.fn(), + fail: vi.fn((msg: string)=>{ throw new Error(`fail:${msg}`); }), + parse_timeout: vi.fn(), + poll_until: vi.fn(), +})); + +vi.mock('../../utils/client', ()=>({get: mocks.get, post: mocks.post})); +vi.mock('../../utils/auth', ()=>({ + ensure_authenticated: mocks.ensure_authenticated, +})); +vi.mock('../../utils/spinner', ()=>({start: mocks.start})); +vi.mock('../../utils/output', ()=>({ + print: mocks.print, + print_table: mocks.print_table, + dim: (s: string)=>s, + fail: mocks.fail, + info: mocks.info, + success: mocks.success, + warn: mocks.warn, +})); +vi.mock('../../utils/polling', ()=>({ + parse_timeout: mocks.parse_timeout, + poll_until: mocks.poll_until, +})); + +import { + handle_list, + handle_fields, + handle_filter, + handle_status, + handle_download, + FEATURED_DATASETS, +} from '../../commands/marketplace'; + +const LINKEDIN_PEOPLE = 'gd_l1viktl72bvl7bjuj0'; + +describe('commands/marketplace', ()=>{ + let exit_spy: ReturnType; + + beforeEach(()=>{ + vi.clearAllMocks(); + mocks.ensure_authenticated.mockReturnValue('api_key'); + mocks.start.mockReturnValue({stop: mocks.stop}); + mocks.parse_timeout.mockReturnValue(600); + exit_spy = vi.spyOn(process, 'exit') + .mockImplementation(((..._a: unknown[])=>undefined) as never); + vi.spyOn(console, 'error').mockImplementation(()=>{}); + }); + + afterEach(()=>{ + vi.restoreAllMocks(); + }); + + describe('the curated alias map', ()=>{ + it('covers 48 datasets and every id looks like a dataset id', ()=>{ + const entries = Object.entries(FEATURED_DATASETS); + expect(entries).toHaveLength(48); + for (const [, id] of entries) + expect(id).toMatch(/^gd_[a-z0-9]+$/); + }); + + it('maps no two names to the same id', ()=>{ + const ids = Object.values(FEATURED_DATASETS); + expect(new Set(ids).size).toBe(ids.length); + }); + }); + + describe('list', ()=>{ + it('fetches the catalogue and renders a table', async()=>{ + mocks.get.mockResolvedValue([ + {id: 'gd_a', name: 'Alpha ', size: 1000}, + ]); + await handle_list({}); + expect(mocks.get).toHaveBeenCalledWith( + 'api_key', '/datasets/list', {timing: undefined}); + // the catalogue carries stray whitespace; display trims it + expect(mocks.print_table).toHaveBeenCalledWith( + [{name: 'Alpha', id: 'gd_a', records: '1,000'}], + ['name', 'id', 'records'] + ); + }); + + it('--featured keeps only curated datasets', async()=>{ + mocks.get.mockResolvedValue([ + {id: LINKEDIN_PEOPLE, name: 'LinkedIn people profiles'}, + {id: 'gd_not_curated', name: 'Something else'}, + ]); + await handle_list({featured: true}); + const rows = mocks.print_table.mock.calls[0][0]; + expect(rows).toHaveLength(1); + expect(rows[0].id).toBe(LINKEDIN_PEOPLE); + }); + + it('--search matches name or id, case-insensitively', async()=>{ + mocks.get.mockResolvedValue([ + {id: 'gd_a', name: 'Zillow properties'}, + {id: 'gd_zillow_b', name: 'Something'}, + {id: 'gd_c', name: 'Unrelated'}, + ]); + await handle_list({search: 'ZILLOW'}); + expect(mocks.print_table.mock.calls[0][0]).toHaveLength(2); + }); + + it('says so when nothing matches instead of printing nothing', + async()=>{ + // print_table returns silently on an empty array, so an empty + // result would otherwise be indistinguishable from a failure. + mocks.get.mockResolvedValue([{id: 'gd_a', name: 'Alpha'}]); + await handle_list({search: 'nothing-matches-this'}); + expect(mocks.info).toHaveBeenCalledWith( + expect.stringContaining('No datasets match')); + expect(mocks.print_table).not.toHaveBeenCalled(); + }); + + it('passes --json straight through to print', async()=>{ + const rows = [{id: 'gd_a', name: 'Alpha'}]; + mocks.get.mockResolvedValue(rows); + await handle_list({json: true}); + expect(mocks.print).toHaveBeenCalledWith( + rows, {json: true, pretty: undefined, output: undefined}); + expect(mocks.print_table).not.toHaveBeenCalled(); + }); + }); + + describe('fields', ()=>{ + it('resolves a curated alias without touching the catalogue', + async()=>{ + mocks.get.mockResolvedValue({fields: {url: {type: 'url'}}}); + await handle_fields('linkedin_people_profiles', {}); + expect(mocks.get).toHaveBeenCalledTimes(1); + expect(mocks.get).toHaveBeenCalledWith( + 'api_key', + `/datasets/${LINKEDIN_PEOPLE}/metadata`, + {timing: undefined} + ); + }); + + it('accepts a raw gd_ id', async()=>{ + mocks.get.mockResolvedValue({fields: {a: {type: 'text'}}}); + await handle_fields('gd_anything', {}); + expect(mocks.get).toHaveBeenCalledWith( + 'api_key', '/datasets/gd_anything/metadata', + {timing: undefined}); + }); + + it('rejects an unknown name and points at list/search', async()=>{ + await expect(handle_fields('not_a_dataset', {})) + .rejects.toThrow(/Unknown dataset "not_a_dataset"/); + expect(mocks.fail).toHaveBeenCalledWith( + expect.stringContaining('--search not_a_dataset')); + expect(mocks.get).not.toHaveBeenCalled(); + }); + + it('truncates long descriptions for the table but not for --json', + async()=>{ + const long = 'x'.repeat(200); + mocks.get.mockResolvedValue({fields: {a: {description: long}}}); + await handle_fields('gd_x', {}); + const row = mocks.print_table.mock.calls[0][0][0]; + expect(row.description.length).toBeLessThan(80); + expect(row.description.endsWith('…')).toBe(true); + }); + }); + + describe('filter — the billable path', ()=>{ + const base = { + dataset: 'crunchbase_companies', + filter: '{"name":"industry","operator":"=","value":"Tech"}', + recordsLimit: '100', + }; + + it('refuses to run without --records-limit', async()=>{ + // The billing guard: omitting the cap means "no limit" on datasets + // holding hundreds of millions of records, and cost is only known + // after the query is committed. + await expect(handle_filter({ + dataset: base.dataset, + filter: base.filter, + })).rejects.toThrow(/--records-limit is required/); + expect(mocks.post).not.toHaveBeenCalled(); + }); + + it('rejects a non-positive --records-limit', async()=>{ + await expect(handle_filter({...base, recordsLimit: '0'})) + .rejects.toThrow(/Invalid --records-limit/); + expect(mocks.post).not.toHaveBeenCalled(); + }); + + it('sends dataset_id, the parsed filter, and the cap', async()=>{ + mocks.post.mockResolvedValue({snapshot_id: 's_1'}); + await handle_filter({...base, async: true}); + expect(mocks.post).toHaveBeenCalledWith( + 'api_key', + '/datasets/filter', + { + dataset_id: FEATURED_DATASETS['crunchbase_companies'], + filter: { + name: 'industry', operator: '=', value: 'Tech', + }, + records_limit: 100, + }, + {timing: undefined} + ); + }); + + it('fails before the network on invalid filter JSON', async()=>{ + await expect(handle_filter({...base, filter: 'not json'})) + .rejects.toThrow(/Invalid JSON in filter/); + expect(mocks.post).not.toHaveBeenCalled(); + }); + + it('rejects --filter and --filter-file together', async()=>{ + await expect(handle_filter({ + ...base, filterFile: 'f.json', + })).rejects.toThrow(/not both/); + expect(mocks.post).not.toHaveBeenCalled(); + }); + + it('surfaces the API error when no snapshot_id comes back', + async()=>{ + mocks.post.mockResolvedValue({error: 'bad filter field'}); + await expect(handle_filter(base)) + .rejects.toThrow(/bad filter field/); + }); + + it('--async prints the id and does not poll', async()=>{ + mocks.post.mockResolvedValue({snapshot_id: 's_1'}); + await handle_filter({...base, async: true}); + expect(mocks.success).toHaveBeenCalledWith( + expect.stringContaining('s_1')); + expect(mocks.poll_until).not.toHaveBeenCalled(); + }); + }); + + describe('status', ()=>{ + it('surfaces cost, record count and file size', async()=>{ + mocks.get.mockResolvedValue({ + status: 'ready', dataset_size: 1234, file_size: 2097152, + cost: 4.5, + }); + await handle_status('s_1', {}); + expect(mocks.get).toHaveBeenCalledWith( + 'api_key', '/datasets/snapshots/s_1', {timing: undefined}); + expect(mocks.info).toHaveBeenCalledWith( + expect.stringContaining('cost 4.5')); + }); + + it('fails loudly on a failed snapshot', async()=>{ + mocks.get.mockResolvedValue({ + status: 'failed', failure_reason: 'filter referenced no field', + }); + await expect(handle_status('s_1', {})) + .rejects.toThrow(/filter referenced no field/); + }); + }); + + describe('download', ()=>{ + it('exits NOT_READY (3) when still building and no --wait', + async()=>{ + mocks.get.mockResolvedValue({status: 'building'}); + await handle_download('s_1', {}); + expect(exit_spy).toHaveBeenCalledWith(3); + expect(mocks.poll_until).not.toHaveBeenCalled(); + expect(mocks.print).not.toHaveBeenCalled(); + }); + + it('downloads a ready snapshot as parsed json', async()=>{ + mocks.get + .mockResolvedValueOnce({status: 'ready', dataset_size: 2}) + .mockResolvedValueOnce([{a: 1}]); + await handle_download('s_1', {format: 'json'}); + expect(mocks.get).toHaveBeenLastCalledWith( + 'api_key', + '/datasets/snapshots/s_1/download?format=json', + {timing: undefined} + ); + expect(mocks.print).toHaveBeenCalledWith( + [{a: 1}], + {json: undefined, pretty: undefined, output: undefined} + ); + }); + + it('requests non-json formats as raw bytes, not parsed json', + async()=>{ + // The endpoint returns content-type application/jsonl, which + // the client's `includes('application/json')` check would + // treat as JSON — parsing the body and re-serialising it as + // indented JSON. raw_buffer keeps it byte-exact. + mocks.get + .mockResolvedValueOnce({status: 'ready'}) + .mockResolvedValueOnce(Buffer.from('{"a":1}\n')); + await handle_download('s_1', {format: 'jsonl'}); + expect(mocks.get).toHaveBeenLastCalledWith( + 'api_key', + '/datasets/snapshots/s_1/download?format=jsonl', + {timing: undefined, raw_buffer: true} + ); + expect(mocks.print).toHaveBeenCalledWith( + '{"a":1}\n', + {json: undefined, pretty: undefined, output: undefined} + ); + }); + + it('reports a zero-match query as an empty result, not a failure', + async()=>{ + // A filter matching nothing comes back as status:failed with + // warning_code no_records_found — a successful empty query. + mocks.get.mockResolvedValue({ + status: 'failed', + warning: 'Provided filter did not match any records', + warning_code: 'no_records_found', + }); + await handle_download('s_1', {}); + expect(mocks.info).toHaveBeenCalledWith( + expect.stringContaining('matched no records')); + expect(exit_spy).not.toHaveBeenCalled(); + }); + + it('--wait polls on the marketplace status vocabulary', async()=>{ + mocks.get.mockResolvedValueOnce({status: 'scheduled'}); + mocks.poll_until.mockResolvedValue({ + result: {status: 'ready'}, attempts: 2, + }); + mocks.get.mockResolvedValueOnce([{a: 1}]); + await handle_download('s_1', {wait: true}); + expect(mocks.poll_until).toHaveBeenCalledWith( + expect.objectContaining({ + running_statuses: ['scheduled', 'building'], + }) + ); + }); + + it('reports an empty snapshot instead of printing nothing', + async()=>{ + mocks.get + .mockResolvedValueOnce({status: 'ready'}) + .mockResolvedValueOnce([]); + await handle_download('s_1', {}); + expect(mocks.info).toHaveBeenCalledWith( + expect.stringContaining('No records')); + expect(mocks.print).not.toHaveBeenCalled(); + }); + + it('fails with the API reason on a failed snapshot', async()=>{ + mocks.get.mockResolvedValue({status: 'failed', error: 'boom'}); + await expect(handle_download('s_1', {})).rejects.toThrow(/boom/); + }); + + it('rejects ndjson, which is a pipelines format', async()=>{ + await expect(handle_download('s_1', {format: 'ndjson'})) + .rejects.toThrow(/Invalid format "ndjson"/); + expect(mocks.get).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/commands/marketplace.ts b/src/commands/marketplace.ts new file mode 100644 index 0000000..705f9d2 --- /dev/null +++ b/src/commands/marketplace.ts @@ -0,0 +1,749 @@ +import fs from 'fs'; +import {Command} from 'commander'; +import {ensure_authenticated} from '../utils/auth'; +import {get, post} from '../utils/client'; +import { + print, print_table, dim, fail, info, success, +} from '../utils/output'; +import {start as start_spinner} from '../utils/spinner'; +import {parse_timeout, poll_until} from '../utils/polling'; +import {add_examples} from '../utils/help'; +import {EXIT} from '../utils/exit-codes'; +import type { + Marketplace_format, + Dataset_info, + Dataset_metadata, + Snapshot_status, + Filter_request, + Filter_response, + Marketplace_opts, +} from '../types/marketplace'; + +// The Dataset Marketplace API: query records Bright Data has already +// collected. Unversioned, and entirely separate from the /datasets/v3/* +// endpoints `pipelines` uses to collect new data. +const LIST_ENDPOINT = '/datasets/list'; +const FILTER_ENDPOINT = '/datasets/filter'; +const metadata_endpoint = (id: string)=>`/datasets/${id}/metadata`; +const snapshot_endpoint = (id: string)=>`/datasets/snapshots/${id}`; +const download_endpoint = (id: string)=>`/datasets/snapshots/${id}/download`; + +// Marketplace status vocabulary — differs from v3's +// starting/building/running. +const RUNNING_STATUSES = ['scheduled', 'building']; +const FAILED_STATUS = 'failed'; +const ALLOWED_FORMATS: Marketplace_format[] = ['json', 'jsonl', 'csv']; +const LIST_WARN_THRESHOLD = 200; + +// Named shortcuts for the datasets people reach for most. This is the ONLY way +// to address a dataset by name: the catalogue's own `name` field is a human +// display string ("Instagram - Profiles", "Facebook - Comments" with a double +// space, "Manta businesses " with a trailing one), 43 names are shared across +// different ids, and the catalogue holds internal entries. So names are for +// reading, not for resolving. Everything outside this map is reachable with +// --dataset-id; `marketplace list --search` finds the id. +// Each comment records the catalogue's own name for that id (validated against +// a live GET /datasets/list on 2026-08-27: 48/48 resolve). +const FEATURED_DATASETS: Record = { + // --- social / content (32) --- + // Bluesky - Posts + bluesky_posts: 'gd_m6hn4r5s27zfhc7w4', + // Top 500 Bluesky Profiles + bluesky_top_profiles: 'gd_m45p78dl1m017wi5lj', + // Facebook - Comments + facebook_comments: 'gd_lkay758p1eanlolqw8', + // Facebook Company Reviews + facebook_company_reviews: 'gd_m0dtqpiu1mbcyc2g86', + // Facebook Events + facebook_events: 'gd_m14sd0to1jz48ppm51', + // Facebook - Posts by group URL + facebook_group_posts: 'gd_lz11l67o2cb3r0lkj3', + // Facebook Marketplace + facebook_marketplace: 'gd_lvt9iwuh6fbcwmx1a', + // Facebook - Pages Posts by Profile URL + facebook_pages_posts: 'gd_lkaxegm826bjpoo9m5', + // Facebook - Pages and Profiles + facebook_pages_profiles: 'gd_mf124a0511bauquyow', + // Facebook - Posts by post URL + facebook_posts_by_url: 'gd_lyclm1571iy3mv57zw', + // Facebook - Profiles + facebook_profiles: 'gd_mf0urb782734ik94dz', + // Facebook - Reels by profile URL + facebook_reels: 'gd_lyclm3ey2q6rww027t', + // Instagram - Comments + instagram_comments: 'gd_ltppn085pokosxh13', + // Instagram - Posts + instagram_posts: 'gd_lk5ns7kz21pck8jpis', + // Instagram - Profiles + instagram_profiles: 'gd_l1vikfch901nx3by4', + // Instagram - Reels + instagram_reels: 'gd_lyclm20il4r5helnj', + // Pinterest - Posts + pinterest_posts: 'gd_lk0sjs4d21kdr7cnlv', + // Pinterest - Profiles + pinterest_profiles: 'gd_lk0zv93c2m9qdph46z', + // Quora posts + quora_posts: 'gd_lvz1rbj81afv3m6n5y', + // Reddit - Comments + reddit_comments: 'gd_lvzdpsdlw09j6t702', + // Reddit- Posts + reddit_posts: 'gd_lvz8ah06191smkebj4', + // Snapchat posts + snapchat_posts: 'gd_ma0ydx431w6stl16ge', + // TikTok - Comments + tiktok_comments: 'gd_lkf2st302ap89utw5k', + // TikTok - Posts + tiktok_posts: 'gd_lu702nij2f790tmv9h', + // TikTok - Profiles + tiktok_profiles: 'gd_l1villgoiiidt09ci', + // TikTok Shop + tiktok_shop: 'gd_m45m1u911dsa4274pi', + // Vimeo - Videos posts + vimeo_videos: 'gd_lxk88z3v1ketji4pn', + // X (formerly Twitter) - Posts + x_twitter_posts: 'gd_lwxkxvnf1cynvib9co', + // X (formerly Twitter) - Profiles + x_twitter_profiles: 'gd_lwxmeb2u1cniijd7t4', + // Youtube - Comments + youtube_comments: 'gd_lk9q0ew71spt1mxywf', + // YouTube - Channels + youtube_profiles: 'gd_lk538t2k2p1k3oos71', + // Youtube - Videos posts + youtube_videos: 'gd_lk56epmy2i5g7lzu0k', + // --- business / people intelligence (16) --- + // Companies information enriched dataset + companies_enriched: 'gd_m3fl0mwzmfpfn4cw4', + // Crunchbase companies information + crunchbase_companies: 'gd_l1vijqt9jfj7olije', + // Employees business enriched dataset + employees_enriched: 'gd_m18zt6ec11wfqohyrs', + // LinkedIn company information + linkedin_company_profiles: 'gd_l1vikfnt1wgvvqz95w', + // Linkedin job listings information + linkedin_job_listings: 'gd_lpfll7v5hcqtkxl6l', + // LinkedIn people profiles + linkedin_people_profiles: 'gd_l1viktl72bvl7bjuj0', + // LinkedIn posts + linkedin_posts: 'gd_lyy3tktm25m4avu764', + // LinkedIn profiles Jobs Listings + linkedin_profiles_job_listings: 'gd_m487ihp32jtc4ujg45', + // Manta businesses + manta_businesses: 'gd_l1vil1d81g0u8763b2', + // Owler companies information + owler_companies: 'gd_l1vilaxi10wutoage7', + // pitchbook companies information + pitchbook_companies: 'gd_m4ijiqfp2n9oe3oluj', + // Slintel 6sense company information + slintel_companies: 'gd_l1vilg5a1decoahvgq', + // US lawyers directory + us_lawyers: 'gd_l1vil5n11okchcbvax', + // VentureRadar company information + ventureradar_companies: 'gd_l1vilsfd1xpsndbtpr', + // Xing social network + xing_profiles: 'gd_l3lh4ev31oqrvvblv6', + // Zoominfo companies information + zoominfo_companies: 'gd_m0ci4a4ivx3j5l6nx', +}; + +const resolve_format = (raw: string|undefined): Marketplace_format=>{ + const format = (raw ?? 'json').toLowerCase(); + if (ALLOWED_FORMATS.includes(format as Marketplace_format)) + return format as Marketplace_format; + fail( + `Invalid format "${format}".\n` + +' Allowed formats: json, jsonl, csv.\n' + +' (ndjson is a pipelines format; the marketplace API does not ' + +'accept it.)' + ); + return 'json'; +}; + +// gd_ id -> used as-is; curated alias -> its id; anything else -> fail with a +// pointer. There is deliberately no lookup against the catalogue's `name` +// field: it is a display string, not an identifier, and 43 names are shared +// across ids, so a "match" could not identify one dataset anyway. +const resolve_dataset_id = (opts: Marketplace_opts): string|undefined=>{ + if (opts.dataset && opts.datasetId) + { + fail('Provide either --dataset or --dataset-id, not both.'); + return undefined; + } + if (opts.datasetId) + return opts.datasetId.trim(); + const ref = opts.dataset?.trim(); + if (!ref) + { + fail( + 'No dataset specified.\n' + +' Use --dataset (see \'brightdata marketplace list ' + +'--featured\')\n' + +' or --dataset-id for any other dataset.' + ); + return undefined; + } + if (ref.startsWith('gd_')) + return ref; + const featured = FEATURED_DATASETS[ref.toLowerCase()]; + if (featured) + return featured; + fail( + `Unknown dataset "${ref}".\n` + +' Named datasets: brightdata marketplace list --featured\n' + +` Anything else: brightdata marketplace list --search ${ref}\n` + +' then pass its id with --dataset-id ' + ); + return undefined; +}; + +const parse_records_limit = (raw: string|undefined): number|undefined=>{ + // Required, unlike the API, which treats it as optional. Omitting it means + // "no cap" on datasets holding hundreds of millions of records, and cost + // is only observable after the query is committed — so there is no way to + // undo an accidentally huge query, and no way to preview its price. + if (raw === undefined) + { + fail( + '--records-limit is required.\n' + +' Marketplace queries are billed by records returned, and some ' + +'datasets\n' + +' hold hundreds of millions. Pass an explicit cap, e.g. ' + +'--records-limit 1000.' + ); + return undefined; + } + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) + { + fail( + `Invalid --records-limit "${raw}".\n` + +' Use a positive integer.' + ); + return undefined; + } + return value; +}; + +const load_filter = (opts: Marketplace_opts): unknown|undefined=>{ + if (opts.filter !== undefined && opts.filterFile !== undefined) + { + fail('Provide either --filter or --filter-file, not both.'); + return undefined; + } + let text: string; + if (opts.filterFile !== undefined) + { + try { + text = fs.readFileSync(opts.filterFile, 'utf8'); + } catch(e) { + fail( + `Cannot read filter file "${opts.filterFile}": ` + +`${(e as Error).message}` + ); + return undefined; + } + } + else if (opts.filter !== undefined) + text = opts.filter; + else + { + fail( + 'No filter provided.\n' + +' Pass --filter \'\' or --filter-file .\n' + +' Example: --filter \'{"name":"industry","operator":"=",' + +'"value":"Technology"}\'' + ); + return undefined; + } + const trimmed = text.trim(); + if (!trimmed) + { + fail('Filter is empty.'); + return undefined; + } + try { + return JSON.parse(trimmed) as unknown; + } catch(e) { + fail( + `Invalid JSON in filter: ${(e as Error).message}\n` + +' The filter is a JSON object, e.g. ' + +'{"name":"followers","operator":">","value":10000}' + ); + return undefined; + } +}; + +const DESCRIPTION_WIDTH = 68; + +const truncate = (text: string|undefined): string=>{ + const clean = (text ?? '').replace(/\s+/g, ' ').trim(); + if (clean.length <= DESCRIPTION_WIDTH) + return clean; + return clean.slice(0, DESCRIPTION_WIDTH-1)+'…'; +}; + +const NO_RECORDS_CODE = 'no_records_found'; + +// The API reports a zero-match query as status:failed with warning_code +// no_records_found. That is a successful query with an empty result, not a +// failure, and it is reported as such (exit 0) — consistent with how +// `marketplace list --search` reports finding nothing. +const is_empty_result = (snap: Snapshot_status): boolean=> + snap.warning_code == NO_RECORDS_CODE; + +const snapshot_error = (snap: Snapshot_status): string=> + snap.error || snap.error_message || snap.failure_reason || snap.warning + || snap.message || 'no reason returned by the API'; + +const handle_list = async(opts: Marketplace_opts)=>{ + const api_key = ensure_authenticated(opts.apiKey); + const spinner = start_spinner('Fetching dataset catalogue...'); + try { + const all = await get(api_key, LIST_ENDPOINT, + {timing: opts.timing}); + spinner.stop(); + let rows = Array.isArray(all) ? all : []; + if (opts.featured) + { + const ids = new Set(Object.values(FEATURED_DATASETS)); + rows = rows.filter(d=>ids.has(d.id)); + } + if (opts.search) + { + const needle = opts.search.toLowerCase(); + rows = rows.filter(d=> + (d.name ?? '').toLowerCase().includes(needle) + || d.id.toLowerCase().includes(needle)); + } + if (!rows.length) + { + info(opts.search + ? `No datasets match "${opts.search}".` + : 'No datasets returned.'); + return; + } + if (opts.json || opts.pretty || opts.output) + { + print(rows, { + json: opts.json, + pretty: opts.pretty, + output: opts.output, + }); + return; + } + // Names carry stray whitespace in the catalogue; trim for display only. + print_table(rows.map(d=>({ + name: (d.name ?? '').trim(), + id: d.id, + records: d.size === undefined ? '' : d.size.toLocaleString(), + })), ['name', 'id', 'records']); + if (rows.length > LIST_WARN_THRESHOLD) + { + info(`${rows.length} datasets listed. Narrow with --search , ` + +'or use --featured for the named shortcuts.'); + } + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + } +}; + +const handle_fields = async(dataset: string, opts: Marketplace_opts)=>{ + const dataset_id = resolve_dataset_id({...opts, dataset}); + if (!dataset_id) + return; + const api_key = ensure_authenticated(opts.apiKey); + const spinner = start_spinner(`Fetching fields for ${dataset_id}...`); + try { + const meta = await get( + api_key, metadata_endpoint(dataset_id), {timing: opts.timing}); + spinner.stop(); + const fields = meta?.fields ?? {}; + const names = Object.keys(fields); + if (!names.length) + { + info(`No fields reported for ${dataset_id}.`); + return; + } + if (opts.json || opts.pretty || opts.output) + { + print(meta, { + json: opts.json, + pretty: opts.pretty, + output: opts.output, + }); + return; + } + print_table(names.sort().map(name=>({ + field: name, + type: fields[name]?.type ?? '', + required: fields[name]?.required ? 'yes' : '', + // Descriptions run to several hundred characters; the table is for + // scanning. --json gives the untruncated text. + description: truncate(fields[name]?.description), + })), ['field', 'type', 'required', 'description']); + info(`${names.length} fields — any of them can be used in a filter.`); + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + } +}; + +const fetch_snapshot_status = ( + api_key: string, + snapshot_id: string, + timing: boolean|undefined +): Promise=> + get(api_key, snapshot_endpoint(snapshot_id), {timing}); + +const print_snapshot_summary = (snap: Snapshot_status)=>{ + const bits: string[] = []; + if (snap.dataset_size !== undefined) + bits.push(`${snap.dataset_size.toLocaleString()} records`); + if (snap.file_size !== undefined) + bits.push(`${(snap.file_size/1024/1024).toFixed(2)} MB`); + if (snap.cost !== undefined) + bits.push(`cost ${snap.cost}`); + if (bits.length) + info(bits.join(' · ')); +}; + +// Waits for a snapshot to leave scheduled/building, then hands back the final +// status. Note the 1s cadence: poll_until counts attempts rather than elapsed +// time, so timeout_seconds only means seconds while the interval stays 1s. +// (PR #20 upstream converts that loop to a wall-clock deadline; once it lands, +// this can move to the SDK's gentler 5s pacing without changing --timeout.) +const wait_for_snapshot = async( + api_key: string, + snapshot_id: string, + timeout: number, + opts: Marketplace_opts +): Promise=>{ + const poll_result = await poll_until({ + timeout_seconds: timeout, + fetch_once: ()=>fetch_snapshot_status(api_key, snapshot_id, + opts.timing), + get_status: snap=>snap.status, + running_statuses: RUNNING_STATUSES, + timeout_label: `snapshot ${snapshot_id}`, + on_running: ({attempt, timeout_seconds, status})=>{ + console.error(dim( + `Status: ${status} - polling again ` + +`(attempt ${attempt}/${timeout_seconds})` + )); + }, + }); + return poll_result.result; +}; + +const download_snapshot = async( + api_key: string, + snapshot_id: string, + format: Marketplace_format, + opts: Marketplace_opts +)=>{ + const endpoint = `${download_endpoint(snapshot_id)}?format=${format}`; + // csv/jsonl must reach the user byte-for-byte. They cannot go through the + // client's JSON path: it selects it with content_type.includes( + // 'application/json'), which is also true for the 'application/jsonl' this + // endpoint returns — so a JSONL body would be parsed into an object and + // re-serialised as indented JSON. raw_buffer bypasses that entirely. + const data = format == 'json' + ? await get(api_key, endpoint, {timing: opts.timing}) + : (await get(api_key, endpoint, + {timing: opts.timing, raw_buffer: true})).toString('utf8'); + // An empty result is a normal outcome of a filter query and must be said + // out loud — otherwise it is indistinguishable from a silent failure. + const empty = data === undefined || data === null || data === '' + || (Array.isArray(data) && !data.length); + if (empty) + { + info('No records in this snapshot (the filter matched nothing).'); + return; + } + print(data, { + json: opts.json, + pretty: opts.pretty, + output: opts.output, + }); +}; + +const handle_status = async(snapshot_id: string, opts: Marketplace_opts)=>{ + const api_key = ensure_authenticated(opts.apiKey); + const spinner = start_spinner(`Checking snapshot "${snapshot_id}"...`); + let snap: Snapshot_status; + // The try covers only the request: a failed *snapshot* is a verdict to + // report, not an exception to swallow. + try { + snap = await fetch_snapshot_status(api_key, snapshot_id, opts.timing); + spinner.stop(); + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + return; + } + if (opts.json || opts.pretty || opts.output) + { + print(snap, { + json: opts.json, + pretty: opts.pretty, + output: opts.output, + }); + return; + } + info(`Status: ${snap.status ?? 'unknown'}`); + print_snapshot_summary(snap); + if (snap.status == FAILED_STATUS && is_empty_result(snap)) + { + info('The filter matched no records.'); + return; + } + if (snap.status == FAILED_STATUS) + fail(`Snapshot ${snapshot_id} failed: ${snapshot_error(snap)}`); +}; + +const handle_download = async(snapshot_id: string, opts: Marketplace_opts)=>{ + const format = resolve_format(opts.format); + const api_key = ensure_authenticated(opts.apiKey); + let timeout = 600; + try { + timeout = parse_timeout(opts.timeout); + } catch(e) { + fail((e as Error).message); + return; + } + const spinner = start_spinner(`Fetching snapshot "${snapshot_id}"...`); + let snap: Snapshot_status; + try { + snap = await fetch_snapshot_status(api_key, snapshot_id, opts.timing); + spinner.stop(); + const running = !!snap.status && RUNNING_STATUSES.includes(snap.status); + if (running && !opts.wait) + { + info(`Snapshot "${snapshot_id}" is not ready yet ` + +`(status: ${snap.status}).`); + info('Re-run with --wait, or check it with: ' + +`brightdata marketplace status ${snapshot_id}`); + // 3 = not ready (retryable), distinct from 1 = error. + process.exit(EXIT.NOT_READY); + return; + } + if (running) + snap = await wait_for_snapshot(api_key, snapshot_id, timeout, opts); + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + return; + } + // A failed snapshot is a verdict, reported outside the request try/catch. + if (snap.status == FAILED_STATUS && is_empty_result(snap)) + { + info('The filter matched no records — nothing to download.'); + return; + } + if (snap.status == FAILED_STATUS) + { + fail(`Snapshot ${snapshot_id} failed: ${snapshot_error(snap)}`); + return; + } + print_snapshot_summary(snap); + try { + await download_snapshot(api_key, snapshot_id, format, opts); + } catch(e) { + console.error((e as Error).message); + process.exit(EXIT.ERROR); + } +}; + +const handle_filter = async(opts: Marketplace_opts)=>{ + const dataset_id = resolve_dataset_id(opts); + if (!dataset_id) + return; + const filter = load_filter(opts); + if (filter === undefined) + return; + const records_limit = parse_records_limit(opts.recordsLimit); + if (records_limit === undefined) + return; + const format = resolve_format(opts.format); + const api_key = ensure_authenticated(opts.apiKey); + let timeout = 600; + try { + timeout = parse_timeout(opts.timeout); + } catch(e) { + fail((e as Error).message); + return; + } + const body: Filter_request = {dataset_id, filter, records_limit}; + const spinner = start_spinner(`Filtering ${dataset_id}...`); + // Each try wraps one request only; verdicts about the result are reported + // after it, so the handler's own catch cannot swallow them. + let res: Filter_response; + try { + res = await post(api_key, FILTER_ENDPOINT, body, + {timing: opts.timing}); + spinner.stop(); + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + return; + } + const snapshot_id = res?.snapshot_id; + if (!snapshot_id) + { + fail( + 'Failed to create snapshot: ' + +(res?.error || res?.message || res?.failure_reason + || 'no snapshot_id in response') + ); + return; + } + if (opts.async) + { + success(`Filter submitted. Snapshot ID: ${snapshot_id}`); + info(`Check it with: brightdata marketplace status ` + +`${snapshot_id}`); + info('Download when ready: brightdata marketplace download ' + +`${snapshot_id} --wait`); + return; + } + console.error(dim(`Created snapshot ${snapshot_id}`)); + let snap: Snapshot_status; + try { + snap = await wait_for_snapshot(api_key, snapshot_id, timeout, opts); + } catch(e) { + console.error((e as Error).message); + process.exit(EXIT.ERROR); + return; + } + if (snap.status == FAILED_STATUS && is_empty_result(snap)) + { + info('The filter matched no records.'); + return; + } + if (snap.status == FAILED_STATUS) + { + fail(`Snapshot ${snapshot_id} failed: ${snapshot_error(snap)}`); + return; + } + print_snapshot_summary(snap); + try { + await download_snapshot(api_key, snapshot_id, format, opts); + } catch(e) { + console.error((e as Error).message); + process.exit(EXIT.ERROR); + } +}; + +const list_command = new Command('list') + .description('List datasets in the marketplace catalogue') + .option('--featured', 'Only the datasets with named shortcuts') + .option('--search ', 'Filter by text in the name or id') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_list); + +const fields_command = new Command('fields') + .description('Show a dataset\'s filterable fields and their types') + .argument('', 'Dataset name (see list --featured) or gd_ id') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_fields); + +const filter_command = new Command('filter') + .description('Query a dataset and download the matching records') + .option('--dataset ', + 'Dataset name (see \'marketplace list --featured\')') + .option('--dataset-id ', 'Raw dataset id (gd_...), for any dataset') + .option('--filter ', 'Filter tree as inline JSON') + .option('--filter-file ', 'Filter tree from a JSON file') + .option('--records-limit ', + 'Maximum records to return (REQUIRED — queries are billed ' + +'by records)') + .option('--format ', 'Result format: json, jsonl, csv (default: json)') + .option('--timeout ', + 'Polling timeout in seconds ' + +'(default: 600 or BRIGHTDATA_POLLING_TIMEOUT)') + .option('--async', 'Submit only; print the snapshot ID and exit') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_filter); + +const status_command = new Command('status') + .description('Check a marketplace snapshot (status, records, size, cost)') + .argument('', 'Snapshot ID returned by marketplace filter') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_status); + +const download_command = new Command('download') + .description( + 'Download a marketplace snapshot ' + +'(exit codes: 0 data, 1 error, 3 not ready yet)' + ) + .argument('', 'Snapshot ID returned by marketplace filter') + .option('--wait', 'Poll until the snapshot is ready, then download') + .option('--format ', 'Result format: json, jsonl, csv (default: json)') + .option('--timeout ', + 'Polling timeout in seconds with --wait ' + +'(default: 600 or BRIGHTDATA_POLLING_TIMEOUT)') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_download); + +const marketplace_command = new Command('marketplace') + .description( + 'Query Bright Data\'s pre-collected datasets ' + +'(pipelines collects new data; this queries data that already exists)' + ) + .addCommand(list_command) + .addCommand(fields_command) + .addCommand(filter_command) + .addCommand(status_command) + .addCommand(download_command); + +add_examples(marketplace_command, [ + { + description: 'See the datasets that have named shortcuts', + command: 'brightdata marketplace list --featured', + }, + { + description: 'Find any other dataset in the catalogue', + command: 'brightdata marketplace list --search zillow', + }, + { + description: 'Inspect what you can filter on before spending anything', + command: 'brightdata marketplace fields linkedin_people_profiles', + }, + { + description: 'Query a dataset (a record cap is always required)', + command: 'brightdata marketplace filter ' + +'--dataset linkedin_company_profiles ' + +'--filter \'{"name":"industry","operator":"=",' + +'"value":"Technology"}\' --records-limit 1000 ' + +'--format jsonl -o tech.jsonl', + }, + { + description: 'Submit now, collect later', + command: 'brightdata marketplace filter --dataset x_twitter_posts ' + +'--filter \'{"name":"likes","operator":">","value":10000}\' ' + +'--records-limit 500 --async', + }, +]); + +export {marketplace_command, handle_list, handle_fields, handle_filter, + handle_status, handle_download, resolve_dataset_id, FEATURED_DATASETS}; diff --git a/src/index.ts b/src/index.ts index 2136762..4ea6912 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import {logout_command} from './commands/logout'; import {scrape_command} from './commands/scrape'; import {search_command} from './commands/search'; import {pipelines_command} from './commands/dataset'; +import {marketplace_command} from './commands/marketplace'; import {status_command} from './commands/status'; import {zones_command} from './commands/zones'; import {config_command} from './commands/config'; @@ -41,6 +42,7 @@ const build_program = ()=>{ program.addCommand(scrape_command); program.addCommand(search_command); program.addCommand(pipelines_command); + program.addCommand(marketplace_command); program.addCommand(status_command); program.addCommand(zones_command); program.addCommand(config_command); diff --git a/src/types/marketplace.ts b/src/types/marketplace.ts new file mode 100644 index 0000000..fcada1d --- /dev/null +++ b/src/types/marketplace.ts @@ -0,0 +1,87 @@ +// Dataset Marketplace: querying pre-collected records. Distinct from the +// /datasets/v3/* Web Scraper API that `pipelines` uses — that one collects new +// data and bills for scraping; this one filters data Bright Data already holds. +// Note the formats differ too: no ndjson here. +type Marketplace_format = 'json'|'jsonl'|'csv'; + +type Dataset_info = { + id: string; + name?: string; + size?: number; // record count +}; + +type Dataset_field = { + type?: string; // text | number | url | array | object | boolean + active?: boolean; + required?: boolean; + description?: string; +}; + +type Dataset_metadata = { + id?: string; + fields?: Record; +}; + +type Snapshot_status = { + id?: string; + snapshot_id?: string; + status?: 'scheduled'|'building'|'ready'|'failed'|string; + dataset_id?: string; + dataset_size?: number; // records + file_size?: number; // bytes + cost?: number; + error?: string; + error_message?: string; + failure_reason?: string; + message?: string; + // A zero-match query comes back as status:failed with these set, rather + // than as a ready snapshot holding no rows. + warning?: string; + warning_code?: string; +}; + +// records_limit is optional in the API but REQUIRED by this CLI: omitting it +// means "no cap" against datasets holding hundreds of millions of records, and +// cost is only observable after the query is committed. +type Filter_request = { + dataset_id: string; + filter: unknown; + records_limit: number; +}; + +type Filter_response = { + snapshot_id?: string; + error?: string; + message?: string; + failure_reason?: string; +}; + +type Marketplace_opts = { + dataset?: string; + datasetId?: string; + filter?: string; + filterFile?: string; + recordsLimit?: string; + featured?: boolean; + search?: string; + format?: string; + timeout?: string; + wait?: boolean; + async?: boolean; + output?: string; + json?: boolean; + pretty?: boolean; + timing?: boolean; + apiKey?: string; +}; + +export type { + Marketplace_format, + Dataset_info, + Dataset_field, + Dataset_metadata, + Snapshot_status, + Filter_request, + Filter_response, + Marketplace_opts, +}; diff --git a/src/utils/exit-codes.ts b/src/utils/exit-codes.ts new file mode 100644 index 0000000..cdf0524 --- /dev/null +++ b/src/utils/exit-codes.ts @@ -0,0 +1,9 @@ +// Exit codes are cross-command public API for shell users. +// 0 success · 1 error (fail() and catch-all paths) · 3 result not ready yet +const EXIT = { + OK: 0, + ERROR: 1, + NOT_READY: 3, +} as const; + +export {EXIT};