{"openapi":"3.1.0","info":{"title":"The Model Company API","version":"0.1.0-draft.2026-10-01","summary":"Create small, specialized models from a task description and examples.","description":"DRAFT — this API is planned and not live. The host below is a placeholder. The web preview at / uses an in-browser simulation of these same contracts.","x-status":"planned"},"servers":[{"url":"https://api.tmc.example","description":"Planned — not live (placeholder host)"}],"security":[{"bearerAuth":[]}],"tags":[{"name":"Discovery"},{"name":"Datasets"},{"name":"Jobs","description":"Long-running training jobs. Create returns 202; poll GET /v1/jobs/{job_id} or subscribe to webhooks."},{"name":"Models"},{"name":"Shadow","description":"Shadow mode: your system sends a copy of each live call with the current model's answer, outcomes follow, and both models are compared on them. Nothing here is on your request path."},{"name":"Swap","description":"Letting the model answer: your code asks POST /v1/models/{model_id}/answer before answering, and the rollout decides which model answers each call. Fall back to your current model on any error or a short timeout."}],"paths":{"/v1/capabilities":{"get":{"tags":["Discovery"],"operationId":"describe_capabilities","summary":"Capability manifest","security":[],"responses":{"200":{"description":"OK"}}}},"/v1/estimates":{"post":{"tags":["Discovery"],"operationId":"estimate_cost","summary":"Free cost and time estimate (dry run)","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EstimateRequest"}}}},"responses":{"200":{"description":"Estimate","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Estimate"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/datasets":{"post":{"tags":["Datasets"],"operationId":"create_dataset","summary":"Register training data and run data-quality checks","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateDatasetInput"}}}},"responses":{"201":{"description":"Created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/DatasetSummary"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/jobs":{"post":{"tags":["Jobs"],"operationId":"create_model_job","summary":"Start a training job","parameters":[{"name":"Idempotency-Key","in":"header","required":true,"description":"Same key + same body replays the original job for 24 h; a different body returns 409.","schema":{"type":"string","minLength":8,"maxLength":128}}],"description":"Reference a dataset registered with POST /v1/datasets. The idempotency key travels in a header here; the MCP tool takes it as an argument.","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateJobRequest"}}}},"responses":{"202":{"description":"Accepted — the job is queued","headers":{"Location":{"schema":{"type":"string"},"description":"/v1/jobs/{job_id}"},"Retry-After":{"schema":{"type":"integer"},"description":"Seconds before the first poll"}},"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Job"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"402":{"description":"The estimate exceeds max_spend_usd, or approval is required","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"409":{"description":"Idempotency key reused with a different body","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}},"get":{"tags":["Jobs"],"operationId":"list_jobs","summary":"List jobs, newest first","responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/Job"}}}}}}}}}},"/v1/jobs/{job_id}":{"get":{"tags":["Jobs"],"operationId":"get_job","summary":"Poll a job","parameters":[{"name":"job_id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"OK","headers":{"Retry-After":{"schema":{"type":"integer"}}},"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Job"}}}},"404":{"description":"Not found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/jobs/{job_id}/cancel":{"post":{"tags":["Jobs"],"operationId":"cancel_job","summary":"Cancel a running job","parameters":[{"name":"job_id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Canceled","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Job"}}}},"409":{"description":"Already finished","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/models":{"get":{"tags":["Models"],"operationId":"list_models","summary":"List models","responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ModelSummary"}}}}}}}}}},"/v1/models/{model_id}":{"get":{"tags":["Models"],"operationId":"get_model","summary":"Model, evaluation and artifacts","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Model"}}}},"404":{"description":"Not found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}},"delete":{"tags":["Models"],"operationId":"delete_model","summary":"Delete a model, its jobs and its training data","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"204":{"description":"Deleted"},"404":{"description":"Not found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"409":{"description":"Its job is still running; cancel it first","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/models/{model_id}/predict":{"post":{"tags":["Models"],"operationId":"predict","summary":"Run one prediction","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PredictRequest"}}}},"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Prediction"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/models/{model_id}/exports":{"post":{"tags":["Models"],"operationId":"export_model","summary":"Signed download for an artifact","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExportRequest"}}}},"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExportResult"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/models/{model_id}/shadow":{"get":{"tags":["Shadow"],"operationId":"get_shadow","summary":"The live comparison: counts, the head-to-head and its verdict","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Shadow"}}}},"404":{"description":"Not found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}},"post":{"tags":["Shadow"],"operationId":"update_shadow","summary":"Name what answers today and set the swap bar","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ShadowSettings"}}}},"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Shadow"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/models/{model_id}/shadow/calls":{"post":{"tags":["Shadow"],"operationId":"record_shadow_calls","summary":"Record live calls with the current model's answers","description":"Send each call after your current model has answered, without waiting for the reply. Ours answers it in the background.","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RecordShadowCalls"}}}},"responses":{"202":{"description":"Recorded; ours answers within seconds","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ShadowReceipt"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"409":{"description":"The model isn't ready yet","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}},"get":{"tags":["Shadow"],"operationId":"list_shadow_calls","summary":"Recent calls, or the calls where the two disagree that nobody has reviewed","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}},{"name":"view","in":"query","schema":{"type":"string","enum":["recent","review"],"default":"recent"}},{"name":"limit","in":"query","schema":{"type":"integer","minimum":1,"maximum":100,"default":25}}],"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ShadowCall"}}}}}}}}}},"/v1/models/{model_id}/shadow/outcomes":{"post":{"tags":["Shadow"],"operationId":"record_shadow_outcomes","summary":"Attach right answers to recorded calls, by your call id","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RecordShadowOutcomes"}}}},"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ShadowOutcomesReceipt"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/models/{model_id}/shadow/calls/{call_id}/review":{"post":{"tags":["Shadow"],"operationId":"review_shadow_call","summary":"A person's answer for one call; it takes precedence over the reported outcome","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}},{"name":"call_id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ShadowReview"}}}},"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ShadowCall"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"404":{"description":"Not found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/models/{model_id}/answer":{"post":{"tags":["Swap"],"operationId":"answer","summary":"Which model answers this call, and this model's answer when it's the one","description":"On your request path. The rollout decides, the same way every time for the same id. When `use` is `incumbent`, answer with your current model and record its answer with POST …/shadow/calls.","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AnswerRequest"}}}},"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Answer"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"409":{"description":"The model isn't ready yet","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/models/{model_id}/rollout":{"post":{"tags":["Swap"],"operationId":"update_rollout","summary":"Who answers: the share of calls, the handback threshold and the guard","parameters":[{"name":"model_id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RolloutSettings"}}}},"responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Shadow"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"403":{"description":"The key lacks the required scope","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"409":{"description":"The guard is on and the model is clearly behind, so it would swap back at once","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"Validation failed","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limited — honour Retry-After","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/workspace":{"get":{"tags":["Discovery"],"operationId":"get_workspace","summary":"The workspace the bearer token opens (invite-only preview)","responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Workspace"}}}},"401":{"description":"Missing or invalid API key","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/v1/access-requests":{"post":{"tags":["Discovery"],"operationId":"request_access","summary":"Request access to real training","security":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccessRequest"}}}},"responses":{"202":{"description":"Received","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccessReceipt"}}}}}}}},"webhooks":{"job.succeeded":{"post":{"summary":"A job finished and the model is ready","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"type":{"const":"job.succeeded"},"job_id":{"type":"string"},"model_id":{"type":"string"}}}}}},"responses":{"200":{"description":"Acknowledge within 10 s"}}}},"job.failed":{"post":{"summary":"A job failed; the payload includes an error code and a fix-it hint","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"type":{"const":"job.failed"},"job_id":{"type":"string"},"error":{"type":"object"}}}}}},"responses":{"200":{"description":"Acknowledge within 10 s"}}}},"job.canceled":{"post":{"summary":"A job was canceled; nothing more is charged","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"type":{"const":"job.canceled"},"job_id":{"type":"string"}}}}}},"responses":{"200":{"description":"Acknowledge within 10 s"}}}}},"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","description":"Scoped API keys. Scopes: models:read (Read models, jobs, estimates and evaluations); datasets:write (Register training data); jobs:create (Start training jobs (spends money, within caps)); jobs:cancel (Cancel running jobs); models:predict (Run predictions (hosted models bill per call)); models:export (Download weights and containers)."}},"schemas":{"EstimateRequest":{"type":"object","properties":{"output_kind":{"type":"string","enum":["single_label","multi_label","extraction"],"description":"single_label: pick exactly one category. multi_label: pick any number of tags. extraction: return named fields."},"label_count":{"type":"integer","minimum":0,"maximum":50},"field_count":{"type":"integer","minimum":0,"maximum":30},"row_count":{"type":"integer","minimum":0,"maximum":9007199254740991},"requirements":{"type":"object","properties":{"min_accuracy":{"type":"number","minimum":0.5,"maximum":0.995,"description":"Quality bar on the held-out test set (0–1)."},"max_latency_ms":{"type":"integer","minimum":1,"maximum":10000,"description":"p95 latency budget per prediction, in ms."},"deployment":{"type":"string","enum":["hosted_api","self_hosted","on_device"],"description":"Where the finished model will run. Affects candidate selection and packaging."},"max_spend_usd":{"type":"number","minimum":0,"maximum":10000,"description":"Hard spending cap for this job. The job stops before exceeding it."},"third_party":{"default":null,"description":"Null (the default): only models that run on our servers can be chosen. Set: models that call a third-party model can be tried, chosen and deployed too. Separate from `compare_with_llms`, which only scores general-purpose LLMs for comparison.","anyOf":[{"type":"object","properties":{"daily_cap_usd":{"type":"number","minimum":0.01,"maximum":10000,"description":"Live calls to such a model stop for the day once they've cost this much, and callers fall back to the model they use today."}},"required":["daily_cap_usd"],"description":"Allows models that call a third-party model, such as Claude through Anthropic's API, for every answer. If one is chosen and deployed, every live call goes to that provider, with emails and long numbers masked."},{"type":"null"}]}},"required":["min_accuracy","max_latency_ms","deployment","max_spend_usd"]},"advanced":{"type":"object","properties":{"base_models":{"anyOf":[{"type":"string","const":"auto"},{"minItems":1,"type":"array","items":{"type":"string"}}],"description":"`auto` lets the planner choose candidates; or pin specific base models."},"max_candidates":{"type":"integer","minimum":1,"maximum":8},"validation_split":{"type":"number","minimum":0.05,"maximum":0.4},"epochs":{"anyOf":[{"type":"string","const":"auto"},{"type":"integer","minimum":1,"maximum":50}]},"learning_rate":{"anyOf":[{"type":"string","const":"auto"},{"type":"number","exclusiveMinimum":0,"maximum":1}]},"quantization":{"type":"string","enum":["none","int8","int4"]},"early_stopping":{"type":"boolean"},"seed":{"type":"integer","minimum":0,"maximum":2147483647}},"required":["base_models","max_candidates","validation_split","epochs","learning_rate","quantization","early_stopping","seed"]},"compare_with_llms":{"description":"Whether the job will also score general-purpose LLMs (see CreateJobRequest).","type":"boolean"}},"required":["output_kind","label_count","field_count","row_count","requirements"]},"Estimate":{"type":"object","properties":{"currency":{"type":"string","const":"USD"},"training_usd_low":{"type":"number"},"training_usd_high":{"type":"number"},"training_minutes_low":{"type":"number"},"training_minutes_high":{"type":"number"},"candidates":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"inference_usd_per_1k":{"type":"number","description":"Hosted inference price; 0 when self-hosted or on-device."},"breakdown":{"type":"array","items":{"type":"object","properties":{"item":{"type":"string"},"usd":{"type":"number"}},"required":["item","usd"],"additionalProperties":false}},"within_cap":{"type":"boolean","description":"Whether the high estimate fits under requirements.max_spend_usd."},"notes":{"type":"array","items":{"type":"string"}}},"required":["currency","training_usd_low","training_usd_high","training_minutes_low","training_minutes_high","candidates","inference_usd_per_1k","breakdown","within_cap","notes"],"additionalProperties":false},"CreateDatasetInput":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":120},"format":{"type":"string","enum":["csv","jsonl","json"]},"text_column":{"description":"For display: the file column the input came from.","type":"string"},"label_column":{"description":"For display: the file column the answers came from; null when there are none.","type":["string","null"]},"source_url":{"description":"HTTPS URL we can fetch the file from (signed URLs welcome).","type":"string","format":"uri"},"incumbent":{"description":"Who gave the examples' `incumbent` answers. Named \"Your current model\" when omitted.","type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"What answers today, e.g. \"Claude Opus 5.5 with our prompt\", \"keyword rules\" or \"the support team\"."},"cost_usd_per_1k":{"description":"Its cost per 1,000 calls, if known. Shown as reported.","type":"number","minimum":0,"maximum":100000},"latency_ms_p95":{"description":"Its p95 latency in ms, if known. Shown as reported.","type":"number","minimum":0,"maximum":600000}},"required":["name"]},"cleaning":{"description":"How to clean each input before storing it. Send the text as it is; cleaning it yourself first is fine too.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"object","properties":{"markup":{"default":false,"description":"HTML to text: block tags become line breaks; styles, scripts, comments and inline-image placeholders are dropped; entities decoded.","type":"boolean"},"quoted_replies":{"default":false,"description":"Drop quoted earlier messages: `>` lines, and everything from a reply header on (-----Original Message-----, On … wrote:, From:/Sent:/To: blocks, and the same in German, French, Spanish, Italian and Dutch).","type":"boolean"},"signatures":{"default":false,"description":"Drop a greeting-only first line and, at the end, a `-- ` signature, a \"Sent from my…\" footer, or a sign-off with the few short lines under it.","type":"boolean"},"drop_lines":{"default":[],"description":"Lines dropped wherever they appear, compared after lowercasing, collapsing spaces and masking email addresses and long numbers.","maxItems":50,"type":"array","items":{"type":"string","minLength":1,"maxLength":1000}}},"description":"How one text input is cleaned: when examples are registered, and on every call a model trained on them answers. Cleaning twice changes nothing, and a step that would empty a value leaves it as it was."}},"label_map":{"description":"Labels to combine before storing the examples.","type":"object","propertyNames":{"type":"string","minLength":1,"maxLength":200},"additionalProperties":{"type":"string","minLength":1,"maxLength":60}},"rows":{"description":"Inline examples, { input, output } or the { text, label } shorthand; only what training needs. Provide this or source_url.","maxItems":10000,"type":"array","items":{"anyOf":[{"type":"object","properties":{"input":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string","maxLength":20000},"description":"Input values keyed by the task's input names. A text input takes a string."},"output":{"default":{},"description":"The right answer, as it turned out. Empty when it isn't known.","type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}}},"incumbent":{"description":"What the model in use today answered. Graded against the right answer to compare with ours; never trained on.","type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}}},"at":{"description":"When the call happened (ISO 8601, e.g. 2026-09-14T10:32:00Z). With times, each label's most recent examples are held out for testing.","type":"string","maxLength":40}},"required":["input"]},{"type":"object","properties":{"text":{"type":"string","maxLength":20000},"label":{"anyOf":[{"type":"string","maxLength":200},{"type":"null"}]}},"required":["text","label"],"description":"Shorthand for one text input and one label: the same as { input: { text }, output: { label } }."}]}}},"required":["name","format"]},"DatasetSummary":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"source":{"type":"string","enum":["upload","sample"]},"format":{"type":"string","enum":["csv","jsonl","json"]},"row_count":{"type":"integer","minimum":0,"maximum":9007199254740991},"input_names":{"type":"array","items":{"type":"string"},"description":"The inputs the examples have values for; a task trained on them reads these."},"text_column":{"type":"string","description":"Where the input came from: the file's column, or the input names."},"label_column":{"type":["string","null"]},"label_counts":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991}},"incumbent":{"anyOf":[{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"What answers today, e.g. \"Claude Opus 5.5 with our prompt\", \"keyword rules\" or \"the support team\"."},"cost_usd_per_1k":{"description":"Its cost per 1,000 calls, if known. Shown as reported.","type":"number","minimum":0,"maximum":100000},"latency_ms_p95":{"description":"Its p95 latency in ms, if known. Shown as reported.","type":"number","minimum":0,"maximum":600000},"answers":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Examples that include its answer."}},"required":["name","answers"],"additionalProperties":false},{"type":"null"}],"description":"The model in use today, when the examples include its answers."},"dated":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Examples with a readable `at` time."},"issues":{"type":"array","items":{"type":"object","properties":{"code":{"type":"string","enum":["missing_labels","duplicates","too_few_examples","imbalanced","short_texts","unknown_labels","labels_without_examples","empty_rows","too_few_rows","masked_personal_data","cleaned","answer_in_input"]},"severity":{"type":"string","enum":["info","warning","error"]},"message":{"type":"string"},"count":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991}},"required":["code","severity","message"],"additionalProperties":false}},"sample_rows":{"maxItems":25,"type":"array","items":{"type":"object","properties":{"input":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string","maxLength":20000},"description":"Input values keyed by the task's input names. A text input takes a string."},"output":{"default":{},"description":"The right answer, as it turned out. Empty when it isn't known.","type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false},"incumbent":{"description":"What the model in use today answered. Graded against the right answer to compare with ours; never trained on.","type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false},"at":{"description":"When the call happened (ISO 8601, e.g. 2026-09-14T10:32:00Z). With times, each label's most recent examples are held out for testing.","type":"string","maxLength":40}},"required":["input","output"],"additionalProperties":false},"description":"A preview of the examples after cleaning, each value cut to 400 characters. In the simulated demo the full file stays in the uploader's browser."},"synthetic":{"type":"boolean","description":"True for the bundled sample datasets, which are synthetic."},"cleaning":{"default":{},"description":"How each input was cleaned. Every model trained on these examples cleans its calls the same way.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"object","properties":{"markup":{"default":false,"description":"HTML to text: block tags become line breaks; styles, scripts, comments and inline-image placeholders are dropped; entities decoded.","type":"boolean"},"quoted_replies":{"default":false,"description":"Drop quoted earlier messages: `>` lines, and everything from a reply header on (-----Original Message-----, On … wrote:, From:/Sent:/To: blocks, and the same in German, French, Spanish, Italian and Dutch).","type":"boolean"},"signatures":{"default":false,"description":"Drop a greeting-only first line and, at the end, a `-- ` signature, a \"Sent from my…\" footer, or a sign-off with the few short lines under it.","type":"boolean"},"drop_lines":{"default":[],"description":"Lines dropped wherever they appear, compared after lowercasing, collapsing spaces and masking email addresses and long numbers.","maxItems":50,"type":"array","items":{"type":"string","minLength":1,"maxLength":1000}}},"required":["markup","quoted_replies","signatures","drop_lines"],"additionalProperties":false,"description":"How one text input is cleaned: when examples are registered, and on every call a model trained on them answers. Cleaning twice changes nothing, and a step that would empty a value leaves it as it was."}},"label_map":{"default":{},"description":"Labels combined when the examples were registered. Shadow mode combines recorded answers the same way.","type":"object","propertyNames":{"type":"string","minLength":1,"maxLength":200},"additionalProperties":{"type":"string","minLength":1,"maxLength":60}},"masking":{"default":{"placeholders":"plain"},"type":"object","properties":{"placeholders":{"type":"string","enum":["plain","numbered"],"description":"plain: every email becomes <email> and every long number <number>. numbered: <email_1>, <number_1>…, the same value getting the same placeholder within an example, so an answer containing a masked value is stored with its placeholder and restored from each call's own values when the model answers."}},"required":["placeholders"],"additionalProperties":false,"description":"How emails and long numbers were masked before the examples were stored. Every model trained on them masks its calls the same way."}},"required":["id","name","source","format","row_count","input_names","text_column","label_column","label_counts","incumbent","dated","issues","sample_rows","synthetic","cleaning","label_map","masking"],"additionalProperties":false},"CreateJobRequest":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80},"task":{"type":"object","properties":{"description":{"type":"string","minLength":8,"maxLength":500,"description":"What the model should do, in plain language."},"template_id":{"type":"string","enum":["support_routing","feedback_classification","invoice_extraction","custom"],"description":"Starting template. `custom` means the task was described from scratch."},"inputs":{"default":[{"name":"text","type":"text"}],"description":"What the model reads. Defaults to one text input named `text`.","minItems":1,"maxItems":10,"type":"array","items":{"type":"object","properties":{"name":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,47}$","description":"Key of this input in an example's or a request's `input`, e.g. `subject`."},"type":{"type":"string","enum":["text"],"description":"What an input holds. Text today; images, audio and documents will be added as more types."},"description":{"description":"What the input holds. Helps the general-purpose LLM comparison.","type":"string","maxLength":240}},"required":["name","type"]}},"output_kind":{"type":"string","enum":["single_label","multi_label","extraction"],"description":"single_label: pick exactly one category. multi_label: pick any number of tags. extraction: return named fields."},"labels":{"default":[],"maxItems":50,"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":60,"description":"Category the model can return."},"description":{"description":"Plain-language definition; improves training.","type":"string","maxLength":240},"metadata":{"type":"object","propertyNames":{"type":"string","minLength":1,"maxLength":40},"additionalProperties":{"type":"string","maxLength":200},"description":"Returned with a prediction of this label: anything your code acts on, such as a queue, an owner or an action."}},"required":["name"]}},"fields":{"default":[],"description":"extraction: the fields to return.","maxItems":30,"type":"array","items":{"type":"object","properties":{"name":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,47}$","description":"Field key in the model's output, e.g. `invoice_number`."},"type":{"type":"string","enum":["text","number","date","money"]},"description":{"type":"string","maxLength":240}},"required":["name","type"]}}},"required":["description","template_id","output_kind"]},"dataset_id":{"type":"string","minLength":1,"description":"A dataset registered with POST /v1/datasets (or a bundled sample id)."},"requirements":{"type":"object","properties":{"min_accuracy":{"type":"number","minimum":0.5,"maximum":0.995,"description":"Quality bar on the held-out test set (0–1)."},"max_latency_ms":{"type":"integer","minimum":1,"maximum":10000,"description":"p95 latency budget per prediction, in ms."},"deployment":{"type":"string","enum":["hosted_api","self_hosted","on_device"],"description":"Where the finished model will run. Affects candidate selection and packaging."},"max_spend_usd":{"type":"number","minimum":0,"maximum":10000,"description":"Hard spending cap for this job. The job stops before exceeding it."},"third_party":{"default":null,"description":"Null (the default): only models that run on our servers can be chosen. Set: models that call a third-party model can be tried, chosen and deployed too. Separate from `compare_with_llms`, which only scores general-purpose LLMs for comparison.","anyOf":[{"type":"object","properties":{"daily_cap_usd":{"type":"number","minimum":0.01,"maximum":10000,"description":"Live calls to such a model stop for the day once they've cost this much, and callers fall back to the model they use today."}},"required":["daily_cap_usd"],"description":"Allows models that call a third-party model, such as Claude through Anthropic's API, for every answer. If one is chosen and deployed, every live call goes to that provider, with emails and long numbers masked."},{"type":"null"}]}},"required":["min_accuracy","max_latency_ms","deployment","max_spend_usd"]},"advanced":{"type":"object","properties":{"base_models":{"anyOf":[{"type":"string","const":"auto"},{"minItems":1,"type":"array","items":{"type":"string"}}],"description":"`auto` lets the planner choose candidates; or pin specific base models."},"max_candidates":{"type":"integer","minimum":1,"maximum":8},"validation_split":{"type":"number","minimum":0.05,"maximum":0.4},"epochs":{"anyOf":[{"type":"string","const":"auto"},{"type":"integer","minimum":1,"maximum":50}]},"learning_rate":{"anyOf":[{"type":"string","const":"auto"},{"type":"number","exclusiveMinimum":0,"maximum":1}]},"quantization":{"type":"string","enum":["none","int8","int4"]},"early_stopping":{"type":"boolean"},"seed":{"type":"integer","minimum":0,"maximum":2147483647}},"required":["base_models","max_candidates","validation_split","epochs","learning_rate","quantization","early_stopping","seed"]},"mode":{"type":"string","enum":["simulated","training"],"description":"`simulated` replays a scripted run with illustrative results and costs nothing. `training` trains a real model."},"compare_with_llms":{"description":"Also score general-purpose LLMs on the same held-out examples, which sends those examples to the LLM provider. Defaults to true. Counts toward max_spend_usd.","type":"boolean"},"swap_bar":{"description":"When the examples include the current model's answers: what ours must show to be ready to replace it. Defaults to at_least_as_good with a 2-point margin.","type":"object","properties":{"rule":{"type":"string","enum":["at_least_as_good","better"],"description":"at_least_as_good: not behind the current model by more than `margin`, with 95% confidence. better: ahead of it, with 95% confidence."},"margin":{"type":"number","minimum":0,"maximum":0.2,"description":"How far behind still counts as as good, as a share of calls (0.02 = 2 points). No label or field may trail by more than this either."}},"required":["rule","margin"]}},"required":["name","task","dataset_id","requirements","mode"]},"Job":{"type":"object","properties":{"id":{"type":"string"},"model_id":{"type":"string"},"model_name":{"type":"string"},"mode":{"type":"string","enum":["simulated","training"],"description":"`simulated` replays a scripted run with illustrative results and costs nothing. `training` trains a real model."},"status":{"type":"string","enum":["queued","running","succeeded","failed","canceled"]},"created_at":{"type":"string"},"updated_at":{"type":"string"},"finished_at":{"type":["string","null"]},"progress":{"type":"number","minimum":0,"maximum":1},"eta_seconds":{"type":["number","null"]},"current_stage":{"anyOf":[{"type":"string","enum":["validate","prepare","select","train","evaluate","package"]},{"type":"null"}]},"requirements":{"type":"object","properties":{"min_accuracy":{"type":"number","minimum":0.5,"maximum":0.995,"description":"Quality bar on the held-out test set (0–1)."},"max_latency_ms":{"type":"integer","minimum":1,"maximum":10000,"description":"p95 latency budget per prediction, in ms."},"deployment":{"type":"string","enum":["hosted_api","self_hosted","on_device"],"description":"Where the finished model will run. Affects candidate selection and packaging."},"max_spend_usd":{"type":"number","minimum":0,"maximum":10000,"description":"Hard spending cap for this job. The job stops before exceeding it."},"third_party":{"default":null,"description":"Null (the default): only models that run on our servers can be chosen. Set: models that call a third-party model can be tried, chosen and deployed too. Separate from `compare_with_llms`, which only scores general-purpose LLMs for comparison.","anyOf":[{"type":"object","properties":{"daily_cap_usd":{"type":"number","minimum":0.01,"maximum":10000,"description":"Live calls to such a model stop for the day once they've cost this much, and callers fall back to the model they use today."}},"required":["daily_cap_usd"],"additionalProperties":false,"description":"Allows models that call a third-party model, such as Claude through Anthropic's API, for every answer. If one is chosen and deployed, every live call goes to that provider, with emails and long numbers masked."},{"type":"null"}]}},"required":["min_accuracy","max_latency_ms","deployment","max_spend_usd","third_party"],"additionalProperties":false,"description":"The bar this job is trained and judged against."},"stages":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string","enum":["validate","prepare","select","train","evaluate","package"]},"status":{"type":"string","enum":["pending","running","done","failed","skipped"]},"progress":{"type":"number","minimum":0,"maximum":1},"started_at":{"type":["string","null"]},"finished_at":{"type":["string","null"]},"summary":{"type":["string","null"]}},"required":["key","status","progress","started_at","finished_at","summary"],"additionalProperties":false}},"candidates":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"recipe":{"default":"encoder_head","type":"string","pattern":"^[a-z][a-z0-9_]{0,47}$","description":"How a model was built, e.g. `encoder_head` (a small encoder with a head trained on your examples) or `prompt` (a general-purpose LLM shown your most similar examples, with no training)."},"base_model":{"type":"string"},"family":{"type":"string","enum":["encoder","decoder"]},"params_m":{"description":"Parameters in millions; null when the provider doesn't publish them.","type":["number","null"]},"status":{"type":"string","enum":["queued","training","done","discarded","selected"]},"val_accuracy":{"description":"Validation score on the task's primary metric: accuracy for one label, F1 for tags, field accuracy for extraction.","type":["number","null"]},"latency_ms_p95":{"type":["number","null"]},"cost_usd_per_1k":{"default":null,"description":"Measured cost per 1,000 answers, for candidates that pay per call.","type":["number","null"]},"third_party":{"default":null,"description":"The third-party model this candidate calls for every answer; null when it runs on our servers.","type":["string","null"]},"curve":{"type":"array","items":{"type":"number"},"description":"Validation accuracy after each evaluation step."},"note":{"type":["string","null"]}},"required":["id","recipe","base_model","family","params_m","status","val_accuracy","latency_ms_p95","cost_usd_per_1k","third_party","curve","note"],"additionalProperties":false}},"events":{"type":"array","items":{"type":"object","properties":{"at":{"type":"string"},"level":{"type":"string","enum":["info","warn","error","success"]},"stage":{"anyOf":[{"type":"string","enum":["validate","prepare","select","train","evaluate","package"]},{"type":"null"}]},"message":{"type":"string"}},"required":["at","level","stage","message"],"additionalProperties":false}},"cost":{"type":"object","properties":{"currency":{"type":"string","const":"USD"},"spent_usd":{"type":"number"},"cap_usd":{"type":"number"},"estimated_usd":{"type":"number","description":"What the run would cost as real training (illustrative for simulated jobs)."}},"required":["currency","spent_usd","cap_usd","estimated_usd"],"additionalProperties":false},"idempotency_key":{"type":"string"},"error":{"anyOf":[{"type":"object","properties":{"code":{"type":"string"},"message":{"type":"string"},"hint":{"type":"string"}},"required":["code","message","hint"],"additionalProperties":false},{"type":"null"}]},"illustrative":{"type":"boolean"}},"required":["id","model_id","model_name","mode","status","created_at","updated_at","finished_at","progress","eta_seconds","current_stage","requirements","stages","candidates","events","cost","idempotency_key","error","illustrative"],"additionalProperties":false},"Model":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"status":{"type":"string","enum":["training","ready","failed","canceled"]},"mode":{"type":"string","enum":["simulated","training"],"description":"`simulated` replays a scripted run with illustrative results and costs nothing. `training` trains a real model."},"version":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"created_at":{"type":"string"},"updated_at":{"type":"string"},"latest_job_id":{"type":"string"},"task":{"type":"object","properties":{"description":{"type":"string","minLength":8,"maxLength":500,"description":"What the model should do, in plain language."},"template_id":{"type":"string","enum":["support_routing","feedback_classification","invoice_extraction","custom"],"description":"Starting template. `custom` means the task was described from scratch."},"inputs":{"default":[{"name":"text","type":"text"}],"description":"What the model reads. Defaults to one text input named `text`.","minItems":1,"maxItems":10,"type":"array","items":{"type":"object","properties":{"name":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,47}$","description":"Key of this input in an example's or a request's `input`, e.g. `subject`."},"type":{"type":"string","enum":["text"],"description":"What an input holds. Text today; images, audio and documents will be added as more types."},"description":{"description":"What the input holds. Helps the general-purpose LLM comparison.","type":"string","maxLength":240}},"required":["name","type"],"additionalProperties":false}},"output_kind":{"type":"string","enum":["single_label","multi_label","extraction"],"description":"single_label: pick exactly one category. multi_label: pick any number of tags. extraction: return named fields."},"labels":{"default":[],"maxItems":50,"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":60,"description":"Category the model can return."},"description":{"description":"Plain-language definition; improves training.","type":"string","maxLength":240},"metadata":{"type":"object","propertyNames":{"type":"string","minLength":1,"maxLength":40},"additionalProperties":{"type":"string","maxLength":200},"description":"Returned with a prediction of this label: anything your code acts on, such as a queue, an owner or an action."}},"required":["name"],"additionalProperties":false}},"fields":{"default":[],"description":"extraction: the fields to return.","maxItems":30,"type":"array","items":{"type":"object","properties":{"name":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,47}$","description":"Field key in the model's output, e.g. `invoice_number`."},"type":{"type":"string","enum":["text","number","date","money"]},"description":{"type":"string","maxLength":240}},"required":["name","type"],"additionalProperties":false}}},"required":["description","template_id","inputs","output_kind","labels","fields"],"additionalProperties":false},"dataset":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"source":{"type":"string","enum":["upload","sample"]},"format":{"type":"string","enum":["csv","jsonl","json"]},"row_count":{"type":"integer","minimum":0,"maximum":9007199254740991},"input_names":{"type":"array","items":{"type":"string"},"description":"The inputs the examples have values for; a task trained on them reads these."},"text_column":{"type":"string","description":"Where the input came from: the file's column, or the input names."},"label_column":{"type":["string","null"]},"label_counts":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991}},"incumbent":{"anyOf":[{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"What answers today, e.g. \"Claude Opus 5.5 with our prompt\", \"keyword rules\" or \"the support team\"."},"cost_usd_per_1k":{"description":"Its cost per 1,000 calls, if known. Shown as reported.","type":"number","minimum":0,"maximum":100000},"latency_ms_p95":{"description":"Its p95 latency in ms, if known. Shown as reported.","type":"number","minimum":0,"maximum":600000},"answers":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Examples that include its answer."}},"required":["name","answers"],"additionalProperties":false},{"type":"null"}],"description":"The model in use today, when the examples include its answers."},"dated":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Examples with a readable `at` time."},"issues":{"type":"array","items":{"type":"object","properties":{"code":{"type":"string","enum":["missing_labels","duplicates","too_few_examples","imbalanced","short_texts","unknown_labels","labels_without_examples","empty_rows","too_few_rows","masked_personal_data","cleaned","answer_in_input"]},"severity":{"type":"string","enum":["info","warning","error"]},"message":{"type":"string"},"count":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991}},"required":["code","severity","message"],"additionalProperties":false}},"sample_rows":{"maxItems":25,"type":"array","items":{"type":"object","properties":{"input":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string","maxLength":20000},"description":"Input values keyed by the task's input names. A text input takes a string."},"output":{"default":{},"description":"The right answer, as it turned out. Empty when it isn't known.","type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false},"incumbent":{"description":"What the model in use today answered. Graded against the right answer to compare with ours; never trained on.","type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false},"at":{"description":"When the call happened (ISO 8601, e.g. 2026-09-14T10:32:00Z). With times, each label's most recent examples are held out for testing.","type":"string","maxLength":40}},"required":["input","output"],"additionalProperties":false},"description":"A preview of the examples after cleaning, each value cut to 400 characters. In the simulated demo the full file stays in the uploader's browser."},"synthetic":{"type":"boolean","description":"True for the bundled sample datasets, which are synthetic."},"cleaning":{"default":{},"description":"How each input was cleaned. Every model trained on these examples cleans its calls the same way.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"object","properties":{"markup":{"default":false,"description":"HTML to text: block tags become line breaks; styles, scripts, comments and inline-image placeholders are dropped; entities decoded.","type":"boolean"},"quoted_replies":{"default":false,"description":"Drop quoted earlier messages: `>` lines, and everything from a reply header on (-----Original Message-----, On … wrote:, From:/Sent:/To: blocks, and the same in German, French, Spanish, Italian and Dutch).","type":"boolean"},"signatures":{"default":false,"description":"Drop a greeting-only first line and, at the end, a `-- ` signature, a \"Sent from my…\" footer, or a sign-off with the few short lines under it.","type":"boolean"},"drop_lines":{"default":[],"description":"Lines dropped wherever they appear, compared after lowercasing, collapsing spaces and masking email addresses and long numbers.","maxItems":50,"type":"array","items":{"type":"string","minLength":1,"maxLength":1000}}},"required":["markup","quoted_replies","signatures","drop_lines"],"additionalProperties":false,"description":"How one text input is cleaned: when examples are registered, and on every call a model trained on them answers. Cleaning twice changes nothing, and a step that would empty a value leaves it as it was."}},"label_map":{"default":{},"description":"Labels combined when the examples were registered. Shadow mode combines recorded answers the same way.","type":"object","propertyNames":{"type":"string","minLength":1,"maxLength":200},"additionalProperties":{"type":"string","minLength":1,"maxLength":60}},"masking":{"default":{"placeholders":"plain"},"type":"object","properties":{"placeholders":{"type":"string","enum":["plain","numbered"],"description":"plain: every email becomes <email> and every long number <number>. numbered: <email_1>, <number_1>…, the same value getting the same placeholder within an example, so an answer containing a masked value is stored with its placeholder and restored from each call's own values when the model answers."}},"required":["placeholders"],"additionalProperties":false,"description":"How emails and long numbers were masked before the examples were stored. Every model trained on them masks its calls the same way."}},"required":["id","name","source","format","row_count","input_names","text_column","label_column","label_counts","incumbent","dated","issues","sample_rows","synthetic","cleaning","label_map","masking"],"additionalProperties":false},"requirements":{"type":"object","properties":{"min_accuracy":{"type":"number","minimum":0.5,"maximum":0.995,"description":"Quality bar on the held-out test set (0–1)."},"max_latency_ms":{"type":"integer","minimum":1,"maximum":10000,"description":"p95 latency budget per prediction, in ms."},"deployment":{"type":"string","enum":["hosted_api","self_hosted","on_device"],"description":"Where the finished model will run. Affects candidate selection and packaging."},"max_spend_usd":{"type":"number","minimum":0,"maximum":10000,"description":"Hard spending cap for this job. The job stops before exceeding it."},"third_party":{"default":null,"description":"Null (the default): only models that run on our servers can be chosen. Set: models that call a third-party model can be tried, chosen and deployed too. Separate from `compare_with_llms`, which only scores general-purpose LLMs for comparison.","anyOf":[{"type":"object","properties":{"daily_cap_usd":{"type":"number","minimum":0.01,"maximum":10000,"description":"Live calls to such a model stop for the day once they've cost this much, and callers fall back to the model they use today."}},"required":["daily_cap_usd"],"additionalProperties":false,"description":"Allows models that call a third-party model, such as Claude through Anthropic's API, for every answer. If one is chosen and deployed, every live call goes to that provider, with emails and long numbers masked."},{"type":"null"}]}},"required":["min_accuracy","max_latency_ms","deployment","max_spend_usd","third_party"],"additionalProperties":false},"base_model":{"type":["string","null"]},"recipe":{"default":null,"description":"How the model was built and what it calls; null for older demo models.","anyOf":[{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,47}$","description":"How a model was built, e.g. `encoder_head` (a small encoder with a head trained on your examples) or `prompt` (a general-purpose LLM shown your most similar examples, with no training)."},"name":{"type":"string","description":"The recipe in plain language."},"third_party":{"description":"The third-party model it calls for every answer, and through whom, e.g. \"Claude Haiku 5.5 via Anthropic's API\". Null when it runs on our servers.","type":["string","null"]},"confidence":{"type":"boolean","description":"Whether its answers come with a confidence. The rollout's handback threshold needs one."}},"required":["id","name","third_party","confidence"],"additionalProperties":false},{"type":"null"}]},"evaluation":{"anyOf":[{"type":"object","properties":{"test_set_size":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"metrics":{"type":"object","properties":{"primary_metric":{"type":"string","enum":["accuracy","f1","field_accuracy"]},"primary_value":{"type":"number"},"accuracy":{"type":["number","null"]},"macro_f1":{"type":"number"},"precision":{"type":"number"},"recall":{"type":"number"},"latency_ms_p50":{"type":"number"},"latency_ms_p95":{"type":"number"},"size_mb":{"type":"number"},"params_m":{"type":["number","null"]},"cost_usd_per_1k":{"type":"number"}},"required":["primary_metric","primary_value","accuracy","macro_f1","precision","recall","latency_ms_p50","latency_ms_p95","size_mb","params_m","cost_usd_per_1k"],"additionalProperties":false},"per_class":{"type":"array","items":{"type":"object","properties":{"label":{"type":"string"},"precision":{"type":"number"},"recall":{"type":"number"},"f1":{"type":"number"},"support":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991}},"required":["label","precision","recall","f1","support"],"additionalProperties":false}},"confusion":{"anyOf":[{"type":"object","properties":{"labels":{"type":"array","items":{"type":"string"}},"matrix":{"type":"array","items":{"type":"array","items":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991}}}},"required":["labels","matrix"],"additionalProperties":false},{"type":"null"}],"description":"Rows are true labels, columns are predictions. Null for extraction tasks."},"baselines":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"kind":{"type":"string","enum":["your_model","general_llm_large","general_llm_small","rules","candidate"],"description":"candidate: the best model another recipe trained in this job, which lost to yours."},"description":{"type":"string"},"primary_value":{"type":"number"},"latency_ms_p95":{"type":"number"},"cost_usd_per_1k":{"type":"number"},"sample_size":{"description":"How many held-out examples this row was scored on, when fewer than the whole test set.","type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"ours_on_sample":{"description":"Your model's score on the same examples, when `sample_size` is set: compare this row with it, not with your model's score on the whole test set.","type":"number"}},"required":["id","name","kind","description","primary_value","latency_ms_p95","cost_usd_per_1k"],"additionalProperties":false}},"requirement_checks":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string","enum":["quality","latency","spend"]},"label":{"type":"string"},"target":{"type":"string"},"actual":{"type":"string"},"passed":{"type":"boolean"}},"required":["key","label","target","actual","passed"],"additionalProperties":false}},"error_examples":{"type":"array","items":{"type":"object","properties":{"input":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string","maxLength":20000},"description":"Input values keyed by the task's input names. A text input takes a string."},"expected":{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false,"description":"An answer, shaped by the task's output_kind. Examples, predictions, other models' answers and outcomes all use this shape."},"predicted":{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false,"description":"An answer, shaped by the task's output_kind. Examples, predictions, other models' answers and outcomes all use this shape."},"confidence":{"type":["number","null"]}},"required":["input","expected","predicted","confidence"],"additionalProperties":false},"description":"Held-out examples the model got wrong, most confident first; long values are shortened for display."},"notes":{"default":[],"description":"How the numbers were measured, and anything left out (e.g. a comparison that didn't run).","type":"array","items":{"type":"string"}},"warnings":{"default":[],"description":"Problems to look at before trusting the result, e.g. labels that don't follow the text.","type":"array","items":{"type":"string"}},"head_to_head":{"default":null,"description":"Against the model in use today, when the examples include its answers.","anyOf":[{"type":"object","properties":{"incumbent":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"What answers today, e.g. \"Claude Opus 5.5 with our prompt\", \"keyword rules\" or \"the support team\"."},"cost_usd_per_1k":{"description":"Its cost per 1,000 calls, if known. Shown as reported.","type":"number","minimum":0,"maximum":100000},"latency_ms_p95":{"description":"Its p95 latency in ms, if known. Shown as reported.","type":"number","minimum":0,"maximum":600000}},"required":["name"],"additionalProperties":false},"compared":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls both models answered. In the backtest, every one has a right answer."},"graded":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Compared calls with a known right answer."},"ours":{"description":"Share of calls our model got exactly right: measured when every compared call has a right answer, otherwise estimated from those that do. Null until it can be estimated.","type":["number","null"]},"theirs":{"description":"The same for the current model.","type":["number","null"]},"ours_only":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Graded calls only our model got right."},"theirs_only":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Graded calls only the current model got right."},"both_right":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"both_wrong":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"disagreements":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Compared calls where the two gave different answers."},"unresolved":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Disagreements without a right answer yet. Calls where the two agree don't need one to compare them: both are right, or both wrong."},"difference":{"type":"object","properties":{"estimate":{"type":"number"},"low":{"type":"number"},"high":{"type":"number"}},"required":["estimate","low","high"],"additionalProperties":false,"description":"Ours minus theirs, as a share of calls, with a 95% interval for paired results (Agresti–Min). Disagreements with a right answer stand in for those without one."},"p_value":{"type":"number","description":"Exact McNemar test on the calls only one model got right: how likely a split this uneven is if they were equally good."},"agreement":{"type":"number","description":"Share of compared calls where the two gave the same answer."},"slices":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string"},"kind":{"type":"string","enum":["label","field"]},"support":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Graded calls whose right answer is this label, or that grade this field."},"ours":{"type":"number"},"theirs":{"type":"number"},"behind":{"type":"boolean","description":"Ours trails by more than the margin, on enough calls to tell."}},"required":["name","kind","support","ours","theirs","behind"],"additionalProperties":false}},"bar":{"type":"object","properties":{"rule":{"type":"string","enum":["at_least_as_good","better"],"description":"at_least_as_good: not behind the current model by more than `margin`, with 95% confidence. better: ahead of it, with 95% confidence."},"margin":{"type":"number","minimum":0,"maximum":0.2,"description":"How far behind still counts as as good, as a share of calls (0.02 = 2 points). No label or field may trail by more than this either."}},"required":["rule","margin"],"additionalProperties":false},"verdict":{"type":"string","enum":["clears_bar","slice_behind","inconclusive","behind"],"description":"clears_bar: ready to swap. slice_behind: clears it overall, but a label or field trails. inconclusive: not enough evidence yet. behind: clearly worse."},"summary":{"type":"string","description":"The result and verdict in plain language."}},"required":["incumbent","compared","graded","ours","theirs","ours_only","theirs_only","both_right","both_wrong","disagreements","unresolved","difference","p_value","agreement","slices","bar","verdict","summary"],"additionalProperties":false,"description":"Our model against the one in use today, on the same calls (held-out in the backtest, live in shadow mode), graded on what really happened."},{"type":"null"}]}},"required":["test_set_size","metrics","per_class","confusion","baselines","requirement_checks","error_examples","notes","warnings","head_to_head"],"additionalProperties":false},{"type":"null"}]},"artifacts":{"type":"array","items":{"type":"object","properties":{"format":{"type":"string","enum":["onnx","safetensors","gguf","docker","hosted_endpoint"]},"filename":{"type":"string"},"size_mb":{"type":"number"},"sha256":{"type":"string"},"status":{"type":"string","enum":["ready","building","unavailable"]}},"required":["format","filename","size_mb","sha256","status"],"additionalProperties":false}},"illustrative":{"type":"boolean","description":"True when metrics come from a simulation or sample data."},"sample":{"type":"boolean","description":"A pre-built example model provided for exploration, not created by you."}},"required":["id","name","status","mode","version","created_at","updated_at","latest_job_id","task","dataset","requirements","base_model","recipe","evaluation","artifacts","illustrative","sample"],"additionalProperties":false},"ModelSummary":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"status":{"type":"string","enum":["training","ready","failed","canceled"]},"mode":{"type":"string","enum":["simulated","training"],"description":"`simulated` replays a scripted run with illustrative results and costs nothing. `training` trains a real model."},"version":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"created_at":{"type":"string"},"updated_at":{"type":"string"},"latest_job_id":{"type":"string"},"base_model":{"type":["string","null"]},"recipe":{"default":null,"description":"How the model was built and what it calls; null for older demo models.","anyOf":[{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,47}$","description":"How a model was built, e.g. `encoder_head` (a small encoder with a head trained on your examples) or `prompt` (a general-purpose LLM shown your most similar examples, with no training)."},"name":{"type":"string","description":"The recipe in plain language."},"third_party":{"description":"The third-party model it calls for every answer, and through whom, e.g. \"Claude Haiku 5.5 via Anthropic's API\". Null when it runs on our servers.","type":["string","null"]},"confidence":{"type":"boolean","description":"Whether its answers come with a confidence. The rollout's handback threshold needs one."}},"required":["id","name","third_party","confidence"],"additionalProperties":false},{"type":"null"}]},"illustrative":{"type":"boolean","description":"True when metrics come from a simulation or sample data."},"sample":{"type":"boolean","description":"A pre-built example model provided for exploration, not created by you."},"template_id":{"type":"string","enum":["support_routing","feedback_classification","invoice_extraction","custom"],"description":"Starting template. `custom` means the task was described from scratch."},"output_kind":{"type":"string","enum":["single_label","multi_label","extraction"],"description":"single_label: pick exactly one category. multi_label: pick any number of tags. extraction: return named fields."},"label_count":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"primary_metric":{"anyOf":[{"type":"string","enum":["accuracy","f1","field_accuracy"]},{"type":"null"}]},"primary_value":{"type":["number","null"]},"latency_ms_p95":{"type":["number","null"]},"job_progress":{"type":["number","null"]}},"required":["id","name","status","mode","version","created_at","updated_at","latest_job_id","base_model","recipe","illustrative","sample","template_id","output_kind","label_count","primary_metric","primary_value","latency_ms_p95","job_progress"],"additionalProperties":false},"PredictRequest":{"anyOf":[{"type":"object","properties":{"input":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string","maxLength":20000},"description":"Values keyed by the task's input names, as they are: the model cleans them the way its examples were cleaned."}},"required":["input"]},{"type":"object","properties":{"text":{"type":"string","minLength":1,"maxLength":20000,"description":"Shorthand for a task with one text input."}},"required":["text"]}],"description":"What to run the model on: `input` by name, or `text` when the task reads one text input."},"Prediction":{"type":"object","properties":{"model_id":{"type":"string"},"output_kind":{"type":"string","enum":["single_label","multi_label","extraction"],"description":"single_label: pick exactly one category. multi_label: pick any number of tags. extraction: return named fields."},"label":{"type":["string","null"]},"labels":{"type":"array","items":{"type":"string"}},"fields":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":["string","null"]}},"metadata":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string"},"description":"single_label: the predicted label's metadata from the task. Empty otherwise."},"scores":{"type":"array","items":{"type":"object","properties":{"label":{"type":"string"},"score":{"type":"number"}},"required":["label","score"],"additionalProperties":false}},"latency_ms":{"type":"number"},"illustrative":{"type":"boolean"}},"required":["model_id","output_kind","label","labels","fields","metadata","scores","latency_ms","illustrative"],"additionalProperties":false},"ExportRequest":{"type":"object","properties":{"format":{"type":"string","enum":["onnx","safetensors","gguf","docker","hosted_endpoint"]}},"required":["format"]},"ExportResult":{"type":"object","properties":{"artifact":{"type":"object","properties":{"format":{"type":"string","enum":["onnx","safetensors","gguf","docker","hosted_endpoint"]},"filename":{"type":"string"},"size_mb":{"type":"number"},"sha256":{"type":"string"},"status":{"type":"string","enum":["ready","building","unavailable"]}},"required":["format","filename","size_mb","sha256","status"],"additionalProperties":false},"download_url":{"description":"Short-lived signed URL. Null for hosted endpoints.","type":["string","null"]},"expires_at":{"type":["string","null"]},"demo_payload":{"anyOf":[{"type":"object","properties":{"filename":{"type":"string"},"mime":{"type":"string"},"content":{"type":"string"}},"required":["filename","mime","content"],"additionalProperties":false},{"type":"null"}]}},"required":["artifact","download_url","expires_at","demo_payload"],"additionalProperties":false},"AccessRequest":{"type":"object","properties":{"email":{"type":"string","format":"email","pattern":"^(?:[A-Za-z0-9_'+\\-]+\\.)*[A-Za-z0-9_'+\\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$"},"use_case":{"type":"string","maxLength":500},"attribution":{"type":"object","properties":{"utm_source":{"type":"string","maxLength":200},"utm_medium":{"type":"string","maxLength":200},"utm_campaign":{"type":"string","maxLength":200},"utm_content":{"type":"string","maxLength":200},"utm_term":{"type":"string","maxLength":200},"landing_path":{"description":"Path and query of the first page this visit landed on.","type":"string","maxLength":500}},"description":"Where the visitor first arrived from this session, so a request can be traced to the campaign behind it."}},"required":["email"]},"AccessReceipt":{"type":"object","properties":{"status":{"type":"string","const":"received"},"position":{"anyOf":[{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},{"type":"null"}],"description":"Place in the early-access queue, when the service tracks one."}},"required":["status","position"],"additionalProperties":false},"Workspace":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"created_at":{"type":"string"},"llm_comparison_available":{"type":"boolean","description":"Whether the service can score general-purpose LLMs (an LLM API key is configured)."},"third_party_models":{"default":false,"description":"Whether models that call a third-party model (Claude with your examples) can be trained and deployed here, when a task allows them.","type":"boolean"},"agents":{"default":false,"description":"Whether agents can work on projects in this workspace.","type":"boolean"}},"required":["id","name","created_at","llm_comparison_available","third_party_models","agents"],"additionalProperties":false},"ShadowSettings":{"type":"object","properties":{"incumbent":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"What answers today, e.g. \"Claude Opus 5.5 with our prompt\", \"keyword rules\" or \"the support team\"."},"cost_usd_per_1k":{"description":"Its cost per 1,000 calls, if known. Shown as reported.","type":"number","minimum":0,"maximum":100000},"latency_ms_p95":{"description":"Its p95 latency in ms, if known. Shown as reported.","type":"number","minimum":0,"maximum":600000}},"required":["name"],"description":"What answers today. Defaults to what the backtest named."},"swap_bar":{"type":"object","properties":{"rule":{"type":"string","enum":["at_least_as_good","better"],"description":"at_least_as_good: not behind the current model by more than `margin`, with 95% confidence. better: ahead of it, with 95% confidence."},"margin":{"type":"number","minimum":0,"maximum":0.2,"description":"How far behind still counts as as good, as a share of calls (0.02 = 2 points). No label or field may trail by more than this either."}},"required":["rule","margin"],"description":"What ours must show to be ready to replace it. Defaults to the bar the model was trained against."}},"required":["incumbent","swap_bar"]},"Shadow":{"type":"object","properties":{"incumbent":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"What answers today, e.g. \"Claude Opus 5.5 with our prompt\", \"keyword rules\" or \"the support team\"."},"cost_usd_per_1k":{"description":"Its cost per 1,000 calls, if known. Shown as reported.","type":"number","minimum":0,"maximum":100000},"latency_ms_p95":{"description":"Its p95 latency in ms, if known. Shown as reported.","type":"number","minimum":0,"maximum":600000}},"required":["name"],"additionalProperties":false,"description":"What answers today. Defaults to what the backtest named."},"swap_bar":{"type":"object","properties":{"rule":{"type":"string","enum":["at_least_as_good","better"],"description":"at_least_as_good: not behind the current model by more than `margin`, with 95% confidence. better: ahead of it, with 95% confidence."},"margin":{"type":"number","minimum":0,"maximum":0.2,"description":"How far behind still counts as as good, as a share of calls (0.02 = 2 points). No label or field may trail by more than this either."}},"required":["rule","margin"],"additionalProperties":false,"description":"What ours must show to be ready to replace it. Defaults to the bar the model was trained against."},"model_id":{"type":"string"},"calls":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls recorded."},"answered":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls our model has answered."},"with_outcome":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls with a reported outcome."},"reviewed":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls a reviewer answered."},"to_review":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls where the two disagree and nobody has reviewed them yet."},"first_call_at":{"type":["string","null"]},"last_call_at":{"type":["string","null"]},"head_to_head":{"anyOf":[{"type":"object","properties":{"incumbent":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"What answers today, e.g. \"Claude Opus 5.5 with our prompt\", \"keyword rules\" or \"the support team\"."},"cost_usd_per_1k":{"description":"Its cost per 1,000 calls, if known. Shown as reported.","type":"number","minimum":0,"maximum":100000},"latency_ms_p95":{"description":"Its p95 latency in ms, if known. Shown as reported.","type":"number","minimum":0,"maximum":600000}},"required":["name"],"additionalProperties":false},"compared":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls both models answered. In the backtest, every one has a right answer."},"graded":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Compared calls with a known right answer."},"ours":{"description":"Share of calls our model got exactly right: measured when every compared call has a right answer, otherwise estimated from those that do. Null until it can be estimated.","type":["number","null"]},"theirs":{"description":"The same for the current model.","type":["number","null"]},"ours_only":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Graded calls only our model got right."},"theirs_only":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Graded calls only the current model got right."},"both_right":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"both_wrong":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"disagreements":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Compared calls where the two gave different answers."},"unresolved":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Disagreements without a right answer yet. Calls where the two agree don't need one to compare them: both are right, or both wrong."},"difference":{"type":"object","properties":{"estimate":{"type":"number"},"low":{"type":"number"},"high":{"type":"number"}},"required":["estimate","low","high"],"additionalProperties":false,"description":"Ours minus theirs, as a share of calls, with a 95% interval for paired results (Agresti–Min). Disagreements with a right answer stand in for those without one."},"p_value":{"type":"number","description":"Exact McNemar test on the calls only one model got right: how likely a split this uneven is if they were equally good."},"agreement":{"type":"number","description":"Share of compared calls where the two gave the same answer."},"slices":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string"},"kind":{"type":"string","enum":["label","field"]},"support":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Graded calls whose right answer is this label, or that grade this field."},"ours":{"type":"number"},"theirs":{"type":"number"},"behind":{"type":"boolean","description":"Ours trails by more than the margin, on enough calls to tell."}},"required":["name","kind","support","ours","theirs","behind"],"additionalProperties":false}},"bar":{"type":"object","properties":{"rule":{"type":"string","enum":["at_least_as_good","better"],"description":"at_least_as_good: not behind the current model by more than `margin`, with 95% confidence. better: ahead of it, with 95% confidence."},"margin":{"type":"number","minimum":0,"maximum":0.2,"description":"How far behind still counts as as good, as a share of calls (0.02 = 2 points). No label or field may trail by more than this either."}},"required":["rule","margin"],"additionalProperties":false},"verdict":{"type":"string","enum":["clears_bar","slice_behind","inconclusive","behind"],"description":"clears_bar: ready to swap. slice_behind: clears it overall, but a label or field trails. inconclusive: not enough evidence yet. behind: clearly worse."},"summary":{"type":"string","description":"The result and verdict in plain language."}},"required":["incumbent","compared","graded","ours","theirs","ours_only","theirs_only","both_right","both_wrong","disagreements","unresolved","difference","p_value","agreement","slices","bar","verdict","summary"],"additionalProperties":false,"description":"Our model against the one in use today, on the same calls (held-out in the backtest, live in shadow mode), graded on what really happened."},{"type":"null"}],"description":"Over every call both models answered that this model would keep (at least as sure as the rollout's handback threshold). Null until there's one."},"notes":{"type":"array","items":{"type":"string"},"description":"How the comparison is measured, in plain language."},"rollout":{"type":"object","properties":{"share":{"type":"number","minimum":0,"maximum":1,"description":"Share of calls through POST …/answer this model answers; the model in use today answers the rest. 0 keeps it in the shadow; 1 is a full swap."},"min_confidence":{"type":"number","minimum":0,"maximum":0.99,"description":"Calls this model is less sure of than this go to the model in use today, whatever the share. 0 hands back none."},"guard":{"type":"boolean","description":"Swap back on its own if the comparison shows this model clearly behind."},"updated_at":{"type":["string","null"]},"history":{"type":"array","items":{"type":"object","properties":{"share":{"type":"number","minimum":0,"maximum":1,"description":"Share of calls through POST …/answer this model answers; the model in use today answers the rest. 0 keeps it in the shadow; 1 is a full swap."},"min_confidence":{"type":"number","minimum":0,"maximum":0.99,"description":"Calls this model is less sure of than this go to the model in use today, whatever the share. 0 hands back none."},"guard":{"type":"boolean","description":"Swap back on its own if the comparison shows this model clearly behind."},"at":{"type":"string"},"by":{"type":"string","enum":["you","guard"]},"note":{"type":"string","description":"What changed, in plain language."}},"required":["share","min_confidence","guard","at","by","note"],"additionalProperties":false},"description":"Changes, newest first (the last 20)."}},"required":["share","min_confidence","guard","updated_at","history"],"additionalProperties":false},"routing":{"type":"object","properties":{"shadow":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls the model in use today answered while this one shadowed it."},"model":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls this model answered (through POST …/answer)."},"incumbent":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls the rollout's share sent to the model in use today."},"handed_back":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls handed back because this model was less sure than the threshold."},"uncompared":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Calls without the current model's answer yet (answered through POST …/answer), so they aren't compared."}},"required":["shadow","model","incumbent","handed_back","uncompared"],"additionalProperties":false,"description":"Who answered the calls recorded so far."},"handback":{"type":"array","items":{"type":"object","properties":{"min_confidence":{"type":"number"},"handed_back":{"type":"number","description":"Share of answered calls this model is less sure of, which would go to the model in use today."},"verdict":{"anyOf":[{"type":"string","enum":["clears_bar","slice_behind","inconclusive","behind"],"description":"clears_bar: ready to swap. slice_behind: clears it overall, but a label or field trails. inconclusive: not enough evidence yet. behind: clearly worse."},{"type":"null"}],"description":"The comparison on the calls it would keep; null without any."},"difference":{"description":"Ours minus theirs on the calls it would keep.","type":["number","null"]}},"required":["min_confidence","handed_back","verdict","difference"],"additionalProperties":false},"description":"What each handback threshold would hand back, and the comparison on the calls kept."}},"required":["incumbent","swap_bar","model_id","calls","answered","with_outcome","reviewed","to_review","first_call_at","last_call_at","head_to_head","notes","rollout","routing","handback"],"additionalProperties":false,"description":"Shadow mode for one model: live calls, with the answers of the model in use today, compared as outcomes and reviews arrive, and the rollout that decides which one answers."},"RecordShadowCalls":{"type":"object","properties":{"calls":{"minItems":1,"maxItems":100,"type":"array","items":{"type":"object","properties":{"id":{"type":"string","minLength":1,"maxLength":200,"description":"Your id for the call, such as a ticket or request id. Outcomes are attached by it, and a repeated id is ignored, so retries are safe; a call that came through POST /v1/models/{model_id}/answer gets the current model's answer filled in instead."},"input":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string","maxLength":20000},"description":"Input values keyed by the task's input names. A text input takes a string."},"incumbent":{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"description":"What the model in use today answered."},"outcome":{"description":"The right answer, if it's already known. Usually sent later, with POST /v1/models/{model_id}/shadow/outcomes.","type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}}},"at":{"description":"When the call happened (ISO 8601). Defaults to when it reached us.","type":"string","maxLength":40}},"required":["id","input","incumbent"]},"description":"Up to 100 calls: send each as it happens, or several at once."}},"required":["calls"]},"ShadowReceipt":{"type":"object","properties":{"recorded":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"New calls stored. Ours answers each within seconds, off your request path."},"duplicates":{"type":"array","items":{"type":"string"},"description":"Ids this model already has; ignored."}},"required":["recorded","duplicates"],"additionalProperties":false},"RecordShadowOutcomes":{"type":"object","properties":{"outcomes":{"minItems":1,"maxItems":500,"type":"array","items":{"type":"object","properties":{"id":{"type":"string","minLength":1,"maxLength":200},"output":{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"description":"The right answer, as it turned out."}},"required":["id","output"]},"description":"Up to 500 outcomes. Sending one again replaces it."}},"required":["outcomes"]},"ShadowOutcomesReceipt":{"type":"object","properties":{"updated":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"unknown":{"type":"array","items":{"type":"string"},"description":"Ids with no recorded call; record the call first."}},"required":["updated","unknown"],"additionalProperties":false},"ShadowReview":{"type":"object","properties":{"output":{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"description":"The right answer, as a person judged it."}},"required":["output"]},"ShadowCall":{"type":"object","properties":{"id":{"type":"string"},"input":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string","maxLength":20000},"description":"As stored: emails and long numbers masked. Lists shorten long values."},"incumbent":{"anyOf":[{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false,"description":"An answer, shaped by the task's output_kind. Examples, predictions, other models' answers and outcomes all use this shape."},{"type":"null"}],"description":"What the model in use today answered; null for a call this model answered until you record that."},"ours":{"anyOf":[{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false,"description":"An answer, shaped by the task's output_kind. Examples, predictions, other models' answers and outcomes all use this shape."},{"type":"null"}],"description":"Our model's answer; null for the few seconds until it answers."},"confidence":{"type":["number","null"]},"agree":{"description":"Whether the two gave the same answer; null until both have answered.","type":["boolean","null"]},"outcome":{"anyOf":[{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false,"description":"An answer, shaped by the task's output_kind. Examples, predictions, other models' answers and outcomes all use this shape."},{"type":"null"}],"description":"The right answer you reported."},"review":{"anyOf":[{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false,"description":"An answer, shaped by the task's output_kind. Examples, predictions, other models' answers and outcomes all use this shape."},{"type":"null"}],"description":"A reviewer's answer. It takes precedence over the reported outcome when grading."},"served":{"type":"string","enum":["model","incumbent"],"description":"Whose answer the call got: this model's, or the model in use today's."},"reason":{"type":"string","enum":["shadow","rollout","low_confidence"],"description":"shadow: the model in use today answered while this one only shadowed it. rollout: the rollout's share decided. low_confidence: this model was less sure than the rollout's handback threshold, so the model in use today answered."},"at":{"type":"string"},"received_at":{"type":"string"}},"required":["id","input","incumbent","ours","confidence","agree","outcome","review","served","reason","at","received_at"],"additionalProperties":false},"AnswerRequest":{"type":"object","properties":{"id":{"type":"string","minLength":1,"maxLength":200,"description":"Your id for the call. The same id always gets the same answer, and outcomes and the current model's answer attach by it."},"input":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string","maxLength":20000},"description":"Input values keyed by the task's input names. A text input takes a string."},"at":{"description":"When the call happened (ISO 8601). Defaults to now.","type":"string","maxLength":40}},"required":["id","input"]},"Answer":{"type":"object","properties":{"id":{"type":"string"},"use":{"type":"string","enum":["model","incumbent"],"description":"model: answer with `output`. incumbent: answer with the model you use today, then record its answer with POST /v1/models/{model_id}/shadow/calls so the comparison continues."},"output":{"anyOf":[{"type":"object","properties":{"label":{"description":"single_label: the one category.","anyOf":[{"type":"string","maxLength":200},{"type":"null"}]},"labels":{"description":"multi_label: every tag that applies; empty when none do.","maxItems":50,"type":"array","items":{"type":"string","maxLength":200}},"fields":{"description":"extraction: each field's value by name; null when the input doesn't have it.","type":"object","propertyNames":{"type":"string"},"additionalProperties":{"anyOf":[{"type":"string","maxLength":2000},{"type":"null"}]}}},"additionalProperties":false,"description":"An answer, shaped by the task's output_kind. Examples, predictions, other models' answers and outcomes all use this shape."},{"type":"null"}],"description":"This model's answer, when `use` is `model`."},"metadata":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":"string"},"description":"single_label: the answer's label metadata, when `use` is `model`."},"confidence":{"description":"How sure this model is of its answer, 0–1; null for models that don't report one.","type":["number","null"]},"reason":{"type":"string","enum":["shadow","rollout","low_confidence"],"description":"shadow: the model in use today answered while this one only shadowed it. rollout: the rollout's share decided. low_confidence: this model was less sure than the rollout's handback threshold, so the model in use today answered."}},"required":["id","use","output","metadata","confidence","reason"],"additionalProperties":false,"description":"Which model answers a live call. Decided by the rollout, the same way every time for the same id."},"RolloutSettings":{"type":"object","properties":{"share":{"type":"number","minimum":0,"maximum":1,"description":"Share of calls through POST …/answer this model answers; the model in use today answers the rest. 0 keeps it in the shadow; 1 is a full swap."},"min_confidence":{"type":"number","minimum":0,"maximum":0.99,"description":"Calls this model is less sure of than this go to the model in use today, whatever the share. 0 hands back none."},"guard":{"type":"boolean","description":"Swap back on its own if the comparison shows this model clearly behind."}},"required":["share","min_confidence","guard"]},"Error":{"type":"object","properties":{"error":{"type":"object","properties":{"code":{"type":"string","enum":["validation_failed","unauthorized","forbidden_scope","spend_limit_exceeded","approval_required","not_found","idempotency_conflict","conflict","rate_limited","model_unavailable","server_error"]},"message":{"type":"string"},"details":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{}}},"required":["code","message"],"additionalProperties":false}},"required":["error"],"additionalProperties":false}}}}