From 66ed78c0a2b613292010632ce9009bb5ef9b02f6 Mon Sep 17 00:00:00 2001 From: Xingkai98 Date: Tue, 10 Nov 2026 21:29:59 +0800 Subject: [PATCH 1/5] feat(contract): stage/route capability mapping table --- docs/README.md | 1 + docs/architecture/capability-contract.md | 101 ++++++++ .../domain/capability_contract.py | 236 ++++++++++++++++++ tests/fixtures/omni-capabilities/absent.json | 90 +++++++ tests/fixtures/omni-capabilities/full.json | 142 +++++++++++ tests/fixtures/omni-capabilities/partial.json | 154 ++++++++++++ tests/test_capability_contract.py | 207 +++++++++++++++ 7 files changed, 931 insertions(+) create mode 100644 docs/architecture/capability-contract.md create mode 100644 packages/core/src/opendirector/domain/capability_contract.py create mode 100644 tests/fixtures/omni-capabilities/absent.json create mode 100644 tests/fixtures/omni-capabilities/full.json create mode 100644 tests/fixtures/omni-capabilities/partial.json create mode 100644 tests/test_capability_contract.py diff --git a/docs/README.md b/docs/README.md index 536a82f3..0a24bfc0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ This directory is the entry point for the repository's shared documentation. The | [Contributing](../CONTRIBUTING.md) | Label taxonomy, how to pick labels, and what a PR must satisfy | Process, labels, or submission requirements change | | [Unified project architecture](architecture/unified-project.md) | Business objects, shared entry points, and interaction rules | Object responsibilities, commands, persistence, or interaction rules change | | [Test contracts](architecture/test-contracts.md) | Test/production boundaries, how to avoid stale assertions and environment-dependent red runs | Test infrastructure, capability gates, or contract ownership change | +| [Capability mapping contract / 能力映射表契约](architecture/capability-contract.md) | `(stage, route) → capability` table, the three degradation states, and how it feeds discovery | Stage/route vocabulary, capability ids, or discovery matching change | | [Roadmap](development/roadmap.md) | Capability dependencies and delivery criteria | Scope, dependencies, or acceptance criteria change | | [Validation guide](development/validation.md) | Reproducible checks and evidence requirements | Tools, runtime requirements, or validation entry points change | | [Issue #87 Windows handoff](development/issue-87-windows-handoff.md) ([中文](development/issue-87-windows-handoff.zh-CN.md)) | Native ACL repair and final text acceptance | The shared checkpoint or native validation scope changes | diff --git a/docs/architecture/capability-contract.md b/docs/architecture/capability-contract.md new file mode 100644 index 00000000..c77590ad --- /dev/null +++ b/docs/architecture/capability-contract.md @@ -0,0 +1,101 @@ +# Capability mapping contract / 能力映射表契约 + +## 中文 + +`GET /v1/omni/capabilities` 只说明部署「自述了什么」；产品要回答的是「这个部署能不能做某件事」。`packages/core/src/opendirector/domain/capability_contract.py` 是唯一把上游 stage 名与路由翻译成 `opendirector.domain.capabilities` id 的地方。它不发起 HTTP，不持有界面文案，也不创建第二套 operation 常量。 + +### 键必须是 (stage, route) 成对 + +一个路径可以同时服务两种能力。`/v1/audio/speech` 既是 TTS 也是音乐生成，`/v1/audio/transcriptions` 既是扒谱（`score`）也是语音识别（`asr`）。只按路径匹配，会让「只声明了 TTS」的服务看起来能生成音乐，反过来也一样；部署级 `exposure` 若按 `(method, path)` 单键下发，关闭语音合成会连带关闭音乐。因此 `STAGE_ROUTES` 的键是成对的 `(upstream stage, upstream route)`，落表后经 `capabilities_for(stage, route)` 查找；`/v1/chat/completions` 这类独占路径同样按对匹配，不为单键开后门。 + +`asr` 目前**不在表里**：能力词表（`domain/capabilities.py`）还没有识别能力 id，把 `/v1/audio/transcriptions` 在 `asr` 下读成 `audio.score` 是错的。它会被如实报成 `unmapped_stage`，等词表有了 `audio.transcribe` 再补一行。 + +### 三种降级状态，含义不同 + +| 状态 | 含义 | 修的地方 | +| --- | --- | --- | +| `unknown_route` | 路径本身不在表里（例如 `/v1/videos/track`、`/v1/embeddings`） | 表或适配器缺这条路径 | +| `unmapped_stage` | 路径认识，stage 不认识（例如 `asr` 打 `/v1/audio/transcriptions`，或 `stage_ids` 是数字 id） | 表缺这个 stage，或上游 stage 名没对齐 | +| `stage_route_mismatch` | stage 与路径都认识，但这一对不成立（例如 `image` 声明 `/v1/videos`） | 部署声明的配对写错了 | + +三者都进 `unresolved`，不被丢弃 —— 调用方可以解释「为什么这个 stage 不可用」，而不是静默隐藏。 + +能力状态另有三档：`available`（有成对声明）、`missing`（部署答了，但没有这一对）、`unknown`（没有足够信息）。请求失败、404、没有 `stages`、空列表、stage 没有名字，全部返回 `endpoint=absent` 与全 `unknown`：**「服务没说」不是「服务不能」**。另有四个能力（`video.first_frame`、`video.edit`、`image.cutout`、`audio.separate`）根本无法由路由表达，它们永远是 `unknown`，并在 `UNROUTED` 里写明原因。 + +### 与 V2 发现契约（#266）的关系 + +这份表是 #266 的**输入**，不是替代品，也不是第二套 schema： + +- #266 的 `domain/omni_capabilities.py` 定义发现文档与匹配；本模块只提供 `(stage, route) → capability` 这一层。两者文件名不同，互不覆盖。 +- #266 的 `Model.stage_ids` 目前没有任何读取方（只有测试夹具填 `['0']`）。两条路：(a) 删掉该字段；(b) 接上 —— 让 `stage_ids` 与每个 operation profile 的 `routes` 交叉出成对声明。本模块的 `declarations()` 已经能读 `models[].stage_ids` × `operations.*.profiles[].routes[]` 这种形状，供 (b) 直接使用；若 `stage_ids` 里是数字 id，结果会如实报 `unmapped_stage`，不会被猜成某个 stage。 +- 本模块不改 `domain/omni_capabilities.py`、`integrations/capability_probe.py`、`application/model_availability.py`。 + +### 如何作为 `requests_for()` 的输入 + +`application/model_availability.py` 现在按 `(kind, protocol)` 硬编码 profile 名，ASR/扒谱因此拿到空列表。表驱动后，调用方只需要自己正在执行的 stage 与将要 POST 的路径： + +```python +from opendirector.domain.capability_contract import capabilities_for + +capabilities = capabilities_for('music', '/v1/audio/speech') # ('audio.music',) +``` + +`resolve(payload)` 返回逐能力状态、逐 stage 状态与 `unresolved` 列表；`classified_capabilities()` 按表顺序给出词表，展示层据此排列，不再自己排序。是否接进 `requests_for()` 由 #266 一侧决定，本契约只保证查找是成对的、稳定的、可测的。 + +### 三档 fixture 与验证 + +`tests/fixtures/omni-capabilities/{full,partial,absent}.json` 是共享样本：`full` 覆盖表里的全部配对与两条共享路径，`partial` 覆盖混合可用、`stage_route_mismatch` 与两种未知，`absent` 覆盖 404、缺字段、空列表。`tests/test_capability_contract.py` 逐档比对期望块，并单独覆盖共享路径不互相覆盖、`asr` 不等于 `score`、`absent` 全 `unknown`、表顺序即输出顺序、`models[]` 文档形状。 + +```bash +PYTHONPATH=packages/core/src python -m pytest tests/test_capability_contract.py -q +``` + +## English + +`GET /v1/omni/capabilities` reports what a deployment says about itself; the product needs to know what that deployment can do. `packages/core/src/opendirector/domain/capability_contract.py` is the only place that translates upstream stage names and routes into `opendirector.domain.capabilities` ids. It performs no HTTP, holds no display text, and introduces no second set of operation constants. + +### The key is the pair + +One path can serve two capabilities. `/v1/audio/speech` carries both speech synthesis and music generation; `/v1/audio/transcriptions` carries both score transcription (`score`) and recognition (`asr`). A path-only key makes a TTS-only deployment look able to generate music, and the reverse; a deployment-level `exposure` published per `(method, path)` disables music when it disables speech. `STAGE_ROUTES` is therefore keyed by the pair `(upstream stage, upstream route)` and read through `capabilities_for(stage, route)`. Paths that happen to be exclusive, such as `/v1/chat/completions`, are matched as pairs too, so there is no second rule to keep in sync. + +`asr` is deliberately absent from the table: the vocabulary in `domain/capabilities.py` has no recognition id yet, and reading `/v1/audio/transcriptions` under `asr` as `audio.score` would be wrong. It is reported as `unmapped_stage` until an `audio.transcribe` capability exists. + +### Three degradation states, three different fixes + +| State | Meaning | What to fix | +| --- | --- | --- | +| `unknown_route` | The path is not in the table (`/v1/videos/track`, `/v1/embeddings`) | The table or an adapter is missing the path | +| `unmapped_stage` | The path is known, the stage is not (`asr` on `/v1/audio/transcriptions`, or a numeric `stage_ids` entry) | The table is missing the stage, or the upstream stage name is not aligned | +| `stage_route_mismatch` | Both are known, but not as this pair (`image` declaring `/v1/videos`) | The deployment declared a wrong pairing | + +All three land in `unresolved` instead of being dropped, so a caller can explain why a stage is unavailable rather than hiding it. + +Capability states are separate: `available` (a matching pair was declared), `missing` (the deployment answered without that pair), `unknown` (not enough information). A failed request, a 404, a body without `stages`, an empty list, or an entry without a stage all return `endpoint=absent` with everything `unknown`: **"the service did not say" is not "the service cannot"**. Four capabilities (`video.first_frame`, `video.edit`, `image.cutout`, `audio.separate`) cannot be expressed by any route; they stay `unknown` forever and `UNROUTED` states why. + +### Relationship to the V2 discovery contract (#266) + +This table is an input to #266, not a replacement and not a second schema: + +- `domain/omni_capabilities.py` in #266 owns the discovery document and matching; this module owns only the `(stage, route) → capability` layer. Different file names, no overwrite. +- `Model.stage_ids` in #266 currently has no reader (tests only fill `['0']`). Either (a) delete the field, or (b) connect it: cross `stage_ids` with the routes of each operation profile to produce paired declarations. `declarations()` already reads the `models[].stage_ids` × `operations.*.profiles[].routes[]` shape, so (b) can use it as is; numeric ids surface as `unmapped_stage` instead of being guessed at. +- This module does not modify `domain/omni_capabilities.py`, `integrations/capability_probe.py`, or `application/model_availability.py`. + +### Feeding `requests_for()` + +`application/model_availability.py` currently hard-codes profile names per `(kind, protocol)`, which is why ASR and score get an empty request list. With a table, the caller needs only the stage it executes and the path it posts to: + +```python +from opendirector.domain.capability_contract import capabilities_for + +capabilities = capabilities_for('music', '/v1/audio/speech') # ('audio.music',) +``` + +`resolve(payload)` returns per-capability states, per-stage states and the `unresolved` list; `classified_capabilities()` returns the vocabulary in table order, so presentation stops sorting on its own. Whether to wire this into `requests_for()` is #266's decision; this contract only guarantees the lookup is paired, ordered and testable. + +### Fixtures and checks + +`tests/fixtures/omni-capabilities/{full,partial,absent}.json` are the shared samples: `full` covers every pair in the table and both shared paths, `partial` covers mixed availability, a `stage_route_mismatch` and both unknown kinds, `absent` covers a 404, a missing field and an empty list. `tests/test_capability_contract.py` compares each tier against its expected block and separately covers shared paths not overwriting each other, `asr` not being `score`, `absent` being all `unknown`, table order as output order, and the `models[]` document shape. + +```bash +PYTHONPATH=packages/core/src python -m pytest tests/test_capability_contract.py -q +``` diff --git a/packages/core/src/opendirector/domain/capability_contract.py b/packages/core/src/opendirector/domain/capability_contract.py new file mode 100644 index 00000000..0d4a2a75 --- /dev/null +++ b/packages/core/src/opendirector/domain/capability_contract.py @@ -0,0 +1,236 @@ +"""The (stage, route) pairs the capability vocabulary understands. + +`GET /v1/omni/capabilities` (vllm-project/vllm-omni#10007) reports what a +deployment serves. This module is the one place that turns those names into +`opendirector.domain.capabilities` ids. It performs no HTTP and holds no +display text: `application.model_availability.requests_for()` consumes the +table, and the presentation layer renders the reason codes through the message +catalogs. + +Three rules keep the table honest: + +* The key is the pair, never the route alone. `/v1/audio/speech` carries both + `tts` and `music`; `/v1/audio/transcriptions` carries both `asr` and `score`. + A declared path without its stage cannot say which capability was meant. +* A capability no route can express stays `unknown` even when the deployment + answers, because it is a request parameter rather than an endpoint: + `video.first_frame` and `video.edit` are `/v1/videos` parameters, and + `image.cutout` is a `/v1/images/generations` parameter. +* A declaration that places no stage is `unknown`, never `missing`: "the + service did not say" is not "the service cannot". + +A declared pair this table cannot translate is reported instead of dropped, so +a caller can show why a stage is unavailable rather than silently hiding it. +""" + +ENDPOINT = '/v1/omni/capabilities' +API_PREFIX = '/v1' + +# (upstream stage, upstream route, capability ids that pair can serve). Order is +# part of the contract: `resolve()` emits capabilities in this order, so a +# presentation change is a diff in one place. The routes are the ones this +# repository's adapters already call, so a declaration published by the service +# and a request we would send name the same path. +STAGE_ROUTES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( + ('text', '/v1/chat/completions', ('text.chat',)), + ('image', '/v1/images/generations', ('image.generate',)), + ('image', '/v1/images/edits', ('image.edit',)), + ('image', '/v1/images/segmentations', ('image.segment',)), + ('video', '/v1/videos', ('video.generate',)), + ('video', '/v1/videos/segmentations', ('video.matting',)), + ('video', '/v1/videos/depth', ('video.motion_capture',)), + ('tts', '/v1/audio/speech', ('audio.speech',)), + ('music', '/v1/audio/speech', ('audio.music',)), + ('score', '/v1/audio/transcriptions', ('audio.score',)), +) + +# `asr` is deliberately absent: recognition has no capability id yet, so +# `/v1/audio/transcriptions` under `asr` is reported as `unmapped_stage` +# instead of being read as `audio.score`. + +# Capability ids no upstream route can express, with the reason to show. Values +# stay stable business codes; wording lives in the message catalogs. +UNROUTED: dict[str, str] = { + 'video.first_frame': 'parameter_only', + 'video.edit': 'parameter_only', + 'image.cutout': 'parameter_only', + 'audio.separate': 'no_local_adapter', +} + +AVAILABLE, MISSING, UNKNOWN = 'available', 'missing', 'unknown' +ABSENT, PRESENT = 'absent', 'present' +PARTIAL, UNMAPPED = 'partial', 'unmapped' + +# Why a declared pair could not be translated. Three distinct cases, because +# they lead to different fixes: add the route to the table, add the stage to the +# table, or correct a deployment that paired a known stage with another stage's +# route. +UNKNOWN_ROUTE, UNMAPPED_STAGE, STAGE_ROUTE_MISMATCH = 'unknown_route', 'unmapped_stage', 'stage_route_mismatch' +REASONS = frozenset({UNKNOWN_ROUTE, UNMAPPED_STAGE, STAGE_ROUTE_MISMATCH}) +PARAMETER_ONLY, NO_LOCAL_ADAPTER = 'parameter_only', 'no_local_adapter' +UNROUTED_REASONS = frozenset({PARAMETER_ONLY, NO_LOCAL_ADAPTER}) + +KNOWN_STAGES = frozenset(stage for stage, _, _ in STAGE_ROUTES) +KNOWN_ROUTES = frozenset(route for _, route, _ in STAGE_ROUTES) +BY_PAIR: dict[tuple[str, str], tuple[str, ...]] = {(stage, route): capabilities + for stage, route, capabilities in STAGE_ROUTES} + + +def named(value): + """Normalized stage name from a string or an object carrying `name`/`stage`.""" + if isinstance(value, dict): + value = value.get('name', value.get('stage')) + return value.strip().lower() if isinstance(value, str) else '' + + +def classified_capabilities(): + """Every capability id this contract places, in table order, without repeats. + + Routed capabilities come first in the order their pairs appear; unrouted + ones follow in declaration order. Callers that render a list use this order + instead of sorting, so the canvas and the contract cannot disagree. + """ + ordered = [] + for _, _, capabilities in STAGE_ROUTES: + for capability in capabilities: + if capability not in ordered: + ordered.append(capability) + ordered.extend(capability for capability in UNROUTED if capability not in ordered) + return tuple(ordered) + + +def route_path(value): + """Canonical `/v1/...` path for a declared route, or None when it is unusable. + + Upstream may report `/v1/audio/speech`, `audio/speech` or an object carrying + `path`; the base URL already contains `/v1`, so the prefix is normalized + here once instead of at every comparison. Query strings are dropped: a + version parameter does not change which capability a path serves. + """ + if isinstance(value, dict): + value = value.get('path') + if not isinstance(value, str): + return None + path = value.strip().split('?', 1)[0].strip() + if not path: + return None + path = '/' + path.lstrip('/') + if len(path) > 1: + path = path.rstrip('/') + if path == API_PREFIX or path.startswith(API_PREFIX + '/'): + return path + return API_PREFIX + path + + +def capabilities_for(stage, route): + """Capability ids for one pair, or `()` when this table does not place it. + + This is the single lookup `requests_for()` needs: given the stage it is + executing and the path it is about to post to, it gets the capabilities the + deployment granted and never has to know profile names. + """ + path = route_path(route) + if path is None: + return () + return BY_PAIR.get((named(stage), path), ()) + + +def _stage_pairs(stages): + """Pairs from a `stages: [{name, routes: [...]}]` body, in document order.""" + if not isinstance(stages, list): + return + for entry in stages: + stage = named(entry) + routes = entry.get('routes') if isinstance(entry, dict) else None + if not stage or not isinstance(routes, list): + continue + for route in routes: + path = route_path(route) + if path: + yield stage, path + + +def _model_pairs(models): + """Pairs from a `models: [{stage_ids, operations}]` body, in document order. + + The discovery document this repository parses today names routes per + operation and profile, and the stages per model. Cross-producting them is + what makes such a document readable as (stage, route) pairs; when + `stage_ids` holds opaque ids the pairs surface as unmapped stages instead of + being guessed at. + """ + if not isinstance(models, list): + return + for model in models: + stages = [stage for stage in map(named, model.get('stage_ids') or ()) if stage] \ + if isinstance(model, dict) else [] + operations = model.get('operations') if isinstance(model, dict) else None + if not stages or not isinstance(operations, dict): + continue + for operation in operations.values(): + profiles = operation.get('profiles') if isinstance(operation, dict) else None + if not isinstance(profiles, list): + continue + for profile in profiles: + routes = profile.get('routes') if isinstance(profile, dict) else None + if not isinstance(routes, list): + continue + for route in routes: + path = route_path(route) + if not path: + continue + for stage in stages: + yield stage, path + + +def declarations(payload): + """Ordered (stage, route) pairs from a capabilities body, or None when it declares none. + + Both published shapes are read: a `stages` list is taken as written, and a + `models` list is expanded through `stage_ids` and its operation profiles. A + body that declares neither returns None, which callers must treat as "no + evidence" rather than "nothing supported". + """ + if not isinstance(payload, dict): + return None + pairs = [] + for pair in (*_stage_pairs(payload.get('stages')), *_model_pairs(payload.get('models'))): + if pair not in pairs: + pairs.append(pair) + return pairs or None + + +def resolve(payload): + """Per-capability availability, the per-stage view, and the unresolved pairs. + + `payload` is the decoded body of `ENDPOINT`, or None when the request failed + (404, timeout, disabled endpoint). Model names and any other provider fields + never reach the result. + """ + vocabulary = classified_capabilities() + pairs = declarations(payload) + if not pairs: + return {'endpoint': ABSENT, 'stages': {}, + 'capabilities': {item: UNKNOWN for item in vocabulary}, 'unresolved': []} + granted, unresolved, stage_flags = set(), [], {} + for stage, route in pairs: + capabilities = capabilities_for(stage, route) + flags = stage_flags.setdefault(stage, []) + if capabilities: + granted.update(capabilities) + flags.append(True) + continue + flags.append(False) + if route not in KNOWN_ROUTES: + reason = UNKNOWN_ROUTE + elif stage not in KNOWN_STAGES: + reason = UNMAPPED_STAGE + else: + reason = STAGE_ROUTE_MISMATCH + unresolved.append({'stage': stage, 'route': route, 'reason': reason}) + stages = {stage: AVAILABLE if all(flags) else PARTIAL if any(flags) else UNMAPPED + for stage, flags in sorted(stage_flags.items())} + capabilities = {item: AVAILABLE if item in granted else UNKNOWN if item in UNROUTED else MISSING + for item in vocabulary} + return {'endpoint': PRESENT, 'stages': stages, 'capabilities': capabilities, + 'unresolved': sorted(unresolved, key=lambda item: (item['stage'], item['route']))} diff --git a/tests/fixtures/omni-capabilities/absent.json b/tests/fixtures/omni-capabilities/absent.json new file mode 100644 index 00000000..f7811dd6 --- /dev/null +++ b/tests/fixtures/omni-capabilities/absent.json @@ -0,0 +1,90 @@ +{ + "tier": "absent", + "upstreamEndpoint": "/v1/omni/capabilities", + "scope": "No declaration: every capability is unknown, so callers must fall back instead of degrading.", + "cases": [ + { + "id": "no-endpoint-404", + "note": "GET returns 404 because the deployment predates the endpoint.", + "payload": null, + "expected": { + "endpoint": "absent", + "stages": {}, + "capabilities": { + "text.chat": "unknown", + "image.generate": "unknown", + "image.edit": "unknown", + "image.segment": "unknown", + "video.generate": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", + "audio.speech": "unknown", + "audio.music": "unknown", + "audio.score": "unknown", + "video.first_frame": "unknown", + "video.edit": "unknown", + "image.cutout": "unknown", + "audio.separate": "unknown" + }, + "unresolved": [] + } + }, + { + "id": "no-stages-field", + "note": "The endpoint answers but the body carries no `stages` field.", + "payload": { + "object": "omni.capabilities" + }, + "expected": { + "endpoint": "absent", + "stages": {}, + "capabilities": { + "text.chat": "unknown", + "image.generate": "unknown", + "image.edit": "unknown", + "image.segment": "unknown", + "video.generate": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", + "audio.speech": "unknown", + "audio.music": "unknown", + "audio.score": "unknown", + "video.first_frame": "unknown", + "video.edit": "unknown", + "image.cutout": "unknown", + "audio.separate": "unknown" + }, + "unresolved": [] + } + }, + { + "id": "empty-stages", + "note": "The endpoint answers with an empty stage list, which declares nothing.", + "payload": { + "object": "omni.capabilities", + "stages": [] + }, + "expected": { + "endpoint": "absent", + "stages": {}, + "capabilities": { + "text.chat": "unknown", + "image.generate": "unknown", + "image.edit": "unknown", + "image.segment": "unknown", + "video.generate": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", + "audio.speech": "unknown", + "audio.music": "unknown", + "audio.score": "unknown", + "video.first_frame": "unknown", + "video.edit": "unknown", + "image.cutout": "unknown", + "audio.separate": "unknown" + }, + "unresolved": [] + } + } + ] +} diff --git a/tests/fixtures/omni-capabilities/full.json b/tests/fixtures/omni-capabilities/full.json new file mode 100644 index 00000000..2e5c193a --- /dev/null +++ b/tests/fixtures/omni-capabilities/full.json @@ -0,0 +1,142 @@ +{ + "tier": "full", + "upstreamEndpoint": "/v1/omni/capabilities", + "scope": "Full declaration: the deployment lists every stage and route this contract knows.", + "cases": [ + { + "id": "every-known-stage", + "note": "Every (stage, route) pair this contract knows is declared; model names are synthetic.", + "payload": { + "object": "omni.capabilities", + "stages": [ + { + "name": "text", + "model": "chat-model", + "routes": [ + "/v1/chat/completions" + ] + }, + { + "name": "image", + "model": "image-model", + "routes": [ + { + "path": "/v1/images/generations", + "methods": [ + "POST" + ] + }, + "/v1/images/edits", + "/v1/images/segmentations" + ] + }, + { + "name": "video", + "model": "video-model", + "routes": [ + "/v1/videos", + "/v1/videos/segmentations", + "/v1/videos/depth" + ] + }, + { + "name": "tts", + "model": "tts-model", + "routes": [ + "/v1/audio/speech" + ] + }, + { + "name": "music", + "model": "music-model", + "routes": [ + "/v1/audio/speech" + ] + }, + { + "name": "score", + "model": "score-model", + "routes": [ + "/v1/audio/transcriptions?version=1" + ] + } + ] + }, + "expected": { + "endpoint": "present", + "stages": { + "image": "available", + "music": "available", + "score": "available", + "text": "available", + "tts": "available", + "video": "available" + }, + "capabilities": { + "text.chat": "available", + "image.generate": "available", + "image.edit": "available", + "image.segment": "available", + "video.generate": "available", + "video.matting": "available", + "video.motion_capture": "available", + "audio.speech": "available", + "audio.music": "available", + "audio.score": "available", + "video.first_frame": "unknown", + "video.edit": "unknown", + "image.cutout": "unknown", + "audio.separate": "unknown" + }, + "unresolved": [] + } + }, + { + "id": "shared-speech-route", + "note": "`tts` and `music` declare the same `/v1/audio/speech` path; both keep their own capability.", + "payload": { + "object": "omni.capabilities", + "stages": [ + { + "name": "tts", + "model": "tts-model", + "routes": [ + "/v1/audio/speech" + ] + }, + { + "name": "music", + "model": "music-model", + "routes": [ + "/v1/audio/speech" + ] + } + ] + }, + "expected": { + "endpoint": "present", + "stages": { + "music": "available", + "tts": "available" + }, + "capabilities": { + "text.chat": "missing", + "image.generate": "missing", + "image.edit": "missing", + "image.segment": "missing", + "video.generate": "missing", + "video.matting": "missing", + "video.motion_capture": "missing", + "audio.speech": "available", + "audio.music": "available", + "audio.score": "missing", + "video.first_frame": "unknown", + "video.edit": "unknown", + "image.cutout": "unknown", + "audio.separate": "unknown" + }, + "unresolved": [] + } + } + ] +} diff --git a/tests/fixtures/omni-capabilities/partial.json b/tests/fixtures/omni-capabilities/partial.json new file mode 100644 index 00000000..1790eb41 --- /dev/null +++ b/tests/fixtures/omni-capabilities/partial.json @@ -0,0 +1,154 @@ +{ + "tier": "partial", + "upstreamEndpoint": "/v1/omni/capabilities", + "scope": "Partial declaration: some stages resolve, some do not, and `asr` must not be read as `audio.score`.", + "cases": [ + { + "id": "mixed-stages", + "note": "One complete stage, one partly understood stage, and three stages this vocabulary cannot express.", + "payload": { + "object": "omni.capabilities", + "stages": [ + { + "name": "tts", + "model": "tts-model", + "routes": [ + "/v1/audio/speech" + ] + }, + { + "name": "video", + "model": "video-model", + "routes": [ + "/v1/videos", + "/v1/videos/depth", + "/v1/videos/track" + ] + }, + { + "name": "asr", + "model": "asr-model", + "routes": [ + "/v1/audio/transcriptions" + ] + }, + { + "name": "image", + "model": "image-model", + "routes": [ + "/v1/videos" + ] + }, + { + "name": "embedding", + "model": "embedding-model", + "routes": [ + "/v1/embeddings" + ] + } + ] + }, + "expected": { + "endpoint": "present", + "stages": { + "asr": "unmapped", + "embedding": "unmapped", + "image": "unmapped", + "tts": "available", + "video": "partial" + }, + "capabilities": { + "text.chat": "missing", + "image.generate": "missing", + "image.edit": "missing", + "image.segment": "missing", + "video.generate": "available", + "video.matting": "missing", + "video.motion_capture": "available", + "audio.speech": "available", + "audio.music": "missing", + "audio.score": "missing", + "video.first_frame": "unknown", + "video.edit": "unknown", + "image.cutout": "unknown", + "audio.separate": "unknown" + }, + "unresolved": [ + { + "stage": "asr", + "route": "/v1/audio/transcriptions", + "reason": "unmapped_stage" + }, + { + "stage": "embedding", + "route": "/v1/embeddings", + "reason": "unknown_route" + }, + { + "stage": "image", + "route": "/v1/videos", + "reason": "stage_route_mismatch" + }, + { + "stage": "video", + "route": "/v1/videos/track", + "reason": "unknown_route" + } + ] + } + }, + { + "id": "shared-transcription-route", + "note": "`score` and `asr` declare the same `/v1/audio/transcriptions` path; only `score` grants a capability.", + "payload": { + "object": "omni.capabilities", + "stages": [ + { + "name": "score", + "model": "score-model", + "routes": [ + "/v1/audio/transcriptions" + ] + }, + { + "name": "asr", + "model": "asr-model", + "routes": [ + "audio/transcriptions" + ] + } + ] + }, + "expected": { + "endpoint": "present", + "stages": { + "asr": "unmapped", + "score": "available" + }, + "capabilities": { + "text.chat": "missing", + "image.generate": "missing", + "image.edit": "missing", + "image.segment": "missing", + "video.generate": "missing", + "video.matting": "missing", + "video.motion_capture": "missing", + "audio.speech": "missing", + "audio.music": "missing", + "audio.score": "available", + "video.first_frame": "unknown", + "video.edit": "unknown", + "image.cutout": "unknown", + "audio.separate": "unknown" + }, + "unresolved": [ + { + "stage": "asr", + "route": "/v1/audio/transcriptions", + "reason": "unmapped_stage" + } + ] + } + } + ] +} diff --git a/tests/test_capability_contract.py b/tests/test_capability_contract.py new file mode 100644 index 00000000..bb1be1d6 --- /dev/null +++ b/tests/test_capability_contract.py @@ -0,0 +1,207 @@ +"""The capability contract translates (stage, route) pairs, and nothing else. + +`tests/fixtures/omni-capabilities/` is the shared sample set: three tiers that +state what a deployment offering those pairs should be understood to provide. +The expected blocks are the reviewed oracle, not a copy of the mapping table, so +changing a mapping has to argue with them. They also stand on their own: with no +resolver at all, a case still says what the deployment offers. + +The last group reads the discovery document this repository parses today +(`models[].stage_ids` with routes per operation profile), because that is the +input `requests_for()` will hand to this table. +""" +import json +from pathlib import Path +from typing import get_args + +import pytest + +from opendirector.domain.capabilities import Capability +from opendirector.domain.capability_contract import (ABSENT, AVAILABLE, ENDPOINT, MISSING, PARTIAL, PRESENT, + REASONS, STAGE_ROUTES, STAGE_ROUTE_MISMATCH, UNKNOWN, UNMAPPED, + UNMAPPED_STAGE, UNKNOWN_ROUTE, UNROUTED, UNROUTED_REASONS, + capabilities_for, classified_capabilities, declarations, + resolve, route_path) + +FIXTURES = Path(__file__).parent / 'fixtures' / 'omni-capabilities' +# Upstream bodies may name models; credentials and raw provider responses never +# belong in a fixture, so their vocabulary is refused outright. +FORBIDDEN = ('apikey', 'api_key', 'authorization', 'bearer', 'secret', 'password', 'cookie', '://') + + +def raw_cases(): + paths = sorted(FIXTURES.glob('*.json')) + assert {path.stem for path in paths} == {'full', 'partial', 'absent'} + for path in paths: + document = json.loads(path.read_text(encoding='utf-8')) + assert document['upstreamEndpoint'] == ENDPOINT + yield document, document['cases'] + + +def fixture_cases(): + for document, cases in raw_cases(): + for case in cases: + yield pytest.param(case['payload'], case['expected'], id=f"{document['tier']}-{case['id']}") + + +@pytest.mark.parametrize('payload, expected', list(fixture_cases())) +def test_declaration_resolves_to_the_reviewed_expectation(payload, expected): + assert resolve(payload) == expected + + +def test_every_capability_id_is_routed_or_explained(): + """Adding a capability to the vocabulary without placing it here fails this.""" + assert set(classified_capabilities()) == set(get_args(Capability)) + + +def test_a_capability_appears_once_in_the_vocabulary_order(): + vocabulary = classified_capabilities() + assert len(vocabulary) == len(set(vocabulary)) + assert tuple(dict.fromkeys(vocabulary)) == vocabulary + + +def test_contract_pairs_and_routes_are_canonical(): + assert len(STAGE_ROUTES) == len({(stage, route) for stage, route, _ in STAGE_ROUTES}) + for stage, route, capabilities in STAGE_ROUTES: + assert route_path(route) == route, (stage, route) + assert route.startswith('/v1/'), (stage, route) + assert stage == stage.strip().lower(), (stage, route) + assert capabilities and all(isinstance(item, str) for item in capabilities), (stage, route) + + +def test_one_path_serves_two_stages_without_overwriting_each_other(): + """`/v1/audio/speech` is both speech and music; the pair decides, the path does not.""" + assert capabilities_for('tts', '/v1/audio/speech') == ('audio.speech',) + assert capabilities_for('music', '/v1/audio/speech') == ('audio.music',) + result = resolve({'stages': [{'name': 'tts', 'routes': ['/v1/audio/speech']}, + {'name': 'music', 'routes': ['/v1/audio/speech']}]}) + assert result['stages'] == {'music': AVAILABLE, 'tts': AVAILABLE} + assert result['capabilities']['audio.speech'] == AVAILABLE + assert result['capabilities']['audio.music'] == AVAILABLE + + +def test_a_shared_path_does_not_grant_the_other_stage(): + """A TTS-only deployment must not look like it can generate music.""" + result = resolve({'stages': [{'name': 'tts', 'routes': ['/v1/audio/speech']}]}) + assert result['capabilities']['audio.speech'] == AVAILABLE + assert result['capabilities']['audio.music'] == MISSING + assert result['unresolved'] == [] + + +def test_transcription_serves_score_but_not_recognition(): + """`/v1/audio/transcriptions` is score under `score` and unmapped under `asr`.""" + assert capabilities_for('score', '/v1/audio/transcriptions') == ('audio.score',) + assert capabilities_for('asr', '/v1/audio/transcriptions') == () + declared = resolve({'stages': [{'name': 'score', 'routes': ['/v1/audio/transcriptions']}, + {'name': 'asr', 'routes': ['/v1/audio/transcriptions']}]}) + assert declared['capabilities']['audio.score'] == AVAILABLE + assert declared['stages'] == {'asr': UNMAPPED, 'score': AVAILABLE} + assert declared['unresolved'] == [{'stage': 'asr', 'route': '/v1/audio/transcriptions', + 'reason': UNMAPPED_STAGE}] + # Recognition alone proves nothing about the score adapter, and vice versa. + recognition_only = resolve({'stages': [{'name': 'asr', 'routes': ['/v1/audio/transcriptions']}]}) + assert recognition_only['capabilities']['audio.score'] == MISSING + + +def test_an_absent_declaration_is_unknown_everywhere_and_never_missing(): + for payload in (None, {'object': 'omni.capabilities'}, {'stages': []}, {'stages': [{'routes': ['/v1/audio/speech']}]}, + {'stages': 'not-a-list'}, {'models': []}): + result = resolve(payload) + assert result['endpoint'] == ABSENT + assert result['stages'] == {} + assert set(result['capabilities'].values()) == {UNKNOWN} + assert MISSING not in result['capabilities'].values() + + +def test_a_known_stage_with_another_stages_route_is_a_mismatch(): + result = resolve({'stages': [{'name': 'image', 'routes': ['/v1/videos']}]}) + assert result['stages'] == {'image': UNMAPPED} + assert result['capabilities']['video.generate'] == MISSING + assert result['unresolved'] == [{'stage': 'image', 'route': '/v1/videos', 'reason': STAGE_ROUTE_MISMATCH}] + + +def test_an_unknown_path_and_an_unknown_stage_stay_distinguishable(): + unknown_path = resolve({'stages': [{'name': 'video', 'routes': ['/v1/videos/track']}]}) + unknown_stage = resolve({'stages': [{'name': 'embedding', 'routes': ['/v1/videos']}]}) + assert unknown_path['unresolved'][0]['reason'] == UNKNOWN_ROUTE + assert unknown_stage['unresolved'][0]['reason'] == UNMAPPED_STAGE + + +def test_capabilities_are_reported_in_table_order(): + """Presentation reads this order; sorting here would make the table's order meaningless.""" + result = resolve({'stages': [{'name': 'score', 'routes': ['/v1/audio/transcriptions']}, + {'name': 'text', 'routes': ['/v1/chat/completions']}]}) + assert list(result['capabilities']) == list(classified_capabilities()) + assert list(result['capabilities'])[:2] == ['text.chat', 'image.generate'] + + +@pytest.mark.parametrize('route', ['audio/speech', '/audio/speech', '/v1/audio/speech/', {'path': '/v1/audio/speech?x=1'}]) +def test_route_spellings_normalize_to_one_path(route): + result = resolve({'stages': [{'name': 'tts', 'routes': [route]}]}) + assert result['stages'] == {'tts': AVAILABLE} + assert result['capabilities']['audio.speech'] == AVAILABLE + + +def test_stage_states_report_partial_understanding(): + result = resolve({'stages': [{'name': 'video', 'routes': ['/v1/videos', '/v1/videos/track']}]}) + assert result['stages'] == {'video': PARTIAL} + assert result['endpoint'] == PRESENT + assert result['unresolved'] == [{'stage': 'video', 'route': '/v1/videos/track', 'reason': UNKNOWN_ROUTE}] + + +def test_a_route_cannot_speak_for_a_parameter_capability(): + """`video.first_frame` is a /v1/videos parameter, so it is never `missing`.""" + assert resolve({'stages': [{'name': 'video', 'routes': ['/v1/videos']}]})['capabilities']['video.first_frame'] == UNKNOWN + + +def test_unrouted_capabilities_state_why_they_are_unreachable(): + assert set(UNROUTED.values()) <= UNROUTED_REASONS + + +def test_model_names_and_other_provider_fields_never_reach_the_result(): + payload = {'stages': [{'name': 'tts', 'model': 'a-private-model-name', 'routes': ['/v1/audio/speech'], + 'extra': {'owner': 'someone'}}]} + assert 'a-private-model-name' not in json.dumps(resolve(payload), ensure_ascii=False) + + +def test_results_use_only_the_contract_vocabulary(): + for _, cases in raw_cases(): + for case in cases: + expected = case['expected'] + assert set(expected['capabilities'].values()) <= {AVAILABLE, MISSING, UNKNOWN} + assert set(expected['stages'].values()) <= {AVAILABLE, PARTIAL, UNMAPPED} + assert {item['reason'] for item in expected['unresolved']} <= REASONS + + +def test_fixtures_carry_no_credentials_or_provider_responses(): + for path in sorted(FIXTURES.glob('*.json')): + text = path.read_text(encoding='utf-8').lower() + assert not [word for word in FORBIDDEN if word in text], path.name + + +def discovery_document(stage_ids, path='/v1/audio/speech', profile='text_to_speech'): + """The body `integrations.capability_probe` parses today: stages per model, routes per profile.""" + return {'object': 'omni.capabilities', 'schema_version': 1, 'observed_at': '2026-11-10T00:00:00+00:00', + 'models': [{'id': 'demo', 'stage_ids': list(stage_ids), 'operations': {'speech': { + 'support': 'supported', 'exposure': 'enabled', 'profiles': [ + {'id': profile, 'support': 'supported', 'routes': [{'method': 'POST', 'path': path}]}]}}}], + 'routes': [{'method': 'POST', 'path': path, 'exposure': 'enabled'}]} + + +def test_a_models_document_yields_the_same_pairs(): + assert declarations(discovery_document(['tts'])) == [('tts', '/v1/audio/speech')] + assert resolve(discovery_document(['tts']))['capabilities']['audio.speech'] == AVAILABLE + assert resolve(discovery_document(['music']))['capabilities']['audio.music'] == AVAILABLE + + +def test_an_opaque_stage_id_is_reported_instead_of_guessed(): + """A numeric `stage_ids` entry cannot be translated, and must not be read as a route match.""" + result = resolve(discovery_document(['0'])) + assert result['unresolved'] == [{'stage': '0', 'route': '/v1/audio/speech', 'reason': UNMAPPED_STAGE}] + assert result['capabilities']['audio.speech'] == MISSING + assert result['capabilities']['video.first_frame'] == UNKNOWN + + +def test_model_pairs_follow_document_order(): + pairs = declarations(discovery_document(['tts', 'music'])) + assert pairs == [('tts', '/v1/audio/speech'), ('music', '/v1/audio/speech')] From cfd25d2bf9d3ac80807a085344adb3b94cda91b1 Mon Sep 17 00:00:00 2001 From: Xingkai98 Date: Wed, 11 Nov 2026 00:22:21 +0800 Subject: [PATCH 2/5] fix(contract): key the table by (operation, profile) per upstream vocabulary vllm-omni#10102 publishes six operations and twelve profiles; `stages[]` is deployment topology (`id`/`type`/`depends_on`, numeric strings, the same full list for every model) and a route cannot separate the three profiles that share `/v1/audio/speech`. The table is therefore keyed by (operation, profile), with OPERATION_ROUTES kept as the ownership check that reports `route_mismatch`. - OPERATION_PROFILES replaces STAGE_ROUTES, covering all twelve profiles - capabilities are available only on a supported profile in an enabled operation; `missing` needs positive evidence, everything else stays `unknown` - segmentations/depth keep a route but no operation, so image.segment, video.matting and video.motion_capture are `unknown` with `no_upstream_profile` - `asr` reports `unmapped_stage`: upstream publishes no recognition operation - fixtures follow the real response shape; five regressions pin the pairs, the shared speech path, topology-only stages and absent fields --- docs/README.md | 2 +- docs/architecture/capability-contract.md | 189 ++++- .../domain/capability_contract.py | 529 ++++++++---- tests/fixtures/omni-capabilities/absent.json | 258 +++++- tests/fixtures/omni-capabilities/full.json | 542 ++++++++++--- tests/fixtures/omni-capabilities/partial.json | 753 ++++++++++++++++-- tests/test_capability_contract.py | 485 ++++++++--- 7 files changed, 2224 insertions(+), 534 deletions(-) diff --git a/docs/README.md b/docs/README.md index 0a24bfc0..3541cc6f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ This directory is the entry point for the repository's shared documentation. The | [Contributing](../CONTRIBUTING.md) | Label taxonomy, how to pick labels, and what a PR must satisfy | Process, labels, or submission requirements change | | [Unified project architecture](architecture/unified-project.md) | Business objects, shared entry points, and interaction rules | Object responsibilities, commands, persistence, or interaction rules change | | [Test contracts](architecture/test-contracts.md) | Test/production boundaries, how to avoid stale assertions and environment-dependent red runs | Test infrastructure, capability gates, or contract ownership change | -| [Capability mapping contract / 能力映射表契约](architecture/capability-contract.md) | `(stage, route) → capability` table, the three degradation states, and how it feeds discovery | Stage/route vocabulary, capability ids, or discovery matching change | +| [Capability mapping contract / 能力映射表契约](architecture/capability-contract.md) | Evidenced upstream vocabulary, the `(operation, profile) → capability` table, the six untranslatable declarations, and how it feeds discovery | Upstream vocabulary, capability ids, or discovery matching change | | [Roadmap](development/roadmap.md) | Capability dependencies and delivery criteria | Scope, dependencies, or acceptance criteria change | | [Validation guide](development/validation.md) | Reproducible checks and evidence requirements | Tools, runtime requirements, or validation entry points change | | [Issue #87 Windows handoff](development/issue-87-windows-handoff.md) ([中文](development/issue-87-windows-handoff.zh-CN.md)) | Native ACL repair and final text acceptance | The shared checkpoint or native validation scope changes | diff --git a/docs/architecture/capability-contract.md b/docs/architecture/capability-contract.md index c77590ad..a72c72b5 100644 --- a/docs/architecture/capability-contract.md +++ b/docs/architecture/capability-contract.md @@ -2,100 +2,209 @@ ## 中文 -`GET /v1/omni/capabilities` 只说明部署「自述了什么」；产品要回答的是「这个部署能不能做某件事」。`packages/core/src/opendirector/domain/capability_contract.py` 是唯一把上游 stage 名与路由翻译成 `opendirector.domain.capabilities` id 的地方。它不发起 HTTP，不持有界面文案，也不创建第二套 operation 常量。 +`GET /v1/omni/capabilities` 只说明部署「自述了什么」；产品要回答的是「这个部署能不能做某件事」。`packages/core/src/opendirector/domain/capability_contract.py` 是唯一把上游词表翻译成 `opendirector.domain.capabilities` id 的地方。它不发起 HTTP，不持有界面文案，也不创建第二套 operation 常量。 -### 键必须是 (stage, route) 成对 +### 上游真实词表（取证） -一个路径可以同时服务两种能力。`/v1/audio/speech` 既是 TTS 也是音乐生成，`/v1/audio/transcriptions` 既是扒谱（`score`）也是语音识别（`asr`）。只按路径匹配，会让「只声明了 TTS」的服务看起来能生成音乐，反过来也一样；部署级 `exposure` 若按 `(method, path)` 单键下发，关闭语音合成会连带关闭音乐。因此 `STAGE_ROUTES` 的键是成对的 `(upstream stage, upstream route)`，落表后经 `capabilities_for(stage, route)` 查找；`/v1/chat/completions` 这类独占路径同样按对匹配，不为单键开后门。 +取证对象：`vllm-project/vllm-omni` PR #10102（issue #10007），head `d1104f8`。下列 `path:line` 均取自 head 原文件。 -`asr` 目前**不在表里**：能力词表（`domain/capabilities.py`）还没有识别能力 id，把 `/v1/audio/transcriptions` 在 `asr` 下读成 `audio.score` 是错的。它会被如实报成 `unmapped_stage`，等词表有了 `audio.transcribe` 再补一行。 +**6 个 operation 是闭合常量**（`vllm_omni/serving_capabilities.py:15,19-26`）： +`chat` / `image_generation` / `image_edit` / `video_generation` / `speech` / `audio_generation`。 +没有 `asr`、`transcription`、`segmentation`、`depth`、`music`。 -### 三种降级状态，含义不同 +**12 个 profile**（`:43-54`），以及本仓对应的 capability id： -| 状态 | 含义 | 修的地方 | +| operation | profile | 本仓 capability id | | --- | --- | --- | -| `unknown_route` | 路径本身不在表里（例如 `/v1/videos/track`、`/v1/embeddings`） | 表或适配器缺这条路径 | -| `unmapped_stage` | 路径认识，stage 不认识（例如 `asr` 打 `/v1/audio/transcriptions`，或 `stage_ids` 是数字 id） | 表缺这个 stage，或上游 stage 名没对齐 | -| `stage_route_mismatch` | stage 与路径都认识，但这一对不成立（例如 `image` 声明 `/v1/videos`） | 部署声明的配对写错了 | +| `chat` | `text_response` | `text.chat` | +| `chat` | `image_response` | `text.chat` | +| `chat` | `audio_response` | `text.chat` | +| `image_generation` | `text_to_image` | `image.generate` | +| `image_edit` | `image_to_image` | `image.edit` | +| `video_generation` | `text_to_video` | `video.generate` | +| `video_generation` | `image_to_video` | `video.generate` | +| `video_generation` | `video_to_video` | `video.generate` | +| `speech` | `text_to_speech` | `audio.speech` | +| `speech` | `lyrics_to_music` | `audio.music` | +| `speech` | `caption_to_audio` | （无）音效/环境声，本仓无对应 id | +| `audio_generation` | `text_to_audio` | （无）扩散侧独立契约，本仓无对应 id | -三者都进 `unresolved`，不被丢弃 —— 调用方可以解释「为什么这个 stage 不可用」，而不是静默隐藏。 +**operation → 路由**（`:42-54`，一致性校验用）： -能力状态另有三档：`available`（有成对声明）、`missing`（部署答了，但没有这一对）、`unknown`（没有足够信息）。请求失败、404、没有 `stages`、空列表、stage 没有名字，全部返回 `endpoint=absent` 与全 `unknown`：**「服务没说」不是「服务不能」**。另有四个能力（`video.first_frame`、`video.edit`、`image.cutout`、`audio.separate`）根本无法由路由表达，它们永远是 `unknown`，并在 `UNROUTED` 里写明原因。 +| operation | 路由 | +| --- | --- | +| `chat` | `POST /v1/chat/completions` | +| `image_generation` | `POST /v1/images/generations` | +| `image_edit` | `POST /v1/images/edits` | +| `video_generation` | `POST /v1/videos`；`POST /v1/videos/sync`；`WEBSOCKET /v1/realtime/video` | +| `speech` | `POST /v1/audio/speech` | +| `audio_generation` | `POST /v1/audio/generate` | + +另外三条事实决定了这份表的形状： + +- 响应顶层只有 `object` / `schema_version` / `observed_at` / `models` / `stages` / `routes`（`protocol/capabilities.py:73-79`），端点是 `response_model=OmniCapabilitiesResponse`、`response_model_exclude_none=True`（`api_server.py:2193`）——**字段缺席是显式设计，等价于 unknown**。 +- `models[].operations` **必须齐全 6 个**，少一个 pydantic 直接抛错（`protocol/capabilities.py:60-64`）。 +- `models[].stage_ids` 是**数字字符串**，而且每个 model 拿到的都是同一份全量列表（`entrypoints/openai/serving_capabilities.py:400`）。 + +### 键是 (operation, profile)，不是 route，也不是 stage + +**为什么不是 route**：`/v1/audio/speech` 一条路径挂 3 个 profile —— `text_to_speech`（配音）、`lyrics_to_music`（音乐）、`caption_to_audio`（音效）。上游自己的测试里，MOSS-SoundEffect 在 speech 上只声明 `caption_to_audio`，同时把 `text_to_speech` 标成 `unsupported`（`tests/config/test_serving_capabilities.py:233,251`）：**同一路径，两种答案**。只按路径匹配，「只做音效」的部署会看起来会配音，反过来也一样。因此 `OPERATION_PROFILES` 的键是成对的 `(operation, profile)`，经 `capabilities_for(operation, profile)` 查找。 + +**为什么不是 stage**：`stages[]` 只有 `id` / `type` / `depends_on`（`protocol/capabilities.py:67-70`），id 是不连续的数字字符串，而且每个 model 都是同一份列表。拿它当键，所有模型会共享一个状态，既区分不了也不能诊断。本模块把 `stages[]` 只放进 `topology` 供展示，**从不用它决定任何能力**；一个 stage 条目即使自带 `routes` 或 `name` 也被忽略，因为上游 schema 没有这两个键。 + +**route 仍然有用**：`OPERATION_ROUTES` 保留 operation 对路由的所有权，用来校验声明的一致性 —— 某个 profile 引用了不属于该 operation 的路径（例如 `text_to_video` 指向 `/v1/images/edits`），该 profile 不产生能力，并报 `route_mismatch`。 + +### 六种无法翻译的声明 + +| reason | 含义 | 修的地方 | +| --- | --- | --- | +| `unknown_route` | 路径不属于任何 operation（`/v1/videos/track`，或已注册但无 operation 的 `/v1/images/segmentations`） | 表缺这个 operation，或声明写错了路径 | +| `route_mismatch` | 路径属于**另一个** operation（`text_to_video` 指向 `/v1/images/edits`） | 部署声明的配对写错了 | +| `unmapped_stage` | 上游没有这个 operation（`asr`、`music`） | 上游词表缺口，不是本表漏行 | +| `unknown_profile` | operation 认识，profile id 上游未发布（`speech_to_speech`） | 上游新增 profile，本表待补 | +| `operation_mismatch` | profile 认识，但挂在别的 operation 下（`lyrics_to_music` 挂在 `audio_generation`） | 声明写错了 operation，或本表把归属搞错了 | +| `no_local_capability` | profile 支持，但本仓没有对应 capability id | 词表待补（`caption_to_audio`、`text_to_audio`） | + +六种都进 `unresolved`，不被丢弃；每条都带 `operation` / `profile` / `route` / `reason` 四键（不适用处为 `null`），调用方可以解释「为什么这条声明没变成能力」，而不是静默隐藏。 + +### 三种能力状态，只有正面证据才是 missing + +- `available`：某个 model 的 profile 明确 `support=supported`，且其 operation 不是 `exposure=disabled`； +- `missing`：某个 profile 或 operation **正面说不**（`support=unsupported` 或 `exposure=disabled`）； +- `unknown`：其余全部 —— 请求失败/404、字段缺席、枚举为 `unknown`、声明无法翻译、能力本身不可路由。 + +**「服务没说」不是「服务不能」**：没有 `models`、没有 operation、`profiles` 被 `exclude_none` 掉，一律 `unknown`，绝不产生 `missing`。 + +### 有路由、没有 operation 的能力 + +`/v1/images/segmentations`、`/v1/videos/segmentations`、`/v1/videos/depth` 是**已注册的端点，但没有任何 operation 拥有它们**（上游文档明示：segmentation 与 depth「不会仅因为出现在 registry 里就获得这些 operation」，`docs/serving/capabilities.md:41-43`）。所以 `image.segment` / `video.matting` / `video.motion_capture` 恒为 `unknown`，reason 是 `no_upstream_profile` —— 端点不发布该能力，无法判定；**不能**写成 `missing`。 + +`UNROUTED` 里其余几项理由不同，不要混： + +| capability | reason | 说明 | +| --- | --- | --- | +| `image.segment` / `video.matting` / `video.motion_capture` | `no_upstream_profile` | 有路由，无 operation | +| `audio.score` | `no_upstream_operation` | 上游没有识别 operation | +| `video.first_frame` / `video.edit` / `image.cutout` | `parameter_only` | 是已发布 operation 的请求参数，不是 profile | +| `audio.separate` | `no_local_adapter` | 上游没说法，本仓也没适配器 | + +### ASR：上游未发布该 operation + +上游词表里没有识别 operation，omni 的 router 也不注册 `/v1/audio/transcriptions`（`api_server.py` 全量路由里没有它），上游自己的 recipe 写明该端点「not currently available with `--omni`」。因此请求侧写 `asr` 得到 `unmapped_stage`：这是**上游没发布这个 operation**，不是本表漏了一行；`audio.score` 因此恒为 `unknown`（`no_upstream_operation`）。等上游发布识别 operation（或本仓补 `audio.transcribe`）再谈映射，绝不把 `/v1/audio/transcriptions` 读成 `audio.score`。 ### 与 V2 发现契约（#266）的关系 这份表是 #266 的**输入**，不是替代品，也不是第二套 schema： -- #266 的 `domain/omni_capabilities.py` 定义发现文档与匹配；本模块只提供 `(stage, route) → capability` 这一层。两者文件名不同，互不覆盖。 -- #266 的 `Model.stage_ids` 目前没有任何读取方（只有测试夹具填 `['0']`）。两条路：(a) 删掉该字段；(b) 接上 —— 让 `stage_ids` 与每个 operation profile 的 `routes` 交叉出成对声明。本模块的 `declarations()` 已经能读 `models[].stage_ids` × `operations.*.profiles[].routes[]` 这种形状，供 (b) 直接使用；若 `stage_ids` 里是数字 id，结果会如实报 `unmapped_stage`，不会被猜成某个 stage。 -- 本模块不改 `domain/omni_capabilities.py`、`integrations/capability_probe.py`、`application/model_availability.py`。 +- #266 的 `domain/omni_capabilities.py` 定义发现文档与匹配；本模块只提供 `(operation, profile) → capability` 这一层。文件名不同，互不覆盖。 +- 本模块**不再**读 `Model.stage_ids`：它是拓扑，不是能力键。`declarations()` 只读上游真实发布的 `models[].operations[].profiles[]`。 +- 本模块不改 `domain/omni_capabilities.py`、`integrations/capability_probe.py`、`application/model_availability.py`、`src/**`。 ### 如何作为 `requests_for()` 的输入 -`application/model_availability.py` 现在按 `(kind, protocol)` 硬编码 profile 名，ASR/扒谱因此拿到空列表。表驱动后，调用方只需要自己正在执行的 stage 与将要 POST 的路径： +`application/model_availability.py` 现在按 `(kind, protocol)` 硬编码 profile 名。表驱动后，调用方只需要自己正在调用的 operation 与 profile： ```python from opendirector.domain.capability_contract import capabilities_for -capabilities = capabilities_for('music', '/v1/audio/speech') # ('audio.music',) +capabilities_for('speech', 'lyrics_to_music') # ('audio.music',) +capabilities_for('speech', 'text_to_speech') # ('audio.speech',) ``` -`resolve(payload)` 返回逐能力状态、逐 stage 状态与 `unresolved` 列表；`classified_capabilities()` 按表顺序给出词表，展示层据此排列，不再自己排序。是否接进 `requests_for()` 由 #266 一侧决定，本契约只保证查找是成对的、稳定的、可测的。 +`resolve(payload)` 返回逐能力状态、`topology`（`stages[]`，仅供展示）与 `unresolved`；`classified_capabilities()` 按表顺序给出词表，展示层据此排列。是否接进 `requests_for()` 由 #266 一侧决定，本契约只保证查找是成对的、稳定的、可测的。 ### 三档 fixture 与验证 -`tests/fixtures/omni-capabilities/{full,partial,absent}.json` 是共享样本：`full` 覆盖表里的全部配对与两条共享路径，`partial` 覆盖混合可用、`stage_route_mismatch` 与两种未知，`absent` 覆盖 404、缺字段、空列表。`tests/test_capability_contract.py` 逐档比对期望块，并单独覆盖共享路径不互相覆盖、`asr` 不等于 `score`、`absent` 全 `unknown`、表顺序即输出顺序、`models[]` 文档形状。 +`tests/fixtures/omni-capabilities/{full,partial,absent}.json` 按**真实响应形状**写：`stages[]` 是 `{id,type,depends_on}`，`models[].operations` 六个齐全、每个含 `support`/`exposure`/`reason_code`/`profiles`/`execution`，profile 带 `routes[]`/`required_inputs`/`outputs`；`absent` 档用**字段缺席**（模拟 `exclude_none=True`）而不是 `null`；`partial` 档覆盖 `support=unsupported`、`exposure=disabled`、`route_mismatch`、`unknown_route`、`unknown_profile`、`operation_mismatch`、`unmapped_stage`、`no_local_capability`。 + +防「假覆盖」的回归测试（`tests/test_capability_contract.py`）： + +| 测试 | 钉住的事实 | +| --- | --- | +| `test_real_shaped_body_resolves_non_empty_pairs` | 真实形状 body 解析出 12 个成对声明，不是空表 | +| `test_route_alone_cannot_distinguish_speech_profiles` | 同一路径 3 个 profile 解析出 3 个不同答案 | +| `test_stage_ids_do_not_collapse_capabilities` | 两个 model 的 `stage_ids` 相同，能力状态仍各自独立 | +| `test_unrouted_capabilities_are_unknown_not_missing` | 有路由无 operation 的能力恒 `unknown` | +| `test_absent_fields_are_unknown` | 字段缺席是 `unknown`，只有明确 `unsupported` 才是 `missing` | ```bash -PYTHONPATH=packages/core/src python -m pytest tests/test_capability_contract.py -q +PYTHONPATH=packages/core/src:apps/api/src python -m pytest tests/test_capability_contract.py -q ``` ## English -`GET /v1/omni/capabilities` reports what a deployment says about itself; the product needs to know what that deployment can do. `packages/core/src/opendirector/domain/capability_contract.py` is the only place that translates upstream stage names and routes into `opendirector.domain.capabilities` ids. It performs no HTTP, holds no display text, and introduces no second set of operation constants. +`GET /v1/omni/capabilities` reports what a deployment says about itself; the product needs to know what that deployment can do. `packages/core/src/opendirector/domain/capability_contract.py` is the only place that translates upstream's vocabulary into `opendirector.domain.capabilities` ids. It performs no HTTP, holds no display text, and introduces no second set of operation constants. -### The key is the pair +### The upstream vocabulary (as evidenced) -One path can serve two capabilities. `/v1/audio/speech` carries both speech synthesis and music generation; `/v1/audio/transcriptions` carries both score transcription (`score`) and recognition (`asr`). A path-only key makes a TTS-only deployment look able to generate music, and the reverse; a deployment-level `exposure` published per `(method, path)` disables music when it disables speech. `STAGE_ROUTES` is therefore keyed by the pair `(upstream stage, upstream route)` and read through `capabilities_for(stage, route)`. Paths that happen to be exclusive, such as `/v1/chat/completions`, are matched as pairs too, so there is no second rule to keep in sync. +Source: `vllm-project/vllm-omni` PR #10102 (issue #10007), head `d1104f8`; every `path:line` below is from the head tree. -`asr` is deliberately absent from the table: the vocabulary in `domain/capabilities.py` has no recognition id yet, and reading `/v1/audio/transcriptions` under `asr` as `audio.score` would be wrong. It is reported as `unmapped_stage` until an `audio.transcribe` capability exists. +**Six closed operations** (`vllm_omni/serving_capabilities.py:15,19-26`): `chat`, `image_generation`, `image_edit`, `video_generation`, `speech`, `audio_generation`. There is no `asr`, `transcription`, `segmentation`, `depth` or `music`. -### Three degradation states, three different fixes +**Twelve profiles** (`:43-54`) and the ids this repository has for them: -| State | Meaning | What to fix | +| operation | profile | capability here | | --- | --- | --- | -| `unknown_route` | The path is not in the table (`/v1/videos/track`, `/v1/embeddings`) | The table or an adapter is missing the path | -| `unmapped_stage` | The path is known, the stage is not (`asr` on `/v1/audio/transcriptions`, or a numeric `stage_ids` entry) | The table is missing the stage, or the upstream stage name is not aligned | -| `stage_route_mismatch` | Both are known, but not as this pair (`image` declaring `/v1/videos`) | The deployment declared a wrong pairing | +| `chat` | `text_response`, `image_response`, `audio_response` | `text.chat` | +| `image_generation` | `text_to_image` | `image.generate` | +| `image_edit` | `image_to_image` | `image.edit` | +| `video_generation` | `text_to_video`, `image_to_video`, `video_to_video` | `video.generate` | +| `speech` | `text_to_speech` | `audio.speech` | +| `speech` | `lyrics_to_music` | `audio.music` | +| `speech` | `caption_to_audio` | none — sound effect / ambience | +| `audio_generation` | `text_to_audio` | none — separate diffusion contract | -All three land in `unresolved` instead of being dropped, so a caller can explain why a stage is unavailable rather than hiding it. +**Operation → routes** (`:42-54`): `chat` → `POST /v1/chat/completions`; `image_generation` → `POST /v1/images/generations`; `image_edit` → `POST /v1/images/edits`; `video_generation` → `POST /v1/videos`, `POST /v1/videos/sync`, `WEBSOCKET /v1/realtime/video`; `speech` → `POST /v1/audio/speech`; `audio_generation` → `POST /v1/audio/generate`. -Capability states are separate: `available` (a matching pair was declared), `missing` (the deployment answered without that pair), `unknown` (not enough information). A failed request, a 404, a body without `stages`, an empty list, or an entry without a stage all return `endpoint=absent` with everything `unknown`: **"the service did not say" is not "the service cannot"**. Four capabilities (`video.first_frame`, `video.edit`, `image.cutout`, `audio.separate`) cannot be expressed by any route; they stay `unknown` forever and `UNROUTED` states why. +Three more facts shape the table: the body has only `object`, `schema_version`, `observed_at`, `models`, `stages`, `routes` (`protocol/capabilities.py:73-79`) and is served with `response_model_exclude_none=True` (`api_server.py:2193`), so **an absent field is the designed way to say unknown**; `models[].operations` **must list all six** or pydantic raises (`protocol/capabilities.py:60-64`); and `models[].stage_ids` are numeric strings, the *same* full list for every model (`entrypoints/openai/serving_capabilities.py:400`). -### Relationship to the V2 discovery contract (#266) +### The key is the pair (operation, profile) + +Not the route: `/v1/audio/speech` is one path with three contracts — `text_to_speech` (voice), `lyrics_to_music` (music), `caption_to_audio` (sound effect). Upstream's own tests show MOSS-SoundEffect serving `caption_to_audio` while reporting `text_to_speech` unsupported on that same path (`tests/config/test_serving_capabilities.py:233,251`): one path, two answers. A path-only key makes a sound-effect deployment look able to speak, and the reverse. + +Not the stage either: `stages[]` publishes only `id` / `type` / `depends_on`, the ids are opaque numeric strings, and every model carries the same list. Keying on stages gives all models one shared state and no diagnosis, so `stages[]` only becomes `topology` here and never decides a capability; a stage entry carrying `routes` or `name` is ignored, because upstream's `CapabilityStage` has no such fields. + +Routes still matter, as a consistency check: `OPERATION_ROUTES` records which operation owns which path, and a profile that references another operation's path (say `text_to_video` → `/v1/images/edits`) grants nothing and is reported as `route_mismatch`. + +### Six untranslatable declarations -This table is an input to #266, not a replacement and not a second schema: +`unknown_route` (the path belongs to no operation, including registered-but-unowned `/v1/images/segmentations`), `route_mismatch` (the path belongs to another operation), `unmapped_stage` (no such operation upstream — `asr`, `music`), `unknown_profile` (operation known, profile id unpublished), `operation_mismatch` (profile known under another operation), `no_local_capability` (profile supported, no id here). All six land in `unresolved` with `operation` / `profile` / `route` / `reason`, never dropped. -- `domain/omni_capabilities.py` in #266 owns the discovery document and matching; this module owns only the `(stage, route) → capability` layer. Different file names, no overwrite. -- `Model.stage_ids` in #266 currently has no reader (tests only fill `['0']`). Either (a) delete the field, or (b) connect it: cross `stage_ids` with the routes of each operation profile to produce paired declarations. `declarations()` already reads the `models[].stage_ids` × `operations.*.profiles[].routes[]` shape, so (b) can use it as is; numeric ids surface as `unmapped_stage` instead of being guessed at. -- This module does not modify `domain/omni_capabilities.py`, `integrations/capability_probe.py`, or `application/model_availability.py`. +### Three capability states, and `missing` needs positive evidence + +`available` when a model's profile says `supported` and its operation is not `exposure=disabled`; `missing` only when a profile or operation says no (`support=unsupported` or `exposure=disabled`); `unknown` for everything else — failed request, absent field, `unknown` enum value, an untranslatable declaration, or a capability no profile can express. **"The service did not say" is not "the service cannot"**: no models, no operations, or a dropped `profiles` field yields `unknown`, never `missing`. + +### Routes without an operation + +`/v1/images/segmentations`, `/v1/videos/segmentations` and `/v1/videos/depth` are registered endpoints that no operation owns; upstream's docs state that "segmentation and depth estimation do not acquire one of these operations merely by being present in a registry" (`docs/serving/capabilities.md:41-43`). `image.segment`, `video.matting` and `video.motion_capture` therefore stay `unknown` with `no_upstream_profile` — the endpoint does not publish the ability, so it cannot be judged, and it must never read as `missing`. `audio.score` is `no_upstream_operation`; `video.first_frame`, `video.edit` and `image.cutout` are `parameter_only` (request fields of a published operation); `audio.separate` is `no_local_adapter`. + +### ASR: an upstream gap, not a missing row + +Upstream publishes no recognition operation and omni's router never registers `/v1/audio/transcriptions` (absent from `api_server.py`; its own recipe says the endpoint is "not currently available with `--omni`"). A caller naming `asr` gets `unmapped_stage` — the upstream vocabulary has no such operation, this table did not lose a row — and `audio.score` stays `unknown` (`no_upstream_operation`). `/v1/audio/transcriptions` is never read as `audio.score`. + +### Relationship to the V2 discovery contract (#266) + +This table is an input to #266, not a replacement and not a second schema: `domain/omni_capabilities.py` owns the discovery document and matching, this module owns only the `(operation, profile) → capability` layer, under a different file name. It no longer reads `Model.stage_ids` — that is topology, not a capability key — and `declarations()` reads only the published `models[].operations[].profiles[]` shape. It does not modify `domain/omni_capabilities.py`, `integrations/capability_probe.py`, `application/model_availability.py` or `src/**`. ### Feeding `requests_for()` -`application/model_availability.py` currently hard-codes profile names per `(kind, protocol)`, which is why ASR and score get an empty request list. With a table, the caller needs only the stage it executes and the path it posts to: +`application/model_availability.py` currently hard-codes profile names per `(kind, protocol)`; with a table the caller needs only the operation and profile it is about to use: ```python from opendirector.domain.capability_contract import capabilities_for -capabilities = capabilities_for('music', '/v1/audio/speech') # ('audio.music',) +capabilities_for('speech', 'lyrics_to_music') # ('audio.music',) +capabilities_for('speech', 'text_to_speech') # ('audio.speech',) ``` -`resolve(payload)` returns per-capability states, per-stage states and the `unresolved` list; `classified_capabilities()` returns the vocabulary in table order, so presentation stops sorting on its own. Whether to wire this into `requests_for()` is #266's decision; this contract only guarantees the lookup is paired, ordered and testable. +`resolve(payload)` returns per-capability states, `topology` (display only) and `unresolved`; `classified_capabilities()` returns the vocabulary in table order. Whether to wire this into `requests_for()` is #266's decision; this contract only guarantees the lookup is paired, ordered and testable. ### Fixtures and checks -`tests/fixtures/omni-capabilities/{full,partial,absent}.json` are the shared samples: `full` covers every pair in the table and both shared paths, `partial` covers mixed availability, a `stage_route_mismatch` and both unknown kinds, `absent` covers a 404, a missing field and an empty list. `tests/test_capability_contract.py` compares each tier against its expected block and separately covers shared paths not overwriting each other, `asr` not being `score`, `absent` being all `unknown`, table order as output order, and the `models[]` document shape. +`tests/fixtures/omni-capabilities/{full,partial,absent}.json` follow the real response shape — `stages[]` as `{id,type,depends_on}`, six complete `operations` per model each with `support`/`exposure`/`reason_code`/`profiles`/`execution`, profiles with `routes[]`/`required_inputs`/`outputs`; the `absent` tier omits fields the way `exclude_none=True` does instead of writing `null`; the `partial` tier covers `support=unsupported`, `exposure=disabled`, `route_mismatch`, `unknown_route`, `unknown_profile`, `operation_mismatch`, `unmapped_stage` and `no_local_capability`. + +Five regressions keep coverage honest: `test_real_shaped_body_resolves_non_empty_pairs` (a real body yields twelve pairs), `test_route_alone_cannot_distinguish_speech_profiles` (one path, three profiles, three answers), `test_stage_ids_do_not_collapse_capabilities` (identical `stage_ids`, independent states), `test_unrouted_capabilities_are_unknown_not_missing`, and `test_absent_fields_are_unknown` (absence is `unknown`; only an explicit `unsupported` is `missing`). ```bash -PYTHONPATH=packages/core/src python -m pytest tests/test_capability_contract.py -q +PYTHONPATH=packages/core/src:apps/api/src python -m pytest tests/test_capability_contract.py -q ``` diff --git a/packages/core/src/opendirector/domain/capability_contract.py b/packages/core/src/opendirector/domain/capability_contract.py index 0d4a2a75..d511c1cf 100644 --- a/packages/core/src/opendirector/domain/capability_contract.py +++ b/packages/core/src/opendirector/domain/capability_contract.py @@ -1,102 +1,171 @@ -"""The (stage, route) pairs the capability vocabulary understands. - -`GET /v1/omni/capabilities` (vllm-project/vllm-omni#10007) reports what a -deployment serves. This module is the one place that turns those names into -`opendirector.domain.capabilities` ids. It performs no HTTP and holds no -display text: `application.model_availability.requests_for()` consumes the -table, and the presentation layer renders the reason codes through the message -catalogs. - -Three rules keep the table honest: - -* The key is the pair, never the route alone. `/v1/audio/speech` carries both - `tts` and `music`; `/v1/audio/transcriptions` carries both `asr` and `score`. - A declared path without its stage cannot say which capability was meant. -* A capability no route can express stays `unknown` even when the deployment - answers, because it is a request parameter rather than an endpoint: - `video.first_frame` and `video.edit` are `/v1/videos` parameters, and - `image.cutout` is a `/v1/images/generations` parameter. -* A declaration that places no stage is `unknown`, never `missing`: "the - service did not say" is not "the service cannot". - -A declared pair this table cannot translate is reported instead of dropped, so -a caller can show why a stage is unavailable rather than silently hiding it. +"""The (operation, profile) pairs the upstream capability vocabulary publishes. + +`GET /v1/omni/capabilities` (vllm-project/vllm-omni#10007, head `d1104f8`) +reports what a deployment serves. This module is the one place that turns that +vocabulary into `opendirector.domain.capabilities` ids. It performs no HTTP and +holds no display text: a caller consumes the table, and the presentation layer +renders reason codes through the message catalogs. + +Upstream publishes exactly six operations and twelve profiles +(`vllm_omni/serving_capabilities.py:19-26`, `:43-54`):: + + chat text_response, image_response, audio_response + image_generation text_to_image + image_edit image_to_image + video_generation text_to_video, image_to_video, video_to_video + speech text_to_speech, lyrics_to_music, caption_to_audio + audio_generation text_to_audio + +Four rules keep the table honest. + +* The key is `(operation, profile)`, never the route, never the stage. + `/v1/audio/speech` is the single route of one operation and carries three + profiles with three different contracts: `text_to_speech` (voice), + `lyrics_to_music` (music) and `caption_to_audio` (sound effect). A path alone + cannot say which one was granted, and a deployment can grant one while + refusing the others (`tests/config/test_serving_capabilities.py:233,251`: + MOSS-SoundEffect serves `caption_to_audio` and reports `text_to_speech` + unsupported on the same path). +* `stages[]` is deployment topology, not a capability. It publishes only + `id` / `type` / `depends_on` (`protocol/capabilities.py:67-70`), the ids are + opaque numeric strings, and every model is handed the *same* full list + (`entrypoints/openai/serving_capabilities.py:400`). Keying capabilities on + stages would therefore give every model one shared state and no diagnosis; + this module returns the topology for display and never reads it to decide a + capability. +* A declaration that says nothing is `unknown`, never `missing`. Absent fields + are the normal way to say "not established" (`response_model_exclude_none=True`, + `api_server.py:2193`) and `unknown` is a first-class upstream value + (`support`, `exposure`, and "Missing operations/profiles are unknown", + `serving_capabilities.py:84`). Only positive evidence — `support=unsupported` + or `exposure=disabled` for the profile — produces `missing`. +* A route without an operation proves nothing. `/v1/images/segmentations`, + `/v1/videos/segmentations` and `/v1/videos/depth` are registered endpoints + that no operation owns (`docs/serving/capabilities.md:41-43`): "segmentation + and depth estimation do not acquire one of these operations merely by being + present in a registry". Those capabilities stay `unknown` with + `no_upstream_profile`. + +`asr` is not an omission on this side: upstream publishes no recognition +operation at all and the omni router never registers `/v1/audio/transcriptions`, +so a caller naming it gets `unmapped_stage` — an upstream gap to be recorded, +not a row we forgot to add, and never silently read as `audio.score`. + +Not yet confirmed upstream (recorded here instead of assumed): the exact +`observed_at` string format, whether `stages[].type` is ever the `"unknown"` +fallback, and whether a multi-model deployment can differentiate `operations` +per model. This module reads only what a body actually states, so those gaps +stay `unknown` rather than becoming policy. """ ENDPOINT = '/v1/omni/capabilities' API_PREFIX = '/v1' -# (upstream stage, upstream route, capability ids that pair can serve). Order is -# part of the contract: `resolve()` emits capabilities in this order, so a -# presentation change is a diff in one place. The routes are the ones this -# repository's adapters already call, so a declaration published by the service -# and a request we would send name the same path. -STAGE_ROUTES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( - ('text', '/v1/chat/completions', ('text.chat',)), - ('image', '/v1/images/generations', ('image.generate',)), - ('image', '/v1/images/edits', ('image.edit',)), - ('image', '/v1/images/segmentations', ('image.segment',)), - ('video', '/v1/videos', ('video.generate',)), - ('video', '/v1/videos/segmentations', ('video.matting',)), - ('video', '/v1/videos/depth', ('video.motion_capture',)), - ('tts', '/v1/audio/speech', ('audio.speech',)), - ('music', '/v1/audio/speech', ('audio.music',)), - ('score', '/v1/audio/transcriptions', ('audio.score',)), +# Upstream's closed operation vocabulary, in its published order. +# (`vllm_omni/serving_capabilities.py:15,19-26`) +OPERATIONS: tuple[str, ...] = ( + 'chat', + 'image_generation', + 'image_edit', + 'video_generation', + 'speech', + 'audio_generation', ) -# `asr` is deliberately absent: recognition has no capability id yet, so -# `/v1/audio/transcriptions` under `asr` is reported as `unmapped_stage` -# instead of being read as `audio.score`. +# (operation, profile, capability ids that profile grants). Order is part of the +# contract: `resolve()` emits capabilities in this order, so a presentation +# change is a diff in one place. Every profile upstream publishes has a row, +# including the two for which this repository has no capability id yet: an empty +# tuple is the honest answer ("upstream says it can, we cannot name it"), and a +# supported declaration of such a profile is reported as `no_local_capability` +# rather than dropped or force-fitted to a neighbouring id. +# +# `caption_to_audio` is a sound-effect contract (upstream requires +# `ambient_sound`, `tests/config/test_serving_capabilities.py:233`), not speech; +# `text_to_audio` is the separate diffusion contract behind +# `/v1/audio/generate` (no voice, no lyrics; `docs/serving/capabilities.md:59-60`). +# Mapping either onto `audio.speech` would let a request carry a voice the +# deployment never accepted. +OPERATION_PROFILES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( + ('chat', 'text_response', ('text.chat',)), + ('chat', 'image_response', ('text.chat',)), + ('chat', 'audio_response', ('text.chat',)), + ('image_generation', 'text_to_image', ('image.generate',)), + ('image_edit', 'image_to_image', ('image.edit',)), + ('video_generation', 'text_to_video', ('video.generate',)), + ('video_generation', 'image_to_video', ('video.generate',)), + ('video_generation', 'video_to_video', ('video.generate',)), + ('speech', 'text_to_speech', ('audio.speech',)), + ('speech', 'lyrics_to_music', ('audio.music',)), + ('speech', 'caption_to_audio', ()), + ('audio_generation', 'text_to_audio', ()), +) -# Capability ids no upstream route can express, with the reason to show. Values -# stay stable business codes; wording lives in the message catalogs. -UNROUTED: dict[str, str] = { - 'video.first_frame': 'parameter_only', - 'video.edit': 'parameter_only', - 'image.cutout': 'parameter_only', - 'audio.separate': 'no_local_adapter', +# operation → the routes that operation owns (`serving_capabilities.py:42-54`). +# Kept beside the profile table so a declared profile can be checked against the +# operation it claims: a profile pointing at another operation's path is a +# declaration error, not a capability. `/v1/realtime/video` is a websocket route, +# compared by path here because the capability question is about the endpoint. +# Routes registered but owned by no operation (`/v1/audio/speech/batch`, +# `/v1/chat/completions/batch`, the `segmentations`/`depth` endpoints, …) belong +# to no value of this map, which is exactly why they cannot grant anything. +OPERATION_ROUTES: dict[str, tuple[str, ...]] = { + 'chat': ('/v1/chat/completions',), + 'image_generation': ('/v1/images/generations',), + 'image_edit': ('/v1/images/edits',), + 'video_generation': ('/v1/videos', '/v1/videos/sync', '/v1/realtime/video'), + 'speech': ('/v1/audio/speech',), + 'audio_generation': ('/v1/audio/generate',), } AVAILABLE, MISSING, UNKNOWN = 'available', 'missing', 'unknown' ABSENT, PRESENT = 'absent', 'present' -PARTIAL, UNMAPPED = 'partial', 'unmapped' - -# Why a declared pair could not be translated. Three distinct cases, because -# they lead to different fixes: add the route to the table, add the stage to the -# table, or correct a deployment that paired a known stage with another stage's -# route. -UNKNOWN_ROUTE, UNMAPPED_STAGE, STAGE_ROUTE_MISMATCH = 'unknown_route', 'unmapped_stage', 'stage_route_mismatch' -REASONS = frozenset({UNKNOWN_ROUTE, UNMAPPED_STAGE, STAGE_ROUTE_MISMATCH}) -PARAMETER_ONLY, NO_LOCAL_ADAPTER = 'parameter_only', 'no_local_adapter' -UNROUTED_REASONS = frozenset({PARAMETER_ONLY, NO_LOCAL_ADAPTER}) - -KNOWN_STAGES = frozenset(stage for stage, _, _ in STAGE_ROUTES) -KNOWN_ROUTES = frozenset(route for _, route, _ in STAGE_ROUTES) -BY_PAIR: dict[tuple[str, str], tuple[str, ...]] = {(stage, route): capabilities - for stage, route, capabilities in STAGE_ROUTES} - - -def named(value): - """Normalized stage name from a string or an object carrying `name`/`stage`.""" - if isinstance(value, dict): - value = value.get('name', value.get('stage')) - return value.strip().lower() if isinstance(value, str) else '' +# Upstream's three-valued vocabulary, shared by `support` and `exposure` +# (`serving_capabilities.py:16`, `protocol/capabilities.py:13`). +SUPPORTED, UNSUPPORTED = 'supported', 'unsupported' +ENABLED, DISABLED = 'enabled', 'disabled' +# `unknown` is the third value of both, and also the state of an absent field. + +# Why a declaration could not be translated. Each names a different fix, so a +# reviewer can tell an upstream gap from a bug in this table. +UNKNOWN_ROUTE = 'unknown_route' # the path belongs to no operation at all +ROUTE_MISMATCH = 'route_mismatch' # the path belongs to a different operation +UNMAPPED_STAGE = 'unmapped_stage' # no such operation upstream (e.g. `asr`) +UNKNOWN_PROFILE = 'unknown_profile' # operation known, profile id not published +OPERATION_MISMATCH = 'operation_mismatch' # profile known under another operation +NO_LOCAL_CAPABILITY = 'no_local_capability' # upstream profile, no id here +REASONS = frozenset({UNKNOWN_ROUTE, ROUTE_MISMATCH, UNMAPPED_STAGE, UNKNOWN_PROFILE, + OPERATION_MISMATCH, NO_LOCAL_CAPABILITY}) + +# Why a capability can never be decided from a declaration here. +NO_UPSTREAM_PROFILE, NO_UPSTREAM_OPERATION = 'no_upstream_profile', 'no_upstream_operation' +PARAMETER_ONLY, NO_LOCAL_ADAPTER = 'parameter_only', 'no_local_adapter' +UNROUTED_REASONS = frozenset({NO_UPSTREAM_PROFILE, NO_UPSTREAM_OPERATION, PARAMETER_ONLY, NO_LOCAL_ADAPTER}) -def classified_capabilities(): - """Every capability id this contract places, in table order, without repeats. +# Capability ids no upstream profile can express, with the reason to show. Values +# stay stable business codes; wording lives in the message catalogs. Every entry +# here resolves to `unknown`: the endpoint does not publish the ability, so +# "cannot judge" is the only defensible state. +UNROUTED: dict[str, str] = { + # Endpoints exist, but no operation owns them (see the module docstring). + 'image.segment': NO_UPSTREAM_PROFILE, + 'video.matting': NO_UPSTREAM_PROFILE, + 'video.motion_capture': NO_UPSTREAM_PROFILE, + # Upstream publishes no transcription operation at all. + 'audio.score': NO_UPSTREAM_OPERATION, + # Request fields of an operation that does publish a profile. + 'video.first_frame': PARAMETER_ONLY, + 'video.edit': PARAMETER_ONLY, + 'image.cutout': PARAMETER_ONLY, + # No adapter here offers it; upstream says nothing either way. + 'audio.separate': NO_LOCAL_ADAPTER, +} - Routed capabilities come first in the order their pairs appear; unrouted - ones follow in declaration order. Callers that render a list use this order - instead of sorting, so the canvas and the contract cannot disagree. - """ - ordered = [] - for _, _, capabilities in STAGE_ROUTES: - for capability in capabilities: - if capability not in ordered: - ordered.append(capability) - ordered.extend(capability for capability in UNROUTED if capability not in ordered) - return tuple(ordered) +BY_PAIR: dict[tuple[str, str], tuple[str, ...]] = {(operation, profile): capabilities + for operation, profile, capabilities in OPERATION_PROFILES} +PROFILE_OPERATIONS: dict[str, str] = {profile: operation for operation, profile, _ in OPERATION_PROFILES} +ROUTED_PATHS = frozenset(path for paths in OPERATION_ROUTES.values() for path in paths) def route_path(value): @@ -105,7 +174,7 @@ def route_path(value): Upstream may report `/v1/audio/speech`, `audio/speech` or an object carrying `path`; the base URL already contains `/v1`, so the prefix is normalized here once instead of at every comparison. Query strings are dropped: a - version parameter does not change which capability a path serves. + version parameter does not change which operation owns a path. """ if isinstance(value, dict): value = value.get('path') @@ -122,115 +191,241 @@ def route_path(value): return API_PREFIX + path -def capabilities_for(stage, route): +def capabilities_for(operation, profile): """Capability ids for one pair, or `()` when this table does not place it. - This is the single lookup `requests_for()` needs: given the stage it is - executing and the path it is about to post to, it gets the capabilities the - deployment granted and never has to know profile names. + This is the single lookup a caller needs: given the operation it believes it + is calling and the profile it intends to use, it gets the capabilities that + profile grants. `()` means either "no such row" or "upstream publishes the + profile but this repository has no id for it" — `resolve()` distinguishes + those two cases in its `unresolved` list. """ - path = route_path(route) - if path is None: + if not isinstance(operation, str) or not isinstance(profile, str): return () - return BY_PAIR.get((named(stage), path), ()) + return BY_PAIR.get((operation.strip().lower(), profile.strip().lower()), ()) -def _stage_pairs(stages): - """Pairs from a `stages: [{name, routes: [...]}]` body, in document order.""" - if not isinstance(stages, list): - return - for entry in stages: - stage = named(entry) - routes = entry.get('routes') if isinstance(entry, dict) else None - if not stage or not isinstance(routes, list): - continue - for route in routes: - path = route_path(route) - if path: - yield stage, path +def classified_capabilities(): + """Every capability id this contract places, in table order, without repeats. + Routed capabilities come first in the order their rows appear; unrouted ones + follow in declaration order. Callers that render a list use this order + instead of sorting, so the canvas and the contract cannot disagree. + """ + ordered = [] + for _, _, capabilities in OPERATION_PROFILES: + for capability in capabilities: + if capability not in ordered: + ordered.append(capability) + ordered.extend(capability for capability in UNROUTED if capability not in ordered) + return tuple(ordered) -def _model_pairs(models): - """Pairs from a `models: [{stage_ids, operations}]` body, in document order. - The discovery document this repository parses today names routes per - operation and profile, and the stages per model. Cross-producting them is - what makes such a document readable as (stage, route) pairs; when - `stage_ids` holds opaque ids the pairs surface as unmapped stages instead of - being guessed at. +def _routes(entry): + """Declared route references of a profile; anything malformed is ignored.""" + routes = entry.get('routes') + return routes if isinstance(routes, list) else [] + + +def _profiles(operation): + """`(profile id, profile entry)` pairs a declared operation body carries.""" + if not isinstance(operation, dict): + return + profiles = operation.get('profiles') + for entry in profiles if isinstance(profiles, list) else []: + profile_id = entry.get('id') if isinstance(entry, dict) else None + if isinstance(profile_id, str) and profile_id.strip(): + yield profile_id.strip().lower(), entry + + +def declarations(payload): + """Ordered (operation, profile) pairs from a capabilities body, or None. + + Only the published shape is read: `models[].operations[].profiles[]`. + `stages[]` is deliberately skipped — it is topology, and reading routes or + names out of it is how the earlier table grew pairs upstream never published. + A body that declares no profile returns None, which callers must treat as + "no evidence" rather than "nothing supported". (`resolve()` still reports + `endpoint=present` for a body that carries operation entries without + profiles: the service answered, it just did not say which profiles it + serves, and that is `unknown`, not `absent`.) """ + if not isinstance(payload, dict): + return None + models = payload.get('models') if not isinstance(models, list): - return + return None + pairs: list[tuple[str, str]] = [] for model in models: - stages = [stage for stage in map(named, model.get('stage_ids') or ()) if stage] \ - if isinstance(model, dict) else [] operations = model.get('operations') if isinstance(model, dict) else None - if not stages or not isinstance(operations, dict): + if not isinstance(operations, dict): continue - for operation in operations.values(): - profiles = operation.get('profiles') if isinstance(operation, dict) else None - if not isinstance(profiles, list): + for operation_id, operation in operations.items(): + if not isinstance(operation_id, str) or not operation_id.strip(): continue - for profile in profiles: - routes = profile.get('routes') if isinstance(profile, dict) else None - if not isinstance(routes, list): - continue - for route in routes: - path = route_path(route) - if not path: - continue - for stage in stages: - yield stage, path + for profile_id, _ in _profiles(operation): + pair = (operation_id.strip().lower(), profile_id) + if pair not in pairs: + pairs.append(pair) + return pairs or None -def declarations(payload): - """Ordered (stage, route) pairs from a capabilities body, or None when it declares none. +def _topology(payload): + """Deployment topology as declared: `[{id, type, depends_on}]`, display only. - Both published shapes are read: a `stages` list is taken as written, and a - `models` list is expanded through `stage_ids` and its operation profiles. A - body that declares neither returns None, which callers must treat as "no - evidence" rather than "nothing supported". + Never consulted for a capability decision. A stage entry that happens to + carry routes or names is ignored on purpose: those keys do not exist in + upstream's `CapabilityStage` (`protocol/capabilities.py:67-70`), so honouring + them would resurrect the stage-keyed reading this module exists to replace. + `type` falls back to `unknown` the way upstream's own projection does + (`entrypoints/openai/serving_capabilities.py:86`). """ - if not isinstance(payload, dict): - return None - pairs = [] - for pair in (*_stage_pairs(payload.get('stages')), *_model_pairs(payload.get('models'))): - if pair not in pairs: - pairs.append(pair) - return pairs or None + stages = payload.get('stages') if isinstance(payload, dict) else None + topology = [] + for entry in stages if isinstance(stages, list) else []: + stage_id = entry.get('id') if isinstance(entry, dict) else None + if stage_id is None: + continue + depends_on = entry.get('depends_on') + stage_type = entry.get('type') + topology.append({ + 'id': str(stage_id), + 'type': stage_type if isinstance(stage_type, str) and stage_type else UNKNOWN, + 'depends_on': [str(item) for item in depends_on] if isinstance(depends_on, list) else [], + }) + return topology + + +def _operation_verdict(operation): + """`yes` / `no` / `unknown` for an operation from its own two enums. + + Upstream keeps `support` and `exposure` independent: a supported operation + can have disabled routes, and an installed route can coexist with an + unsupported operation (`docs/serving/capabilities.md:45-51`). Either one + saying no is a definitive no for the deployment as it stands; anything less + than both saying yes is not enough to claim availability. + """ + support, exposure = operation.get('support'), operation.get('exposure') + if support == UNSUPPORTED or exposure == DISABLED: + return 'no' + if support == SUPPORTED and exposure == ENABLED: + return 'yes' + return 'unknown' + + +def _profile_verdict(entry, operation_verdict): + """`yes` / `no` / `unknown` for one profile inside an already-judged operation.""" + if operation_verdict == 'no': + return 'no' + support = entry.get('support') + if support == UNSUPPORTED: + return 'no' + if support != SUPPORTED or operation_verdict != 'yes': + return 'unknown' + return 'yes' + + +def _note(unresolved, operation, profile, route, reason): + """Append a translation failure once; models repeat the same declarations.""" + entry = {'operation': operation, 'profile': profile, 'route': route, 'reason': reason} + if entry not in unresolved: + unresolved.append(entry) + + +def _check_route(operation, profile, path, unresolved): + """True when `path` is one the operation owns; otherwise report why not.""" + if path in OPERATION_ROUTES[operation]: + return True + _note(unresolved, operation, profile, path, + ROUTE_MISMATCH if path in ROUTED_PATHS else UNKNOWN_ROUTE) + return False def resolve(payload): - """Per-capability availability, the per-stage view, and the unresolved pairs. + """Per-capability availability, the declared topology, and untranslated claims. `payload` is the decoded body of `ENDPOINT`, or None when the request failed (404, timeout, disabled endpoint). Model names and any other provider fields never reach the result. + + `endpoint` is `absent` when the body carries no operation entry at all, and + then every capability is `unknown` and nothing is listed as unresolved. + + A capability is `available` only when some model's profile says so and its + operation is enabled; `missing` only when a profile or operation positively + says no; `unknown` otherwise — no body, an absent field, an unknown enum, an + unrouted capability, or a declaration this table could not translate. """ vocabulary = classified_capabilities() - pairs = declarations(payload) - if not pairs: - return {'endpoint': ABSENT, 'stages': {}, - 'capabilities': {item: UNKNOWN for item in vocabulary}, 'unresolved': []} - granted, unresolved, stage_flags = set(), [], {} - for stage, route in pairs: - capabilities = capabilities_for(stage, route) - flags = stage_flags.setdefault(stage, []) - if capabilities: - granted.update(capabilities) - flags.append(True) + unresolved: list[dict] = [] + topology = _topology(payload) + evidence: dict[str, set[str]] = {} + + def record(capabilities, verdict): + for capability in capabilities: + evidence.setdefault(capability, set()).add(verdict) + + models = payload.get('models') if isinstance(payload, dict) else None + declared = False + for model in models if isinstance(models, list) else []: + operations = model.get('operations') if isinstance(model, dict) else None + if not isinstance(operations, dict) or not operations: continue - flags.append(False) - if route not in KNOWN_ROUTES: - reason = UNKNOWN_ROUTE - elif stage not in KNOWN_STAGES: - reason = UNMAPPED_STAGE + declared = True + for operation_id, operation in operations.items(): + key = operation_id.strip().lower() if isinstance(operation_id, str) else '' + if not key: + continue + if key not in OPERATIONS: + _note(unresolved, key, None, None, UNMAPPED_STAGE) + continue + if not isinstance(operation, dict): + for row_operation, _, row_capabilities in OPERATION_PROFILES: + if row_operation == key: + record(row_capabilities, 'unknown') + continue + operation_verdict = _operation_verdict(operation) + seen = set() + for profile_id, entry in _profiles(operation): + seen.add(profile_id) + owner = PROFILE_OPERATIONS.get(profile_id) + if owner is None: + _note(unresolved, key, profile_id, None, UNKNOWN_PROFILE) + continue + if owner != key: + _note(unresolved, key, profile_id, None, OPERATION_MISMATCH) + continue + paths = [path for path in map(route_path, _routes(entry)) if path] + broken = [path for path in paths if not _check_route(key, profile_id, path, unresolved)] + verdict = _profile_verdict(entry, operation_verdict) + capabilities = BY_PAIR[(key, profile_id)] + # Without a route we cannot confirm reachability, and a profile + # pointing at another operation's path tells us nothing reliable. + if verdict != 'no' and (broken or not paths): + verdict = 'unknown' + if capabilities: + record(capabilities, verdict) + elif verdict == 'yes': + _note(unresolved, key, profile_id, None, NO_LOCAL_CAPABILITY) + # A profile the operation never mentions establishes nothing, unless + # the operation itself refused (`declare_capabilities` fills omitted + # standard profiles with unsupported upstream, so silence is silence). + for row_operation, profile_id, row_capabilities in OPERATION_PROFILES: + if row_operation == key and profile_id not in seen: + record(row_capabilities, 'no' if operation_verdict == 'no' else 'unknown') + + if not declared: + return {'endpoint': ABSENT, 'topology': topology, + 'capabilities': {item: UNKNOWN for item in vocabulary}, 'unresolved': []} + capabilities = {} + for item in vocabulary: + verdicts = evidence.get(item, set()) + if 'yes' in verdicts: + capabilities[item] = AVAILABLE + elif item in UNROUTED or 'unknown' in verdicts or not verdicts: + capabilities[item] = UNKNOWN else: - reason = STAGE_ROUTE_MISMATCH - unresolved.append({'stage': stage, 'route': route, 'reason': reason}) - stages = {stage: AVAILABLE if all(flags) else PARTIAL if any(flags) else UNMAPPED - for stage, flags in sorted(stage_flags.items())} - capabilities = {item: AVAILABLE if item in granted else UNKNOWN if item in UNROUTED else MISSING - for item in vocabulary} - return {'endpoint': PRESENT, 'stages': stages, 'capabilities': capabilities, - 'unresolved': sorted(unresolved, key=lambda item: (item['stage'], item['route']))} + capabilities[item] = MISSING + return {'endpoint': PRESENT, 'topology': topology, 'capabilities': capabilities, + 'unresolved': sorted(unresolved, key=lambda item: (item['operation'] or '', item['profile'] or '', + item['route'] or '', item['reason']))} diff --git a/tests/fixtures/omni-capabilities/absent.json b/tests/fixtures/omni-capabilities/absent.json index f7811dd6..63aa799c 100644 --- a/tests/fixtures/omni-capabilities/absent.json +++ b/tests/fixtures/omni-capabilities/absent.json @@ -1,25 +1,25 @@ { "tier": "absent", "upstreamEndpoint": "/v1/omni/capabilities", - "scope": "No declaration: every capability is unknown, so callers must fall back instead of degrading.", + "scope": "No usable capability evidence: an unreachable endpoint, a topology-only body, and a body whose optional fields were dropped by exclude_none.", "cases": [ { - "id": "no-endpoint-404", - "note": "GET returns 404 because the deployment predates the endpoint.", + "id": "endpoint-unreachable", + "note": "404, timeout or a disabled endpoint: nothing is claimed, nothing is missing.", "payload": null, "expected": { "endpoint": "absent", - "stages": {}, + "topology": [], "capabilities": { "text.chat": "unknown", "image.generate": "unknown", "image.edit": "unknown", - "image.segment": "unknown", "video.generate": "unknown", - "video.matting": "unknown", - "video.motion_capture": "unknown", "audio.speech": "unknown", "audio.music": "unknown", + "image.segment": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", "audio.score": "unknown", "video.first_frame": "unknown", "video.edit": "unknown", @@ -30,24 +30,122 @@ } }, { - "id": "no-stages-field", - "note": "The endpoint answers but the body carries no `stages` field.", + "id": "topology-only-body", + "note": "Stages are published but no model and no operation: topology is display, so it grants nothing.", "payload": { - "object": "omni.capabilities" + "object": "omni.capabilities", + "schema_version": 1, + "observed_at": "2026-11-10T00:00:00Z", + "stages": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], + "models": [], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/depth", + "exposure": "enabled" + } + ] }, "expected": { "endpoint": "absent", - "stages": {}, + "topology": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], "capabilities": { "text.chat": "unknown", "image.generate": "unknown", "image.edit": "unknown", - "image.segment": "unknown", "video.generate": "unknown", - "video.matting": "unknown", - "video.motion_capture": "unknown", "audio.speech": "unknown", "audio.music": "unknown", + "image.segment": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", "audio.score": "unknown", "video.first_frame": "unknown", "video.edit": "unknown", @@ -58,25 +156,141 @@ } }, { - "id": "empty-stages", - "note": "The endpoint answers with an empty stage list, which declares nothing.", + "id": "operations-without-profiles", + "note": "The operation answered but `profiles` was dropped: an absent field means unknown, not unsupported.", "payload": { "object": "omni.capabilities", - "stages": [] + "schema_version": 1, + "observed_at": "2026-11-10T00:00:00Z", + "stages": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], + "models": [ + { + "id": "model-d", + "stage_ids": [ + "7" + ], + "operations": { + "chat": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + } + } + } + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/depth", + "exposure": "enabled" + } + ] }, "expected": { - "endpoint": "absent", - "stages": {}, + "endpoint": "present", + "topology": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], "capabilities": { "text.chat": "unknown", "image.generate": "unknown", "image.edit": "unknown", - "image.segment": "unknown", "video.generate": "unknown", - "video.matting": "unknown", - "video.motion_capture": "unknown", "audio.speech": "unknown", "audio.music": "unknown", + "image.segment": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", "audio.score": "unknown", "video.first_frame": "unknown", "video.edit": "unknown", diff --git a/tests/fixtures/omni-capabilities/full.json b/tests/fixtures/omni-capabilities/full.json index 2e5c193a..e15058f0 100644 --- a/tests/fixtures/omni-capabilities/full.json +++ b/tests/fixtures/omni-capabilities/full.json @@ -1,141 +1,487 @@ { "tier": "full", "upstreamEndpoint": "/v1/omni/capabilities", - "scope": "Full declaration: the deployment lists every stage and route this contract knows.", + "scope": "A deployment that answers yes to every operation and profile upstream publishes, on a two-stage llm+diffusion topology.", "cases": [ { - "id": "every-known-stage", - "note": "Every (stage, route) pair this contract knows is declared; model names are synthetic.", + "id": "every-operation-and-profile", + "note": "All six operations supported and enabled, all twelve profiles declared; `text_to_audio` is supported but this repository has no id for it.", "payload": { "object": "omni.capabilities", + "schema_version": 1, + "observed_at": "2026-11-10T00:00:00Z", "stages": [ { - "name": "text", - "model": "chat-model", - "routes": [ - "/v1/chat/completions" - ] + "id": "7", + "type": "llm", + "depends_on": [] }, { - "name": "image", - "model": "image-model", - "routes": [ - { - "path": "/v1/images/generations", - "methods": [ - "POST" - ] - }, - "/v1/images/edits", - "/v1/images/segmentations" + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" ] + } + ], + "models": [ + { + "id": "model-a", + "stage_ids": [ + "7", + "42" + ], + "operations": { + "chat": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_response", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "text" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "image_response", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "text", + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "audio_response", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "text", + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_image", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_edit": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "image_to_image", + "support": "supported", + "required_inputs": [ + "image", + "text" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "video_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_video", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "image_to_video", + "support": "supported", + "required_inputs": [ + "image", + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "video_to_video", + "support": "supported", + "required_inputs": [ + "video", + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "speech": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_speech", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "lyrics_to_music", + "support": "supported", + "required_inputs": [ + "text", + "instructions" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "caption_to_audio", + "support": "unsupported", + "required_inputs": [], + "outputs": [], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_not_supported", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "audio_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_audio", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + } + } + } + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" }, { - "name": "video", - "model": "video-model", - "routes": [ - "/v1/videos", - "/v1/videos/segmentations", - "/v1/videos/depth" - ] + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" }, { - "name": "tts", - "model": "tts-model", - "routes": [ - "/v1/audio/speech" - ] + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" }, { - "name": "music", - "model": "music-model", - "routes": [ - "/v1/audio/speech" - ] + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" }, { - "name": "score", - "model": "score-model", - "routes": [ - "/v1/audio/transcriptions?version=1" - ] + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/depth", + "exposure": "enabled" } ] }, "expected": { "endpoint": "present", - "stages": { - "image": "available", - "music": "available", - "score": "available", - "text": "available", - "tts": "available", - "video": "available" - }, + "topology": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], "capabilities": { "text.chat": "available", "image.generate": "available", "image.edit": "available", - "image.segment": "available", "video.generate": "available", - "video.matting": "available", - "video.motion_capture": "available", "audio.speech": "available", "audio.music": "available", - "audio.score": "available", + "image.segment": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", + "audio.score": "unknown", "video.first_frame": "unknown", "video.edit": "unknown", "image.cutout": "unknown", "audio.separate": "unknown" }, - "unresolved": [] - } - }, - { - "id": "shared-speech-route", - "note": "`tts` and `music` declare the same `/v1/audio/speech` path; both keep their own capability.", - "payload": { - "object": "omni.capabilities", - "stages": [ - { - "name": "tts", - "model": "tts-model", - "routes": [ - "/v1/audio/speech" - ] - }, + "unresolved": [ { - "name": "music", - "model": "music-model", - "routes": [ - "/v1/audio/speech" - ] + "operation": "audio_generation", + "profile": "text_to_audio", + "route": null, + "reason": "no_local_capability" } ] - }, - "expected": { - "endpoint": "present", - "stages": { - "music": "available", - "tts": "available" - }, - "capabilities": { - "text.chat": "missing", - "image.generate": "missing", - "image.edit": "missing", - "image.segment": "missing", - "video.generate": "missing", - "video.matting": "missing", - "video.motion_capture": "missing", - "audio.speech": "available", - "audio.music": "available", - "audio.score": "missing", - "video.first_frame": "unknown", - "video.edit": "unknown", - "image.cutout": "unknown", - "audio.separate": "unknown" - }, - "unresolved": [] } } ] diff --git a/tests/fixtures/omni-capabilities/partial.json b/tests/fixtures/omni-capabilities/partial.json index 1790eb41..50c88c4b 100644 --- a/tests/fixtures/omni-capabilities/partial.json +++ b/tests/fixtures/omni-capabilities/partial.json @@ -1,73 +1,430 @@ { "tier": "partial", "upstreamEndpoint": "/v1/omni/capabilities", - "scope": "Partial declaration: some stages resolve, some do not, and `asr` must not be read as `audio.score`.", + "scope": "Bodies that answer, but disagree with themselves or with the vocabulary: unsupported, disabled, a misplaced route, an omitted field, unknown names.", "cases": [ { - "id": "mixed-stages", - "note": "One complete stage, one partly understood stage, and three stages this vocabulary cannot express.", + "id": "mixed-operations", + "note": "`image_generation` is supported but disabled, `image_edit` unsupported, one video profile declares an image path (route_mismatch), one profile omits `support`, `caption_to_audio` is supported with no local id.", "payload": { "object": "omni.capabilities", + "schema_version": 1, + "observed_at": "2026-11-10T00:00:00Z", "stages": [ { - "name": "tts", - "model": "tts-model", - "routes": [ - "/v1/audio/speech" - ] + "id": "7", + "type": "llm", + "depends_on": [] }, { - "name": "video", - "model": "video-model", - "routes": [ - "/v1/videos", - "/v1/videos/depth", - "/v1/videos/track" + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" ] + } + ], + "models": [ + { + "id": "model-b", + "stage_ids": [ + "7" + ], + "operations": { + "chat": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_response", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "text" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "image_response", + "support": "unsupported", + "required_inputs": [], + "outputs": [], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "chat_prompt_contract_undeclared", + "constraints": {} + }, + { + "id": "audio_response", + "required_inputs": [ + "text" + ], + "outputs": [ + "text", + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "request_contract_unverified", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_generation": { + "support": "supported", + "exposure": "disabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_image", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_edit": { + "support": "unsupported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "image_to_image", + "support": "unsupported", + "required_inputs": [], + "outputs": [], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + } + ], + "reason_code": "model_not_supported", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "video_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_video", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "image_to_video", + "support": "supported", + "required_inputs": [ + "image", + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "video_to_video", + "support": "unsupported", + "required_inputs": [], + "outputs": [], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + } + ], + "reason_code": "checkpoint_requires_expand_timesteps", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "speech": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_speech", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "lyrics_to_music", + "support": "unsupported", + "required_inputs": [], + "outputs": [], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_not_supported", + "constraints": {} + }, + { + "id": "caption_to_audio", + "support": "supported", + "required_inputs": [ + "ambient_sound" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "audio_generation": { + "reason_code": "request_contract_unverified", + "profiles": [ + { + "id": "text_to_audio", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + } + } + } + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" }, { - "name": "asr", - "model": "asr-model", - "routes": [ - "/v1/audio/transcriptions" - ] + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" }, { - "name": "image", - "model": "image-model", - "routes": [ - "/v1/videos" - ] + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" }, { - "name": "embedding", - "model": "embedding-model", - "routes": [ - "/v1/embeddings" - ] + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/depth", + "exposure": "enabled" } ] }, "expected": { "endpoint": "present", - "stages": { - "asr": "unmapped", - "embedding": "unmapped", - "image": "unmapped", - "tts": "available", - "video": "partial" - }, + "topology": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], "capabilities": { - "text.chat": "missing", + "text.chat": "available", "image.generate": "missing", "image.edit": "missing", - "image.segment": "missing", "video.generate": "available", - "video.matting": "missing", - "video.motion_capture": "available", "audio.speech": "available", "audio.music": "missing", - "audio.score": "missing", + "image.segment": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", + "audio.score": "unknown", "video.first_frame": "unknown", "video.edit": "unknown", "image.cutout": "unknown", @@ -75,67 +432,276 @@ }, "unresolved": [ { - "stage": "asr", - "route": "/v1/audio/transcriptions", - "reason": "unmapped_stage" - }, - { - "stage": "embedding", - "route": "/v1/embeddings", - "reason": "unknown_route" + "operation": "speech", + "profile": "caption_to_audio", + "route": null, + "reason": "no_local_capability" }, { - "stage": "image", - "route": "/v1/videos", - "reason": "stage_route_mismatch" - }, - { - "stage": "video", - "route": "/v1/videos/track", - "reason": "unknown_route" + "operation": "video_generation", + "profile": "text_to_video", + "route": "/v1/images/edits", + "reason": "route_mismatch" } ] } }, { - "id": "shared-transcription-route", - "note": "`score` and `asr` declare the same `/v1/audio/transcriptions` path; only `score` grants a capability.", + "id": "unknown-vocabulary", + "note": "`asr` is not an upstream operation, `speech_to_speech` is not a profile, `lyrics_to_music` is filed under the wrong operation, and `/v1/videos/track` belongs to no operation at all.", "payload": { "object": "omni.capabilities", + "schema_version": 1, + "observed_at": "2026-11-10T00:00:00Z", "stages": [ { - "name": "score", - "model": "score-model", - "routes": [ - "/v1/audio/transcriptions" - ] + "id": "7", + "type": "llm", + "depends_on": [] }, { - "name": "asr", - "model": "asr-model", - "routes": [ - "audio/transcriptions" + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" ] } + ], + "models": [ + { + "id": "model-c", + "stage_ids": [ + "7" + ], + "operations": { + "asr": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "speech": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_speech", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "speech_to_speech", + "support": "supported", + "required_inputs": [ + "audio", + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "audio_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "lyrics_to_music", + "support": "supported", + "required_inputs": [ + "text", + "instructions" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "video_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_video", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/track", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + } + } + } + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/depth", + "exposure": "enabled" + } ] }, "expected": { "endpoint": "present", - "stages": { - "asr": "unmapped", - "score": "available" - }, + "topology": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], "capabilities": { - "text.chat": "missing", - "image.generate": "missing", - "image.edit": "missing", - "image.segment": "missing", - "video.generate": "missing", - "video.matting": "missing", - "video.motion_capture": "missing", - "audio.speech": "missing", - "audio.music": "missing", - "audio.score": "available", + "text.chat": "unknown", + "image.generate": "unknown", + "image.edit": "unknown", + "video.generate": "unknown", + "audio.speech": "available", + "audio.music": "unknown", + "image.segment": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", + "audio.score": "unknown", "video.first_frame": "unknown", "video.edit": "unknown", "image.cutout": "unknown", @@ -143,9 +709,28 @@ }, "unresolved": [ { - "stage": "asr", - "route": "/v1/audio/transcriptions", + "operation": "asr", + "profile": null, + "route": null, "reason": "unmapped_stage" + }, + { + "operation": "audio_generation", + "profile": "lyrics_to_music", + "route": null, + "reason": "operation_mismatch" + }, + { + "operation": "speech", + "profile": "speech_to_speech", + "route": null, + "reason": "unknown_profile" + }, + { + "operation": "video_generation", + "profile": "text_to_video", + "route": "/v1/videos/track", + "reason": "unknown_route" } ] } diff --git a/tests/test_capability_contract.py b/tests/test_capability_contract.py index bb1be1d6..5c86e0da 100644 --- a/tests/test_capability_contract.py +++ b/tests/test_capability_contract.py @@ -1,14 +1,16 @@ -"""The capability contract translates (stage, route) pairs, and nothing else. - -`tests/fixtures/omni-capabilities/` is the shared sample set: three tiers that -state what a deployment offering those pairs should be understood to provide. -The expected blocks are the reviewed oracle, not a copy of the mapping table, so -changing a mapping has to argue with them. They also stand on their own: with no -resolver at all, a case still says what the deployment offers. - -The last group reads the discovery document this repository parses today -(`models[].stage_ids` with routes per operation profile), because that is the -input `requests_for()` will hand to this table. +"""The capability contract translates upstream's (operation, profile) vocabulary. + +`tests/fixtures/omni-capabilities/` is the shared sample set: three tiers built +from the response shape upstream actually publishes — `stages[]` as topology, +`models[].operations[].profiles[]` as the declaration, absent fields +instead of nulls. The expected blocks are the reviewed oracle, not a copy of the +mapping table, so changing a mapping has to argue with them. + +The last group guards the failure this table exists to prevent: a route (or a +stage) that is read as if it named a capability. `/v1/audio/speech` carries +three profiles, `stages[].id` is opaque topology, and a registered endpoint +without an operation proves nothing — each of those gets its own regression +test below. """ import json from pathlib import Path @@ -17,17 +19,21 @@ import pytest from opendirector.domain.capabilities import Capability -from opendirector.domain.capability_contract import (ABSENT, AVAILABLE, ENDPOINT, MISSING, PARTIAL, PRESENT, - REASONS, STAGE_ROUTES, STAGE_ROUTE_MISMATCH, UNKNOWN, UNMAPPED, - UNMAPPED_STAGE, UNKNOWN_ROUTE, UNROUTED, UNROUTED_REASONS, - capabilities_for, classified_capabilities, declarations, - resolve, route_path) +from opendirector.domain.capability_contract import (ABSENT, AVAILABLE, ENDPOINT, MISSING, NO_LOCAL_CAPABILITY, + OPERATION_MISMATCH, OPERATION_PROFILES, OPERATION_ROUTES, + OPERATIONS, PRESENT, REASONS, ROUTED_PATHS, ROUTE_MISMATCH, + UNKNOWN, UNKNOWN_PROFILE, UNKNOWN_ROUTE, UNMAPPED_STAGE, UNROUTED, + UNROUTED_REASONS, capabilities_for, classified_capabilities, + declarations, resolve, route_path) FIXTURES = Path(__file__).parent / 'fixtures' / 'omni-capabilities' # Upstream bodies may name models; credentials and raw provider responses never # belong in a fixture, so their vocabulary is refused outright. FORBIDDEN = ('apikey', 'api_key', 'authorization', 'bearer', 'secret', 'password', 'cookie', '://') +OBSERVED = '2026-11-10T00:00:00Z' +EXECUTION = {'state': 'unpaused', 'scope': 'worker', 'observed_at': OBSERVED} + def raw_cases(): paths = sorted(FIXTURES.glob('*.json')) @@ -44,14 +50,62 @@ def fixture_cases(): yield pytest.param(case['payload'], case['expected'], id=f"{document['tier']}-{case['id']}") +def route(path, exposure='enabled', method='POST', transport='http'): + return {'transport': transport, 'method': method, 'path': path, 'exposure': exposure} + + +def profile(profile_id, support='supported', paths=(), inputs=('text',), outputs=('text',), **extra): + entry = {'id': profile_id, 'support': support, 'required_inputs': list(inputs), 'outputs': list(outputs), + 'routes': [route(path) for path in paths], 'reason_code': 'model_declared', 'constraints': {}} + entry.update(extra) + return entry + + +def operation(support='supported', exposure='enabled', profiles=(), **extra): + entry = {'support': support, 'exposure': exposure, 'reason_code': 'model_declared', + 'profiles': list(profiles), 'execution': dict(EXECUTION)} + entry.update(extra) + return entry + + +def model(operations=None, model_id='model-a', stage_ids=('7',)): + return {'id': model_id, 'stage_ids': list(stage_ids), 'operations': dict(operations or {})} + + +def body(operations=None, models=None, stages=None): + return {'object': 'omni.capabilities', 'schema_version': 1, 'observed_at': OBSERVED, + 'stages': [{'id': '7', 'type': 'llm', 'depends_on': []}] if stages is None else stages, + 'models': list(models) if models is not None else [model(operations)], + 'routes': [route('/v1/chat/completions'), route('/v1/audio/speech')]} + + +def every_operation(): + """All six operations supported and enabled, every profile they publish.""" + return {operation_id: operation(profiles=[profile(profile_id, paths=OPERATION_ROUTES[operation_id]) + for owner, profile_id, _ in OPERATION_PROFILES + if owner == operation_id]) + for operation_id in OPERATIONS} + + +def routes_of(operation_id, profile_id): + return [path for owner, row_profile, _ in OPERATION_PROFILES + if (owner, row_profile) == (operation_id, profile_id) + for path in OPERATION_ROUTES[operation_id]] + + @pytest.mark.parametrize('payload, expected', list(fixture_cases())) def test_declaration_resolves_to_the_reviewed_expectation(payload, expected): assert resolve(payload) == expected +# --- the table itself ----------------------------------------------------- + def test_every_capability_id_is_routed_or_explained(): """Adding a capability to the vocabulary without placing it here fails this.""" + placed = {capability for _, _, capabilities in OPERATION_PROFILES for capability in capabilities} assert set(classified_capabilities()) == set(get_args(Capability)) + assert not placed & set(UNROUTED), 'a capability is either routed or unrouted, never both' + assert placed | set(UNROUTED) == set(get_args(Capability)) def test_a_capability_appears_once_in_the_vocabulary_order(): @@ -60,117 +114,325 @@ def test_a_capability_appears_once_in_the_vocabulary_order(): assert tuple(dict.fromkeys(vocabulary)) == vocabulary -def test_contract_pairs_and_routes_are_canonical(): - assert len(STAGE_ROUTES) == len({(stage, route) for stage, route, _ in STAGE_ROUTES}) - for stage, route, capabilities in STAGE_ROUTES: - assert route_path(route) == route, (stage, route) - assert route.startswith('/v1/'), (stage, route) - assert stage == stage.strip().lower(), (stage, route) - assert capabilities and all(isinstance(item, str) for item in capabilities), (stage, route) - - -def test_one_path_serves_two_stages_without_overwriting_each_other(): - """`/v1/audio/speech` is both speech and music; the pair decides, the path does not.""" - assert capabilities_for('tts', '/v1/audio/speech') == ('audio.speech',) - assert capabilities_for('music', '/v1/audio/speech') == ('audio.music',) - result = resolve({'stages': [{'name': 'tts', 'routes': ['/v1/audio/speech']}, - {'name': 'music', 'routes': ['/v1/audio/speech']}]}) - assert result['stages'] == {'music': AVAILABLE, 'tts': AVAILABLE} - assert result['capabilities']['audio.speech'] == AVAILABLE - assert result['capabilities']['audio.music'] == AVAILABLE - - -def test_a_shared_path_does_not_grant_the_other_stage(): - """A TTS-only deployment must not look like it can generate music.""" - result = resolve({'stages': [{'name': 'tts', 'routes': ['/v1/audio/speech']}]}) - assert result['capabilities']['audio.speech'] == AVAILABLE +def test_the_table_covers_the_twelve_published_profiles(): + pairs = [(operation_id, profile_id) for operation_id, profile_id, _ in OPERATION_PROFILES] + assert len(pairs) == 12 == len(set(pairs)) + assert {operation_id for operation_id, _, _ in OPERATION_PROFILES} == set(OPERATIONS) + per_operation = {operation_id: [profile_id for owner, profile_id, _ in OPERATION_PROFILES if owner == operation_id] + for operation_id in OPERATIONS} + assert per_operation == {'chat': ['text_response', 'image_response', 'audio_response'], + 'image_generation': ['text_to_image'], + 'image_edit': ['image_to_image'], + 'video_generation': ['text_to_video', 'image_to_video', 'video_to_video'], + 'speech': ['text_to_speech', 'lyrics_to_music', 'caption_to_audio'], + 'audio_generation': ['text_to_audio']} + + +def test_rows_and_routes_are_canonical(): + for operation_id, profile_id, capabilities in OPERATION_PROFILES: + assert operation_id == operation_id.strip().lower(), (operation_id, profile_id) + assert profile_id == profile_id.strip().lower(), (operation_id, profile_id) + assert capabilities == tuple(capabilities), (operation_id, profile_id) + for operation_id, paths in OPERATION_ROUTES.items(): + assert paths, operation_id + for path in paths: + assert route_path(path) == path, path + assert path.startswith('/v1/'), path + + +def test_operation_routes_follow_upstream_ownership(): + assert set(OPERATION_ROUTES) == set(OPERATIONS) + assert OPERATION_ROUTES['chat'] == ('/v1/chat/completions',) + assert OPERATION_ROUTES['image_generation'] == ('/v1/images/generations',) + assert OPERATION_ROUTES['image_edit'] == ('/v1/images/edits',) + # The only operation with more than one route, including the websocket one. + assert OPERATION_ROUTES['video_generation'] == ('/v1/videos', '/v1/videos/sync', '/v1/realtime/video') + assert OPERATION_ROUTES['speech'] == ('/v1/audio/speech',) + assert OPERATION_ROUTES['audio_generation'] == ('/v1/audio/generate',) + assert {'/v1/videos/sync', '/v1/realtime/video'} <= ROUTED_PATHS + + +def test_unowned_endpoints_belong_to_no_operation(): + """Registered routes without an operation must not be readable as capabilities.""" + for path in ('/v1/images/segmentations', '/v1/videos/segmentations', '/v1/videos/depth', + '/v1/audio/transcriptions', '/v1/audio/speech/batch', '/v1/chat/completions/batch'): + assert path not in ROUTED_PATHS, path + + +def test_music_is_a_speech_profile_on_the_speech_route(): + """There is no music operation: MiniMax Music3 declares `lyrics_to_music` under `speech`.""" + assert capabilities_for('speech', 'lyrics_to_music') == ('audio.music',) + assert capabilities_for('audio_generation', 'lyrics_to_music') == () + assert 'music' not in OPERATIONS + assert '/v1/audio/generate' not in OPERATION_ROUTES['speech'] + assert '/v1/audio/speech' not in OPERATION_ROUTES['audio_generation'] + assert '/v1/audio/music' not in ROUTED_PATHS and '/v1/audio/music/generations' not in ROUTED_PATHS + + +# --- the five regressions that keep coverage honest ---------------------- + +def test_real_shaped_body_resolves_non_empty_pairs(): + """A body in the published shape must produce pairs and capabilities, not silence.""" + payload = body(every_operation()) + pairs = declarations(payload) + assert pairs and len(pairs) == 12 + assert set(pairs) == {(operation_id, profile_id) for operation_id, profile_id, _ in OPERATION_PROFILES} + result = resolve(payload) + assert result['endpoint'] == PRESENT + for capability in ('text.chat', 'image.generate', 'image.edit', 'video.generate', 'audio.speech', 'audio.music'): + assert result['capabilities'][capability] == AVAILABLE, capability + assert result['topology'] == [{'id': '7', 'type': 'llm', 'depends_on': []}] + + +def test_route_alone_cannot_distinguish_speech_profiles(): + """`/v1/audio/speech` is one path and three contracts; only the profile decides.""" + speech_routes = OPERATION_ROUTES['speech'] + assert len(speech_routes) == 1 + resolved = {profile_id: capabilities_for('speech', profile_id) + for profile_id in ('text_to_speech', 'lyrics_to_music', 'caption_to_audio')} + assert resolved == {'text_to_speech': ('audio.speech',), 'lyrics_to_music': ('audio.music',), + 'caption_to_audio': ()} + assert len(set(resolved.values())) == 3 + # MOSS-SoundEffect serves `caption_to_audio` and reports `text_to_speech` unsupported + # on the same path (upstream test_serving_capabilities.py:233,251). + payload = body({'speech': operation(profiles=[ + profile('text_to_speech', 'unsupported', paths=speech_routes), + profile('lyrics_to_music', 'unsupported', paths=speech_routes), + profile('caption_to_audio', paths=speech_routes, inputs=('ambient_sound',), outputs=('audio',))])}) + result = resolve(payload) + assert result['capabilities']['audio.speech'] == MISSING assert result['capabilities']['audio.music'] == MISSING + assert result['unresolved'] == [{'operation': 'speech', 'profile': 'caption_to_audio', 'route': None, + 'reason': NO_LOCAL_CAPABILITY}] + + +def test_stage_ids_do_not_collapse_capabilities(): + """Every model gets the same `stage_ids`; the operations must still decide separately.""" + first = model({'chat': operation(profiles=[profile('text_response', paths=routes_of('chat', 'text_response'))]), + 'image_generation': operation(exposure='disabled', + profiles=[profile('text_to_image', + paths=routes_of('image_generation', + 'text_to_image'))])}, + model_id='model-a', stage_ids=('7', '42')) + second = model({'chat': operation(support='unsupported', + profiles=[profile('text_response', 'unsupported', + paths=routes_of('chat', 'text_response'))]), + 'image_generation': operation(profiles=[profile('text_to_image', + paths=routes_of('image_generation', + 'text_to_image'))])}, + model_id='model-b', stage_ids=('7', '42')) + assert first['stage_ids'] == second['stage_ids'] + result = resolve(body(models=[first, second])) + # chat comes from the model that serves it, image generation from the other one. + assert result['capabilities']['text.chat'] == AVAILABLE + assert result['capabilities']['image.generate'] == AVAILABLE + # Opaque numeric stage ids are topology, so they never surface as failures. assert result['unresolved'] == [] + assert resolve(body(models=[second]))['capabilities']['text.chat'] == MISSING + + +def test_unrouted_capabilities_are_unknown_not_missing(): + """An endpoint that no operation publishes cannot be judged, so it is never `missing`.""" + result = resolve(body(every_operation())) + for capability, reason in UNROUTED.items(): + assert result['capabilities'][capability] == UNKNOWN, capability + assert reason in UNROUTED_REASONS, capability + assert {UNROUTED[capability] for capability in ('image.segment', 'video.matting', 'video.motion_capture')} == { + 'no_upstream_profile'} + assert 'parameter_only' not in {UNROUTED[capability] + for capability in ('image.segment', 'video.matting', 'video.motion_capture')} + assert MISSING not in {result['capabilities'][capability] for capability in UNROUTED} + + +def test_absent_fields_are_unknown(): + """`response_model_exclude_none` drops fields; absence is not a refusal.""" + omitted = body({'chat': {'reason_code': 'request_contract_unverified', + 'profiles': [{'id': 'text_response', 'routes': [route('/v1/chat/completions')]}]}, + 'speech': operation(profiles=[{'id': 'text_to_speech'}])}) + result = resolve(omitted) + assert result['endpoint'] == PRESENT + assert result['capabilities']['text.chat'] == UNKNOWN + assert result['capabilities']['audio.speech'] == UNKNOWN + assert MISSING not in result['capabilities'].values() + # The same body that states `unsupported` is a refusal, and does become missing. + stated = body({'chat': operation(support='unsupported', + profiles=[profile('text_response', 'unsupported', + paths=['/v1/chat/completions'])])}) + assert resolve(stated)['capabilities']['text.chat'] == MISSING -def test_transcription_serves_score_but_not_recognition(): - """`/v1/audio/transcriptions` is score under `score` and unmapped under `asr`.""" - assert capabilities_for('score', '/v1/audio/transcriptions') == ('audio.score',) - assert capabilities_for('asr', '/v1/audio/transcriptions') == () - declared = resolve({'stages': [{'name': 'score', 'routes': ['/v1/audio/transcriptions']}, - {'name': 'asr', 'routes': ['/v1/audio/transcriptions']}]}) - assert declared['capabilities']['audio.score'] == AVAILABLE - assert declared['stages'] == {'asr': UNMAPPED, 'score': AVAILABLE} - assert declared['unresolved'] == [{'stage': 'asr', 'route': '/v1/audio/transcriptions', - 'reason': UNMAPPED_STAGE}] - # Recognition alone proves nothing about the score adapter, and vice versa. - recognition_only = resolve({'stages': [{'name': 'asr', 'routes': ['/v1/audio/transcriptions']}]}) - assert recognition_only['capabilities']['audio.score'] == MISSING - +# --- states --------------------------------------------------------------- def test_an_absent_declaration_is_unknown_everywhere_and_never_missing(): - for payload in (None, {'object': 'omni.capabilities'}, {'stages': []}, {'stages': [{'routes': ['/v1/audio/speech']}]}, - {'stages': 'not-a-list'}, {'models': []}): + for payload in (None, {}, {'object': 'omni.capabilities', 'schema_version': 1}, + {'models': []}, {'models': 'not-a-list'}, + {'models': [{'id': 'm', 'stage_ids': ['7']}]}, + {'stages': [{'id': '7', 'type': 'llm', 'depends_on': []}]}): result = resolve(payload) assert result['endpoint'] == ABSENT - assert result['stages'] == {} assert set(result['capabilities'].values()) == {UNKNOWN} assert MISSING not in result['capabilities'].values() + assert result['unresolved'] == [] + + +def test_a_topology_only_body_grants_nothing(): + """`stages[]` is display data; a stage that carries routes and a name is still ignored.""" + payload = body(models=[], stages=[{'id': '7', 'type': 'llm', 'depends_on': []}, + {'id': '42', 'type': 'diffusion', 'depends_on': ['7']}, + {'name': 'tts', 'routes': ['/v1/audio/speech']}]) + result = resolve(payload) + assert result['topology'] == [{'id': '7', 'type': 'llm', 'depends_on': []}, + {'id': '42', 'type': 'diffusion', 'depends_on': ['7']}] + assert result['capabilities']['audio.speech'] == UNKNOWN + assert resolve(body(models=[], stages=[{'id': '99'}]))['topology'] == [ + {'id': '99', 'type': 'unknown', 'depends_on': []}] + + +def test_a_supported_but_disabled_operation_is_missing(): + result = resolve(body({'image_generation': operation(exposure='disabled', + profiles=[profile('text_to_image', + paths=['/v1/images/generations'])])})) + assert result['capabilities']['image.generate'] == MISSING + assert result['unresolved'] == [] + +def test_an_unsupported_profile_is_missing_in_a_supported_operation(): + result = resolve(body({'speech': operation(profiles=[ + profile('text_to_speech', paths=['/v1/audio/speech']), + profile('lyrics_to_music', 'unsupported', paths=['/v1/audio/speech'])])})) + assert result['capabilities']['audio.speech'] == AVAILABLE + assert result['capabilities']['audio.music'] == MISSING + + +def test_a_capability_no_profile_mentions_is_unknown_not_missing(): + result = resolve(body({'chat': operation(profiles=[profile('text_response', paths=['/v1/chat/completions'])])})) + assert result['capabilities']['video.generate'] == UNKNOWN + assert result['capabilities']['audio.music'] == UNKNOWN + + +# --- untranslatable declarations ----------------------------------------- + +def test_a_profile_declaring_another_operations_route_is_a_route_mismatch(): + payload = body({'video_generation': operation(profiles=[ + profile('text_to_video', paths=['/v1/images/edits']), + profile('image_to_video', paths=['/v1/videos'])])}) + result = resolve(payload) + # The coherent profile still counts; the broken one does not. + assert result['capabilities']['video.generate'] == AVAILABLE + assert result['unresolved'] == [{'operation': 'video_generation', 'profile': 'text_to_video', + 'route': '/v1/images/edits', 'reason': ROUTE_MISMATCH}] + alone = resolve(body({'video_generation': operation(profiles=[profile('text_to_video', + paths=['/v1/images/edits'])])})) + assert alone['capabilities']['video.generate'] == UNKNOWN + + +def test_a_route_that_no_operation_owns_is_an_unknown_route(): + for path in ('/v1/videos/track', '/v1/images/segmentations'): + result = resolve(body({'image_generation': operation( + profiles=[profile('text_to_image', paths=[path])])})) + assert result['unresolved'] == [{'operation': 'image_generation', 'profile': 'text_to_image', + 'route': path, 'reason': UNKNOWN_ROUTE}], path + assert result['capabilities']['image.generate'] == UNKNOWN + assert result['capabilities']['image.segment'] == UNKNOWN + + +def test_a_profile_filed_under_the_wrong_operation_is_an_operation_mismatch(): + result = resolve(body({'audio_generation': operation(profiles=[ + profile('lyrics_to_music', paths=['/v1/audio/speech'])])})) + assert result['unresolved'] == [{'operation': 'audio_generation', 'profile': 'lyrics_to_music', + 'route': None, 'reason': OPERATION_MISMATCH}] + assert result['capabilities']['audio.music'] == UNKNOWN + assert result['capabilities']['audio.speech'] == UNKNOWN + + +def test_unknown_operation_and_unknown_profile_stay_distinguishable(): + result = resolve(body({'speech': operation(profiles=[profile('speech_to_speech', + paths=['/v1/audio/speech'])])})) + assert result['unresolved'] == [{'operation': 'speech', 'profile': 'speech_to_speech', + 'route': None, 'reason': UNKNOWN_PROFILE}] + wrong_operation = resolve(body({'asr': operation(profiles=[])})) + assert wrong_operation['unresolved'] == [{'operation': 'asr', 'profile': None, + 'route': None, 'reason': UNMAPPED_STAGE}] + + +def test_asr_is_an_upstream_gap_not_a_missing_row(): + """Recognition has no upstream operation and no registered route; that is not our omission.""" + assert 'asr' not in OPERATIONS + assert '/v1/audio/transcriptions' not in ROUTED_PATHS + assert UNROUTED['audio.score'] == 'no_upstream_operation' + result = resolve(body({'asr': operation(profiles=[profile('asr', paths=['/v1/audio/transcriptions'])])})) + assert [item['reason'] for item in result['unresolved']] == [UNMAPPED_STAGE] + assert result['capabilities']['audio.score'] == UNKNOWN + + +def test_a_supported_profile_without_a_local_capability_is_reported(): + result = resolve(body({'audio_generation': operation(profiles=[ + profile('text_to_audio', paths=['/v1/audio/generate'], inputs=('text',), outputs=('audio',))])})) + assert result['unresolved'] == [{'operation': 'audio_generation', 'profile': 'text_to_audio', + 'route': None, 'reason': NO_LOCAL_CAPABILITY}] + assert result['capabilities']['audio.speech'] == UNKNOWN + assert MISSING not in result['capabilities'].values() + + +def test_an_unsupported_profile_without_a_local_capability_stays_quiet(): + result = resolve(body({'speech': operation(profiles=[ + profile('caption_to_audio', 'unsupported', paths=['/v1/audio/speech'])])})) + assert result['unresolved'] == [] -def test_a_known_stage_with_another_stages_route_is_a_mismatch(): - result = resolve({'stages': [{'name': 'image', 'routes': ['/v1/videos']}]}) - assert result['stages'] == {'image': UNMAPPED} - assert result['capabilities']['video.generate'] == MISSING - assert result['unresolved'] == [{'stage': 'image', 'route': '/v1/videos', 'reason': STAGE_ROUTE_MISMATCH}] +def test_two_models_repeating_one_declaration_are_reported_once(): + declaration = {'video_generation': operation(profiles=[profile('text_to_video', paths=['/v1/images/edits'])])} + result = resolve(body(models=[model(declaration, 'model-a', ('7',)), + model(declaration, 'model-b', ('7',))])) + assert result['unresolved'] == [{'operation': 'video_generation', 'profile': 'text_to_video', + 'route': '/v1/images/edits', 'reason': ROUTE_MISMATCH}] -def test_an_unknown_path_and_an_unknown_stage_stay_distinguishable(): - unknown_path = resolve({'stages': [{'name': 'video', 'routes': ['/v1/videos/track']}]}) - unknown_stage = resolve({'stages': [{'name': 'embedding', 'routes': ['/v1/videos']}]}) - assert unknown_path['unresolved'][0]['reason'] == UNKNOWN_ROUTE - assert unknown_stage['unresolved'][0]['reason'] == UNMAPPED_STAGE +# --- shape and plumbing --------------------------------------------------- def test_capabilities_are_reported_in_table_order(): """Presentation reads this order; sorting here would make the table's order meaningless.""" - result = resolve({'stages': [{'name': 'score', 'routes': ['/v1/audio/transcriptions']}, - {'name': 'text', 'routes': ['/v1/chat/completions']}]}) + result = resolve(body({'speech': operation(profiles=[profile('lyrics_to_music', paths=['/v1/audio/speech'])]), + 'chat': operation(profiles=[profile('text_response', paths=['/v1/chat/completions'])])})) assert list(result['capabilities']) == list(classified_capabilities()) assert list(result['capabilities'])[:2] == ['text.chat', 'image.generate'] -@pytest.mark.parametrize('route', ['audio/speech', '/audio/speech', '/v1/audio/speech/', {'path': '/v1/audio/speech?x=1'}]) -def test_route_spellings_normalize_to_one_path(route): - result = resolve({'stages': [{'name': 'tts', 'routes': [route]}]}) - assert result['stages'] == {'tts': AVAILABLE} +def test_declarations_keep_document_order_and_drop_repeats(): + operations = {'speech': operation(profiles=[profile('lyrics_to_music', paths=['/v1/audio/speech']), + profile('text_to_speech', paths=['/v1/audio/speech'])]), + 'chat': operation(profiles=[profile('text_response', paths=['/v1/chat/completions'])])} + assert declarations(body(operations)) == [('speech', 'lyrics_to_music'), ('speech', 'text_to_speech'), + ('chat', 'text_response')] + twin = body(models=[model(operations, 'model-a', ('7',)), model(operations, 'model-b', ('7',))]) + assert declarations(twin) == declarations(body(operations)) + assert declarations({'models': [{'id': 'm', 'operations': {}}]}) is None + + +@pytest.mark.parametrize('spelling', ['audio/speech', '/audio/speech', '/v1/audio/speech/', + {'path': '/v1/audio/speech?x=1'}]) +def test_route_spellings_normalize_to_one_path(spelling): + assert route_path(spelling) == '/v1/audio/speech' + result = resolve(body({'speech': operation(profiles=[ + profile('text_to_speech', paths=[], + routes=[spelling if isinstance(spelling, dict) else route(spelling)])])})) assert result['capabilities']['audio.speech'] == AVAILABLE - - -def test_stage_states_report_partial_understanding(): - result = resolve({'stages': [{'name': 'video', 'routes': ['/v1/videos', '/v1/videos/track']}]}) - assert result['stages'] == {'video': PARTIAL} - assert result['endpoint'] == PRESENT - assert result['unresolved'] == [{'stage': 'video', 'route': '/v1/videos/track', 'reason': UNKNOWN_ROUTE}] - - -def test_a_route_cannot_speak_for_a_parameter_capability(): - """`video.first_frame` is a /v1/videos parameter, so it is never `missing`.""" - assert resolve({'stages': [{'name': 'video', 'routes': ['/v1/videos']}]})['capabilities']['video.first_frame'] == UNKNOWN - - -def test_unrouted_capabilities_state_why_they_are_unreachable(): - assert set(UNROUTED.values()) <= UNROUTED_REASONS - - -def test_model_names_and_other_provider_fields_never_reach_the_result(): - payload = {'stages': [{'name': 'tts', 'model': 'a-private-model-name', 'routes': ['/v1/audio/speech'], - 'extra': {'owner': 'someone'}}]} - assert 'a-private-model-name' not in json.dumps(resolve(payload), ensure_ascii=False) + assert result['unresolved'] == [] def test_results_use_only_the_contract_vocabulary(): for _, cases in raw_cases(): for case in cases: expected = case['expected'] + assert set(expected) == {'endpoint', 'topology', 'capabilities', 'unresolved'} + assert expected['endpoint'] in {ABSENT, PRESENT} + assert set(expected['capabilities']) == set(classified_capabilities()) assert set(expected['capabilities'].values()) <= {AVAILABLE, MISSING, UNKNOWN} - assert set(expected['stages'].values()) <= {AVAILABLE, PARTIAL, UNMAPPED} assert {item['reason'] for item in expected['unresolved']} <= REASONS + assert {tuple(sorted(item)) for item in expected['unresolved']} <= { + ('operation', 'profile', 'reason', 'route')} + for stage in expected['topology']: + assert set(stage) == {'id', 'type', 'depends_on'} def test_fixtures_carry_no_credentials_or_provider_responses(): @@ -179,29 +441,8 @@ def test_fixtures_carry_no_credentials_or_provider_responses(): assert not [word for word in FORBIDDEN if word in text], path.name -def discovery_document(stage_ids, path='/v1/audio/speech', profile='text_to_speech'): - """The body `integrations.capability_probe` parses today: stages per model, routes per profile.""" - return {'object': 'omni.capabilities', 'schema_version': 1, 'observed_at': '2026-11-10T00:00:00+00:00', - 'models': [{'id': 'demo', 'stage_ids': list(stage_ids), 'operations': {'speech': { - 'support': 'supported', 'exposure': 'enabled', 'profiles': [ - {'id': profile, 'support': 'supported', 'routes': [{'method': 'POST', 'path': path}]}]}}}], - 'routes': [{'method': 'POST', 'path': path, 'exposure': 'enabled'}]} - - -def test_a_models_document_yields_the_same_pairs(): - assert declarations(discovery_document(['tts'])) == [('tts', '/v1/audio/speech')] - assert resolve(discovery_document(['tts']))['capabilities']['audio.speech'] == AVAILABLE - assert resolve(discovery_document(['music']))['capabilities']['audio.music'] == AVAILABLE - - -def test_an_opaque_stage_id_is_reported_instead_of_guessed(): - """A numeric `stage_ids` entry cannot be translated, and must not be read as a route match.""" - result = resolve(discovery_document(['0'])) - assert result['unresolved'] == [{'stage': '0', 'route': '/v1/audio/speech', 'reason': UNMAPPED_STAGE}] - assert result['capabilities']['audio.speech'] == MISSING - assert result['capabilities']['video.first_frame'] == UNKNOWN - - -def test_model_pairs_follow_document_order(): - pairs = declarations(discovery_document(['tts', 'music'])) - assert pairs == [('tts', '/v1/audio/speech'), ('music', '/v1/audio/speech')] +def test_model_names_and_other_provider_fields_never_reach_the_result(): + payload = body({'chat': operation(profiles=[profile('text_response', paths=['/v1/chat/completions'])])}) + payload['models'][0]['id'] = 'a-private-model-name' + payload['models'][0]['extra'] = {'owner': 'someone'} + assert 'a-private-model-name' not in json.dumps(resolve(payload), ensure_ascii=False) From 0c004558504b1a20017043dbaa3463e84f5534bf Mon Sep 17 00:00:00 2001 From: OpenDirector Bot Date: Wed, 11 Nov 2026 00:39:36 +0800 Subject: [PATCH 3/5] fix(contract): separate states, validate transport/method, make fixtures schema-valid Round-2 fixes for the three independent reviews of #278. Product decisions (P1-P5): - P1: image_response / audio_response no longer map to text.chat; they stay unknown with no_local_capability, since text.chat is served by a text-only adapter and an image-only deployment must not read as able to chat. - P2: text_to_audio (audio_generation, /v1/audio/generate) maps to audio.music alongside lyrics_to_music; the StableAudio/LTX2-t2a shape no longer reads audio.music as missing. - P3: support=unsupported and exposure=disabled stay "missing" but are told apart: resolve() now returns a `reasons` map (unsupported / endpoint_disabled / route_disabled) instead of dropping reason_code. - P4: profiles[*].routes[*].exposure is read: all routes disabled -> missing + route_disabled, all unknown or routes: [] -> unknown, one enabled -> available. - P5: one unrecognised route no longer voids a profile that answered; only when nothing usable remains does it fall to unknown. Contract and implementation (I1-I3): - I1: OPERATION_ROUTES is (transport, method, path); _check_route compares all three, so DELETE /v1/chat/completions and a websocket-declared http route are route_mismatch. - I2: endpoint distinguishes absent (no answer: None, no models, models not a list) from empty (answered, no operation declared) from present. - I3: the unreachable `item in UNROUTED` branch is deleted; unrouted ids never collect evidence, and the disjointness invariant is enforced by test_every_capability_id_is_routed_or_explained instead of dead runtime code. Malformed routes and supported profiles with no route are now reported (malformed_route / no_route_declared) instead of silently unknown. Fixtures (F1-F3): - All three fixtures are upstream-shaped: six operations per model, support and exposure on every operation, profile support present, profiles: [] instead of a missing key, image_response/audio_response outputs ["image"]/["audio"]. - Defensive shapes (asr key, misplaced profile, absent required support) moved out of the fixtures into the tests that defend against malformed input. - test_fixtures_are_schema_valid asserts upstream's structural invariants only. - exclude_none is described correctly (it drops None-valued optional fields such as `request`; default_factory lists serialize as []). Tests (T1-T5): - The tautological order assertion is now a literal vocabulary list, and the twelve pairs are literal too instead of an echo of OPERATION_PROFILES. - test_a_profile_without_support_is_unknown_even_when_its_operation_is_supported covers profile-level absence (old N4 mutant now dies). - test_a_profile_that_declares_no_route_is_unknown covers `supported + routes: []` (old M7/P3 mutant now dies). - New cases: profile routes all disabled / all unknown, one unknown route does not void a profile, DELETE and websocket mismatches, image-only chat, text_to_audio -> audio.music, absent vs empty endpoints, reason separation, non-dict operation values, profile order independence. - UNROUTED expectations are a literal table, not recomputed from the constant. Docs (D1-D2): - Corrected the exclude_none claim and the audio.score / /v1/audio/transcriptions statement (the capabilities body never publishes that path, so audio.score can only be unknown). - Added the status/reason mapping table (CN + EN) covering every declaration shape, plus the profile/route/profile-output tables. Tests: 60 passed (was 40); related subset 122 passed, 1 skipped. Nine mutations of the new rules (including removing route_disabled) all fail the suite. --- docs/README.md | 2 +- docs/architecture/capability-contract.md | 226 +++++--- .../domain/capability_contract.py | 298 ++++++++--- tests/fixtures/omni-capabilities/absent.json | 67 ++- tests/fixtures/omni-capabilities/full.json | 23 +- tests/fixtures/omni-capabilities/partial.json | 368 ++++++++++--- tests/test_capability_contract.py | 497 ++++++++++++++---- 7 files changed, 1146 insertions(+), 335 deletions(-) diff --git a/docs/README.md b/docs/README.md index 3541cc6f..5bdf783d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ This directory is the entry point for the repository's shared documentation. The | [Contributing](../CONTRIBUTING.md) | Label taxonomy, how to pick labels, and what a PR must satisfy | Process, labels, or submission requirements change | | [Unified project architecture](architecture/unified-project.md) | Business objects, shared entry points, and interaction rules | Object responsibilities, commands, persistence, or interaction rules change | | [Test contracts](architecture/test-contracts.md) | Test/production boundaries, how to avoid stale assertions and environment-dependent red runs | Test infrastructure, capability gates, or contract ownership change | -| [Capability mapping contract / 能力映射表契约](architecture/capability-contract.md) | Evidenced upstream vocabulary, the `(operation, profile) → capability` table, the six untranslatable declarations, and how it feeds discovery | Upstream vocabulary, capability ids, or discovery matching change | +| [Capability mapping contract / 能力映射表契约](architecture/capability-contract.md) | Evidenced upstream vocabulary, the `(operation, profile) → capability` table, the eight untranslatable declarations, and how it feeds discovery | Upstream vocabulary, capability ids, or discovery matching change | | [Roadmap](development/roadmap.md) | Capability dependencies and delivery criteria | Scope, dependencies, or acceptance criteria change | | [Validation guide](development/validation.md) | Reproducible checks and evidence requirements | Tools, runtime requirements, or validation entry points change | | [Issue #87 Windows handoff](development/issue-87-windows-handoff.md) ([中文](development/issue-87-windows-handoff.zh-CN.md)) | Native ACL repair and final text acceptance | The shared checkpoint or native validation scope changes | diff --git a/docs/architecture/capability-contract.md b/docs/architecture/capability-contract.md index a72c72b5..a03bb2f0 100644 --- a/docs/architecture/capability-contract.md +++ b/docs/architecture/capability-contract.md @@ -12,39 +12,46 @@ `chat` / `image_generation` / `image_edit` / `video_generation` / `speech` / `audio_generation`。 没有 `asr`、`transcription`、`segmentation`、`depth`、`music`。 -**12 个 profile**（`:43-54`），以及本仓对应的 capability id： - -| operation | profile | 本仓 capability id | -| --- | --- | --- | -| `chat` | `text_response` | `text.chat` | -| `chat` | `image_response` | `text.chat` | -| `chat` | `audio_response` | `text.chat` | -| `image_generation` | `text_to_image` | `image.generate` | -| `image_edit` | `image_to_image` | `image.edit` | -| `video_generation` | `text_to_video` | `video.generate` | -| `video_generation` | `image_to_video` | `video.generate` | -| `video_generation` | `video_to_video` | `video.generate` | -| `speech` | `text_to_speech` | `audio.speech` | -| `speech` | `lyrics_to_music` | `audio.music` | -| `speech` | `caption_to_audio` | （无）音效/环境声，本仓无对应 id | -| `audio_generation` | `text_to_audio` | （无）扩散侧独立契约，本仓无对应 id | - -**operation → 路由**（`:42-54`，一致性校验用）： - -| operation | 路由 | +**12 个 profile**（`:43-54`），以及本仓对应的 capability id。**输出模态不同的 profile 不合并**：`text.chat` 在本仓由纯文本适配器消费（`integrations/model_adapters.py:566-580` 直接取 `choices[0]["message"]["content"]` 当字符串），把「只出图」的 chat 部署折进 `text.chat` 会让它看起来能聊天。 + +| operation | profile | 上游 `required_inputs` → `outputs` | 本仓 capability id | +| --- | --- | --- | --- | +| `chat` | `text_response` | `("text",)` → `("text",)` | `text.chat` | +| `chat` | `image_response` | `("text",)` → `("image",)` | （无）输出是图像，不是本仓 `text.chat` | +| `chat` | `audio_response` | `("text",)` → `("audio",)` | （无）输出是音频，不是本仓 `text.chat` | +| `image_generation` | `text_to_image` | `("text",)` → `("image",)` | `image.generate` | +| `image_edit` | `image_to_image` | `("image","text")` → `("image",)` | `image.edit` | +| `video_generation` | `text_to_video` | `("text",)` → `("video",)` | `video.generate` | +| `video_generation` | `image_to_video` | `("image","text")` → `("video",)` | `video.generate` | +| `video_generation` | `video_to_video` | `("video","text")` → `("video",)` | `video.generate` | +| `speech` | `text_to_speech` | `("text",)` → `("audio",)` | `audio.speech` | +| `speech` | `lyrics_to_music` | `("text","instructions")` → `("audio",)` | `audio.music` | +| `speech` | `caption_to_audio` | `("text",)` → `("audio",)` | （无）音效/环境声，本仓无对应 id | +| `audio_generation` | `text_to_audio` | `("text",)` → `("audio",)` | `audio.music`（同为「文本生成音频」） | + +两个「沉默」的行要一起读： + +- `image_response` / `audio_response` 上游确实发布，但本仓没有「图片/音频版 chat」id，所以**保持 `unknown`**，`unresolved` 里报 `no_local_capability`。绝不折进 `text.chat`。 +- `text_to_audio` 走的是扩散侧独立请求契约（`/v1/audio/generate`，无 voice/lyrics/instructions 映射，`docs/serving/capabilities.md:59-60`），但语义就是「文本生成音频」，与 `lyrics_to_music` 同属音频生成，因此映射到 `audio.music`。否则 StableAudio / LTX2-t2a 这种形状（`speech/text_to_speech=unsupported`、`audio_generation/text_to_audio=supported`）会把 `audio.music` 反判成 `missing`。 + +**operation → 路由**，三个维度都要对（`:42-54`）。上游的 `RouteKey` 是 `(transport, method, path)`，websocket 路由的 method 字面量就是 `WEBSOCKET`： + +| operation | 路由（transport, method, path） | | --- | --- | -| `chat` | `POST /v1/chat/completions` | -| `image_generation` | `POST /v1/images/generations` | -| `image_edit` | `POST /v1/images/edits` | -| `video_generation` | `POST /v1/videos`；`POST /v1/videos/sync`；`WEBSOCKET /v1/realtime/video` | -| `speech` | `POST /v1/audio/speech` | -| `audio_generation` | `POST /v1/audio/generate` | +| `chat` | `http POST /v1/chat/completions` | +| `image_generation` | `http POST /v1/images/generations` | +| `image_edit` | `http POST /v1/images/edits` | +| `video_generation` | `http POST /v1/videos`；`http POST /v1/videos/sync`；`websocket WEBSOCKET /v1/realtime/video` | +| `speech` | `http POST /v1/audio/speech` | +| `audio_generation` | `http POST /v1/audio/generate` | + +`method` 不是装饰：`DELETE /v1/chat/completions` 会得到 405，`transport=websocket` 声明在 http 路由上是错的声明，两者都报 `route_mismatch`。`transport` 上游有默认值 `http`（`protocol/capabilities.py:16-21`），缺席不算不符；`method` 没有默认值，缺席即无法确认，按不符处理。 -另外三条事实决定了这份表的形状： +三条 schema 事实决定了这份表的形状： -- 响应顶层只有 `object` / `schema_version` / `observed_at` / `models` / `stages` / `routes`（`protocol/capabilities.py:73-79`），端点是 `response_model=OmniCapabilitiesResponse`、`response_model_exclude_none=True`（`api_server.py:2193`）——**字段缺席是显式设计，等价于 unknown**。 -- `models[].operations` **必须齐全 6 个**，少一个 pydantic 直接抛错（`protocol/capabilities.py:60-64`）。 -- `models[].stage_ids` 是**数字字符串**，而且每个 model 拿到的都是同一份全量列表（`entrypoints/openai/serving_capabilities.py:400`）。 +- 响应顶层只有 `object` / `schema_version` / `observed_at` / `models` / `stages` / `routes`（`protocol/capabilities.py:73-79`），端点是 `response_model=OmniCapabilitiesResponse`、`response_model_exclude_none=True`（`api_server.py:2193`）。 +- **`exclude_none` 只能省掉值为 `None` 的可选字段**，也就是 profile 的 `request`（`RequestConstraints | None = None`）。列表/字典字段是 `default_factory`，空的时候序列化成 `[]`/`{}`，**不会缺席**：`profiles: []` 而不是「没有 `profiles` 键」。`support` / `exposure` / `reason_code` 在 operation 和 profile 上都是必填（`protocol/capabilities.py:36-52`），永远不会缺席；少了它们就不是上游能发出的 body。 +- `models[].operations` **必须齐全 6 个**，少一个 pydantic 直接抛错（`protocol/capabilities.py:60-64`）；`models[].stage_ids` 是**数字字符串**，而且每个 model 拿到的都是同一份全量列表（`entrypoints/openai/serving_capabilities.py:400`）。 ### 键是 (operation, profile)，不是 route，也不是 stage @@ -52,28 +59,60 @@ **为什么不是 stage**：`stages[]` 只有 `id` / `type` / `depends_on`（`protocol/capabilities.py:67-70`），id 是不连续的数字字符串，而且每个 model 都是同一份列表。拿它当键，所有模型会共享一个状态，既区分不了也不能诊断。本模块把 `stages[]` 只放进 `topology` 供展示，**从不用它决定任何能力**；一个 stage 条目即使自带 `routes` 或 `name` 也被忽略，因为上游 schema 没有这两个键。 -**route 仍然有用**：`OPERATION_ROUTES` 保留 operation 对路由的所有权，用来校验声明的一致性 —— 某个 profile 引用了不属于该 operation 的路径（例如 `text_to_video` 指向 `/v1/images/edits`），该 profile 不产生能力，并报 `route_mismatch`。 +**route 仍然有用**：`OPERATION_ROUTES` 保留每个 operation 对 `(transport, method, path)` 的所有权，用来校验声明的一致性 —— 某个 profile 引用了不属于该 operation 的路径、或用错了 method/transport，该 profile 不产生能力，并报 `route_mismatch`。它同时决定「profile 自己说的可达性」：`profiles[*].routes[*].exposure` 是上游第三层 exposure，会按 profile 覆写（例如需要 video/audio 输入的 profile 在 `/v1/realtime/video` 上被强制 `disabled`，`serving_capabilities.py:341-343`）。 -### 六种无法翻译的声明 +### 八种无法翻译的声明 | reason | 含义 | 修的地方 | | --- | --- | --- | -| `unknown_route` | 路径不属于任何 operation（`/v1/videos/track`，或已注册但无 operation 的 `/v1/images/segmentations`） | 表缺这个 operation，或声明写错了路径 | -| `route_mismatch` | 路径属于**另一个** operation（`text_to_video` 指向 `/v1/images/edits`） | 部署声明的配对写错了 | -| `unmapped_stage` | 上游没有这个 operation（`asr`、`music`） | 上游词表缺口，不是本表漏行 | +| `unknown_route` | 路径不属于任何 operation（`/v1/videos/segmentations`，或已注册但无 operation 的 `/v1/images/segmentations`） | 表缺这个 operation，或声明写错了路径 | +| `route_mismatch` | 路径属于**另一个** operation，或 method/transport 与该 operation 的路由不符（`text_to_video` 指向 `/v1/images/edits`、`DELETE /v1/chat/completions`） | 部署声明的配对写错了 | +| `unmapped_stage` | 上游没有这个 operation（`asr`、`music`）。名字沿用上一版 stage-keyed 设计，含义是 operation 层缺口 | 上游词表缺口，不是本表漏行 | | `unknown_profile` | operation 认识，profile id 上游未发布（`speech_to_speech`） | 上游新增 profile，本表待补 | | `operation_mismatch` | profile 认识，但挂在别的 operation 下（`lyrics_to_music` 挂在 `audio_generation`） | 声明写错了 operation，或本表把归属搞错了 | -| `no_local_capability` | profile 支持，但本仓没有对应 capability id | 词表待补（`caption_to_audio`、`text_to_audio`） | +| `no_local_capability` | profile 支持，但本仓没有对应 capability id（`caption_to_audio`、`image_response`、`audio_response`） | 词表待补 | +| `no_route_declared` | profile 说自己支持，却一条路由都没声明（`routes: []` 是上游合法默认值） | 声明不完整；可达性无法确认 | +| `malformed_route` | 路由条目里没有可用的 `path`（畸形 body） | 上游 bug 或代理改写 | -六种都进 `unresolved`，不被丢弃；每条都带 `operation` / `profile` / `route` / `reason` 四键（不适用处为 `null`），调用方可以解释「为什么这条声明没变成能力」，而不是静默隐藏。 +八种都进 `unresolved`，不被丢弃；每条都带 `operation` / `profile` / `route` / `reason` 四键（不适用处为 `null`），调用方可以解释「为什么这条声明没变成能力」，而不是静默隐藏。 ### 三种能力状态，只有正面证据才是 missing -- `available`：某个 model 的 profile 明确 `support=supported`，且其 operation 不是 `exposure=disabled`； -- `missing`：某个 profile 或 operation **正面说不**（`support=unsupported` 或 `exposure=disabled`）； -- `unknown`：其余全部 —— 请求失败/404、字段缺席、枚举为 `unknown`、声明无法翻译、能力本身不可路由。 +- `available`：某个 model 的 profile 明确 `support=supported`，其 operation 是 `supported` + `exposure=enabled`，且该 profile **至少有一条自己的路由 `exposure=enabled`**； +- `missing`：某个 profile 或 operation **正面说不**（`support=unsupported` / `exposure=disabled` / 自己的路由全 `disabled`）； +- `unknown`：其余全部 —— 请求失败/404、字段缺席、枚举为 `unknown`、路由可达性无法确认、声明无法翻译、能力本身不可路由。 + +`resolve()` 的返回值除 `capabilities` 外，还有 **`reasons`**：只给 `missing` 的能力一个原因码（`unsupported` / `endpoint_disabled` / `route_disabled`）。`support=unsupported` 与 `exposure=disabled` 都是「正面说不」，但事实不同——一个是模型不行，一个是管理员把路由关了——所以不再坍缩成同一个 `missing`。 + +`unknown` 的原因不在 `reasons` 里，而在 `unresolved`（声明级）或 `UNROUTED` 表（能力级），见下表。 -**「服务没说」不是「服务不能」**：没有 `models`、没有 operation、`profiles` 被 `exclude_none` 掉，一律 `unknown`，绝不产生 `missing`。 +### 状态与 reason 的映射表（契约） + +维护者改这张表前先看这一节：每一种上游形状落到哪个 status、从哪个出口能查到原因。 + +| 上游声明 | status | 原因出口 | +| --- | --- | --- | +| profile `support=unsupported` | `missing` | `reasons[cid] = "unsupported"` | +| operation `support=unsupported`（profile 说什么都一样） | `missing` | `reasons[cid] = "unsupported"` | +| operation `support=supported` + `exposure=disabled`（部署关了路由） | `missing` | `reasons[cid] = "endpoint_disabled"` | +| profile `support=supported`，但它声明的路由**全部** `exposure=disabled` | `missing` | `reasons[cid] = "route_disabled"` | +| profile 声明的路由全部 `exposure=unknown`（上游默认值） | `unknown` | 无（可达性无法确认，不是拒绝） | +| profile `routes: []`（上游默认空列表） | `unknown` | `unresolved: no_route_declared` | +| 路由条目没有可用 `path` | `unknown` | `unresolved: malformed_route` | +| 路由 path 不认得，且没有别的可用路由 | `unknown` | `unresolved: unknown_route` | +| 路由 path 属于别的 operation，或 method/transport 不符 | `unknown` | `unresolved: route_mismatch` | +| 上游有 profile，本仓没有 id（`caption_to_audio` 等） | `unknown` | `unresolved: no_local_capability` | +| operation 认不出（`asr`）/ profile 认不出 / profile 挂错 operation | 该能力无证据 → `unknown` | `unresolved: unmapped_stage` / `unknown_profile` / `operation_mismatch` | +| `support=supported` + `exposure=enabled` + 至少一条自己的路由 `enabled` | `available` | 无 | +| 能力本身没有上游 profile（`image.segment` 等 8 项） | `unknown` | `UNROUTED[cid]`（`no_upstream_profile` / `no_upstream_operation` / `parameter_only` / `no_local_adapter`） | +| body 没答（`None`、无 `models` 键、`models` 不是列表） | 全部 `unknown` | `endpoint = "absent"` | +| 答了，但没有 model 声明任何 operation（含 `models: []`） | 全部 `unknown` | `endpoint = "empty"` | +| operation 答了但 `profiles: []` | 该 operation 的能力 `unknown` | `endpoint = "present"` | + +两条边界值得单独记住： + +1. **一条不认得的路由不打穿已确认的 profile**。上游会加路由，本表会落后；只有当该 profile **全部**路由都不认得（或全部被拒）时才降为 `unknown`，否则保留判定并在 `unresolved` 里记一条。 +2. **`endpoint` 三值**区分「没答」与「答了但没声明」：`absent` / `empty` / `present`。前者该降级或报错，后者该提示「服务未声明能力」，两者对用户的意义完全不同。 ### 有路由、没有 operation 的能力 @@ -88,9 +127,13 @@ | `video.first_frame` / `video.edit` / `image.cutout` | `parameter_only` | 是已发布 operation 的请求参数，不是 profile | | `audio.separate` | `no_local_adapter` | 上游没说法，本仓也没适配器 | +这 8 个 id **不出现在** `OPERATION_PROFILES` 的值域里，所以它们永远收不到证据，`unknown` 是「没有证据」的自然结果，而不是 `resolve()` 里的一个特例分支（这条不变量由 `test_every_capability_id_is_routed_or_explained` 守着）。 + ### ASR：上游未发布该 operation -上游词表里没有识别 operation，omni 的 router 也不注册 `/v1/audio/transcriptions`（`api_server.py` 全量路由里没有它），上游自己的 recipe 写明该端点「not currently available with `--omni`」。因此请求侧写 `asr` 得到 `unmapped_stage`：这是**上游没发布这个 operation**，不是本表漏了一行；`audio.score` 因此恒为 `unknown`（`no_upstream_operation`）。等上游发布识别 operation（或本仓补 `audio.transcribe`）再谈映射，绝不把 `/v1/audio/transcriptions` 读成 `audio.score`。 +上游词表里没有识别 operation，omni 的 router 也不注册 `/v1/audio/transcriptions`（`api_server.py` 全量路由里没有它），上游自己的 recipe 写明该端点「not currently available with `--omni`」。因此 body 里出现 `asr` 得到 `unmapped_stage`：这是**上游没发布这个 operation**，不是本表漏了一行。 + +这不等于「本仓不认 `audio.score`」——恰恰相反，本仓 `VllmOmniScore` 适配器 POST 的正是 `/audio/transcriptions`（`integrations/model_adapters.py:547,556`），对端是上游的独立工具 `tools/sheetsage2_serve.py:124`。但 **capabilities body 从不会提到这条路径**，所以 `audio.score` 只能判 `unknown`（`no_upstream_operation`）：本表不从「我们知道某条路由」反推能力，那样会把部署策略、代理改写和表 bug 混成同一个结论。等上游发布识别 operation（或把该路径纳入词表）再谈映射。 ### 与 V2 发现契约（#266）的关系 @@ -109,13 +152,20 @@ from opendirector.domain.capability_contract import capabilities_for capabilities_for('speech', 'lyrics_to_music') # ('audio.music',) capabilities_for('speech', 'text_to_speech') # ('audio.speech',) +capabilities_for('audio_generation', 'text_to_audio') # ('audio.music',) ``` -`resolve(payload)` 返回逐能力状态、`topology`（`stages[]`，仅供展示）与 `unresolved`；`classified_capabilities()` 按表顺序给出词表，展示层据此排列。是否接进 `requests_for()` 由 #266 一侧决定，本契约只保证查找是成对的、稳定的、可测的。 +`resolve(payload)` 返回逐能力状态（`capabilities`）、缺失原因（`reasons`）、`topology`（`stages[]`，仅供展示）与 `unresolved`；`classified_capabilities()` 按表顺序给出词表，展示层据此排列。是否接进 `requests_for()` 由 #266 一侧决定，本契约只保证查找是成对的、稳定的、可测的。 ### 三档 fixture 与验证 -`tests/fixtures/omni-capabilities/{full,partial,absent}.json` 按**真实响应形状**写：`stages[]` 是 `{id,type,depends_on}`，`models[].operations` 六个齐全、每个含 `support`/`exposure`/`reason_code`/`profiles`/`execution`，profile 带 `routes[]`/`required_inputs`/`outputs`；`absent` 档用**字段缺席**（模拟 `exclude_none=True`）而不是 `null`；`partial` 档覆盖 `support=unsupported`、`exposure=disabled`、`route_mismatch`、`unknown_route`、`unknown_profile`、`operation_mismatch`、`unmapped_stage`、`no_local_capability`。 +`tests/fixtures/omni-capabilities/{full,partial,absent}.json` 按**真实响应形状**写：`stages[]` 是 `{id,type,depends_on}` 且 id 为数字字符串，`models[].operations` 六个齐全、每个含 `support`/`exposure`/`reason_code`/`profiles`/`execution`，每个 profile 都带 `support` 与 `routes[]`（可以是 `[]`），`image_response`/`audio_response` 的 `outputs` 照上游 `STANDARD_PROFILES` 分别是 `["image"]`/`["audio"]`。 + +- `full`：全支持全启用，12 个 profile 齐全；`image_response`/`audio_response`/`caption_to_audio` 报 `no_local_capability`，`text_to_audio` 让 `audio.music` 为 `available`。 +- `partial`：`support=unsupported`、operation `exposure=disabled`（`endpoint_disabled`）、profile 自己的路由 `disabled`（`route_disabled`）、`route_mismatch`、`no_local_capability`，以及整个 body 都是 `unknown` 枚举的形态。 +- `absent`：没答（`absent`）、只答拓扑（`empty`）、六个 operation 都答了但 `profiles: []`（`present` 且全 `unknown`）。 + +**上游不可能发出的形状**（`asr` operation 键、profile 挂在错 operation、缺必填 `support`、`profiles` 整键缺席）**不进 fixture**，只在对应测试里内联构造 —— 它们是防御性输入的用例，混进 fixture 会让「fixture = 真实样本」这句话变成假话。`test_fixtures_are_schema_valid` 只断言上游 schema 的**结构不变量**（六操作齐全、必填字段在、`profiles` 是列表、stages 形状、id 为数字字符串），不把上游 pydantic 源码抄一份进本仓。 防「假覆盖」的回归测试（`tests/test_capability_contract.py`）： @@ -124,8 +174,16 @@ capabilities_for('speech', 'text_to_speech') # ('audio.speech',) | `test_real_shaped_body_resolves_non_empty_pairs` | 真实形状 body 解析出 12 个成对声明，不是空表 | | `test_route_alone_cannot_distinguish_speech_profiles` | 同一路径 3 个 profile 解析出 3 个不同答案 | | `test_stage_ids_do_not_collapse_capabilities` | 两个 model 的 `stage_ids` 相同，能力状态仍各自独立 | -| `test_unrouted_capabilities_are_unknown_not_missing` | 有路由无 operation 的能力恒 `unknown` | -| `test_absent_fields_are_unknown` | 字段缺席是 `unknown`，只有明确 `unsupported` 才是 `missing` | +| `test_capabilities_without_an_upstream_profile_are_unknown_not_missing` | 8 个无上游 profile 的 id 恒 `unknown`，且 `reasons` 里不出现 | +| `test_a_profile_without_support_is_unknown_even_when_its_operation_is_supported` | profile 级字段缺席是 `unknown`（不是 `supported`） | +| `test_a_profile_that_declares_no_route_is_unknown` | `supported + routes: []` 不能判 `available` | +| `test_a_profile_whose_only_route_is_disabled_is_missing` | profile 路由全 `disabled` → `missing` + `route_disabled` | +| `test_a_profile_whose_routes_are_all_unknown_cannot_be_confirmed` | 路由全 `unknown` → `unknown` | +| `test_one_unrecognised_route_does_not_void_a_profile_that_answered` | 一条不认得的路由不打穿已确认的 profile | +| `test_a_wrong_method_is_a_route_mismatch` | `DELETE /v1/chat/completions` 不授予 chat | +| `test_an_image_only_chat_deployment_does_not_read_as_text_chat` | `image_response` 不折进 `text.chat` | +| `test_text_to_audio_grants_audio_music` | `text_to_audio` → `audio.music` | +| `test_a_refused_model_and_a_closed_route_are_told_apart` | `unsupported` / `endpoint_disabled` / `route_disabled` 三码可分 | ```bash PYTHONPATH=packages/core/src:apps/api/src python -m pytest tests/test_capability_contract.py -q @@ -141,22 +199,28 @@ Source: `vllm-project/vllm-omni` PR #10102 (issue #10007), head `d1104f8`; every **Six closed operations** (`vllm_omni/serving_capabilities.py:15,19-26`): `chat`, `image_generation`, `image_edit`, `video_generation`, `speech`, `audio_generation`. There is no `asr`, `transcription`, `segmentation`, `depth` or `music`. -**Twelve profiles** (`:43-54`) and the ids this repository has for them: +**Twelve profiles** (`:43-54`) and the ids this repository has for them. Profiles with a different output modality are not merged: `text.chat` here is consumed by a text-only adapter (`integrations/model_adapters.py:566-580` reads `choices[0]["message"]["content"]` as a string), so an image-only chat deployment must not read as able to chat. -| operation | profile | capability here | -| --- | --- | --- | -| `chat` | `text_response`, `image_response`, `audio_response` | `text.chat` | -| `image_generation` | `text_to_image` | `image.generate` | -| `image_edit` | `image_to_image` | `image.edit` | -| `video_generation` | `text_to_video`, `image_to_video`, `video_to_video` | `video.generate` | -| `speech` | `text_to_speech` | `audio.speech` | -| `speech` | `lyrics_to_music` | `audio.music` | -| `speech` | `caption_to_audio` | none — sound effect / ambience | -| `audio_generation` | `text_to_audio` | none — separate diffusion contract | +| operation | profile | upstream `required_inputs` → `outputs` | capability here | +| --- | --- | --- | --- | +| `chat` | `text_response` | `("text",)` → `("text",)` | `text.chat` | +| `chat` | `image_response` | `("text",)` → `("image",)` | none — the output is an image, not this repo's `text.chat` | +| `chat` | `audio_response` | `("text",)` → `("audio",)` | none — the output is audio, not this repo's `text.chat` | +| `image_generation` | `text_to_image` | `("text",)` → `("image",)` | `image.generate` | +| `image_edit` | `image_to_image` | `("image","text")` → `("image",)` | `image.edit` | +| `video_generation` | `text_to_video` | `("text",)` → `("video",)` | `video.generate` | +| `video_generation` | `image_to_video` | `("image","text")` → `("video",)` | `video.generate` | +| `video_generation` | `video_to_video` | `("video","text")` → `("video",)` | `video.generate` | +| `speech` | `text_to_speech` | `("text",)` → `("audio",)` | `audio.speech` | +| `speech` | `lyrics_to_music` | `("text","instructions")` → `("audio",)` | `audio.music` | +| `speech` | `caption_to_audio` | `("text",)` → `("audio",)` | none — sound effect / ambience | +| `audio_generation` | `text_to_audio` | `("text",)` → `("audio",)` | `audio.music` (text to audio, same family) | + +`image_response` and `audio_response` are published upstream but have no id here: they stay `unknown` and are reported as `no_local_capability` instead of being folded into `text.chat`. `text_to_audio` uses the separate diffusion request contract behind `/v1/audio/generate` (no voice, no lyrics; `docs/serving/capabilities.md:59-60`) yet is still text-to-audio, so it grants `audio.music` alongside `lyrics_to_music`; without that row a StableAudio / LTX2-t2a shape (`speech/text_to_speech=unsupported`, `audio_generation/text_to_audio=supported`) would report `audio.music=missing`. -**Operation → routes** (`:42-54`): `chat` → `POST /v1/chat/completions`; `image_generation` → `POST /v1/images/generations`; `image_edit` → `POST /v1/images/edits`; `video_generation` → `POST /v1/videos`, `POST /v1/videos/sync`, `WEBSOCKET /v1/realtime/video`; `speech` → `POST /v1/audio/speech`; `audio_generation` → `POST /v1/audio/generate`. +**Operation → routes** (`:42-54`) as `(transport, method, path)`, because upstream's `RouteKey` is exactly that and the websocket method literal is `WEBSOCKET`: `chat` → `http POST /v1/chat/completions`; `image_generation` → `http POST /v1/images/generations`; `image_edit` → `http POST /v1/images/edits`; `video_generation` → `http POST /v1/videos`, `http POST /v1/videos/sync`, `websocket WEBSOCKET /v1/realtime/video`; `speech` → `http POST /v1/audio/speech`; `audio_generation` → `http POST /v1/audio/generate`. `DELETE /v1/chat/completions` would be a 405 and a `websocket` declaration on an http route is wrong, so both are `route_mismatch`. `transport` defaults to `http` upstream (`protocol/capabilities.py:16-21`) and an absent transport is not a mismatch; `method` has no default and an absent one cannot be confirmed. -Three more facts shape the table: the body has only `object`, `schema_version`, `observed_at`, `models`, `stages`, `routes` (`protocol/capabilities.py:73-79`) and is served with `response_model_exclude_none=True` (`api_server.py:2193`), so **an absent field is the designed way to say unknown**; `models[].operations` **must list all six** or pydantic raises (`protocol/capabilities.py:60-64`); and `models[].stage_ids` are numeric strings, the *same* full list for every model (`entrypoints/openai/serving_capabilities.py:400`). +Three schema facts shape the table: the body has only `object`, `schema_version`, `observed_at`, `models`, `stages`, `routes` (`protocol/capabilities.py:73-79`) and is served with `response_model_exclude_none=True` (`api_server.py:2193`); **`exclude_none` drops only `None`-valued optional fields** — in practice the profile's `request` — while `default_factory` lists serialize as `[]` (`profiles: []`, never a missing key) and `support`/`exposure`/`reason_code` are required on every operation and profile (`protocol/capabilities.py:36-52`), so a body lacking them is not a shape upstream can emit; and `models[].operations` **must list all six** or pydantic raises (`:60-64`), with `stage_ids` numeric strings, the *same* full list for every model (`entrypoints/openai/serving_capabilities.py:400`). ### The key is the pair (operation, profile) @@ -164,23 +228,46 @@ Not the route: `/v1/audio/speech` is one path with three contracts — `text_to_ Not the stage either: `stages[]` publishes only `id` / `type` / `depends_on`, the ids are opaque numeric strings, and every model carries the same list. Keying on stages gives all models one shared state and no diagnosis, so `stages[]` only becomes `topology` here and never decides a capability; a stage entry carrying `routes` or `name` is ignored, because upstream's `CapabilityStage` has no such fields. -Routes still matter, as a consistency check: `OPERATION_ROUTES` records which operation owns which path, and a profile that references another operation's path (say `text_to_video` → `/v1/images/edits`) grants nothing and is reported as `route_mismatch`. +Routes still matter, in two ways: `OPERATION_ROUTES` records which operation owns which `(transport, method, path)` and a profile referencing another operation's path — or the wrong method — grants nothing and is reported as `route_mismatch`; and `profiles[*].routes[*].exposure` is upstream's third exposure layer, overridden per profile (a profile needing video/audio input is forced `disabled` on `/v1/realtime/video`, `serving_capabilities.py:341-343`). -### Six untranslatable declarations +### Eight untranslatable declarations -`unknown_route` (the path belongs to no operation, including registered-but-unowned `/v1/images/segmentations`), `route_mismatch` (the path belongs to another operation), `unmapped_stage` (no such operation upstream — `asr`, `music`), `unknown_profile` (operation known, profile id unpublished), `operation_mismatch` (profile known under another operation), `no_local_capability` (profile supported, no id here). All six land in `unresolved` with `operation` / `profile` / `route` / `reason`, never dropped. +`unknown_route` (the path belongs to no operation, including registered-but-unowned `/v1/images/segmentations`), `route_mismatch` (another operation's path, or a wrong method/transport), `unmapped_stage` (no such operation upstream — `asr`, `music`; the name predates the pair-keyed table), `unknown_profile` (operation known, profile id unpublished), `operation_mismatch` (profile known under another operation), `no_local_capability` (profile supported, no id here), `no_route_declared` (a supported profile with `routes: []`), `malformed_route` (a route entry with no usable `path`). All eight land in `unresolved` with `operation` / `profile` / `route` / `reason`, never dropped. ### Three capability states, and `missing` needs positive evidence -`available` when a model's profile says `supported` and its operation is not `exposure=disabled`; `missing` only when a profile or operation says no (`support=unsupported` or `exposure=disabled`); `unknown` for everything else — failed request, absent field, `unknown` enum value, an untranslatable declaration, or a capability no profile can express. **"The service did not say" is not "the service cannot"**: no models, no operations, or a dropped `profiles` field yields `unknown`, never `missing`. +`available` when a model's profile says `supported`, its operation is `supported`+`enabled`, and at least one of the profile's own routes is `enabled`; `missing` only when a profile or operation says no; `unknown` for everything else — failed request, absent field, `unknown` enum value, unconfirmed reachability, an untranslatable declaration, or a capability no profile can express. `resolve()` also returns `reasons`, which names the kind of refusal for every `missing` capability: `unsupported` (the model refused), `endpoint_disabled` (the deployment closed the operation's routes) or `route_disabled` (every route the profile declares is closed). Reasons for `unknown` live in `unresolved` (declaration level) or in `UNROUTED` (capability level). + +### Status and reason mapping (the contract) + +| Upstream declaration | status | where the reason lives | +| --- | --- | --- | +| profile `support=unsupported` | `missing` | `reasons[cid] = "unsupported"` | +| operation `support=unsupported` | `missing` | `reasons[cid] = "unsupported"` | +| operation `supported` + `exposure=disabled` | `missing` | `reasons[cid] = "endpoint_disabled"` | +| all of the profile's routes `exposure=disabled` | `missing` | `reasons[cid] = "route_disabled"` | +| all of the profile's routes `exposure=unknown` | `unknown` | none — reachability unconfirmed, not a refusal | +| profile `routes: []` | `unknown` | `unresolved: no_route_declared` | +| a route entry with no usable `path` | `unknown` | `unresolved: malformed_route` | +| an unrecognised path, and nothing else usable | `unknown` | `unresolved: unknown_route` | +| another operation's path, or a wrong method/transport | `unknown` | `unresolved: route_mismatch` | +| upstream profile, no id here | `unknown` | `unresolved: no_local_capability` | +| unknown operation / profile / misplaced profile | no evidence → `unknown` | `unresolved: unmapped_stage` / `unknown_profile` / `operation_mismatch` | +| `supported`, `enabled`, one `enabled` route | `available` | none | +| a capability no upstream profile can express | `unknown` | `UNROUTED[cid]` (`no_upstream_profile` / `no_upstream_operation` / `parameter_only` / `no_local_adapter`) | +| no answer (`None`, no `models`, `models` not a list) | all `unknown` | `endpoint = "absent"` | +| answered, but no model declared an operation | all `unknown` | `endpoint = "empty"` | +| operation answered with `profiles: []` | `unknown` | `endpoint = "present"` | + +Two boundaries to keep: one unrecognised route must not void a profile that answered (upstream adds routes; only when nothing usable remains does the profile fall to `unknown`), and `endpoint` distinguishes "did not answer" (`absent`) from "answered without declaring" (`empty`) from "declared" (`present`). ### Routes without an operation -`/v1/images/segmentations`, `/v1/videos/segmentations` and `/v1/videos/depth` are registered endpoints that no operation owns; upstream's docs state that "segmentation and depth estimation do not acquire one of these operations merely by being present in a registry" (`docs/serving/capabilities.md:41-43`). `image.segment`, `video.matting` and `video.motion_capture` therefore stay `unknown` with `no_upstream_profile` — the endpoint does not publish the ability, so it cannot be judged, and it must never read as `missing`. `audio.score` is `no_upstream_operation`; `video.first_frame`, `video.edit` and `image.cutout` are `parameter_only` (request fields of a published operation); `audio.separate` is `no_local_adapter`. +`/v1/images/segmentations`, `/v1/videos/segmentations` and `/v1/videos/depth` are registered endpoints that no operation owns; upstream's docs state that "segmentation and depth estimation do not acquire one of these operations merely by being present in a registry" (`docs/serving/capabilities.md:41-43`). `image.segment`, `video.matting` and `video.motion_capture` therefore stay `unknown` with `no_upstream_profile` — the endpoint does not publish the ability, so it cannot be judged, and it must never read as `missing`. `audio.score` is `no_upstream_operation`; `video.first_frame`, `video.edit` and `image.cutout` are `parameter_only` (request fields of a published operation); `audio.separate` is `no_local_adapter`. None of these eight ids is a value of `OPERATION_PROFILES`, so `unknown` falls out of "no evidence" rather than a special case. ### ASR: an upstream gap, not a missing row -Upstream publishes no recognition operation and omni's router never registers `/v1/audio/transcriptions` (absent from `api_server.py`; its own recipe says the endpoint is "not currently available with `--omni`"). A caller naming `asr` gets `unmapped_stage` — the upstream vocabulary has no such operation, this table did not lose a row — and `audio.score` stays `unknown` (`no_upstream_operation`). `/v1/audio/transcriptions` is never read as `audio.score`. +Upstream publishes no recognition operation and omni's router never registers `/v1/audio/transcriptions` (absent from `api_server.py`; its own recipe says the endpoint is "not currently available with `--omni`"). A body naming `asr` gets `unmapped_stage` — the upstream vocabulary has no such operation, this table did not lose a row. That is not a claim that this repository ignores `audio.score`: the `VllmOmniScore` adapter does post to `/audio/transcriptions` (`integrations/model_adapters.py:547,556`), whose peer is the standalone `tools/sheetsage2_serve.py:124`. But a capabilities body never mentions that path, so `audio.score` can only be `unknown` (`no_upstream_operation`): a route we happen to know is not evidence of a published capability. ### Relationship to the V2 discovery contract (#266) @@ -195,15 +282,18 @@ from opendirector.domain.capability_contract import capabilities_for capabilities_for('speech', 'lyrics_to_music') # ('audio.music',) capabilities_for('speech', 'text_to_speech') # ('audio.speech',) +capabilities_for('audio_generation', 'text_to_audio') # ('audio.music',) ``` -`resolve(payload)` returns per-capability states, `topology` (display only) and `unresolved`; `classified_capabilities()` returns the vocabulary in table order. Whether to wire this into `requests_for()` is #266's decision; this contract only guarantees the lookup is paired, ordered and testable. +`resolve(payload)` returns per-capability states, per-capability `reasons`, `topology` (display only) and `unresolved`; `classified_capabilities()` returns the vocabulary in table order. Whether to wire this into `requests_for()` is #266's decision; this contract only guarantees the lookup is paired, ordered and testable. ### Fixtures and checks -`tests/fixtures/omni-capabilities/{full,partial,absent}.json` follow the real response shape — `stages[]` as `{id,type,depends_on}`, six complete `operations` per model each with `support`/`exposure`/`reason_code`/`profiles`/`execution`, profiles with `routes[]`/`required_inputs`/`outputs`; the `absent` tier omits fields the way `exclude_none=True` does instead of writing `null`; the `partial` tier covers `support=unsupported`, `exposure=disabled`, `route_mismatch`, `unknown_route`, `unknown_profile`, `operation_mismatch`, `unmapped_stage` and `no_local_capability`. +`tests/fixtures/omni-capabilities/{full,partial,absent}.json` follow the real response shape — `stages[]` as `{id,type,depends_on}` with numeric-string ids, six complete `operations` per model each with `support`/`exposure`/`reason_code`/`profiles`/`execution`, every profile carrying `support` and `routes` (possibly `[]`), and `image_response`/`audio_response` outputs `["image"]`/`["audio"]` as published. `full` is the all-yes deployment, `partial` mixes refusals, a policy-disabled operation, a profile whose own route is disabled, a misplaced route and an all-`unknown` body, and `absent` covers no answer (`absent`), topology only (`empty`) and six operations that declare `profiles: []` (`present`, all `unknown`). + +Shapes upstream can never emit — an `asr` operation key, a profile filed under the wrong operation, a missing required `support` — are built inline in the tests that defend against malformed input, never in a fixture, so "the fixtures are a real sample" stays true. `test_fixtures_are_schema_valid` asserts only the structural invariants of upstream's model, not a copy of it. -Five regressions keep coverage honest: `test_real_shaped_body_resolves_non_empty_pairs` (a real body yields twelve pairs), `test_route_alone_cannot_distinguish_speech_profiles` (one path, three profiles, three answers), `test_stage_ids_do_not_collapse_capabilities` (identical `stage_ids`, independent states), `test_unrouted_capabilities_are_unknown_not_missing`, and `test_absent_fields_are_unknown` (absence is `unknown`; only an explicit `unsupported` is `missing`). +Five regressions keep coverage honest: `test_real_shaped_body_resolves_non_empty_pairs`, `test_route_alone_cannot_distinguish_speech_profiles`, `test_stage_ids_do_not_collapse_capabilities`, `test_capabilities_without_an_upstream_profile_are_unknown_not_missing`, and `test_a_profile_without_support_is_unknown_even_when_its_operation_is_supported`; the route-level exposure, transport/method, chat-modality and missing-reason cases each have their own test, listed in the Chinese section above. ```bash PYTHONPATH=packages/core/src:apps/api/src python -m pytest tests/test_capability_contract.py -q diff --git a/packages/core/src/opendirector/domain/capability_contract.py b/packages/core/src/opendirector/domain/capability_contract.py index d511c1cf..9eed655b 100644 --- a/packages/core/src/opendirector/domain/capability_contract.py +++ b/packages/core/src/opendirector/domain/capability_contract.py @@ -16,7 +16,7 @@ speech text_to_speech, lyrics_to_music, caption_to_audio audio_generation text_to_audio -Four rules keep the table honest. +Five rules keep the table honest. * The key is `(operation, profile)`, never the route, never the stage. `/v1/audio/speech` is the single route of one operation and carries three @@ -26,30 +26,55 @@ refusing the others (`tests/config/test_serving_capabilities.py:233,251`: MOSS-SoundEffect serves `caption_to_audio` and reports `text_to_speech` unsupported on the same path). -* `stages[]` is deployment topology, not a capability. It publishes only - `id` / `type` / `depends_on` (`protocol/capabilities.py:67-70`), the ids are - opaque numeric strings, and every model is handed the *same* full list - (`entrypoints/openai/serving_capabilities.py:400`). Keying capabilities on - stages would therefore give every model one shared state and no diagnosis; - this module returns the topology for display and never reads it to decide a - capability. -* A declaration that says nothing is `unknown`, never `missing`. Absent fields - are the normal way to say "not established" (`response_model_exclude_none=True`, +* A profile is never force-fitted onto a neighbouring id. The three `chat` + profiles publish different output modalities — `text_response` is `("text",)`, + `image_response` is `("image",)` and `audio_response` is `("audio",)` + (`serving_capabilities.py:43-45`) — and `text.chat` is consumed by a + text-only adapter (`integrations/model_adapters.py:566-580` reads + `choices[0]["message"]["content"]` as a string), so only `text_response` + grants it: an image-only chat deployment must not read as able to chat. + `caption_to_audio` (sound effect) has no id here either. `text_to_audio` does + grant `audio.music`: it is text-to-audio, the same family as + `lyrics_to_music`, and the only difference is the diffusion request contract + behind `/v1/audio/generate` (`docs/serving/capabilities.md:59-60`). +* A declaration that says nothing is `unknown`, never `missing`. Optional fields + are simply absent from a published body (`response_model_exclude_none=True`, `api_server.py:2193`) and `unknown` is a first-class upstream value - (`support`, `exposure`, and "Missing operations/profiles are unknown", - `serving_capabilities.py:84`). Only positive evidence — `support=unsupported` - or `exposure=disabled` for the profile — produces `missing`. + (`support`, `exposure`; "Missing operations/profiles are unknown", + `serving_capabilities.py:84`). Required fields are never absent: `support`, + `exposure` and `reason_code` are mandatory on every operation and profile, and + all six operations must be listed (`protocol/capabilities.py:36-64`). A body + that omits them is malformed input to be defended against, not a shape + upstream publishes. Only positive evidence produces `missing`, and the result + keeps the two kinds apart in its `reasons` map: `unsupported` (the model + refused the profile), `endpoint_disabled` (the deployment switched the + operation's routes off) and `route_disabled` (every route the profile itself + declares is switched off). * A route without an operation proves nothing. `/v1/images/segmentations`, `/v1/videos/segmentations` and `/v1/videos/depth` are registered endpoints that no operation owns (`docs/serving/capabilities.md:41-43`): "segmentation and depth estimation do not acquire one of these operations merely by being present in a registry". Those capabilities stay `unknown` with `no_upstream_profile`. +* Reachability is decided on the profile's own route references, and their + `exposure` is read. Every declared route `disabled` means `missing` with + `route_disabled`; every route `unknown` (or no route at all) means `unknown`, + because reachability cannot be confirmed; one `enabled` route is enough. A + route this table does not recognise is reported in `unresolved`, but only + makes the profile `unknown` when nothing else was declared: upstream adds + routes over time, and one new path must not void a profile that already + answered. `asr` is not an omission on this side: upstream publishes no recognition -operation at all and the omni router never registers `/v1/audio/transcriptions`, -so a caller naming it gets `unmapped_stage` — an upstream gap to be recorded, -not a row we forgot to add, and never silently read as `audio.score`. +operation at all (`OperationId` is a closed `Literal`) and the omni router never +registers `/v1/audio/transcriptions` (no `@router` decorator in `api_server.py`; +`recipes/inclusionAI/Ming-3.0-audio-flash.md:331` says the endpoint "is not +currently available with `--omni`"). A body naming it is outside the vocabulary +and is recorded as `unmapped_stage`. `audio.score` therefore cannot be judged +from this endpoint: this repository's `VllmOmniScore` adapter does post to +`/audio/transcriptions` (`integrations/model_adapters.py:547,556`), but a +capabilities body never mentions that path, so the id stays `unknown` with +`no_upstream_operation` instead of being derived from a route we happen to know. Not yet confirmed upstream (recorded here instead of assumed): the exact `observed_at` string format, whether `stages[].type` is ever the `"unknown"` @@ -75,21 +100,30 @@ # (operation, profile, capability ids that profile grants). Order is part of the # contract: `resolve()` emits capabilities in this order, so a presentation # change is a diff in one place. Every profile upstream publishes has a row, -# including the two for which this repository has no capability id yet: an empty +# including the ones for which this repository has no capability id yet: an empty # tuple is the honest answer ("upstream says it can, we cannot name it"), and a # supported declaration of such a profile is reported as `no_local_capability` # rather than dropped or force-fitted to a neighbouring id. # +# `image_response` / `audio_response` are empty on purpose. They are chat +# profiles whose published output is an image or an audio clip +# (`serving_capabilities.py:44-45`), while `text.chat` here is served by a +# text-only adapter: mapping them would make an image-only deployment read as +# able to chat, and would contradict this module's own refusal to push +# `caption_to_audio` onto `audio.speech`. +# # `caption_to_audio` is a sound-effect contract (upstream requires # `ambient_sound`, `tests/config/test_serving_capabilities.py:233`), not speech; -# `text_to_audio` is the separate diffusion contract behind -# `/v1/audio/generate` (no voice, no lyrics; `docs/serving/capabilities.md:59-60`). -# Mapping either onto `audio.speech` would let a request carry a voice the -# deployment never accepted. +# mapping it onto `audio.speech` would let a request carry a voice the deployment +# never accepted. `text_to_audio` is the separate diffusion contract behind +# `/v1/audio/generate` (no voice, no lyrics; `docs/serving/capabilities.md:59-60`) +# but it is still text-to-audio, so it grants `audio.music` alongside +# `lyrics_to_music`: without that row a StableAudio/LTX2-t2a shape would report +# `audio.music=missing` while the deployment says it generates audio. OPERATION_PROFILES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( ('chat', 'text_response', ('text.chat',)), - ('chat', 'image_response', ('text.chat',)), - ('chat', 'audio_response', ('text.chat',)), + ('chat', 'image_response', ()), + ('chat', 'audio_response', ()), ('image_generation', 'text_to_image', ('image.generate',)), ('image_edit', 'image_to_image', ('image.edit',)), ('video_generation', 'text_to_video', ('video.generate',)), @@ -98,55 +132,81 @@ ('speech', 'text_to_speech', ('audio.speech',)), ('speech', 'lyrics_to_music', ('audio.music',)), ('speech', 'caption_to_audio', ()), - ('audio_generation', 'text_to_audio', ()), + ('audio_generation', 'text_to_audio', ('audio.music',)), ) -# operation → the routes that operation owns (`serving_capabilities.py:42-54`). -# Kept beside the profile table so a declared profile can be checked against the -# operation it claims: a profile pointing at another operation's path is a -# declaration error, not a capability. `/v1/realtime/video` is a websocket route, -# compared by path here because the capability question is about the endpoint. +# operation → the routes that operation owns, as `(transport, method, path)` +# (`entrypoints/openai/serving_capabilities.py:42-54`). Kept beside the profile +# table so a declared profile can be checked against the operation it claims: a +# profile pointing at another operation's path, or declaring a method or a +# transport the operation does not serve, is a declaration error, not a +# capability. The websocket route's `method` literal is `WEBSOCKET` +# (`protocol/capabilities.py:17`), which is why the method is compared and not +# merely the path: `DELETE /v1/chat/completions` is a 405, not chat. # Routes registered but owned by no operation (`/v1/audio/speech/batch`, # `/v1/chat/completions/batch`, the `segmentations`/`depth` endpoints, …) belong # to no value of this map, which is exactly why they cannot grant anything. -OPERATION_ROUTES: dict[str, tuple[str, ...]] = { - 'chat': ('/v1/chat/completions',), - 'image_generation': ('/v1/images/generations',), - 'image_edit': ('/v1/images/edits',), - 'video_generation': ('/v1/videos', '/v1/videos/sync', '/v1/realtime/video'), - 'speech': ('/v1/audio/speech',), - 'audio_generation': ('/v1/audio/generate',), +OPERATION_ROUTES: dict[str, tuple[tuple[str, str, str], ...]] = { + 'chat': (('http', 'POST', '/v1/chat/completions'),), + 'image_generation': (('http', 'POST', '/v1/images/generations'),), + 'image_edit': (('http', 'POST', '/v1/images/edits'),), + 'video_generation': (('http', 'POST', '/v1/videos'), + ('http', 'POST', '/v1/videos/sync'), + ('websocket', 'WEBSOCKET', '/v1/realtime/video')), + 'speech': (('http', 'POST', '/v1/audio/speech'),), + 'audio_generation': (('http', 'POST', '/v1/audio/generate'),), } AVAILABLE, MISSING, UNKNOWN = 'available', 'missing', 'unknown' -ABSENT, PRESENT = 'absent', 'present' +# `absent`: the endpoint did not answer (404, timeout, no capability body). +# `empty`: it answered with a model list but no model declared an operation. +# `present`: at least one model declared operations. +ABSENT, EMPTY, PRESENT = 'absent', 'empty', 'present' # Upstream's three-valued vocabulary, shared by `support` and `exposure` # (`serving_capabilities.py:16`, `protocol/capabilities.py:13`). SUPPORTED, UNSUPPORTED = 'supported', 'unsupported' ENABLED, DISABLED = 'enabled', 'disabled' # `unknown` is the third value of both, and also the state of an absent field. +ROUTE_UNKNOWN = UNKNOWN + +# Upstream's `transport` vocabulary (`protocol/capabilities.py:17`). +HTTP, WEBSOCKET = 'http', 'websocket' # Why a declaration could not be translated. Each names a different fix, so a # reviewer can tell an upstream gap from a bug in this table. UNKNOWN_ROUTE = 'unknown_route' # the path belongs to no operation at all -ROUTE_MISMATCH = 'route_mismatch' # the path belongs to a different operation -UNMAPPED_STAGE = 'unmapped_stage' # no such operation upstream (e.g. `asr`) +ROUTE_MISMATCH = 'route_mismatch' # path, transport or method belongs to another operation +UNMAPPED_STAGE = 'unmapped_stage' # no such operation upstream (e.g. `asr`); `stage` is the old name UNKNOWN_PROFILE = 'unknown_profile' # operation known, profile id not published OPERATION_MISMATCH = 'operation_mismatch' # profile known under another operation NO_LOCAL_CAPABILITY = 'no_local_capability' # upstream profile, no id here -REASONS = frozenset({UNKNOWN_ROUTE, ROUTE_MISMATCH, UNMAPPED_STAGE, UNKNOWN_PROFILE, - OPERATION_MISMATCH, NO_LOCAL_CAPABILITY}) +NO_ROUTE_DECLARED = 'no_route_declared' # supported profile that declares no route at all +MALFORMED_ROUTE = 'malformed_route' # a route entry without a usable path +REASONS = frozenset({UNKNOWN_ROUTE, ROUTE_MISMATCH, UNMAPPED_STAGE, UNKNOWN_PROFILE, OPERATION_MISMATCH, + NO_LOCAL_CAPABILITY, NO_ROUTE_DECLARED, MALFORMED_ROUTE}) # Why a capability can never be decided from a declaration here. NO_UPSTREAM_PROFILE, NO_UPSTREAM_OPERATION = 'no_upstream_profile', 'no_upstream_operation' PARAMETER_ONLY, NO_LOCAL_ADAPTER = 'parameter_only', 'no_local_adapter' UNROUTED_REASONS = frozenset({NO_UPSTREAM_PROFILE, NO_UPSTREAM_OPERATION, PARAMETER_ONLY, NO_LOCAL_ADAPTER}) +# Why a capability is `missing`. `support=unsupported` and `exposure=disabled` +# are both a positive no, but they are not the same fact: one is the model, the +# other is the deployment's switch. Kept apart in `resolve()['reasons']`, most +# specific first, so a screen can say "this deployment turned it off" instead of +# "this model cannot do it". +UNSUPPORTED_REASON = 'unsupported' +ENDPOINT_DISABLED_REASON = 'endpoint_disabled' +ROUTE_DISABLED_REASON = 'route_disabled' +MISSING_REASONS = (UNSUPPORTED_REASON, ENDPOINT_DISABLED_REASON, ROUTE_DISABLED_REASON) + # Capability ids no upstream profile can express, with the reason to show. Values # stay stable business codes; wording lives in the message catalogs. Every entry # here resolves to `unknown`: the endpoint does not publish the ability, so -# "cannot judge" is the only defensible state. +# "cannot judge" is the only defensible state. None of these ids is a value of +# `OPERATION_PROFILES`, which is what makes `unknown` fall out of "no evidence" +# rather than out of a special case in `resolve()`. UNROUTED: dict[str, str] = { # Endpoints exist, but no operation owns them (see the module docstring). 'image.segment': NO_UPSTREAM_PROFILE, @@ -165,7 +225,7 @@ BY_PAIR: dict[tuple[str, str], tuple[str, ...]] = {(operation, profile): capabilities for operation, profile, capabilities in OPERATION_PROFILES} PROFILE_OPERATIONS: dict[str, str] = {profile: operation for operation, profile, _ in OPERATION_PROFILES} -ROUTED_PATHS = frozenset(path for paths in OPERATION_ROUTES.values() for path in paths) +ROUTED_PATHS = frozenset(path for routes in OPERATION_ROUTES.values() for _, _, path in routes) def route_path(value): @@ -191,6 +251,27 @@ def route_path(value): return API_PREFIX + path +def _route_key(value): + """`(transport, method, path)` of a declared route; path is None when unusable. + + `transport` defaults to `http` upstream (`protocol/capabilities.py:16-21`), so + an absent transport is not a mismatch. `method` has no default there: an + absent or blank method cannot be confirmed, and is compared as an empty + string that matches no owned route. + """ + transport = value.get('transport') if isinstance(value, dict) else None + method = value.get('method') if isinstance(value, dict) else None + transport = transport.strip().lower() if isinstance(transport, str) and transport.strip() else HTTP + method = method.strip().upper() if isinstance(method, str) and method.strip() else '' + return transport, method, route_path(value) + + +def _route_exposure(value): + """Declared exposure of one route reference; absent is `unknown` (the upstream default).""" + exposure = value.get('exposure') if isinstance(value, dict) else None + return exposure if exposure in (ENABLED, DISABLED) else ROUTE_UNKNOWN + + def capabilities_for(operation, profile): """Capability ids for one pair, or `()` when this table does not place it. @@ -297,32 +378,44 @@ def _topology(payload): def _operation_verdict(operation): - """`yes` / `no` / `unknown` for an operation from its own two enums. + """`(verdict, reason)` for an operation from its own two enums. Upstream keeps `support` and `exposure` independent: a supported operation can have disabled routes, and an installed route can coexist with an unsupported operation (`docs/serving/capabilities.md:45-51`). Either one - saying no is a definitive no for the deployment as it stands; anything less - than both saying yes is not enough to claim availability. + saying no is a definitive no for the deployment as it stands — but the reason + says which: the model refused, or the deployment closed the route. """ support, exposure = operation.get('support'), operation.get('exposure') - if support == UNSUPPORTED or exposure == DISABLED: - return 'no' + if support == UNSUPPORTED: + return 'no', UNSUPPORTED_REASON + if exposure == DISABLED: + return 'no', ENDPOINT_DISABLED_REASON if support == SUPPORTED and exposure == ENABLED: - return 'yes' - return 'unknown' + return 'yes', None + return 'unknown', None -def _profile_verdict(entry, operation_verdict): - """`yes` / `no` / `unknown` for one profile inside an already-judged operation.""" - if operation_verdict == 'no': - return 'no' +def _profile_verdict(entry, operation_verdict, operation_reason, exposures): + """`(verdict, reason)` for one profile inside an already-judged operation. + + `exposures` holds one value per route reference the operation owns: an empty + list means the profile declared no usable route of its own. + """ support = entry.get('support') if support == UNSUPPORTED: - return 'no' + return 'no', UNSUPPORTED_REASON + if operation_verdict == 'no': + return 'no', operation_reason if support != SUPPORTED or operation_verdict != 'yes': - return 'unknown' - return 'yes' + return 'unknown', None + if not exposures: + return 'unknown', None + if ENABLED in exposures: + return 'yes', None + if ROUTE_UNKNOWN in exposures: + return 'unknown', None + return 'no', ROUTE_DISABLED_REASON def _note(unresolved, operation, profile, route, reason): @@ -332,15 +425,36 @@ def _note(unresolved, operation, profile, route, reason): unresolved.append(entry) -def _check_route(operation, profile, path, unresolved): - """True when `path` is one the operation owns; otherwise report why not.""" - if path in OPERATION_ROUTES[operation]: +def _check_route(operation, profile, key, unresolved): + """True when `(transport, method, path)` is one the operation owns; else report why not.""" + if key in OPERATION_ROUTES[operation]: return True + path = key[2] _note(unresolved, operation, profile, path, ROUTE_MISMATCH if path in ROUTED_PATHS else UNKNOWN_ROUTE) return False +def _route_exposures(operation, profile, entry, unresolved): + """`(exposures of the routes this operation owns, whether any path was declared)`. + + A route reference whose path cannot be read is reported rather than dropped: + silently ignoring it would turn a broken declaration into `unknown` with no + explanation, which is the failure this module exists to prevent. + """ + exposures = [] + declared_path = False + for value in _routes(entry): + key = _route_key(value) + if key[2] is None: + _note(unresolved, operation, profile, None, MALFORMED_ROUTE) + continue + declared_path = True + if _check_route(operation, profile, key, unresolved): + exposures.append(_route_exposure(value)) + return exposures, declared_path + + def resolve(payload): """Per-capability availability, the declared topology, and untranslated claims. @@ -348,22 +462,31 @@ def resolve(payload): (404, timeout, disabled endpoint). Model names and any other provider fields never reach the result. - `endpoint` is `absent` when the body carries no operation entry at all, and - then every capability is `unknown` and nothing is listed as unresolved. - - A capability is `available` only when some model's profile says so and its - operation is enabled; `missing` only when a profile or operation positively - says no; `unknown` otherwise — no body, an absent field, an unknown enum, an - unrouted capability, or a declaration this table could not translate. + `endpoint` separates "did not answer" from "answered without declaring": + `absent` when there is no body or no model list at all, `empty` when a model + list came back but no model declared an operation, `present` otherwise. In + the first two cases every capability is `unknown`, `reasons` is empty and + nothing is listed as unresolved. + + A capability is `available` only when some model's profile says so, its + operation is enabled and at least one of its own routes is enabled; + `missing` only when a profile or operation positively says no, with the kind + of refusal in `reasons` (`unsupported`, `endpoint_disabled`, + `route_disabled`); `unknown` otherwise — no body, an absent field, an unknown + enum, an unrouted capability, or a declaration this table could not + translate. """ vocabulary = classified_capabilities() unresolved: list[dict] = [] topology = _topology(payload) evidence: dict[str, set[str]] = {} + refusals: dict[str, set[str]] = {} - def record(capabilities, verdict): + def record(capabilities, verdict, reason=None): for capability in capabilities: evidence.setdefault(capability, set()).add(verdict) + if verdict == 'no' and reason: + refusals.setdefault(capability, set()).add(reason) models = payload.get('models') if isinstance(payload, dict) else None declared = False @@ -384,7 +507,7 @@ def record(capabilities, verdict): if row_operation == key: record(row_capabilities, 'unknown') continue - operation_verdict = _operation_verdict(operation) + operation_verdict, operation_reason = _operation_verdict(operation) seen = set() for profile_id, entry in _profiles(operation): seen.add(profile_id) @@ -395,16 +518,15 @@ def record(capabilities, verdict): if owner != key: _note(unresolved, key, profile_id, None, OPERATION_MISMATCH) continue - paths = [path for path in map(route_path, _routes(entry)) if path] - broken = [path for path in paths if not _check_route(key, profile_id, path, unresolved)] - verdict = _profile_verdict(entry, operation_verdict) + exposures, declared_path = _route_exposures(key, profile_id, entry, unresolved) + verdict, reason = _profile_verdict(entry, operation_verdict, operation_reason, exposures) + # A supported profile that declares no route at all: reachability + # cannot be confirmed, and nothing else explains the `unknown`. + if verdict == 'unknown' and not exposures and not declared_path: + _note(unresolved, key, profile_id, None, NO_ROUTE_DECLARED) capabilities = BY_PAIR[(key, profile_id)] - # Without a route we cannot confirm reachability, and a profile - # pointing at another operation's path tells us nothing reliable. - if verdict != 'no' and (broken or not paths): - verdict = 'unknown' if capabilities: - record(capabilities, verdict) + record(capabilities, verdict, reason) elif verdict == 'yes': _note(unresolved, key, profile_id, None, NO_LOCAL_CAPABILITY) # A profile the operation never mentions establishes nothing, unless @@ -412,20 +534,32 @@ def record(capabilities, verdict): # standard profiles with unsupported upstream, so silence is silence). for row_operation, profile_id, row_capabilities in OPERATION_PROFILES: if row_operation == key and profile_id not in seen: - record(row_capabilities, 'no' if operation_verdict == 'no' else 'unknown') + if operation_verdict == 'no': + record(row_capabilities, 'no', operation_reason) + else: + record(row_capabilities, 'unknown') if not declared: - return {'endpoint': ABSENT, 'topology': topology, - 'capabilities': {item: UNKNOWN for item in vocabulary}, 'unresolved': []} + # `models` being a list is the service answering; no operations at all + # means it answered without declaring anything, which is not the same as + # never having answered. + endpoint = EMPTY if isinstance(models, list) else ABSENT + return {'endpoint': endpoint, 'topology': topology, 'capabilities': {item: UNKNOWN for item in vocabulary}, + 'reasons': {}, 'unresolved': []} capabilities = {} + reasons = {} for item in vocabulary: verdicts = evidence.get(item, set()) if 'yes' in verdicts: capabilities[item] = AVAILABLE - elif item in UNROUTED or 'unknown' in verdicts or not verdicts: + elif ROUTE_UNKNOWN in verdicts or not verdicts: capabilities[item] = UNKNOWN else: capabilities[item] = MISSING - return {'endpoint': PRESENT, 'topology': topology, 'capabilities': capabilities, + refusal = refusals.get(item, set()) + reason = next((name for name in MISSING_REASONS if name in refusal), None) + if reason: + reasons[item] = reason + return {'endpoint': PRESENT, 'topology': topology, 'capabilities': capabilities, 'reasons': reasons, 'unresolved': sorted(unresolved, key=lambda item: (item['operation'] or '', item['profile'] or '', item['route'] or '', item['reason']))} diff --git a/tests/fixtures/omni-capabilities/absent.json b/tests/fixtures/omni-capabilities/absent.json index 63aa799c..efe9c6df 100644 --- a/tests/fixtures/omni-capabilities/absent.json +++ b/tests/fixtures/omni-capabilities/absent.json @@ -1,7 +1,7 @@ { "tier": "absent", "upstreamEndpoint": "/v1/omni/capabilities", - "scope": "No usable capability evidence: an unreachable endpoint, a topology-only body, and a body whose optional fields were dropped by exclude_none.", + "scope": "No usable capability evidence: no answer at all, an answer that carries only topology, and an answer whose operations declare no profile.", "cases": [ { "id": "endpoint-unreachable", @@ -26,12 +26,13 @@ "image.cutout": "unknown", "audio.separate": "unknown" }, + "reasons": {}, "unresolved": [] } }, { "id": "topology-only-body", - "note": "Stages are published but no model and no operation: topology is display, so it grants nothing.", + "note": "Stages are published but no model declared an operation: the service answered without declaring anything (`empty`, not `absent`), and topology is display.", "payload": { "object": "omni.capabilities", "schema_version": 1, @@ -121,7 +122,7 @@ ] }, "expected": { - "endpoint": "absent", + "endpoint": "empty", "topology": [ { "id": "7", @@ -152,12 +153,13 @@ "image.cutout": "unknown", "audio.separate": "unknown" }, + "reasons": {}, "unresolved": [] } }, { "id": "operations-without-profiles", - "note": "The operation answered but `profiles` was dropped: an absent field means unknown, not unsupported.", + "note": "Six operations answered `supported`/`enabled` and each declares `profiles: []` — the published shape of \"no profile declared\" (`default_factory=list`), so every capability is `unknown`, not `missing`.", "payload": { "object": "omni.capabilities", "schema_version": 1, @@ -187,6 +189,62 @@ "support": "supported", "exposure": "enabled", "reason_code": "model_declared", + "profiles": [], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_edit": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "video_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "speech": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "audio_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [], "execution": { "state": "unpaused", "scope": "worker", @@ -297,6 +355,7 @@ "image.cutout": "unknown", "audio.separate": "unknown" }, + "reasons": {}, "unresolved": [] } } diff --git a/tests/fixtures/omni-capabilities/full.json b/tests/fixtures/omni-capabilities/full.json index e15058f0..f447fdad 100644 --- a/tests/fixtures/omni-capabilities/full.json +++ b/tests/fixtures/omni-capabilities/full.json @@ -5,7 +5,7 @@ "cases": [ { "id": "every-operation-and-profile", - "note": "All six operations supported and enabled, all twelve profiles declared; `text_to_audio` is supported but this repository has no id for it.", + "note": "All six operations supported and enabled, all twelve profiles declared. Only `text_response` grants `text.chat`: the image and audio chat profiles are reported as `no_local_capability`, and `text_to_audio` grants `audio.music`.", "payload": { "object": "omni.capabilities", "schema_version": 1, @@ -64,7 +64,6 @@ "text" ], "outputs": [ - "text", "image" ], "routes": [ @@ -85,7 +84,6 @@ "text" ], "outputs": [ - "text", "audio" ], "routes": [ @@ -318,8 +316,12 @@ { "id": "caption_to_audio", "support": "unsupported", - "required_inputs": [], - "outputs": [], + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], "routes": [ { "transport": "http", @@ -474,10 +476,17 @@ "image.cutout": "unknown", "audio.separate": "unknown" }, + "reasons": {}, "unresolved": [ { - "operation": "audio_generation", - "profile": "text_to_audio", + "operation": "chat", + "profile": "audio_response", + "route": null, + "reason": "no_local_capability" + }, + { + "operation": "chat", + "profile": "image_response", "route": null, "reason": "no_local_capability" } diff --git a/tests/fixtures/omni-capabilities/partial.json b/tests/fixtures/omni-capabilities/partial.json index 50c88c4b..be260db1 100644 --- a/tests/fixtures/omni-capabilities/partial.json +++ b/tests/fixtures/omni-capabilities/partial.json @@ -1,11 +1,11 @@ { "tier": "partial", "upstreamEndpoint": "/v1/omni/capabilities", - "scope": "Bodies that answer, but disagree with themselves or with the vocabulary: unsupported, disabled, a misplaced route, an omitted field, unknown names.", + "scope": "Bodies that answer, but disagree with themselves: a policy-disabled operation, a profile whose own route is disabled, a misplaced route, an unsupported profile, and an operation that states `unknown`.", "cases": [ { "id": "mixed-operations", - "note": "`image_generation` is supported but disabled, `image_edit` unsupported, one video profile declares an image path (route_mismatch), one profile omits `support`, `caption_to_audio` is supported with no local id.", + "note": "`image_generation` is supported but disabled (`endpoint_disabled`), `image_edit` is supported but its only route is `disabled` (`route_disabled`), one video profile declares an image path (`route_mismatch`), `lyrics_to_music` is unsupported while `text_to_audio` is supported, and `caption_to_audio` has no local id.", "payload": { "object": "omni.capabilities", "schema_version": 1, @@ -59,8 +59,12 @@ { "id": "image_response", "support": "unsupported", - "required_inputs": [], - "outputs": [], + "required_inputs": [ + "text" + ], + "outputs": [ + "image" + ], "routes": [ { "transport": "http", @@ -74,11 +78,11 @@ }, { "id": "audio_response", + "support": "unknown", "required_inputs": [ "text" ], "outputs": [ - "text", "audio" ], "routes": [ @@ -89,7 +93,7 @@ "exposure": "enabled" } ], - "reason_code": "request_contract_unverified", + "reason_code": "chat_prompt_contract_undeclared", "constraints": {} } ], @@ -132,24 +136,29 @@ } }, "image_edit": { - "support": "unsupported", + "support": "supported", "exposure": "enabled", "reason_code": "model_declared", "profiles": [ { "id": "image_to_image", - "support": "unsupported", - "required_inputs": [], - "outputs": [], + "support": "supported", + "required_inputs": [ + "image", + "text" + ], + "outputs": [ + "image" + ], "routes": [ { "transport": "http", "method": "POST", "path": "/v1/images/edits", - "exposure": "enabled" + "exposure": "disabled" } ], - "reason_code": "model_not_supported", + "reason_code": "model_declared", "constraints": {} } ], @@ -208,8 +217,13 @@ { "id": "video_to_video", "support": "unsupported", - "required_inputs": [], - "outputs": [], + "required_inputs": [ + "video", + "text" + ], + "outputs": [ + "video" + ], "routes": [ { "transport": "http", @@ -256,8 +270,13 @@ { "id": "lyrics_to_music", "support": "unsupported", - "required_inputs": [], - "outputs": [], + "required_inputs": [ + "text", + "instructions" + ], + "outputs": [ + "audio" + ], "routes": [ { "transport": "http", @@ -297,7 +316,9 @@ } }, "audio_generation": { - "reason_code": "request_contract_unverified", + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", "profiles": [ { "id": "text_to_audio", @@ -420,7 +441,7 @@ "image.edit": "missing", "video.generate": "available", "audio.speech": "available", - "audio.music": "missing", + "audio.music": "available", "image.segment": "unknown", "video.matting": "unknown", "video.motion_capture": "unknown", @@ -430,6 +451,10 @@ "image.cutout": "unknown", "audio.separate": "unknown" }, + "reasons": { + "image.generate": "endpoint_disabled", + "image.edit": "route_disabled" + }, "unresolved": [ { "operation": "speech", @@ -447,8 +472,8 @@ } }, { - "id": "unknown-vocabulary", - "note": "`asr` is not an upstream operation, `speech_to_speech` is not a profile, `lyrics_to_music` is filed under the wrong operation, and `/v1/videos/track` belongs to no operation at all.", + "id": "unknown-enums", + "note": "Every operation and profile states `unknown`: the third upstream value is a real answer, so nothing is claimed and nothing is refused.", "payload": { "object": "omni.capabilities", "schema_version": 1, @@ -474,61 +499,229 @@ "7" ], "operations": { - "asr": { - "support": "supported", - "exposure": "enabled", + "chat": { + "support": "unknown", + "exposure": "unknown", "reason_code": "model_declared", - "profiles": [], + "profiles": [ + { + "id": "text_response", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "text" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "request_contract_unverified", + "constraints": {} + }, + { + "id": "image_response", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "request_contract_unverified", + "constraints": {} + }, + { + "id": "audio_response", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "request_contract_unverified", + "constraints": {} + } + ], "execution": { "state": "unpaused", "scope": "worker", "observed_at": "2026-11-10T00:00:00Z" } }, - "speech": { - "support": "supported", - "exposure": "enabled", + "image_generation": { + "support": "unknown", + "exposure": "unknown", "reason_code": "model_declared", "profiles": [ { - "id": "text_to_speech", - "support": "supported", + "id": "text_to_image", + "support": "unknown", "required_inputs": [ "text" ], "outputs": [ - "audio" + "image" ], "routes": [ { "transport": "http", "method": "POST", - "path": "/v1/audio/speech", + "path": "/v1/images/generations", "exposure": "enabled" } ], - "reason_code": "model_declared", + "reason_code": "request_contract_unverified", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_edit": { + "support": "unknown", + "exposure": "unknown", + "reason_code": "model_declared", + "profiles": [ + { + "id": "image_to_image", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + } + ], + "reason_code": "request_contract_unverified", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "video_generation": { + "support": "unknown", + "exposure": "unknown", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_video", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + } + ], + "reason_code": "request_contract_unverified", "constraints": {} }, { - "id": "speech_to_speech", - "support": "supported", + "id": "image_to_video", + "support": "unknown", "required_inputs": [ - "audio", "text" ], "outputs": [ - "audio" + "video" ], "routes": [ { "transport": "http", "method": "POST", - "path": "/v1/audio/speech", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", "exposure": "enabled" } ], - "reason_code": "model_declared", + "reason_code": "request_contract_unverified", + "constraints": {} + }, + { + "id": "video_to_video", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + } + ], + "reason_code": "request_contract_unverified", "constraints": {} } ], @@ -538,17 +731,36 @@ "observed_at": "2026-11-10T00:00:00Z" } }, - "audio_generation": { - "support": "supported", - "exposure": "enabled", + "speech": { + "support": "unknown", + "exposure": "unknown", "reason_code": "model_declared", "profiles": [ + { + "id": "text_to_speech", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "request_contract_unverified", + "constraints": {} + }, { "id": "lyrics_to_music", - "support": "supported", + "support": "unknown", "required_inputs": [ - "text", - "instructions" + "text" ], "outputs": [ "audio" @@ -561,7 +773,27 @@ "exposure": "enabled" } ], - "reason_code": "model_declared", + "reason_code": "request_contract_unverified", + "constraints": {} + }, + { + "id": "caption_to_audio", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "request_contract_unverified", "constraints": {} } ], @@ -571,29 +803,29 @@ "observed_at": "2026-11-10T00:00:00Z" } }, - "video_generation": { - "support": "supported", - "exposure": "enabled", + "audio_generation": { + "support": "unknown", + "exposure": "unknown", "reason_code": "model_declared", "profiles": [ { - "id": "text_to_video", - "support": "supported", + "id": "text_to_audio", + "support": "unknown", "required_inputs": [ "text" ], "outputs": [ - "video" + "audio" ], "routes": [ { "transport": "http", "method": "POST", - "path": "/v1/videos/track", + "path": "/v1/audio/generate", "exposure": "enabled" } ], - "reason_code": "model_declared", + "reason_code": "request_contract_unverified", "constraints": {} } ], @@ -696,7 +928,7 @@ "image.generate": "unknown", "image.edit": "unknown", "video.generate": "unknown", - "audio.speech": "available", + "audio.speech": "unknown", "audio.music": "unknown", "image.segment": "unknown", "video.matting": "unknown", @@ -707,32 +939,8 @@ "image.cutout": "unknown", "audio.separate": "unknown" }, - "unresolved": [ - { - "operation": "asr", - "profile": null, - "route": null, - "reason": "unmapped_stage" - }, - { - "operation": "audio_generation", - "profile": "lyrics_to_music", - "route": null, - "reason": "operation_mismatch" - }, - { - "operation": "speech", - "profile": "speech_to_speech", - "route": null, - "reason": "unknown_profile" - }, - { - "operation": "video_generation", - "profile": "text_to_video", - "route": "/v1/videos/track", - "reason": "unknown_route" - } - ] + "reasons": {}, + "unresolved": [] } } ] diff --git a/tests/test_capability_contract.py b/tests/test_capability_contract.py index 5c86e0da..db3e84a1 100644 --- a/tests/test_capability_contract.py +++ b/tests/test_capability_contract.py @@ -2,10 +2,18 @@ `tests/fixtures/omni-capabilities/` is the shared sample set: three tiers built from the response shape upstream actually publishes — `stages[]` as topology, -`models[].operations[].profiles[]` as the declaration, absent fields -instead of nulls. The expected blocks are the reviewed oracle, not a copy of the +`models[].operations[].profiles[]` as the declaration, and only +optional fields ever absent. `test_fixtures_are_schema_valid` checks the +structural invariants of that shape (`protocol/capabilities.py:16-79`), so a +fixture that upstream's own pydantic model would reject cannot be frozen into +the oracle. The expected blocks are the reviewed oracle, not a copy of the mapping table, so changing a mapping has to argue with them. +Shapes upstream can never publish — an `asr` operation key, a profile filed under +the wrong operation, a missing required `support` — are built inline in the +tests that defend against malformed input, never in a fixture, so the fixtures +stay an honest sample of real bodies. + The last group guards the failure this table exists to prevent: a route (or a stage) that is read as if it named a capability. `/v1/audio/speech` carries three profiles, `stages[].id` is opaque topology, and a registered endpoint @@ -19,7 +27,8 @@ import pytest from opendirector.domain.capabilities import Capability -from opendirector.domain.capability_contract import (ABSENT, AVAILABLE, ENDPOINT, MISSING, NO_LOCAL_CAPABILITY, +from opendirector.domain.capability_contract import (ABSENT, AVAILABLE, EMPTY, ENDPOINT, MALFORMED_ROUTE, MISSING, + MISSING_REASONS, NO_LOCAL_CAPABILITY, NO_ROUTE_DECLARED, OPERATION_MISMATCH, OPERATION_PROFILES, OPERATION_ROUTES, OPERATIONS, PRESENT, REASONS, ROUTED_PATHS, ROUTE_MISMATCH, UNKNOWN, UNKNOWN_PROFILE, UNKNOWN_ROUTE, UNMAPPED_STAGE, UNROUTED, @@ -34,6 +43,25 @@ OBSERVED = '2026-11-10T00:00:00Z' EXECUTION = {'state': 'unpaused', 'scope': 'worker', 'observed_at': OBSERVED} +# The vocabulary in contract order: routed ids in table order, then the unrouted +# ids in declaration order. Written out instead of derived, so a reordering of +# the tables is a visible decision rather than a silent presentation change. +VOCABULARY_ORDER = ['text.chat', 'image.generate', 'image.edit', 'video.generate', 'audio.speech', 'audio.music', + 'image.segment', 'video.matting', 'video.motion_capture', 'audio.score', 'video.first_frame', + 'video.edit', 'image.cutout', 'audio.separate'] + +# Every capability id that no upstream profile can express, with its reason. +UNROUTED_EXPECTED = { + 'image.segment': 'no_upstream_profile', + 'video.matting': 'no_upstream_profile', + 'video.motion_capture': 'no_upstream_profile', + 'audio.score': 'no_upstream_operation', + 'video.first_frame': 'parameter_only', + 'video.edit': 'parameter_only', + 'image.cutout': 'parameter_only', + 'audio.separate': 'no_local_adapter', +} + def raw_cases(): paths = sorted(FIXTURES.glob('*.json')) @@ -54,9 +82,20 @@ def route(path, exposure='enabled', method='POST', transport='http'): return {'transport': transport, 'method': method, 'path': path, 'exposure': exposure} -def profile(profile_id, support='supported', paths=(), inputs=('text',), outputs=('text',), **extra): +def table_routes(operation_id): + """The operation's owned routes as declarations, straight from the table.""" + return [route(path, method=method, transport=transport) + for transport, method, path in OPERATION_ROUTES[operation_id]] + + +def table_paths(operation_id): + return [path for _, _, path in OPERATION_ROUTES[operation_id]] + + +def profile(profile_id, support='supported', paths=(), inputs=('text',), outputs=('text',), routes=None, **extra): entry = {'id': profile_id, 'support': support, 'required_inputs': list(inputs), 'outputs': list(outputs), - 'routes': [route(path) for path in paths], 'reason_code': 'model_declared', 'constraints': {}} + 'routes': [route(path) for path in paths] if routes is None else list(routes), + 'reason_code': 'model_declared', 'constraints': {}} entry.update(extra) return entry @@ -81,18 +120,12 @@ def body(operations=None, models=None, stages=None): def every_operation(): """All six operations supported and enabled, every profile they publish.""" - return {operation_id: operation(profiles=[profile(profile_id, paths=OPERATION_ROUTES[operation_id]) + return {operation_id: operation(profiles=[profile(profile_id, routes=table_routes(operation_id)) for owner, profile_id, _ in OPERATION_PROFILES if owner == operation_id]) for operation_id in OPERATIONS} -def routes_of(operation_id, profile_id): - return [path for owner, row_profile, _ in OPERATION_PROFILES - if (owner, row_profile) == (operation_id, profile_id) - for path in OPERATION_ROUTES[operation_id]] - - @pytest.mark.parametrize('payload, expected', list(fixture_cases())) def test_declaration_resolves_to_the_reviewed_expectation(payload, expected): assert resolve(payload) == expected @@ -101,7 +134,13 @@ def test_declaration_resolves_to_the_reviewed_expectation(payload, expected): # --- the table itself ----------------------------------------------------- def test_every_capability_id_is_routed_or_explained(): - """Adding a capability to the vocabulary without placing it here fails this.""" + """Adding a capability to the vocabulary without placing it here fails this. + + This is also the invariant that keeps an unrouted id out of `resolve()`: no + row of `OPERATION_PROFILES` maps to an id in `UNROUTED`, so an unrouted id + can never collect evidence and is `unknown` by "no evidence", not by a + special case. + """ placed = {capability for _, _, capabilities in OPERATION_PROFILES for capability in capabilities} assert set(classified_capabilities()) == set(get_args(Capability)) assert not placed & set(UNROUTED), 'a capability is either routed or unrouted, never both' @@ -133,30 +172,44 @@ def test_rows_and_routes_are_canonical(): assert operation_id == operation_id.strip().lower(), (operation_id, profile_id) assert profile_id == profile_id.strip().lower(), (operation_id, profile_id) assert capabilities == tuple(capabilities), (operation_id, profile_id) - for operation_id, paths in OPERATION_ROUTES.items(): - assert paths, operation_id - for path in paths: + for operation_id, routes in OPERATION_ROUTES.items(): + assert routes, operation_id + for transport, method, path in routes: + assert transport in {'http', 'websocket'}, (operation_id, transport) + assert method and method == method.upper(), (operation_id, method) assert route_path(path) == path, path assert path.startswith('/v1/'), path def test_operation_routes_follow_upstream_ownership(): assert set(OPERATION_ROUTES) == set(OPERATIONS) - assert OPERATION_ROUTES['chat'] == ('/v1/chat/completions',) - assert OPERATION_ROUTES['image_generation'] == ('/v1/images/generations',) - assert OPERATION_ROUTES['image_edit'] == ('/v1/images/edits',) + assert OPERATION_ROUTES['chat'] == (('http', 'POST', '/v1/chat/completions'),) + assert OPERATION_ROUTES['image_generation'] == (('http', 'POST', '/v1/images/generations'),) + assert OPERATION_ROUTES['image_edit'] == (('http', 'POST', '/v1/images/edits'),) # The only operation with more than one route, including the websocket one. - assert OPERATION_ROUTES['video_generation'] == ('/v1/videos', '/v1/videos/sync', '/v1/realtime/video') - assert OPERATION_ROUTES['speech'] == ('/v1/audio/speech',) - assert OPERATION_ROUTES['audio_generation'] == ('/v1/audio/generate',) - assert {'/v1/videos/sync', '/v1/realtime/video'} <= ROUTED_PATHS - - -def test_unowned_endpoints_belong_to_no_operation(): + assert OPERATION_ROUTES['video_generation'] == (('http', 'POST', '/v1/videos'), + ('http', 'POST', '/v1/videos/sync'), + ('websocket', 'WEBSOCKET', '/v1/realtime/video')) + assert OPERATION_ROUTES['speech'] == (('http', 'POST', '/v1/audio/speech'),) + assert OPERATION_ROUTES['audio_generation'] == (('http', 'POST', '/v1/audio/generate'),) + # `WEBSOCKET` is a method literal, not a transport of its own: exactly one + # route carries it, and only video generation owns that route. + assert [operation_id for operation_id, routes in OPERATION_ROUTES.items() + if any(method == 'WEBSOCKET' for _, method, _ in routes)] == ['video_generation'] + + +def test_unowned_endpoints_are_not_in_the_route_table(): """Registered routes without an operation must not be readable as capabilities.""" - for path in ('/v1/images/segmentations', '/v1/videos/segmentations', '/v1/videos/depth', - '/v1/audio/transcriptions', '/v1/audio/speech/batch', '/v1/chat/completions/batch'): + unowned = ('/v1/images/segmentations', '/v1/videos/segmentations', '/v1/videos/depth', + '/v1/audio/transcriptions', '/v1/audio/speech/batch', '/v1/chat/completions/batch') + for path in unowned: assert path not in ROUTED_PATHS, path + # Publishing them in the body's top-level `routes[]` still grants nothing. + payload = body(models=[]) + payload['routes'] = [route(path) for path in unowned] + result = resolve(payload) + for capability in ('image.segment', 'video.matting', 'video.motion_capture', 'audio.score'): + assert result['capabilities'][capability] == UNKNOWN, capability def test_music_is_a_speech_profile_on_the_speech_route(): @@ -164,8 +217,8 @@ def test_music_is_a_speech_profile_on_the_speech_route(): assert capabilities_for('speech', 'lyrics_to_music') == ('audio.music',) assert capabilities_for('audio_generation', 'lyrics_to_music') == () assert 'music' not in OPERATIONS - assert '/v1/audio/generate' not in OPERATION_ROUTES['speech'] - assert '/v1/audio/speech' not in OPERATION_ROUTES['audio_generation'] + assert '/v1/audio/generate' not in table_paths('speech') + assert '/v1/audio/speech' not in table_paths('audio_generation') assert '/v1/audio/music' not in ROUTED_PATHS and '/v1/audio/music/generations' not in ROUTED_PATHS @@ -176,17 +229,32 @@ def test_real_shaped_body_resolves_non_empty_pairs(): payload = body(every_operation()) pairs = declarations(payload) assert pairs and len(pairs) == 12 - assert set(pairs) == {(operation_id, profile_id) for operation_id, profile_id, _ in OPERATION_PROFILES} + # Literal on purpose: recomputing this from `OPERATION_PROFILES` would only + # prove the table echoes itself. + assert set(pairs) == {('chat', 'text_response'), ('chat', 'image_response'), ('chat', 'audio_response'), + ('image_generation', 'text_to_image'), ('image_edit', 'image_to_image'), + ('video_generation', 'text_to_video'), ('video_generation', 'image_to_video'), + ('video_generation', 'video_to_video'), ('speech', 'text_to_speech'), + ('speech', 'lyrics_to_music'), ('speech', 'caption_to_audio'), + ('audio_generation', 'text_to_audio')} result = resolve(payload) assert result['endpoint'] == PRESENT for capability in ('text.chat', 'image.generate', 'image.edit', 'video.generate', 'audio.speech', 'audio.music'): assert result['capabilities'][capability] == AVAILABLE, capability assert result['topology'] == [{'id': '7', 'type': 'llm', 'depends_on': []}] + # `not_declared_locally`: the chat profiles whose output is not text carry no + # id here, so they are reported instead of being folded into `text.chat`. + assert result['unresolved'] == [{'operation': 'chat', 'profile': 'audio_response', 'route': None, + 'reason': NO_LOCAL_CAPABILITY}, + {'operation': 'chat', 'profile': 'image_response', 'route': None, + 'reason': NO_LOCAL_CAPABILITY}, + {'operation': 'speech', 'profile': 'caption_to_audio', 'route': None, + 'reason': NO_LOCAL_CAPABILITY}] def test_route_alone_cannot_distinguish_speech_profiles(): """`/v1/audio/speech` is one path and three contracts; only the profile decides.""" - speech_routes = OPERATION_ROUTES['speech'] + speech_routes = table_paths('speech') assert len(speech_routes) == 1 resolved = {profile_id: capabilities_for('speech', profile_id) for profile_id in ('text_to_speech', 'lyrics_to_music', 'caption_to_audio')} @@ -208,18 +276,16 @@ def test_route_alone_cannot_distinguish_speech_profiles(): def test_stage_ids_do_not_collapse_capabilities(): """Every model gets the same `stage_ids`; the operations must still decide separately.""" - first = model({'chat': operation(profiles=[profile('text_response', paths=routes_of('chat', 'text_response'))]), + first = model({'chat': operation(profiles=[profile('text_response', paths=table_paths('chat'))]), 'image_generation': operation(exposure='disabled', profiles=[profile('text_to_image', - paths=routes_of('image_generation', - 'text_to_image'))])}, + paths=table_paths('image_generation'))])}, model_id='model-a', stage_ids=('7', '42')) second = model({'chat': operation(support='unsupported', profiles=[profile('text_response', 'unsupported', - paths=routes_of('chat', 'text_response'))]), + paths=table_paths('chat'))]), 'image_generation': operation(profiles=[profile('text_to_image', - paths=routes_of('image_generation', - 'text_to_image'))])}, + paths=table_paths('image_generation'))])}, model_id='model-b', stage_ids=('7', '42')) assert first['stage_ids'] == second['stage_ids'] result = resolve(body(models=[first, second])) @@ -231,48 +297,38 @@ def test_stage_ids_do_not_collapse_capabilities(): assert resolve(body(models=[second]))['capabilities']['text.chat'] == MISSING -def test_unrouted_capabilities_are_unknown_not_missing(): +def test_capabilities_without_an_upstream_profile_are_unknown_not_missing(): """An endpoint that no operation publishes cannot be judged, so it is never `missing`.""" + assert UNROUTED == UNROUTED_EXPECTED + assert set(UNROUTED.values()) <= UNROUTED_REASONS result = resolve(body(every_operation())) - for capability, reason in UNROUTED.items(): - assert result['capabilities'][capability] == UNKNOWN, capability - assert reason in UNROUTED_REASONS, capability - assert {UNROUTED[capability] for capability in ('image.segment', 'video.matting', 'video.motion_capture')} == { - 'no_upstream_profile'} - assert 'parameter_only' not in {UNROUTED[capability] - for capability in ('image.segment', 'video.matting', 'video.motion_capture')} - assert MISSING not in {result['capabilities'][capability] for capability in UNROUTED} - - -def test_absent_fields_are_unknown(): - """`response_model_exclude_none` drops fields; absence is not a refusal.""" - omitted = body({'chat': {'reason_code': 'request_contract_unverified', - 'profiles': [{'id': 'text_response', 'routes': [route('/v1/chat/completions')]}]}, - 'speech': operation(profiles=[{'id': 'text_to_speech'}])}) - result = resolve(omitted) - assert result['endpoint'] == PRESENT - assert result['capabilities']['text.chat'] == UNKNOWN - assert result['capabilities']['audio.speech'] == UNKNOWN - assert MISSING not in result['capabilities'].values() - # The same body that states `unsupported` is a refusal, and does become missing. - stated = body({'chat': operation(support='unsupported', - profiles=[profile('text_response', 'unsupported', - paths=['/v1/chat/completions'])])}) - assert resolve(stated)['capabilities']['text.chat'] == MISSING + assert {capability: result['capabilities'][capability] for capability in UNROUTED} == { + capability: UNKNOWN for capability in UNROUTED} + assert not set(UNROUTED) & set(result['reasons']) # --- states --------------------------------------------------------------- -def test_an_absent_declaration_is_unknown_everywhere_and_never_missing(): +def test_an_unanswered_endpoint_is_absent_and_unknown(): + """No body, no model list: the service did not answer, so nothing is claimed.""" for payload in (None, {}, {'object': 'omni.capabilities', 'schema_version': 1}, - {'models': []}, {'models': 'not-a-list'}, - {'models': [{'id': 'm', 'stage_ids': ['7']}]}, - {'stages': [{'id': '7', 'type': 'llm', 'depends_on': []}]}): + {'models': 'not-a-list'}, {'stages': [{'id': '7', 'type': 'llm', 'depends_on': []}]}): + result = resolve(payload) + assert result['endpoint'] == ABSENT, payload + assert set(result['capabilities'].values()) == {UNKNOWN} + assert result['reasons'] == {} + assert result['unresolved'] == [] + + +def test_an_endpoint_that_answered_without_declaring_is_empty(): + """A model list with no operation is an answer, and a different one from silence.""" + for payload in ({'models': []}, {'models': [{'id': 'm', 'stage_ids': ['7']}]}, + {'models': [{'id': 'm', 'operations': {}}]}): result = resolve(payload) - assert result['endpoint'] == ABSENT + assert result['endpoint'] == EMPTY, payload assert set(result['capabilities'].values()) == {UNKNOWN} - assert MISSING not in result['capabilities'].values() assert result['unresolved'] == [] + assert resolve(None)['endpoint'] != resolve({'models': []})['endpoint'] def test_a_topology_only_body_grants_nothing(): @@ -288,26 +344,219 @@ def test_a_topology_only_body_grants_nothing(): {'id': '99', 'type': 'unknown', 'depends_on': []}] +def test_an_unanswered_operation_field_is_unknown_not_supported(): + """Defensive: an operation that states neither enum. Upstream always states both.""" + payload = body({'chat': {'reason_code': 'request_contract_unverified', + 'profiles': [profile('text_response', paths=table_paths('chat'))]}}) + result = resolve(payload) + assert result['endpoint'] == PRESENT + assert result['capabilities']['text.chat'] == UNKNOWN + assert MISSING not in result['capabilities'].values() + + +def test_an_operation_value_that_is_not_a_body_records_unknown(): + """Defensive: `{"chat": null}` answers nothing about chat, so its rows stay unknown.""" + for payload in (body({'chat': None}), body({'speech': 'not-a-body'})): + result = resolve(payload) + assert result['endpoint'] == PRESENT, payload + assert set(result['capabilities'].values()) == {UNKNOWN} + assert result['unresolved'] == [] + + +def test_a_profile_without_support_is_unknown_even_when_its_operation_is_supported(): + """Defensive: `support` is required upstream, so only a malformed body omits it. + + The operation says `supported`/`enabled` and the route is the right one, so + the only thing standing between this profile and `available` is its own + missing `support`. Treating an absent field as consent would make this + `available`. + """ + entry = profile('text_response', paths=table_paths('chat')) + del entry['support'] + result = resolve(body({'chat': operation(profiles=[entry])})) + assert result['endpoint'] == PRESENT + assert result['capabilities']['text.chat'] == UNKNOWN + assert result['capabilities']['text.chat'] != AVAILABLE + assert MISSING not in result['capabilities'].values() + # The same body that states `unsupported` is a refusal, and does become missing. + stated = body({'chat': operation(profiles=[profile('text_response', 'unsupported', paths=table_paths('chat'))])}) + assert resolve(stated)['capabilities']['text.chat'] == MISSING + assert resolve(stated)['reasons'] == {'text.chat': 'unsupported'} + + def test_a_supported_but_disabled_operation_is_missing(): result = resolve(body({'image_generation': operation(exposure='disabled', profiles=[profile('text_to_image', - paths=['/v1/images/generations'])])})) + paths=table_paths('image_generation'))])})) assert result['capabilities']['image.generate'] == MISSING + assert result['reasons'] == {'image.generate': 'endpoint_disabled'} assert result['unresolved'] == [] def test_an_unsupported_profile_is_missing_in_a_supported_operation(): result = resolve(body({'speech': operation(profiles=[ - profile('text_to_speech', paths=['/v1/audio/speech']), - profile('lyrics_to_music', 'unsupported', paths=['/v1/audio/speech'])])})) + profile('text_to_speech', paths=table_paths('speech')), + profile('lyrics_to_music', 'unsupported', paths=table_paths('speech'))])})) assert result['capabilities']['audio.speech'] == AVAILABLE assert result['capabilities']['audio.music'] == MISSING + assert result['reasons'] == {'audio.music': 'unsupported'} + + +def test_a_refused_model_and_a_closed_route_are_told_apart(): + """Both are a positive no, but the deployment did different things.""" + refused = resolve(body({'image_edit': operation(support='unsupported', profiles=[ + profile('image_to_image', 'unsupported', paths=table_paths('image_edit'))])})) + assert refused['capabilities']['image.edit'] == MISSING + assert refused['reasons'] == {'image.edit': 'unsupported'} + closed = resolve(body({'image_edit': operation(exposure='disabled', profiles=[ + profile('image_to_image', paths=table_paths('image_edit'))])})) + assert closed['capabilities']['image.edit'] == MISSING + assert closed['reasons'] == {'image.edit': 'endpoint_disabled'} + route_closed = resolve(body({'image_edit': operation(profiles=[ + profile('image_to_image', routes=[route('/v1/images/edits', exposure='disabled')])])})) + assert route_closed['capabilities']['image.edit'] == MISSING + assert route_closed['reasons'] == {'image.edit': 'route_disabled'} + assert set(refused['reasons'].values()) <= set(MISSING_REASONS) + + +def test_reasons_never_explain_an_available_or_unknown_capability(): + result = resolve(body(every_operation())) + assert set(result['reasons']) <= {item for item, state in result['capabilities'].items() if state == MISSING} def test_a_capability_no_profile_mentions_is_unknown_not_missing(): - result = resolve(body({'chat': operation(profiles=[profile('text_response', paths=['/v1/chat/completions'])])})) + result = resolve(body({'chat': operation(profiles=[profile('text_response', paths=table_paths('chat'))])})) assert result['capabilities']['video.generate'] == UNKNOWN assert result['capabilities']['audio.music'] == UNKNOWN + assert result['reasons'] == {} + + +# --- profile-level route exposure ---------------------------------------- + +def test_a_profile_whose_only_route_is_disabled_is_missing(): + """`support=supported` is not enough: the profile's own route says `disabled`.""" + result = resolve(body({'image_generation': operation(profiles=[ + profile('text_to_image', routes=[route('/v1/images/generations', exposure='disabled')])])})) + assert result['capabilities']['image.generate'] == MISSING + assert result['reasons'] == {'image.generate': 'route_disabled'} + assert result['unresolved'] == [] + + +def test_a_profile_whose_routes_are_all_unknown_cannot_be_confirmed(): + result = resolve(body({'image_generation': operation(profiles=[ + profile('text_to_image', routes=[route('/v1/images/generations', exposure='unknown')])])})) + assert result['capabilities']['image.generate'] == UNKNOWN + assert result['reasons'] == {} + assert result['unresolved'] == [] + + +def test_one_enabled_route_is_enough_when_another_is_disabled(): + result = resolve(body({'video_generation': operation(profiles=[ + profile('text_to_video', routes=[route('/v1/videos', exposure='disabled'), + route('/v1/videos/sync')])])})) + assert result['capabilities']['video.generate'] == AVAILABLE + + +def test_a_profile_that_declares_no_route_is_unknown(): + """`routes` is `default_factory=list` upstream: `[]` is a legal shape, not a refusal.""" + entry = profile('text_to_speech', paths=[]) + assert entry['routes'] == [] + result = resolve(body({'speech': operation(profiles=[entry])})) + assert result['capabilities']['audio.speech'] == UNKNOWN + assert result['unresolved'] == [{'operation': 'speech', 'profile': 'text_to_speech', 'route': None, + 'reason': NO_ROUTE_DECLARED}] + + +def test_one_unrecognised_route_does_not_void_a_profile_that_answered(): + """Upstream adds routes; one new path must not turn a confirmed profile into `unknown`.""" + result = resolve(body({'image_generation': operation(profiles=[ + profile('text_to_image', paths=['/v1/images/generations', '/v1/images/generations/stream'])])})) + assert result['capabilities']['image.generate'] == AVAILABLE + assert result['unresolved'] == [{'operation': 'image_generation', 'profile': 'text_to_image', + 'route': '/v1/images/generations/stream', 'reason': UNKNOWN_ROUTE}] + # With nothing usable left, the profile cannot be confirmed. + alone = resolve(body({'image_generation': operation(profiles=[ + profile('text_to_image', paths=['/v1/images/generations/stream'])])})) + assert alone['capabilities']['image.generate'] == UNKNOWN + + +def test_a_route_entry_without_a_usable_path_is_reported_not_dropped(): + entry = profile('text_to_speech') + entry['routes'] = [{'method': 'POST'}, {'path': 123}] + result = resolve(body({'speech': operation(profiles=[entry])})) + assert result['capabilities']['audio.speech'] == UNKNOWN + # Both facts hold: nothing readable was declared, and what was declared is broken. + assert result['unresolved'] == [{'operation': 'speech', 'profile': 'text_to_speech', 'route': None, + 'reason': MALFORMED_ROUTE}, + {'operation': 'speech', 'profile': 'text_to_speech', 'route': None, + 'reason': NO_ROUTE_DECLARED}] + + +# --- transport / method --------------------------------------------------- + +def test_a_wrong_method_is_a_route_mismatch(): + """`DELETE /v1/chat/completions` would be a 405, not a chat request.""" + result = resolve(body({'chat': operation(profiles=[ + profile('text_response', routes=[route('/v1/chat/completions', method='DELETE')])])})) + assert result['capabilities']['text.chat'] == UNKNOWN + assert result['unresolved'] == [{'operation': 'chat', 'profile': 'text_response', + 'route': '/v1/chat/completions', 'reason': ROUTE_MISMATCH}] + + +def test_a_wrong_transport_is_a_route_mismatch(): + """The websocket method literal belongs to one route only.""" + result = resolve(body({'speech': operation(profiles=[ + profile('text_to_speech', routes=[route('/v1/audio/speech', method='WEBSOCKET', transport='websocket')])])})) + assert result['capabilities']['audio.speech'] == UNKNOWN + assert result['unresolved'] == [{'operation': 'speech', 'profile': 'text_to_speech', + 'route': '/v1/audio/speech', 'reason': ROUTE_MISMATCH}] + + +def test_the_video_websocket_route_needs_its_declared_method(): + accepted = resolve(body({'video_generation': operation(profiles=[ + profile('text_to_video', routes=[route('/v1/realtime/video', method='WEBSOCKET', transport='websocket')])])})) + assert accepted['capabilities']['video.generate'] == AVAILABLE + http_spelling = resolve(body({'video_generation': operation(profiles=[ + profile('text_to_video', routes=[route('/v1/realtime/video', method='POST')])])})) + assert http_spelling['capabilities']['video.generate'] == UNKNOWN + assert http_spelling['unresolved'] == [{'operation': 'video_generation', 'profile': 'text_to_video', + 'route': '/v1/realtime/video', 'reason': ROUTE_MISMATCH}] + + +# --- what each chat profile grants --------------------------------------- + +def test_an_image_only_chat_deployment_does_not_read_as_text_chat(): + """`image_response` output is an image; `text.chat` is served by a text-only adapter.""" + result = resolve(body({'chat': operation(profiles=[ + profile('image_response', outputs=('image',), paths=table_paths('chat'))])})) + assert result['capabilities']['text.chat'] != AVAILABLE + assert result['capabilities']['text.chat'] == UNKNOWN + assert result['unresolved'] == [{'operation': 'chat', 'profile': 'image_response', 'route': None, + 'reason': NO_LOCAL_CAPABILITY}] + + +def test_an_audio_only_chat_deployment_does_not_read_as_text_chat(): + result = resolve(body({'chat': operation(profiles=[ + profile('audio_response', outputs=('audio',), paths=table_paths('chat'))])})) + assert result['capabilities']['text.chat'] == UNKNOWN + assert result['unresolved'] == [{'operation': 'chat', 'profile': 'audio_response', 'route': None, + 'reason': NO_LOCAL_CAPABILITY}] + + +def test_a_text_chat_deployment_still_reads_as_text_chat(): + result = resolve(body({'chat': operation(profiles=[ + profile('text_response', paths=table_paths('chat'))])})) + assert result['capabilities']['text.chat'] == AVAILABLE + assert result['unresolved'] == [] + + +def test_text_to_audio_grants_audio_music(): + """StableAudio / LTX2-t2a serve `/v1/audio/generate`: text to audio is audio.music here.""" + result = resolve(body({'audio_generation': operation(profiles=[ + profile('text_to_audio', paths=table_paths('audio_generation'), outputs=('audio',))])})) + assert result['capabilities']['audio.music'] == AVAILABLE + assert result['capabilities']['audio.speech'] == UNKNOWN + assert result['unresolved'] == [] # --- untranslatable declarations ----------------------------------------- @@ -327,7 +576,7 @@ def test_a_profile_declaring_another_operations_route_is_a_route_mismatch(): def test_a_route_that_no_operation_owns_is_an_unknown_route(): - for path in ('/v1/videos/track', '/v1/images/segmentations'): + for path in ('/v1/videos/segmentations', '/v1/images/segmentations'): result = resolve(body({'image_generation': operation( profiles=[profile('text_to_image', paths=[path])])})) assert result['unresolved'] == [{'operation': 'image_generation', 'profile': 'text_to_image', @@ -338,7 +587,7 @@ def test_a_route_that_no_operation_owns_is_an_unknown_route(): def test_a_profile_filed_under_the_wrong_operation_is_an_operation_mismatch(): result = resolve(body({'audio_generation': operation(profiles=[ - profile('lyrics_to_music', paths=['/v1/audio/speech'])])})) + profile('lyrics_to_music', paths=table_paths('speech'))])})) assert result['unresolved'] == [{'operation': 'audio_generation', 'profile': 'lyrics_to_music', 'route': None, 'reason': OPERATION_MISMATCH}] assert result['capabilities']['audio.music'] == UNKNOWN @@ -347,7 +596,7 @@ def test_a_profile_filed_under_the_wrong_operation_is_an_operation_mismatch(): def test_unknown_operation_and_unknown_profile_stay_distinguishable(): result = resolve(body({'speech': operation(profiles=[profile('speech_to_speech', - paths=['/v1/audio/speech'])])})) + paths=table_paths('speech'))])})) assert result['unresolved'] == [{'operation': 'speech', 'profile': 'speech_to_speech', 'route': None, 'reason': UNKNOWN_PROFILE}] wrong_operation = resolve(body({'asr': operation(profiles=[])})) @@ -366,9 +615,9 @@ def test_asr_is_an_upstream_gap_not_a_missing_row(): def test_a_supported_profile_without_a_local_capability_is_reported(): - result = resolve(body({'audio_generation': operation(profiles=[ - profile('text_to_audio', paths=['/v1/audio/generate'], inputs=('text',), outputs=('audio',))])})) - assert result['unresolved'] == [{'operation': 'audio_generation', 'profile': 'text_to_audio', + result = resolve(body({'speech': operation(profiles=[ + profile('caption_to_audio', paths=table_paths('speech'), inputs=('ambient_sound',), outputs=('audio',))])})) + assert result['unresolved'] == [{'operation': 'speech', 'profile': 'caption_to_audio', 'route': None, 'reason': NO_LOCAL_CAPABILITY}] assert result['capabilities']['audio.speech'] == UNKNOWN assert MISSING not in result['capabilities'].values() @@ -376,10 +625,19 @@ def test_a_supported_profile_without_a_local_capability_is_reported(): def test_an_unsupported_profile_without_a_local_capability_stays_quiet(): result = resolve(body({'speech': operation(profiles=[ - profile('caption_to_audio', 'unsupported', paths=['/v1/audio/speech'])])})) + profile('caption_to_audio', 'unsupported', paths=table_paths('speech'))])})) assert result['unresolved'] == [] +def test_profile_order_inside_a_model_does_not_change_the_result(): + declared = [profile('text_to_speech', paths=table_paths('speech')), + profile('lyrics_to_music', 'unsupported', paths=table_paths('speech')), + profile('caption_to_audio', paths=table_paths('speech'), inputs=('ambient_sound',))] + forward = resolve(body({'speech': operation(profiles=declared)})) + backward = resolve(body({'speech': operation(profiles=list(reversed(declared)))})) + assert forward == backward + + def test_two_models_repeating_one_declaration_are_reported_once(): declaration = {'video_generation': operation(profiles=[profile('text_to_video', paths=['/v1/images/edits'])])} result = resolve(body(models=[model(declaration, 'model-a', ('7',)), @@ -392,16 +650,16 @@ def test_two_models_repeating_one_declaration_are_reported_once(): def test_capabilities_are_reported_in_table_order(): """Presentation reads this order; sorting here would make the table's order meaningless.""" - result = resolve(body({'speech': operation(profiles=[profile('lyrics_to_music', paths=['/v1/audio/speech'])]), - 'chat': operation(profiles=[profile('text_response', paths=['/v1/chat/completions'])])})) - assert list(result['capabilities']) == list(classified_capabilities()) + result = resolve(body({'speech': operation(profiles=[profile('lyrics_to_music', paths=table_paths('speech'))]), + 'chat': operation(profiles=[profile('text_response', paths=table_paths('chat'))])})) + assert list(result['capabilities']) == VOCABULARY_ORDER assert list(result['capabilities'])[:2] == ['text.chat', 'image.generate'] def test_declarations_keep_document_order_and_drop_repeats(): - operations = {'speech': operation(profiles=[profile('lyrics_to_music', paths=['/v1/audio/speech']), - profile('text_to_speech', paths=['/v1/audio/speech'])]), - 'chat': operation(profiles=[profile('text_response', paths=['/v1/chat/completions'])])} + operations = {'speech': operation(profiles=[profile('lyrics_to_music', paths=table_paths('speech')), + profile('text_to_speech', paths=table_paths('speech'))]), + 'chat': operation(profiles=[profile('text_response', paths=table_paths('chat'))])} assert declarations(body(operations)) == [('speech', 'lyrics_to_music'), ('speech', 'text_to_speech'), ('chat', 'text_response')] twin = body(models=[model(operations, 'model-a', ('7',)), model(operations, 'model-b', ('7',))]) @@ -410,12 +668,12 @@ def test_declarations_keep_document_order_and_drop_repeats(): @pytest.mark.parametrize('spelling', ['audio/speech', '/audio/speech', '/v1/audio/speech/', - {'path': '/v1/audio/speech?x=1'}]) + '/v1/audio/speech?x=1']) def test_route_spellings_normalize_to_one_path(spelling): assert route_path(spelling) == '/v1/audio/speech' + assert route_path({'path': spelling}) == '/v1/audio/speech' result = resolve(body({'speech': operation(profiles=[ - profile('text_to_speech', paths=[], - routes=[spelling if isinstance(spelling, dict) else route(spelling)])])})) + profile('text_to_speech', routes=[route(spelling)])])})) assert result['capabilities']['audio.speech'] == AVAILABLE assert result['unresolved'] == [] @@ -424,10 +682,13 @@ def test_results_use_only_the_contract_vocabulary(): for _, cases in raw_cases(): for case in cases: expected = case['expected'] - assert set(expected) == {'endpoint', 'topology', 'capabilities', 'unresolved'} - assert expected['endpoint'] in {ABSENT, PRESENT} + assert set(expected) == {'endpoint', 'topology', 'capabilities', 'reasons', 'unresolved'} + assert expected['endpoint'] in {ABSENT, EMPTY, PRESENT} assert set(expected['capabilities']) == set(classified_capabilities()) assert set(expected['capabilities'].values()) <= {AVAILABLE, MISSING, UNKNOWN} + assert set(expected['reasons']) <= {item for item, state in expected['capabilities'].items() + if state == MISSING} + assert set(expected['reasons'].values()) <= set(MISSING_REASONS) assert {item['reason'] for item in expected['unresolved']} <= REASONS assert {tuple(sorted(item)) for item in expected['unresolved']} <= { ('operation', 'profile', 'reason', 'route')} @@ -435,6 +696,56 @@ def test_results_use_only_the_contract_vocabulary(): assert set(stage) == {'id', 'type', 'depends_on'} +def test_fixtures_are_schema_valid(): + """Structural invariants of upstream's response model, checked on our fixtures. + + Upstream ships these as pydantic models (`protocol/capabilities.py:16-79`, + head `d1104f8`). Re-implementing them here would be a second copy to keep in + sync, so this test only asserts the invariants a rejected body would break: + all six operations, `support`/`exposure` on every operation, `profiles` as a + list (it serializes as `[]`, it never disappears), the profile `support` + required, stages as `{id, type, depends_on}` with numeric-string ids. + """ + operations = set(OPERATIONS) + seen_body = 0 + for document, cases in raw_cases(): + for case in cases: + payload = case['payload'] + if payload is None: + assert case['expected']['endpoint'] == ABSENT, case['id'] + continue + seen_body += 1 + assert set(payload) == {'object', 'schema_version', 'observed_at', 'stages', 'models', 'routes'}, case['id'] + assert payload['object'] == 'omni.capabilities' and payload['schema_version'] == 1 + for stage in payload['stages']: + assert set(stage) == {'id', 'type', 'depends_on'}, case['id'] + assert isinstance(stage['id'], str) and stage['id'].isdigit(), stage + assert isinstance(stage['depends_on'], list), stage + assert stage['type'] in {'llm', 'diffusion', 'unknown'}, stage + for model in payload['models']: + assert {'id', 'stage_ids', 'operations'} <= set(model), case['id'] + assert isinstance(model['stage_ids'], list), model + assert all(isinstance(stage_id, str) and stage_id.isdigit() for stage_id in model['stage_ids']) + assert set(model['operations']) == operations, (case['id'], sorted(model['operations'])) + for operation_id, declared in model['operations'].items(): + assert declared['support'] in {'supported', 'unsupported', 'unknown'}, (case['id'], operation_id) + assert declared['exposure'] in {'enabled', 'disabled', 'unknown'}, (case['id'], operation_id) + assert isinstance(declared.get('profiles'), list), (case['id'], operation_id) + for entry in declared['profiles']: + assert entry['support'] in {'supported', 'unsupported', 'unknown'}, (case['id'], entry['id']) + assert isinstance(entry.get('routes'), list), entry + assert isinstance(entry.get('required_inputs'), list), entry + # Published `STANDARD_PROFILES` outputs (`serving_capabilities.py:43-45`). + if entry['id'] == 'image_response': + assert entry['outputs'] == ['image'], entry + if entry['id'] == 'audio_response': + assert entry['outputs'] == ['audio'], entry + for declared_route in payload['routes']: + assert set(declared_route) == {'transport', 'method', 'path', 'exposure'}, case['id'] + assert declared_route['transport'] in {'http', 'websocket'}, declared_route + assert seen_body >= 5 + + def test_fixtures_carry_no_credentials_or_provider_responses(): for path in sorted(FIXTURES.glob('*.json')): text = path.read_text(encoding='utf-8').lower() @@ -442,7 +753,7 @@ def test_fixtures_carry_no_credentials_or_provider_responses(): def test_model_names_and_other_provider_fields_never_reach_the_result(): - payload = body({'chat': operation(profiles=[profile('text_response', paths=['/v1/chat/completions'])])}) + payload = body({'chat': operation(profiles=[profile('text_response', paths=table_paths('chat'))])}) payload['models'][0]['id'] = 'a-private-model-name' payload['models'][0]['extra'] = {'owner': 'someone'} assert 'a-private-model-name' not in json.dumps(resolve(payload), ensure_ascii=False) From 6a19cc44ea4e04bd99b43368145626eaf9f6d9ee Mon Sep 17 00:00:00 2001 From: Xingkai98 Date: Wed, 11 Nov 2026 00:59:14 +0800 Subject: [PATCH 4/5] fix(contract): keep audio_generation local-unknown, align required_inputs with upstream --- docs/architecture/capability-contract.md | 65 +- .../domain/capability_contract.py | 92 +- tests/fixtures/omni-capabilities/full.json | 19 +- .../fixtures/omni-capabilities/malformed.json | 972 ++++++++++++++++++ tests/fixtures/omni-capabilities/partial.json | 59 +- tests/test_capability_contract.py | 375 +++++-- 6 files changed, 1429 insertions(+), 153 deletions(-) create mode 100644 tests/fixtures/omni-capabilities/malformed.json diff --git a/docs/architecture/capability-contract.md b/docs/architecture/capability-contract.md index a03bb2f0..e2c39181 100644 --- a/docs/architecture/capability-contract.md +++ b/docs/architecture/capability-contract.md @@ -20,19 +20,19 @@ | `chat` | `image_response` | `("text",)` → `("image",)` | （无）输出是图像，不是本仓 `text.chat` | | `chat` | `audio_response` | `("text",)` → `("audio",)` | （无）输出是音频，不是本仓 `text.chat` | | `image_generation` | `text_to_image` | `("text",)` → `("image",)` | `image.generate` | -| `image_edit` | `image_to_image` | `("image","text")` → `("image",)` | `image.edit` | +| `image_edit` | `image_to_image` | `("text","image")` → `("image",)` | `image.edit` | | `video_generation` | `text_to_video` | `("text",)` → `("video",)` | `video.generate` | -| `video_generation` | `image_to_video` | `("image","text")` → `("video",)` | `video.generate` | -| `video_generation` | `video_to_video` | `("video","text")` → `("video",)` | `video.generate` | +| `video_generation` | `image_to_video` | `("text","image")` → `("video",)` | `video.generate` | +| `video_generation` | `video_to_video` | `("video",)` → `("video",)` | `video.generate` | | `speech` | `text_to_speech` | `("text",)` → `("audio",)` | `audio.speech` | | `speech` | `lyrics_to_music` | `("text","instructions")` → `("audio",)` | `audio.music` | | `speech` | `caption_to_audio` | `("text",)` → `("audio",)` | （无）音效/环境声，本仓无对应 id | -| `audio_generation` | `text_to_audio` | `("text",)` → `("audio",)` | `audio.music`（同为「文本生成音频」） | +| `audio_generation` | `text_to_audio` | `("text",)` → `("audio",)` | （无）扩散侧 `/v1/audio/generate` 契约，不是本仓 `audio.music` | -两个「沉默」的行要一起读： +本仓没有 id 的这四行（`image_response` / `audio_response` / `caption_to_audio` / `text_to_audio`）要一起读： - `image_response` / `audio_response` 上游确实发布，但本仓没有「图片/音频版 chat」id，所以**保持 `unknown`**，`unresolved` 里报 `no_local_capability`。绝不折进 `text.chat`。 -- `text_to_audio` 走的是扩散侧独立请求契约（`/v1/audio/generate`，无 voice/lyrics/instructions 映射，`docs/serving/capabilities.md:59-60`），但语义就是「文本生成音频」，与 `lyrics_to_music` 同属音频生成，因此映射到 `audio.music`。否则 StableAudio / LTX2-t2a 这种形状（`speech/text_to_speech=unsupported`、`audio_generation/text_to_audio=supported`）会把 `audio.music` 反判成 `missing`。 +- `text_to_audio` 走的是扩散侧独立请求契约（`/v1/audio/generate`，无 voice/lyrics/instructions 映射，`docs/serving/capabilities.md:59-60`），而本仓 `audio.music` 的**唯一**适配器 `VllmOmniMusic`（`integrations/model_adapters.py:510-537`）POST 的是 `/v1/audio/speech`，payload 必带歌词与 `instructions`。上游说这个 profile 没有这两样、本仓却要这两样，把它映射成 `audio.music` 就是承诺一个该部署从未接受的请求。所以 `text_to_audio` 也留空：`support=supported` 时 `unresolved` 报 `no_local_capability`，`audio.music` 只由 `lyrics_to_music` 决定，没有证据就是 `unknown`。这与「绝不把 `caption_to_audio` 塞进 `audio.speech`」是同一条规则。 **operation → 路由**，三个维度都要对（`:42-54`）。上游的 `RouteKey` 是 `(transport, method, path)`，websocket 路由的 method 字面量就是 `WEBSOCKET`： @@ -49,8 +49,8 @@ 三条 schema 事实决定了这份表的形状： -- 响应顶层只有 `object` / `schema_version` / `observed_at` / `models` / `stages` / `routes`（`protocol/capabilities.py:73-79`），端点是 `response_model=OmniCapabilitiesResponse`、`response_model_exclude_none=True`（`api_server.py:2193`）。 -- **`exclude_none` 只能省掉值为 `None` 的可选字段**，也就是 profile 的 `request`（`RequestConstraints | None = None`）。列表/字典字段是 `default_factory`，空的时候序列化成 `[]`/`{}`，**不会缺席**：`profiles: []` 而不是「没有 `profiles` 键」。`support` / `exposure` / `reason_code` 在 operation 和 profile 上都是必填（`protocol/capabilities.py:36-52`），永远不会缺席；少了它们就不是上游能发出的 body。 +- 响应顶层只有 `object` / `schema_version` / `observed_at` / `models` / `stages` / `routes`（`protocol/capabilities.py:73-79`），端点是 `response_model=OmniCapabilitiesResponse`、`response_model_exclude_none=True`（`api_server.py:2193`）。顶层 `routes[]` 只是**该进程注册了哪些路径的展示清单**：`resolve()` 从不读它，所以把一个没有 operation 的路径写进去不会改变任何能力判定。 +- **`exclude_none` 只能省掉值为 `None` 的可选字段**，也就是 profile 的 `request`（`RequestConstraints | None = None`）。列表/字典字段是 `default_factory`，空的时候序列化成 `[]`/`{}`，**不会缺席**：`profiles: []` 而不是「没有 `profiles` 键」。`support` / `reason_code` 在 operation 与 profile 上都是必填，`exposure` 在 operation 与顶层路由上是必填（`protocol/capabilities.py:24-52`），永远不会缺席；少了它们就不是上游能发出的 body。**profile 自己没有 `exposure` 字段**（`:36-44`）：某个 profile 可达不可达，看的是它引用的各条路由的 `exposure` 聚合，而不是 profile 上的一个开关。 - `models[].operations` **必须齐全 6 个**，少一个 pydantic 直接抛错（`protocol/capabilities.py:60-64`）；`models[].stage_ids` 是**数字字符串**，而且每个 model 拿到的都是同一份全量列表（`entrypoints/openai/serving_capabilities.py:400`）。 ### 键是 (operation, profile)，不是 route，也不是 stage @@ -70,7 +70,7 @@ | `unmapped_stage` | 上游没有这个 operation（`asr`、`music`）。名字沿用上一版 stage-keyed 设计，含义是 operation 层缺口 | 上游词表缺口，不是本表漏行 | | `unknown_profile` | operation 认识，profile id 上游未发布（`speech_to_speech`） | 上游新增 profile，本表待补 | | `operation_mismatch` | profile 认识，但挂在别的 operation 下（`lyrics_to_music` 挂在 `audio_generation`） | 声明写错了 operation，或本表把归属搞错了 | -| `no_local_capability` | profile 支持，但本仓没有对应 capability id（`caption_to_audio`、`image_response`、`audio_response`） | 词表待补 | +| `no_local_capability` | profile 支持，但本仓没有对应 capability id（`caption_to_audio`、`text_to_audio`、`image_response`、`audio_response`） | 词表待补 | | `no_route_declared` | profile 说自己支持，却一条路由都没声明（`routes: []` 是上游合法默认值） | 声明不完整；可达性无法确认 | | `malformed_route` | 路由条目里没有可用的 `path`（畸形 body） | 上游 bug 或代理改写 | @@ -96,6 +96,7 @@ | operation `support=unsupported`（profile 说什么都一样） | `missing` | `reasons[cid] = "unsupported"` | | operation `support=supported` + `exposure=disabled`（部署关了路由） | `missing` | `reasons[cid] = "endpoint_disabled"` | | profile `support=supported`，但它声明的路由**全部** `exposure=disabled` | `missing` | `reasons[cid] = "route_disabled"` | +| 自有路由全 `disabled`，但另有一条本表不认识的 `enabled`（或没写 `exposure`）路由 | `unknown` | 无（存在判断不了的通道，不是拒绝） | | profile 声明的路由全部 `exposure=unknown`（上游默认值） | `unknown` | 无（可达性无法确认，不是拒绝） | | profile `routes: []`（上游默认空列表） | `unknown` | `unresolved: no_route_declared` | | 路由条目没有可用 `path` | `unknown` | `unresolved: malformed_route` | @@ -111,7 +112,7 @@ 两条边界值得单独记住： -1. **一条不认得的路由不打穿已确认的 profile**。上游会加路由，本表会落后；只有当该 profile **全部**路由都不认得（或全部被拒）时才降为 `unknown`，否则保留判定并在 `unresolved` 里记一条。 +1. **一条不认得的路由既不打穿已确认的 profile，也不把「自有路由全 disabled」变成 `missing`**。上游会加路由，本表会落后：不认得的路由自身若说 `enabled`（或没写 `exposure`，上游默认即 `unknown`），就说明存在一条本表判断不了的通道，自有路由全关也就不构成「正面说不」，结果是 `unknown`；只有它自己也说 `disabled` 时，全 `disabled` 才仍然是 `missing`。已经确认（有 `enabled` 自有路由）的 profile 不受影响，继续 `available`。 2. **`endpoint` 三值**区分「没答」与「答了但没声明」：`absent` / `empty` / `present`。前者该降级或报错，后者该提示「服务未声明能力」，两者对用户的意义完全不同。 ### 有路由、没有 operation 的能力 @@ -152,20 +153,22 @@ from opendirector.domain.capability_contract import capabilities_for capabilities_for('speech', 'lyrics_to_music') # ('audio.music',) capabilities_for('speech', 'text_to_speech') # ('audio.speech',) -capabilities_for('audio_generation', 'text_to_audio') # ('audio.music',) +capabilities_for('audio_generation', 'text_to_audio') # ()：上游有 profile，本仓没有对应 id ``` `resolve(payload)` 返回逐能力状态（`capabilities`）、缺失原因（`reasons`）、`topology`（`stages[]`，仅供展示）与 `unresolved`；`classified_capabilities()` 按表顺序给出词表，展示层据此排列。是否接进 `requests_for()` 由 #266 一侧决定，本契约只保证查找是成对的、稳定的、可测的。 -### 三档 fixture 与验证 +### 三档 fixture（外加一档防御输入）与验证 -`tests/fixtures/omni-capabilities/{full,partial,absent}.json` 按**真实响应形状**写：`stages[]` 是 `{id,type,depends_on}` 且 id 为数字字符串，`models[].operations` 六个齐全、每个含 `support`/`exposure`/`reason_code`/`profiles`/`execution`，每个 profile 都带 `support` 与 `routes[]`（可以是 `[]`），`image_response`/`audio_response` 的 `outputs` 照上游 `STANDARD_PROFILES` 分别是 `["image"]`/`["audio"]`。 +`tests/fixtures/omni-capabilities/{full,partial,absent}.json` 是**求值样本**，按真实响应形状写：`stages[]` 是 `{id,type,depends_on}` 且 id 为数字字符串，`models[].operations` 六个齐全、每个含 `support`/`exposure`/`reason_code`/`profiles`/`execution`，每个 profile 都带 `support`、`reason_code` 与 `routes[]`（可以是 `[]`），`required_inputs`/`outputs` 逐条照上游 `STANDARD_PROFILES`（含顺序；`image_response`/`audio_response` 分别是 `["image"]`/`["audio"]`）。 -- `full`：全支持全启用，12 个 profile 齐全；`image_response`/`audio_response`/`caption_to_audio` 报 `no_local_capability`，`text_to_audio` 让 `audio.music` 为 `available`。 -- `partial`：`support=unsupported`、operation `exposure=disabled`（`endpoint_disabled`）、profile 自己的路由 `disabled`（`route_disabled`）、`route_mismatch`、`no_local_capability`，以及整个 body 都是 `unknown` 枚举的形态。 +- `full`：全支持全启用，12 个 profile 齐全；`image_response`/`audio_response`/`caption_to_audio`/`text_to_audio` 报 `no_local_capability`（`text_to_audio` **不**授 `audio.music`，`audio.music` 由 `lyrics_to_music` 供货）。 +- `partial`：`support=unsupported`、operation `exposure=disabled`（`endpoint_disabled`）、profile 自己的路由 `disabled`（`route_disabled`）、`no_local_capability`，以及整个 body 都是 `unknown` 枚举的形态。 - `absent`：没答（`absent`）、只答拓扑（`empty`）、六个 operation 都答了但 `profiles: []`（`present` 且全 `unknown`）。 -**上游不可能发出的形状**（`asr` operation 键、profile 挂在错 operation、缺必填 `support`、`profiles` 整键缺席）**不进 fixture**，只在对应测试里内联构造 —— 它们是防御性输入的用例，混进 fixture 会让「fixture = 真实样本」这句话变成假话。`test_fixtures_are_schema_valid` 只断言上游 schema 的**结构不变量**（六操作齐全、必填字段在、`profiles` 是列表、stages 形状、id 为数字字符串），不把上游 pydantic 源码抄一份进本仓。 +第四档 `malformed.json` **只用于防御输入**，不代表上游可能产出：里面的 body 是合法 pydantic（结构断言照样过），但上游投影发不出来（例如 profile 引用了别的 operation 的路由，或自有路由全 `disabled` 旁边挂着一条本表不认识的 `enabled` 路径）。代理、缓存、旧版本才可能把这种 body 交给我们，所以它只被防御性与结构断言读取，不进样本 oracle，也不参与「fixture = 真实样本」这句话。 + +**上游不可能发出的形状**（`asr` operation 键、profile 挂在错 operation、缺必填 `support`、`profiles` 整键缺席）**不进 fixture**，只在对应测试里内联构造 —— 它们是防御性输入的用例，混进样本会让「fixture = 真实样本」这句话变成假话。`test_fixtures_are_schema_valid` 只断言上游 schema 的**结构不变量**（六键、`observed_at` 可解析且带时区、`reason_code` 在 operation 与 profile 上都在、`execution` 为 `{state,scope,observed_at}`、枚举值域、`profiles` 是列表、route 的 `transport`/`method`/`path` 齐全、stages 形状与数字字符串 id），不把上游 pydantic 源码抄一份进本仓。 防「假覆盖」的回归测试（`tests/test_capability_contract.py`）： @@ -178,11 +181,14 @@ capabilities_for('audio_generation', 'text_to_audio') # ('audio.music',) | `test_a_profile_without_support_is_unknown_even_when_its_operation_is_supported` | profile 级字段缺席是 `unknown`（不是 `supported`） | | `test_a_profile_that_declares_no_route_is_unknown` | `supported + routes: []` 不能判 `available` | | `test_a_profile_whose_only_route_is_disabled_is_missing` | profile 路由全 `disabled` → `missing` + `route_disabled` | +| `test_a_disabled_profile_with_an_unknown_enabled_route_is_unknown` | 自有路由全关 + 一条不认识的 `enabled` 路由 → `unknown`（不是 `missing`） | | `test_a_profile_whose_routes_are_all_unknown_cannot_be_confirmed` | 路由全 `unknown` → `unknown` | | `test_one_unrecognised_route_does_not_void_a_profile_that_answered` | 一条不认得的路由不打穿已确认的 profile | | `test_a_wrong_method_is_a_route_mismatch` | `DELETE /v1/chat/completions` 不授予 chat | | `test_an_image_only_chat_deployment_does_not_read_as_text_chat` | `image_response` 不折进 `text.chat` | -| `test_text_to_audio_grants_audio_music` | `text_to_audio` → `audio.music` | +| `test_text_to_audio_alone_does_not_make_audio_music_available` | `text_to_audio` 不授 `audio.music`，只报 `no_local_capability` | +| `test_documented_required_inputs_match_fixtures` | 12 行上游 `required_inputs`/`outputs` 字面量表 = 两份文档表格 = 所有 fixture | +| `test_fixtures_are_schema_valid` | 三档样本 + 防御档的结构不变量（`observed_at`、`reason_code`、`execution`、route 三元组） | | `test_a_refused_model_and_a_closed_route_are_told_apart` | `unsupported` / `endpoint_disabled` / `route_disabled` 三码可分 | ```bash @@ -207,20 +213,20 @@ Source: `vllm-project/vllm-omni` PR #10102 (issue #10007), head `d1104f8`; every | `chat` | `image_response` | `("text",)` → `("image",)` | none — the output is an image, not this repo's `text.chat` | | `chat` | `audio_response` | `("text",)` → `("audio",)` | none — the output is audio, not this repo's `text.chat` | | `image_generation` | `text_to_image` | `("text",)` → `("image",)` | `image.generate` | -| `image_edit` | `image_to_image` | `("image","text")` → `("image",)` | `image.edit` | +| `image_edit` | `image_to_image` | `("text","image")` → `("image",)` | `image.edit` | | `video_generation` | `text_to_video` | `("text",)` → `("video",)` | `video.generate` | -| `video_generation` | `image_to_video` | `("image","text")` → `("video",)` | `video.generate` | -| `video_generation` | `video_to_video` | `("video","text")` → `("video",)` | `video.generate` | +| `video_generation` | `image_to_video` | `("text","image")` → `("video",)` | `video.generate` | +| `video_generation` | `video_to_video` | `("video",)` → `("video",)` | `video.generate` | | `speech` | `text_to_speech` | `("text",)` → `("audio",)` | `audio.speech` | | `speech` | `lyrics_to_music` | `("text","instructions")` → `("audio",)` | `audio.music` | | `speech` | `caption_to_audio` | `("text",)` → `("audio",)` | none — sound effect / ambience | -| `audio_generation` | `text_to_audio` | `("text",)` → `("audio",)` | `audio.music` (text to audio, same family) | +| `audio_generation` | `text_to_audio` | `("text",)` → `("audio",)` | none — the diffusion `/v1/audio/generate` contract, not this repo's `audio.music` | -`image_response` and `audio_response` are published upstream but have no id here: they stay `unknown` and are reported as `no_local_capability` instead of being folded into `text.chat`. `text_to_audio` uses the separate diffusion request contract behind `/v1/audio/generate` (no voice, no lyrics; `docs/serving/capabilities.md:59-60`) yet is still text-to-audio, so it grants `audio.music` alongside `lyrics_to_music`; without that row a StableAudio / LTX2-t2a shape (`speech/text_to_speech=unsupported`, `audio_generation/text_to_audio=supported`) would report `audio.music=missing`. +Four rows carry no id here (`image_response`, `audio_response`, `caption_to_audio`, `text_to_audio`), and they read together: `image_response` and `audio_response` are published upstream but have no id here, so they stay `unknown` and are reported as `no_local_capability` instead of being folded into `text.chat`. `text_to_audio` uses the separate diffusion request contract behind `/v1/audio/generate` (no voice, no lyrics, no instructions; `docs/serving/capabilities.md:59-60`), while the *only* `audio.music` adapter in this repository (`VllmOmniMusic`, `integrations/model_adapters.py:510-537`) posts `/v1/audio/speech` with lyrics **and** `instructions`, both required. Upstream says that profile has neither and this repository demands both, so mapping it to `audio.music` would promise a request the deployment never accepted: the profile is reported as `no_local_capability` and `audio.music` is decided by `lyrics_to_music` alone (no evidence → `unknown`). That is the same rule that keeps `caption_to_audio` off `audio.speech`. **Operation → routes** (`:42-54`) as `(transport, method, path)`, because upstream's `RouteKey` is exactly that and the websocket method literal is `WEBSOCKET`: `chat` → `http POST /v1/chat/completions`; `image_generation` → `http POST /v1/images/generations`; `image_edit` → `http POST /v1/images/edits`; `video_generation` → `http POST /v1/videos`, `http POST /v1/videos/sync`, `websocket WEBSOCKET /v1/realtime/video`; `speech` → `http POST /v1/audio/speech`; `audio_generation` → `http POST /v1/audio/generate`. `DELETE /v1/chat/completions` would be a 405 and a `websocket` declaration on an http route is wrong, so both are `route_mismatch`. `transport` defaults to `http` upstream (`protocol/capabilities.py:16-21`) and an absent transport is not a mismatch; `method` has no default and an absent one cannot be confirmed. -Three schema facts shape the table: the body has only `object`, `schema_version`, `observed_at`, `models`, `stages`, `routes` (`protocol/capabilities.py:73-79`) and is served with `response_model_exclude_none=True` (`api_server.py:2193`); **`exclude_none` drops only `None`-valued optional fields** — in practice the profile's `request` — while `default_factory` lists serialize as `[]` (`profiles: []`, never a missing key) and `support`/`exposure`/`reason_code` are required on every operation and profile (`protocol/capabilities.py:36-52`), so a body lacking them is not a shape upstream can emit; and `models[].operations` **must list all six** or pydantic raises (`:60-64`), with `stage_ids` numeric strings, the *same* full list for every model (`entrypoints/openai/serving_capabilities.py:400`). +Three schema facts shape the table: the body has only `object`, `schema_version`, `observed_at`, `models`, `stages`, `routes` (`protocol/capabilities.py:73-79`) and is served with `response_model_exclude_none=True` (`api_server.py:2193`) — the top-level `routes[]` is a display list of what the process registered, and `resolve()` never reads it, so publishing an unowned path there changes no verdict; **`exclude_none` drops only `None`-valued optional fields** — in practice the profile's `request` — while `default_factory` lists serialize as `[]` (`profiles: []`, never a missing key), `support`/`reason_code` are required on every operation and profile, `exposure` is required on every operation and top-level route (`protocol/capabilities.py:24-52`), and a profile has **no `exposure` field of its own** (`:36-44`): its reachability is the aggregation of the exposures of the routes it references; and `models[].operations` **must list all six** or pydantic raises (`:60-64`), with `stage_ids` numeric strings, the *same* full list for every model (`entrypoints/openai/serving_capabilities.py:400`). ### The key is the pair (operation, profile) @@ -246,6 +252,7 @@ Routes still matter, in two ways: `OPERATION_ROUTES` records which operation own | operation `support=unsupported` | `missing` | `reasons[cid] = "unsupported"` | | operation `supported` + `exposure=disabled` | `missing` | `reasons[cid] = "endpoint_disabled"` | | all of the profile's routes `exposure=disabled` | `missing` | `reasons[cid] = "route_disabled"` | +| owned routes all `disabled`, plus an `enabled` route this table cannot place (or one with no `exposure`) | `unknown` | none — a channel that cannot be judged, not a refusal | | all of the profile's routes `exposure=unknown` | `unknown` | none — reachability unconfirmed, not a refusal | | profile `routes: []` | `unknown` | `unresolved: no_route_declared` | | a route entry with no usable `path` | `unknown` | `unresolved: malformed_route` | @@ -259,7 +266,7 @@ Routes still matter, in two ways: `OPERATION_ROUTES` records which operation own | answered, but no model declared an operation | all `unknown` | `endpoint = "empty"` | | operation answered with `profiles: []` | `unknown` | `endpoint = "present"` | -Two boundaries to keep: one unrecognised route must not void a profile that answered (upstream adds routes; only when nothing usable remains does the profile fall to `unknown`), and `endpoint` distinguishes "did not answer" (`absent`) from "answered without declaring" (`empty`) from "declared" (`present`). +Two boundaries to keep: one unrecognised route must not void a profile that answered, and must not turn "owned routes all disabled" into `missing` — if that route itself says `enabled` (or omits `exposure`, whose upstream default is `unknown`) a channel exists that this table cannot judge, so `unknown` is the only honest answer; only a route that also says `disabled` leaves all-`disabled` a refusal. And `endpoint` distinguishes "did not answer" (`absent`) from "answered without declaring" (`empty`) from "declared" (`present`). ### Routes without an operation @@ -282,18 +289,20 @@ from opendirector.domain.capability_contract import capabilities_for capabilities_for('speech', 'lyrics_to_music') # ('audio.music',) capabilities_for('speech', 'text_to_speech') # ('audio.speech',) -capabilities_for('audio_generation', 'text_to_audio') # ('audio.music',) +capabilities_for('audio_generation', 'text_to_audio') # () — upstream publishes it, no id here ``` `resolve(payload)` returns per-capability states, per-capability `reasons`, `topology` (display only) and `unresolved`; `classified_capabilities()` returns the vocabulary in table order. Whether to wire this into `requests_for()` is #266's decision; this contract only guarantees the lookup is paired, ordered and testable. ### Fixtures and checks -`tests/fixtures/omni-capabilities/{full,partial,absent}.json` follow the real response shape — `stages[]` as `{id,type,depends_on}` with numeric-string ids, six complete `operations` per model each with `support`/`exposure`/`reason_code`/`profiles`/`execution`, every profile carrying `support` and `routes` (possibly `[]`), and `image_response`/`audio_response` outputs `["image"]`/`["audio"]` as published. `full` is the all-yes deployment, `partial` mixes refusals, a policy-disabled operation, a profile whose own route is disabled, a misplaced route and an all-`unknown` body, and `absent` covers no answer (`absent`), topology only (`empty`) and six operations that declare `profiles: []` (`present`, all `unknown`). +`tests/fixtures/omni-capabilities/{full,partial,absent}.json` are the **evaluated sample** and follow the real response shape — `stages[]` as `{id,type,depends_on}` with numeric-string ids, six complete `operations` per model each with `support`/`exposure`/`reason_code`/`profiles`/`execution`, every profile carrying `support`, `reason_code` and `routes` (possibly `[]`), and `required_inputs`/`outputs` transcribed from upstream's `STANDARD_PROFILES` including order (`image_response`/`audio_response` are `["image"]`/`["audio"]`). `full` is the all-yes deployment, `partial` mixes refusals, a policy-disabled operation, a profile whose own route is disabled and an all-`unknown` body, and `absent` covers no answer (`absent`), topology only (`empty`) and six operations that declare `profiles: []` (`present`, all `unknown`). In `full`, four profiles report `no_local_capability` (`image_response`, `audio_response`, `caption_to_audio`, `text_to_audio`) — `text_to_audio` grants no `audio.music`; `audio.music` is supplied by `lyrics_to_music`. + +A fourth file, `malformed.json`, is **defensive input only and not a sample of what upstream can produce**: its bodies are valid pydantic (the structural checks pass) but upstream's projection cannot emit them — a profile pointing at another operation's route, or owned routes all `disabled` beside an `enabled` path this table does not know. Only a proxy, a cache or an older revision can hand us one, so it is read by the defensive and structural checks alone, never by the sample oracle. -Shapes upstream can never emit — an `asr` operation key, a profile filed under the wrong operation, a missing required `support` — are built inline in the tests that defend against malformed input, never in a fixture, so "the fixtures are a real sample" stays true. `test_fixtures_are_schema_valid` asserts only the structural invariants of upstream's model, not a copy of it. +Shapes upstream can never emit — an `asr` operation key, a profile filed under the wrong operation, a missing required `support` — are built inline in the tests that defend against malformed input, never in the sample fixtures. `test_fixtures_are_schema_valid` asserts only the structural invariants of upstream's model (six keys, `observed_at` parseable and timezone-aware, `reason_code` wherever `support` is, `execution` as `{state,scope,observed_at}`, the enum domains, `profiles` as a list, routes carrying `transport`/`method`/`path`, stage shape with numeric-string ids), not a copy of upstream's pydantic source. -Five regressions keep coverage honest: `test_real_shaped_body_resolves_non_empty_pairs`, `test_route_alone_cannot_distinguish_speech_profiles`, `test_stage_ids_do_not_collapse_capabilities`, `test_capabilities_without_an_upstream_profile_are_unknown_not_missing`, and `test_a_profile_without_support_is_unknown_even_when_its_operation_is_supported`; the route-level exposure, transport/method, chat-modality and missing-reason cases each have their own test, listed in the Chinese section above. +Five regressions keep coverage honest: `test_real_shaped_body_resolves_non_empty_pairs`, `test_route_alone_cannot_distinguish_speech_profiles`, `test_stage_ids_do_not_collapse_capabilities`, `test_capabilities_without_an_upstream_profile_are_unknown_not_missing`, and `test_a_profile_without_support_is_unknown_even_when_its_operation_is_supported`; the route-level exposure, transport/method, chat-modality, `text_to_audio` and missing-reason cases each have their own test, and `test_documented_required_inputs_match_fixtures` pins the twelve `required_inputs`/`outputs` rows against both doc tables and every fixture. ```bash PYTHONPATH=packages/core/src:apps/api/src python -m pytest tests/test_capability_contract.py -q diff --git a/packages/core/src/opendirector/domain/capability_contract.py b/packages/core/src/opendirector/domain/capability_contract.py index 9eed655b..177f4e63 100644 --- a/packages/core/src/opendirector/domain/capability_contract.py +++ b/packages/core/src/opendirector/domain/capability_contract.py @@ -33,18 +33,24 @@ text-only adapter (`integrations/model_adapters.py:566-580` reads `choices[0]["message"]["content"]` as a string), so only `text_response` grants it: an image-only chat deployment must not read as able to chat. - `caption_to_audio` (sound effect) has no id here either. `text_to_audio` does - grant `audio.music`: it is text-to-audio, the same family as - `lyrics_to_music`, and the only difference is the diffusion request contract - behind `/v1/audio/generate` (`docs/serving/capabilities.md:59-60`). + `caption_to_audio` (sound effect) has no id here either. Neither does + `text_to_audio`: it is the diffusion contract behind `/v1/audio/generate` + with no lyrics and no `instructions` (`docs/serving/capabilities.md:59-60`), + while this repository's only `audio.music` adapter posts `/v1/audio/speech` + with both (`VllmOmniMusic`, `integrations/model_adapters.py:510-537`). + Mapping it would promise a request the deployment never accepted — the same + error as pushing `caption_to_audio` onto `audio.speech`. * A declaration that says nothing is `unknown`, never `missing`. Optional fields are simply absent from a published body (`response_model_exclude_none=True`, `api_server.py:2193`) and `unknown` is a first-class upstream value (`support`, `exposure`; "Missing operations/profiles are unknown", - `serving_capabilities.py:84`). Required fields are never absent: `support`, - `exposure` and `reason_code` are mandatory on every operation and profile, and - all six operations must be listed (`protocol/capabilities.py:36-64`). A body - that omits them is malformed input to be defended against, not a shape + `serving_capabilities.py:84`). Required fields are never absent: `support` and + `reason_code` are mandatory on every operation and on every profile, + `exposure` on every operation and on every top-level route, and all six + operations must be listed (`protocol/capabilities.py:24-64`). A profile has no + `exposure` field of its own (`:36-44`): how reachable it is comes from + aggregating the exposures of the routes it references. A body that omits a + required field is malformed input to be defended against, not a shape upstream publishes. Only positive evidence produces `missing`, and the result keeps the two kinds apart in its `reasons` map: `unsupported` (the model refused the profile), `endpoint_disabled` (the deployment switched the @@ -59,11 +65,13 @@ * Reachability is decided on the profile's own route references, and their `exposure` is read. Every declared route `disabled` means `missing` with `route_disabled`; every route `unknown` (or no route at all) means `unknown`, - because reachability cannot be confirmed; one `enabled` route is enough. A - route this table does not recognise is reported in `unresolved`, but only - makes the profile `unknown` when nothing else was declared: upstream adds - routes over time, and one new path must not void a profile that already - answered. + because reachability cannot be confirmed; one `enabled` owned route is enough. + A route this table does not recognise is reported in `unresolved`, and its own + `exposure` still counts: such a route saying `enabled` (or nothing) is a + channel this table cannot judge, so owned routes that are all `disabled` are + not proof of absence and the profile stays `unknown` instead of `missing`. A + profile that already answered stays `available` either way: upstream adds + routes over time, and one new path must not void a confirmed profile. `asr` is not an omission on this side: upstream publishes no recognition operation at all (`OperationId` is a closed `Literal`) and the omni router never @@ -112,14 +120,20 @@ # able to chat, and would contradict this module's own refusal to push # `caption_to_audio` onto `audio.speech`. # -# `caption_to_audio` is a sound-effect contract (upstream requires -# `ambient_sound`, `tests/config/test_serving_capabilities.py:233`), not speech; -# mapping it onto `audio.speech` would let a request carry a voice the deployment -# never accepted. `text_to_audio` is the separate diffusion contract behind -# `/v1/audio/generate` (no voice, no lyrics; `docs/serving/capabilities.md:59-60`) -# but it is still text-to-audio, so it grants `audio.music` alongside -# `lyrics_to_music`: without that row a StableAudio/LTX2-t2a shape would report -# `audio.music=missing` while the deployment says it generates audio. +# `caption_to_audio` is a sound-effect contract, not speech: MOSS-SoundEffect +# overrides its `required_inputs` with `ambient_sound` +# (`model_executor/models/moss_tts/pipeline.py:205`), and mapping it onto +# `audio.speech` would let a request carry a voice the deployment never accepted. +# `text_to_audio` is empty for the same kind of reason. It is the separate +# diffusion contract behind `/v1/audio/generate`, with no lyrics and no +# `instructions` (`docs/serving/capabilities.md:59-60`), while the only +# `audio.music` adapter here (`VllmOmniMusic`, `integrations/model_adapters.py:510-537`) +# posts `/v1/audio/speech` with lyrics *and* `instructions`, both required. A +# deployment publishing `text_to_audio` cannot answer that request, so calling it +# `audio.music` would be a false `available` — the same error as pushing +# `caption_to_audio` onto `audio.speech`. Such a deployment reports +# `no_local_capability`, and `audio.music` keeps whatever `lyrics_to_music` +# says (no evidence -> `unknown`) rather than being granted by a neighbour. OPERATION_PROFILES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( ('chat', 'text_response', ('text.chat',)), ('chat', 'image_response', ()), @@ -132,7 +146,7 @@ ('speech', 'text_to_speech', ('audio.speech',)), ('speech', 'lyrics_to_music', ('audio.music',)), ('speech', 'caption_to_audio', ()), - ('audio_generation', 'text_to_audio', ('audio.music',)), + ('audio_generation', 'text_to_audio', ()), ) # operation → the routes that operation owns, as `(transport, method, path)` @@ -396,11 +410,12 @@ def _operation_verdict(operation): return 'unknown', None -def _profile_verdict(entry, operation_verdict, operation_reason, exposures): +def _profile_verdict(entry, operation_verdict, operation_reason, exposures, unrecognised=()): """`(verdict, reason)` for one profile inside an already-judged operation. - `exposures` holds one value per route reference the operation owns: an empty - list means the profile declared no usable route of its own. + `exposures` holds one value per route reference the operation owns, and + `unrecognised` one per declared route this table could not place: an empty + `exposures` means the profile declared no usable route of its own. """ support = entry.get('support') if support == UNSUPPORTED: @@ -409,10 +424,16 @@ def _profile_verdict(entry, operation_verdict, operation_reason, exposures): return 'no', operation_reason if support != SUPPORTED or operation_verdict != 'yes': return 'unknown', None - if not exposures: - return 'unknown', None if ENABLED in exposures: return 'yes', None + # A route this table cannot place still states whether it is switched on. + # One saying `enabled` (or nothing) is a channel whose reachability cannot be + # confirmed, so owned routes that are all `disabled` are not proof of + # absence: `unknown`, never `missing`. + if ENABLED in unrecognised or ROUTE_UNKNOWN in unrecognised: + return 'unknown', None + if not exposures: + return 'unknown', None if ROUTE_UNKNOWN in exposures: return 'unknown', None return 'no', ROUTE_DISABLED_REASON @@ -436,13 +457,19 @@ def _check_route(operation, profile, key, unresolved): def _route_exposures(operation, profile, entry, unresolved): - """`(exposures of the routes this operation owns, whether any path was declared)`. + """`(owned route exposures, unrecognised route exposures, any path declared)`. A route reference whose path cannot be read is reported rather than dropped: silently ignoring it would turn a broken declaration into `unknown` with no explanation, which is the failure this module exists to prevent. + + Routes the operation does not own are reported too, and their `exposure` is + kept in its own list: they may not grant a capability, but they can still + say that a channel exists, which is what keeps known-but-off routes from + being read as a refusal (see `_profile_verdict`). """ exposures = [] + unrecognised = [] declared_path = False for value in _routes(entry): key = _route_key(value) @@ -452,7 +479,9 @@ def _route_exposures(operation, profile, entry, unresolved): declared_path = True if _check_route(operation, profile, key, unresolved): exposures.append(_route_exposure(value)) - return exposures, declared_path + else: + unrecognised.append(_route_exposure(value)) + return exposures, unrecognised, declared_path def resolve(payload): @@ -518,8 +547,9 @@ def record(capabilities, verdict, reason=None): if owner != key: _note(unresolved, key, profile_id, None, OPERATION_MISMATCH) continue - exposures, declared_path = _route_exposures(key, profile_id, entry, unresolved) - verdict, reason = _profile_verdict(entry, operation_verdict, operation_reason, exposures) + exposures, unrecognised, declared_path = _route_exposures(key, profile_id, entry, unresolved) + verdict, reason = _profile_verdict(entry, operation_verdict, operation_reason, exposures, + unrecognised) # A supported profile that declares no route at all: reachability # cannot be confirmed, and nothing else explains the `unknown`. if verdict == 'unknown' and not exposures and not declared_path: diff --git a/tests/fixtures/omni-capabilities/full.json b/tests/fixtures/omni-capabilities/full.json index f447fdad..828fab6e 100644 --- a/tests/fixtures/omni-capabilities/full.json +++ b/tests/fixtures/omni-capabilities/full.json @@ -5,7 +5,7 @@ "cases": [ { "id": "every-operation-and-profile", - "note": "All six operations supported and enabled, all twelve profiles declared. Only `text_response` grants `text.chat`: the image and audio chat profiles are reported as `no_local_capability`, and `text_to_audio` grants `audio.music`.", + "note": "All six operations supported and enabled, all twelve profiles declared. Only `text_response` grants `text.chat`, and only `lyrics_to_music` grants `audio.music`: the image/audio chat profiles and `text_to_audio` are reported as `no_local_capability`.", "payload": { "object": "omni.capabilities", "schema_version": 1, @@ -145,8 +145,8 @@ "id": "image_to_image", "support": "supported", "required_inputs": [ - "image", - "text" + "text", + "image" ], "outputs": [ "image" @@ -210,8 +210,8 @@ "id": "image_to_video", "support": "supported", "required_inputs": [ - "image", - "text" + "text", + "image" ], "outputs": [ "video" @@ -237,8 +237,7 @@ "id": "video_to_video", "support": "supported", "required_inputs": [ - "video", - "text" + "video" ], "outputs": [ "video" @@ -478,6 +477,12 @@ }, "reasons": {}, "unresolved": [ + { + "operation": "audio_generation", + "profile": "text_to_audio", + "route": null, + "reason": "no_local_capability" + }, { "operation": "chat", "profile": "audio_response", diff --git a/tests/fixtures/omni-capabilities/malformed.json b/tests/fixtures/omni-capabilities/malformed.json new file mode 100644 index 00000000..4ec16f9e --- /dev/null +++ b/tests/fixtures/omni-capabilities/malformed.json @@ -0,0 +1,972 @@ +{ + "tier": "malformed", + "upstreamEndpoint": "/v1/omni/capabilities", + "scope": "Defensive bodies only: shapes upstream's own projection cannot produce (a profile pointing at another operation's route), because it appends only the routes of the profile's own operation (`entrypoints/openai/serving_capabilities.py:339-359`). They are still valid pydantic, so a proxy, a cache or an older revision can hand them to us. Never treat them as a sample of a real response.", + "cases": [ + { + "id": "profile-on-another-operations-route", + "note": "`text_to_video` declares `/v1/images/edits`, which belongs to `image_edit`: the profile grants nothing and is reported as `route_mismatch`, while `image_to_video` keeps `video.generate` available.", + "payload": { + "object": "omni.capabilities", + "schema_version": 1, + "observed_at": "2026-11-10T00:00:00Z", + "stages": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], + "models": [ + { + "id": "model-b", + "stage_ids": [ + "7" + ], + "operations": { + "chat": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_response", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "text" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "image_response", + "support": "unsupported", + "required_inputs": [ + "text" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "chat_prompt_contract_undeclared", + "constraints": {} + }, + { + "id": "audio_response", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "chat_prompt_contract_undeclared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_generation": { + "support": "supported", + "exposure": "disabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_image", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_edit": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "image_to_image", + "support": "supported", + "required_inputs": [ + "text", + "image" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "disabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "video_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_video", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "image_to_video", + "support": "supported", + "required_inputs": [ + "text", + "image" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "video_to_video", + "support": "unsupported", + "required_inputs": [ + "video" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + } + ], + "reason_code": "checkpoint_requires_expand_timesteps", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "speech": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_speech", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "lyrics_to_music", + "support": "unsupported", + "required_inputs": [ + "text", + "instructions" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_not_supported", + "constraints": {} + }, + { + "id": "caption_to_audio", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "audio_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_audio", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + } + } + } + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/depth", + "exposure": "enabled" + } + ] + }, + "expected": { + "endpoint": "present", + "topology": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], + "capabilities": { + "text.chat": "available", + "image.generate": "missing", + "image.edit": "missing", + "video.generate": "available", + "audio.speech": "available", + "audio.music": "missing", + "image.segment": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", + "audio.score": "unknown", + "video.first_frame": "unknown", + "video.edit": "unknown", + "image.cutout": "unknown", + "audio.separate": "unknown" + }, + "reasons": { + "image.generate": "endpoint_disabled", + "image.edit": "route_disabled", + "audio.music": "unsupported" + }, + "unresolved": [ + { + "operation": "audio_generation", + "profile": "text_to_audio", + "route": null, + "reason": "no_local_capability" + }, + { + "operation": "speech", + "profile": "caption_to_audio", + "route": null, + "reason": "no_local_capability" + }, + { + "operation": "video_generation", + "profile": "text_to_video", + "route": "/v1/images/edits", + "reason": "route_mismatch" + } + ] + } + }, + { + "id": "unrecognised-route-beside-a-disabled-route", + "note": "`image_to_image` turns its only owned route off but declares an `enabled` path this table does not know: reachability cannot be confirmed, so `image.edit` is `unknown`, not `missing`.", + "payload": { + "object": "omni.capabilities", + "schema_version": 1, + "observed_at": "2026-11-10T00:00:00Z", + "stages": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], + "models": [ + { + "id": "model-b", + "stage_ids": [ + "7" + ], + "operations": { + "chat": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_response", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "text" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "image_response", + "support": "unsupported", + "required_inputs": [ + "text" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "chat_prompt_contract_undeclared", + "constraints": {} + }, + { + "id": "audio_response", + "support": "unknown", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + } + ], + "reason_code": "chat_prompt_contract_undeclared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_generation": { + "support": "supported", + "exposure": "disabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_image", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "image_edit": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "image_to_image", + "support": "supported", + "required_inputs": [ + "text", + "image" + ], + "outputs": [ + "image" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "disabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits/stream", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "video_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_video", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "image_to_video", + "support": "supported", + "required_inputs": [ + "text", + "image" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "video_to_video", + "support": "unsupported", + "required_inputs": [ + "video" + ], + "outputs": [ + "video" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + } + ], + "reason_code": "checkpoint_requires_expand_timesteps", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "speech": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_speech", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + }, + { + "id": "lyrics_to_music", + "support": "unsupported", + "required_inputs": [ + "text", + "instructions" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_not_supported", + "constraints": {} + }, + { + "id": "caption_to_audio", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + }, + "audio_generation": { + "support": "supported", + "exposure": "enabled", + "reason_code": "model_declared", + "profiles": [ + { + "id": "text_to_audio", + "support": "supported", + "required_inputs": [ + "text" + ], + "outputs": [ + "audio" + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + } + ], + "reason_code": "model_declared", + "constraints": {} + } + ], + "execution": { + "state": "unpaused", + "scope": "worker", + "observed_at": "2026-11-10T00:00:00Z" + } + } + } + } + ], + "routes": [ + { + "transport": "http", + "method": "POST", + "path": "/v1/chat/completions", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/generations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/edits", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/speech", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/audio/generate", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/images/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/segmentations", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/depth", + "exposure": "enabled" + } + ] + }, + "expected": { + "endpoint": "present", + "topology": [ + { + "id": "7", + "type": "llm", + "depends_on": [] + }, + { + "id": "42", + "type": "diffusion", + "depends_on": [ + "7" + ] + } + ], + "capabilities": { + "text.chat": "available", + "image.generate": "missing", + "image.edit": "unknown", + "video.generate": "available", + "audio.speech": "available", + "audio.music": "missing", + "image.segment": "unknown", + "video.matting": "unknown", + "video.motion_capture": "unknown", + "audio.score": "unknown", + "video.first_frame": "unknown", + "video.edit": "unknown", + "image.cutout": "unknown", + "audio.separate": "unknown" + }, + "reasons": { + "image.generate": "endpoint_disabled", + "audio.music": "unsupported" + }, + "unresolved": [ + { + "operation": "audio_generation", + "profile": "text_to_audio", + "route": null, + "reason": "no_local_capability" + }, + { + "operation": "image_edit", + "profile": "image_to_image", + "route": "/v1/images/edits/stream", + "reason": "unknown_route" + }, + { + "operation": "speech", + "profile": "caption_to_audio", + "route": null, + "reason": "no_local_capability" + } + ] + } + } + ] +} diff --git a/tests/fixtures/omni-capabilities/partial.json b/tests/fixtures/omni-capabilities/partial.json index be260db1..578d2069 100644 --- a/tests/fixtures/omni-capabilities/partial.json +++ b/tests/fixtures/omni-capabilities/partial.json @@ -1,11 +1,11 @@ { "tier": "partial", "upstreamEndpoint": "/v1/omni/capabilities", - "scope": "Bodies that answer, but disagree with themselves: a policy-disabled operation, a profile whose own route is disabled, a misplaced route, an unsupported profile, and an operation that states `unknown`.", + "scope": "Bodies that answer, but disagree with themselves: a policy-disabled operation, a profile whose own route is disabled, an unsupported profile, a profile with no local id, and an operation that states `unknown`.", "cases": [ { "id": "mixed-operations", - "note": "`image_generation` is supported but disabled (`endpoint_disabled`), `image_edit` is supported but its only route is `disabled` (`route_disabled`), one video profile declares an image path (`route_mismatch`), `lyrics_to_music` is unsupported while `text_to_audio` is supported, and `caption_to_audio` has no local id.", + "note": "`image_generation` is supported but disabled (`endpoint_disabled`), `image_edit` is supported but its only route is `disabled` (`route_disabled`), `lyrics_to_music` is unsupported while `text_to_audio` is supported (no local id, `no_local_capability`), and `caption_to_audio` has no local id either.", "payload": { "object": "omni.capabilities", "schema_version": 1, @@ -144,8 +144,8 @@ "id": "image_to_image", "support": "supported", "required_inputs": [ - "image", - "text" + "text", + "image" ], "outputs": [ "image" @@ -186,7 +186,19 @@ { "transport": "http", "method": "POST", - "path": "/v1/images/edits", + "path": "/v1/videos", + "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", "exposure": "enabled" } ], @@ -197,8 +209,8 @@ "id": "image_to_video", "support": "supported", "required_inputs": [ - "image", - "text" + "text", + "image" ], "outputs": [ "video" @@ -218,8 +230,7 @@ "id": "video_to_video", "support": "unsupported", "required_inputs": [ - "video", - "text" + "video" ], "outputs": [ "video" @@ -292,7 +303,7 @@ "id": "caption_to_audio", "support": "supported", "required_inputs": [ - "ambient_sound" + "text" ], "outputs": [ "audio" @@ -441,7 +452,7 @@ "image.edit": "missing", "video.generate": "available", "audio.speech": "available", - "audio.music": "available", + "audio.music": "missing", "image.segment": "unknown", "video.matting": "unknown", "video.motion_capture": "unknown", @@ -453,20 +464,21 @@ }, "reasons": { "image.generate": "endpoint_disabled", - "image.edit": "route_disabled" + "image.edit": "route_disabled", + "audio.music": "unsupported" }, "unresolved": [ { - "operation": "speech", - "profile": "caption_to_audio", + "operation": "audio_generation", + "profile": "text_to_audio", "route": null, "reason": "no_local_capability" }, { - "operation": "video_generation", - "profile": "text_to_video", - "route": "/v1/images/edits", - "reason": "route_mismatch" + "operation": "speech", + "profile": "caption_to_audio", + "route": null, + "reason": "no_local_capability" } ] } @@ -612,7 +624,8 @@ "id": "image_to_image", "support": "unknown", "required_inputs": [ - "text" + "text", + "image" ], "outputs": [ "image" @@ -676,7 +689,8 @@ "id": "image_to_video", "support": "unknown", "required_inputs": [ - "text" + "text", + "image" ], "outputs": [ "video" @@ -702,7 +716,7 @@ "id": "video_to_video", "support": "unknown", "required_inputs": [ - "text" + "video" ], "outputs": [ "video" @@ -760,7 +774,8 @@ "id": "lyrics_to_music", "support": "unknown", "required_inputs": [ - "text" + "text", + "instructions" ], "outputs": [ "audio" diff --git a/tests/test_capability_contract.py b/tests/test_capability_contract.py index db3e84a1..98f73ab4 100644 --- a/tests/test_capability_contract.py +++ b/tests/test_capability_contract.py @@ -1,7 +1,8 @@ """The capability contract translates upstream's (operation, profile) vocabulary. -`tests/fixtures/omni-capabilities/` is the shared sample set: three tiers built -from the response shape upstream actually publishes — `stages[]` as topology, +`tests/fixtures/omni-capabilities/` is the shared sample set: three tiers +(`full`, `partial`, `absent`) built from the response shape upstream actually +publishes — `stages[]` as topology, `models[].operations[].profiles[]` as the declaration, and only optional fields ever absent. `test_fixtures_are_schema_valid` checks the structural invariants of that shape (`protocol/capabilities.py:16-79`), so a @@ -9,10 +10,13 @@ the oracle. The expected blocks are the reviewed oracle, not a copy of the mapping table, so changing a mapping has to argue with them. -Shapes upstream can never publish — an `asr` operation key, a profile filed under -the wrong operation, a missing required `support` — are built inline in the -tests that defend against malformed input, never in a fixture, so the fixtures -stay an honest sample of real bodies. +A fourth file, `malformed.json`, holds bodies only a rewrite can produce — a +profile pointing at another operation's route, a disabled route beside a path +this table does not know. It is read by the defensive and structural checks +alone, never by the sample oracle: it is input to defend against, not evidence +of what upstream publishes. The shapes upstream can never emit — an `asr` +operation key, a profile filed under the wrong operation, a missing required +`support` — stay inline in the tests that defend against them. The last group guards the failure this table exists to prevent: a route (or a stage) that is read as if it named a capability. `/v1/audio/speech` carries @@ -20,6 +24,7 @@ without an operation proves nothing — each of those gets its own regression test below. """ +import datetime import json from pathlib import Path from typing import get_args @@ -30,12 +35,17 @@ from opendirector.domain.capability_contract import (ABSENT, AVAILABLE, EMPTY, ENDPOINT, MALFORMED_ROUTE, MISSING, MISSING_REASONS, NO_LOCAL_CAPABILITY, NO_ROUTE_DECLARED, OPERATION_MISMATCH, OPERATION_PROFILES, OPERATION_ROUTES, - OPERATIONS, PRESENT, REASONS, ROUTED_PATHS, ROUTE_MISMATCH, - UNKNOWN, UNKNOWN_PROFILE, UNKNOWN_ROUTE, UNMAPPED_STAGE, UNROUTED, - UNROUTED_REASONS, capabilities_for, classified_capabilities, - declarations, resolve, route_path) + OPERATIONS, PRESENT, PROFILE_OPERATIONS, REASONS, ROUTED_PATHS, + ROUTE_MISMATCH, UNKNOWN, UNKNOWN_PROFILE, UNKNOWN_ROUTE, + UNMAPPED_STAGE, UNROUTED, UNROUTED_REASONS, capabilities_for, + classified_capabilities, declarations, resolve, route_path) FIXTURES = Path(__file__).parent / 'fixtures' / 'omni-capabilities' +CONTRACT_DOC = Path(__file__).resolve().parents[1] / 'docs' / 'architecture' / 'capability-contract.md' +# The published sample. `malformed` is deliberately not among them: defining it +# as a fourth tier here is what keeps defensive bodies out of the oracle. +TIERS = ('absent', 'full', 'partial') +DEFENSIVE_TIER = 'malformed' # Upstream bodies may name models; credentials and raw provider responses never # belong in a fixture, so their vocabulary is refused outright. FORBIDDEN = ('apikey', 'api_key', 'authorization', 'bearer', 'secret', 'password', 'cookie', '://') @@ -63,21 +73,69 @@ } -def raw_cases(): +# Upstream's `STANDARD_PROFILES` (`vllm_omni/serving_capabilities.py:43-54`, head +# `d1104f8`), transcribed as `(profile, required_inputs, outputs)`. Literal on +# purpose: this is the one place in the change that claims to quote upstream, so +# deriving it from the module under test would only prove the table echoes +# itself. Order inside a tuple is part of the declaration (`image_to_image` is +# text-then-image) and `image_response`/`audio_response` publish a non-text +# output, which is why they grant nothing here. +UPSTREAM_PROFILES = ( + ('text_response', ('text',), ('text',)), + ('image_response', ('text',), ('image',)), + ('audio_response', ('text',), ('audio',)), + ('text_to_image', ('text',), ('image',)), + ('image_to_image', ('text', 'image'), ('image',)), + ('text_to_video', ('text',), ('video',)), + ('image_to_video', ('text', 'image'), ('video',)), + ('video_to_video', ('video',), ('video',)), + ('text_to_speech', ('text',), ('audio',)), + ('lyrics_to_music', ('text', 'instructions'), ('audio',)), + ('caption_to_audio', ('text',), ('audio',)), + ('text_to_audio', ('text',), ('audio',)), +) + + +def parse_observed(value): + """A datetime from an upstream `observed_at`, raising if it is not one.""" + assert isinstance(value, str) and value, value + return datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) + + +def raw_documents(tiers): + """Fixture documents of the requested tiers, in filename order.""" paths = sorted(FIXTURES.glob('*.json')) - assert {path.stem for path in paths} == {'full', 'partial', 'absent'} + assert {path.stem for path in paths} == set(TIERS) | {DEFENSIVE_TIER} for path in paths: + if path.stem not in tiers: + continue document = json.loads(path.read_text(encoding='utf-8')) - assert document['upstreamEndpoint'] == ENDPOINT + assert document['tier'] == path.stem, path.stem + assert document['upstreamEndpoint'] == ENDPOINT, path.stem yield document, document['cases'] +def raw_cases(): + """The published sample: bodies upstream's own projection can emit.""" + return raw_documents(TIERS) + + +def defensive_cases(): + """Bodies a proxy, cache or older revision can hand us; never a sample.""" + return raw_documents((DEFENSIVE_TIER,)) + + def fixture_cases(): for document, cases in raw_cases(): for case in cases: yield pytest.param(case['payload'], case['expected'], id=f"{document['tier']}-{case['id']}") +def defensive_params(): + return [pytest.param(case['payload'], case['expected'], id=f"{document['tier']}-{case['id']}") + for document, cases in defensive_cases() for case in cases] + + def route(path, exposure='enabled', method='POST', transport='http'): return {'transport': transport, 'method': method, 'path': path, 'exposure': exposure} @@ -131,6 +189,17 @@ def test_declaration_resolves_to_the_reviewed_expectation(payload, expected): assert resolve(payload) == expected +@pytest.mark.parametrize('payload, expected', defensive_params()) +def test_defensive_fixture_bodies_resolve_to_the_reviewed_expectation(payload, expected): + """`malformed.json` is input to defend against, not a sample of a real body. + + It is kept apart from the sample oracle precisely so the three tiers above + can claim to be published shapes; these cases only prove the defensive + branches still answer as reviewed. + """ + assert resolve(payload) == expected + + # --- the table itself ----------------------------------------------------- def test_every_capability_id_is_routed_or_explained(): @@ -199,17 +268,28 @@ def test_operation_routes_follow_upstream_ownership(): def test_unowned_endpoints_are_not_in_the_route_table(): - """Registered routes without an operation must not be readable as capabilities.""" + """Registered routes without an operation must not be readable as capabilities. + + The top-level `routes[]` is display data: `resolve()` never consults it, which + is why this test resolves one set of declarations three times — with the + published routes, with none, and with only unowned paths — and requires one + identical result. Asserting "an unowned path is not in `ROUTED_PATHS`" alone + would only test a constant; the point is that publishing such a path in the + body changes no verdict. + """ unowned = ('/v1/images/segmentations', '/v1/videos/segmentations', '/v1/videos/depth', '/v1/audio/transcriptions', '/v1/audio/speech/batch', '/v1/chat/completions/batch') for path in unowned: assert path not in ROUTED_PATHS, path - # Publishing them in the body's top-level `routes[]` still grants nothing. - payload = body(models=[]) - payload['routes'] = [route(path) for path in unowned] - result = resolve(payload) + declared = body(every_operation()) + from_unowned = {**declared, 'routes': [route(path) for path in unowned]} + from_nothing = {**declared, 'routes': []} + assert resolve(from_unowned) == resolve(from_nothing) == resolve(declared) for capability in ('image.segment', 'video.matting', 'video.motion_capture', 'audio.score'): - assert result['capabilities'][capability] == UNKNOWN, capability + assert resolve(from_unowned)['capabilities'][capability] == UNKNOWN, capability + # A body that carries only unowned routes and no model declaration answered + # nothing at all: it is `empty`, not a list of capabilities. + assert resolve(body(models=[], stages=[]) | {'routes': [route(path) for path in unowned]})['endpoint'] == EMPTY def test_music_is_a_speech_profile_on_the_speech_route(): @@ -242,9 +322,12 @@ def test_real_shaped_body_resolves_non_empty_pairs(): for capability in ('text.chat', 'image.generate', 'image.edit', 'video.generate', 'audio.speech', 'audio.music'): assert result['capabilities'][capability] == AVAILABLE, capability assert result['topology'] == [{'id': '7', 'type': 'llm', 'depends_on': []}] - # `not_declared_locally`: the chat profiles whose output is not text carry no - # id here, so they are reported instead of being folded into `text.chat`. - assert result['unresolved'] == [{'operation': 'chat', 'profile': 'audio_response', 'route': None, + # `no_local_capability`: the chat profiles whose output is not text, and the + # diffusion `text_to_audio`, carry no id here, so they are reported instead + # of being folded into `text.chat` or `audio.music`. + assert result['unresolved'] == [{'operation': 'audio_generation', 'profile': 'text_to_audio', 'route': None, + 'reason': NO_LOCAL_CAPABILITY}, + {'operation': 'chat', 'profile': 'audio_response', 'route': None, 'reason': NO_LOCAL_CAPABILITY}, {'operation': 'chat', 'profile': 'image_response', 'route': None, 'reason': NO_LOCAL_CAPABILITY}, @@ -442,6 +525,30 @@ def test_a_profile_whose_only_route_is_disabled_is_missing(): assert result['unresolved'] == [] +def test_a_disabled_profile_with_an_unknown_enabled_route_is_unknown(): + """Every known route off, plus a path this table cannot place that says `enabled`. + + Upstream adds routes over time and this table can lag behind. An unrecognised + route that reports `enabled` is a channel whose reachability cannot be + confirmed, so the known-but-off routes are not proof of absence: `unknown`, + never `missing`. A route that cannot be placed is no excuse for a positive no. + """ + result = resolve(body({'image_generation': operation(profiles=[ + profile('text_to_image', routes=[route('/v1/images/generations', exposure='disabled'), + route('/v1/images/generations/stream')])])})) + assert result['capabilities']['image.generate'] == UNKNOWN + assert result['reasons'] == {} + assert result['unresolved'] == [{'operation': 'image_generation', 'profile': 'text_to_image', + 'route': '/v1/images/generations/stream', 'reason': UNKNOWN_ROUTE}] + # An unrecognised route that is itself `disabled` adds no channel, so the + # known-off routes remain a refusal. + still_missing = resolve(body({'image_generation': operation(profiles=[ + profile('text_to_image', routes=[route('/v1/images/generations', exposure='disabled'), + route('/v1/images/generations/stream', exposure='disabled')])])})) + assert still_missing['capabilities']['image.generate'] == MISSING + assert still_missing['reasons'] == {'image.generate': 'route_disabled'} + + def test_a_profile_whose_routes_are_all_unknown_cannot_be_confirmed(): result = resolve(body({'image_generation': operation(profiles=[ profile('text_to_image', routes=[route('/v1/images/generations', exposure='unknown')])])})) @@ -550,13 +657,28 @@ def test_a_text_chat_deployment_still_reads_as_text_chat(): assert result['unresolved'] == [] -def test_text_to_audio_grants_audio_music(): - """StableAudio / LTX2-t2a serve `/v1/audio/generate`: text to audio is audio.music here.""" +def test_text_to_audio_alone_does_not_make_audio_music_available(): + """`text_to_audio` is the diffusion `/v1/audio/generate` contract, not music. + + It carries no lyrics and no `instructions`, while the only `audio.music` + adapter here (`VllmOmniMusic`) posts `/v1/audio/speech` with both required. + A deployment that publishes only `text_to_audio` therefore cannot answer an + `audio.music` request: the profile is reported and the capability stays + `unknown`, never a false `available`. + """ + assert capabilities_for('audio_generation', 'text_to_audio') == () result = resolve(body({'audio_generation': operation(profiles=[ profile('text_to_audio', paths=table_paths('audio_generation'), outputs=('audio',))])})) - assert result['capabilities']['audio.music'] == AVAILABLE + assert result['capabilities']['audio.music'] != AVAILABLE + assert result['capabilities']['audio.music'] == UNKNOWN + assert {'operation': 'audio_generation', 'profile': 'text_to_audio', 'route': None, + 'reason': NO_LOCAL_CAPABILITY} in result['unresolved'] assert result['capabilities']['audio.speech'] == UNKNOWN - assert result['unresolved'] == [] + # The music path is still the speech profile, on the speech route. + music = resolve(body({'speech': operation(profiles=[ + profile('lyrics_to_music', paths=table_paths('speech'), inputs=('text', 'instructions'), + outputs=('audio',))])})) + assert music['capabilities']['audio.music'] == AVAILABLE # --- untranslatable declarations ----------------------------------------- @@ -696,56 +818,179 @@ def test_results_use_only_the_contract_vocabulary(): assert set(stage) == {'id', 'type', 'depends_on'} -def test_fixtures_are_schema_valid(): - """Structural invariants of upstream's response model, checked on our fixtures. - - Upstream ships these as pydantic models (`protocol/capabilities.py:16-79`, - head `d1104f8`). Re-implementing them here would be a second copy to keep in - sync, so this test only asserts the invariants a rejected body would break: - all six operations, `support`/`exposure` on every operation, `profiles` as a - list (it serializes as `[]`, it never disappears), the profile `support` - required, stages as `{id, type, depends_on}` with numeric-string ids. +def schema_problems(payload): + """Structural invariants upstream's pydantic models impose on a body. + + Upstream ships `protocol/capabilities.py:16-79` as pydantic models. Copying + them here would be a second implementation to keep in sync, so this only + checks the invariants a rejected body would break: the six keys, `observed_at` + as a datetime, `reason_code` present wherever `support` is, `execution` as + `{state, scope, observed_at}`, `support`/`exposure` domains, `profiles` as a + list (it serializes as `[]`, it never disappears), and routes as + `{transport, method, path, exposure}` with a method. """ - operations = set(OPERATIONS) + problems = [] + if set(payload) != {'object', 'schema_version', 'observed_at', 'stages', 'models', 'routes'}: + problems.append(f'keys {sorted(payload)}') + if payload.get('object') != 'omni.capabilities' or payload.get('schema_version') != 1: + problems.append('object/schema_version') + try: + assert parse_observed(payload['observed_at']).tzinfo is not None + except (AssertionError, KeyError, TypeError, ValueError) as error: + problems.append(f'observed_at: {error!r}') + for stage in payload.get('stages', []): + if set(stage) != {'id', 'type', 'depends_on'}: + problems.append(f'stage keys {sorted(stage)}') + if not (isinstance(stage.get('id'), str) and stage['id'].isdigit()): + problems.append(f'stage id {stage.get("id")!r}') + if not isinstance(stage.get('depends_on'), list): + problems.append(f'stage depends_on {stage.get("depends_on")!r}') + if stage.get('type') not in {'llm', 'diffusion', 'unknown'}: + problems.append(f'stage type {stage.get("type")!r}') + for model in payload.get('models', []): + if not {'id', 'stage_ids', 'operations'} <= set(model): + problems.append(f'model keys {sorted(model)}') + stage_ids = model.get('stage_ids') + if not isinstance(stage_ids, list) or not all(isinstance(item, str) and item.isdigit() + for item in stage_ids): + problems.append(f'model stage_ids {stage_ids!r}') + operations = model.get('operations') + if not isinstance(operations, dict) or set(operations) != set(OPERATIONS): + problems.append(f'operations {sorted(operations or ())}') + continue + for operation_id, declared in operations.items(): + if declared.get('support') not in {'supported', 'unsupported', 'unknown'}: + problems.append(f'{operation_id} support {declared.get("support")!r}') + if declared.get('exposure') not in {'enabled', 'disabled', 'unknown'}: + problems.append(f'{operation_id} exposure {declared.get("exposure")!r}') + if not (isinstance(declared.get('reason_code'), str) and declared['reason_code']): + problems.append(f'{operation_id} reason_code {declared.get("reason_code")!r}') + execution = declared.get('execution') + if not isinstance(execution, dict) or set(execution) != {'state', 'scope', 'observed_at'}: + problems.append(f'{operation_id} execution {execution!r}') + continue + if execution['state'] not in {'unpaused', 'paused', 'unknown'}: + problems.append(f'{operation_id} execution.state {execution["state"]!r}') + if execution['scope'] not in {'deployment', 'worker'}: + problems.append(f'{operation_id} execution.scope {execution["scope"]!r}') + try: + assert parse_observed(execution['observed_at']).tzinfo is not None + except (AssertionError, TypeError, ValueError) as error: + problems.append(f'{operation_id} execution.observed_at: {error!r}') + if not isinstance(declared.get('profiles'), list): + problems.append(f'{operation_id} profiles {declared.get("profiles")!r}') + continue + for entry in declared['profiles']: + if entry.get('support') not in {'supported', 'unsupported', 'unknown'}: + problems.append(f'{operation_id}/{entry.get("id")} support {entry.get("support")!r}') + if not (isinstance(entry.get('reason_code'), str) and entry['reason_code']): + problems.append(f'{operation_id}/{entry.get("id")} reason_code') + if not isinstance(entry.get('required_inputs'), list): + problems.append(f'{operation_id}/{entry.get("id")} required_inputs') + if not isinstance(entry.get('outputs'), list): + problems.append(f'{operation_id}/{entry.get("id")} outputs') + if not isinstance(entry.get('routes'), list): + problems.append(f'{operation_id}/{entry.get("id")} routes') + continue + for reference in entry['routes']: + if set(reference) != {'transport', 'method', 'path', 'exposure'}: + problems.append(f'{operation_id}/{entry.get("id")} route keys {sorted(reference)}') + if reference.get('transport') not in {'http', 'websocket'}: + problems.append(f'{operation_id}/{entry.get("id")} transport {reference.get("transport")!r}') + if not (isinstance(reference.get('method'), str) and reference['method']): + problems.append(f'{operation_id}/{entry.get("id")} method {reference.get("method")!r}') + if not isinstance(reference.get('path'), str): + problems.append(f'{operation_id}/{entry.get("id")} path {reference.get("path")!r}') + if reference.get('exposure') not in {'enabled', 'disabled', 'unknown'}: + problems.append(f'{operation_id}/{entry.get("id")} exposure {reference.get("exposure")!r}') + # The top-level list is a different model (`CapabilityRoute`) with a required + # `exposure`; it is display data, but it still has to be structurally legal. + for declared_route in payload.get('routes', []): + if set(declared_route) != {'transport', 'method', 'path', 'exposure'}: + problems.append(f'top-level route keys {sorted(declared_route)}') + if declared_route.get('transport') not in {'http', 'websocket'}: + problems.append(f'top-level route transport {declared_route.get("transport")!r}') + if not (isinstance(declared_route.get('method'), str) and declared_route['method']): + problems.append(f'top-level route method {declared_route.get("method")!r}') + return problems + + +def test_fixtures_are_schema_valid(): + """Every fixture body, defensive ones included, is structurally legal.""" seen_body = 0 - for document, cases in raw_cases(): + for document, cases in raw_documents(TIERS + (DEFENSIVE_TIER,)): for case in cases: payload = case['payload'] if payload is None: assert case['expected']['endpoint'] == ABSENT, case['id'] continue seen_body += 1 - assert set(payload) == {'object', 'schema_version', 'observed_at', 'stages', 'models', 'routes'}, case['id'] - assert payload['object'] == 'omni.capabilities' and payload['schema_version'] == 1 - for stage in payload['stages']: - assert set(stage) == {'id', 'type', 'depends_on'}, case['id'] - assert isinstance(stage['id'], str) and stage['id'].isdigit(), stage - assert isinstance(stage['depends_on'], list), stage - assert stage['type'] in {'llm', 'diffusion', 'unknown'}, stage - for model in payload['models']: - assert {'id', 'stage_ids', 'operations'} <= set(model), case['id'] - assert isinstance(model['stage_ids'], list), model - assert all(isinstance(stage_id, str) and stage_id.isdigit() for stage_id in model['stage_ids']) - assert set(model['operations']) == operations, (case['id'], sorted(model['operations'])) - for operation_id, declared in model['operations'].items(): - assert declared['support'] in {'supported', 'unsupported', 'unknown'}, (case['id'], operation_id) - assert declared['exposure'] in {'enabled', 'disabled', 'unknown'}, (case['id'], operation_id) - assert isinstance(declared.get('profiles'), list), (case['id'], operation_id) - for entry in declared['profiles']: - assert entry['support'] in {'supported', 'unsupported', 'unknown'}, (case['id'], entry['id']) - assert isinstance(entry.get('routes'), list), entry - assert isinstance(entry.get('required_inputs'), list), entry - # Published `STANDARD_PROFILES` outputs (`serving_capabilities.py:43-45`). - if entry['id'] == 'image_response': - assert entry['outputs'] == ['image'], entry - if entry['id'] == 'audio_response': - assert entry['outputs'] == ['audio'], entry - for declared_route in payload['routes']: - assert set(declared_route) == {'transport', 'method', 'path', 'exposure'}, case['id'] - assert declared_route['transport'] in {'http', 'websocket'}, declared_route + assert schema_problems(payload) == [], (document['tier'], case['id']) assert seen_body >= 5 +def rendered_cell(inputs, outputs): + """The doc's spelling of one profile row: `("text","image")` → `("image",)`.""" + + def cell(items): + body = ','.join(f'"{item}"' for item in items) + return '(' + body + (',' if len(items) == 1 else '') + ')' + + return cell(inputs) + '→' + cell(outputs) + + +def documented_profiles(): + """`(profile, required_inputs → outputs cell)` pairs from the contract doc.""" + cells = [] + for line in CONTRACT_DOC.read_text(encoding='utf-8').splitlines(): + parts = [part.strip() for part in line.split('|')] + # A four-column row splits into six parts: two outer empties plus cells. + if len(parts) != 6 or '→' not in parts[3]: + continue + profile_id = parts[2].strip('`') + if profile_id == 'profile': # the header row + continue + cells.append((profile_id, parts[3].replace('`', '').replace(' ', ''))) + return cells + + +def test_documented_required_inputs_match_fixtures(): + """The literal upstream table, both doc tables and every sample fixture agree. + + `required_inputs` order is part of the declaration (`image_to_image` is + text-then-image), `video_to_video` needs no text, `lyrics_to_music` needs + `instructions`, and `image_response`/`audio_response` publish a non-text + output. `resolve()` reads none of it, so nothing else can catch rot here. + `malformed.json` is left out on purpose: it is defensive input, guarded by + the schema test, not part of the published sample this row set describes. + """ + assert len(UPSTREAM_PROFILES) == 12 + assert {profile_id for profile_id, _, _ in UPSTREAM_PROFILES} == set(PROFILE_OPERATIONS) + expected = {profile_id: (inputs, outputs) for profile_id, inputs, outputs in UPSTREAM_PROFILES} + + cells = documented_profiles() + assert len(cells) == 24, 'the Chinese and the English table must each list all twelve profiles' + assert {profile_id for profile_id, _ in cells} == set(expected) + for profile_id, cell in cells: + inputs, outputs = expected[profile_id] + assert cell == rendered_cell(inputs, outputs), profile_id + + seen = set() + for _, cases in raw_cases(): + for case in cases: + payload = case['payload'] + if not isinstance(payload, dict): + continue + for model_entry in payload['models']: + for declared in model_entry['operations'].values(): + for entry in declared['profiles']: + inputs, outputs = expected[entry['id']] + assert (tuple(entry['required_inputs']), tuple(entry['outputs'])) == (inputs, outputs), \ + (case['id'], entry['id']) + seen.add(entry['id']) + assert seen == set(expected), f'fixtures never exercise {sorted(set(expected) - seen)}' + + def test_fixtures_carry_no_credentials_or_provider_responses(): for path in sorted(FIXTURES.glob('*.json')): text = path.read_text(encoding='utf-8').lower() From d9506135c7343da1cf52ec219087a0eb46154887 Mon Sep 17 00:00:00 2001 From: Xingkai98 Date: Wed, 11 Nov 2026 01:14:06 +0800 Subject: [PATCH 5/5] fix(contract): fixture construction invariants and unrecognised-route exposure cases --- docs/architecture/capability-contract.md | 8 +- tests/fixtures/omni-capabilities/full.json | 22 +++- tests/fixtures/omni-capabilities/partial.json | 77 ++++++++---- tests/test_capability_contract.py | 113 +++++++++++++++--- 4 files changed, 178 insertions(+), 42 deletions(-) diff --git a/docs/architecture/capability-contract.md b/docs/architecture/capability-contract.md index e2c39181..a0a88864 100644 --- a/docs/architecture/capability-contract.md +++ b/docs/architecture/capability-contract.md @@ -163,10 +163,10 @@ capabilities_for('audio_generation', 'text_to_audio') # ()：上游有 profile `tests/fixtures/omni-capabilities/{full,partial,absent}.json` 是**求值样本**，按真实响应形状写：`stages[]` 是 `{id,type,depends_on}` 且 id 为数字字符串，`models[].operations` 六个齐全、每个含 `support`/`exposure`/`reason_code`/`profiles`/`execution`，每个 profile 都带 `support`、`reason_code` 与 `routes[]`（可以是 `[]`），`required_inputs`/`outputs` 逐条照上游 `STANDARD_PROFILES`（含顺序；`image_response`/`audio_response` 分别是 `["image"]`/`["audio"]`）。 - `full`：全支持全启用，12 个 profile 齐全；`image_response`/`audio_response`/`caption_to_audio`/`text_to_audio` 报 `no_local_capability`（`text_to_audio` **不**授 `audio.music`，`audio.music` 由 `lyrics_to_music` 供货）。 -- `partial`：`support=unsupported`、operation `exposure=disabled`（`endpoint_disabled`）、profile 自己的路由 `disabled`（`route_disabled`）、`no_local_capability`，以及整个 body 都是 `unknown` 枚举的形态。 +- `partial`：`support=unsupported`、operation `exposure=disabled`（`endpoint_disabled`）、`no_local_capability`，以及整个 body 都是 `unknown` 枚举的形态；`route_disabled` 只在 `malformed.json` 里出现——上游把「自有路由全 `disabled`」聚合成 operation `exposure=disabled`，样本档发不出这个组合。 - `absent`：没答（`absent`）、只答拓扑（`empty`）、六个 operation 都答了但 `profiles: []`（`present` 且全 `unknown`）。 -第四档 `malformed.json` **只用于防御输入**，不代表上游可能产出：里面的 body 是合法 pydantic（结构断言照样过），但上游投影发不出来（例如 profile 引用了别的 operation 的路由，或自有路由全 `disabled` 旁边挂着一条本表不认识的 `enabled` 路径）。代理、缓存、旧版本才可能把这种 body 交给我们，所以它只被防御性与结构断言读取，不进样本 oracle，也不参与「fixture = 真实样本」这句话。 +第四档 `malformed.json` **只用于防御输入**，不代表上游可能产出：里面的 body 是合法 pydantic（结构断言照样过），但上游投影发不出来（例如 profile 引用了别的 operation 的路由，或自有路由全 `disabled` 旁边挂着一条本表不认识的 `enabled` 路径）。代理、缓存、旧版本才可能把这种 body 交给我们，所以它只被防御性、结构性与 profile 行校验读取，不进样本 oracle，也不参与「fixture = 真实样本」这句话。 **上游不可能发出的形状**（`asr` operation 键、profile 挂在错 operation、缺必填 `support`、`profiles` 整键缺席）**不进 fixture**，只在对应测试里内联构造 —— 它们是防御性输入的用例，混进样本会让「fixture = 真实样本」这句话变成假话。`test_fixtures_are_schema_valid` 只断言上游 schema 的**结构不变量**（六键、`observed_at` 可解析且带时区、`reason_code` 在 operation 与 profile 上都在、`execution` 为 `{state,scope,observed_at}`、枚举值域、`profiles` 是列表、route 的 `transport`/`method`/`path` 齐全、stages 形状与数字字符串 id），不把上游 pydantic 源码抄一份进本仓。 @@ -296,9 +296,9 @@ capabilities_for('audio_generation', 'text_to_audio') # () — upstream publish ### Fixtures and checks -`tests/fixtures/omni-capabilities/{full,partial,absent}.json` are the **evaluated sample** and follow the real response shape — `stages[]` as `{id,type,depends_on}` with numeric-string ids, six complete `operations` per model each with `support`/`exposure`/`reason_code`/`profiles`/`execution`, every profile carrying `support`, `reason_code` and `routes` (possibly `[]`), and `required_inputs`/`outputs` transcribed from upstream's `STANDARD_PROFILES` including order (`image_response`/`audio_response` are `["image"]`/`["audio"]`). `full` is the all-yes deployment, `partial` mixes refusals, a policy-disabled operation, a profile whose own route is disabled and an all-`unknown` body, and `absent` covers no answer (`absent`), topology only (`empty`) and six operations that declare `profiles: []` (`present`, all `unknown`). In `full`, four profiles report `no_local_capability` (`image_response`, `audio_response`, `caption_to_audio`, `text_to_audio`) — `text_to_audio` grants no `audio.music`; `audio.music` is supplied by `lyrics_to_music`. +`tests/fixtures/omni-capabilities/{full,partial,absent}.json` are the **evaluated sample** and follow the real response shape — `stages[]` as `{id,type,depends_on}` with numeric-string ids, six complete `operations` per model each with `support`/`exposure`/`reason_code`/`profiles`/`execution`, every profile carrying `support`, `reason_code` and `routes` (possibly `[]`), and `required_inputs`/`outputs` transcribed from upstream's `STANDARD_PROFILES` including order (`image_response`/`audio_response` are `["image"]`/`["audio"]`). `full` is the all-yes deployment, `partial` mixes refusals, a policy-disabled operation and an all-`unknown` body (with the routes stating `unknown` too), and `absent` covers no answer (`absent`), topology only (`empty`) and six operations that declare `profiles: []` (`present`, all `unknown`). `route_disabled` appears only in `malformed.json`: upstream aggregates "all of the operation's routes `disabled`" into `exposure=disabled`, so a sample cannot carry that combination. In `full`, four profiles report `no_local_capability` (`image_response`, `audio_response`, `caption_to_audio`, `text_to_audio`) — `text_to_audio` grants no `audio.music`; `audio.music` is supplied by `lyrics_to_music`. -A fourth file, `malformed.json`, is **defensive input only and not a sample of what upstream can produce**: its bodies are valid pydantic (the structural checks pass) but upstream's projection cannot emit them — a profile pointing at another operation's route, or owned routes all `disabled` beside an `enabled` path this table does not know. Only a proxy, a cache or an older revision can hand us one, so it is read by the defensive and structural checks alone, never by the sample oracle. +A fourth file, `malformed.json`, is **defensive input only and not a sample of what upstream can produce**: its bodies are valid pydantic (the structural checks pass) but upstream's projection cannot emit them — a profile pointing at another operation's route, or owned routes all `disabled` beside an `enabled` path this table does not know. Only a proxy, a cache or an older revision can hand us one, so it is read by the defensive, structural and profile-row checks alone, never by the sample oracle. Shapes upstream can never emit — an `asr` operation key, a profile filed under the wrong operation, a missing required `support` — are built inline in the tests that defend against malformed input, never in the sample fixtures. `test_fixtures_are_schema_valid` asserts only the structural invariants of upstream's model (six keys, `observed_at` parseable and timezone-aware, `reason_code` wherever `support` is, `execution` as `{state,scope,observed_at}`, the enum domains, `profiles` as a list, routes carrying `transport`/`method`/`path`, stage shape with numeric-string ids), not a copy of upstream's pydantic source. diff --git a/tests/fixtures/omni-capabilities/full.json b/tests/fixtures/omni-capabilities/full.json index 828fab6e..c54fddf1 100644 --- a/tests/fixtures/omni-capabilities/full.json +++ b/tests/fixtures/omni-capabilities/full.json @@ -5,7 +5,7 @@ "cases": [ { "id": "every-operation-and-profile", - "note": "All six operations supported and enabled, all twelve profiles declared. Only `text_response` grants `text.chat`, and only `lyrics_to_music` grants `audio.music`: the image/audio chat profiles and `text_to_audio` are reported as `no_local_capability`.", + "note": "All six operations supported and enabled, all twelve profiles declared. Only `text_response` grants `text.chat`, and only `lyrics_to_music` grants `audio.music`: the image/audio chat profiles, `caption_to_audio` and `text_to_audio` are reported as `no_local_capability`.", "payload": { "object": "omni.capabilities", "schema_version": 1, @@ -228,6 +228,12 @@ "method": "POST", "path": "/v1/videos/sync", "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "disabled" } ], "reason_code": "model_declared", @@ -254,6 +260,12 @@ "method": "POST", "path": "/v1/videos/sync", "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "disabled" } ], "reason_code": "model_declared", @@ -314,7 +326,7 @@ }, { "id": "caption_to_audio", - "support": "unsupported", + "support": "supported", "required_inputs": [ "text" ], @@ -494,6 +506,12 @@ "profile": "image_response", "route": null, "reason": "no_local_capability" + }, + { + "operation": "speech", + "profile": "caption_to_audio", + "route": null, + "reason": "no_local_capability" } ] } diff --git a/tests/fixtures/omni-capabilities/partial.json b/tests/fixtures/omni-capabilities/partial.json index 578d2069..8aa91696 100644 --- a/tests/fixtures/omni-capabilities/partial.json +++ b/tests/fixtures/omni-capabilities/partial.json @@ -5,7 +5,7 @@ "cases": [ { "id": "mixed-operations", - "note": "`image_generation` is supported but disabled (`endpoint_disabled`), `image_edit` is supported but its only route is `disabled` (`route_disabled`), `lyrics_to_music` is unsupported while `text_to_audio` is supported (no local id, `no_local_capability`), and `caption_to_audio` has no local id either.", + "note": "`image_generation` is supported but disabled (`endpoint_disabled`), `image_edit` is served, `lyrics_to_music` is unsupported while `text_to_audio` is supported (no local id, `no_local_capability`), and `caption_to_audio` has no local id either.", "payload": { "object": "omni.capabilities", "schema_version": 1, @@ -122,7 +122,7 @@ "transport": "http", "method": "POST", "path": "/v1/images/generations", - "exposure": "enabled" + "exposure": "disabled" } ], "reason_code": "model_declared", @@ -155,7 +155,7 @@ "transport": "http", "method": "POST", "path": "/v1/images/edits", - "exposure": "disabled" + "exposure": "enabled" } ], "reason_code": "model_declared", @@ -221,6 +221,18 @@ "method": "POST", "path": "/v1/videos", "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "disabled" } ], "reason_code": "model_declared", @@ -241,6 +253,18 @@ "method": "POST", "path": "/v1/videos", "exposure": "enabled" + }, + { + "transport": "http", + "method": "POST", + "path": "/v1/videos/sync", + "exposure": "enabled" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "disabled" } ], "reason_code": "checkpoint_requires_expand_timesteps", @@ -449,7 +473,7 @@ "capabilities": { "text.chat": "available", "image.generate": "missing", - "image.edit": "missing", + "image.edit": "available", "video.generate": "available", "audio.speech": "available", "audio.music": "missing", @@ -464,7 +488,6 @@ }, "reasons": { "image.generate": "endpoint_disabled", - "image.edit": "route_disabled", "audio.music": "unsupported" }, "unresolved": [ @@ -530,7 +553,7 @@ "transport": "http", "method": "POST", "path": "/v1/chat/completions", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -550,7 +573,7 @@ "transport": "http", "method": "POST", "path": "/v1/chat/completions", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -570,7 +593,7 @@ "transport": "http", "method": "POST", "path": "/v1/chat/completions", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -602,7 +625,7 @@ "transport": "http", "method": "POST", "path": "/v1/images/generations", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -635,7 +658,7 @@ "transport": "http", "method": "POST", "path": "/v1/images/edits", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -667,19 +690,19 @@ "transport": "http", "method": "POST", "path": "/v1/videos", - "exposure": "enabled" + "exposure": "unknown" }, { "transport": "http", "method": "POST", "path": "/v1/videos/sync", - "exposure": "enabled" + "exposure": "unknown" }, { "transport": "websocket", "method": "WEBSOCKET", "path": "/v1/realtime/video", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -700,13 +723,19 @@ "transport": "http", "method": "POST", "path": "/v1/videos", - "exposure": "enabled" + "exposure": "unknown" }, { "transport": "http", "method": "POST", "path": "/v1/videos/sync", - "exposure": "enabled" + "exposure": "unknown" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -726,13 +755,19 @@ "transport": "http", "method": "POST", "path": "/v1/videos", - "exposure": "enabled" + "exposure": "unknown" }, { "transport": "http", "method": "POST", "path": "/v1/videos/sync", - "exposure": "enabled" + "exposure": "unknown" + }, + { + "transport": "websocket", + "method": "WEBSOCKET", + "path": "/v1/realtime/video", + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -764,7 +799,7 @@ "transport": "http", "method": "POST", "path": "/v1/audio/speech", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -785,7 +820,7 @@ "transport": "http", "method": "POST", "path": "/v1/audio/speech", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -805,7 +840,7 @@ "transport": "http", "method": "POST", "path": "/v1/audio/speech", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", @@ -837,7 +872,7 @@ "transport": "http", "method": "POST", "path": "/v1/audio/generate", - "exposure": "enabled" + "exposure": "unknown" } ], "reason_code": "request_contract_unverified", diff --git a/tests/test_capability_contract.py b/tests/test_capability_contract.py index 98f73ab4..644f4abd 100644 --- a/tests/test_capability_contract.py +++ b/tests/test_capability_contract.py @@ -12,11 +12,11 @@ A fourth file, `malformed.json`, holds bodies only a rewrite can produce — a profile pointing at another operation's route, a disabled route beside a path -this table does not know. It is read by the defensive and structural checks -alone, never by the sample oracle: it is input to defend against, not evidence -of what upstream publishes. The shapes upstream can never emit — an `asr` -operation key, a profile filed under the wrong operation, a missing required -`support` — stay inline in the tests that defend against them. +this table does not know. It is read by the defensive, structural and +profile-row checks, never by the sample oracle: it is input to defend against, +not evidence of what upstream publishes. The shapes upstream can never emit — +an `asr` operation key, a profile filed under the wrong operation, a missing +required `support` — stay inline in the tests that defend against them. The last group guards the failure this table exists to prevent: a route (or a stage) that is read as if it named a capability. `/v1/audio/speech` carries @@ -549,6 +549,36 @@ def test_a_disabled_profile_with_an_unknown_enabled_route_is_unknown(): assert still_missing['reasons'] == {'image.generate': 'route_disabled'} +@pytest.mark.parametrize('exposure', ['enabled', 'unknown', None, 'disabled']) +def test_unrecognised_route_outcome_depends_on_its_exposure(exposure): + """A path this table cannot place still states whether it is switched on. + + Upstream sets a route's `exposure` to `unknown` when the route is registered + but its reachability is not settled (`serving_capabilities.py:267,304`), so + "every known route of the profile is `disabled`, plus an unrecognised route + whose `exposure` is `unknown`" is a body upstream really publishes, and so + is the same body with the field absent (upstream's default). Reading either + as a refusal would be the false `missing` this table exists to prevent. + Only an unrecognised route that itself says `disabled` adds no channel, and + then the known-but-off routes remain a positive no. + """ + declared = route('/v1/images/generations/stream') + if exposure is None: + del declared['exposure'] + else: + declared['exposure'] = exposure + result = resolve(body({'image_generation': operation(profiles=[ + profile('text_to_image', routes=[route('/v1/images/generations', exposure='disabled'), declared])])})) + if exposure == 'disabled': + assert result['capabilities']['image.generate'] == 'missing' + assert result['reasons'] == {'image.generate': 'route_disabled'} + else: + assert result['capabilities']['image.generate'] == 'unknown' + assert result['reasons'] == {} + assert result['unresolved'] == [{'operation': 'image_generation', 'profile': 'text_to_image', + 'route': '/v1/images/generations/stream', 'reason': 'unknown_route'}] + + def test_a_profile_whose_routes_are_all_unknown_cannot_be_confirmed(): result = resolve(body({'image_generation': operation(profiles=[ profile('text_to_image', routes=[route('/v1/images/generations', exposure='unknown')])])})) @@ -636,8 +666,7 @@ def test_an_image_only_chat_deployment_does_not_read_as_text_chat(): """`image_response` output is an image; `text.chat` is served by a text-only adapter.""" result = resolve(body({'chat': operation(profiles=[ profile('image_response', outputs=('image',), paths=table_paths('chat'))])})) - assert result['capabilities']['text.chat'] != AVAILABLE - assert result['capabilities']['text.chat'] == UNKNOWN + assert result['capabilities']['text.chat'] == 'unknown' assert result['unresolved'] == [{'operation': 'chat', 'profile': 'image_response', 'route': None, 'reason': NO_LOCAL_CAPABILITY}] @@ -669,11 +698,10 @@ def test_text_to_audio_alone_does_not_make_audio_music_available(): assert capabilities_for('audio_generation', 'text_to_audio') == () result = resolve(body({'audio_generation': operation(profiles=[ profile('text_to_audio', paths=table_paths('audio_generation'), outputs=('audio',))])})) - assert result['capabilities']['audio.music'] != AVAILABLE - assert result['capabilities']['audio.music'] == UNKNOWN + assert result['capabilities']['audio.music'] == 'unknown' assert {'operation': 'audio_generation', 'profile': 'text_to_audio', 'route': None, - 'reason': NO_LOCAL_CAPABILITY} in result['unresolved'] - assert result['capabilities']['audio.speech'] == UNKNOWN + 'reason': 'no_local_capability'} in result['unresolved'] + assert result['capabilities']['audio.speech'] == 'unknown' # The music path is still the speech profile, on the speech route. music = resolve(body({'speech': operation(profiles=[ profile('lyrics_to_music', paths=table_paths('speech'), inputs=('text', 'instructions'), @@ -929,6 +957,59 @@ def test_fixtures_are_schema_valid(): assert seen_body >= 5 +def test_fixtures_obey_upstream_construction_invariants(): + """The three sample tiers only carry shapes upstream's projection can emit. + + Upstream derives an operation's `exposure` from that operation's own routes + (`entrypoints/openai/serving_capabilities.py:384-387`: any `enabled` → + `enabled`, else any `unknown` → `unknown`, else `disabled`) and hands every + profile of the operation the *full* route list of its operation (`:320-329`, + with `/v1/realtime/video` forced `disabled` for the profiles that take video + or audio input). "Operation says off, route says on" and "profile references + half of the video routes" are therefore bodies upstream cannot publish, and + a fixture claiming to be a real response shape cannot contain them. + + `malformed.json` is excluded on purpose: it is the one file that exists to + hold shapes a proxy, cache or older revision can hand us — exactly the + bodies this test forbids in a sample. The defensive cases pin their own + expectations instead. + """ + checked = 0 + for document, cases in raw_cases(): + for case in cases: + payload = case['payload'] + if not isinstance(payload, dict): + continue + for model_entry in payload['models']: + for operation_id, declared in model_entry['operations'].items(): + if operation_id not in OPERATION_ROUTES: + continue + legal = set(OPERATION_ROUTES[operation_id]) + exposures = [] + for entry in declared['profiles']: + keys = {(reference['transport'], reference['method'], reference['path']) + for reference in entry['routes']} + assert keys, (case['id'], operation_id, entry['id'], 'no route declared') + assert keys <= legal, (case['id'], operation_id, entry['id'], sorted(keys - legal)) + exposures.extend(reference['exposure'] for reference in entry['routes']) + if not exposures: + # No profile, hence no route: nothing to aggregate. + continue + aggregate = ('enabled' if 'enabled' in exposures else + 'unknown' if 'unknown' in exposures else 'disabled') + assert declared['exposure'] == aggregate, (case['id'], operation_id, + declared['exposure'], aggregate) + if operation_id == 'video_generation' and declared['exposure'] != 'disabled': + # Three routes are owned here; a profile that references + # a subset would hide a channel upstream does publish. + for entry in declared['profiles']: + keys = {(reference['transport'], reference['method'], reference['path']) + for reference in entry['routes']} + assert keys == legal, (case['id'], entry['id'], sorted(legal - keys)) + checked += 1 + assert checked >= 5 + + def rendered_cell(inputs, outputs): """The doc's spelling of one profile row: `("text","image")` → `("image",)`.""" @@ -955,14 +1036,16 @@ def documented_profiles(): def test_documented_required_inputs_match_fixtures(): - """The literal upstream table, both doc tables and every sample fixture agree. + """The literal upstream table, both doc tables and every fixture agree. `required_inputs` order is part of the declaration (`image_to_image` is text-then-image), `video_to_video` needs no text, `lyrics_to_music` needs `instructions`, and `image_response`/`audio_response` publish a non-text output. `resolve()` reads none of it, so nothing else can catch rot here. - `malformed.json` is left out on purpose: it is defensive input, guarded by - the schema test, not part of the published sample this row set describes. + `malformed.json` is included: its bodies are structurally legal pydantic, so + the profiles they carry are still upstream's `STANDARD_PROFILES` rows, and + leaving them out only let that file rot silently. What is defensive about it + is the routes and the exposures, not the profile rows. """ assert len(UPSTREAM_PROFILES) == 12 assert {profile_id for profile_id, _, _ in UPSTREAM_PROFILES} == set(PROFILE_OPERATIONS) @@ -976,7 +1059,7 @@ def test_documented_required_inputs_match_fixtures(): assert cell == rendered_cell(inputs, outputs), profile_id seen = set() - for _, cases in raw_cases(): + for _, cases in raw_documents(TIERS + (DEFENSIVE_TIER,)): for case in cases: payload = case['payload'] if not isinstance(payload, dict):