diff --git a/apps/api/src/opendirector_api/agent.py b/apps/api/src/opendirector_api/agent.py index e4e0ec68..2a1d3385 100644 --- a/apps/api/src/opendirector_api/agent.py +++ b/apps/api/src/opendirector_api/agent.py @@ -17,6 +17,7 @@ from opendirector.agent.runtime import AgentRuntimes, runtime_available, configured_runtimes, runtime_spec, runtime_catalog, runtime_status from opendirector.agent.skills import skills from opendirector.domain.projects import ProjectError +from opendirector.application.model_availability import requests_for from opendirector.domain.rule_facts import MAX_REFERENCES, PROMPT_MAX_LENGTH from opendirector.infrastructure.creation_projects import durable_json from opendirector.infrastructure.local_projects import child, read_json @@ -71,7 +72,8 @@ def path(self, project_id: UUID, session: str | None = None) -> Path: return directory def update(self, project_id: UUID, session: str | None = None, *, draft: Draft | None = None, - begin: dict | None = None, finish: dict | None = None, conversation_id: UUID | None = None) -> dict: + begin: dict | None = None, finish: dict | None = None, conversation_id: UUID | None = None, + before_begin=None) -> dict: with self.projects.local.transaction() as db: self.projects._recover() directory, _, _ = self.projects._load(db, project_id) @@ -92,6 +94,8 @@ def update(self, project_id: UUID, session: str | None = None, *, draft: Draft | if begin is not None: if any(message['id'] == begin['id'] for message in value['messages']): raise ProjectError('这条消息已接受，请读取对话，不要重复发送。', 409) + if before_begin is not None: + before_begin() value['messages'].append(begin) value['messages'].append({'id': str(uuid5(UUID(begin['id']), 'assistant')), 'role': 'assistant', 'text': '', 'status': 'streaming', @@ -304,6 +308,11 @@ async def turn(project_id: UUID, value: Turn, x_opendirector_session: UUID = Hea if value.runtime == 'jiuwenswarm' and value.thinking != 'auto': raise ProjectError('JiuwenSwarm 当前使用原生默认推理强度。', 409) config = await asyncio.to_thread(configs.resolve, str(value.modelId), 'agent') if spec.modelSource == 'settings' else None + if config is not None: + prepared = await asyncio.to_thread(generation.availability.prepare, config) + current = await asyncio.to_thread(configs.resolve, str(value.modelId), 'agent') + generation.availability.require(current, requests_for(current, inputs={'text'})[0], prepared) + config = current available = await skills(projects.local.repository, directory) if spec.supportsSkills else [] if set(value.skills) - {item['name'] for item in available}: raise ProjectError('技能已失效，请重新选择。', 409) @@ -318,6 +327,10 @@ async def turn(project_id: UUID, value: Turn, x_opendirector_session: UUID = Hea preferences = {kind: {'modelConfigId': model_id, 'modelName': model_names.get(model_id)} for kind, model_id in model_selections.items() if model_id} events = RunEvents() + def validate_connection(): + if config is not None: + current = configs.resolve(str(value.modelId), 'agent') + generation.availability.require(current, requests_for(current, inputs={'text'})[0], prepared) async def produce() -> None: text, status, error, accepted = '', 'complete', None, False activities = [] @@ -330,7 +343,8 @@ async def produce() -> None: 'skills': value.skills, 'runtime': value.runtime, 'runtimeModel': value.runtimeModel, 'modelSelections': dict(value.modelSelections), 'model': model, 'status': 'complete'} - saved = await asyncio.to_thread(conversations.update, project_id, session, begin=user, conversation_id=value.conversationId) + saved = await asyncio.to_thread(conversations.update, project_id, session, begin=user, + conversation_id=value.conversationId, before_begin=validate_connection) accepted = True events.put({'type': 'start', 'conversation': saved}) context = [] diff --git a/apps/api/src/opendirector_api/localization.py b/apps/api/src/opendirector_api/localization.py index c5a4cbf6..91090b9c 100644 --- a/apps/api/src/opendirector_api/localization.py +++ b/apps/api/src/opendirector_api/localization.py @@ -10,6 +10,8 @@ from opendirector.domain.timeline_errors import timeline_error_message from opendirector.domain.creation.commands import CommandReceipt +from opendirector.domain.capabilities import Capability +from opendirector.domain.capability_messages import english_capability_message, missing_tool_model_message ENGLISH_BY_CODE = { 'execution_file_link': 'Execution files must not have hard links.', @@ -21,6 +23,14 @@ 'kling_aspect': 'Kling keyframe generation follows the image framing. Choose Auto for frames, or a supported ratio for text and image references.', 'kling_output': 'Choose an output resolution and integer duration supported by the selected Kling model.', 'kling_cancel': 'Remote Kling cancellation is not connected. The task may continue running and incur charges.', + 'capability_unavailable': 'This request is not supported.', + 'capability_paused': 'Service paused. Resume it and try again.', + 'image_input_count': 'Too many input images.', + 'image_input_bytes': 'The input images exceed the size limit.', + 'image_input_mask': 'Check the image selection.', + 'tool_source_count': 'Check the number of source images.', + 'tool_source_type': 'Source assets must be images.', + 'capability_connection_changed': 'The model connection changed during the capability check. Submit again.', 'tool_concept_translation': 'Use a short English subject description, or add a chat model in Models to translate it automatically.', 'tool_no_video_match': 'The requested subject was not found in the video. Try a different description.', 'voice_upload_type': 'Upload the voice recording using multipart/form-data.', @@ -106,12 +116,19 @@ def localized_error_for_language( chinese: str, status: int, code: str = "operation_rejected", + required_capability: Capability | None = None, ) -> str: specific = timeline_error_message(code, language) if specific is not None: return specific if not wants_english_language(language): return chinese + if code == 'tool_model_missing': + return missing_tool_model_message(required_capability, english=True) + if code.startswith(('capability_', 'image_input_', 'tool_source_')): + translated = english_capability_message(chinese) + if translated: + return translated # A generic operation code must not hide a more specific HTTP status. generic = code in {"operation_rejected", "invalid_input"} if generic and status in {413, 502, 504}: @@ -134,12 +151,14 @@ def localized_error( chinese: str, status: int, code: str = "operation_rejected", + required_capability: Capability | None = None, ) -> str: return localized_error_for_language( request.headers.get("accept-language"), chinese=chinese, status=status, code=code, + required_capability=required_capability, ) diff --git a/apps/api/src/opendirector_api/main.py b/apps/api/src/opendirector_api/main.py index 8bbc9af8..e652e8e3 100644 --- a/apps/api/src/opendirector_api/main.py +++ b/apps/api/src/opendirector_api/main.py @@ -12,10 +12,12 @@ from opendirector.installation import data_root, web_root from opendirector.wiring import configure_runtime from opendirector.domain.projects import ProjectError, identifier +from opendirector.domain.capability_messages import MissingToolModelError from opendirector.domain.creation.validation import ContractError from opendirector.domain.settings import GeneralSettings, GeneralSettingsPatch from opendirector.application.creation_generation import CreationGeneration from opendirector.application.model_catalog import ModelCatalog +from opendirector.application.model_availability import ModelAvailability from opendirector.application.preferences import Preferences from opendirector.application.tool_tasks import ToolTasks from opendirector.application.creation_tool_assets import CreationToolAssets @@ -30,7 +32,8 @@ from opendirector_api.localization import localized_error from opendirector_api.models import (ModelConfigInput, ModelConfigPublic, AgentModelTest, ModelCapabilityRequest, ModelDiscoveryRequest, ModelBatchInput, - ToolTaskCreate, ToolPlacement, ImageSelectionCreate) + ToolTaskCreate, ToolPlacement, ImageSelectionCreate, AvailabilityRefresh, AvailabilityCatalog, + ModelAvailability as ModelAvailabilityResponse) def video_capabilities(value): @@ -41,7 +44,7 @@ def model_options(value: ModelBatchInput): return {key: item.model_dump(exclude_unset=True) for key, item in value.modelOptions.items()} -def create_app(store: LocalProjects | None = None, chat_provider=None, generation_provider=None) -> FastAPI: +def create_app(store: LocalProjects | None = None, chat_provider=None, generation_provider=None, capability_probe=None) -> FastAPI: configure_runtime() repository = data_root() if store is None: @@ -49,9 +52,10 @@ def create_app(store: LocalProjects | None = None, chat_provider=None, generatio model_configs = ModelConfigs(store.repository) preferences = Preferences(store.repository) creation_projects = CreationProjects(store) - creation_generation = CreationGeneration(creation_projects, model_configs, generation_provider) - creation_tools = ToolTasks(store, model_configs, CreationToolAssets(creation_projects), projects=creation_projects) - catalog = ModelCatalog(model_configs, chat_provider or ChatCompletions()) + availability = ModelAvailability(model_configs, capability_probe) + creation_generation = CreationGeneration(creation_projects, model_configs, generation_provider, availability=availability) + creation_tools = ToolTasks(store, model_configs, CreationToolAssets(creation_projects), projects=creation_projects, availability=availability) + catalog = ModelCatalog(model_configs, chat_provider or ChatCompletions(), availability) @asynccontextmanager async def lifespan(app): @@ -64,6 +68,7 @@ async def lifespan(app): await creation_tools.close() finally: await creation_generation.close() + availability.probe.close() app = FastAPI(title="OpenDirector Local API", version="0.1.2", lifespan=lifespan) app.include_router(creation_routes(creation_projects, creation_generation)) @@ -87,6 +92,7 @@ async def invalid_request(request, error): @app.exception_handler(ProjectError) async def project_error(request, error): + required_capability = error.capability if isinstance(error, MissingToolModelError) else None return JSONResponse( { "detail": localized_error( @@ -94,8 +100,10 @@ async def project_error(request, error): chinese=str(error), status=error.status, code=error.code, + required_capability=required_capability, ), "code": error.code, + **({'requiredCapability': required_capability} if required_capability else {}), }, status_code=error.status, ) @@ -192,6 +200,21 @@ def health(): def list_model_configs(): return catalog.list() + @app.get('/api/model-configs/availability', response_model=AvailabilityCatalog) + def model_availability(): + return availability.list() + + @app.post('/api/model-configs/availability', response_model=AvailabilityCatalog) + def refresh_model_availability(value: AvailabilityRefresh): + return availability.refresh([str(item) for item in value.configIds] if value.configIds is not None else None, manual=value.manual) + + @app.post('/api/model-configs/availability/check', response_model=ModelAvailabilityResponse) + async def check_model_availability(value: ModelDiscoveryRequest): + return await catalog.check_availability(value.kind, value.url, value.modelId, value.apiKey.get_secret_value(), + str(value.configId) if value.configId else None, connection_type=value.connectionType, protocol=value.protocol, + auth_mode=value.authMode, proxy_mode=value.proxyMode, video_adapter=value.videoAdapter, + capability_auto_check=value.capabilityAutoCheck) + @app.get('/api/settings', response_model=GeneralSettings) def get_settings(): return preferences.get() @@ -211,7 +234,7 @@ def save_model_batch(value: ModelBatchInput): protocol=value.protocol, auth_mode=value.authMode, proxy_mode=value.proxyMode, video_adapter=value.videoAdapter, video_capabilities=video_capabilities(value), input_modalities=value.inputModalities, model_options=model_options(value), - capabilities_supplied='videoCapabilities' in value.model_fields_set) + capabilities_supplied='videoCapabilities' in value.model_fields_set, capability_auto_check=value.capabilityAutoCheck) @app.post('/api/model-configs/discover') async def discover_models(value: ModelDiscoveryRequest): @@ -224,14 +247,16 @@ async def discover_models(value: ModelDiscoveryRequest): async def test_agent_model(value: AgentModelTest): elapsed = await catalog.test_agent(value.url, value.modelId, value.apiKey.get_secret_value(), str(value.configId) if value.configId else None, connection_type=value.connectionType, - protocol=value.protocol, auth_mode=value.authMode, proxy_mode=value.proxyMode) + protocol=value.protocol, auth_mode=value.authMode, proxy_mode=value.proxyMode, + capability_auto_check=value.capabilityAutoCheck) return {'elapsedMs': elapsed} @app.post("/api/model-configs", response_model=ModelConfigPublic) def create_model_config(value: ModelConfigInput): return catalog.save(value.kind, value.url, value.modelId, value.apiKey.get_secret_value(), connection_type=value.connectionType, protocol=value.protocol, auth_mode=value.authMode, proxy_mode=value.proxyMode, video_adapter=value.videoAdapter, video_capabilities=video_capabilities(value), - capabilities_supplied='videoCapabilities' in value.model_fields_set, input_modalities=value.inputModalities) + capabilities_supplied='videoCapabilities' in value.model_fields_set, input_modalities=value.inputModalities, + capability_auto_check=value.capabilityAutoCheck) @app.post('/api/model-configs/capabilities') async def model_capabilities(value: ModelCapabilityRequest): @@ -245,7 +270,8 @@ async def model_capabilities(value: ModelCapabilityRequest): def update_model_config(config_id: UUID, value: ModelConfigInput): return catalog.save(value.kind, value.url, value.modelId, value.apiKey.get_secret_value(), str(config_id), connection_type=value.connectionType, protocol=value.protocol, auth_mode=value.authMode, proxy_mode=value.proxyMode, video_adapter=value.videoAdapter, video_capabilities=video_capabilities(value), - capabilities_supplied='videoCapabilities' in value.model_fields_set, input_modalities=value.inputModalities) + capabilities_supplied='videoCapabilities' in value.model_fields_set, input_modalities=value.inputModalities, + capability_auto_check=value.capabilityAutoCheck) @app.delete("/api/model-configs/{config_id}", status_code=204) def delete_model_config(config_id: UUID): diff --git a/apps/api/src/opendirector_api/models.py b/apps/api/src/opendirector_api/models.py index b45d4c82..3007d213 100644 --- a/apps/api/src/opendirector_api/models.py +++ b/apps/api/src/opendirector_api/models.py @@ -5,6 +5,7 @@ from opendirector.domain.creation.editors import ToolDraftBinding from opendirector.domain.capabilities import AgentInputModality from opendirector.domain.video_capabilities import VideoAdapter, VideoCapabilities +from opendirector.domain.creation.models import ModelAvailability, AvailabilityCatalog class Model(BaseModel): @@ -27,6 +28,12 @@ class ModelConnection(Model): protocol: Literal['openai', 'volcengine', 'kling_video', 'minimax_video_v2', 'vllm_omni', 'vllm_omni_segmentation', 'vllm_omni_music', 'vllm_omni_depth', 'vllm_omni_score'] | None = None authMode: Literal['none', 'bearer'] | None = None proxyMode: Literal['system', 'direct'] | None = None + capabilityAutoCheck: bool | None = Field(default=None, strict=True, exclude_if=lambda value: value is None) + + +class AvailabilityRefresh(Model): + configIds: list[UUID] | None = Field(default=None, min_length=1, max_length=200) + manual: bool = False class ModelConfigInput(ModelConnection): @@ -68,6 +75,7 @@ class ModelCapabilityRequest(ModelConfigInput): class ModelConfigPublic(ModelConnection): + availability: ModelAvailability | None = Field(default=None, exclude_if=lambda value: value is None) providerId: UUID | None = Field(default=None, exclude_if=lambda value: value is None) videoAdapter: VideoAdapter = 'auto' videoCapabilities: VideoCapabilities | None = None diff --git a/docs/README.md b/docs/README.md index 536a82f3..69cee9b8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,6 +22,7 @@ This directory is the entry point for the repository's shared documentation. The | [Business CLI](guides/cli.md) | Querying, editing, and generating through a terminal or Agent Shell | Commands, authorization, or receipt semantics change | | [DeepSeek Harness plugin](../plugins/deepseek-harness/README.md) | Installing the DSH bundle and editing an OpenDirector project from DSH | Plugin setup, tools, or creation workflows change | | [Video model capabilities](guides/video-model-capabilities.md) | Declaring output specifications supported by a service | Configuration fields or capability integration changes | +| [Model availability / 模型可用状态](guides/model-availability.md) | Optional runtime declarations, automatic checks, shared admission and API/CLI/Agent queries | Capability protocol, refresh, or acceptance rules change | | [OpenAI Images](guides/openai-images.md) | Wiring gpt-image as an optional image service, request shape and failures | Integration fields, capability scope, or the OpenAI interface change | | [Kling video](guides/kling-video.md) ([中文](guides/kling-video.zh-CN.md)) | Optional Kling 4.0 and Flash setup, model-specific inputs, and recovery limits | Kling inputs, model declarations, or recovery behavior change | | [Agent runtime](architecture/agent-runtime.md) | Agent runtime architecture | Runtime architecture changes | diff --git a/docs/development/validation.md b/docs/development/validation.md index 71e2d2b5..23419b0e 100644 --- a/docs/development/validation.md +++ b/docs/development/validation.md @@ -35,6 +35,8 @@ Consult the [development lessons](lessons.md) when selecting relevant scenarios, ## 按风险选择验证 +能力与提示修改还需检查实际页面和发送内容：共同失败原因只呈现一次，提示说明用户能够采取的操作，调研记录和测试配置不进入产品文案。分别查看中英文页面的正常、未知、过期、暂停和失败状态。按最终请求统计素材，确认超量输入零新增任务、零生成请求；默认参数与服务允许范围、工具来源与模型输入分别核对。 + | 修改范围 | 应核对的重点 | | --- | --- | | 业务规则、参考及上传 | 实际类型、字节预算、来源身份、跟随与固定、拒绝后状态不变 | diff --git a/docs/guides/images/model-availability-en-US.png b/docs/guides/images/model-availability-en-US.png new file mode 100644 index 00000000..2037c809 Binary files /dev/null and b/docs/guides/images/model-availability-en-US.png differ diff --git a/docs/guides/images/model-availability-zh-CN.png b/docs/guides/images/model-availability-zh-CN.png new file mode 100644 index 00000000..5405baaf Binary files /dev/null and b/docs/guides/images/model-availability-zh-CN.png differ diff --git a/docs/guides/model-availability.md b/docs/guides/model-availability.md new file mode 100644 index 00000000..16506a1f --- /dev/null +++ b/docs/guides/model-availability.md @@ -0,0 +1,122 @@ +# Model availability / 模型可用状态 + +## 中文 + +能力卡示例，展示操作状态和服务声明的输入限制： + +![中文服务能力卡](images/model-availability-zh-CN.png) + +模型接入可以读取服务的可选能力声明。自托管连接默认自动检查，其他连接默认关闭；服务级 `capabilityAutoCheck` 可以覆盖默认值。省略该字段会保留已有设置，旧配置无需转换。共享同一 `providerId` 的模型使用相同设置和查询缓存。独立服务及旧的独立配置保留自己的归属，相同 URL 和密钥不会合并它们。自动检查只查询能力接口，不生成内容、不唤醒模型。 + +“检查服务能力”可以检查未保存的表单。草稿检查不会保存配置，也不会更新已保存模型的状态。显式刷新已保存模型可以检查关闭自动检查的服务。 + +在“设置 → 模型接入”选择模型类别，展开服务卡片的“服务能力”，即可查看每种操作的状态和检查时间。编辑服务时，可在高级连接设置中调整“自动检查服务能力”。输入过程中不会查询；保存后在后台检查。生成按钮和助手会说明明确不可用的原因，并保留当前输入和选择。暂停观察到期后会解除该观察产生的禁用，提交时仍执行后端检查。 + +状态有三种：`available` 表示该请求具有匹配声明；`unavailable` 表示服务明确不支持或暂停当前执行入口；`unknown` 表示没有足够信息。旧服务没有接口、超时、断网、鉴权失败和无法理解的版本都保留手动配置与现有请求路径。错误原因分别显示，响应正文与凭据不进入状态数据。 + +声明按精确 Model ID、输入模式、输出类型和实际 HTTP 路由匹配。一个模型支持聊天协议，不能证明它会返回助手需要的文字。音乐适配器通过 `/audio/speech` 发送歌词 `input`、风格 `instructions` 和音频帧数 `max_new_tokens`；其 `lyrics_to_music` 模式独立于普通 TTS 的 `text_to_speech`。`/audio/generate` 的通用音频生成声明不会启用现有音乐适配器。 + +提交时还会核对该模式声明的必要输入。例如图生视频声明同时需要图片和音频，只提供图片会在接受任务前被拒绝；补充实际传输的音频后可以提交。仅作为上下文的素材不计入传输输入。目录没有具体任务输入，不会因此禁用整个模型；无法理解的必要输入名称保留 `unknown`。 + +声明在内存中保留 30 秒；可信部署暂停状态最多保留 5 秒。内部使用本地单调时钟计算有效期，系统时钟调整和服务时钟偏差不改变到期规则；对外时间用于界面展示。单次查询或一次多服务刷新共用 2 秒总等待预算，包含等待已有共享请求的时间。独立服务通过同一个最多 4 个线程的查询池并发读取，排队数量和缓存均有容量限制。失败可以短时间后重试，进程重启后重新获得观察。目录读取不会访问模型服务，到期会改变目录版本。 + +生成、工具和使用设置模型的助手共享接受规则。服务查询位于项目存储事务之外，接受新任务时再次核对连接身份。明确拒绝不会创建生成任务，也不会调用生成接口。工具的本地操作和可选翻译保留原有行为。已接受任务的快照、恢复、下载、取消和同一命令重放继续使用原有规则；最新能力状态不会重新拒绝它们。能力状态不会清除音色或替换用户选择。 + +## English + +Example capability card showing operation states and declared input limits: + +![English service capability card](images/model-availability-en-US.png) + +Model connections can read an optional service capability declaration. Self-hosted connections check automatically by default; other connections require explicit enablement or a manual check. The optional service-level `capabilityAutoCheck` preference and query cache are shared by models with the same `providerId`. Independent providers and legacy standalone configuration IDs stay isolated even at the same URL with the same credential. Checks query declarations without inference or wake-up requests. + +The unsaved form check uses an isolated draft identity. It saves neither configuration nor observations for saved connections. Explicit refresh can check selected saved connections even when automatic checks are disabled. + +In Settings → Model connections, select a category and expand Service capabilities on a connection card to see operation states and check times. Advanced connection settings contain Automatically check service capabilities. Typing does not query the service; saving starts a background check. Generation and assistant controls explain explicit unavailability while retaining inputs and selections. Pause observations expire locally; submission still runs the backend check. + +`available`, `unavailable`, and `unknown` describe evidence for a specific request. Matching requires the exact model, operation profile, consumed output and actual HTTP route. Unknown or failed discovery preserves manual configuration and the existing request path. Authentication, connection, timeout, unsupported-version and invalid-response reasons are distinct; credentials and response bodies are never included in observations. + +Text responses and image responses on Chat Completions are distinct. Music uses the `lyrics_to_music` profile on `/audio/speech`, with lyrics, style and an audio-frame budget; ordinary TTS uses `text_to_speech`. Generic `/audio/generate` declarations do not create a consumer for that API in OpenDirector. + +Admission also checks a matched profile's required inputs against the actual adapter request. An image-to-video profile requiring audio rejects a submission without transmitted audio, while the catalog remains selectable. Context-only references are not transmitted inputs. Unrecognized required-input names remain unknown. + +Declarations expire after 30 seconds. A proven deployment-wide pause expires after at most 5 seconds. Internal expiration uses a local monotonic clock; service timestamps identify observations, and wall-clock timestamps are presentation data. Each query or multi-service refresh shares one two-second wait budget, including existing shared requests. Independent services use the same four-worker pool with bounded pending requests and cache. Concurrent reads are coalesced and failures permit short retries. Catalog reads never contact providers; expiration changes their revision. + +Generation, tools and settings-backed assistant turns share acceptance rules. Network reads occur outside project transactions; connection identity is checked again when accepting work. Explicit rejection creates no job and submits no inference. Local tools and optional translation retain their behavior. Accepted snapshots, recovery, downloads, cancellation and idempotent replay do not undergo new capability admission. Observations never clear voice selections or silently choose another model. + +## HTTP, CLI and Agent interfaces + +| Interface | Behavior | +| --- | --- | +| `GET /api/model-configs/availability` | Current observations without network access | +| `POST /api/model-configs/availability` | Refresh eligible services; accepts optional `configIds` and `manual` | +| `POST /api/model-configs/availability/check` | Check an unsaved model form with existing connection/authentication fields; optional `configId` reuses a key only at its saved URL | +| `GET /api/model-configs` | Saved models with an optional current `availability` projection | +| Agent `model_availability` | Same read/refresh service; accepts `refresh`, `configIds`, `manual` | + +Manual refresh requires explicit `configIds`. The existing `/api/model-configs/capabilities` endpoint retains its video-output specification behavior. + +```sh +opendirector model availability +opendirector model availability --refresh +opendirector model availability --refresh --manual --model MODEL_CONFIG_UUID +``` + +Responses include a `revision` and model observations containing `identity`, `status`, `reasonCode`, `checkedAt`, `expiresAt` and exact operation decisions. Times use Unix milliseconds. `configId` is null for drafts. `capabilityAutoCheck` and runtime observations are excluded from accepted job execution snapshots. + +When a tool needs an unconfigured model, its HTTP request returns 409 with `code="tool_model_missing"` and `requiredCapability`, such as `image.edit`. The localized `detail` names the required operation and directs the user to Model connections. No tool task or provider request is created. + +工具缺少所需模型时，HTTP 返回 409、`code="tool_model_missing"` 和 `requiredCapability`（例如 `image.edit`）。`detail` 按界面语言说明需要的操作和添加模型的位置，不创建工具任务或发送模型请求。 + +The remote protocol is `GET /omni/capabilities`, with `object="omni.capabilities"` and `schema_version=1`. Missing models, profiles, routes or output information remain unknown. A worker-only pause does not prove that another request will use that worker. No result implies that every possible parameter combination or GPU resource requirement has been verified. + +## Development checks / 开发验证 + +### Input constraints and region editing / 输入限制与选区编辑 + +图片能力分别显示文字生图、单图编辑和多图编辑。已知限制会说明图片数量、选区蒙版及独立参考图；缺失数量声明不会被解释为无限数量。服务能力与应用处理限制分别显示，临时检查不会修改保存的配置。 + +`profile.request` 描述模型输入；`profile.routes[].request` 是对应请求路径的完整有效约束。消费者只使用实际调用路径的声明，不将模型级约束外推到没有声明的路径。`inputs` 描述角色、支持情况及最少/最多数量，`required_any_of` 表示至少提供其中一种输入，`parameters` 描述明确的枚举或数值限制。字段缺失保持未知。默认参数不代表唯一支持值。 + +字符串枚举可以声明 `enum_normalization`：`lower` 忽略大小写，`title` 按服务使用的标题大小写比较，`strip_lower` 同时忽略首尾空白与大小写；缺失或 `null` 保持精确比较。规则只用于比较，不改写用户输入或已接受任务。未知规则或非字符串枚举会使该次发现结果保持未知。应用原有的音色身份等输入规则仍然有效；空音色占位值不会显示为可选音色。 + +只有 `content` 参考计入实际媒体输入。文字生成中的音视频素材信息使用 `metadata`，H3 接续来源使用 `latent`；二者不会满足上传音视频的要求。接续请求保留上一任务的远端标识；没有匹配的接续能力声明时保持未知，同时仍检查实际 `/videos` 路径的部署暂停与明确关闭状态。已关闭路径不会只提示恢复服务。 + +普通生成与工具都按最终发送的图片校验数量；主图和普通参考图合计，蒙版及独立标注参考分别统计。商品替换会先合成场景与商品，因此两个来源对应一张模型输入。超量输入会被拒绝，不会截取图片列表。Z-Image-Turbo 支持一张图片的编辑请求。 + +选区、消除、扩图和商品替换会在服务明确支持、本地适配器能够传输蒙版、输入与尺寸符合要求时使用原生蒙版。蒙版为灰度 PNG，白色表示修改，黑色表示保留；图片、蒙版与请求尺寸一致。当前请求不适用时使用原有编辑方式，保留用户指定输出尺寸。原生请求失败不会自动再发送普通编辑请求。结果仍经过原有选区合成，消除和商品替换保留已有的边缘柔化。 + +新工具任务保存 `imageExecutionPolicy` 和 `imageExecution`，固定请求方式、输入角色及必要限制。运行状态和期限不写入这些记录。旧任务没有这些字段时沿用已接受的请求行为；恢复或重放不会重新选择编辑方式。 + +普通编辑与原生选区都使用适配器实际发送的尺寸和参数进行接受检查；批量增强逐张检查，预设使用自身的请求尺寸。模型渲染尺寸与工具最后保存的尺寸可以不同，检查不会修改用户要求的输出尺寸。`imageExecutionPolicy.requestConstraints` 可选保存精确路由的 `RequestConstraints` JSON；执行时只检查这份已接受的限制，不读取最新声明。旧记录缺失该字段时不会追加限制。 + +The image card distinguishes text-to-image, single-image editing and multi-image editing. Known limits describe image counts, selection masks and independent references. Missing limits remain unknown. Service declarations and application limits are separate and do not rewrite saved settings. + +Only the matched `profile.routes[].request` supplies effective request constraints. `profile.request` describes model facts without proving that every route implements them. Input roles carry explicit counts, `required_any_of` describes alternative required inputs, and parameters carry known enums or numeric bounds. Defaults are not allowed-value lists. + +String enums may declare `enum_normalization`: `lower`, `title`, or `strip_lower`. Missing or null rules retain exact comparison. Both the supplied value and enum candidates are compared using the declared rule, without rewriting input or accepted tasks. Unknown rules or normalization on a non-string enum make discovery unknown. Existing application input rules, including voice identity validation, still apply. An empty voice placeholder is not presented as a selectable voice. + +Only `content` references count as transmitted media. Audio/video metadata in text generation uses `metadata`, and an H3 continuation source uses `latent`; neither satisfies an uploaded-media requirement. Continuation keeps the previous remote job ID. An undeclared continuation profile remains unknown while a deployment pause or an explicitly disabled actual `/videos` endpoint still prevents new work. A disabled endpoint retains its specific rejection reason even during a pause. + +Generation and tools count the images actually sent. A mask and an independent annotation reference have separate roles. Product replacement combines its two source assets into one model image. Excess images are rejected rather than truncated. Z-Image-Turbo accepts a single-image editing request. + +Existing region-editing tools automatically use native masks when the declared route, local adapter transport and unchanged requested dimensions allow it. Masks use grayscale PNG: white edits and black preserves. Otherwise the existing editing method remains available. A failed native request never triggers another generation automatically. Existing final compositing and edge feathering are retained. Accepted tools freeze the chosen request policy and input roles; historical tasks and replay retain their original behavior. + +Ordinary edits and native selections both check the adapter's actual render dimensions and transmitted parameters before acceptance. Batch enhancement checks each image; presets use their own request geometry. Render dimensions may differ from the tool's final saved dimensions, and validation never changes the requested output size. The optional `imageExecutionPolicy.requestConstraints` stores validated `RequestConstraints` JSON for the exact route. Execution checks this accepted snapshot without querying current declarations. Existing records without the field acquire no additional limits. + +Declaration and execution observations have independent expiry fields. Expiring a pause restores the still-valid declaration. The interface displays a shared expiry or query failure once and does not repeat it for every operation. + +```sh +.venv/bin/python -m pytest tests/test_model_availability.py tests/test_capability_acceptance.py -q +.venv/bin/python -m pytest tests/test_capability_input_projection.py tests/test_vllm_omni_h3.py -q +.venv/bin/python -m pytest tests/test_image_input_policy.py tests/test_tool_native_mask.py tests/test_request_capability_acceptance.py -q +.venv/bin/python -m pytest tests/test_tool_model_errors.py tests/test_localization.py -q +.venv/bin/python -m pytest tests/test_model_configs.py tests/test_model_catalog.py tests/test_creation_generation_service.py tests/test_tool_tasks.py tests/test_assistant_conversations.py tests/test_cli.py -q +node --test tests/modelAvailability.test.ts +# With the isolated API and Vite running as described in development validation: +node tests/browser/capability_discovery.mjs +``` + +These tests use isolated configuration/project directories, model doubles and a local HTTP server. They cover identity changes, draft isolation, operation matching, TTL expiration, coalescing, errors, zero-job rejection, local tools and replay. They do not validate real model outputs. See [development validation](../development/validation.md) for browser, contracts and import-layer checks. + +The bilingual browser workflow checks editing without queries, stale draft responses, saving an unsupported configuration, zero-job rejection, UI submission, decoding the saved fixture image, pause expiration, project reopening and unchanged historical input. The frontend clock tests include system-clock changes and repeated observations. 浏览器用例实际保存并解码测试图片，模型输出来自测试替身；前端时钟测试包含系统时间变化和重复响应，不能作为真实模型验证结果。 diff --git a/packages/core/src/opendirector/agent/gateway.py b/packages/core/src/opendirector/agent/gateway.py index d508a441..6f11d131 100644 --- a/packages/core/src/opendirector/agent/gateway.py +++ b/packages/core/src/opendirector/agent/gateway.py @@ -19,9 +19,17 @@ class ToolResult: error: str | None = None def event(self, name: str) -> AgentEvent: + code = self.value.get('code') if isinstance(self.value, dict) else None + if code is None and isinstance(self.value, dict): + issues = self.value.get('issues') + if issues is None and isinstance(self.value.get('receipt'), dict): + issues = self.value['receipt'].get('issues') + if isinstance(issues, list) and len(issues) == 1 and isinstance(issues[0], dict): + code = issues[0].get('code') return {'type': 'tool', 'name': name, 'status': 'failed' if self.error else 'complete', - **({'error': self.error} if self.error else {})} + **({'error': self.error} if self.error else {}), + **({'code': code} if self.error and isinstance(code, str) else {})} class ToolGateway: @@ -88,10 +96,10 @@ async def execute(self, call_id: str, name: str, arguments: dict[str, JsonValue] error = '; '.join(messages) return ToolResult(value={'error': error}, error=error) except (ProjectError, ValueError, KeyError, OSError, UnicodeError) as failure: - value = {'error': str(failure)} + details = {'error': str(failure)} if isinstance(failure, ProjectError): - value['code'] = failure.code - return ToolResult(value=value, error=str(failure)) + details['code'] = failure.code + return ToolResult(value=details, error=str(failure)) async def close(self) -> None: self.closed = True diff --git a/packages/core/src/opendirector/agent/tools.py b/packages/core/src/opendirector/agent/tools.py index 64da5bd1..d9ff99ff 100644 --- a/packages/core/src/opendirector/agent/tools.py +++ b/packages/core/src/opendirector/agent/tools.py @@ -1,5 +1,6 @@ """Business commands shared by every agent runtime.""" import asyncio +from opendirector.application.model_availability import ModelAvailability import difflib import shutil from pathlib import Path @@ -233,6 +234,13 @@ def tools() -> list[dict]: {'type': 'object', 'properties': {}, 'additionalProperties': False}), tool_schema('read', 'Read a discovered skill file/resource, or text asset via asset:. No scripts.', {'type': 'object', 'properties': {'path': {'type': 'string'}}, 'required': ['path'], 'additionalProperties': False})] + result.append(tool_schema('model_availability', 'Read current model operation availability without network access. ' + 'Set refresh=true to query declarations without inference. Manual refresh requires explicit configIds. ' + 'Unknown preserves manual configuration; unavailable blocks only new matching operations.', + {'type': 'object', 'properties': {'refresh': {'type': 'boolean', 'default': False}, + 'manual': {'type': 'boolean', 'default': False}, + 'configIds': {'type': 'array', 'items': {'type': 'string', 'format': 'uuid'}, 'minItems': 1, 'maxItems': 200}}, + 'additionalProperties': False})) query_schema = TypeAdapter(Query).json_schema() query_schema['type'] = 'object' for definition in query_schema['$defs'].values(): @@ -283,6 +291,7 @@ def __init__(self, projects, generation, configs): self.projects = projects self.generation = generation self.configs = configs + self.availability = getattr(generation, 'availability', None) or ModelAvailability(configs) self.assets = CreationAssets(projects) self.voices = getattr(generation, 'voices', None) or VoiceLibrary(configs) @@ -322,8 +331,16 @@ def recording(): ref_text=arguments.get('refText'), speaker_description=arguments.get('speakerDescription')) if name == 'read_project': snapshot = await asyncio.to_thread(self.projects.snapshot, project_id) - return {'project': snapshot.model_dump(mode='json'), 'models': self.configs.list(), 'commands': list(COMMANDS), + return {'project': snapshot.model_dump(mode='json'), 'models': self.availability.catalog(), 'commands': list(COMMANDS), 'commandMetadata': {kind: {'description': model.__doc__} for kind, model in COMMANDS.items()}} + if name == 'model_availability': + refresh = TypeAdapter(bool).validate_python(arguments.get('refresh', False), strict=True) + manual = TypeAdapter(bool).validate_python(arguments.get('manual', False), strict=True) + config_ids = TypeAdapter(list[UUID]).validate_python(arguments['configIds']) if 'configIds' in arguments else None + if refresh: + return await asyncio.to_thread(self.availability.refresh, + [str(item) for item in config_ids] if config_ids is not None else None, manual=manual) + return self.availability.list() if name == 'query': if {'projectId', 'queryId'} & arguments.keys(): raise ProjectError('查询身份与工程范围由宿主提供。') diff --git a/packages/core/src/opendirector/application/creation_generation.py b/packages/core/src/opendirector/application/creation_generation.py index 6f35659f..5c668296 100644 --- a/packages/core/src/opendirector/application/creation_generation.py +++ b/packages/core/src/opendirector/application/creation_generation.py @@ -15,6 +15,7 @@ from opendirector.application.creation_assets import CreationAssets from opendirector.application.creation_canvas import CreationCanvas, asset_ref, current_generation_inputs, visible_item_ids from opendirector.application.voice_library import VoiceLibrary +from opendirector.application.model_availability import ModelAvailability, generation_request from opendirector.application.generation_runtime import GenerationRuntime, _self_hosted_video from opendirector.domain.creation.assets import AssetProvenance from opendirector.domain.creation.canvas import TextCanvasItem, ImageCanvasItem, VideoCanvasItem, AudioCanvasItem @@ -80,7 +81,7 @@ def frozen_model(self, frozen): class CreationGeneration(GenerationRuntime): - def __init__(self, projects, configs, provider=None): + def __init__(self, projects, configs, provider=None, *, availability=None): super().__init__() self.projects, self.configs = projects, configs self.assets = CreationAssets(projects) @@ -88,6 +89,7 @@ def __init__(self, projects, configs, provider=None): self.settings = gateway().settings(projects.local.repository) self.provider = provider or GenerationProviders(self.store) self.voices = VoiceLibrary(configs) + self.availability = availability or ModelAvailability(configs) def directory(self, project_id): with self.projects.local.transaction() as db: @@ -149,7 +151,8 @@ def _parameters(self, spec, config, node, records): if spec.kind == 'image': from opendirector.domain.image_output import resolve_image_output params['size'] = resolve_image_output(config, input_rules(config), intent) - if input_rules(config)['profile'] == 'vllm-omni-image': + if (input_rules(config)['profile'] == 'vllm-omni-image' + and not any(records[key]['kind'] == 'image' for key in node['referenceIds'])): from opendirector.integrations.vllm_omni_image import SAMPLING params.update(SAMPLING) if spec.kind == 'audio' and intent['creationType'] == 'music': @@ -298,6 +301,14 @@ def prepare(self, command, *, session=None, source='ui'): # again under commit so a slow/offline provider cannot freeze editing. initial = self.projects.snapshot(command.projectId) voice_spec = next((spec for spec in initial.generationSpecs if spec.id == command.payload.specId), None) + capability_identity = None + capability_failure = None + if voice_spec is not None and voice_spec.modelConfigId: + try: + capability_config = self.configs.resolve(str(voice_spec.modelConfigId), 'agent' if voice_spec.kind == 'text' else voice_spec.kind) + capability_identity = self.availability.prepare(capability_config) + except ProjectError as error: + capability_failure = error voice_config, voice_failure = None, None if voice_spec is not None and voice_spec.kind == 'audio' and voice_spec.parameters.creationType == 'audio': try: @@ -340,7 +351,7 @@ def accept(before): intermediate = ProjectSnapshot.model_validate(data) spec = next(s for s in intermediate.generationSpecs if s.id == old.id) config = self.configs.resolve(str(spec.modelConfigId), 'agent' if spec.kind == 'text' else spec.kind) - public = ModelConfigPublic.model_validate({key: value for key, value in config.items() if key != 'apiKey'} | {'inputRules': input_rules(config)}) + public = ModelConfigPublic.model_validate({key: value for key, value in config.items() if key != 'apiKey'} | {'inputRules': self.availability.effective_input_rules(config)}) spec, resolved = current_generation_inputs(intermediate, spec, public.inputRules) spec = execution_spec(spec, resolved, public.inputRules) try: @@ -350,6 +361,10 @@ def accept(before): raise ContractError('revision_conflict', '音色验证期间生成配置或模型服务已变化，请重新提交。', spec.id) if voice_failure is not None: raise voice_failure + if capability_failure is not None: + raise capability_failure + if capability_identity is not None: + self.availability.require(config, generation_request(config, spec, resolved), capability_identity) except (ValueError, ProjectError) as error: raise ContractError.from_error(error, spec.id) from error for reference in resolved: @@ -373,6 +388,8 @@ def accept(before): if hasattr(self.provider, 'prepare_inputs'): self.provider.prepare_inputs(str(command.projectId), task, config) parameters = self._parameters(spec, config, task, records) + if capability_identity is not None: + self.availability.require(config, generation_request(config, spec, resolved, parameters=parameters), capability_identity) except ProjectError as error: raise ContractError.from_error(error, spec.id) from error job = Job.model_validate({**stub.model_dump(), 'adapterParameters': parameters, 'execution': self._execution(spec.kind, parameters, config)}) diff --git a/packages/core/src/opendirector/application/creation_tool_drafts.py b/packages/core/src/opendirector/application/creation_tool_drafts.py index a8fcc38c..55e4baed 100644 --- a/packages/core/src/opendirector/application/creation_tool_drafts.py +++ b/packages/core/src/opendirector/application/creation_tool_drafts.py @@ -63,6 +63,7 @@ def freeze_inputs(service, project_id, value): raise ProjectError('工具来源已变化，不能用旧选区提交新素材。', 409) sources.append(asset_ref(asset).model_dump(mode='json')) first = service.assets.assets.get(project_id, value['sourceAssetIds'][0]) + auxiliary_assets = {} for key in ('maskAssetId', 'annotationAssetId'): if value.get(key): auxiliary = service.assets.assets.get(project_id, value[key]) @@ -70,6 +71,7 @@ def freeze_inputs(service, project_id, value): if (auxiliary.kind != 'image' or first.kind != 'image' or (auxiliary.metadata.width, auxiliary.metadata.height) != (first.metadata.width, first.metadata.height)): raise ProjectError('选区或标注尺寸与实际来源图片不一致。', 422) + auxiliary_assets[key] = asset_ref(auxiliary).model_dump(mode='json') for box in value.get('boxes', []): if (first.kind not in {'image', 'video'} or box['x'] < 0 or box['y'] < 0 or box['width'] <= 0 or box['height'] <= 0 or box['x'] + box['width'] > first.metadata.width or box['y'] + box['height'] > first.metadata.height): @@ -81,6 +83,7 @@ def freeze_inputs(service, project_id, value): raise ProjectError('工具结果占位节点已变化。', 409) return dict(draftSnapshot=document.model_dump(mode='json') if document else None, draftBinding=value.get('draft'), sourceAssets=sources, + auxiliaryAssets=auxiliary_assets, targetSnapshot=target.model_dump(mode='json') if target else None, targetIntent=target_intent(snapshot, target) if target else None) diff --git a/packages/core/src/opendirector/application/model_availability.py b/packages/core/src/opendirector/application/model_availability.py new file mode 100644 index 00000000..e23c7081 --- /dev/null +++ b/packages/core/src/opendirector/application/model_availability.py @@ -0,0 +1,395 @@ +"""Shared optional capability checks for new work; catalogs never perform I/O. + +Configured adapters still own request construction and manual input rules. +Remote observations can reject a precisely matched request, but cannot add +adapters, rewrite configuration, or change an already accepted execution. +""" +from dataclasses import dataclass, replace +from collections import Counter +from math import ceil, lcm +import hashlib +import json +from urllib.parse import urlsplit + +from opendirector.domain.omni_capabilities import timestamp +from opendirector.domain.image_inputs import image_input_policy +from opendirector.domain.input_rules import input_rules +from opendirector.domain.request_constraints import INPUT_ROLES, constraint_failure, input_details +from opendirector.domain.capability_messages import constraint_rejection +from opendirector.domain.voices import effective_speech_voice +from opendirector.domain.projects import ProjectError +from opendirector.integrations.capability_probe import CapabilityProbe, PAUSE_SECONDS, automatic, identity +from opendirector.integrations.generation_capability_parameters import generation_capability_parameters +from opendirector.integrations.volcengine import endpoint + + +class AvailabilityError(ProjectError): + def __init__(self, code, message): + super().__init__(message, 409) + self.code = code + + +@dataclass(frozen=True) +class RequestOperation: + operation: str + profile: str + path: str + output: str + method: str = 'POST' + # None is a directory projection; an actual request fixes transmitted inputs. + inputs: frozenset[str] | None = None + input_counts: tuple[tuple[str, int], ...] | None = None + parameters: tuple[tuple[str, str | int | float | bool], ...] = () + + +def requests_for(config, *, inputs=None): + """Only the protocols this application actually sends, including music.""" + kind, protocol = config['kind'], config.get('protocol', 'openai') + def request(operation, profile, suffix, output): + return RequestOperation(operation, profile, urlsplit(endpoint(config['url'], suffix)).path, output, + inputs=frozenset(inputs) if inputs is not None else None) + if kind == 'agent': + return [request('chat', 'text_response', '/chat/completions', 'text')] + if kind == 'image' and protocol == 'openai': + return [request('image_generation', 'text_to_image', '/images/generations', 'image'), + request('image_edit', 'image_to_image', '/images/edits', 'image')] + if kind == 'video' and protocol == 'vllm_omni': + return [request('video_generation', profile, '/videos', 'video') + for profile in ('text_to_video', 'image_to_video', 'video_to_video')] + if kind == 'audio' and protocol in {'openai', 'vllm_omni_music'}: + return [request('speech', 'lyrics_to_music' if protocol == 'vllm_omni_music' else 'text_to_speech', '/audio/speech', 'audio')] + # Ark/Minimax and specialized segmentation, depth, score protocols have + # different wire contracts. A shared media label proves no Omni operation. + return [] + + +def generation_request(config, spec, references, *, parameters=None): + candidates = requests_for(config) + if not candidates: + return None + transmitted = [item for item in references if item.transmission == 'content'] + kinds = {item.asset.kind for item in transmitted} + if spec.kind == 'image': + profile = 'image_to_image' if 'image' in kinds else 'text_to_image' + elif spec.kind == 'video': + profile = 'video_to_video' if 'video' in kinds else 'image_to_video' if 'image' in kinds else 'text_to_video' + elif spec.kind == 'audio': + profile = 'lyrics_to_music' if spec.parameters.creationType == 'music' else 'text_to_speech' + else: + profile = 'text_response' + request = next((item for item in candidates if item.profile == profile), None) + if spec.kind == 'video' and any(item.transmission == 'latent' for item in references): + # A cached continuation ID is not a video upload. Keep its profile + # unknown unless declared, while retaining the actual route for pause. + request = replace(candidates[0], profile='video_continuation') + # The resolved reference transmission is the same one frozen for execution. + # Music sends the prompt as instructions and lyrics (or arrangement tags + # for instrumental music) as the text field on /audio/speech. + inputs = {'text'} + if spec.kind in {'text', 'video'}: + inputs.update(kinds) + elif spec.kind == 'image': + inputs.update(kinds & {'image'}) + if profile == 'lyrics_to_music': + inputs.add('instructions') + counts = Counter(item.asset.kind for item in transmitted if item.asset.kind in inputs) + counts['text'] = 1 + if profile == 'lyrics_to_music': + counts['instructions'] = 1 + actual = generation_capability_parameters(config, spec.kind, parameters, has_images='image' in kinds) if parameters is not None else {} + if profile == 'text_to_speech': + voice = effective_speech_voice(config, getattr(getattr(spec, 'parameters', None), 'voice', None)) + counts['voice'] = 1 + inputs.add('voice') + actual['voice'] = voice + return replace(request, inputs=frozenset(inputs), input_counts=tuple(counts.items()), + parameters=tuple(actual.items())) if request else None + + +def tool_request(config, capability, *, actual=False): + profile = {'text.chat': 'text_response', 'image.generate': 'text_to_image', 'image.edit': 'image_to_image', 'image.cutout': 'image_to_image', + 'video.generate': 'text_to_video', 'video.first_frame': 'image_to_video', + 'video.edit': 'video_to_video', + 'audio.speech': 'text_to_speech', 'audio.music': 'lyrics_to_music'}.get(capability) + request = next((item for item in requests_for(config) if item.profile == profile), None) + if request is None or not actual: + return request + # These are the arguments of the existing tool adapters, not all source + # assets: e.g. a video-to-video tool does not transmit an audio attachment. + inputs = {'text'} + if profile in {'image_to_image', 'image_to_video'}: + inputs.add('image') + elif profile == 'video_to_video': + inputs.add('video') + elif profile == 'lyrics_to_music': + inputs.add('instructions') + return replace(request, inputs=frozenset(inputs)) + + +def _revision(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()[:24] + + +class ModelAvailability: + def __init__(self, configs, probe=None): + self.configs = configs + self.probe = probe or CapabilityProbe() + + def prepare(self, config, *, manual=False): + self.probe.query(config, manual=manual) + return identity(config) + + def prepare_many(self, configs): + """Check one acceptance's dependencies within a shared query budget.""" + configs = list(configs) + self.probe.query_many(configs) + return [identity(config) for config in configs] + + def assert_identity(self, config, prepared): + if identity(config) != prepared: + raise AvailabilityError('capability_connection_changed', '能力检查期间模型服务已变化，请重新提交。') + + def request_constraints(self, config, request): + """Return only the complete constraint set for the actual HTTP route.""" + observation = self.probe.peek(config) + if not observation or not observation.document or request is None: + return None + document = observation.document + model = next((value for value in document.models if value.id == config['modelId']), None) + operation = model.operations.get(request.operation) if model else None + profile = next((value for value in operation.profiles if value.id == request.profile), None) if operation else None + if not profile or operation.support == 'unsupported' or profile.support == 'unsupported': + return None + route = next((value for value in profile.routes if value.transport == 'http' + and value.path == request.path and value.method == request.method), None) + exposed = next((value for value in document.routes if value.transport == 'http' + and value.path == request.path and value.method == request.method), None) + if (route is None or route.exposure == 'disabled' or operation.exposure == 'disabled' + or (exposed and exposed.exposure == 'disabled')): + return None + return route.request + + def image_execution_policy(self, config): + policy = image_input_policy(config) + request = next((value for value in requests_for(config) if value.profile == 'image_to_image'), None) + contract = self.request_constraints(config, request) + if contract is None: + return policy + images = contract.inputs.get('image') + if images is not None: + if images.support == 'unsupported': + policy['maxImages'] = 0 + elif images.max_count is not None: + policy['maxImages'] = min(policy['maxImages'], images.max_count) + # A remote declaration cannot add a transport operation to an adapter. + from opendirector.integrations.model_adapters import adapter_for + adapter = adapter_for(config) + if adapter is None or not adapter.supports_native_mask: + return policy + mask, role = contract.mask, contract.inputs.get('mask_image') + if (mask is None or mask.support != 'supported' or role is None or role.support != 'supported' + or mask.source_role != 'image' or mask.white != 'edit' or mask.black != 'preserve' + or mask.binarize_threshold != 0.5 or mask.requires_same_size is not True + or mask.geometry.state != 'known' or mask.geometry.multiple_of is None + or mask.max_source_count is None or mask.source_resize is None or mask.source_resize.max_pixels is None + or mask.source_resize.policy != 'downscale_preserve_aspect_ratio' + or (role.max_count is not None and role.max_count < 1) or (role.min_count or 0) > 1 + or (images is not None and (images.min_count or 0) > 1)): + return policy + reference = contract.inputs.get('reference_image') + max_reference = (reference.max_count if reference and reference.support == 'supported' + and reference.max_count is not None else 0) + width, height = contract.parameters.get('width'), contract.parameters.get('height') + policy['nativeMask'] = dict(field='mask_image', referenceField='reference_image' if max_reference else None, + encoding='grayscale_white_edit', minImages=1, maxImages=min(policy['maxImages'], mask.max_source_count), + maxReferenceImages=max_reference, + minReferenceImages=reference.min_count or 0 if reference else 0, + widthMultiple=lcm(mask.geometry.multiple_of, int(width.step)) if width and width.step and float(width.step).is_integer() else mask.geometry.multiple_of, + heightMultiple=lcm(mask.geometry.multiple_of, int(height.step)) if height and height.step and float(height.step).is_integer() else mask.geometry.multiple_of, + minWidth=max(1, ceil(width.minimum)) if width and width.minimum is not None else 1, + minHeight=max(1, ceil(height.minimum)) if height and height.minimum is not None else 1, + maxWidth=width.maximum if width else None, maxHeight=height.maximum if height else None, + widthValues=width.enum if width else None, heightValues=height.enum if height else None, + widthStep=width.step if width else None, heightStep=height.step if height else None, + maxPixels=min(mask.source_resize.max_pixels, contract.max_output_pixels or mask.source_resize.max_pixels), sameSize=True) + return policy + + def effective_input_rules(self, config): + """Compose acceptance limits without writing them into saved settings.""" + rules = input_rules(config) + if config['kind'] == 'image': + rules['maxImages'] = self.image_execution_policy(config)['maxImages'] + return rules + + @staticmethod + def _declaration(result): + result['declaration'] = {key: result[key] for key in ('status', 'reasonCode', 'checkedAt', 'expiresAt')} + return result + + def decision(self, config, request): + observation = self.probe.peek(config) + result = dict(operation=request.operation, profile=request.profile, method=request.method, + path=request.path, output=request.output, status='unknown', reasonCode='not_checked', + checkedAt=None, expiresAt=None, inputDetails=[]) + if not observation: + return self._declaration(result) + result.update(checkedAt=round(observation.checked * 1000), expiresAt=round(observation.expires * 1000), reasonCode=observation.reason) + document = observation.document + if document is None: + return self._declaration(result) + model = next((item for item in document.models if item.id == config['modelId']), None) + operation = model.operations.get(request.operation) if model else None + if operation is None: + result['reasonCode'] = 'model_not_declared' if model is None else 'operation_not_declared' + return self._declaration(result) + profile = next((item for item in operation.profiles if item.id == request.profile), None) + route = None + exposed = next((item for item in document.routes if item.transport == 'http' + and item.method == request.method and item.path == request.path), None) + if operation.support == 'unsupported': + result.update(status='unavailable', reasonCode='operation_unsupported') + elif operation.exposure == 'disabled': + result.update(status='unavailable', reasonCode='operation_disabled') + elif profile is None: + result['reasonCode'] = 'profile_not_declared' + route = next((route for item in operation.profiles for route in item.routes + if route.transport == 'http' and route.method == request.method and route.path == request.path), None) + elif profile.support == 'unsupported': + result.update(status='unavailable', reasonCode='profile_unsupported') + else: + route = next((item for item in profile.routes if item.transport == 'http' + and item.method == request.method and item.path == request.path), None) + if route is None: + result['reasonCode'] = 'route_not_declared' + elif route.exposure == 'disabled' or (exposed and exposed.exposure == 'disabled'): + result.update(status='unavailable', reasonCode='route_disabled') + elif operation.support == profile.support == 'supported' and profile.outputs and request.output not in profile.outputs: + result.update(status='unavailable', reasonCode='output_unsupported') + elif operation.support == profile.support == 'supported' and request.output in profile.outputs: + required = set(profile.required_inputs) + if required - INPUT_ROLES: + result['reasonCode'] = 'input_contract_unknown' + elif request.inputs is not None and required - request.inputs: + result.update(status='unavailable', reasonCode='required_inputs_missing') + else: + result.update(status='available', reasonCode='supported') + else: + result['reasonCode'] = 'declaration_incomplete' + contract = route.request if route is not None else None + if contract is not None: + result['inputDetails'] = input_details(contract) + if config['kind'] == 'image' and 'image' in contract.inputs: + maximum = contract.inputs['image'].max_count + local_maximum = input_rules(config)['maxImages'] + if maximum is not None and maximum > local_maximum: + result['inputDetails'].append(dict(name='image', source='application', + support='supported' if local_maximum else 'unsupported', maxCount=local_maximum)) + counts = dict(request.input_counts) if request.input_counts is not None else ( + {role: 1 for role in request.inputs} if request.inputs is not None else None) + failure = constraint_failure(contract, counts, dict(request.parameters)) + if failure and result['status'] != 'unavailable': + result.update(status='unavailable', reasonCode=failure[0]) + if exposed and exposed.exposure == 'disabled' and result['status'] != 'unavailable': + result.update(status='unavailable', reasonCode='route_disabled') + self._declaration(result) + # Execution is an operation fact on the actual endpoint. An unknown + # input profile does not invalidate a fresh deployment pause there. + execution = operation.execution + if route: + try: + timestamp(execution.observed_at) + active = self.probe.clock() < observation.received + PAUSE_SECONDS + result['execution'] = dict(state=execution.state if active else 'unknown', scope=execution.scope, + checkedAt=round(observation.checked * 1000), + expiresAt=round((observation.checked + PAUSE_SECONDS) * 1000) if active else None) + if active and execution.state == 'paused' and execution.scope == 'deployment' and result['reasonCode'] != 'route_disabled': + result.update(status='unavailable', reasonCode='execution_paused', expiresAt=round((observation.checked + PAUSE_SECONDS) * 1000)) + except (ValueError, TypeError, AttributeError): + pass + return result + + def require(self, config, request, prepared=None): + if prepared is not None: + self.assert_identity(config, prepared) + if request is None: + return + decision = self.decision(config, request) + if decision['status'] == 'unavailable': + code = 'capability_paused' if decision['reasonCode'] == 'execution_paused' else 'capability_unavailable' + labels = {'text_response': '文字回复', 'image_response': '图片回复', 'text_to_image': '图片生成', + 'image_to_image': '图片编辑', 'text_to_video': '文字生成视频', 'image_to_video': '图片生成视频', + 'video_to_video': '视频编辑', 'text_to_speech': '语音合成', 'lyrics_to_music': '音乐生成'} + label = labels.get(request.profile, '此操作') + message = ('服务已暂停，请恢复后重试。' if code == 'capability_paused' + else f'服务未开放{label}。' if decision['reasonCode'] in {'route_disabled', 'operation_disabled'} + else '请补充所需素材。' if decision['reasonCode'] == 'required_inputs_missing' + else f'该模型不支持{label}。') + contract = self.request_constraints(config, request) + counts = dict(request.input_counts) if request.input_counts is not None else ( + {role: 1 for role in request.inputs} if request.inputs is not None else None) + failure = constraint_failure(contract, counts, dict(request.parameters)) if contract else None + if failure and decision['reasonCode'] == failure[0]: + code, message = constraint_rejection(*failure) + raise AvailabilityError(code, message) + + def model(self, config): + operations = [self.decision(config, request) for request in requests_for(config)] + for index, value in enumerate(operations): + if value['profile'] != 'image_to_image': + continue + request = next(item for item in requests_for(config) if item.profile == 'image_to_image') + contract = self.request_constraints(config, request) + constraint = contract.inputs.get('image') if contract else None + single = self.decision(config, replace(request, inputs=frozenset({'text', 'image'}), + input_counts=(('text', 1), ('image', 1)))) + operations[index] = single + multi = {**value, 'profile': 'images_to_image'} + declaration = dict(value['declaration']) + if declaration['status'] != 'unavailable': + if constraint and (constraint.support == 'unsupported' or constraint.max_count is not None and constraint.max_count < 2): + declaration.update(status='unavailable', reasonCode='input_count_unsupported') + elif not constraint or (constraint.max_count is None and (constraint.min_count or 0) < 2): + declaration.update(status='unknown', reasonCode='input_count_unknown') + multi['declaration'] = declaration + if multi['reasonCode'] != 'execution_paused': + multi.update(declaration) + operations.insert(index + 1, multi) + break + states = {item['status'] for item in operations} + state = ('unavailable' if states == {'unavailable'} else 'available' if states == {'available'} else 'unknown') + observation = self.probe.peek(config) + value = dict(configId=config.get('id'), modelId=config['modelId'], identity=identity(config), autoCheck=automatic(config), + status=state, reasonCode=(operations[0]['reasonCode'] if operations and len(states) == 1 else 'mixed_operations' if operations else 'adapter_not_mapped'), + checkedAt=round(observation.checked * 1000) if observation else None, + expiresAt=min((item['expiresAt'] for item in operations if item['expiresAt'] is not None), default=None), operations=operations) + return {**value, 'revision': _revision(value)} + + def list(self): + values = [self.model(private) for _, private in self.configs.catalog_snapshot()] + return {'revision': _revision(values), 'models': values} + + def catalog(self): + return [{**public, 'availability': self.model(private)} for public, private in self.configs.catalog_snapshot()] + + def refresh(self, config_ids=None, *, manual=False): + if type(manual) is not bool or (config_ids is not None and not 1 <= len(config_ids) <= 200): + raise ProjectError('能力刷新参数无效，请指定 1–200 个模型配置。', 422) + if manual and not config_ids: + raise ProjectError('手动检查需要指定模型配置。', 422) + selected = set(config_ids) if config_ids is not None else None + values = self.configs.list() + if selected is not None and selected - {item['id'] for item in values}: + raise ProjectError('模型配置已被删除，请重新加载。', 404) + seen, configs = set(), [] + for item in values: + if selected is not None and item['id'] not in selected: + continue + config = self.configs.resolve(item['id'], item['kind']) + if not manual and not automatic(config): + continue + key = identity(config, service=True) + if key not in seen: + configs.append(config) + seen.add(key) + self.probe.query_many(configs, manual=manual) + return self.list() diff --git a/packages/core/src/opendirector/application/model_catalog.py b/packages/core/src/opendirector/application/model_catalog.py index d5df3a3c..99eb4ac7 100644 --- a/packages/core/src/opendirector/application/model_catalog.py +++ b/packages/core/src/opendirector/application/model_catalog.py @@ -11,6 +11,7 @@ from opendirector.domain.projects import ProjectError from opendirector.integrations.chat_completions import ChatCompletions, chat_payload from opendirector.integrations.model_capabilities import read_service_models, read_video_capabilities +from opendirector.application.model_availability import ModelAvailability, requests_for _SAVED_KEY = '请填写该服务的 API Key；新地址不能复用原地址的密钥。' @@ -32,12 +33,23 @@ def _reuse_saved_key(config, existing, *, require_key): class ModelCatalog: """Saved model configuration plus read-only service probes.""" - def __init__(self, configs, tester=None): + def __init__(self, configs, tester=None, availability=None): self.configs = configs self.tester = tester or ChatCompletions() + self.availability = availability or ModelAvailability(configs) def list(self): - return self.configs.list() + return self.availability.catalog() + + async def check_availability(self, kind, url, model_id, api_key, config_id=None, *, connection_type='api', + protocol=None, auth_mode=None, proxy_mode=None, video_adapter=None, capability_auto_check=None): + config = _draft(kind, url, model_id, api_key, connection_type, protocol, auth_mode, proxy_mode, + videoAdapter=video_adapter or 'auto', _capability_namespace='draft') + if capability_auto_check is not None: + config['capabilityAutoCheck'] = capability_auto_check + config = await self._with_saved_key(config, config_id, kind, require_key=True, threaded=True) + await asyncio.to_thread(self.availability.prepare, config, manual=True) + return self.availability.model(config) def save(self, kind, url, model_id, api_key, config_id=None, **options): return self.configs.save(kind, url, model_id, api_key, config_id, **options) @@ -74,9 +86,13 @@ async def discover(self, kind, url, model_id, api_key, config_id=None, *, connec return list(dict.fromkeys(item['id'] for item in models)) async def test_agent(self, url, model_id, api_key, config_id=None, *, connection_type='api', protocol=None, - auth_mode=None, proxy_mode=None, timeout=AGENT_TEST_TIMEOUT_SECONDS): - config = _draft('agent', url, model_id, api_key, connection_type, protocol, auth_mode, proxy_mode) + auth_mode=None, proxy_mode=None, timeout=AGENT_TEST_TIMEOUT_SECONDS, capability_auto_check=None): + config = _draft('agent', url, model_id, api_key, connection_type, protocol, auth_mode, proxy_mode, _capability_namespace='draft') + if capability_auto_check is not None: + config['capabilityAutoCheck'] = capability_auto_check config = await self._with_saved_key(config, config_id, 'agent', require_key=True, threaded=True) + await asyncio.to_thread(self.availability.prepare, config) + self.availability.require(config, requests_for(config, inputs={'text'})[0]) payload = chat_payload(config, [{'role': 'user', 'content': 'Reply with OK only.'}], 'auto') payload['max_tokens'] = 1024 started = time.monotonic() diff --git a/packages/core/src/opendirector/application/tool_context.py b/packages/core/src/opendirector/application/tool_context.py index 592c9da6..b1835436 100644 --- a/packages/core/src/opendirector/application/tool_context.py +++ b/packages/core/src/opendirector/application/tool_context.py @@ -44,11 +44,14 @@ class ToolContext: def __init__(self, *, sources, mask, boxes, parameters, image_model, video_model, progress, video_submitted=None, cutout_model=None, edit_model=None, edit_seconds=(4.0, 15.0), chat_model=None, segment_model=None, track_model=None, - edit_keeps_layout=True, annotation=None, edit_max_images=1, segment_available=False, depth_model=None): + edit_keeps_layout=True, annotation=None, edit_max_images=1, segment_available=False, depth_model=None, + native_mask=False): self.sources: list[Source] = sources self.mask: bytes | None = mask self.annotation: bytes | None = annotation self.edit_max_images = edit_max_images + # Chosen and frozen at acceptance, never switched after a provider failure. + self.native_mask = native_mask self.segment_available = segment_available self.boxes: list[dict] = boxes self.parameters: dict = parameters @@ -83,10 +86,13 @@ def choice(self, key, options, default=None): async def progress(self, message): await self._progress(message) - async def edit(self, image: Image.Image, prompt: str, strength: float, *, references=()) -> Image.Image: + async def edit(self, image: Image.Image, prompt: str, strength: float, *, references=(), mask=None, + reference=None) -> Image.Image: """Image-to-image at the source's aspect; the result is resampled back to the source size.""" result = await self._image(ImageRequest(prompt=prompt, width=image.width, height=image.height, - image=media.rgb_png(image), strength=strength, references=tuple(references))) + image=media.rgb_png(image), strength=strength, references=tuple(references), + mask=media.png(mask.convert('L')) if mask is not None and self.native_mask else None, + reference=reference if self.native_mask else None)) return media.resize(media.open_image(result.data), image.width, image.height) async def reframe(self, image: Image.Image, width: int, height: int, prompt: str) -> Image.Image: diff --git a/packages/core/src/opendirector/application/tool_image_edits.py b/packages/core/src/opendirector/application/tool_image_edits.py index d5229cef..17ab112c 100644 --- a/packages/core/src/opendirector/application/tool_image_edits.py +++ b/packages/core/src/opendirector/application/tool_image_edits.py @@ -27,15 +27,16 @@ async def _extend(ctx: ToolContext, image: Image.Image, pads, prompt: str) -> Im Instruction editors ignore strength and recompose a padded canvas instead of filling it, so they are asked to pull the camera back over the original. """ - if not ctx.edit_keeps_layout: + if not ctx.edit_keeps_layout and not ctx.native_mask: await ctx.progress('正在扩展画面') width, height = image.width + pads[0] + pads[2], image.height + pads[1] + pads[3] return await ctx.reframe(image, width, height, _with(prompt, ( f'镜头拉远，扩展画面：{_placement(pads, image.width, image.height)}。保持原图中的人物、物体、服饰、场景、色调与光线完全一致，' '延续原有的透视与风格，画面完整自然，不要边框、白边或拼接痕迹'))) - canvas, _ = media.extend_canvas(image, *pads) + canvas, region = media.extend_canvas(image, *pads) await ctx.progress('正在扩展画面') - drawn = await ctx.edit(canvas, _with(prompt, '自然延展画面四周，与原图内容、风格、透视和光线保持一致，无边框无拼接痕迹'), 0.9) + drawn = await ctx.edit(canvas, _with(prompt, '自然延展画面四周，与原图内容、风格、透视和光线保持一致，无边框无拼接痕迹'), 0.9, + mask=region) # The redraw shifts tone slightly; a wide ramp inside the original hides the seam. inset = max(8, min(image.size) // 8) left, top, right, bottom = pads @@ -62,12 +63,15 @@ async def smart_edit(ctx: ToolContext): if not instruction: raise ProjectError('请描述要如何修改图片。', 422) await ctx.progress('正在按描述改图') - input_image, references = image, () + input_image, references, reference = image, (), None if mode != 'selection' and ctx.annotation: marked = media.open_image(ctx.annotation) if marked.size != image.size: raise ProjectError('标注图片尺寸必须与原图一致。', 422) - if ctx.edit_max_images > 1: + if ctx.native_mask: + reference = media.rgb_png(marked) + instruction += '。参考图是原图上的编辑标记，请根据标记理解位置与意图' + elif ctx.edit_max_images > 1: references = (media.rgb_png(marked),) instruction += '。第二张图是第一张图上的编辑标记，请根据标记理解位置与意图' else: @@ -76,10 +80,12 @@ async def smart_edit(ctx: ToolContext): instruction += '，完成修改后移除这些辅助标记' if mode != 'annotation' and ctx.mask: instruction += '。这是原图上的局部修改，请严格保持原图构图、镜头、主体位置、大小和姿势，使编辑结果与原图逐像素对齐' - edited = await ctx.edit(input_image, instruction + '。保持未提及的内容不变', 0.78, references=references) + region = media.mask_from(ctx.mask, image.size) if mode != 'annotation' and ctx.mask else None + edited = await ctx.edit(input_image, instruction + '。保持未提及的内容不变', 0.78, + references=references, mask=region, reference=reference) if mode != 'annotation' and ctx.mask: # A selection is an exact editing boundary: pixels outside it stay original. - edited = media.blend(image, edited, media.mask_from(ctx.mask, image.size)) + edited = media.blend(image, edited, region) return [image_output(edited, '智能改图')] async def outpaint(ctx: ToolContext): @@ -117,7 +123,8 @@ async def inpainting_remove(ctx: ToolContext): raise ProjectError('请先涂抹需要消除的区域。', 422) mask = media.mask_from(ctx.mask, image.size) await ctx.progress('正在消除并填补') - drawn = await ctx.edit(media.soften_region(image, mask), '移除涂抹区域中的物体与文字，用周围环境自然填补，画面干净完整', 0.8) + drawn = await ctx.edit(image if ctx.native_mask else media.soften_region(image, mask), + '移除涂抹区域中的物体与文字，用周围环境自然填补，画面干净完整', 0.8, mask=mask) return [image_output(media.blend(image, drawn, media.feather(mask, max(3, min(image.size) / 120))), '消除笔')] def _yaw_text(yaw): @@ -242,9 +249,10 @@ async def product_replacement(ctx: ToolContext): placed = scene.copy() placed.paste(layer, (x, y), layer) await ctx.progress('正在融合商品与场景') - drawn = await ctx.edit(placed, _with(ctx.text('prompt'), '商品自然融入场景，光影、透视和色调一致，保留商品外观和文字'), 0.42) region = Image.new('L', scene.size, 0) region.paste(255, (x, y, x + width, y + height)) + drawn = await ctx.edit(placed, _with(ctx.text('prompt'), '商品自然融入场景，光影、透视和色调一致，保留商品外观和文字'), 0.42, + mask=region) result = media.blend(scene, drawn, media.feather(region, max(4, min(width, height) / 12))) edge = {'4K': 4096, '2K': 2048}.get(ctx.parameters.get('resolution')) if edge and max(result.size) < edge: diff --git a/packages/core/src/opendirector/application/tool_image_inputs.py b/packages/core/src/opendirector/application/tool_image_inputs.py new file mode 100644 index 00000000..4ada84de --- /dev/null +++ b/packages/core/src/opendirector/application/tool_image_inputs.py @@ -0,0 +1,119 @@ +"""Image-tool source roles and the execution choice fixed at acceptance. + +Source assets are recipe inputs: product replacement composes two sources into +one model image, while upscaling makes a separate request for each source. +The shared image-input policy owns the limits on those serialized requests. +""" +import math + +from opendirector.domain.image_inputs import native_mask_applicable +from opendirector.domain.projects import ProjectError + + +def validate_image_sources(tool, sources): + expected = 2 if tool == 'product-replacement' else None if tool == 'image-hd' else 1 + if expected is not None and len(sources) != expected: + raise ProjectError(f'该工具需要 {expected} 张来源图片。', 422, code='tool_source_count') + if any(source['kind'] != 'image' for source in sources): + raise ProjectError('来源素材必须是图片。', 422, code='tool_source_type') + + +def local_image_resize(tool, parameters, source): + """The same-aspect multi-size branch performs no model operation.""" + if tool != 'multi-size': + return False + width, height = source.get('width'), source.get('height') + target_width, target_height = parameters.get('outputWidth', width), parameters.get('outputHeight', height) + if not all(type(value) in (int, float) and math.isfinite(value) and value > 0 + for value in (width, height, target_width, target_height)): + return False + if not all(64 <= value <= 8192 for value in (target_width, target_height)): + return False + # The recipe validates numbers and converts them to integer pixel sizes. + return abs(int(target_width) / int(target_height) - width / height) < .01 + + +def image_execution(tool, parameters, source, *, has_mask, has_annotation, policy): + """Only choose native masking when the entire requested geometry applies.""" + width, height = source.get('width'), source.get('height') + result = {'mode': 'legacy', 'roles': ['image']} + if not isinstance(width, int) or not isinstance(height, int) or min(width, height) <= 0: + return result + reference_count = 0 + candidate = False + if tool == 'smart-edit': + mode = parameters.get('editMode', 'combined') + candidate = mode != 'annotation' and has_mask + reference_count = int(mode != 'selection' and has_annotation) + if reference_count: + result['roles'] = ['image', 'annotation'] if policy['maxImages'] > 1 else ['annotation'] + elif tool == 'inpainting-remove': + candidate = has_mask + elif tool == 'product-replacement': + candidate = True # The recipe builds one image and a rectangle mask. + edge = {'4K': 4096, '2K': 2048}.get(parameters.get('resolution')) + if edge and max(width, height) < edge: + candidate = False # Do not generate a smaller image then upscale it. + elif tool == 'outpaint': + sides = [parameters.get(key, 0) for key in ('left', 'top', 'right', 'bottom')] + if all(isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + and 0 <= value <= 4 for value in sides) and any(sides): + width, height = (width + round(sides[0] * width) + round(sides[2] * width), + height + round(sides[1] * height) + round(sides[3] * height)) + candidate = (parameters.get('outputWidth', width), parameters.get('outputHeight', height)) == (width, height) + elif tool == 'multi-size': + target_width, target_height = parameters.get('outputWidth', width), parameters.get('outputHeight', height) + if all(type(value) in (int, float) and math.isfinite(value) and 64 <= value <= 8192 + for value in (target_width, target_height)): + target_width, target_height = int(target_width), int(target_height) + target, current = target_width / target_height, width / height + if abs(target - current) >= .01: + width, height = ((round(height * target), height) if target > current + else (width, round(width / target))) + candidate = (target_width, target_height) == (width, height) + elif tool == 'panorama': + original = (width, height) + width = max(width, height * 2) + height = width // 2 + edge = {'4K': 4096, '2K': 2048}.get(parameters.get('resolution'), 2048) + candidate = (width, height) != original and (width, height) == (edge, edge // 2) + elif tool == 'image-presets': + from opendirector.application.tool_image_edits import PRESETS + preset = PRESETS.get(parameters.get('preset')) + if preset is not None: + width, height = preset.size + elif tool == 'image-hd' and max(width, height) > 1536: + scale = 1536 / max(width, height) + width, height = max(1, round(width * scale)), max(1, round(height * scale)) + result.update(width=width, height=height) + if candidate and native_mask_applicable(policy, width, height, image_count=1, reference_count=reference_count): + result.update(mode='native_mask', roles=['image', 'mask'] + (['reference'] if reference_count else [])) + return result + + +def image_request_parameters(tool, parameters, sources, plan, *, adapter, capability, policy): + """Project each recipe request through its actual adapter, before acceptance. + + A batch upscale submits one image per source. Preset variants share one + geometry. Product replacement submits the composed scene, not two images. + """ + if local_image_resize(tool, parameters, sources[0]): + return [] + strength = {'smart-edit': .78, 'inpainting-remove': .8, 'view-control': .8, 'lighting': .72, + 'product-replacement': .42, 'layer-decompose': .85}.get(tool) + if tool == 'image-hd': + detail = parameters.get('detailStrength', .5) + if type(detail) in (int, float) and math.isfinite(detail): + strength = .12 + .3 * detail + if tool in {'outpaint', 'multi-size', 'panorama'} and (getattr(adapter, 'strength', False) or plan['mode'] == 'native_mask'): + strength = .9 + plans = ([image_execution(tool, parameters, source, has_mask=False, has_annotation=False, policy=policy) + for source in sources] if tool == 'image-hd' else [plan]) + result = [] + for request in plans: + if 'width' not in request or 'height' not in request: + continue + result.append(adapter.image_parameters(request['width'], request['height'], + has_images=capability != 'image.generate', native_mask=request['mode'] == 'native_mask', strength=strength, + extra={'background': 'transparent'} if capability == 'image.cutout' else None)) + return result diff --git a/packages/core/src/opendirector/application/tool_tasks.py b/packages/core/src/opendirector/application/tool_tasks.py index 4194d41d..0eda469c 100644 --- a/packages/core/src/opendirector/application/tool_tasks.py +++ b/packages/core/src/opendirector/application/tool_tasks.py @@ -7,13 +7,18 @@ import asyncio import base64 +from dataclasses import replace from datetime import datetime, timezone from uuid import UUID, uuid5 from PIL import Image from opendirector.application.tool_operations import TOOLS, Source, ToolContext, select_image_objects +from opendirector.application.model_availability import ModelAvailability, tool_request +from opendirector.application.tool_image_inputs import image_execution, image_request_parameters, local_image_resize, validate_image_sources from opendirector.domain.capabilities import LABELS +from opendirector.domain.capability_messages import MissingToolModelError +from opendirector.domain.image_inputs import image_input_policy from opendirector.domain.generation import GenerationError from opendirector.domain.projects import ProjectError, identifier from opendirector.ports.gateway import gateway @@ -47,12 +52,13 @@ def output_item_id(task, index, kind): class ToolTasks(ToolTaskLifecycle): - def __init__(self, store, configs, assets, *, transport=None, poll_interval=3, concurrency=2, projects): + def __init__(self, store, configs, assets, *, transport=None, poll_interval=3, concurrency=2, projects, availability=None): self.store, self.configs, self.assets = store, configs, assets self.transport, self.poll_interval = transport, poll_interval self.jobs: dict[tuple[str, str], asyncio.Task] = {} self.limit = asyncio.Semaphore(concurrency) self.projects = projects + self.availability = availability or ModelAvailability(configs) self.accepting = set() self._init_lifecycle() @@ -72,24 +78,43 @@ def catalog(self): for spec in TOOLS.values(): needs = spec.requires({}) missing = [item for item in needs if item not in served] + # The panel must open before its chosen dimensions determine + # whether multi-size needs an image editor or only local resizing. + local_path = spec.id == 'multi-size' entry = {'id': spec.id, 'requires': [{'capability': item, 'label': LABELS[item]} for item in needs], - 'available': spec.recipe is not None and not missing, 'parametersConfirmed': spec.recipe is not None, + 'available': spec.recipe is not None and (local_path or not missing), 'parametersConfirmed': spec.recipe is not None, 'fields': list(spec.fields), 'note': spec.note, 'models': {item: [{'id': config['id'], 'label': _model_label(config)} for config in served.get(item, [])] for item in needs}} - if spec.recipe is None or missing: + for capability, choices in entry['models'].items(): + for choice in choices: + config = next(item for item in served[capability] if item['id'] == choice['id']) + private = self.configs.resolve(config['id'], config['kind']) + request = tool_request(private, capability) + if request: + choice['availability'] = self.availability.decision(private, request) + if not local_path and choices and all(item.get('availability', {}).get('status') == 'unavailable' for item in choices): + selected = next(item for item in served[capability] if item['id'] == choices[0]['id']) + private = self.configs.resolve(selected['id'], selected['kind']) + try: + self.availability.require(private, tool_request(private, capability)) + except ProjectError as error: + entry.update(available=False, reason=str(error), reasonCode=choices[0]['availability']['reasonCode']) + if spec.recipe is None or (missing and not local_path): entry['reason'] = f"需要接入具备「{'、'.join(LABELS[item] for item in missing or needs)}」能力的模型。" + entry.pop('reasonCode', None) tools.append(entry) selection = {'available': 'image.segment' in served} if not selection['available']: selection['reason'] = '接入图像分割模型后可自动识别选区；仍可手动画选区。' return {'capabilities': {item: [{'id': config['id'], 'label': _model_label(config)} for config in configs] - for item, configs in served.items()}, 'tools': tools, 'selectionPreview': selection} + for item, configs in served.items()}, 'tools': tools, 'selectionPreview': selection, + 'availabilityRevision': self.availability.list()['revision']} def _model(self, capability, chosen): candidates = self.providers().get(capability, []) public = next((item for item in candidates if item['id'] == chosen), None) if chosen else (candidates[0] if candidates else None) if public is None: - raise ProjectError(f'没有可用的「{LABELS[capability]}」模型，请在“模型接入”中添加。', 409) + raise MissingToolModelError(capability) config = self.configs.resolve(public['id'], public['kind']) return config, adapter_for(config, self.transport, self.poll_interval) @@ -117,6 +142,9 @@ async def progress(_): if self.providers().get('text.chat'): async def chat(request): chat_config, chat_adapter = self._model('text.chat', None) + prepared = await asyncio.to_thread(self.availability.prepare, chat_config) + current, _ = self._model('text.chat', chat_config['id']) + self.availability.require(current, tool_request(current, 'text.chat', actual=True), prepared) return await chat_adapter.chat(chat_config, request) context = ToolContext(sources=[], mask=None, boxes=[], parameters={}, image_model=None, video_model=None, @@ -217,9 +245,24 @@ def _prepare(self, project_id, value, session): # Snapshot validation and accepted ledger publication share the project # lock. A source cannot be replaced between checking and acceptance. with self.store.transaction(): - return self._prepare_locked(project_id, value, session) + initial = self._prepare_locked(project_id, value, session, inspect=True) + if isinstance(initial, tuple): + return initial # A replay does not query or reaccept its frozen job. + uses_mask = (value.get('tool') == 'inpainting-remove' or + value.get('tool') == 'smart-edit' and (value.get('parameters') or {}).get('editMode', 'combined') != 'annotation') + if uses_mask and value.get('maskAssetId'): + source = self.assets.assets.get(project_id, value['sourceAssetIds'][0]) + # Decode outside the storage transaction; freeze_inputs rechecks the + # immutable asset identities and matching dimensions before acceptance. + media.mask_from(self.assets.read_bytes(project_id, value['maskAssetId']), + (source.metadata.width, source.metadata.height)) + selected = {capability: config for capability, config in initial.items() + if tool_request(config, capability) is not None} + prepared = dict(zip(selected, self.availability.prepare_many(selected.values()), strict=True)) + with self.store.transaction(): + return self._prepare_locked(project_id, value, session, prepared=prepared) - def _prepare_locked(self, project_id, value, session): + def _prepare_locked(self, project_id, value, session, *, inspect=False, prepared=None): if self.closing: raise ProjectError('工具服务正在关闭，请稍后查询原任务。', 503) self._ensure_owner() @@ -253,37 +296,87 @@ def _prepare_locked(self, project_id, value, session): or existing.get('requestedModelConfigIds', existing.get('modelConfigIds', {})) != chosen): raise ProjectError('工具任务标识已使用。', 409) return existing, directory, None - for capability in needs: - self._model(capability, chosen.get(capability)) records = self.assets._records(directory, project_id) for asset_id in [*source_ids, *([mask_id] if mask_id else []), *([annotation_id] if annotation_id else [])]: if asset_id not in records: raise ProjectError('来源素材不存在，请重新上传后提交。', 409) + if spec.sources == ('image',): + validate_image_sources(spec.id, [records[asset_id] for asset_id in source_ids]) if records[source_ids[0]]['kind'] not in spec.sources: raise ProjectError('来源素材类型与工具不匹配。', 422) if mask_id and records[mask_id]['kind'] != 'image': raise ProjectError('选区必须是图片。', 422) if annotation_id and records[annotation_id]['kind'] != 'image': raise ProjectError('标注必须是图片。', 422) + if local_image_resize(spec.id, parameters, records[source_ids[0]]): + needs = () canvas = value.get('canvas') if canvas and (len(canvas['sourceNodeIds']) != len(source_ids) or canvas['sourceNodeIds'][0] != canvas['nodeId']): raise ProjectError('工具任务的画布来源不匹配。', 422) - from .creation_tool_drafts import freeze_inputs - frozen = freeze_inputs(self, project_id, value) models = {capability: self._model(capability, chosen.get(capability))[0] for capability in needs} - if self.providers().get('text.chat'): + if spec.id == 'layer-decompose' and self.providers().get('image.edit'): + models['image.edit'] = self._model('image.edit', chosen.get('image.edit'))[0] + if needs and self.providers().get('text.chat'): models['text.chat'] = self._model('text.chat', chosen.get('text.chat'))[0] + if inspect: + return models + image_policy, image_plan = None, None + image_capability = next((name for name in ('image.edit', 'image.cutout', 'image.generate') if name in models), None) + if image_capability is not None: + image_policy = (image_input_policy(models[image_capability]) if image_capability == 'image.generate' + else self.availability.image_execution_policy(models[image_capability])) + if image_capability != 'image.edit': + image_policy = {**image_policy, 'nativeMask': None} + image_plan = image_execution(spec.id, parameters, records[source_ids[0]], has_mask=bool(mask_id), + has_annotation=bool(annotation_id), policy=image_policy) + if image_capability == 'image.generate': + image_plan['roles'] = [] + for capability, config in list(models.items()): + request = tool_request(config, capability, actual=True) + if request is None: + continue + parameters_to_check = [{}] + if capability == image_capability: + counts = {'text': 1, 'image': sum(role in {'image', 'annotation'} for role in image_plan['roles'])} + if image_plan['mode'] == 'native_mask': + counts['mask_image'] = 1 + if 'reference' in image_plan['roles']: + counts['reference_image'] = 1 + request = replace(request, inputs=frozenset(counts), input_counts=tuple(counts.items())) + painter = adapter_for(config, image_policy=image_policy) + parameters_to_check = image_request_parameters(spec.id, parameters, [records[item] for item in source_ids], + image_plan, adapter=painter, capability=capability, policy=image_policy) or [{}] + contract = self.availability.request_constraints(config, request) + if contract is not None: + image_policy['requestConstraints'] = contract.model_dump(mode='json', exclude_none=True) + if capability not in (prepared or {}): + raise ProjectError('能力检查期间模型服务已变化，请重新提交。', 409, code='capability_connection_changed') + self.availability.assert_identity(config, prepared[capability]) + for actual in parameters_to_check: + actual_request = replace(request, parameters=tuple(actual.items())) if actual else request + if capability in {'text.chat', 'image.edit'} and capability not in needs: + if self.availability.decision(config, actual_request)['status'] == 'unavailable': + del models[capability] + break + else: + self.availability.require(config, actual_request) + from .creation_tool_drafts import freeze_inputs + frozen = freeze_inputs(self, project_id, value) task = {'schemaVersion': 1, 'id': task_id, 'projectId': project_id, 'tool': spec.id, 'status': 'queued', 'phase': '排队中', 'sourceAssetIds': source_ids, 'maskAssetId': mask_id, 'annotationAssetId': annotation_id, 'boxes': value.get('boxes') or [], 'parameters': parameters, 'modelConfigIds': {item: config['id'] for item, config in models.items()}, - 'modelSnapshots': {item: {key: val for key, val in config.items() if key != 'apiKey'} for item, config in models.items()}, 'requestedModelConfigIds': chosen, 'requires': list(needs), + 'modelSnapshots': {item: {key: val for key, val in config.items() if key not in {'apiKey', 'capabilityAutoCheck', 'availability'}} for item, config in models.items()}, 'requestedModelConfigIds': chosen, 'requires': list(needs), 'outputs': [], 'error': None, 'createdAt': _now(), 'finishedAt': None, 'revision': 1, 'workerId': self.worker_id, 'localState': 'active', 'cancelRequested': False, 'modelCalls': [], 'canvas': canvas, 'sourceFingerprint': records[source_ids[0]]['fingerprint'], 'sourceKind': records[source_ids[0]]['kind'], 'placement': 'pending'} task.update(frozen) + if spec.id == 'layer-decompose': + task['backgroundCompletion'] = 'model' if 'image.edit' in models else 'transparent' + if image_policy is not None and image_capability in models: + task.update(imageExecutionPolicy=image_policy, imageExecution=image_plan) self._write(directory, task) return task, directory, records @@ -345,13 +438,22 @@ async def progress(message): await self._update(directory, task, phase=message) def frozen_model(capability): + if capability == 'image.edit' and task.get('backgroundCompletion') == 'transparent': + # This task accepted a transparent background. A model + # added later cannot turn it into a new remote request. + raise ProjectError('本次拆分保留透明背景。', 409) config, adapter = self._model(capability, task['modelConfigIds'].get(capability)) frozen = task.get('modelSnapshots', {}).get(capability) if frozen and any(config.get(key) != frozen.get(key) for key in ('id', 'modelId', 'url', 'protocol', 'connectionType', 'videoAdapter')): raise ProjectError('已接受工具任务的模型配置已变化，请新建尝试。', 409) # Runtime credentials remain separate; execution options # come from acceptance, not later configuration edits. - return ({**frozen, 'apiKey': config.get('apiKey', '')} if frozen else config), adapter + config = {**frozen, 'apiKey': config.get('apiKey', '')} if frozen else config + if capability in {'image.edit', 'image.generate', 'image.cutout'}: + adapter = adapter_for(config, self.transport, self.poll_interval, + image_policy=task.get('imageExecutionPolicy'), + legacy_image_policy=task.get('imageExecutionPolicy') is None) + return config, adapter async def image_model(request): config, adapter = frozen_model('image.edit' if request.image is not None else 'image.generate') @@ -412,6 +514,7 @@ async def chat_model(request): edit_model=edit_model, edit_seconds=limits, chat_model=chat_model, segment_model=segment_model, track_model=track_model, edit_keeps_layout=keeps_layout, annotation=annotation, edit_max_images=edit_max_images, + native_mask=task.get('imageExecutionPolicy') is not None and task.get('imageExecution', {}).get('mode') == 'native_mask', segment_available='image.segment' in task['requires'], depth_model=depth_model) outputs = await spec.recipe(context) # Preserve already-produced bytes even if cancellation arrives diff --git a/packages/core/src/opendirector/cli.py b/packages/core/src/opendirector/cli.py index 2b66ec8a..9562a947 100644 --- a/packages/core/src/opendirector/cli.py +++ b/packages/core/src/opendirector/cli.py @@ -349,6 +349,26 @@ def select_model(models, selector): return matches[0] +@group('model').command('availability', help='Read model operation availability; optionally refresh declarations without running models.') +def model_availability(ctx: typer.Context, + refresh: Annotated[bool, typer.Option(help='Query configured services using automatic-check preferences.')] = False, + model: Annotated[list[UUID] | None, typer.Option(help='Restrict to saved model configuration UUIDs.')] = None, + manual: Annotated[bool, typer.Option(help='Explicitly check selected models even when automatic checks are disabled. Requires --refresh and --model.')] = False): + client = ctx.obj + client.command = ctx.command_path + if manual and (not refresh or not model): + raise typer.BadParameter('--manual requires --refresh and --model') + arguments = {'manual': manual} + if model: + arguments['configIds'] = [str(item) for item in model] + if client.live: + emit(client.call('model_availability', {**arguments, 'refresh': refresh})) + elif refresh: + emit(client.request('POST', '/api/model-configs/availability', json=arguments)) + else: + emit(client.request('GET', '/api/model-configs/availability')) + + @group("model").command("list", help="Find model configurations by media kind or model name.") def model_list(ctx: typer.Context, kind: Annotated[Literal["agent", "image", "video", "audio", "vision"] | None, diff --git a/packages/core/src/opendirector/domain/capabilities.py b/packages/core/src/opendirector/domain/capabilities.py index 924e598c..bae07f9f 100644 --- a/packages/core/src/opendirector/domain/capabilities.py +++ b/packages/core/src/opendirector/domain/capabilities.py @@ -46,6 +46,11 @@ class ImageRequest: seed: int | None = None # Further condition images (style, product, character) after the main image. references: tuple[bytes, ...] = () + # Canonical selection: grayscale PNG, white edits and black preserves. + # It is an auxiliary input, never another ordinary condition image. + mask: bytes | None = None + # A separately declared reference_image role, distinct from repeated image parts. + reference: bytes | None = None @dataclass(frozen=True) diff --git a/packages/core/src/opendirector/domain/capability_messages.py b/packages/core/src/opendirector/domain/capability_messages.py new file mode 100644 index 00000000..dca29de4 --- /dev/null +++ b/packages/core/src/opendirector/domain/capability_messages.py @@ -0,0 +1,107 @@ +"""Translations for concrete capability failures, preserving their input detail.""" +import re + +from opendirector.domain.capabilities import Capability, LABELS +from opendirector.domain.projects import ProjectError + + +_TOOL_OPERATIONS = { + 'text.chat': 'chat', 'image.generate': 'text-to-image generation', 'image.edit': 'image editing', + 'image.cutout': 'subject cutout', 'video.generate': 'text-to-video generation', + 'video.first_frame': 'first-frame video generation', 'video.edit': 'video inpainting', + 'audio.speech': 'speech synthesis', 'audio.music': 'music generation', 'audio.score': 'music transcription', + 'image.segment': 'image segmentation', 'video.matting': 'video matting', + 'video.motion_capture': 'depth motion capture', 'audio.separate': 'audio separation', +} + + +def missing_tool_model_message(capability: Capability | None, *, english=False): + if english: + operation = _TOOL_OPERATIONS.get(capability, 'suitable') + return f'No {operation} model is available. Add one in Model connections.' + return f'没有可用的「{LABELS[capability]}」模型，请在“模型接入”中添加。' + + +class MissingToolModelError(ProjectError): + """Keep the required operation separate from its presentation language.""" + def __init__(self, capability: Capability): + super().__init__(missing_tool_model_message(capability), 409, code='tool_model_missing') + self.capability = capability + + +_MESSAGES = { + '服务已暂停，请恢复后重试。': 'Service paused. Resume it and try again.', + '请补充所需素材。': 'Add the required inputs.', + '图片文件过大，请压缩后重试。': 'The image is too large. Compress it and try again.', + '图片总大小超出限制，请减少图片或压缩后重试。': 'The images exceed the total size limit. Remove or compress images.', + '标注参考图片需要配合选区使用。': 'Add a selection to use the annotation reference.', + '当前图片尺寸或数量不支持选区输入。': 'The image dimensions or count do not support selection input.', + '选区与图片的尺寸不一致。': 'The selection and image dimensions must match.', + '请先选择需要修改的区域。': 'Select the area to edit.', + '无法读取图片或选区，请重新选择。': 'Cannot read the image or selection. Select it again.', + '来源素材必须是图片。': 'Source assets must be images.', +} +_OPERATIONS = {'文字回复': 'text replies', '图片回复': 'image replies', '图片生成': 'image generation', + '图片编辑': 'image editing', '文字生成视频': 'text-to-video generation', '图片生成视频': 'image-to-video generation', + '视频编辑': 'video editing', '语音合成': 'speech synthesis', '音乐生成': 'music generation', '此操作': 'this operation'} +_INPUTS = {'图片': 'images', '音频': 'audio', '视频': 'video', '文字': 'text', '音色': 'a voice', + '参考音频': 'reference audio', '风格说明': 'a style description', '参考文本': 'a reference transcript', + '选区': 'a selection', '标注参考图片': 'an annotation reference', '素材': 'inputs'} +_PARAMETERS = {'图片尺寸': 'Image size', '图片宽度': 'Image width', '图片高度': 'Image height', '时长': 'Duration', '音色': 'Voice', + '语速': 'Speech speed', '语言': 'Language', '帧率': 'Frame rate', '帧数': 'Frame count', + '音频生成选项': 'Audio generation setting', '所选参数': 'The selected parameter'} +for _label, _english in _OPERATIONS.items(): + _MESSAGES[f'该模型不支持{_label}。'] = f'This model does not support {_english}.' + _MESSAGES[f'服务未开放{_label}。'] = f'The service has not enabled {_english}.' +for _label, _english in _INPUTS.items(): + _MESSAGES[f'该模型不支持{_label}输入。'] = f'This model does not accept {_english}.' + _MESSAGES[f'请添加{_label}。'] = f'Add {_english}.' +for _label, _english in _PARAMETERS.items(): + _MESSAGES[f'{_label}不符合服务要求。'] = f'{_english} does not meet the service requirements.' +_MESSAGES['图片尺寸不符合服务要求。'] = 'Image dimensions do not meet the service requirements.' + + +def constraint_rejection(reason, details): + """The same specific diagnostic for admission and frozen execution checks.""" + code = 'capability_unavailable' + roles = {'image': '图片', 'audio': '音频', 'video': '视频', 'text': '文字', + 'voice': '音色', 'reference_audio': '参考音频', 'instructions': '风格说明', + 'reference_text': '参考文本', 'mask_image': '选区', 'reference_image': '标注参考图片'} + name = roles.get(details.get('input'), '素材') + if reason == 'input_count_exceeded': + if details['input'] == 'image': + return 'image_input_count', f"最多支持 {details['maximum']} 张图片，当前为 {details['actual']} 张。" + message = f"最多支持 {details['maximum']} 个{name}输入，当前为 {details['actual']} 个。" + elif reason == 'input_unsupported': + message = f'该模型不支持{name}输入。' + elif reason == 'required_inputs_missing': + if 'inputs' in details: + choices = '或'.join(roles.get(role, '素材') for role in details['inputs']) + message = f'请提供{choices}。' + else: + message = f'请添加{name}。' + else: + parameter = {'size': '图片尺寸', 'width': '图片宽度', 'height': '图片高度', 'duration': '时长', 'seconds': '时长', + 'voice': '音色', 'speed': '语速', 'language': '语言', 'fps': '帧率', + 'num_frames': '帧数', 'generate_audio': '音频生成选项'}.get(details.get('parameter'), '所选参数') + message = f'{parameter}不符合服务要求。' + return code, message + + +def english_capability_message(message): + if message in _MESSAGES: + return _MESSAGES[message] + count = re.fullmatch(r'最多支持 (\d+) 张图片，当前为 (\d+) 张。', message) + if count: + return f'Up to {count[1]} images; {count[2]} added.' + count = re.fullmatch(r'该工具需要 (\d+) 张来源图片。', message) + if count: + return f'This tool requires {count[1]} source images.' + count = re.fullmatch(r'最多支持 (\d+) 个(.+)输入，当前为 (\d+) 个。', message) + if count and count[2] in _INPUTS: + return f'Up to {count[1]} {_INPUTS[count[2]]} inputs; {count[3]} added.' + if message.startswith('请提供') and message.endswith('。'): + roles = message[3:-1].split('或') + if all(role in _INPUTS for role in roles): + return 'Provide ' + ' or '.join(_INPUTS[role] for role in roles) + '.' + return None diff --git a/packages/core/src/opendirector/domain/creation/commands.py b/packages/core/src/opendirector/domain/creation/commands.py index d712a054..bee6b2a4 100644 --- a/packages/core/src/opendirector/domain/creation/commands.py +++ b/packages/core/src/opendirector/domain/creation/commands.py @@ -398,6 +398,7 @@ def command_fingerprint(command: Command) -> str: IssueCode = Literal[TimelineIssueCode, 'invalid_input', 'revision_conflict', 'role_restricted', 'missing_object', 'reference_invalid', 'kling_model', 'kling_mode', 'kling_image', 'kling_frames', 'kling_prompt', 'kling_aspect', 'kling_output', 'kling_cancel', + 'capability_unavailable', 'capability_paused', 'capability_connection_changed', 'capability_unsupported', 'preview_expired', 'storage_unavailable', 'submission_unknown', 'provider_error', 'moss_speech_text', 'moss_speech_voice', 'moss_speech_reference', 'moss_prompt_limit', 'moss_image_size', 'moss_image_reference', 'moss_layout_reference', diff --git a/packages/core/src/opendirector/domain/creation/generation.py b/packages/core/src/opendirector/domain/creation/generation.py index 4040b3b0..d484d9d1 100644 --- a/packages/core/src/opendirector/domain/creation/generation.py +++ b/packages/core/src/opendirector/domain/creation/generation.py @@ -8,6 +8,7 @@ from pydantic import Field, JsonValue, model_validator from opendirector.domain.minimax_models import MINIMAX_MODELS +from opendirector.domain.projects import ProjectError from opendirector.domain.generation import GenerationFailure, ResultCheck from opendirector.domain.rule_facts import ( FASTH3_MEDIA_REFERENCE_LIMIT, FRAME_IMAGE_LIMIT, LYRICS_MAX_LENGTH, MAX_REFERENCES, @@ -401,6 +402,8 @@ def validate_generation(spec: GenerationSpec, model: ModelConfigPublic, referenc count = sum(ref.asset.kind == kind for ref in transmitted) if count > limit: label = {'image': '图片', 'video': '视频', 'audio': '音频'}[kind] + if kind == 'image': + raise ProjectError(f'最多支持 {limit} 张图片，当前为 {count} 张。', code='image_input_count') raise ValueError(f'{label}参考素材数量超过上限。') if spec.kind == 'image': from opendirector.domain.image_output import resolve_image_output diff --git a/packages/core/src/opendirector/domain/creation/models.py b/packages/core/src/opendirector/domain/creation/models.py index 937a2d0f..29a26880 100644 --- a/packages/core/src/opendirector/domain/creation/models.py +++ b/packages/core/src/opendirector/domain/creation/models.py @@ -141,16 +141,84 @@ def connection(self): return self +class CapabilityDeclaration(Contract): + status: Literal['available', 'unavailable', 'unknown'] = Field(description='Support decision excluding execution state.') + reasonCode: str = Field(description='Localizable declaration decision.') + checkedAt: int | None = Field(description='Completion time in Unix milliseconds.') + expiresAt: int | None = Field(description='Declaration expiry in Unix milliseconds.') + + +class CapabilityExecution(Contract): + state: Literal['unpaused', 'paused', 'unknown'] = Field(description='Observed execution pause state.') + scope: Literal['deployment', 'worker'] | None = Field(description='Scope proved by the service observation.') + checkedAt: int | None = Field(description='Observation receipt time in Unix milliseconds.') + expiresAt: int | None = Field(description='Execution observation expiry in Unix milliseconds.') + + +class CapabilityInputDetail(Contract): + name: str = Field(description='Canonical input role or parameter name.') + support: Literal['supported', 'unsupported', 'unknown'] = Field(description='Explicit fact, with omission kept unknown.') + source: Literal['service', 'application'] = Field(default='service', description='Owner of this restriction.') + minCount: int | None = Field(default=None, description='Minimum transmitted inputs of this role.') + maxCount: int | None = Field(default=None, description='Maximum transmitted inputs of this role.') + values: tuple[str | int | float | bool, ...] | None = Field(default=None, description='Explicit allowed parameter values.') + min: float | None = Field(default=None, description='Inclusive parameter lower bound.') + max: float | None = Field(default=None, description='Inclusive parameter upper bound.') + multipleOf: float | None = Field(default=None, description='Parameter step from zero.') + + +class OperationAvailability(Contract): + operation: str = Field(description='Canonical remote operation identifier.') + profile: str = Field(description='Input and output mode used by the configured adapter.') + method: str = Field(description='Actual HTTP method used for submission.') + path: str = Field(description='Actual HTTP submission path.') + output: str = Field(description='Output modality consumed by the adapter.') + status: Literal['available', 'unavailable', 'unknown'] = Field(description='Current evidence for this exact request.') + reasonCode: str = Field(description='Stable, localizable decision reason.') + checkedAt: int | None = Field(description='Observation completion time in Unix milliseconds.') + expiresAt: int | None = Field(description='Decision expiry in Unix milliseconds; recompute after expiry.') + declaration: CapabilityDeclaration | None = Field(default=None, exclude_if=lambda value: value is None, + description='Request support independently expires from the short execution observation.') + execution: CapabilityExecution | None = Field(default=None, exclude_if=lambda value: value is None, + description='Short-lived execution observation, separate from declared support.') + inputDetails: tuple[CapabilityInputDetail, ...] = Field(default=(), + description='Known role and parameter facts for the actual request route; no saved configuration changes.') + + +class ModelAvailability(Contract): + configId: UUID | None = Field(description='Saved model identity; null for an unsaved draft.') + modelId: str = Field(description='Exact remote model identifier requested.') + identity: str = Field(description='Opaque identity of the checked connection and model.') + autoCheck: bool = Field(description='Effective service automatic-check preference.') + status: Literal['available', 'unavailable', 'unknown'] = Field(description='Summary across the configured adapter operations.') + reasonCode: str = Field(description='Stable, localizable summary reason.') + checkedAt: int | None = Field(description='Observation completion time in Unix milliseconds.') + expiresAt: int | None = Field(description='Earliest operation expiry in Unix milliseconds.') + revision: str = Field(description='Projection version; changes on observations, identity changes and expiration.') + operations: tuple[OperationAvailability, ...] = Field(description='Exact operations consumed by the configured adapter.') + + +class AvailabilityCatalog(Contract): + revision: str = Field(description='Version of the complete observation projection.') + models: tuple[ModelAvailability, ...] = Field(description='Current saved model observations; reading does not query providers.') + + class ModelConfigPublic(ModelConnection): """Application-level, never embedded in an editable project's credential data.""" id: UUID = Field(description='Stable object ID.') providerId: UUID | None = Field(default=None, exclude=True, description='Shared provider connection ID.') inputRules: InputRules = Field(description='Verified model input limits and output choices.') + capabilityAutoCheck: bool | None = Field(default=None, strict=True, exclude_if=lambda value: value is None, + description='Service preference; omitted defaults to automatic for self-hosted connections.') + availability: ModelAvailability | None = Field(default=None, exclude_if=lambda value: value is None, + description='Ephemeral observation, never persisted in project snapshots.') class ModelConfigWrite(ModelConnection): """Only the model-settings write boundary accepts a credential.""" apiKey: SecretStr | None = Field(default=None, max_length=8192, description='Provider credential; omitted from public model records.') + capabilityAutoCheck: bool | None = Field(default=None, strict=True, exclude_if=lambda value: value is None, + description='Service automatic-check override; omission preserves the existing preference.') @field_validator('apiKey') @classmethod diff --git a/packages/core/src/opendirector/domain/image_inputs.py b/packages/core/src/opendirector/domain/image_inputs.py new file mode 100644 index 00000000..70e1aea9 --- /dev/null +++ b/packages/core/src/opendirector/domain/image_inputs.py @@ -0,0 +1,90 @@ +"""Shared image request limits; execution consumes the policy frozen at acceptance.""" +from io import BytesIO +from math import isclose +from typing import Any, Literal, NotRequired, TypedDict + +from PIL import Image, UnidentifiedImageError + +from opendirector.domain.input_rules import input_rules +from opendirector.domain.projects import ProjectError + + +class ImageExecutionPolicy(TypedDict): + """JSON persisted with accepted image work; missing limits stay unknown. + + requestConstraints is the serialized, validated RequestConstraints for the + exact route. It contains no discovery timestamps or connection state. + """ + version: Literal[1] + maxImages: int + imageBytes: int + requestBytes: int + nativeMask: dict[str, Any] | None + requestConstraints: NotRequired[dict[str, Any]] + + +class ImageInputError(ProjectError): + def __init__(self, code, message): + super().__init__(message) + self.code = code + + +def image_input_policy(config) -> ImageExecutionPolicy: + rules = input_rules(config) + return {'version': 1, 'maxImages': rules['maxImages'], 'imageBytes': rules['imageBytes'], + 'requestBytes': rules['requestBytes'], 'nativeMask': None} + + +def native_mask_applicable(policy, width, height, image_count=1, reference_count=0): + """Native selection never resizes the requested canvas to satisfy a model.""" + mask = policy.get('nativeMask') or {} + required = ('widthMultiple', 'heightMultiple', 'minWidth', 'minHeight', 'minImages', 'maxImages', 'maxReferenceImages') + if (mask.get('field') != 'mask_image' or mask.get('encoding') != 'grayscale_white_edit' + or mask.get('sameSize') is not True or any(type(mask.get(key)) is not int for key in required)): + return False + if (type(width) is not int or type(height) is not int + or mask['widthMultiple'] <= 0 or mask['heightMultiple'] <= 0): + return False + maximum = mask.get('maxPixels') + for dimension, key in ((width, 'width'), (height, 'height')): + allowed, step = mask.get(key + 'Values'), mask.get(key + 'Step') + if allowed is not None and dimension not in allowed: + return False + if step is not None and (step <= 0 or not isclose(dimension / step, round(dimension / step), abs_tol=1e-8)): + return False + return (width >= mask['minWidth'] and height >= mask['minHeight'] + and (mask.get('maxWidth') is None or width <= mask['maxWidth']) + and (mask.get('maxHeight') is None or height <= mask['maxHeight']) + and width % mask['widthMultiple'] == 0 and height % mask['heightMultiple'] == 0 + and mask['minImages'] <= image_count <= mask['maxImages'] + and image_count <= policy.get('maxImages', 0) + and mask.get('minReferenceImages', 0) <= reference_count <= mask['maxReferenceImages'] + and (not reference_count or mask.get('referenceField') == 'reference_image') + and (maximum is None or (type(maximum) is int and width * height <= maximum))) + + +def validate_image_request(request, policy): + """Validate actual wire roles, without slicing or modifying caller input.""" + images = ([request.image] if request.image is not None else []) + list(request.references) + mask, reference = getattr(request, 'mask', None), getattr(request, 'reference', None) + if len(images) > policy['maxImages']: + raise ImageInputError('image_input_count', f"最多支持 {policy['maxImages']} 张图片，当前为 {len(images)} 张。") + files = images + ([mask] if mask is not None else []) + ([reference] if reference is not None else []) + if any(len(data) > policy['imageBytes'] for data in files): + raise ImageInputError('image_input_bytes', '图片文件过大，请压缩后重试。') + if sum(map(len, files)) > policy['requestBytes']: + raise ImageInputError('image_input_bytes', '图片总大小超出限制，请减少图片或压缩后重试。') + if reference is not None and mask is None: + raise ImageInputError('image_input_mask', '标注参考图片需要配合选区使用。') + if mask is None: + return + if not native_mask_applicable(policy, request.width, request.height, len(images), int(reference is not None)): + raise ImageInputError('image_input_mask', '当前图片尺寸或数量不支持选区输入。') + try: + with Image.open(BytesIO(mask)) as selection, Image.open(BytesIO(images[0])) as source: + if selection.mode != 'L' or selection.size != source.size or source.size != (request.width, request.height): + raise ImageInputError('image_input_mask', '选区与图片的尺寸不一致。') + if not selection.getbbox(): + raise ImageInputError('image_input_mask', '请先选择需要修改的区域。') + except (UnidentifiedImageError, OSError, Image.DecompressionBombError): + raise ImageInputError('image_input_mask', '无法读取图片或选区，请重新选择。') from None diff --git a/packages/core/src/opendirector/domain/input_rules.py b/packages/core/src/opendirector/domain/input_rules.py index 80532a38..92b09a5e 100644 --- a/packages/core/src/opendirector/domain/input_rules.py +++ b/packages/core/src/opendirector/domain/input_rules.py @@ -110,9 +110,8 @@ def _adapter_rules(config): rules.update(profile='vllm-omni-qwen-image', allowedKinds=['text', 'image'], maxImages=4, imageBytes=30 * 1024 * 1024, requestBytes=64 * 1024 * 1024) if kind == 'image' and model.rsplit('/', 1)[-1].lower() == 'z-image-turbo': - # Self-hosted vLLM-Omni text-to-image: it draws from the prompt alone, so - # documents are inlined as text and reference images are not accepted. - rules.update(profile='vllm-omni-image', allowedKinds=['text'], maxImages=0) + # The Omni edits endpoint accepts one source image and strength. + rules.update(profile='vllm-omni-image', allowedKinds=['text', 'image'], maxImages=1) if kind == 'image' and config.get('protocol') in {None, 'openai'} and name.startswith(('gpt-image', 'chatgpt-image')): # OpenAI gpt-image (and the chatgpt-image-* aliases on the same Images API): # documented text-to-image plus reference editing with at diff --git a/packages/core/src/opendirector/domain/omni_capabilities.py b/packages/core/src/opendirector/domain/omni_capabilities.py new file mode 100644 index 00000000..ce81ff42 --- /dev/null +++ b/packages/core/src/opendirector/domain/omni_capabilities.py @@ -0,0 +1,161 @@ +"""The optional Omni discovery document. Missing declarations remain unknown.""" +from datetime import datetime +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +Support = Literal['supported', 'unsupported', 'unknown'] +Exposure = Literal['enabled', 'disabled', 'unknown'] + + +class Declaration(BaseModel): + model_config = ConfigDict(extra='ignore', strict=True) + + +Number = Annotated[int | float, Field(allow_inf_nan=False)] + + +class InputConstraint(Declaration): + support: Support = 'unknown' + min_count: int | None = Field(default=None, ge=0) + max_count: int | None = Field(default=None, ge=0) + + @model_validator(mode='after') + def bounds(self): + if self.min_count is not None and self.max_count is not None and self.min_count > self.max_count: + raise ValueError('inverted input bounds') + return self + + +class ParameterConstraint(Declaration): + support: Support = 'unknown' + type: Literal['string', 'integer', 'number', 'boolean'] | None = None + enum: list[str | int | float | bool] | None = Field(default=None, max_length=1024) + enum_normalization: Literal['lower', 'title', 'strip_lower'] | None = None + minimum: Number | None = None + maximum: Number | None = None + step: Number | None = Field(default=None, gt=0) + + @model_validator(mode='after') + def bounds(self): + if self.enum_normalization is not None and ( + self.type != 'string' or self.enum is None or not all(isinstance(value, str) for value in self.enum)): + raise ValueError('enum normalization requires a string enum') + if self.minimum is not None and self.maximum is not None and self.minimum > self.maximum: + raise ValueError('inverted parameter bounds') + return self + + +class MaskGeometry(Declaration): + state: Literal['known', 'unknown'] = 'unknown' + multiple_of: int | None = Field(default=None, ge=1) + + +class SourceResize(Declaration): + max_pixels: int | None = Field(default=None, ge=1) + policy: str | None = None + + +class MaskConstraint(Declaration): + support: Support = 'unknown' + source_role: str | None = None + max_source_count: int | None = Field(default=None, ge=1) + white: str | None = None + black: str | None = None + binarize_threshold: Number | None = Field(default=None, ge=0, le=1) + requires_same_size: bool | None = None + geometry: MaskGeometry = Field(default_factory=MaskGeometry) + source_resize: SourceResize | None = None + + +class RequestConstraints(Declaration): + inputs: dict[str, InputConstraint] = Field(default_factory=dict, max_length=64) + required_any_of: list[list[str]] = Field(default_factory=list, max_length=32) + parameters: dict[str, ParameterConstraint] = Field(default_factory=dict, max_length=128) + mask: MaskConstraint | None = None + max_output_pixels: int | None = Field(default=None, ge=1) + + @model_validator(mode='after') + def groups(self): + if any(not group or len(group) > 32 or len(set(group)) != len(group) for group in self.required_any_of): + raise ValueError('invalid alternative input group') + return self + + +class Route(Declaration): + method: str = Field(max_length=16) + path: str = Field(max_length=2048) + transport: str = 'http' + exposure: Exposure = 'unknown' + request: RequestConstraints | None = None + + +class Profile(Declaration): + id: str = Field(min_length=1, max_length=100) + support: Support = 'unknown' + required_inputs: list[str] = Field(default_factory=list, max_length=32) + outputs: list[str] = Field(default_factory=list, max_length=32) + routes: list[Route] = Field(default_factory=list, max_length=32) + reason_code: str = Field(default='declaration_incomplete', max_length=200) + request: RequestConstraints | None = None + + @model_validator(mode='after') + def unique_routes(self): + check_routes(self.routes) + return self + + +class Execution(Declaration): + state: Literal['unpaused', 'paused', 'unknown'] = 'unknown' + scope: Literal['deployment', 'worker'] | None = None + observed_at: str | None = Field(default=None, max_length=100) + + +class Operation(Declaration): + support: Support = 'unknown' + exposure: Exposure = 'unknown' + reason_code: str = Field(default='declaration_incomplete', max_length=200) + profiles: list[Profile] = Field(default_factory=list, max_length=64) + execution: Execution = Field(default_factory=Execution) + + @model_validator(mode='after') + def unique_profiles(self): + if len({item.id for item in self.profiles}) != len(self.profiles): + raise ValueError('duplicate profiles') + if self.support == 'unsupported' and any(item.support == 'supported' for item in self.profiles): + raise ValueError('unsupported operation has a supported profile') + return self + + +class Model(Declaration): + id: str = Field(min_length=1, max_length=200) + stage_ids: list[str] = Field(default_factory=list, max_length=256) + operations: dict[str, Operation] = Field(default_factory=dict, max_length=64) + + +class OmniCapabilities(Declaration): + object: Literal['omni.capabilities'] + schema_version: Literal[1] + observed_at: str = Field(max_length=100) + models: list[Model] = Field(max_length=512) + routes: list[Route] = Field(default_factory=list, max_length=256) + + @model_validator(mode='after') + def unique_models(self): + if len({item.id for item in self.models}) != len(self.models): + raise ValueError('duplicate models') + timestamp(self.observed_at) + check_routes(self.routes) + return self + + +def check_routes(routes): + if len({(item.transport, item.method, item.path) for item in routes}) != len(routes): + raise ValueError('duplicate routes') + + +def timestamp(value): + value = datetime.fromisoformat(value.replace('Z', '+00:00')) + if value.tzinfo is None: + raise ValueError('timestamp requires a timezone') + return value.timestamp() diff --git a/packages/core/src/opendirector/domain/request_constraints.py b/packages/core/src/opendirector/domain/request_constraints.py new file mode 100644 index 00000000..603ef916 --- /dev/null +++ b/packages/core/src/opendirector/domain/request_constraints.py @@ -0,0 +1,77 @@ +"""Check explicit constraints against transmitted roles and parameters.""" +from math import isclose + +from opendirector.domain.omni_capabilities import RequestConstraints + + +INPUT_ROLES = frozenset({'text', 'image', 'video', 'audio', 'instructions', 'voice', + 'reference_audio', 'reference_text', 'reference_embedding', 'speaker_embedding', 'mask_image', 'reference_image', + 'first_frame', 'last_frame'}) + + +def _enum_value(value, normalization): + if not isinstance(value, str) or normalization is None: + return value + if normalization == 'strip_lower': + return value.strip().lower() + return value.lower() if normalization == 'lower' else value.title() + + +def constraint_failure(contract: RequestConstraints, counts: dict[str, int] | None, parameters: dict): + """Return a localizable cause and arguments. Missing facts impose no limit.""" + if counts is not None: + for role, constraint in contract.inputs.items(): + if role not in INPUT_ROLES: + continue + count = counts.get(role, 0) + if count and constraint.support == 'unsupported': + return 'input_unsupported', {'input': role} + if constraint.min_count is not None and count < constraint.min_count: + return 'required_inputs_missing', {'input': role, 'minimum': constraint.min_count, 'actual': count} + if constraint.max_count is not None and count > constraint.max_count: + return 'input_count_exceeded', {'input': role, 'maximum': constraint.max_count, 'actual': count} + for group in contract.required_any_of: + if set(group) <= INPUT_ROLES and not any(counts.get(role, 0) for role in group): + return 'required_inputs_missing', {'inputs': group} + width, height = parameters.get('width'), parameters.get('height') + if (contract.max_output_pixels is not None and type(width) is int and type(height) is int + and width * height > contract.max_output_pixels): + return 'parameter_value_unsupported', {'parameter': 'size'} + for name, value in parameters.items(): + constraint = contract.parameters.get(name) + if constraint is None or value is None: + continue + if constraint.support == 'unsupported': + return 'parameter_unsupported', {'parameter': name} + if constraint.enum is not None and not any( + isinstance(value, bool) == isinstance(option, bool) + and _enum_value(value, constraint.enum_normalization) == _enum_value(option, constraint.enum_normalization) + for option in constraint.enum): + return 'parameter_value_unsupported', {'parameter': name} + numeric = type(value) in {int, float} + if ((constraint.type == 'integer' and type(value) is not int) + or (constraint.type == 'number' and not numeric) + or (constraint.type == 'boolean' and type(value) is not bool) + or (constraint.type == 'string' and not isinstance(value, str))): + return 'parameter_value_unsupported', {'parameter': name} + if numeric: + if ((constraint.minimum is not None and value < constraint.minimum) + or (constraint.maximum is not None and value > constraint.maximum) + or (constraint.step is not None and not isclose(value / constraint.step, round(value / constraint.step), abs_tol=1e-8))): + return 'parameter_value_unsupported', {'parameter': name} + return None + + +def input_details(contract): + if contract is None: + return [] + details = [dict(name=name, support=value.support, source='service', minCount=value.min_count, + maxCount=value.max_count) for name, value in contract.inputs.items()] + details.extend(dict(name=name, support=value.support, source='service', + values=([option for option in value.enum if option != ''] + if name == 'voice' and value.enum is not None else value.enum), + min=value.minimum, max=value.maximum, multipleOf=value.step) + for name, value in contract.parameters.items()) + if contract.max_output_pixels is not None: + details.append(dict(name='output_pixels', support='supported', source='service', max=contract.max_output_pixels)) + return details diff --git a/packages/core/src/opendirector/domain/voices.py b/packages/core/src/opendirector/domain/voices.py index 791bcd53..81186414 100644 --- a/packages/core/src/opendirector/domain/voices.py +++ b/packages/core/src/opendirector/domain/voices.py @@ -35,6 +35,11 @@ } +def effective_speech_voice(config, voice): + """The voice value sent by the OpenAI-compatible speech adapter.""" + return voice or ('default' if config.get('connectionType') == 'self_hosted' else 'alloy') + + def voice_error(code, status=422): return ProjectError(VOICE_MESSAGES[code], status, code=code) diff --git a/packages/core/src/opendirector/infrastructure/model_configs.py b/packages/core/src/opendirector/infrastructure/model_configs.py index c928272d..9494704c 100644 --- a/packages/core/src/opendirector/infrastructure/model_configs.py +++ b/packages/core/src/opendirector/infrastructure/model_configs.py @@ -39,7 +39,8 @@ def _read(self) -> list[dict]: if value["version"] != 1 or not isinstance(value["models"], list): raise ValueError() services = value.get('services', {}) - if not isinstance(services, dict) or any(set(service) != {'url', 'apiKey', 'connectionType', 'authMode'} for service in services.values()): + if not isinstance(services, dict) or any(not {'url', 'apiKey', 'connectionType', 'authMode'} <= set(service) + or set(service) - {'url', 'apiKey', 'connectionType', 'authMode', 'capabilityAutoCheck'} for service in services.values()): raise ValueError() for item in value['models']: if 'providerId' in item: @@ -47,7 +48,9 @@ def _read(self) -> list[dict]: item.update(services[item['providerId']]) seen = set() for item in value["models"]: - if not {'id', 'kind', 'url', 'modelId', 'apiKey'} <= set(item) or set(item) - {'id', 'kind', 'url', 'modelId', 'apiKey', 'connectionType', 'protocol', 'authMode', 'proxyMode', 'videoAdapter', 'videoCapabilities', 'providerId', 'inputModalities'}: + if not {'id', 'kind', 'url', 'modelId', 'apiKey'} <= set(item) or set(item) - {'id', 'kind', 'url', 'modelId', 'apiKey', 'connectionType', 'protocol', 'authMode', 'proxyMode', 'videoAdapter', 'videoCapabilities', 'providerId', 'inputModalities', 'capabilityAutoCheck'}: + raise ValueError() + if 'capabilityAutoCheck' in item and type(item['capabilityAutoCheck']) is not bool: raise ValueError() identifier(item["id"]) if item["id"] in seen or item["kind"] not in MODEL_KINDS: @@ -70,12 +73,23 @@ def _read(self) -> list[dict]: @staticmethod def _public(item: dict) -> dict: - return {**({'providerId': item['providerId']} if 'providerId' in item else {}), **{key: item[key] for key in ("id", "kind", "url", "modelId", "connectionType", "protocol", "authMode", "proxyMode", "videoAdapter", "videoCapabilities", "inputModalities")}, 'inputRules': input_rules(item)} + return {**({'providerId': item['providerId']} if 'providerId' in item else {}), + **({'capabilityAutoCheck': item['capabilityAutoCheck']} if 'capabilityAutoCheck' in item else {}), + **{key: item[key] for key in ("id", "kind", "url", "modelId", "connectionType", "protocol", "authMode", "proxyMode", "videoAdapter", "videoCapabilities", "inputModalities")}, 'inputRules': input_rules(item)} def list(self) -> list[dict]: with self.lock: return [self._public(item) for item in self._read()] + def catalog_snapshot(self) -> list[tuple[dict, dict]]: + """Public rows and their private identities from one locked read. + + Callers may project in-memory observations after releasing this lock; + no provider request belongs in the configuration transaction. + """ + with self.lock: + return [(self._public(item), dict(item)) for item in self._read()] + def agent(self, config_id: str) -> dict: """Resolve a private credential snapshot for a backend call only.""" return self.resolve(config_id, "agent") @@ -87,7 +101,9 @@ def resolve(self, config_id: str, kind: str) -> dict: raise ProjectError("所选模型不存在或类别不符，请在设置中配置并重新选择。", 409) return dict(item) - def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str | None = None, *, connection_type='api', protocol=None, auth_mode=None, proxy_mode=None, video_adapter=None, video_capabilities=None, capabilities_supplied=False, input_modalities=None) -> dict: + def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str | None = None, *, connection_type='api', protocol=None, auth_mode=None, proxy_mode=None, video_adapter=None, video_capabilities=None, capabilities_supplied=False, input_modalities=None, capability_auto_check=None) -> dict: + if capability_auto_check is not None and type(capability_auto_check) is not bool: + raise ProjectError('自动能力检查设置必须为布尔值。') if kind not in MODEL_KINDS: raise ProjectError("无效的模型类别。") fields = model_fields(url, model_id, api_key) @@ -111,6 +127,9 @@ def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str raise ProjectError('请填写 API Key，或将鉴权方式设为无需鉴权。') fields['apiKey'] = existing['apiKey'] item = {'id': config_id or str(uuid4()), 'kind': kind, **fields, **connection} + auto_check = capability_auto_check if capability_auto_check is not None else (existing or {}).get('capabilityAutoCheck') + if auto_check is not None: + item['capabilityAutoCheck'] = auto_check item['inputModalities'] = agent_input_modalities(kind, input_modalities if input_modalities is not None else (existing or {}).get('inputModalities')) item['videoAdapter'] = video_adapter if video_adapter is not None else (existing or {}).get('videoAdapter', 'auto') raw = video_capabilities if capabilities_supplied or video_capabilities is not None else (existing or {}).get('videoCapabilities') @@ -127,6 +146,8 @@ def save(self, kind: str, url: str, model_id: str, api_key: str, config_id: str for peer in models: if peer.get('providerId') == item['providerId']: peer.update({key: item[key] for key in ('url', 'apiKey', 'connectionType', 'authMode', 'proxyMode')}) + if auto_check is not None: + peer['capabilityAutoCheck'] = auto_check self._write(models) return self._public(item) @@ -145,12 +166,16 @@ def _write(self, models: list[dict]) -> None: item = dict(model) if 'providerId' in item: services[item['providerId']] = {key: item.pop(key) for key in ('url', 'apiKey', 'connectionType', 'authMode')} + if 'capabilityAutoCheck' in item: + services[item['providerId']]['capabilityAutoCheck'] = item.pop('capabilityAutoCheck') stored.append(item) write_json(self.path, {'version': 1, 'models': stored, 'services': services}) def save_many(self, kind: str, url: str, model_ids: list[str], api_key: str, config_id: str | None = None, *, connection_type='api', protocol=None, - auth_mode=None, proxy_mode=None, video_adapter=None, video_capabilities=None, input_modalities=None, model_options=None, capabilities_supplied=False) -> list[dict]: + auth_mode=None, proxy_mode=None, video_adapter=None, video_capabilities=None, input_modalities=None, model_options=None, capabilities_supplied=False, capability_auto_check=None) -> list[dict]: + if capability_auto_check is not None and type(capability_auto_check) is not bool: + raise ProjectError('自动能力检查设置必须为布尔值。') if kind not in MODEL_KINDS or not 1 <= len(model_ids) <= 200: raise ProjectError('请选择 1—200 个模型。') model_options = model_options or {} @@ -178,6 +203,7 @@ def save_many(self, kind: str, url: str, model_ids: list[str], api_key: str, raise ProjectError('请填写该服务的 API Key；新地址不能复用原地址的密钥。') key = existing['apiKey'] provider_id = existing.get('providerId', existing['id']) if existing else str(uuid4()) + auto_check = capability_auto_check if capability_auto_check is not None else (existing or {}).get('capabilityAutoCheck') previous = [item for item in models if item.get('providerId') == provider_id or item['id'] == config_id] peers = {item['modelId']: item for item in previous} saved = [] @@ -187,6 +213,8 @@ def save_many(self, kind: str, url: str, model_ids: list[str], api_key: str, old = existing item = {'id': old['id'] if old else str(uuid4()), 'kind': kind, **field, **connection, 'apiKey': key, 'providerId': provider_id, 'videoAdapter': video_adapter or 'auto'} + if auto_check is not None: + item['capabilityAutoCheck'] = auto_check options = model_options.get(field['modelId'], {}) item['videoAdapter'] = options.get('videoAdapter') or video_adapter or (old or {}).get('videoAdapter', 'auto') modalities = options.get('inputModalities', input_modalities if input_modalities is not None else (old or {}).get('inputModalities')) @@ -201,6 +229,8 @@ def save_many(self, kind: str, url: str, model_ids: list[str], api_key: str, for item in remaining: if item.get('providerId') == provider_id: item.update(url=fields[0]['url'], apiKey=key, connectionType=connection['connectionType'], authMode=connection['authMode'], proxyMode=connection['proxyMode']) + if auto_check is not None: + item['capabilityAutoCheck'] = auto_check self._write(remaining + saved) return [self._public(item) for item in saved] diff --git a/packages/core/src/opendirector/integrations/capability_probe.py b/packages/core/src/opendirector/integrations/capability_probe.py new file mode 100644 index 00000000..29f7694a --- /dev/null +++ b/packages/core/src/opendirector/integrations/capability_probe.py @@ -0,0 +1,185 @@ +"""Bounded, process-local, coalesced reads of optional Omni declarations. + +This service never wakes a model and never submits inference. A worker owns +each HTTP request; callers share its future with a total two-second deadline. +""" +from collections import OrderedDict +from concurrent.futures import Future, ThreadPoolExecutor, TimeoutError +from dataclasses import dataclass +import hashlib +import json +import threading +import time + +import httpx + +from opendirector.domain.omni_capabilities import OmniCapabilities +from opendirector.integrations.http_settings import auth_headers, trust_environment +from opendirector.integrations.volcengine import endpoint + +DECLARATION_SECONDS = 30 +PAUSE_SECONDS = 5 +QUERY_SECONDS = 2 +FAILURE_SECONDS = 2 + + +def identity(config, *, service=False): + keys = ('url', 'apiKey', 'authMode', 'connectionType', 'proxyMode', '_capability_namespace') + if not service: + keys += ('id', 'providerId', 'kind', 'modelId', 'protocol', 'videoAdapter', 'inputModalities', + 'videoCapabilities', 'capabilityAutoCheck') + value = {key: config.get(key) for key in keys} + value['serviceId'] = config.get('providerId') or config.get('id') + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def automatic(config): + return config.get('capabilityAutoCheck', config.get('connectionType') == 'self_hosted') + + +@dataclass(frozen=True) +class Observation: + document: OmniCapabilities | None + reason: str + checked: float + expires: float + received: float + valid_until: float + + +class CapabilityProbe: + def __init__(self, *, transport=None, clock=time.monotonic, wall_clock=time.time, timeout=QUERY_SECONDS, capacity=128): + self.transport, self.clock, self.timeout, self.capacity = transport, clock, timeout, capacity + self.wall_clock = wall_clock + self._lock = threading.Lock() + self._cache = OrderedDict() + self._pending: dict[str, Future] = {} + self._workers = ThreadPoolExecutor(max_workers=4, thread_name_prefix='capabilities') + + def close(self): + self._workers.shutdown(wait=False, cancel_futures=True) + + def _observation(self, document, reason): + lifetime = DECLARATION_SECONDS if document else FAILURE_SECONDS + received, checked = self.clock(), self.wall_clock() + return Observation(document, reason, checked, checked + lifetime, received, received + lifetime) + + def peek(self, config): + key = identity(config, service=True) + with self._lock: + observation = self._cache.get(key) + return observation if observation and observation.valid_until > self.clock() else None + + def query(self, config, *, manual=False): + return self.query_many([config], manual=manual)[0] + + def query_many(self, configs, *, manual=False): + """Start independent services together and share one caller deadline. + + The same bounded worker pool and pending map serve both one-model + admission and directory refreshes; queued requests inherit the deadline. + """ + deadline = time.monotonic() + self.timeout + pending = [(config, self._start(config, manual, deadline)) for config in configs] + return [self._wait(config, value, deadline) if isinstance(value, Future) else value + for config, value in pending] + + def _start(self, config, manual, deadline): + if not manual and not automatic(config): + return self.peek(config) + key = identity(config, service=True) + with self._lock: + observation = self._cache.get(key) + # Paused observations need another read after their shorter lifetime. + paused = observation and observation.document and any( + op.execution.state == 'paused' for model in observation.document.models for op in model.operations.values()) + if (not manual and observation and observation.valid_until > self.clock() + and (not paused or observation.received + PAUSE_SECONDS > self.clock())): + return observation + future = self._pending.get(key) + if future is None: + if len(self._pending) >= self.capacity or time.monotonic() >= deadline: + value = self._observation(None, 'query_busy' if len(self._pending) >= self.capacity else 'query_timeout') + self._cache[key] = value + while len(self._cache) > self.capacity: + self._cache.popitem(last=False) + return value + future = self._workers.submit(self._read, dict(config), deadline) + self._pending[key] = future + # Register outside the lock: already-completed futures invoke callbacks inline. + register = True + else: + register = False + if register: + future.add_done_callback(lambda done: self._finish(key, done)) + return future + + def _wait(self, config, future, deadline): + key = identity(config, service=True) + try: + value = future.result(timeout=max(0, deadline - time.monotonic())) + # Future.result may wake before its completion callback publishes. + self._finish(key, future) + return value + except TimeoutError: + value = self._observation(None, 'query_timeout') + with self._lock: + if self._pending.get(key) is future: + self._cache[key] = value + while len(self._cache) > self.capacity: + self._cache.popitem(last=False) + return value + + def _finish(self, key, future): + try: + value = future.result() + except Exception: + value = self._observation(None, 'query_failed') + with self._lock: + if self._pending.get(key) is not future: + return # A waiter already published it; do not overwrite a newer refresh. + self._pending.pop(key, None) + self._cache[key] = value + self._cache.move_to_end(key) + while len(self._cache) > self.capacity: + self._cache.popitem(last=False) + + def _read(self, config, deadline): + reason, document = 'query_failed', None + try: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError() + with httpx.Client(transport=self.transport, trust_env=trust_environment(config['url'], config=config), + timeout=remaining, follow_redirects=False) as client: + with client.stream('GET', endpoint(config['url'], '/omni/capabilities'), headers=auth_headers(config)) as response: + if response.status_code != 200: + reason = ('authentication_failed' if response.status_code in {401, 403} + else 'protocol_unavailable' if response.status_code == 404 else 'query_failed') + else: + data = bytearray() + for chunk in response.iter_bytes(): + if time.monotonic() >= deadline: + raise TimeoutError() + data.extend(chunk) + if len(data) > 1024 * 1024: + raise ValueError('response too large') + decoded = json.loads(data) + if isinstance(decoded, dict) and decoded.get('object') != 'omni.capabilities': + reason = 'protocol_unavailable' + elif isinstance(decoded, dict) and type(decoded.get('schema_version')) is not int: + reason = 'invalid_response' + elif isinstance(decoded, dict) and decoded.get('schema_version') != 1: + reason = 'unsupported_version' + else: + document = OmniCapabilities.model_validate(decoded) + reason = 'observed' + if time.monotonic() > deadline: + raise TimeoutError() + except (httpx.TimeoutException, TimeoutError): + reason, document = 'query_timeout', None + except httpx.HTTPError: + reason, document = 'connection_failed', None + except (ValueError, TypeError, RecursionError): + reason, document = 'invalid_response', None + return self._observation(document, reason) diff --git a/packages/core/src/opendirector/integrations/generation_capability_parameters.py b/packages/core/src/opendirector/integrations/generation_capability_parameters.py new file mode 100644 index 00000000..12f4e577 --- /dev/null +++ b/packages/core/src/opendirector/integrations/generation_capability_parameters.py @@ -0,0 +1,66 @@ +"""Project prepared generation values onto the fields sent by each adapter. + +Preparation includes UI and recovery fields. Only values actually transmitted +may be checked against a service's request contract. +""" +from opendirector.domain.input_rules import input_rules +from opendirector.domain.rule_facts import MUSIC_SECONDS_INITIAL +from opendirector.domain.voices import effective_speech_voice + + +def generation_capability_parameters(config, kind, parameters, *, has_images=False): + profile = input_rules(config)['profile'] + if kind == 'image': + size = parameters.get('size', '') + if profile == 'minimax-h3-image': + from opendirector.integrations.model_adapters import MinimaxH3Images + result = MinimaxH3Images.generation_parameters(size) + elif profile in {'vllm-omni-image', 'vllm-omni-qwen-image'}: + from opendirector.integrations.model_adapters import adapter_for + from opendirector.integrations.vllm_omni_image import WIDTH, HEIGHT + width, height = map(int, size.split('x')) if 'x' in size else (WIDTH, HEIGHT) + width, height = adapter_for(config).size(width, height) + result = dict(size=f'{width}x{height}', response_format='b64_json', output_format='png') + if profile == 'vllm-omni-image': + from opendirector.integrations.vllm_omni_image import SAMPLING_FIELDS + result.update({key: parameters[key] for key in SAMPLING_FIELDS if key in parameters}) + else: + # The generic OpenAI generation provider does not transmit its + # internal response_format/output_format preparation defaults. + result = dict(size=size if 'x' in size else '1024x1024') + if profile == 'moss-image': + result.update(response_format='url', prompt_enhance=False) + if not has_images: + result['n'] = 1 + elif kind == 'audio': + result = dict(response_format='mp3') + if config.get('protocol') == 'vllm_omni_music': + from opendirector.integrations.model_adapters import VllmOmniMusic + result['max_new_tokens'] = round(parameters.get('duration', MUSIC_SECONDS_INITIAL) * VllmOmniMusic.FRAMES_PER_SECOND) + elif profile == 'moss-speech': + result.update(voice_id=parameters.get('voice'), delivery_method='url', stream=False) + else: + result['voice'] = effective_speech_voice(config, parameters.get('voice')) + elif kind == 'video': + if profile == 'vllm-omni-h3': + from opendirector.integrations.vllm_omni import FPS, h3_frames + result = dict(fps=parameters.get('fps', FPS), + num_frames=parameters.get('num_frames', h3_frames(parameters.get('duration', 5)))) + if parameters.get('resolution') and parameters.get('size'): + result['size'] = parameters['size'] + elif profile == 'fasth3-omni': + result = dict(seconds=float(parameters.get('seconds', 5)), size=parameters.get('size', '832x480'), + num_inference_steps=8, guidance_scale=1, fps=24, seed=parameters.get('seed', 1234)) + else: + result = {key: parameters[key] for key in ('size',) if key in parameters} + if 'duration' in parameters: + result['seconds'] = parameters['duration'] + else: + result = {key: parameters[key] for key in ('max_tokens',) if key in parameters} + size = result.get('size') + if isinstance(size, str) and 'x' in size: + try: + result['width'], result['height'] = map(int, size.split('x')) + except ValueError: + pass + return {key: value for key, value in result.items() if type(value) in {str, int, float, bool}} diff --git a/packages/core/src/opendirector/integrations/generation_errors.py b/packages/core/src/opendirector/integrations/generation_errors.py index a5a73cb1..826bbb8f 100644 --- a/packages/core/src/opendirector/integrations/generation_errors.py +++ b/packages/core/src/opendirector/integrations/generation_errors.py @@ -17,7 +17,7 @@ 'authentication': ('生成服务鉴权失败。', '请在设置中检查 API Key。'), 'permission': ('生成服务拒绝了访问请求。', '请检查 API Key 是否正确及有效、模型调用权限和服务安全策略。'), 'quota': ('模型账户余额或套餐额度不足。', '请核对账户余额和套餐额度。'), - 'rate_limit': ('模型调用频率或配额受限。', '请核对调用频率及套餐额度后再操作。'), + 'rate_limit': ('模型服务繁忙或请求受限。', '请稍后再试；若持续发生，请检查服务状态、调用频率和账户额度。'), 'temporary': ('生成服务暂时不可用。', '请先核对已有任务，再处理当前步骤。'), 'provider': ('生成服务未能完成当前操作。', '请核对原任务；错误码和请求标识可用于向供应商排查。'), } @@ -173,10 +173,10 @@ def transport_failure(stage, *, timeout=False, status=None): status=status or (504 if timeout else 503)) -def protocol_failure(stage, *, message='生成服务返回了无法识别或不完整的结果。'): +def protocol_failure(stage, *, message='生成服务返回了无法识别或不完整的结果。', suggestion=None): context = MODEL_JOB_CONTEXT.get() logger.error(f'model.error {json.dumps({"project_id": context[0] if context else None, "job_id": context[1] if context else None, "stage": stage, "error_type": "ProtocolError"})}') unknown = stage == 'submit' return GenerationError(GenerationFailure(category='submission_unknown' if unknown else 'protocol', stage=stage, message=message, - suggestion='请先核对请求是否已被接收，避免重复生成。' if unknown else '请核对原任务状态和接口协议。')) + suggestion=suggestion or ('请先核对请求是否已被接收，避免重复生成。' if unknown else '请核对原任务状态和接口协议。'))) diff --git a/packages/core/src/opendirector/integrations/model_adapters.py b/packages/core/src/opendirector/integrations/model_adapters.py index a78204ab..185796ea 100644 --- a/packages/core/src/opendirector/integrations/model_adapters.py +++ b/packages/core/src/opendirector/integrations/model_adapters.py @@ -17,14 +17,18 @@ from PIL import Image from opendirector.domain.assets import MAX_ASSET_BYTES +from opendirector.domain.capability_messages import constraint_rejection from opendirector.domain.capabilities import (ChatRequest, CutoutRequest, DepthRequest, EditMask, ImageRequest, MediaResult, MusicRequest, ScoreRequest, Segment, SegmentRequest, SpeechRequest, TrackRequest, Tracks, VideoEditRequest, VideoRequest) from opendirector.domain.input_rules import input_rules +from opendirector.domain.image_inputs import image_input_policy, validate_image_request from opendirector.domain.minimax_image import validate_minimax_image_reference from opendirector.domain.moss_image import validate_moss_image_prompt, validate_moss_image_size from opendirector.domain.moss_speech import validate_moss_speech_input +from opendirector.domain.omni_capabilities import RequestConstraints from opendirector.domain.projects import ProjectError +from opendirector.domain.request_constraints import constraint_failure from opendirector.integrations.generation_errors import checked, protocol_failure, provider_failure, transport_failure from opendirector.integrations.http_settings import MODEL_HTTP_HOOKS, auth_headers, trust_environment from opendirector.integrations.reference_inputs import data_uri, image_type, request_size @@ -102,10 +106,15 @@ async def read(): class ModelAdapter: capabilities: frozenset[str] = frozenset() + supports_native_mask = False def __init__(self, transport=None, poll_interval=3): self.transport, self.poll_interval = transport, poll_interval + def image_parameters(self, width, height, **options): + """Only adapters with an exact image-route projection supply fields.""" + return {} + async def image(self, config, request: ImageRequest) -> MediaResult: raise ProjectError('所选模型不支持图片生成。', 409) @@ -217,7 +226,22 @@ class MinimaxH3Images(ModelAdapter): capabilities = frozenset({'image.generate', 'image.edit'}) max_images = 16 + @staticmethod + def generation_parameters(size): + return dict(size=size, n=1, output_format='png', response_format='url') + + def image_parameters(self, width, height, *, has_images=True, native_mask=False, strength=None, seed=None, extra=None): + if native_mask: + raise ProjectError('当前图片服务不支持选区输入。', 422, code='image_input_mask') + ratio = min(3, max(1 / 3, width / height)) + scale = 2048 / math.sqrt(ratio) + width, height = round(scale * ratio), round(scale) + width, height = min(width, 3 * height), min(height, 3 * width) + return {**self.generation_parameters(f'{width}x{height}'), 'width': width, 'height': height} + async def image(self, config, request: ImageRequest) -> MediaResult: + if request.mask is not None or request.reference is not None: + raise ProjectError('当前图片服务不支持选区输入。', 422, code='image_input_mask') images = [*([request.image] if request.image is not None else []), *request.references] if len(images) > self.max_images: raise ProjectError('MiniMax H3 图片编辑最多接受 16 张参考图。', 422, code='minimax_image_count') @@ -233,16 +257,11 @@ async def image(self, config, request: ImageRequest) -> MediaResult: if len(data) > 64 * 1024 * 1024: raise ProjectError('MiniMax H3 图片参考请求超过 64 MiB。', 413, code='minimax_image_bytes') encoded.append(data_uri(data, mime)) - width, height = request.width, request.height - ratio = min(3, max(1 / 3, width / height)) - scale = 2048 / math.sqrt(ratio) - width, height = round(scale * ratio), round(scale) - size = f'{min(width, 3 * height)}x{min(height, 3 * width)}' - return await self.generate(config, request.prompt, size, encoded) + parameters = self.image_parameters(request.width, request.height, has_images=bool(images)) + return await self.generate(config, request.prompt, parameters['size'], encoded) async def generate(self, config, prompt, size, images, *, result_ready=None): - payload = {'model': config['modelId'], 'prompt': prompt, 'size': size, 'n': 1, - 'output_format': 'png', 'response_format': 'url'} + payload = {'model': config['modelId'], 'prompt': prompt, **self.generation_parameters(size)} route = '/images/generations' if images: route = '/images/edits' @@ -283,16 +302,19 @@ class OpenAIImages(ModelAdapter): """ capabilities = frozenset({'image.generate', 'image.edit'}) + supports_native_mask = True OPENAI_SIZES = ((1024, 1024), (1536, 1024), (1024, 1536)) def __init__(self, transport=None, poll_interval=3, *, free_size=False, max_edge=1536, grid=16, strength=False, max_images=1, - transparent=False, response_format=True, reference_field='image'): + transparent=False, response_format=True, reference_field='image', image_policy=None, legacy_image_policy=False): super().__init__(transport, poll_interval) self.free_size, self.max_edge, self.grid = free_size, max_edge, grid # img2img services honour strength; instruction editors ignore it. self.strength, self.max_images = strength, max_images # gpt-image omits `response_format`: it is documented base64 output only. self.response_format, self.reference_field = response_format, reference_field + self.image_policy = image_policy + self.legacy_image_policy = legacy_image_policy # Models with an RGBA decoder can lift a subject onto transparency. self.capabilities = type(self).capabilities | ({'image.cutout'} if transparent else set()) @@ -305,6 +327,22 @@ def size(self, width, height): # Diffusion latents need both axes on the service's pixel grid. return tuple(max(256, int(round(value * scale / self.grid)) * self.grid) for value in (width, height)) + def image_parameters(self, width, height, *, has_images=True, native_mask=False, strength=None, seed=None, extra=None): + """The same scalar values drive capability checks and HTTP serialization.""" + width, height = (width, height) if native_mask else self.size(width, height) + parameters = {'width': width, 'height': height, 'size': f'{width}x{height}', **(extra or {})} + if not has_images: + parameters['n'] = 1 + if self.response_format: + parameters['response_format'] = 'b64_json' + if self.free_size: + parameters['output_format'] = 'png' + if seed is not None: + parameters['seed'] = seed + if has_images and strength is not None and (self.strength or native_mask): + parameters['strength'] = round(min(max(strength, 0.0), 1.0), 3) + return parameters + async def cutout(self, config, request: CutoutRequest) -> MediaResult: if 'image.cutout' not in self.capabilities: return await super().cutout(config, request) @@ -316,33 +354,44 @@ async def cutout(self, config, request: CutoutRequest) -> MediaResult: extra={'background': 'transparent'}) async def image(self, config, request: ImageRequest, *, extra=None) -> MediaResult: - width, height = self.size(request.width, request.height) + policy = self.image_policy or {**image_input_policy(config), 'maxImages': self.max_images} + if not self.legacy_image_policy: + validate_image_request(request, policy) secrets = (config.get('apiKey', ''),) - images = [*([request.image] if request.image is not None else []), *request.references][:self.max_images] - extra = dict(extra or {}) - if self.free_size: - extra['output_format'] = 'png' - if request.seed is not None: - extra['seed'] = request.seed + images = [*([request.image] if request.image is not None else []), *request.references] + if self.legacy_image_policy: + # Only persisted tasks accepted before image policies existed use + # the original request shape, including its original image count. + images = images[:self.max_images] + parameters = self.image_parameters(request.width, request.height, has_images=bool(images), + native_mask=request.mask is not None, strength=request.strength, seed=request.seed, extra=extra) + frozen = policy.get('requestConstraints') + if frozen is not None and not self.legacy_image_policy: + counts = {'text': 1, 'image': len(images), 'mask_image': int(request.mask is not None), + 'reference_image': int(request.reference is not None)} + failure = constraint_failure(RequestConstraints.model_validate(frozen), counts, parameters) + if failure: + code, message = constraint_rejection(*failure) + raise ProjectError(message, 422, code=code) + # Width/height describe the parsed size; the image API accepts one size field. + fields = {key: value for key, value in parameters.items() if key not in {'width', 'height'}} async def work(): async with _client(self, config) as client: if not images: - payload = {'model': config['modelId'], 'prompt': request.prompt, 'size': f'{width}x{height}', - 'n': 1, **extra} - if self.response_format: - payload['response_format'] = 'b64_json' + payload = {'model': config['modelId'], 'prompt': request.prompt, **fields} async with client.stream('POST', endpoint(config['url'], '/images/generations'), headers=auth_headers(config), json=payload) as response: value = json.loads(await bounded(response, stage='submit', secrets=secrets)) else: - form = {'model': config['modelId'], 'prompt': request.prompt, 'size': f'{width}x{height}', - **{key: str(item) for key, item in extra.items()}} - if self.response_format: - form['response_format'] = 'b64_json' - if request.strength is not None and self.strength: - form['strength'] = f'{min(max(request.strength, 0.0), 1.0):.3f}' + form = {'model': config['modelId'], 'prompt': request.prompt, + **{key: f'{item:.3f}' if key == 'strength' else str(item) for key, item in fields.items()}} files = image_parts(images, self.reference_field) + if request.mask is not None: + native = policy['nativeMask'] + files.append((native['field'], ('mask.png', request.mask, 'image/png'))) + if request.reference is not None: + files.append((native['referenceField'], ('reference.png', request.reference, 'image/png'))) async with client.stream('POST', endpoint(config['url'], '/images/edits'), headers=auth_headers(config), data=form, files=files, extensions={'model_parameters': {'fields': form, 'file_count': len(files)}}) as response: value = json.loads(await bounded(response, stage='submit', secrets=secrets)) @@ -371,7 +420,17 @@ class ArkImageEdits(ModelAdapter): # Use the shared reference count limit across Ark's Seedream variants. max_images = 10 + def __init__(self, transport=None, poll_interval=3, *, image_policy=None, legacy_image_policy=False): + super().__init__(transport, poll_interval) + self.image_policy = image_policy + self.legacy_image_policy = legacy_image_policy + if image_policy is not None: + self.max_images = min(type(self).max_images, image_policy['maxImages']) + async def image(self, config, request: ImageRequest) -> MediaResult: + policy = self.image_policy or image_input_policy(config) + if not self.legacy_image_policy: + validate_image_request(request, {**policy, 'maxImages': self.max_images, 'nativeMask': None}) width, height = request.width, request.height if not 1 / 16 <= width / height <= 16: raise ProjectError('火山方舟图片编辑的输出宽高比须在 1:16 至 16:1 之间。', 422) @@ -381,8 +440,8 @@ async def image(self, config, request: ImageRequest) -> MediaResult: width, height = round(width * scale), round(height * scale) width, height = min(width, 16 * height), min(height, 16 * width) images = [request.image, *request.references] if request.image is not None else list(request.references) - if len(images) > self.max_images: - raise ProjectError('火山方舟图片编辑最多接受 10 张参考图片。', 422) + if self.legacy_image_policy and len(images) > self.max_images: + raise ProjectError('火山方舟图片编辑最多支持 10 张参考图片。', 422) refs = [] for image in images: mime = image_type(image) @@ -559,7 +618,8 @@ class OpenAISpeech(ModelAdapter): async def speech(self, config, request: SpeechRequest) -> MediaResult: # Self-hosted speech servers name their own speakers; "default" selects the model's. - voice = request.voice or ('default' if config.get('connectionType') == 'self_hosted' else 'alloy') + from opendirector.domain.voices import effective_speech_voice + voice = effective_speech_voice(config, request.voice) payload = {'model': config['modelId'], 'input': request.text, 'voice': voice, 'response_format': 'mp3'} async def work(): @@ -721,10 +781,18 @@ def _inverse(data, near, far): return media.disparity(work.write('meters.mp4', data), work.path / 'inverse.mp4', near=near, far=far) -def adapter_for(config, transport=None, poll_interval=3) -> ModelAdapter | None: +def adapter_for(config, transport=None, poll_interval=3, *, image_policy=None, legacy_image_policy=False) -> ModelAdapter | None: """The adapter serving a configuration, chosen from its explicit profile and protocol.""" kind, protocol = config['kind'], config.get('protocol') profile = input_rules(config)['profile'] + policy = image_policy if image_policy is not None else image_input_policy(config) if kind == 'image' else None + if legacy_image_policy and kind == 'image': + # These are the historical transport limits, used exclusively by tasks + # already accepted without a frozen image policy. + old_max = 10 if protocol == 'volcengine' else {'vllm-omni-qwen-image': 4, 'openai-gpt-image': 16}.get(profile, 1) + policy = {**policy, 'maxImages': old_max, 'nativeMask': None} + image_options = {'max_images': policy['maxImages'], 'image_policy': policy, + 'legacy_image_policy': legacy_image_policy} if policy is not None else {} if profile == 'vllm-omni-segmentation': return VllmOmniSegmentation(transport, poll_interval) if profile == 'vllm-omni-depth': @@ -740,20 +808,20 @@ def adapter_for(config, transport=None, poll_interval=3) -> ModelAdapter | None: if profile == 'moss-speech': return MossSpeech(transport, poll_interval) if kind == 'image' and profile == 'vllm-omni-image': - return OpenAIImages(transport, poll_interval, free_size=True, strength=True) + return OpenAIImages(transport, poll_interval, free_size=True, strength=True, **image_options) if kind == 'image' and profile == 'vllm-omni-qwen-image': - return OpenAIImages(transport, poll_interval, free_size=True, max_edge=2048, grid=32, max_images=4, transparent=True) + return OpenAIImages(transport, poll_interval, free_size=True, max_edge=2048, grid=32, transparent=True, **image_options) if kind == 'image' and protocol == 'volcengine': - return ArkImageEdits(transport, poll_interval) + return ArkImageEdits(transport, poll_interval, image_policy=policy, legacy_image_policy=legacy_image_policy) if kind == 'image' and profile == 'openai-gpt-image': # gpt-image: base64 answers only, 16 reference images, documented sizes. # No subject cutout: its edits redraw and move the subject, so a returned # alpha does not line up with the source pixels, and gpt-image-2 rejects # transparent backgrounds outright. - return OpenAIImages(transport, poll_interval, max_images=16, + return OpenAIImages(transport, poll_interval, **image_options, response_format=False, reference_field='image[]') if kind == 'image' and protocol == 'openai': - return OpenAIImages(transport, poll_interval) + return OpenAIImages(transport, poll_interval, **image_options) if kind == 'video' and profile == 'vllm-omni-h3': return VllmOmniVideo(transport, poll_interval) if kind == 'audio' and profile == 'vllm-omni-music': diff --git a/packages/core/src/opendirector/integrations/providers.py b/packages/core/src/opendirector/integrations/providers.py index 3c96dd02..e5723528 100644 --- a/packages/core/src/opendirector/integrations/providers.py +++ b/packages/core/src/opendirector/integrations/providers.py @@ -18,10 +18,9 @@ from opendirector.integrations.kling_video import KlingVideoGeneration from opendirector.integrations.vllm_omni import VllmOmniGeneration from opendirector.integrations.vllm_omni_image import VllmOmniImageGeneration -from opendirector.integrations.model_adapters import MinimaxH3Images, MossImages, MossSpeech, OpenAISpeech, VllmOmniMusic, VllmOmniScore +from opendirector.integrations.model_adapters import (MinimaxH3Images, MossImages, MossSpeech, OpenAISpeech, + VllmOmniMusic, VllmOmniScore, image_parts) from opendirector.domain.capabilities import ImageRequest, MusicRequest, ScoreRequest, SpeechRequest -from opendirector.integrations.model_adapters import OpenAISpeech, VllmOmniMusic, VllmOmniScore, image_parts -from opendirector.domain.capabilities import MusicRequest, ScoreRequest, SpeechRequest from opendirector.media.probe import require_ffprobe diff --git a/packages/core/src/opendirector/integrations/vllm_omni_image.py b/packages/core/src/opendirector/integrations/vllm_omni_image.py index 96a125a0..241299fa 100644 --- a/packages/core/src/opendirector/integrations/vllm_omni_image.py +++ b/packages/core/src/opendirector/integrations/vllm_omni_image.py @@ -1,7 +1,7 @@ """Self-hosted vLLM-Omni image models (Z-Image-Turbo, Qwen-Image-2.1) for canvas nodes. -Reference documents are inlined as text. Z-Image-Turbo draws from the prompt -alone; Qwen-Image-2.1 also takes connected images as condition images. Automatic +Reference documents are inlined as text. Z-Image-Turbo accepts one input image; +Qwen-Image-2.1 accepts up to four condition images. Automatic output uses H3's 16:9 canvas; explicit output uses the frozen request dimensions. The model API itself is reached through the capability adapter. """ @@ -19,8 +19,8 @@ # Z-Image-Turbo is a few-step distilled model meant to run without CFG; vLLM-Omni's recipe uses 9 steps # and guidance 0. Left out, vLLM-Omni runs 50 steps with CFG at 1.0, about ten times the work. # Fixed into a job's parameters when it is accepted; see CreationGeneration._parameters. -# Every new job sends these, which is safe only while the vllm-omni-image profile takes no reference images -# (domain/input_rules.py): vLLM-Omni starts an edit at step int(steps * (1 - strength)), so 9 steps would +# Apply only to text-to-image jobs. Image edits retain the service defaults: +# vLLM-Omni starts an edit at step int(steps * (1 - strength)), so 9 steps would # start a strong edit from almost pure noise. SAMPLING = {'num_inference_steps': 9, 'guidance_scale': 0.0} SAMPLING_FIELDS = ('num_inference_steps', 'guidance_scale') diff --git a/packages/core/src/opendirector/integrations/volcengine.py b/packages/core/src/opendirector/integrations/volcengine.py index 56008418..ca9cced5 100644 --- a/packages/core/src/opendirector/integrations/volcengine.py +++ b/packages/core/src/opendirector/integrations/volcengine.py @@ -59,7 +59,9 @@ async def _public_result_url(url): checked_address = getattr(address, 'ipv4_mapped', None) or address if (not checked_address.is_global or checked_address.is_multicast or checked_address.is_reserved): - raise protocol_failure('download') + raise protocol_failure('download', + message='生成结果地址解析为非公网地址，下载已停止。', + suggestion='请检查服务返回的地址或本机域名解析设置，再恢复原任务的下载。') if not addresses: raise protocol_failure('download') # A pooled connection to one IP must not reuse another hostname's TLS identity. diff --git a/scripts/check-i18n.mjs b/scripts/check-i18n.mjs index af2d3e63..0add7e50 100644 --- a/scripts/check-i18n.mjs +++ b/scripts/check-i18n.mjs @@ -29,6 +29,7 @@ const skip = [ /[\\/]modelDescriptions\.ts$/, /[\\/]creationPreviewText\.ts$/, /[\\/]assistantToolText\.ts$/, + /[\\/]capabilityErrorText\.ts$/, /[\\/]generationFailureText\.ts$/, /\.test\./, ]; diff --git a/scripts/run-browser-suite.mjs b/scripts/run-browser-suite.mjs index 70e56877..e7973f8d 100644 --- a/scripts/run-browser-suite.mjs +++ b/scripts/run-browser-suite.mjs @@ -22,6 +22,7 @@ const workflows = [ }, { name: "minimax h3", script: "tests/browser/minimax_h3.mjs" }, { name: "provider settings", script: "tests/browser/provider_settings.mjs" }, + { name: "capability discovery", script: "tests/browser/capability_discovery.mjs" }, { name: "created node focus", script: "tests/browser/created_node_focus.mjs" }, { name: "cross canvas paste", script: "tests/browser/cross_canvas_paste.mjs" }, { name: "reference reconnect", script: "tests/browser/reference_reconnect.mjs" }, diff --git a/src/backend/availability.ts b/src/backend/availability.ts new file mode 100644 index 00000000..60772dc8 --- /dev/null +++ b/src/backend/availability.ts @@ -0,0 +1,331 @@ +import type { + ModelAvailability, + OperationAvailability, +} from "../contracts/generated.ts"; +import type { ModelConfig } from "./rules.ts"; +import { hasMessage, tNow } from "../i18n/messages.ts"; + +export type { ModelAvailability, OperationAvailability }; + +/** A transport failure clears decisions while retaining the saved connection identity for retry. */ +export function failedAvailability( + value: ModelAvailability, +): ModelAvailability { + return { + ...value, + status: "unknown", + reasonCode: "query_failed", + checkedAt: null, + expiresAt: null, + operations: value.operations.map((item) => ({ + ...item, + status: "unknown", + reasonCode: "query_failed", + checkedAt: null, + expiresAt: null, + ...(item.declaration + ? { + declaration: { + ...item.declaration, + status: "unknown" as const, + reasonCode: "query_failed", + checkedAt: null, + expiresAt: null, + }, + } + : {}), + ...(item.execution + ? { + execution: { + ...item.execution, + state: "unknown" as const, + checkedAt: null, + expiresAt: null, + }, + } + : {}), + })), + }; +} + +export interface AvailabilityClock { + wallNow(): number; + monotonicNow(): number; +} + +const clock: AvailabilityClock = { + wallNow: () => Date.now(), + monotonicNow: () => performance.now(), +}; +type TimedObservation = Pick; +const deadlines = new WeakMap< + TimedObservation, + { clock: AvailabilityClock; at: number } +>(); +const observations = new WeakMap>(); + +function deadline( + value: TimedObservation, + timing: AvailabilityClock, +): number | null { + if (value.expiresAt === null) return null; + const existing = deadlines.get(value); + if (existing?.clock === timing) return existing.at; + const maximum = + value.checkedAt === null + ? 30_000 + : Math.max(0, value.expiresAt - value.checkedAt); + const at = + timing.monotonicNow() + + Math.max(0, Math.min(maximum, value.expiresAt - timing.wallNow())); + deadlines.set(value, { clock: timing, at }); + return at; +} + +/** Call when a response arrives. Re-reading the same observation cannot renew its lifetime. */ +export function receiveAvailability( + value: ModelAvailability, + timing: AvailabilityClock = clock, +): ModelAvailability { + let known = observations.get(timing); + if (!known) { + known = new Map(); + observations.set(timing, known); + } + for (const operation of value.operations) { + const parts: [string, TimedObservation][] = operation.declaration + ? [ + ["declaration", operation.declaration], + ...(operation.execution + ? [["execution", operation.execution] as [string, TimedObservation]] + : []), + ] + : [["combined", operation]]; + for (const [part, observation] of parts) { + if (observation.expiresAt === null) continue; + const key = JSON.stringify([ + value.identity, + operation.operation, + operation.profile, + operation.method, + operation.path, + operation.output, + part, + observation, + ]); + const previous = known.get(key); + const at = previous ?? deadline(observation, timing)!; + deadlines.set(observation, { clock: timing, at }); + known.delete(key); + known.set(key, at); + if (known.size > 1024) known.delete(known.keys().next().value!); + } + } + return value; +} + +/** Remaining elapsed time, independent of later wall-clock changes. */ +export function availabilityDelay( + value: ModelAvailability, + timing: AvailabilityClock = clock, +): number | null { + receiveAvailability(value, timing); + const now = timing.monotonicNow(); + const remaining = value.operations + .flatMap((item) => + item.declaration + ? [item.declaration, ...(item.execution ? [item.execution] : [])] + : [item], + ) + .map((item) => deadline(item, timing)) + .filter((at): at is number => at !== null && at > now) + .map((at) => at - now); + return remaining.length ? Math.min(...remaining) : null; +} + +/** The server owns decisions. The browser only expires and presents observations. */ +export function currentOperation( + value: OperationAvailability, + timing: AvailabilityClock = clock, +): OperationAvailability { + if (value.declaration) { + const declarationAt = deadline(value.declaration, timing); + const executionAt = value.execution + ? deadline(value.execution, timing) + : null; + const declarationExpired = + declarationAt !== null && declarationAt <= timing.monotonicNow(); + const executionExpired = + executionAt !== null && executionAt <= timing.monotonicNow(); + if (!declarationExpired && !executionExpired) return value; + const declaration = declarationExpired + ? { + ...value.declaration, + status: "unknown" as const, + reasonCode: "observation_expired", + expiresAt: null, + } + : value.declaration; + // Restore the server's declaration when its shorter execution observation + // expires. The browser never infers support from a model name or input. + return { + ...value, + ...declaration, + declaration, + ...(executionExpired && value.execution + ? { + execution: { + ...value.execution, + state: "unknown" as const, + expiresAt: null, + }, + } + : {}), + }; + } + const at = deadline(value, timing); + return at !== null && at <= timing.monotonicNow() + ? { + ...value, + status: "unknown", + reasonCode: "observation_expired", + expiresAt: null, + } + : value; +} + +export function currentAvailability( + value: ModelAvailability, + timing: AvailabilityClock = clock, +): ModelAvailability { + receiveAvailability(value, timing); + const operations = value.operations.map((item) => + currentOperation(item, timing), + ); + if (operations.every((item, index) => item === value.operations[index])) + return value; + const states = new Set(operations.map((item) => item.status)); + const reasons = new Set(operations.map((item) => item.reasonCode)); + return { + ...value, + operations, + status: states.size === 1 ? operations[0].status : "unknown", + reasonCode: + reasons.size === 1 + ? operations[0].reasonCode + : reasons.has("observation_expired") + ? "observation_expired" + : "mixed_operations", + }; +} + +export function availabilityReason(code: string): string { + if (code === "mixed_operations") return tNow("availability.status.unknown"); + const key = `availability.reason.${code}`; + return hasMessage(key) + ? tNow(key) + : tNow("availability.reason.declaration_incomplete"); +} + +export function availabilityProfile(profile: string): string { + const key = `availability.profile.${profile}`; + return hasMessage(key) ? tNow(key) : tNow("availability.profile.other"); +} + +export function availabilityBlocker( + model: ModelConfig | undefined, + profile: string, + timing: AvailabilityClock = clock, +): string | null { + if (model?.availability) receiveAvailability(model.availability, timing); + const item = model?.availability?.operations.find( + (value) => value.profile === profile, + ); + if (!item) return null; + const current = currentOperation(item, timing); + return current.status === "unavailable" + ? availabilityReason(current.reasonCode) + : null; +} + +/** Service limits are temporary observations, never saved input rules. */ +export function availabilityInputLimit( + model: ModelConfig | undefined, + profile: string, + role: "image" | "video" | "audio", + timing: AvailabilityClock = clock, +): number | null { + if (!model?.availability) return null; + receiveAvailability(model.availability, timing); + const item = model.availability.operations.find( + (value) => value.profile === profile, + ); + if (!item) return null; + const current = currentOperation(item, timing); + const declaration = current.declaration ?? current; + if (declaration.status === "unknown" || declaration.expiresAt === null) + return null; + const limits = (current.inputDetails ?? []) + .filter( + (detail) => + detail.name === role && + detail.source === "service" && + detail.support === "supported" && + detail.maxCount != null, + ) + .map((detail) => detail.maxCount!); + return limits.length ? Math.min(...limits) : null; +} + +/** Local identity excludes credentials; saves additionally invalidate pending reads. */ +export function availabilityModelKey(model: ModelConfig): string { + return JSON.stringify([ + model.id, + model.providerId, + model.kind, + model.url, + model.modelId, + model.protocol, + model.connectionType, + model.authMode, + model.proxyMode, + model.videoAdapter, + model.inputModalities, + model.videoCapabilities, + model.capabilityAutoCheck, + ]); +} + +/** Saved observations require the opaque identity captured before the request. */ +export function availabilityMatchesModel( + model: ModelConfig, + observation: ModelAvailability, + expectedIdentity: string | undefined = model.availability?.identity, +): boolean { + return ( + expectedIdentity !== undefined && + observation.configId === model.id && + observation.modelId === model.modelId && + observation.identity === expectedIdentity + ); +} + +export function nextAvailabilityDelay( + models: readonly ModelConfig[], + timing: AvailabilityClock = clock, +): number | null { + const remaining = models + .map((model) => + model.availability ? availabilityDelay(model.availability, timing) : null, + ) + .filter((value): value is number => value !== null); + return remaining.length ? Math.min(...remaining) : null; +} + +export function nextAvailabilityExpiry( + models: readonly ModelConfig[], + timing: AvailabilityClock = clock, +): number | null { + const delay = nextAvailabilityDelay(models, timing); + return delay === null ? null : timing.wallNow() + delay; +} diff --git a/src/backend/chat.ts b/src/backend/chat.ts index 93de2b62..995a244f 100644 --- a/src/backend/chat.ts +++ b/src/backend/chat.ts @@ -39,6 +39,7 @@ export interface ChatTool { status: "running" | "complete" | "failed" | "interrupted"; textOffset: number; error?: string | null; + code?: string; message?: string | null; taskId?: string | null; arguments?: Record; diff --git a/src/backend/creationJobs.ts b/src/backend/creationJobs.ts index 88e40baf..f7c2a9d4 100644 --- a/src/backend/creationJobs.ts +++ b/src/backend/creationJobs.ts @@ -44,6 +44,7 @@ export function creationJobs( : undefined; return { id: job.id, + revision: job.revision, nodeId: job.originItemId ?? spec.id, title: spec.prompt.slice(0, 60) || tNow("errors.backend.generationTask"), taskId: job.id, diff --git a/src/backend/models.ts b/src/backend/models.ts index 7f506230..2de554c0 100644 --- a/src/backend/models.ts +++ b/src/backend/models.ts @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { errorMessage, json, request } from "./api.ts"; import type { ModelConfig, @@ -6,6 +6,16 @@ import type { VideoAdapter, VideoCapabilities, } from "./rules.ts"; +import { + availabilityModelKey, + availabilityMatchesModel, + currentAvailability, + failedAvailability, + nextAvailabilityDelay, + receiveAvailability, + type ModelAvailability, +} from "./availability.ts"; +import type { AvailabilityCatalog } from "../contracts/generated.ts"; import type { MessageKey } from "../i18n/types.ts"; export const MODEL_KINDS: readonly { @@ -35,9 +45,13 @@ export const MODEL_KINDS: readonly { }, ]; -export type ModelOptions = Pick; +export type ModelOptions = Pick< + ModelConfigInput, + "inputModalities" | "videoAdapter" | "videoCapabilities" +>; export interface ModelConfigInput { + capabilityAutoCheck?: boolean | null; kind: ModelKind; url: string; modelId: string; @@ -80,43 +94,177 @@ export interface GeneralSettings { removeWatermark: boolean; } -/** Model availability is independent of the filesystem project's editable draft. */ +export function checkDraftAvailability( + value: ModelConfigInput, + configId?: string, +) { + return request( + "/model-configs/availability/check", + json("POST", { ...value, configId }), + ).then((value) => receiveAvailability(value)); +} + +/** Saved configurations remain distinct from disposable capability observations. */ export function useBackendModels() { const [models, setModels] = useState([]); + const modelsRef = useRef([]); + const epoch = useRef(0); + const querySequence = useRef(0); + const latestQuery = useRef(new Map()); + const alive = useRef(true); + const [availabilityRevision, setAvailabilityRevision] = useState(0); + const [availabilityError, setAvailabilityError] = useState(""); + const [checkingIds, setCheckingIds] = useState>( + new Set(), + ); const [settings, setSettings] = useState(null); const [error, setError] = useState(""); const [loading, setLoading] = useState(true); const [connected, setConnected] = useState(false); + const storeModels = useCallback((next: ModelConfig[]) => { + if (!alive.current) return; + modelsRef.current = next; + setModels(next); + setAvailabilityRevision((value) => value + 1); + }, []); + useEffect(() => { + alive.current = true; + return () => { + alive.current = false; + epoch.current += 1; + }; + }, []); + const refreshAvailability = useCallback( + async (ids: readonly string[], manual = false) => { + if (!ids.length) return; + const serial = ++querySequence.current; + const initialEpoch = epoch.current; + const initial = new Map( + modelsRef.current + .filter((model) => ids.includes(model.id)) + .map((model) => [ + model.id, + { + key: availabilityModelKey(model), + identity: model.availability?.identity, + }, + ]), + ); + if (!initial.size) return; + initial.forEach((_, id) => latestQuery.current.set(id, serial)); + setCheckingIds((current) => new Set([...current, ...initial.keys()])); + try { + const result = await request( + "/model-configs/availability", + json("POST", { configIds: [...initial.keys()], manual }), + ); + result.models.forEach((value) => receiveAvailability(value)); + if (!alive.current || epoch.current !== initialEpoch) return; + const received = new Map( + result.models.map((item) => [item.configId, item]), + ); + storeModels( + modelsRef.current.map((model) => { + const observation = received.get(model.id); + const expected = initial.get(model.id); + return expected?.key === availabilityModelKey(model) && + latestQuery.current.get(model.id) === serial && + observation && + availabilityMatchesModel(model, observation, expected.identity) + ? { ...model, availability: observation } + : model; + }), + ); + setAvailabilityError(""); + } catch (failure) { + if (!alive.current || epoch.current !== initialEpoch) return; + // An unsuccessful check cannot keep an earlier denial active. + storeModels( + modelsRef.current.map((model) => + initial.get(model.id)?.key === availabilityModelKey(model) && + latestQuery.current.get(model.id) === serial + ? { + ...model, + availability: model.availability + ? failedAvailability(model.availability) + : undefined, + } + : model, + ), + ); + setAvailabilityError(errorMessage(failure)); + } finally { + if (alive.current) + setCheckingIds( + (current) => + new Set( + [...current].filter( + (id) => + !initial.has(id) || latestQuery.current.get(id) !== serial, + ), + ), + ); + } + }, + [storeModels], + ); const refresh = useCallback(async () => { + const initialEpoch = ++epoch.current; try { const [list, general] = await Promise.all([ - request("/model-configs"), + request("/model-configs").then((list) => { + list.forEach((model) => { + if (model.availability) receiveAvailability(model.availability); + }); + return list; + }), request("/settings"), ]); - setModels(list); + if (!alive.current || initialEpoch !== epoch.current) return; + storeModels(list); setSettings(general); setConnected(true); setError(""); } catch (failure) { + if (!alive.current || initialEpoch !== epoch.current) return; setConnected(false); setError(errorMessage(failure)); } finally { - setLoading(false); + if (alive.current) setLoading(false); } - }, []); + }, [storeModels]); useEffect(() => { void refresh(); }, [refresh]); + useEffect(() => { + const delay = nextAvailabilityDelay(models); + if (delay === null) return; + const timer = setTimeout( + () => { + storeModels( + modelsRef.current.map((model) => + model.availability + ? { + ...model, + availability: currentAvailability(model.availability), + } + : model, + ), + ); + }, + Math.max(1, delay + 1), + ); + return () => clearTimeout(timer); + }, [models, storeModels]); const save = async (value: ModelConfigInput, id?: string) => { + epoch.current += 1; const saved = await request( `/model-configs${id ? `/${id}` : ""}`, json(id ? "PATCH" : "POST", value), ); - setModels((current) => - id - ? current.map((item) => (item.id === id ? saved : item)) - : [...current, saved], - ); + // The directory supplies the opaque server identity, including credentials. + await refresh(); + void refreshAvailability([saved.id]); }; const saveMany = async ( value: ModelConfigInput, @@ -124,19 +272,23 @@ export function useBackendModels() { id?: string, modelOptions: Record = {}, ) => { - await request( + epoch.current += 1; + const saved = await request( "/model-configs/batch", json("POST", { ...value, modelIds, modelOptions, configId: id }), ); await refresh(); + void refreshAvailability(saved.map((model) => model.id)); }; const removeService = async (id: string) => { + epoch.current += 1; await request(`/model-services/${id}`, { method: "DELETE" }); await refresh(); }; const remove = async (id: string) => { + epoch.current += 1; await request(`/model-configs/${id}`, { method: "DELETE" }); - setModels((current) => current.filter((item) => item.id !== id)); + storeModels(modelsRef.current.filter((item) => item.id !== id)); }; const updateSettings = async (patch: GeneralSettings) => { try { @@ -144,12 +296,11 @@ export function useBackendModels() { await request("/settings", json("PATCH", patch)), ); } catch (failure) { - // Reconcile a lost response before displaying a previous value as truth. setSettings(null); try { setSettings(await request("/settings")); } catch { - /* Await a successful refresh. */ + /* A later refresh reconciles the save. */ } throw failure; } @@ -166,6 +317,10 @@ export function useBackendModels() { remove, removeService, updateSettings, + availabilityRevision, + availabilityError, + checkingIds, + refreshAvailability, }; } diff --git a/src/backend/rules.ts b/src/backend/rules.ts index 378e4ceb..a31f0fc0 100644 --- a/src/backend/rules.ts +++ b/src/backend/rules.ts @@ -1,3 +1,4 @@ +import { availabilityBlocker, availabilityInputLimit } from "./availability.ts"; import { voiceIssue, type VoiceCapabilities } from "./voices.ts"; import { nodeByteSize } from "../canvas/uploadPolicy.ts"; import type { CanvasNode, GenerationSettings } from "../canvas/model.ts"; @@ -15,6 +16,7 @@ import { frameImageLimitIssue } from "../canvas/frameReferences.ts"; import { nodeDownstreamKinds } from "../canvas/connectionPolicy.ts"; import { tNow } from "../i18n/messages.ts"; import type { MessageKey } from "../i18n/types.ts"; +import type { ModelAvailability } from "../contracts/generated.ts"; /** Model configurations and input rules published by the local API. */ export type ModelKind = "agent" | "image" | "video" | "audio" | "vision"; @@ -57,6 +59,8 @@ export interface InputRules { } export interface ModelConfig { + capabilityAutoCheck?: boolean | null; + availability?: ModelAvailability | null; providerId?: string | null; id: string; kind: ModelKind; @@ -515,6 +519,13 @@ export function backendBlocker( return tNow("errors.rules.transcriptionModel"); if (!isGenerationModel(config)) return tNow("errors.rules.segmentationModel"); + const transmitted = backendReferences(settings, incoming, config); + const profile = settings.mode === "audio" ? (isMusicModel(config) ? "lyrics_to_music" : "text_to_speech") + : settings.mode === "image" ? (transmitted.some((ref) => ref.node.data.kind === "image") ? "image_to_image" : "text_to_image") + : transmitted.some((ref) => ref.node.data.kind === "video") ? "video_to_video" + : transmitted.some((ref) => ref.node.data.kind === "image") ? "image_to_video" : "text_to_video"; + const capabilityIssue = availabilityBlocker(config, profile); + if (capabilityIssue) return capabilityIssue; if (!settings.prompt.trim()) return settings.mode !== "audio" ? tNow("errors.rules.promptRequired") @@ -646,6 +657,14 @@ export function backendBlocker( references.filter( (item) => item.role !== "continuation" && item.node.data.kind === kind, ).length; + for (const kind of ["image", "video", "audio"] as const) { + const limit = availabilityInputLimit(config, profile, kind); + const current = references.filter((item) => item.node.data.kind === kind).length; + if (limit !== null && current > limit) + return tNow(kind === "image" ? "errors.capability.imageCount" : "errors.rules.serviceInputLimit", { + kind: tNow(LABELS[kind] ?? "errors.rules.kindImage"), limit, current, + }); + } for (const [kind, limit] of [ ["image", rules.maxImages], ["video", rules.maxVideos], diff --git a/src/backend/tools.ts b/src/backend/tools.ts index 940157e3..f5770a1d 100644 --- a/src/backend/tools.ts +++ b/src/backend/tools.ts @@ -17,6 +17,8 @@ import { } from "../canvas/toolTaskText.ts"; import { toolReasonText } from "../canvas/toolReasonText.ts"; import { tNow } from "../i18n/messages.ts"; +import type { Translate } from "../i18n/I18nProvider.tsx"; +import { defaultTranslate } from "../canvas/assistantTranslate.ts"; import { EXTENSIONS, fileName, @@ -39,6 +41,7 @@ export interface ToolCatalogEntry { id: CanvasToolId; available: boolean; reason?: string; + reasonCode?: string; note: string; parametersConfirmed: boolean; fields: ToolField[]; @@ -167,7 +170,7 @@ export interface ToolRequest { } /** Availability and service-supplied options for every tool, refreshed with the model list. */ -export function useToolCatalog(connected: boolean, revision: unknown) { +export function useToolCatalog(connected: boolean, revision: unknown, t: Translate = defaultTranslate) { const [tools, setTools] = useState([]); const [error, setError] = useState(""); const [selectionPreview, setSelectionPreview] = useState<{ @@ -180,16 +183,8 @@ export function useToolCatalog(connected: boolean, revision: unknown) { tools: ToolCatalogEntry[]; selectionPreview?: { available: boolean; reason?: string }; }>("/tools"); - setTools( - value.tools.map((entry) => ({ - ...entry, - reason: toolReasonText(entry.reason), - })), - ); - setSelectionPreview({ - ...(value.selectionPreview ?? { available: false }), - reason: toolReasonText(value.selectionPreview?.reason), - }); + setTools(value.tools); + setSelectionPreview(value.selectionPreview ?? { available: false }); setError(""); } catch (failure) { setTools([]); @@ -204,19 +199,22 @@ export function useToolCatalog(connected: boolean, revision: unknown) { setSelectionPreview({ available: false }); } }, [connected, revision, refresh]); + const displayedTools = tools.map((entry) => ({ ...entry, reason: toolReasonText(entry.reason, t, entry.reasonCode) })); const capabilities: ToolCapabilities = Object.fromEntries( - tools.map((entry) => [ + displayedTools.map((entry) => [ entry.id, { available: entry.available, reason: entry.reason, + reasonCode: entry.reasonCode, fields: entry.fields, models: entry.models, parametersConfirmed: entry.parametersConfirmed, }, ]), ); - return { tools, capabilities, selectionPreview, error, refresh }; + return { tools: displayedTools, capabilities, + selectionPreview: { ...selectionPreview, reason: toolReasonText(selectionPreview.reason, t) }, error, refresh }; } /** Sources and masks become project assets once; identical files are reused. */ diff --git a/src/canvas/AssistantPanel.tsx b/src/canvas/AssistantPanel.tsx index 3efd0d22..28116f6c 100644 --- a/src/canvas/AssistantPanel.tsx +++ b/src/canvas/AssistantPanel.tsx @@ -751,7 +751,7 @@ export function AssistantPanel({
- {!chat.loaded ? ( + {chat.availabilityIssue ? chat.availabilityIssue : !chat.loaded ? ( chat.changing ? t("assistant.panel.switching") : chat.error diff --git a/src/canvas/AssistantReply.tsx b/src/canvas/AssistantReply.tsx index e60d4e7e..13d7d57c 100644 --- a/src/canvas/AssistantReply.tsx +++ b/src/canvas/AssistantReply.tsx @@ -29,7 +29,7 @@ function ToolActivity({ tool, t }: { tool: ChatTool; t: Translate }) {
{(tool.error || tool.message) && ( -

{assistantToolText(tool.error || tool.message!, t)}

+

{assistantToolText(tool.error || tool.message!, t, tool.code)}

)} {tool.arguments &&
{t("assistant.reply.arguments")} diff --git a/src/canvas/DraftAvailabilityCheck.tsx b/src/canvas/DraftAvailabilityCheck.tsx new file mode 100644 index 00000000..6230c01f --- /dev/null +++ b/src/canvas/DraftAvailabilityCheck.tsx @@ -0,0 +1,76 @@ +import { useEffect, useRef, useState } from "react"; +import { + checkDraftAvailability, + type ModelConfigInput, +} from "../backend/models.ts"; +import type { ModelAvailability } from "../backend/availability.ts"; +import { errorMessage } from "../backend/api.ts"; +import { ModelAvailabilityCard } from "./ModelAvailabilityCard"; + +/** A draft check never publishes an observation into the saved model catalog. */ +export function DraftAvailabilityCheck({ + connection, + configId, + modelId, +}: { + connection: ModelConfigInput; + configId?: string; + modelId: string; +}) { + const signature = JSON.stringify([connection, configId, modelId]); + const latest = useRef(signature); + latest.current = signature; + const serial = useRef(0); + const alive = useRef(true); + const [value, setValue] = useState<{ + signature: string; + observation: ModelAvailability; + } | null>(null); + const [pending, setPending] = useState(null); + const [failure, setFailure] = useState<{ + signature: string; + message: string; + } | null>(null); + useEffect(() => { + alive.current = true; + return () => { + alive.current = false; + }; + }, []); + const check = async () => { + const attempt = ++serial.current; + setPending(signature); + setFailure(null); + try { + const observation = await checkDraftAvailability( + { ...connection, modelId }, + configId, + ); + if ( + alive.current && + latest.current === signature && + serial.current === attempt + ) + setValue({ signature, observation }); + } catch (error) { + if ( + alive.current && + latest.current === signature && + serial.current === attempt + ) { + setValue(null); + setFailure({ signature, message: errorMessage(error) }); + } + } finally { + if (alive.current && serial.current === attempt) setPending(null); + } + }; + return ( + void check()} + error={failure?.signature === signature ? failure.message : undefined} + /> + ); +} diff --git a/src/canvas/ModelAvailabilityCard.tsx b/src/canvas/ModelAvailabilityCard.tsx new file mode 100644 index 00000000..fbc31136 --- /dev/null +++ b/src/canvas/ModelAvailabilityCard.tsx @@ -0,0 +1,133 @@ +import { useEffect, useState } from "react"; +import { + availabilityDelay, + availabilityProfile, + availabilityReason, + currentAvailability, + type ModelAvailability, +} from "../backend/availability.ts"; +import { useI18n } from "../i18n/I18nProvider"; +import { + availabilityCardDetails, + availabilityNotice, + availabilityRow, +} from "./availabilityPresentation.ts"; + +/** Observations do not edit the model's saved input rules or voice selection. */ +export function ModelAvailabilityCard({ + value, + onRefresh, + checking = false, + title, + error, +}: { + value?: ModelAvailability | null; + onRefresh?: () => void; + checking?: boolean; + title?: string; + error?: string; +}) { + const { t, locale } = useI18n(); + const [tick, setTick] = useState(0); + useEffect(() => { + const delay = value ? availabilityDelay(value) : null; + if (delay === null) return; + const timer = setTimeout( + () => setTick((previous) => previous + 1), + Math.max(1, delay + 1), + ); + return () => clearTimeout(timer); + }, [value, tick]); + const current = value ? currentAvailability(value) : null; + const notice = availabilityNotice(current); + const message = checking + ? null + : error || (notice ? availabilityReason(notice) : null); + const details = availabilityCardDetails(current?.operations ?? []); + const commonSource = + details.length > 0 && + details.every((detail) => detail.source === details[0].source) + ? details[0].source + : null; + return ( +
+
+ {title ?? t("availability.title")} + {onRefresh && ( + + )} +
+ {message &&

{message}

} + {current?.operations.length ? ( +
    + {current.operations.map((item) => { + const row = availabilityRow(item, notice); + return ( +
  • + {availabilityProfile(item.profile)} +
    + {row.text} +
    +
    • + ); + })} +
+ ) : null} + {details.length > 0 && ( +
+ + {t("availability.details")} + {commonSource && {commonSource}} + +
+ {details.map((detail, index) => ( +
+
{detail.label}
+
+ {detail.text} + {detail.scope && {detail.scope}} + {!commonSource && {detail.source}} +
+
+ ))} +
+
+ )} + {current?.checkedAt != null && ( + + {t("availability.checkedAt", { + time: new Date(current.checkedAt).toLocaleTimeString(locale), + })} + + )} +
+ ); +} diff --git a/src/canvas/ModelServiceForm.tsx b/src/canvas/ModelServiceForm.tsx index c3d775ad..3fe3b3d0 100644 --- a/src/canvas/ModelServiceForm.tsx +++ b/src/canvas/ModelServiceForm.tsx @@ -1,3 +1,4 @@ +import { DraftAvailabilityCheck } from "./DraftAvailabilityCheck"; import { ProviderPicker } from "./ProviderPicker"; import { useId, useState, type FormEvent } from "react"; import { errorMessage } from "../backend/api.ts"; @@ -52,26 +53,6 @@ const PROTOCOLS: Record< ], }; -const PLACEHOLDERS: Record = { - agent: ["http://127.0.0.1:8100/v1", "Qwen/Qwen3-30B-A3B"], - image: ["http://127.0.0.1:8200/v1", "Tongyi-MAI/Z-Image-Turbo"], - video: ["http://127.0.0.1:8010/v1", "MiniMaxAI/MiniMax-H3"], - audio: ["http://127.0.0.1:8220/v1", "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"], - vision: ["http://127.0.0.1:8230/v1", "facebook/sam3"], -}; -const MUSIC_PLACEHOLDERS: [string, string] = [ - "http://127.0.0.1:8240/v1", - "m-a-p/YuE2-3B", -]; -const SCORE_PLACEHOLDERS: [string, string] = [ - "http://127.0.0.1:8260/v1", - "m-a-p/SheetSage2", -]; -const DEPTH_PLACEHOLDERS: [string, string] = [ - "http://127.0.0.1:8250/v1", - "depth-anything/Video-Depth-Anything-Small", -]; - export function ModelServiceForm({ kind, existing, @@ -107,6 +88,7 @@ export function ModelServiceForm({ modelId: existing?.modelId ?? "", apiKey: "", connectionType: existing?.connectionType ?? "self_hosted", + capabilityAutoCheck: existing?.capabilityAutoCheck, protocol: existing && isSegmentationModel(existing) ? "vllm_omni_segmentation" @@ -144,18 +126,6 @@ export function ModelServiceForm({ const segmentation = value.protocol === "vllm_omni_segmentation"; const depth = value.protocol === "vllm_omni_depth"; const score = value.protocol === "vllm_omni_score"; - const placeholders = - value.protocol === "minimax_video_v2" - ? ["https://api.minimax.cn/v2", "MiniMax-H3"] - : value.protocol === "kling_video" - ? ["https://api-beijing.klingai.com", "kling-4.0"] - : value.protocol === "vllm_omni_music" - ? MUSIC_PLACEHOLDERS - : score - ? SCORE_PLACEHOLDERS - : depth - ? DEPTH_PLACEHOLDERS - : PLACEHOLDERS[segmentation ? "vision" : kind]; const hasSavedKey = Boolean( connection && connection.authMode === "bearer" && @@ -311,6 +281,7 @@ export function ModelServiceForm({ url: model.url, modelId: model.modelId, connectionType: model.connectionType, + capabilityAutoCheck: model.capabilityAutoCheck, inputModalities: model.inputModalities, videoAdapter: model.videoAdapter, videoCapabilities: model.videoCapabilities, @@ -431,7 +402,7 @@ export function ModelServiceForm({ required maxLength={2048} value={value.url} - placeholder={placeholders[0]} + placeholder={t("models.service.urlPlaceholder")} aria-describedby={`${connectionFieldId}-url-note`} onChange={(event) => update({ url: event.target.value })} /> @@ -464,6 +435,10 @@ export function ModelServiceForm({ {t("models.service.proxyNote")}

+
+ +
{t("models.service.minimaxNote")}

)} ( + <> + ({ ...current, [id]: message })) } /> + )} /> {segmentation && ( diff --git a/src/canvas/ModelSettings.tsx b/src/canvas/ModelSettings.tsx index 060bd2de..cecabaa3 100644 --- a/src/canvas/ModelSettings.tsx +++ b/src/canvas/ModelSettings.tsx @@ -1,7 +1,8 @@ +import { ModelAvailabilityCard } from "./ModelAvailabilityCard"; import { ModelSetup, modelSetupMissing, type SetupKind } from "./ModelSetup"; import { ModelLogo } from "./ModelLogo"; import { providerLabel } from "./providerPresets"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { errorMessage } from "../backend/api.ts"; import { MODEL_KINDS, type BackendModels } from "../backend/models.ts"; import type { ModelConfig } from "../backend/rules.ts"; @@ -44,6 +45,11 @@ export function ModelSettings({ catalog, onClose, guided = false }: { catalog: B models.findIndex((item) => item.providerId === model.providerId) === index, ); + const visibleIds = models.map((model) => model.id).join(","); + const checkAvailability = catalog.refreshAvailability; + useEffect(() => { + if (visibleIds && catalog.connected) void checkAvailability(visibleIds.split(",")); + }, [visibleIds, catalog.connected, checkAvailability]); const [skipped, setSkipped] = useState([]); const missing = modelSetupMissing(catalog.models); const pending = missing.filter(kind => !skipped.includes(kind)); @@ -185,6 +191,13 @@ export function ModelSettings({ catalog, onClose, guided = false }: { catalog: B ))} {members.length > 3 && +{members.length - 3}}
+
+ {t("availability.title")} + {members.map((member) => void catalog.refreshAvailability([member.id], true)} />)} +
{providerLabel(model)}{" "} diff --git a/src/canvas/NodeComposer.tsx b/src/canvas/NodeComposer.tsx index 7bbe4eb5..467d8fc0 100644 --- a/src/canvas/NodeComposer.tsx +++ b/src/canvas/NodeComposer.tsx @@ -1,3 +1,4 @@ +import { availabilitySummary } from "./availabilityPresentation.ts"; import { VoiceLibrary } from "./VoiceLibrary"; import { modelDescription } from "./modelDescriptions"; import { SearchableSelect } from "./SearchableSelect"; @@ -61,6 +62,7 @@ export interface NodeComposerProps { running?: boolean; /** Models configured in the local API, usable alongside the catalog. */ backendModels?: readonly ModelConfig[]; + onCheckModels?: (ids: readonly string[]) => Promise; referenceControls?: GenerationReferenceControls; } @@ -85,11 +87,16 @@ export function NodeComposer({ onPickReference, running = false, backendModels = [], + onCheckModels, referenceControls, }: NodeComposerProps) { const { t } = useI18n(); const [openPanel, setPanel] = useState(null); const panel = running ? null : openPanel; + const visibleModelIds = backendModels.filter((item) => item.kind === selectedNode?.data.kind).map((item) => item.id).join(","); + useEffect(() => { + if (selectedNode && visibleModelIds) void onCheckModels?.(visibleModelIds.split(",")); + }, [selectedNode?.id, visibleModelIds, onCheckModels]); const rootRef = useRef(null); const panelAnchor = useRef(null); const mentions = useRef(null); @@ -487,7 +494,7 @@ export function NodeComposer({ id: `config:${item.id}`, name: modelLabel(item), modelId: item.modelId, - description: modelDescription(item), + description: [modelDescription(item), availabilitySummary(item.availability)].filter(Boolean).join(" · ") || undefined, group: providerLabel(item), })), ]} diff --git a/src/canvas/NodeTaskPanel.tsx b/src/canvas/NodeTaskPanel.tsx index e9746e62..71898847 100644 --- a/src/canvas/NodeTaskPanel.tsx +++ b/src/canvas/NodeTaskPanel.tsx @@ -17,6 +17,7 @@ import { import { translateTool, translateToolText } from "./toolText.ts"; import { displayNodeTitle } from "./nodeTitle.ts"; import { useI18n } from "../i18n/I18nProvider"; +import { toolReasonText } from "./toolReasonText.ts"; import "./node-task-panel.css"; /** A single source-bound operation using the service's capability catalog. */ @@ -103,7 +104,7 @@ export function NodeTaskPanel({ errors[0] ?? (runnable ? t("tools.nodeTask.presetNote") - : (capability?.reason ?? t("tools.nodeTask.notConnected"))); + : (toolReasonText(capability?.reason, t) ?? t("tools.nodeTask.notConnected"))); const models = capability?.models?.["image.edit"] ?? []; return (
{runnable ? (errors[0] ?? t("tools.nodeTask.remoteNote")) - : `${capability?.reason ?? t("tools.nodeTask.unavailable")}${t("tools.nodeTask.saveHint")}`} + : `${toolReasonText(capability?.reason, t) ?? t("tools.nodeTask.unavailable")}${t("tools.nodeTask.saveHint")}`}

{ onDraftChange(task); diff --git a/src/canvas/TextNodeComposer.tsx b/src/canvas/TextNodeComposer.tsx index 6c18cedf..2a610ff8 100644 --- a/src/canvas/TextNodeComposer.tsx +++ b/src/canvas/TextNodeComposer.tsx @@ -20,6 +20,7 @@ export function TextNodeComposer({ node, incoming, models, + onCheckModels, onUpdate, onRun, onCancel, @@ -30,6 +31,7 @@ export function TextNodeComposer({ node: CanvasNode; incoming: CanvasNode[]; models: readonly ModelConfig[]; + onCheckModels?: (ids: readonly string[]) => Promise; onUpdate: (data: Partial) => void; onRun: () => void; onCancel: () => void; @@ -38,6 +40,8 @@ export function TextNodeComposer({ referenceControls?: GenerationReferenceControls; }) { const t = useT(); + const agentIds = models.filter((model) => model.kind === "agent").map((model) => model.id).join(","); + useEffect(() => { if (agentIds) void onCheckModels?.(agentIds.split(",")); }, [node.id, agentIds, onCheckModels]); const root = useRef(null), anchor = useRef(null); const [openPanel, setPanel] = useState<"parameters" | null>(null); diff --git a/src/canvas/assistantToolText.ts b/src/canvas/assistantToolText.ts index cdb3b718..9c659c4f 100644 --- a/src/canvas/assistantToolText.ts +++ b/src/canvas/assistantToolText.ts @@ -3,6 +3,7 @@ import type { Translate } from "../i18n/I18nProvider.tsx"; import { getActiveLocale } from "../i18n/messages.ts"; import type { MessageKey } from "../i18n/types.ts"; import { defaultTranslate } from "./assistantTranslate.ts"; +import { capabilityErrorText } from "./capabilityErrorText.ts"; const CJK = /[\u3400-\u9fff]/; @@ -24,7 +25,12 @@ const BY_MESSAGE: Record = { export function assistantToolText( text: string, t: Translate = defaultTranslate, + code?: string, ): string { if (getActiveLocale() !== "en-US" || !CJK.test(text)) return text; + const capability = capabilityErrorText(text, t); + if (capability) return capability; + // Codes describe the category; known detail above always takes priority. + if (code === "capability_paused") return t("errors.capability.paused"); return t(BY_MESSAGE[text] ?? "assistant.reply.toolError"); } diff --git a/src/canvas/availabilityPresentation.ts b/src/canvas/availabilityPresentation.ts new file mode 100644 index 00000000..a8845ce8 --- /dev/null +++ b/src/canvas/availabilityPresentation.ts @@ -0,0 +1,191 @@ +import { + availabilityProfile, + availabilityReason, + currentAvailability, + type ModelAvailability, + type OperationAvailability, +} from "../backend/availability.ts"; +import { hasMessage, tNow } from "../i18n/messages.ts"; + +const SHARED_REASONS = new Set([ + "not_checked", + "execution_paused", + "observation_expired", + "query_timeout", + "query_busy", + "query_failed", + "connection_failed", + "authentication_failed", + "protocol_unavailable", + "invalid_response", + "unsupported_version", + "model_not_declared", +]); + +/** This only groups server decisions for display; it never decides support. */ +export function availabilityNotice( + value: ModelAvailability | null, +): string | null { + if (!value) return "not_checked"; + if (SHARED_REASONS.has(value.reasonCode)) return value.reasonCode; + const reasons = new Set(value.operations.map((item) => item.reasonCode)); + const reason = value.operations[0]?.reasonCode; + return reasons.size === 1 && + reason && + value.operations.every((item) => item.status !== "available") + ? reason + : null; +} + +export function availabilityRow( + item: OperationAvailability, + notice: string | null, +) { + const displayed = + notice === "execution_paused" && + item.reasonCode === "execution_paused" && + item.declaration + ? item.declaration + : item; + const shared = + displayed.reasonCode === notice || + (displayed.status === "unknown" && + notice !== null && + SHARED_REASONS.has(notice)); + return { + status: displayed.status, + muted: shared && displayed.status === "unknown", + text: shared + ? "—" + : displayed.status === "available" + ? tNow("availability.status.available") + : availabilityReason(displayed.reasonCode), + }; +} + +/** A selector describes the affected modes rather than the aggregate status. */ +export function availabilitySummary( + value?: ModelAvailability | null, +): string | undefined { + if (!value) return undefined; + const current = currentAvailability(value); + const notice = availabilityNotice(current); + if (notice) return availabilityReason(notice); + const details = current.operations + .filter((item) => item.status !== "available") + .map( + (item) => + `${availabilityProfile(item.profile)}: ${availabilityReason(item.reasonCode)}`, + ); + return details.length ? details.join("; ") : undefined; +} + +export function availabilityInputDetails(item: OperationAvailability) { + return (item.inputDetails ?? []).flatMap((detail) => { + const key = `availability.input.${detail.name}`; + if (!hasMessage(key)) return []; + const values: string[] = []; + if (detail.support === "unsupported") + values.push(tNow("availability.detail.unsupported")); + else if (detail.support === "unknown") + values.push(tNow("availability.status.unknown")); + else { + const countKind = + detail.name === "image" || detail.name === "reference_image" + ? "image" + : "count"; + if (detail.minCount != null && detail.maxCount != null) + values.push( + tNow(`availability.detail.${countKind}Range`, { + min: detail.minCount, + max: detail.maxCount, + }), + ); + else if (detail.maxCount != null) + values.push( + tNow( + countKind === "image" + ? "availability.detail.maxImages" + : "availability.detail.maxCount", + { count: detail.maxCount }, + ), + ); + else if (detail.minCount != null) + values.push( + tNow( + countKind === "image" + ? "availability.detail.minImages" + : "availability.detail.minCount", + { count: detail.minCount }, + ), + ); + if (detail.min != null && detail.max != null) + values.push(`${detail.min}–${detail.max}`); + else if (detail.min != null) + values.push(tNow("availability.detail.minimum", { value: detail.min })); + else if (detail.max != null) + values.push(tNow("availability.detail.maximum", { value: detail.max })); + if (detail.multipleOf != null) + values.push( + tNow("availability.detail.multiple", { value: detail.multipleOf }), + ); + if (detail.values?.length) + values.push( + detail.values + .map((value) => + typeof value === "boolean" + ? tNow( + value + ? "availability.detail.yes" + : "availability.detail.no", + ) + : String(value), + ) + .join(", "), + ); + if (!values.length) values.push(tNow("availability.status.available")); + } + return [ + { + name: detail.name, + label: tNow(key), + text: values.join(" · "), + source: tNow( + detail.source === "service" + ? "availability.detail.service" + : "availability.detail.application", + ), + }, + ]; + }); +} + +/** Shared limits appear once; differing route requirements retain their mode label. */ +export function availabilityCardDetails( + items: readonly OperationAvailability[], +) { + const details = new Map< + string, + ReturnType[number] & { profiles: string[] } + >(); + for (const item of items) + for (const detail of availabilityInputDetails(item)) { + const key = JSON.stringify([detail.name, detail.text, detail.source]); + const existing = details.get(key); + if (existing) existing.profiles.push(availabilityProfile(item.profile)); + else + details.set(key, { + ...detail, + profiles: [availabilityProfile(item.profile)], + }); + } + const values = [...details.values()]; + return values.map((detail) => ({ + ...detail, + scope: values.some( + (other) => other !== detail && other.name === detail.name, + ) + ? [...new Set(detail.profiles)].join(" / ") + : undefined, + })); +} diff --git a/src/canvas/capabilityErrorText.ts b/src/canvas/capabilityErrorText.ts new file mode 100644 index 00000000..87e52768 --- /dev/null +++ b/src/canvas/capabilityErrorText.ts @@ -0,0 +1,94 @@ +/** Translate stored diagnostics without discarding their operation, input, or count. */ +import type { Translate } from "../i18n/I18nProvider.tsx"; +import type { MessageKey } from "../i18n/types.ts"; +import { defaultTranslate } from "./assistantTranslate.ts"; + +const EXACT: Record = { + "服务已暂停，请恢复后重试。": "paused", + "请补充所需素材。": "required", + "图片文件过大，请压缩后重试。": "imageBytes", + "图片总大小超出限制，请减少图片或压缩后重试。": "totalBytes", + "标注参考图片需要配合选区使用。": "annotationSelection", + "当前图片尺寸或数量不支持选区输入。": "selectionInputs", + "选区与图片的尺寸不一致。": "selectionDimensions", + "请先选择需要修改的区域。": "selectionRequired", + "无法读取图片或选区，请重新选择。": "selectionUnreadable", + "来源素材必须是图片。": "sourceType", + "图片尺寸不符合服务要求。": "dimensions", +}; +const LABELS: Record = { + 文字回复: "textReplies", + 图片回复: "imageReplies", + 图片生成: "imageGeneration", + 图片编辑: "imageEditing", + 文字生成视频: "textVideo", + 图片生成视频: "imageVideo", + 视频编辑: "videoEditing", + 语音合成: "speech", + 音乐生成: "music", + 此操作: "operation", + 图片: "image", + 音频: "audio", + 视频: "video", + 文字: "text", + 音色: "voice", + 参考音频: "referenceAudio", + 风格说明: "style", + 参考文本: "transcript", + 选区: "selection", + 标注参考图片: "annotation", + 素材: "inputs", + 所需素材: "inputs", + 图片宽度: "width", + 图片高度: "height", + 图片尺寸: "size", + 时长: "duration", + 语速: "speed", + 语言: "language", + 帧率: "fps", + 帧数: "frames", + 音频生成选项: "generateAudio", + 所选参数: "parameter", +}; + +export function capabilityErrorText( + text: string, + t: Translate = defaultTranslate, +): string | undefined { + const key = (name: string) => `errors.capability.${name}` as MessageKey; + if (EXACT[text]) return t(key(EXACT[text])); + let match = /^最多支持 (\d+) 张图片，当前为 (\d+) 张。$/.exec(text); + if (match) + return t(key("imageCount"), { limit: match[1]!, current: match[2]! }); + match = /^该工具需要 (\d+) 张来源图片。$/.exec(text); + if (match) return t(key("sourceCount"), { count: match[1]! }); + match = /^最多支持 (\d+) 个(.+)输入，当前为 (\d+) 个。$/.exec(text); + if (match && LABELS[match[2]!]) + return t(key("inputCount"), { + limit: match[1]!, + input: t(key(`label.${LABELS[match[2]!]}`)), + current: match[3]!, + }); + for (const [pattern, template] of [ + [/^该模型不支持(.+)输入。$/, "inputUnsupported"], + [/^该模型不支持(.+)。$/, "unsupported"], + [/^服务未开放(.+)。$/, "disabled"], + [/^请添加(.+)。$/, "add"], + [/^(.+)不符合服务要求。$/, "parameter"], + ] as const) { + match = pattern.exec(text); + if (match && LABELS[match[1]!]) + return t(key(template), { label: t(key(`label.${LABELS[match[1]!]}`)) }); + } + match = /^请提供(.+)。$/.exec(text); + if (match) { + const labels = match[1]!.split("或"); + if (labels.every((label) => LABELS[label])) + return t(key("provide"), { + inputs: labels + .map((label) => t(key(`label.${LABELS[label]}`))) + .join(t(key("or"))), + }); + } + return undefined; +} diff --git a/src/canvas/generationFailureText.ts b/src/canvas/generationFailureText.ts index 23246b64..0ecb8fd0 100644 --- a/src/canvas/generationFailureText.ts +++ b/src/canvas/generationFailureText.ts @@ -3,6 +3,7 @@ import type { GenerationFailure } from "../contracts/generated.ts"; import type { Translate } from "../i18n/I18nProvider.tsx"; import type { MessageKey } from "../i18n/types.ts"; import { defaultTranslate } from "./assistantTranslate.ts"; +import { capabilityErrorText } from "./capabilityErrorText.ts"; /** * The backend stores Chinese diagnostics in the job record. Known messages map @@ -17,6 +18,7 @@ const BY_MESSAGE: Record = { "当前账户没有调用该模型的权限。": "errors.generationFailure.permission", "模型账户余额或套餐额度不足。": "errors.generationFailure.quota", "模型调用频率或配额受限。": "errors.generationFailure.rateLimit", + "模型服务繁忙或请求受限。": "errors.generationFailure.rateLimit", "生成服务暂时不可用。": "errors.generationFailure.temporary", "生成服务未能完成当前操作。": "errors.generationFailure.provider", "音频参考不能单独使用，请搭配参考图片。": @@ -34,6 +36,7 @@ const BY_MESSAGE: Record = { "供应商未找到该任务。": "errors.generationFailure.notFoundTask", "生成接口或模型不存在。": "errors.generationFailure.notFoundEndpoint", "生成结果下载失败。": "errors.generationFailure.download", + "生成结果地址解析为非公网地址，下载已停止。": "errors.generationFailure.downloadAddress", "生成服务任务已失败。": "errors.generationFailure.failed", "生成请求的提交结果不明。": "errors.generationFailure.submissionUnknown", "生成服务连接超时或中断。": "errors.generationFailure.temporary", @@ -59,6 +62,8 @@ export function generationFailureMessage( failure: GenerationFailure, t: Translate = defaultTranslate, ): string { + const capability = capabilityErrorText(failure.message, t); + if (capability) return capability; if (failure.providerCode === "voice_not_found") return t("errors.generationFailure.voiceNotFound"); const exact = BY_MESSAGE[failure.message]; diff --git a/src/canvas/generationHistory.ts b/src/canvas/generationHistory.ts index c70daaa6..f1f1bfed 100644 --- a/src/canvas/generationHistory.ts +++ b/src/canvas/generationHistory.ts @@ -17,6 +17,8 @@ import type { CanvasToolTask } from "./toolCatalog.ts"; export interface GenerationJob { id: string; + /** Authoritative execution revision, when supplied by the service ledger. */ + revision?: number; nodeId: string; title: string; status: diff --git a/src/canvas/model-settings.css b/src/canvas/model-settings.css index 21397785..0be4112d 100644 --- a/src/canvas/model-settings.css +++ b/src/canvas/model-settings.css @@ -661,3 +661,21 @@ .cw-model-editor-scroll > .cw-model-connection-details { margin: 0 0 16px; } .cw-model-editor-scroll > .cw-provider-key { margin: 0 0 18px; } + +.cw-model-availability { padding: 12px 0; font-size: var(--settings-label-size, 13px); } +.cw-model-option > .cw-model-availability { flex-basis: 100%; width: 100%; min-width: 0; border-top: 1px solid var(--settings-border); } +.cw-model-availability-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; } +.cw-model-availability p, .cw-model-availability small { color: var(--settings-muted); } +.cw-model-availability ul { display: grid; gap: 8px; margin: 10px 0; padding: 0; list-style: none; } +.cw-model-availability li { display: grid; grid-template-columns: minmax(80px, 1fr) minmax(0, 3fr); gap: 12px; overflow-wrap: anywhere; } +.cw-model-availability-details { padding: 12px 16px; border-top: 1px solid var(--settings-border); } +.cw-model-availability-details summary { cursor: pointer; } +.cw-model-availability [data-availability="unavailable"] { color: var(--cw-danger, #b45336); } +.cw-model-availability [data-stale] { opacity: .55; } +.cw-model-input-details { margin-top: 4px; color: var(--settings-muted); } +.cw-model-input-details summary { cursor: pointer; } +.cw-model-input-details dl { margin: 6px 0; } +.cw-model-input-details dl > div { display: grid; grid-template-columns: minmax(80px, 1fr) minmax(0, 2fr); gap: 8px; } +.cw-model-input-details dd { margin: 0; } +.cw-model-input-details small { display: block; } +.cw-model-input-details summary small { display: inline; margin-inline-start: 8px; } diff --git a/src/canvas/textGeneration.ts b/src/canvas/textGeneration.ts index 4f668e10..4aecccac 100644 --- a/src/canvas/textGeneration.ts +++ b/src/canvas/textGeneration.ts @@ -1,3 +1,4 @@ +import { availabilityBlocker } from "../backend/availability.ts"; import type { ModelConfig } from "../backend/rules.ts"; import { RULE_FACTS } from "../contracts/ruleFacts.generated.ts"; import type { CanvasDocument, CanvasNode, AgentNodeSettings } from "./model.ts"; @@ -21,6 +22,8 @@ export function agentGenerationBlocker( ) { if (!model || model.kind !== "agent") return tNow("errors.text.noAgent"); + const capabilityIssue = availabilityBlocker(model, "text_response"); + if (capabilityIssue) return capabilityIssue; if (!node.data.prompt.trim()) return tNow("errors.text.promptRequired"); const settings = agentNodeSettings(node); if ( diff --git a/src/canvas/toolCatalog.ts b/src/canvas/toolCatalog.ts index 7d66769f..891fb35b 100644 --- a/src/canvas/toolCatalog.ts +++ b/src/canvas/toolCatalog.ts @@ -96,6 +96,7 @@ export interface CanvasToolTask { export interface ToolCapability { available: boolean; reason?: string; + reasonCode?: string; /** Adapter-supplied fields own dynamic model/option constraints. */ fields?: readonly ToolField[]; parametersConfirmed?: boolean; @@ -267,7 +268,7 @@ export const CANVAS_TOOLS: readonly CanvasToolDefinition[] = [ tool( "layer-decompose", "图层分解", - "原站实测提供智能拆分、框选拆分和按描述拆分。框选需原图矩形选区；实际拆分及报价由真实服务提供。", + "按对象、选区或描述拆分图层；可用方式由连接的服务提供。", [ { key: "mode", @@ -300,14 +301,14 @@ export const CANVAS_TOOLS: readonly CanvasToolDefinition[] = [ tool( "product-replacement", "商品替换", - "使用场景图和商品图替换主体，支持选区与提示词；原站已实测自动主体识别和 2K 结果。", + "使用场景图和商品图替换主体，支持选区与提示词。", [prompt, choice("resolution", "清晰度", ["2K", "4K"])], { minSources: 2, maxSources: 2 }, ), tool( "multi-size", "智能多尺寸", - "按原站实测画幅和像素尺寸创建草稿；自定义范围与实际服务协议尚未接入。", + "设置目标画幅与输出尺寸。", [ { key: "aspectRatio", @@ -345,7 +346,7 @@ export const CANVAS_TOOLS: readonly CanvasToolDefinition[] = [ tool( "dynamic-poster", "动态海报", - "基于原图比例创建动态海报请求；本轮原站入口仅提供 Seedance 2.5，实际生成需连接服务。", + "按原图比例生成动态海报，需要连接视频生成服务。", [ prompt, choice("model", "模型", ["Seedance 2.5"]), @@ -362,7 +363,7 @@ export const CANVAS_TOOLS: readonly CanvasToolDefinition[] = [ choice("resolution", "清晰度", ["1.5K", "2K"]), { ...choice("aspectRatio", "画幅", ["1:1"]), - hint: "本轮只核对 1:1；其他画幅待服务确认。", + hint: "当前预设为 1:1。", }, ], { maxSources: 6, parameterCoverage: "entry-only" }, @@ -409,12 +410,12 @@ export const CANVAS_TOOLS: readonly CanvasToolDefinition[] = [ tool( "video-edit", "视频编辑", - "原站实测支持 4–30 秒素材、提示词及矩形/画笔/箭头/文字/橡皮擦/标记；本轮未见模型选择。这里只记录意图，视频标注数据尚未接入。", + "设置需要重拍的片段及修改要求；可用编辑方式由服务提供。", [ { ...prompt, required: true }, { key: "annotationTool", - label: "原站标注工具", + label: "标注工具", type: "select", options: [ { value: "rectangle", label: "矩形" }, @@ -439,14 +440,14 @@ export const CANVAS_TOOLS: readonly CanvasToolDefinition[] = [ tool( "video-subtitle-erasure", "字幕擦除", - "本轮原站实测没有额外设置；实际视频处理依赖真实服务。", + "擦除视频中的字幕，需要连接视频处理服务。", [], { category: "video", sources: ["video"], parameterCoverage: "entry-only" }, ), tool( "video-matting", "视频抠像", - "原站实测可选黑色、绿色或透明背景；输出格式与处理协议由真实服务确定。", + "选择黑色、绿色或透明背景；输出格式由服务提供。", [ { key: "background", @@ -465,7 +466,7 @@ export const CANVAS_TOOLS: readonly CanvasToolDefinition[] = [ tool( "video-translation", "视频翻译", - "静态代码存在候选入口，本轮原站未见；当前可用性和参数未知。", + "需要连接提供此功能的服务。", [], { category: "video", sources: ["video"], parameterCoverage: "entry-only" }, ), @@ -508,7 +509,7 @@ export const CANVAS_TOOLS: readonly CanvasToolDefinition[] = [ tool( "extract-audio", "提取音频", - "将视频音轨另存为音频素材；对应本轮原站「声音分离 → 分离音频」。", + "将视频音轨另存为音频素材。", [], { category: "audio", @@ -520,7 +521,7 @@ export const CANVAS_TOOLS: readonly CanvasToolDefinition[] = [ tool( "audio-separation", "音频分离", - "仅保留静态候选入口。本轮未见人声/伴奏分离；视频「声音分离」实测仅为提取音频，不能据此确认此能力。", + "分离人声与伴奏，需要连接音轨分离服务。", [], { category: "audio", diff --git a/src/canvas/toolReasonText.ts b/src/canvas/toolReasonText.ts index e454cd1a..5225f229 100644 --- a/src/canvas/toolReasonText.ts +++ b/src/canvas/toolReasonText.ts @@ -2,6 +2,8 @@ import type { Translate } from "../i18n/I18nProvider.tsx"; import { getActiveLocale } from "../i18n/messages.ts"; import { defaultTranslate } from "./assistantTranslate.ts"; +import { capabilityErrorText } from "./capabilityErrorText.ts"; +import { availabilityReason } from "../backend/availability.ts"; const CAPABILITY: Record = { 对话: "chat", @@ -27,9 +29,13 @@ const NEED = /^需要接入具备「(.+)」能力的模型。$/; export function toolReasonText( text: string | undefined, t: Translate = defaultTranslate, + reasonCode?: string, ): string | undefined { if (!text || getActiveLocale() !== "en-US" || !/[\u3400-\u9fff]/.test(text)) return text; + const capability = capabilityErrorText(text, t); + if (capability) return capability; + if (reasonCode) return availabilityReason(reasonCode); if (text.startsWith("接入图像分割模型")) { return t("errors.tools.needSelectionModel"); } diff --git a/src/canvas/toolTaskText.ts b/src/canvas/toolTaskText.ts index 9f7eeafd..d7b288ea 100644 --- a/src/canvas/toolTaskText.ts +++ b/src/canvas/toolTaskText.ts @@ -4,6 +4,7 @@ import type { MessageKey, MessageParams } from "../i18n/types.ts"; import { defaultTranslate } from "./assistantTranslate.ts"; import { getActiveLocale } from "../i18n/messages.ts"; import { translateToolText } from "./toolText.ts"; +import { capabilityErrorText } from "./capabilityErrorText.ts"; const CJK = /[\u3400-\u9fff]/; @@ -125,6 +126,8 @@ export function toolTaskError( t: Translate = defaultTranslate, failure?: ToolTaskFailure, ): string | undefined { + const capability = text && capabilityErrorText(text, t); + if (capability) return capability; const key = failure && CODE[failure.code]; if (key) return t(key, failure?.params); if (!text) return undefined; diff --git a/src/canvas/toolText.ts b/src/canvas/toolText.ts index 3c6f97f9..60f5704f 100644 --- a/src/canvas/toolText.ts +++ b/src/canvas/toolText.ts @@ -75,8 +75,8 @@ const EN: Record = { "光质": "Light quality", "光源颜色": "Light color", "图层分解": "Layer Decompose", - "原站实测提供智能拆分、框选拆分和按描述拆分。框选需原图矩形选区；实际拆分及报价由真实服务提供。": - "The source site provides smart, box, and description-based splitting. Box splitting needs a rectangle on the original image; actual splitting and pricing come from the live service.", + "按对象、选区或描述拆分图层；可用方式由连接的服务提供。": + "Split layers by object, selection, or description using the connected service.", "拆分模式": "Split mode", "智能拆分": "Smart split", "框选拆分": "Box split", @@ -89,18 +89,18 @@ const EN: Record = { "从单张图片生成 2:1 全景；模型和清晰度由服务提供。": "Generate a 2:1 panorama from one image; the service provides models and resolution.", "商品替换": "Product Replacement", - "使用场景图和商品图替换主体，支持选区与提示词；原站已实测自动主体识别和 2K 结果。": - "Replace the subject using a scene and product image, with selection and prompt support; the source site verified automatic subject detection and 2K results.", + "使用场景图和商品图替换主体，支持选区与提示词。": + "Replace a subject using scene and product images, with a selection and prompt.", "智能多尺寸": "Smart Multi-size", - "按原站实测画幅和像素尺寸创建草稿；自定义范围与实际服务协议尚未接入。": - "Create drafts from the aspect ratios and pixel sizes verified on the source site; custom ranges and the live service protocol are not connected yet.", + "设置目标画幅与输出尺寸。": + "Set the target aspect ratio and output dimensions.", "输出尺寸": "Output size", "自定义": "Custom", "自定义宽度": "Custom width", "自定义高度": "Custom height", "动态海报": "Dynamic Poster", - "基于原图比例创建动态海报请求；本轮原站入口仅提供 Seedance 2.5，实际生成需连接服务。": - "Create a dynamic poster request at the source aspect ratio; the source entry only offers Seedance 2.5, and generation requires a connected service.", + "按原图比例生成动态海报，需要连接视频生成服务。": + "Animate a poster at its original aspect ratio using a video generation service.", "电商主图": "E-commerce Hero", "整理电商主图请求；支持最多 6 张参考图，实际生成需连接服务。": "Prepare an e-commerce hero request; up to 6 references are supported, and generation requires a connected service.", @@ -110,8 +110,8 @@ const EN: Record = { "侧视": "Side", "俯视": "Top", "画幅": "Aspect ratio", - "本轮只核对 1:1；其他画幅待服务确认。": - "Only 1:1 is verified; other ratios await service confirmation.", + "当前预设为 1:1。": + "The current preset uses a 1:1 aspect ratio.", "图像预设": "Image Presets", "三视图、连续分镜、多机位等预设由模型能力加载。": "Presets such as three-view, storyboard, and multi-camera come from model capabilities.", @@ -129,9 +129,9 @@ const EN: Record = { "此功能有独立任务与能力约束，参数需由已连接服务提供。": "This feature has its own task and capability constraints; parameters come from a connected service.", "视频编辑": "Video Edit", - "原站实测支持 4–30 秒素材、提示词及矩形/画笔/箭头/文字/橡皮擦/标记；本轮未见模型选择。这里只记录意图，视频标注数据尚未接入。": - "The source site supports 4–30 second media, prompts, and rectangle/brush/arrow/text/eraser/mark tools; no model selection was seen. This only records intent; video annotation data is not connected yet.", - "原站标注工具": "Source annotation tools", + "设置需要重拍的片段及修改要求；可用编辑方式由服务提供。": + "Choose the segment and describe the changes; editing options depend on the service.", + "标注工具": "Annotation tools", "矩形": "Rectangle", "画笔": "Brush", "箭头": "Arrow", @@ -144,18 +144,18 @@ const EN: Record = { "输出 480P / 720P 灰度深度视频（近处亮、远处暗），素材最长 30 秒；需要接入 vLLM-Omni 深度接口。": "Outputs 480P/720P grayscale depth video (near is bright, far is dark) for media up to 30 seconds; requires the vLLM-Omni depth endpoint.", "字幕擦除": "Subtitle Erasure", - "本轮原站实测没有额外设置；实际视频处理依赖真实服务。": - "No extra settings were found on the source site; actual video processing depends on the live service.", + "擦除视频中的字幕，需要连接视频处理服务。": + "Remove subtitles from video using a video processing service.", "视频抠像": "Video Matting", - "原站实测可选黑色、绿色或透明背景；输出格式与处理协议由真实服务确定。": - "The source site offers black, green, or transparent backgrounds; the live service defines output formats and protocol.", + "选择黑色、绿色或透明背景；输出格式由服务提供。": + "Choose a black, green, or transparent background; output formats depend on the service.", "输出背景": "Output background", "黑色": "Black", "绿色": "Green", "透明": "Transparent", "视频翻译": "Video Translation", - "静态代码存在候选入口，本轮原站未见；当前可用性和参数未知。": - "A candidate entry exists in static code but was not seen on the source site; availability and parameters are unknown.", + "需要连接提供此功能的服务。": + "Connect a service that provides this feature.", "视频超清": "Video Upscale", "视频增强有 2K 与 4K 输出路径，支持条件由服务判断。": "Video enhancement offers 2K and 4K outputs; the service decides whether they are supported.", @@ -170,11 +170,11 @@ const EN: Record = { "Crop to source pixels, keep a capturable audio track, and re-encode WebM locally.", "打开对应的本地媒体编辑器。": "Open the matching local media editor.", "提取音频": "Extract Audio", - "将视频音轨另存为音频素材；对应本轮原站「声音分离 → 分离音频」。": - "Save the video audio track as an audio asset; corresponds to the source site's \"Audio Separation → Extract Audio\".", + "将视频音轨另存为音频素材。": + "Save a video's soundtrack as an audio asset.", "音频分离": "Audio Separation", - "仅保留静态候选入口。本轮未见人声/伴奏分离；视频「声音分离」实测仅为提取音频，不能据此确认此能力。": - "Static candidate entry only. Vocal/accompaniment separation was not seen; the video \"audio separation\" flow only extracts audio, which does not confirm this capability.", + "分离人声与伴奏，需要连接音轨分离服务。": + "Separate vocals and accompaniment using an audio separation service.", "音频裁剪": "Audio Trim", "裁剪音频时长并导出新素材。": "Trim audio duration and export a new asset.", "导演台": "Director Stage", diff --git a/src/canvas/useAssistantChat.ts b/src/canvas/useAssistantChat.ts index b2a00e49..72f0bca9 100644 --- a/src/canvas/useAssistantChat.ts +++ b/src/canvas/useAssistantChat.ts @@ -1,3 +1,4 @@ +import { availabilityBlocker } from "../backend/availability.ts"; import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"; import { rememberAgentSelection, restoreAgentSelection, builtinThinking } from "./agentPreferences"; import { errorMessage } from "../backend/api"; @@ -312,7 +313,13 @@ export function useAssistantChat( return () => abort.abort(); }, [open, loaded, runtimesLoading, runtimeToCheck, refreshRuntimes]); + const checkCapabilities = catalog.refreshAvailability; + useEffect(() => { + if (open && currentRuntime?.modelSource === "settings" && agent) void checkCapabilities([agent.id]); + }, [open, currentRuntime?.modelSource, agent?.id, checkCapabilities]); + const availabilityIssue = currentRuntime?.modelSource === "settings" ? availabilityBlocker(agent, "text_response") : null; const canSend = + !availabilityIssue && currentRuntime?.status === "ready" && !!currentRuntime.available && (currentRuntime.modelSource === "cli" || !!agent); @@ -664,6 +671,7 @@ export function useAssistantChat( setError, agent, canSend, + availabilityIssue, adding, upload, addNodes, diff --git a/src/canvas/useGenerationJobs.ts b/src/canvas/useGenerationJobs.ts index 4111ff11..24878c36 100644 --- a/src/canvas/useGenerationJobs.ts +++ b/src/canvas/useGenerationJobs.ts @@ -14,6 +14,7 @@ import { import { abortable } from "../backend/resilientRead.ts"; import { toolHistoryJobs } from "../backend/tools.ts"; import { type GenerationJob } from "./generationHistory"; +import { useI18n } from "../i18n/I18nProvider.tsx"; /** Each callback retains its project owner, including a service result arriving after switching. */ export function useGenerationJobs( @@ -21,9 +22,11 @@ export function useGenerationJobs( onError: (message: string) => void, connected = true, ) { + const { locale } = useI18n(); const cache = useRef(new Map()); const [revision, setRevision] = useState(0); const cacheCursors = useRef(new Map()); + const cacheLocales = useRef(new Map()); const reads = useRef(new Map()); const previews = useRef(new Map>()); const error = useRef(onError); @@ -41,7 +44,8 @@ export function useGenerationJobs( }; if ( snapshot && - snapshot.eventCursor > (cacheCursors.current.get(documentId) ?? -1) + (snapshot.eventCursor > (cacheCursors.current.get(documentId) ?? -1) || + cacheLocales.current.get(documentId) !== locale) ) { const current = cache.current.get(documentId) ?? []; publish([ @@ -49,6 +53,9 @@ export function useGenerationJobs( ...current.filter((job) => job.tool), ]); cacheCursors.current.set(documentId, snapshot.eventCursor); + // Messages are a local projection. A language change must refresh them + // even when the persisted project has no new execution events. + cacheLocales.current.set(documentId, locale); } // Each preview hydrates independently. Late files only patch matching // preview identities, never status, phase, adoption or cancellation. @@ -100,7 +107,7 @@ export function useGenerationJobs( publish(next); return cache.current.get(documentId) ?? []; }, - [documentId, connected], + [documentId, connected, locale], ); useEffect(() => { let active = true; diff --git a/src/canvas/workspace/WorkspaceView.tsx b/src/canvas/workspace/WorkspaceView.tsx index 99493f7b..4a0b738a 100644 --- a/src/canvas/workspace/WorkspaceView.tsx +++ b/src/canvas/workspace/WorkspaceView.tsx @@ -201,6 +201,7 @@ export function WorkspaceView({ rt }: { rt: WorkspaceRuntime }) { projectRenaming, projected, readJobs, + recoverServiceTask, ready, referenceEdges, referencePreviews, @@ -1590,6 +1591,7 @@ export function WorkspaceView({ rt }: { rt: WorkspaceRuntime }) { incoming={composerIncoming} referenceControls={generationReferenceControls(composerNode)} models={backendCatalog.models} + onCheckModels={backendCatalog.refreshAvailability} onUpdate={(patch) => command({ type: "update-node", @@ -1630,6 +1632,7 @@ export function WorkspaceView({ rt }: { rt: WorkspaceRuntime }) { }} running={composerRunning} backendModels={backendCatalog.models} + onCheckModels={backendCatalog.refreshAvailability} /> )}
@@ -2056,10 +2059,8 @@ export function WorkspaceView({ rt }: { rt: WorkspaceRuntime }) { return; } { - await creationClient(doc.id).control( - job.taskId, - action, - ); + if (action === "recover") await recoverServiceTask(job); + else await creationClient(doc.id).control(job.taskId, action); return; } } catch (error) { diff --git a/src/canvas/workspace/jobFailureNotifications.ts b/src/canvas/workspace/jobFailureNotifications.ts new file mode 100644 index 00000000..a5fad4ad --- /dev/null +++ b/src/canvas/workspace/jobFailureNotifications.ts @@ -0,0 +1,113 @@ +import type { GenerationJob } from "../generationHistory.ts"; + +type JobState = Pick; + +/** Submission and background observers share ownership of one task failure. + * Observing a revision does not mean its failure was announced. Only claim() + * owns the notification; a confirmed recovery permits a later failure version. + */ +export class JobFailureNotifications { + private openedAt: number; + private states = new Map< + string, + { + observedStatus?: GenerationJob["status"]; + revision?: number; + /** Set only when the observer actually claims this version's failure. */ + reportedRevision?: number; + recoveredAfter?: number; + reported: boolean; + } + >(); + + constructor(openedAt = Date.now()) { + this.openedAt = openedAt; + } + + /** Claim before publishing failed state, which can immediately wake observers. */ + claim(id: string): boolean { + const previous = this.states.get(id); + if (previous?.reported) return false; + // A callback can report before React observes the failed projection. Keep + // the last observed status so an older running render is not a retry. + this.states.set(id, { ...previous, reported: true }); + return true; + } + + /** A confirmed recovery may finish again before a running snapshot arrives. */ + recovered(id: string, failedRevision: number) { + const previous = this.states.get(id); + if ( + previous?.reported && + (previous.reportedRevision ?? -1) > failedRevision && + (previous.observedStatus === "failed" || + previous.observedStatus === "interrupted") + ) + return; // The observer already announced this recovery's failure. + this.states.set(id, { + ...previous, + reported: false, + recoveredAfter: failedRevision, + }); + } + + observe(job: JobState): boolean { + const previous = this.states.get(job.id); + if ( + job.revision !== undefined && + previous?.revision !== undefined && + job.revision < previous.revision + ) + return false; + if ( + previous?.recoveredAfter !== undefined && + (job.revision === undefined || job.revision <= previous.recoveredAfter) + ) + return false; // A replay of the failure preceding the accepted recovery. + if (job.status === "running") { + // A resumed execution may keep its task ID. Its next failure is new. + const resumed = + previous?.observedStatus === "failed" || + previous?.observedStatus === "interrupted"; + this.states.set(job.id, { + ...previous, + observedStatus: job.status, + revision: job.revision, + reported: resumed ? false : (previous?.reported ?? false), + }); + return false; + } + if (job.status !== "failed" && job.status !== "interrupted") { + this.states.set(job.id, { + ...previous, + observedStatus: job.status, + revision: job.revision, + reported: previous?.reported ?? false, + }); + return false; + } + // Reopening a page must not announce a failure that was already history, + // even when it happened only milliseconds before this page opened. + if ( + previous?.observedStatus !== "running" && + previous?.recoveredAfter === undefined && + (job.createdAt ?? 0) < this.openedAt + ) { + this.states.set(job.id, { + ...previous, + observedStatus: job.status, + revision: job.revision, + reported: true, + }); + return false; + } + const report = this.claim(job.id); + this.states.set(job.id, { + observedStatus: job.status, + revision: job.revision, + reportedRevision: report ? job.revision : previous?.reportedRevision, + reported: true, + }); + return report; + } +} diff --git a/src/canvas/workspace/runtimeTypes.ts b/src/canvas/workspace/runtimeTypes.ts index 4763f4a7..0bd1b5f8 100644 --- a/src/canvas/workspace/runtimeTypes.ts +++ b/src/canvas/workspace/runtimeTypes.ts @@ -50,6 +50,7 @@ export interface WorkspaceRuntime extends WorkspaceState, WorkspaceSetters { jobs: GenerationJob[]; setJobs: Dispatch>; readJobs: (signal?: AbortSignal) => Promise; + recoverServiceTask: (job: GenerationJob) => Promise; command: (value: CanvasCommand) => void; change: (fn: (document: CanvasDocument) => CanvasDocument) => void; begin: () => void; diff --git a/src/canvas/workspace/useWorkspaceGeneration.ts b/src/canvas/workspace/useWorkspaceGeneration.ts index 9406382a..2c5616df 100644 --- a/src/canvas/workspace/useWorkspaceGeneration.ts +++ b/src/canvas/workspace/useWorkspaceGeneration.ts @@ -18,6 +18,7 @@ import { type CanvasNode } from "../model"; import { getNodeGenerationSettings } from "../NodeComposer"; import { serviceGenerationPlan, textGenerationPlan } from "../workspaceInteraction.ts"; import { generationPrepareBlock } from "./entryPoints.ts"; +import { JobFailureNotifications } from "./jobFailureNotifications.ts"; import { CANVAS_TOOLS, type CanvasToolTask } from "../toolCatalog"; import { translateToolText } from "../toolText.ts"; @@ -56,6 +57,23 @@ export function useWorkspaceGeneration(rt: WorkspaceRuntime) { t, toolCatalog, } = rt; + const failureWatch = useRef({ + documentId: doc.id, + notifications: new JobFailureNotifications(), + }); + if (failureWatch.current.documentId !== doc.id) + failureWatch.current = { + documentId: doc.id, + notifications: new JobFailureNotifications(), + }; + const failureNotifications = failureWatch.current.notifications; + const recoverServiceTask = async (job: Job) => { + if (!job.taskId) return; + const receipt = await creationClient(doc.id).control(job.taskId, "recover"); + if (receipt.status !== "rejected" && job.revision !== undefined) + failureNotifications.recovered(job.id, job.revision); + setJobs(await readJobs()); + }; useEffect(() => { if (!arrivingNodes.size) return; @@ -362,6 +380,10 @@ export function useWorkspaceGeneration(rt: WorkspaceRuntime) { setJobs(await readJobs()); } catch (error) { const uncertain = error instanceof ToolSubmissionError && error.uncertain; + // Admission errors remain visible even if a project switch removes this + // local job. Claim first so its failed projection cannot also notify. + const report = !run.controller.signal.aborted && + (uncertain || failureNotifications.claim(id)); updateJob(id, { status: uncertain ? "running" @@ -386,7 +408,7 @@ export function useWorkspaceGeneration(rt: WorkspaceRuntime) { change(() => cleaned); await persist(cleaned); } - if (!run.controller.signal.aborted) + if (report) notify(`「${title}」：${errorMessage(error)}`); } finally { serviceRuns.current.delete(id); @@ -419,20 +441,9 @@ export function useWorkspaceGeneration(rt: WorkspaceRuntime) { /** A job the service gives up on must say so: the node otherwise just stays * empty and the prompt looks like it did nothing. Failures that predate this * session are history, not news; a job we watched run always qualifies. */ - const failureWatch = useRef({ - mountedAt: Date.now(), - active: new Set(), - reported: new Set(), - }); useEffect(() => { - const watch = failureWatch.current; for (const job of jobs) { - if (job.status === "running") watch.active.add(job.id); - if (job.status !== "failed" && job.status !== "interrupted") continue; - if (watch.reported.has(job.id)) continue; - const recent = (job.createdAt ?? 0) >= watch.mountedAt - 5_000; - if (!recent && !watch.active.has(job.id)) continue; - watch.reported.add(job.id); + if (!failureNotifications.observe(job)) continue; const node = [...doc.nodes, ...(doc.hiddenNodes ?? [])].find( (item) => item.id === (job.targetNodeId ?? job.nodeId), ); @@ -446,7 +457,7 @@ export function useWorkspaceGeneration(rt: WorkspaceRuntime) { : t("generation.error.jobFailed", { name }), ); } - }, [jobs, doc.nodes, doc.hiddenNodes, notify, t]); + }, [jobs, doc.nodes, doc.hiddenNodes, failureNotifications, notify, t]); const toolPlacementQueue = useRef>(Promise.resolve()); const placingTools = useRef(new Set()); @@ -683,6 +694,7 @@ export function useWorkspaceGeneration(rt: WorkspaceRuntime) { rt.revealResults = revealResults; rt.syncServiceResults = syncServiceResults; rt.runServiceTask = runServiceTask; + rt.recoverServiceTask = recoverServiceTask; rt.runToolTask = runToolTask; rt.observedRuns = observedRuns; rt.failureWatch = failureWatch; diff --git a/src/canvas/workspace/useWorkspaceRuntime.ts b/src/canvas/workspace/useWorkspaceRuntime.ts index 21883871..74932fd2 100644 --- a/src/canvas/workspace/useWorkspaceRuntime.ts +++ b/src/canvas/workspace/useWorkspaceRuntime.ts @@ -147,7 +147,8 @@ export function useWorkspaceRuntime(): WorkspaceRuntime { ); const toolCatalog = useToolCatalog( backendCatalog.connected, - backendCatalog.models, + backendCatalog.availabilityRevision, + t, ); const setupChecked = useRef(false); const theme = useTheme(); diff --git a/src/contracts/creation.schema.json b/src/contracts/creation.schema.json index 5ff5a4d7..9eee5d20 100644 --- a/src/contracts/creation.schema.json +++ b/src/contracts/creation.schema.json @@ -1417,6 +1417,30 @@ "title": "AudioParameters", "type": "object" }, + "AvailabilityCatalog": { + "additionalProperties": false, + "properties": { + "revision": { + "description": "Version of the complete observation projection.", + "title": "Revision", + "type": "string" + }, + "models": { + "description": "Current saved model observations; reading does not query providers.", + "items": { + "$ref": "#/$defs/ModelAvailability" + }, + "title": "Models", + "type": "array" + } + }, + "required": [ + "revision", + "models" + ], + "title": "AvailabilityCatalog", + "type": "object" + }, "BatchCommand": { "additionalProperties": false, "description": "Apply multiple operations atomically; all succeed or none are committed.", @@ -3252,6 +3276,251 @@ "title": "CanvasUpdateCommandOperation", "type": "object" }, + "CapabilityDeclaration": { + "additionalProperties": false, + "properties": { + "status": { + "description": "Support decision excluding execution state.", + "enum": [ + "available", + "unavailable", + "unknown" + ], + "title": "Status", + "type": "string" + }, + "reasonCode": { + "description": "Localizable declaration decision.", + "title": "Reasoncode", + "type": "string" + }, + "checkedAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Completion time in Unix milliseconds.", + "title": "Checkedat" + }, + "expiresAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Declaration expiry in Unix milliseconds.", + "title": "Expiresat" + } + }, + "required": [ + "status", + "reasonCode", + "checkedAt", + "expiresAt" + ], + "title": "CapabilityDeclaration", + "type": "object" + }, + "CapabilityExecution": { + "additionalProperties": false, + "properties": { + "state": { + "description": "Observed execution pause state.", + "enum": [ + "unpaused", + "paused", + "unknown" + ], + "title": "State", + "type": "string" + }, + "scope": { + "anyOf": [ + { + "enum": [ + "deployment", + "worker" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Scope proved by the service observation.", + "title": "Scope" + }, + "checkedAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Observation receipt time in Unix milliseconds.", + "title": "Checkedat" + }, + "expiresAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Execution observation expiry in Unix milliseconds.", + "title": "Expiresat" + } + }, + "required": [ + "state", + "scope", + "checkedAt", + "expiresAt" + ], + "title": "CapabilityExecution", + "type": "object" + }, + "CapabilityInputDetail": { + "additionalProperties": false, + "properties": { + "name": { + "description": "Canonical input role or parameter name.", + "title": "Name", + "type": "string" + }, + "support": { + "description": "Explicit fact, with omission kept unknown.", + "enum": [ + "supported", + "unsupported", + "unknown" + ], + "title": "Support", + "type": "string" + }, + "source": { + "default": "service", + "description": "Owner of this restriction.", + "enum": [ + "service", + "application" + ], + "title": "Source", + "type": "string" + }, + "minCount": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum transmitted inputs of this role.", + "title": "Mincount" + }, + "maxCount": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum transmitted inputs of this role.", + "title": "Maxcount" + }, + "values": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Explicit allowed parameter values.", + "title": "Values" + }, + "min": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inclusive parameter lower bound.", + "title": "Min" + }, + "max": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inclusive parameter upper bound.", + "title": "Max" + }, + "multipleOf": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Parameter step from zero.", + "title": "Multipleof" + } + }, + "required": [ + "name", + "support" + ], + "title": "CapabilityInputDetail", + "type": "object" + }, "ChangedObject": { "additionalProperties": false, "properties": { @@ -3472,6 +3741,9 @@ "kling_aspect", "kling_output", "kling_cancel", + "capability_unavailable", + "capability_paused", + "capability_connection_changed", "capability_unsupported", "preview_expired", "storage_unavailable", @@ -8031,6 +8303,105 @@ "title": "MediaPublicationOutput", "type": "object" }, + "ModelAvailability": { + "additionalProperties": false, + "properties": { + "configId": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Saved model identity; null for an unsaved draft.", + "title": "Configid" + }, + "modelId": { + "description": "Exact remote model identifier requested.", + "title": "Modelid", + "type": "string" + }, + "identity": { + "description": "Opaque identity of the checked connection and model.", + "title": "Identity", + "type": "string" + }, + "autoCheck": { + "description": "Effective service automatic-check preference.", + "title": "Autocheck", + "type": "boolean" + }, + "status": { + "description": "Summary across the configured adapter operations.", + "enum": [ + "available", + "unavailable", + "unknown" + ], + "title": "Status", + "type": "string" + }, + "reasonCode": { + "description": "Stable, localizable summary reason.", + "title": "Reasoncode", + "type": "string" + }, + "checkedAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Observation completion time in Unix milliseconds.", + "title": "Checkedat" + }, + "expiresAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Earliest operation expiry in Unix milliseconds.", + "title": "Expiresat" + }, + "revision": { + "description": "Projection version; changes on observations, identity changes and expiration.", + "title": "Revision", + "type": "string" + }, + "operations": { + "description": "Exact operations consumed by the configured adapter.", + "items": { + "$ref": "#/$defs/OperationAvailability" + }, + "title": "Operations", + "type": "array" + } + }, + "required": [ + "configId", + "modelId", + "identity", + "autoCheck", + "status", + "reasonCode", + "checkedAt", + "expiresAt", + "revision", + "operations" + ], + "title": "ModelAvailability", + "type": "object" + }, "ModelConfigPublic": { "additionalProperties": false, "description": "Application-level, never embedded in an editable project's credential data.", @@ -8172,9 +8543,34 @@ "inputRules": { "$ref": "#/$defs/InputRules", "description": "Verified model input limits and output choices." - } - }, - "required": [ + }, + "capabilityAutoCheck": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Service preference; omitted defaults to automatic for self-hosted connections.", + "title": "Capabilityautocheck" + }, + "availability": { + "anyOf": [ + { + "$ref": "#/$defs/ModelAvailability" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Ephemeral observation, never persisted in project snapshots." + } + }, + "required": [ "kind", "url", "modelId", @@ -8320,6 +8716,19 @@ "default": null, "description": "Provider credential; omitted from public model records.", "title": "Apikey" + }, + "capabilityAutoCheck": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Service automatic-check override; omission preserves the existing preference.", + "title": "Capabilityautocheck" } }, "required": [ @@ -8865,6 +9274,121 @@ "title": "ObjectVersion", "type": "object" }, + "OperationAvailability": { + "additionalProperties": false, + "properties": { + "operation": { + "description": "Canonical remote operation identifier.", + "title": "Operation", + "type": "string" + }, + "profile": { + "description": "Input and output mode used by the configured adapter.", + "title": "Profile", + "type": "string" + }, + "method": { + "description": "Actual HTTP method used for submission.", + "title": "Method", + "type": "string" + }, + "path": { + "description": "Actual HTTP submission path.", + "title": "Path", + "type": "string" + }, + "output": { + "description": "Output modality consumed by the adapter.", + "title": "Output", + "type": "string" + }, + "status": { + "description": "Current evidence for this exact request.", + "enum": [ + "available", + "unavailable", + "unknown" + ], + "title": "Status", + "type": "string" + }, + "reasonCode": { + "description": "Stable, localizable decision reason.", + "title": "Reasoncode", + "type": "string" + }, + "checkedAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Observation completion time in Unix milliseconds.", + "title": "Checkedat" + }, + "expiresAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Decision expiry in Unix milliseconds; recompute after expiry.", + "title": "Expiresat" + }, + "declaration": { + "anyOf": [ + { + "$ref": "#/$defs/CapabilityDeclaration" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Request support independently expires from the short execution observation." + }, + "execution": { + "anyOf": [ + { + "$ref": "#/$defs/CapabilityExecution" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Short-lived execution observation, separate from declared support." + }, + "inputDetails": { + "default": [], + "description": "Known role and parameter facts for the actual request route; no saved configuration changes.", + "items": { + "$ref": "#/$defs/CapabilityInputDetail" + }, + "title": "Inputdetails", + "type": "array" + } + }, + "required": [ + "operation", + "profile", + "method", + "path", + "output", + "status", + "reasonCode", + "checkedAt", + "expiresAt" + ], + "title": "OperationAvailability", + "type": "object" + }, "PaletteData": { "additionalProperties": false, "properties": { @@ -16530,6 +17054,30 @@ "title": "AudioParameters", "type": "object" }, + "AvailabilityCatalog": { + "additionalProperties": false, + "properties": { + "revision": { + "description": "Version of the complete observation projection.", + "title": "Revision", + "type": "string" + }, + "models": { + "description": "Current saved model observations; reading does not query providers.", + "items": { + "$ref": "#/$defs/ModelAvailability" + }, + "title": "Models", + "type": "array" + } + }, + "required": [ + "revision", + "models" + ], + "title": "AvailabilityCatalog", + "type": "object" + }, "BatchCommand": { "additionalProperties": false, "description": "Apply multiple operations atomically; all succeed or none are committed.", @@ -18412,6 +18960,258 @@ "title": "CanvasUpdateCommandOperation", "type": "object" }, + "CapabilityDeclaration": { + "additionalProperties": false, + "properties": { + "status": { + "description": "Support decision excluding execution state.", + "enum": [ + "available", + "unavailable", + "unknown" + ], + "title": "Status", + "type": "string" + }, + "reasonCode": { + "description": "Localizable declaration decision.", + "title": "Reasoncode", + "type": "string" + }, + "checkedAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Completion time in Unix milliseconds.", + "title": "Checkedat" + }, + "expiresAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Declaration expiry in Unix milliseconds.", + "title": "Expiresat" + } + }, + "required": [ + "status", + "reasonCode", + "checkedAt", + "expiresAt" + ], + "title": "CapabilityDeclaration", + "type": "object" + }, + "CapabilityExecution": { + "additionalProperties": false, + "properties": { + "state": { + "description": "Observed execution pause state.", + "enum": [ + "unpaused", + "paused", + "unknown" + ], + "title": "State", + "type": "string" + }, + "scope": { + "anyOf": [ + { + "enum": [ + "deployment", + "worker" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Scope proved by the service observation.", + "title": "Scope" + }, + "checkedAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Observation receipt time in Unix milliseconds.", + "title": "Checkedat" + }, + "expiresAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Execution observation expiry in Unix milliseconds.", + "title": "Expiresat" + } + }, + "required": [ + "state", + "scope", + "checkedAt", + "expiresAt" + ], + "title": "CapabilityExecution", + "type": "object" + }, + "CapabilityInputDetail": { + "additionalProperties": false, + "properties": { + "name": { + "description": "Canonical input role or parameter name.", + "title": "Name", + "type": "string" + }, + "support": { + "description": "Explicit fact, with omission kept unknown.", + "enum": [ + "supported", + "unsupported", + "unknown" + ], + "title": "Support", + "type": "string" + }, + "source": { + "default": "service", + "description": "Owner of this restriction.", + "enum": [ + "service", + "application" + ], + "title": "Source", + "type": "string" + }, + "minCount": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum transmitted inputs of this role.", + "title": "Mincount" + }, + "maxCount": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum transmitted inputs of this role.", + "title": "Maxcount" + }, + "values": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Explicit allowed parameter values.", + "title": "Values" + }, + "min": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inclusive parameter lower bound.", + "title": "Min" + }, + "max": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inclusive parameter upper bound.", + "title": "Max" + }, + "multipleOf": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Parameter step from zero.", + "title": "Multipleof" + } + }, + "required": [ + "name", + "support", + "source", + "minCount", + "maxCount", + "values", + "min", + "max", + "multipleOf" + ], + "title": "CapabilityInputDetail", + "type": "object" + }, "ChangedObject": { "additionalProperties": false, "properties": { @@ -18638,6 +19438,9 @@ "kling_aspect", "kling_output", "kling_cancel", + "capability_unavailable", + "capability_paused", + "capability_connection_changed", "capability_unsupported", "preview_expired", "storage_unavailable", @@ -23404,6 +24207,105 @@ "title": "MediaPublicationOutput", "type": "object" }, + "ModelAvailability": { + "additionalProperties": false, + "properties": { + "configId": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Saved model identity; null for an unsaved draft.", + "title": "Configid" + }, + "modelId": { + "description": "Exact remote model identifier requested.", + "title": "Modelid", + "type": "string" + }, + "identity": { + "description": "Opaque identity of the checked connection and model.", + "title": "Identity", + "type": "string" + }, + "autoCheck": { + "description": "Effective service automatic-check preference.", + "title": "Autocheck", + "type": "boolean" + }, + "status": { + "description": "Summary across the configured adapter operations.", + "enum": [ + "available", + "unavailable", + "unknown" + ], + "title": "Status", + "type": "string" + }, + "reasonCode": { + "description": "Stable, localizable summary reason.", + "title": "Reasoncode", + "type": "string" + }, + "checkedAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Observation completion time in Unix milliseconds.", + "title": "Checkedat" + }, + "expiresAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Earliest operation expiry in Unix milliseconds.", + "title": "Expiresat" + }, + "revision": { + "description": "Projection version; changes on observations, identity changes and expiration.", + "title": "Revision", + "type": "string" + }, + "operations": { + "description": "Exact operations consumed by the configured adapter.", + "items": { + "$ref": "#/$defs/OperationAvailability" + }, + "title": "Operations", + "type": "array" + } + }, + "required": [ + "configId", + "modelId", + "identity", + "autoCheck", + "status", + "reasonCode", + "checkedAt", + "expiresAt", + "revision", + "operations" + ], + "title": "ModelAvailability", + "type": "object" + }, "ModelConfigPublic": { "additionalProperties": false, "description": "Application-level, never embedded in an editable project's credential data.", @@ -23531,6 +24433,31 @@ "inputRules": { "$ref": "#/$defs/InputRules", "description": "Verified model input limits and output choices." + }, + "capabilityAutoCheck": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Service preference; omitted defaults to automatic for self-hosted connections.", + "title": "Capabilityautocheck" + }, + "availability": { + "anyOf": [ + { + "$ref": "#/$defs/ModelAvailability" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Ephemeral observation, never persisted in project snapshots." } }, "required": [ @@ -23683,6 +24610,19 @@ "default": null, "description": "Provider credential; omitted from public model records.", "title": "Apikey" + }, + "capabilityAutoCheck": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Service automatic-check override; omission preserves the existing preference.", + "title": "Capabilityautocheck" } }, "required": [ @@ -24243,6 +25183,122 @@ "title": "ObjectVersion", "type": "object" }, + "OperationAvailability": { + "additionalProperties": false, + "properties": { + "operation": { + "description": "Canonical remote operation identifier.", + "title": "Operation", + "type": "string" + }, + "profile": { + "description": "Input and output mode used by the configured adapter.", + "title": "Profile", + "type": "string" + }, + "method": { + "description": "Actual HTTP method used for submission.", + "title": "Method", + "type": "string" + }, + "path": { + "description": "Actual HTTP submission path.", + "title": "Path", + "type": "string" + }, + "output": { + "description": "Output modality consumed by the adapter.", + "title": "Output", + "type": "string" + }, + "status": { + "description": "Current evidence for this exact request.", + "enum": [ + "available", + "unavailable", + "unknown" + ], + "title": "Status", + "type": "string" + }, + "reasonCode": { + "description": "Stable, localizable decision reason.", + "title": "Reasoncode", + "type": "string" + }, + "checkedAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Observation completion time in Unix milliseconds.", + "title": "Checkedat" + }, + "expiresAt": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Decision expiry in Unix milliseconds; recompute after expiry.", + "title": "Expiresat" + }, + "declaration": { + "anyOf": [ + { + "$ref": "#/$defs/CapabilityDeclaration" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Request support independently expires from the short execution observation." + }, + "execution": { + "anyOf": [ + { + "$ref": "#/$defs/CapabilityExecution" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Short-lived execution observation, separate from declared support." + }, + "inputDetails": { + "default": [], + "description": "Known role and parameter facts for the actual request route; no saved configuration changes.", + "items": { + "$ref": "#/$defs/CapabilityInputDetail" + }, + "title": "Inputdetails", + "type": "array" + } + }, + "required": [ + "operation", + "profile", + "method", + "path", + "output", + "status", + "reasonCode", + "checkedAt", + "expiresAt", + "inputDetails" + ], + "title": "OperationAvailability", + "type": "object" + }, "PaletteData": { "additionalProperties": false, "properties": { diff --git a/src/contracts/generated.ts b/src/contracts/generated.ts index ebf5b9c8..0e008478 100644 --- a/src/contracts/generated.ts +++ b/src/contracts/generated.ts @@ -140,6 +140,11 @@ export interface AudioParametersInput extends ContractInput { readonly durationFromReference?: boolean | null; } +export interface AvailabilityCatalogInput extends ContractInput { + readonly revision: string; + readonly models: ReadonlyArray; +} + export interface BatchCommandInput extends CommandBaseInput { readonly type: "creation.batch"; readonly payload: BatchPayloadInput; @@ -355,6 +360,32 @@ export interface CanvasUpdateCommandOperationInput { readonly payload: CanvasPresentationUpdateInput; } +export interface CapabilityDeclarationInput extends ContractInput { + readonly status: "available" | "unavailable" | "unknown"; + readonly reasonCode: string; + readonly checkedAt: number | null; + readonly expiresAt: number | null; +} + +export interface CapabilityExecutionInput extends ContractInput { + readonly state: "unpaused" | "paused" | "unknown"; + readonly scope: "deployment" | "worker" | null; + readonly checkedAt: number | null; + readonly expiresAt: number | null; +} + +export interface CapabilityInputDetailInput extends ContractInput { + readonly name: string; + readonly support: "supported" | "unsupported" | "unknown"; + readonly source?: "service" | "application"; + readonly minCount?: number | null; + readonly maxCount?: number | null; + readonly values?: ReadonlyArray | null; + readonly min?: number | null; + readonly max?: number | null; + readonly multipleOf?: number | null; +} + export interface ChangedObjectInput extends ContractInput { readonly version: ObjectVersionInput; readonly change: "created" | "updated" | "hidden" | "deleted"; @@ -387,7 +418,7 @@ export interface ContractInput { } export interface ContractIssueInput extends ContractInput { - readonly code: "timeline_package_invalid" | "timeline_package_too_large" | "timeline_revision_missing" | "timeline_import_missing" | "timeline_import_conflict" | "timeline_registration_rejected" | "timeline_font_too_large" | "timeline_font_unavailable" | "timeline_font_unsupported" | "timeline_font_invalid" | "timeline_font_fingerprint" | "timeline_font_coverage" | "timeline_font_total_bytes" | "timeline_font_unverified" | "timeline_font_mime" | "timeline_bundled_font_fingerprint" | "timeline_bundled_font_unavailable" | "invalid_input" | "revision_conflict" | "role_restricted" | "missing_object" | "reference_invalid" | "kling_model" | "kling_mode" | "kling_image" | "kling_frames" | "kling_prompt" | "kling_aspect" | "kling_output" | "kling_cancel" | "capability_unsupported" | "preview_expired" | "storage_unavailable" | "submission_unknown" | "provider_error" | "moss_speech_text" | "moss_speech_voice" | "moss_speech_reference" | "moss_prompt_limit" | "moss_image_size" | "moss_image_reference" | "moss_layout_reference" | "minimax_image_reference" | "minimax_image_count" | "minimax_image_bytes" | "voice_library_unsupported" | "voice_connection_changed" | "voice_reference_required" | "voice_transcript_required" | "voice_transcript_too_long" | "voice_description_too_long" | "voice_not_found" | "voice_delete_unsupported" | "voice_consent_required" | "voice_name_invalid" | "voice_audio_invalid" | "voice_format" | "voice_size" | "voice_duration" | "voice_provider_protocol" | "voice_provider_error" | "voice_provider_timeout" | "voice_model_unsupported"; + readonly code: "timeline_package_invalid" | "timeline_package_too_large" | "timeline_revision_missing" | "timeline_import_missing" | "timeline_import_conflict" | "timeline_registration_rejected" | "timeline_font_too_large" | "timeline_font_unavailable" | "timeline_font_unsupported" | "timeline_font_invalid" | "timeline_font_fingerprint" | "timeline_font_coverage" | "timeline_font_total_bytes" | "timeline_font_unverified" | "timeline_font_mime" | "timeline_bundled_font_fingerprint" | "timeline_bundled_font_unavailable" | "invalid_input" | "revision_conflict" | "role_restricted" | "missing_object" | "reference_invalid" | "kling_model" | "kling_mode" | "kling_image" | "kling_frames" | "kling_prompt" | "kling_aspect" | "kling_output" | "kling_cancel" | "capability_unavailable" | "capability_paused" | "capability_connection_changed" | "capability_unsupported" | "preview_expired" | "storage_unavailable" | "submission_unknown" | "provider_error" | "moss_speech_text" | "moss_speech_voice" | "moss_speech_reference" | "moss_prompt_limit" | "moss_image_size" | "moss_image_reference" | "moss_layout_reference" | "minimax_image_reference" | "minimax_image_count" | "minimax_image_bytes" | "voice_library_unsupported" | "voice_connection_changed" | "voice_reference_required" | "voice_transcript_required" | "voice_transcript_too_long" | "voice_description_too_long" | "voice_not_found" | "voice_delete_unsupported" | "voice_consent_required" | "voice_name_invalid" | "voice_audio_invalid" | "voice_format" | "voice_size" | "voice_duration" | "voice_provider_protocol" | "voice_provider_error" | "voice_provider_timeout" | "voice_model_unsupported"; readonly message: string; readonly objectId?: string | null; readonly field?: string | null; @@ -810,14 +841,30 @@ export interface MediaPublicationOutputInput extends ContractInput { readonly height?: number; } +export interface ModelAvailabilityInput extends ContractInput { + readonly configId: string | null; + readonly modelId: string; + readonly identity: string; + readonly autoCheck: boolean; + readonly status: "available" | "unavailable" | "unknown"; + readonly reasonCode: string; + readonly checkedAt: number | null; + readonly expiresAt: number | null; + readonly revision: string; + readonly operations: ReadonlyArray; +} + export interface ModelConfigPublicInput extends ModelConnectionInput { readonly id: string; readonly providerId?: string | null; readonly inputRules: InputRulesInput; + readonly capabilityAutoCheck?: boolean | null; + readonly availability?: ModelAvailabilityInput | null; } export interface ModelConfigWriteInput extends ModelConnectionInput { readonly apiKey?: string | null; + readonly capabilityAutoCheck?: boolean | null; } export interface ModelConnectionInput extends ContractInput { @@ -887,6 +934,21 @@ export interface ObjectVersionInput extends ContractInput { readonly revision: number; } +export interface OperationAvailabilityInput extends ContractInput { + readonly operation: string; + readonly profile: string; + readonly method: string; + readonly path: string; + readonly output: string; + readonly status: "available" | "unavailable" | "unknown"; + readonly reasonCode: string; + readonly checkedAt: number | null; + readonly expiresAt: number | null; + readonly declaration?: CapabilityDeclarationInput | null; + readonly execution?: CapabilityExecutionInput | null; + readonly inputDetails?: ReadonlyArray; +} + export interface PaletteDataInput extends ContractInput { readonly name?: string | null; readonly sourceImage?: AssetRefInput | null; @@ -1712,6 +1774,11 @@ export interface AudioParameters extends Contract { readonly durationFromReference?: boolean | null; } +export interface AvailabilityCatalog extends Contract { + readonly revision: string; + readonly models: ReadonlyArray; +} + export interface BatchCommand extends CommandBase { readonly type: "creation.batch"; readonly payload: BatchPayload; @@ -1927,6 +1994,32 @@ export interface CanvasUpdateCommandOperation { readonly payload: CanvasPresentationUpdate; } +export interface CapabilityDeclaration extends Contract { + readonly status: "available" | "unavailable" | "unknown"; + readonly reasonCode: string; + readonly checkedAt: number | null; + readonly expiresAt: number | null; +} + +export interface CapabilityExecution extends Contract { + readonly state: "unpaused" | "paused" | "unknown"; + readonly scope: "deployment" | "worker" | null; + readonly checkedAt: number | null; + readonly expiresAt: number | null; +} + +export interface CapabilityInputDetail extends Contract { + readonly name: string; + readonly support: "supported" | "unsupported" | "unknown"; + readonly source: "service" | "application"; + readonly minCount: number | null; + readonly maxCount: number | null; + readonly values: ReadonlyArray | null; + readonly min: number | null; + readonly max: number | null; + readonly multipleOf: number | null; +} + export interface ChangedObject extends Contract { readonly version: ObjectVersion; readonly change: "created" | "updated" | "hidden" | "deleted"; @@ -1959,7 +2052,7 @@ export interface Contract { } export interface ContractIssue extends Contract { - readonly code: "timeline_package_invalid" | "timeline_package_too_large" | "timeline_revision_missing" | "timeline_import_missing" | "timeline_import_conflict" | "timeline_registration_rejected" | "timeline_font_too_large" | "timeline_font_unavailable" | "timeline_font_unsupported" | "timeline_font_invalid" | "timeline_font_fingerprint" | "timeline_font_coverage" | "timeline_font_total_bytes" | "timeline_font_unverified" | "timeline_font_mime" | "timeline_bundled_font_fingerprint" | "timeline_bundled_font_unavailable" | "invalid_input" | "revision_conflict" | "role_restricted" | "missing_object" | "reference_invalid" | "kling_model" | "kling_mode" | "kling_image" | "kling_frames" | "kling_prompt" | "kling_aspect" | "kling_output" | "kling_cancel" | "capability_unsupported" | "preview_expired" | "storage_unavailable" | "submission_unknown" | "provider_error" | "moss_speech_text" | "moss_speech_voice" | "moss_speech_reference" | "moss_prompt_limit" | "moss_image_size" | "moss_image_reference" | "moss_layout_reference" | "minimax_image_reference" | "minimax_image_count" | "minimax_image_bytes" | "voice_library_unsupported" | "voice_connection_changed" | "voice_reference_required" | "voice_transcript_required" | "voice_transcript_too_long" | "voice_description_too_long" | "voice_not_found" | "voice_delete_unsupported" | "voice_consent_required" | "voice_name_invalid" | "voice_audio_invalid" | "voice_format" | "voice_size" | "voice_duration" | "voice_provider_protocol" | "voice_provider_error" | "voice_provider_timeout" | "voice_model_unsupported"; + readonly code: "timeline_package_invalid" | "timeline_package_too_large" | "timeline_revision_missing" | "timeline_import_missing" | "timeline_import_conflict" | "timeline_registration_rejected" | "timeline_font_too_large" | "timeline_font_unavailable" | "timeline_font_unsupported" | "timeline_font_invalid" | "timeline_font_fingerprint" | "timeline_font_coverage" | "timeline_font_total_bytes" | "timeline_font_unverified" | "timeline_font_mime" | "timeline_bundled_font_fingerprint" | "timeline_bundled_font_unavailable" | "invalid_input" | "revision_conflict" | "role_restricted" | "missing_object" | "reference_invalid" | "kling_model" | "kling_mode" | "kling_image" | "kling_frames" | "kling_prompt" | "kling_aspect" | "kling_output" | "kling_cancel" | "capability_unavailable" | "capability_paused" | "capability_connection_changed" | "capability_unsupported" | "preview_expired" | "storage_unavailable" | "submission_unknown" | "provider_error" | "moss_speech_text" | "moss_speech_voice" | "moss_speech_reference" | "moss_prompt_limit" | "moss_image_size" | "moss_image_reference" | "moss_layout_reference" | "minimax_image_reference" | "minimax_image_count" | "minimax_image_bytes" | "voice_library_unsupported" | "voice_connection_changed" | "voice_reference_required" | "voice_transcript_required" | "voice_transcript_too_long" | "voice_description_too_long" | "voice_not_found" | "voice_delete_unsupported" | "voice_consent_required" | "voice_name_invalid" | "voice_audio_invalid" | "voice_format" | "voice_size" | "voice_duration" | "voice_provider_protocol" | "voice_provider_error" | "voice_provider_timeout" | "voice_model_unsupported"; readonly message: string; readonly objectId: string | null; readonly field: string | null; @@ -2382,13 +2475,29 @@ export interface MediaPublicationOutput extends Contract { readonly height: number; } +export interface ModelAvailability extends Contract { + readonly configId: string | null; + readonly modelId: string; + readonly identity: string; + readonly autoCheck: boolean; + readonly status: "available" | "unavailable" | "unknown"; + readonly reasonCode: string; + readonly checkedAt: number | null; + readonly expiresAt: number | null; + readonly revision: string; + readonly operations: ReadonlyArray; +} + export interface ModelConfigPublic extends ModelConnection { readonly id: string; readonly inputRules: InputRules; + readonly capabilityAutoCheck?: boolean | null; + readonly availability?: ModelAvailability | null; } export interface ModelConfigWrite extends ModelConnection { readonly apiKey: string | null; + readonly capabilityAutoCheck?: boolean | null; } export interface ModelConnection extends Contract { @@ -2458,6 +2567,21 @@ export interface ObjectVersion extends Contract { readonly revision: number; } +export interface OperationAvailability extends Contract { + readonly operation: string; + readonly profile: string; + readonly method: string; + readonly path: string; + readonly output: string; + readonly status: "available" | "unavailable" | "unknown"; + readonly reasonCode: string; + readonly checkedAt: number | null; + readonly expiresAt: number | null; + readonly declaration?: CapabilityDeclaration | null; + readonly execution?: CapabilityExecution | null; + readonly inputDetails: ReadonlyArray; +} + export interface PaletteData extends Contract { readonly name: string | null; readonly sourceImage: AssetRef | null; diff --git a/src/i18n/messages/en-US/availability.ts b/src/i18n/messages/en-US/availability.ts new file mode 100644 index 00000000..c9849ea9 --- /dev/null +++ b/src/i18n/messages/en-US/availability.ts @@ -0,0 +1,92 @@ +export const availability = { + "availability.reason.query_timeout": "Check timed out", + "availability.reason.query_busy": "Service busy. Try again shortly", + "availability.reason.query_failed": "Check failed", + + "availability.title": "Service capabilities", + "availability.check": "Check", + "availability.checking": "Checking…", + "availability.refresh": "Refresh", + "availability.retry": "Retry", + "availability.details": "Input requirements", + "availability.detail.service": "Service declaration", + "availability.detail.application": "Application limit", + "availability.detail.unsupported": "Not supported", + "availability.detail.countRange": "Count {min}–{max}", + "availability.detail.maxCount": "Up to {count}", + "availability.detail.minCount": "At least {count}", + "availability.detail.imageRange": "Count {min}–{max}", + "availability.detail.maxImages": "Up to {count}", + "availability.detail.minImages": "At least {count}", + "availability.detail.minimum": "Minimum {value}", + "availability.detail.maximum": "Maximum {value}", + "availability.detail.multiple": "Multiple of {value}", + "availability.detail.yes": "Yes", + "availability.detail.no": "No", + "availability.input.image": "Images", + "availability.input.reference_image": "Reference images", + "availability.input.video": "Video", + "availability.input.audio": "Audio", + "availability.input.mask": "Region input", + "availability.input.mask_image": "Region input", + "availability.input.output_pixels": "Output pixel count", + "availability.input.width": "Width", + "availability.input.height": "Height", + "availability.input.duration": "Duration", + "availability.input.speed": "Speech speed", + "availability.input.voice": "Voice", + "availability.autoCheck": "Automatically check service capabilities", + "availability.checkedAt": "Last checked: {time}", + "availability.status.available": "Supported", + "availability.status.unknown": "Unknown", + "availability.reason.not_checked": "Not checked", + "availability.reason.supported": "Supported", + "availability.reason.operation_unsupported": "Not supported", + "availability.reason.profile_unsupported": "Not supported", + "availability.reason.operation_disabled": "Not enabled on the service", + "availability.reason.route_disabled": "Endpoint not enabled", + "availability.reason.output_unsupported": "Output type not supported", + "availability.reason.execution_paused": "Service paused", + "availability.reason.observation_expired": + "Results expired. Refresh to check again.", + "availability.reason.connection_failed": + "Cannot connect. Check the service URL.", + "availability.reason.authentication_failed": + "Authentication failed. Check the API key.", + "availability.reason.protocol_unavailable": + "No capability declaration from this service", + "availability.reason.invalid_response": + "Could not read the capability declaration", + "availability.reason.unsupported_version": + "Declaration version not recognized", + "availability.reason.declaration_incomplete": "Incomplete declaration", + "availability.reason.required_inputs_missing": "Add the required inputs", + "availability.reason.input_contract_unknown": + "Input requirements not confirmed", + "availability.reason.model_not_declared": "No declaration for this model", + "availability.reason.operation_not_declared": "Not declared", + "availability.reason.profile_not_declared": "Not declared", + "availability.reason.route_not_declared": + "Endpoint requirements not confirmed", + "availability.reason.adapter_not_mapped": "Using saved configuration", + "availability.profile.text_response": "Text response", + "availability.profile.image_response": "Image response through chat", + "availability.profile.text_to_image": "Text to image", + "availability.profile.image_to_image": "Single-image editing", + "availability.profile.images_to_image": "Multi-image editing", + "availability.profile.text_to_video": "Text to video", + "availability.profile.image_to_video": "Image to video", + "availability.profile.video_to_video": "Video transformation", + "availability.profile.text_to_speech": "Speech synthesis", + "availability.profile.lyrics_to_music": "Music from lyrics and style", + "availability.profile.text_to_audio": "General audio generation", + "availability.profile.other": "Other request mode", + "availability.reason.input_count_unsupported": + "Multiple images not supported", + "availability.reason.input_count_unknown": "Image count not confirmed", + "availability.reason.input_unsupported": "Input not supported", + "availability.reason.input_count_exceeded": "Too many inputs", + "availability.reason.parameter_unsupported": "Parameter not supported", + "availability.reason.parameter_value_unsupported": + "Parameter outside service limits", +} as const; diff --git a/src/i18n/messages/en-US/errors.ts b/src/i18n/messages/en-US/errors.ts index 9b43de92..b8793860 100644 --- a/src/i18n/messages/en-US/errors.ts +++ b/src/i18n/messages/en-US/errors.ts @@ -156,6 +156,7 @@ export const errors = { "The first-frame media is disconnected or missing. Choose it again.", "errors.rules.continuationSource": "Continuation needs a video generated by the current H3 service as the previous segment.", + "errors.rules.serviceInputLimit": "Up to {limit} {kind} inputs; currently {current}.", "errors.rules.referenceLimit": "Too many {kind} references for this model (limit: {limit}).", "errors.rules.referenceFileSize": @@ -675,7 +676,7 @@ export const errors = { "errors.generationFailure.quota": "The model account balance or plan quota is insufficient.", "errors.generationFailure.rateLimit": - "The model call rate or quota is limited. Try again later.", + "The model service is busy or limiting requests. Try again later. If this persists, check service status, request limits, and account quota.", "errors.generationFailure.temporary": "The generation service is temporarily unavailable.", "errors.generationFailure.provider": @@ -687,6 +688,8 @@ export const errors = { "The generation service returned an unrecognized or incomplete result.", "errors.generationFailure.download": "The generated result could not be downloaded.", + "errors.generationFailure.downloadAddress": + "The result URL resolves to a non-public address, so the download stopped. Check the service's result URL or your DNS settings, then resume the original download.", "errors.generationFailure.notFoundTask": "The provider could not find this task.", "errors.generationFailure.notFoundEndpoint": @@ -915,4 +918,56 @@ export const errors = { "Music generation · 30 s · measured in this run", "errors.catalog.contextVoice": "SpongeBob voice read · measured with this input, not priced per character", + "errors.capability.paused": "Service paused. Resume it and try again.", + "errors.capability.required": "Add the required inputs.", + "errors.capability.imageBytes": "The image is too large. Compress it and try again.", + "errors.capability.totalBytes": "The images exceed the total size limit. Remove or compress images.", + "errors.capability.annotationSelection": "Add a selection to use the annotation reference.", + "errors.capability.selectionInputs": "The image dimensions or count do not support selection input.", + "errors.capability.selectionDimensions": "The selection and image dimensions must match.", + "errors.capability.selectionRequired": "Select the area to edit.", + "errors.capability.selectionUnreadable": "Cannot read the image or selection. Select it again.", + "errors.capability.sourceType": "Source assets must be images.", + "errors.capability.imageCount": "Up to {limit} images; {current} added.", + "errors.capability.sourceCount": "This tool requires {count} source images.", + "errors.capability.inputCount": "Up to {limit} {input} inputs; {current} added.", + "errors.capability.inputUnsupported": "This model does not accept {label}.", + "errors.capability.unsupported": "This model does not support {label}.", + "errors.capability.disabled": "The service has not enabled {label}.", + "errors.capability.add": "Add {label}.", + "errors.capability.parameter": "{label} does not meet the service requirements.", + "errors.capability.provide": "Provide {inputs}.", + "errors.capability.or": " or ", + "errors.capability.label.textReplies": "text replies", + "errors.capability.label.imageReplies": "image replies", + "errors.capability.label.imageGeneration": "image generation", + "errors.capability.label.imageEditing": "image editing", + "errors.capability.label.textVideo": "text-to-video generation", + "errors.capability.label.imageVideo": "image-to-video generation", + "errors.capability.label.videoEditing": "video editing", + "errors.capability.label.speech": "speech synthesis", + "errors.capability.label.music": "music generation", + "errors.capability.label.operation": "this operation", + "errors.capability.label.image": "images", + "errors.capability.label.audio": "audio", + "errors.capability.label.video": "video", + "errors.capability.label.text": "text", + "errors.capability.label.voice": "a voice", + "errors.capability.label.referenceAudio": "reference audio", + "errors.capability.label.style": "a style description", + "errors.capability.label.transcript": "a reference transcript", + "errors.capability.label.selection": "a selection", + "errors.capability.label.annotation": "an annotation reference", + "errors.capability.label.inputs": "inputs", + "errors.capability.label.width": "Image width", + "errors.capability.label.height": "Image height", + "errors.capability.label.size": "Image dimensions", + "errors.capability.dimensions": "Image dimensions do not meet the service requirements.", + "errors.capability.label.duration": "Duration", + "errors.capability.label.speed": "Speech speed", + "errors.capability.label.language": "Language", + "errors.capability.label.fps": "Frame rate", + "errors.capability.label.frames": "Frame count", + "errors.capability.label.generateAudio": "Audio generation setting", + "errors.capability.label.parameter": "The selected parameter", } as const; diff --git a/src/i18n/messages/en-US/index.ts b/src/i18n/messages/en-US/index.ts index 29898459..475ef04b 100644 --- a/src/i18n/messages/en-US/index.ts +++ b/src/i18n/messages/en-US/index.ts @@ -1,3 +1,4 @@ +import { availability } from "./availability.ts"; import { voiceLibrary } from "./voiceLibrary.ts"; import { trace } from "./trace.ts"; import { modelSetup } from "./modelSetup.ts"; @@ -23,6 +24,7 @@ type ZhCatalog = typeof import("../zh-CN/index.ts").zhCN; /** `Record` makes a missing English key a compile error. */ export const enUS: Record = { ...common, + ...availability, ...voiceLibrary, ...settings, ...modelSetup, diff --git a/src/i18n/messages/en-US/models.ts b/src/i18n/messages/en-US/models.ts index 6e2d6c99..123ebc62 100644 --- a/src/i18n/messages/en-US/models.ts +++ b/src/i18n/messages/en-US/models.ts @@ -1,5 +1,7 @@ /** Model connection and service forms. */ export const models = { + "models.service.urlPlaceholder": "Enter your service's API base URL", + "models.service.modelPlaceholder": "Enter the model ID from your service", "models.service.proxyMode": "Network connection mode", "models.service.proxySystem": "Use system proxy", "models.service.proxyDirect": "Direct (no proxy)", @@ -71,7 +73,7 @@ export const models = { "models.service.hide": "Hide", "models.service.show": "Show", "models.service.urlNote": - "Self-hosted services usually use a /v1 URL; when connecting through an SSH tunnel, enter the local forwarded address.", + "Enter the API base URL provided by your service.", "models.service.inputCapabilities": "Input capabilities", "models.service.referenceOutput": "References and output", "models.service.textInput": "Text", diff --git a/src/i18n/messages/zh-CN/availability.ts b/src/i18n/messages/zh-CN/availability.ts new file mode 100644 index 00000000..2ba80dff --- /dev/null +++ b/src/i18n/messages/zh-CN/availability.ts @@ -0,0 +1,82 @@ +export const availability = { + "availability.reason.query_timeout": "检查超时", + "availability.reason.query_busy": "服务繁忙，请稍后重试", + "availability.reason.query_failed": "检查失败", + + "availability.title": "服务能力", + "availability.check": "检查", + "availability.checking": "检查中…", + "availability.refresh": "刷新", + "availability.retry": "重试", + "availability.details": "输入要求", + "availability.detail.service": "服务声明", + "availability.detail.application": "应用限制", + "availability.detail.unsupported": "不支持", + "availability.detail.countRange": "数量 {min}–{max}", + "availability.detail.maxCount": "最多 {count} 个", + "availability.detail.minCount": "至少 {count} 个", + "availability.detail.imageRange": "{min}–{max} 张", + "availability.detail.maxImages": "最多 {count} 张", + "availability.detail.minImages": "至少 {count} 张", + "availability.detail.minimum": "最小 {value}", + "availability.detail.maximum": "最大 {value}", + "availability.detail.multiple": "{value} 的倍数", + "availability.detail.yes": "是", + "availability.detail.no": "否", + "availability.input.image": "图片", + "availability.input.reference_image": "参考图片", + "availability.input.video": "视频", + "availability.input.audio": "音频", + "availability.input.mask": "选区输入", + "availability.input.mask_image": "选区输入", + "availability.input.output_pixels": "输出像素总数", + "availability.input.width": "宽度", + "availability.input.height": "高度", + "availability.input.duration": "时长", + "availability.input.speed": "语速", + "availability.input.voice": "音色", + "availability.autoCheck": "自动检查服务能力", + "availability.checkedAt": "最近检查：{time}", + "availability.status.available": "支持", + "availability.status.unknown": "待确认", + "availability.reason.not_checked": "尚未检查", + "availability.reason.supported": "支持", + "availability.reason.operation_unsupported": "不支持", + "availability.reason.profile_unsupported": "不支持", + "availability.reason.operation_disabled": "服务未开放", + "availability.reason.route_disabled": "当前接口未开放", + "availability.reason.output_unsupported": "不支持此输出类型", + "availability.reason.execution_paused": "服务已暂停", + "availability.reason.observation_expired": "检查结果已过期，请刷新", + "availability.reason.connection_failed": "无法连接服务，请检查地址", + "availability.reason.authentication_failed": "认证失败，请检查密钥", + "availability.reason.protocol_unavailable": "服务未提供能力声明", + "availability.reason.invalid_response": "无法读取能力声明", + "availability.reason.unsupported_version": "无法识别声明版本", + "availability.reason.declaration_incomplete": "声明不完整", + "availability.reason.required_inputs_missing": "请补充必要输入", + "availability.reason.input_contract_unknown": "输入要求待确认", + "availability.reason.model_not_declared": "服务未声明此模型", + "availability.reason.operation_not_declared": "未声明", + "availability.reason.profile_not_declared": "未声明", + "availability.reason.route_not_declared": "接口要求待确认", + "availability.reason.adapter_not_mapped": "使用已保存的配置", + "availability.profile.text_response": "文字响应", + "availability.profile.image_response": "聊天图片响应", + "availability.profile.text_to_image": "文字生图", + "availability.profile.image_to_image": "单图编辑", + "availability.profile.images_to_image": "多图编辑", + "availability.profile.text_to_video": "文字生成视频", + "availability.profile.image_to_video": "图片生成视频", + "availability.profile.video_to_video": "视频变换", + "availability.profile.text_to_speech": "语音朗读", + "availability.profile.lyrics_to_music": "歌词与风格生成音乐", + "availability.profile.text_to_audio": "通用音频生成", + "availability.profile.other": "其他请求方式", + "availability.reason.input_count_unsupported": "不支持多图", + "availability.reason.input_count_unknown": "多图数量待确认", + "availability.reason.input_unsupported": "不支持此输入", + "availability.reason.input_count_exceeded": "输入数量超限", + "availability.reason.parameter_unsupported": "不支持此参数", + "availability.reason.parameter_value_unsupported": "参数不符合要求", +} as const; diff --git a/src/i18n/messages/zh-CN/errors.ts b/src/i18n/messages/zh-CN/errors.ts index 77a9cc13..18526c16 100644 --- a/src/i18n/messages/zh-CN/errors.ts +++ b/src/i18n/messages/zh-CN/errors.ts @@ -110,6 +110,7 @@ export const errors = { "errors.rules.continuationSource": "接续需要连接一段由当前 H3 服务生成的视频作为上一段。", "errors.rules.referenceLimit": "参考{kind}超过当前模型上限（{limit} 个）。", + "errors.rules.serviceInputLimit": "最多支持 {limit} 个{kind}，当前为 {current} 个。", "errors.rules.referenceFileSize": "参考文件超过当前模型的大小限制", "errors.rules.referenceBudget": "参考总大小超过当前模型的请求预算", "errors.rules.referenceCount": "参考数量超出当前模型限制", @@ -512,13 +513,14 @@ export const errors = { "errors.generationFailure.authentication": "生成服务鉴权失败。", "errors.generationFailure.permission": "生成服务拒绝了访问请求。请检查 API Key 是否正确及有效、模型调用权限和服务安全策略。", "errors.generationFailure.quota": "模型账户余额或套餐额度不足。", - "errors.generationFailure.rateLimit": "模型调用频率或配额受限。", + "errors.generationFailure.rateLimit": "模型服务繁忙或请求受限，请稍后再试。若持续发生，请检查服务状态、调用频率和账户额度。", "errors.generationFailure.temporary": "生成服务暂时不可用。", "errors.generationFailure.provider": "生成服务未能完成当前操作。", "errors.generationFailure.submissionUnknown": "生成请求的提交结果不明。", "errors.generationFailure.local": "本地生成准备失败。", "errors.generationFailure.protocol": "生成服务返回了无法识别或不完整的结果。", "errors.generationFailure.download": "生成结果下载失败。", + "errors.generationFailure.downloadAddress": "生成结果地址解析为非公网地址，下载已停止。请检查服务返回的地址或本机域名解析设置，再恢复原任务的下载。", "errors.generationFailure.notFoundTask": "供应商未找到该任务。", "errors.generationFailure.notFoundEndpoint": "生成接口或模型不存在。", "errors.generationFailure.cancelled": "供应商任务已取消。", @@ -697,4 +699,56 @@ export const errors = { "errors.catalog.contextFirstLast": "首尾帧 · 5 秒 · 720P · 1 条", "errors.catalog.contextMusic": "音乐生成 · 30 秒 · 本次实测", "errors.catalog.contextVoice": "动漫海绵朗读 · 本次输入实测，非按字数报价", + "errors.capability.paused": "服务已暂停，请恢复后重试。", + "errors.capability.required": "请补充所需素材。", + "errors.capability.imageBytes": "图片文件过大，请压缩后重试。", + "errors.capability.totalBytes": "图片总大小超出限制，请减少图片或压缩后重试。", + "errors.capability.annotationSelection": "标注参考图片需要配合选区使用。", + "errors.capability.selectionInputs": "当前图片尺寸或数量不支持选区输入。", + "errors.capability.selectionDimensions": "选区与图片的尺寸不一致。", + "errors.capability.selectionRequired": "请先选择需要修改的区域。", + "errors.capability.selectionUnreadable": "无法读取图片或选区，请重新选择。", + "errors.capability.sourceType": "来源素材必须是图片。", + "errors.capability.imageCount": "最多支持 {limit} 张图片，当前为 {current} 张。", + "errors.capability.sourceCount": "该工具需要 {count} 张来源图片。", + "errors.capability.inputCount": "最多支持 {limit} 个{input}输入，当前为 {current} 个。", + "errors.capability.inputUnsupported": "该模型不支持{label}输入。", + "errors.capability.unsupported": "该模型不支持{label}。", + "errors.capability.disabled": "服务未开放{label}。", + "errors.capability.add": "请添加{label}。", + "errors.capability.parameter": "{label}不符合服务要求。", + "errors.capability.provide": "请提供{inputs}。", + "errors.capability.or": "或", + "errors.capability.label.textReplies": "文字回复", + "errors.capability.label.imageReplies": "图片回复", + "errors.capability.label.imageGeneration": "图片生成", + "errors.capability.label.imageEditing": "图片编辑", + "errors.capability.label.textVideo": "文字生成视频", + "errors.capability.label.imageVideo": "图片生成视频", + "errors.capability.label.videoEditing": "视频编辑", + "errors.capability.label.speech": "语音合成", + "errors.capability.label.music": "音乐生成", + "errors.capability.label.operation": "此操作", + "errors.capability.label.image": "图片", + "errors.capability.label.audio": "音频", + "errors.capability.label.video": "视频", + "errors.capability.label.text": "文字", + "errors.capability.label.voice": "音色", + "errors.capability.label.referenceAudio": "参考音频", + "errors.capability.label.style": "风格说明", + "errors.capability.label.transcript": "参考文本", + "errors.capability.label.selection": "选区", + "errors.capability.label.annotation": "标注参考图片", + "errors.capability.label.inputs": "素材", + "errors.capability.label.width": "图片宽度", + "errors.capability.label.height": "图片高度", + "errors.capability.label.size": "图片尺寸", + "errors.capability.dimensions": "图片尺寸不符合服务要求。", + "errors.capability.label.duration": "时长", + "errors.capability.label.speed": "语速", + "errors.capability.label.language": "语言", + "errors.capability.label.fps": "帧率", + "errors.capability.label.frames": "帧数", + "errors.capability.label.generateAudio": "音频生成选项", + "errors.capability.label.parameter": "所选参数", } as const; diff --git a/src/i18n/messages/zh-CN/index.ts b/src/i18n/messages/zh-CN/index.ts index 3561f1e8..d427f360 100644 --- a/src/i18n/messages/zh-CN/index.ts +++ b/src/i18n/messages/zh-CN/index.ts @@ -1,3 +1,4 @@ +import { availability } from "./availability.ts"; import { voiceLibrary } from "./voiceLibrary.ts"; import { trace } from "./trace.ts"; import { modelSetup } from "./modelSetup.ts"; @@ -20,6 +21,7 @@ import { tools } from "./tools.ts"; /** Canonical copy. Every other locale must provide the same keys. */ export const zhCN = { ...common, + ...availability, ...voiceLibrary, ...settings, ...modelSetup, diff --git a/src/i18n/messages/zh-CN/models.ts b/src/i18n/messages/zh-CN/models.ts index 2e9c132c..875fae1c 100644 --- a/src/i18n/messages/zh-CN/models.ts +++ b/src/i18n/messages/zh-CN/models.ts @@ -52,6 +52,8 @@ export const models = { "models.service.providerApi": "服务商 API", "models.service.provider": "提供商", "models.service.url": "服务地址（Base URL）", + "models.service.urlPlaceholder": "输入服务的 API 根地址", + "models.service.modelPlaceholder": "输入服务中的模型 ID", "models.service.noKey": "无需密钥", "models.service.keyConfigured": "密钥已配置", "models.service.noAuthPlaceholder": "此服务无需鉴权", @@ -62,7 +64,7 @@ export const models = { "models.service.hide": "隐藏", "models.service.show": "显示", "models.service.urlNote": - "自部署服务通常填写 /v1 地址；通过 SSH 隧道连接时填写本地转发地址。", + "填写服务提供的 API 根地址。", "models.service.inputCapabilities": "输入能力", "models.service.referenceOutput": "参考与输出能力", "models.service.textInput": "文本", diff --git a/tests/assistantToolText.test.ts b/tests/assistantToolText.test.ts index e1742880..9a701c6b 100644 --- a/tests/assistantToolText.test.ts +++ b/tests/assistantToolText.test.ts @@ -24,3 +24,16 @@ test("agent tool errors never show Chinese in the English reply card", () => { setActiveLocale("zh-CN"); assert.equal(assistantToolText("本轮工具入口已关闭。"), "本轮工具入口已关闭。"); }); + +test("image geometry rejections retain their concrete cause in both languages", () => { + setActiveLocale("en-US"); + assert.equal( + assistantToolText("图片尺寸不符合服务要求。", undefined, "capability_unavailable"), + "Image dimensions do not meet the service requirements.", + ); + setActiveLocale("zh-CN"); + assert.equal( + assistantToolText("图片尺寸不符合服务要求。", undefined, "capability_unavailable"), + "图片尺寸不符合服务要求。", + ); +}); diff --git a/tests/backendRules.test.ts b/tests/backendRules.test.ts index 85b01159..b3770a2b 100644 --- a/tests/backendRules.test.ts +++ b/tests/backendRules.test.ts @@ -14,8 +14,40 @@ import { backendReferences, backendVideoMode, isGenerationModel, + serviceGenerationSettings, type ModelConfig, } from "../src/backend/rules.ts"; +import { failedAvailability } from "../src/backend/availability.ts"; +import { setActiveLocale } from "../src/i18n/messages.ts"; + +test("fresh service counts validate transmitted images without rewriting saved rules", () => { + const configured = config("image", { + profile: "openai-image", allowedKinds: ["text", "image"], maxImages: 4, + imageOutputs: [{ aspectRatio: "1:1", size: "768x768" }], + }); + const now = Date.now(); + configured.availability = { + configId: configured.id, modelId: configured.modelId, identity: "fresh-count", + autoCheck: false, status: "available", reasonCode: "supported", checkedAt: now, + expiresAt: now + 30000, revision: "test", operations: [{ + operation: "image_edit", profile: "image_to_image", method: "POST", path: "/v1/images/edits", output: "image", + status: "available", reasonCode: "supported", checkedAt: now, expiresAt: now + 30000, + declaration: { status: "available", reasonCode: "supported", checkedAt: now, expiresAt: now + 30000 }, + inputDetails: [{ name: "image", source: "service", support: "supported", minCount: 1, maxCount: 1, + min: null, max: null, multipleOf: null, values: null }], + }], + }; + const settings = serviceGenerationSettings({ ...generationDefaults("image"), modelConfigId: configured.id, prompt: "Keep the draft" }, configured); + const references = [media("image"), media("image")]; + setActiveLocale("zh-CN"); + assert.equal(backendBlocker(settings, references, [configured]), "最多支持 1 张图片，当前为 2 张。"); + setActiveLocale("en-US"); + assert.equal(backendBlocker(settings, references, [configured]), "Up to 1 images; 2 added."); + configured.availability = failedAvailability(configured.availability); + assert.equal(backendBlocker(settings, references, [configured]), null); + assert.equal(configured.inputRules.maxImages, 4); + setActiveLocale("zh-CN"); +}); const config = ( kind: ModelConfig["kind"], diff --git a/tests/browser/capability_discovery.mjs b/tests/browser/capability_discovery.mjs new file mode 100644 index 00000000..d6e73581 --- /dev/null +++ b/tests/browser/capability_discovery.mjs @@ -0,0 +1,932 @@ +/** Real UI and application, with a controllable HTTP capability service and fixture generation. */ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { once } from "node:events"; +import { dirname, extname, basename, join } from "node:path"; +import { launchBrowser } from "./engine.mjs"; + +const base = process.env.CREATION_WEB_URL ?? "http://127.0.0.1:5179"; +const api = process.env.CREATION_API_URL ?? "http://127.0.0.1:8019"; +const headers = { + "X-OpenDirector-Request": "1", + "Content-Type": "application/json", +}; +async function call(path, body, method = body ? "POST" : "GET") { + const requestHeaders = { ...headers }; + if (body?.type === "project.create") + delete requestHeaders["X-OpenDirector-Session"]; + const response = await fetch(api + path, { + method, + headers: requestHeaders, + ...(body ? { body: JSON.stringify(body) } : {}), + }); + assert.ok( + response.ok, + `${response.status}: ${await response.clone().text()}`, + ); + return response.status === 204 ? null : response.json(); +} +async function until(check) { + const deadline = performance.now() + 20000; + while (performance.now() < deadline) { + const value = await check(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw Error("Expected persisted capability workflow state was not observed"); +} +assert.equal((await call("/api/creation-test-health")).isolated, true); +let mode = "unsupported", + delay = 0, + queries = 0; +const queryEvents = []; +const served = `capability-image-${crypto.randomUUID().slice(0, 8)}`; +function document() { + const route = { + transport: "http", + method: "POST", + path: "/v1/images/generations", + exposure: "enabled", + }; + const operations = Object.fromEntries( + [ + "chat", + "image_generation", + "image_edit", + "video_generation", + "speech", + "audio_generation", + ].map((id) => [ + id, + { support: "unknown", exposure: "unknown", profiles: [] }, + ]), + ); + operations.image_generation = { + support: mode === "unsupported" ? "unsupported" : "supported", + exposure: "enabled", + profiles: [ + { + id: "text_to_image", + support: mode === "unsupported" ? "unsupported" : "supported", + required_inputs: ["text"], + outputs: ["image"], + routes: [route], + }, + ], + execution: { + state: mode === "paused" ? "paused" : "unpaused", + scope: "deployment", + observed_at: new Date().toISOString(), + }, + }; + const editRoute = { + ...route, + path: "/v1/images/edits", + request: { + inputs: { + image: { + support: "supported", + min_count: 1, + ...(mode === "partial" ? {} : { max_count: 3 }), + }, + mask_image: { support: "supported", max_count: 1 }, + }, + parameters: { + width: { support: "supported", minimum: 64, maximum: 2048, step: 16 }, + }, + }, + }; + operations.image_edit = { + support: mode === "unsupported" ? "unsupported" : "supported", + exposure: "enabled", + profiles: [ + { + id: "image_to_image", + support: mode === "unsupported" ? "unsupported" : "supported", + required_inputs: ["text", "image"], + outputs: ["image"], + routes: [editRoute], + }, + ], + execution: operations.image_generation.execution, + }; + return { + object: "omni.capabilities", + schema_version: 1, + observed_at: new Date().toISOString(), + models: [{ id: served, stage_ids: ["7"], operations }], + stages: [{ id: "7", type: "diffusion", depends_on: [] }], + routes: [route, editRoute], + }; +} +const service = createServer((request, response) => { + if (request.url !== "/v1/omni/capabilities") { + response.writeHead(404).end(); + return; + } + queries += 1; + const queryEvent = { mode, received: Date.now() }; + queryEvents.push(queryEvent); + if (mode === "connection") { + request.socket.destroy(); + return; + } + const body = document(), + status = + mode === "legacy" + ? 404 + : mode === "authentication" + ? 401 + : mode === "failure" + ? 503 + : 200; + setTimeout(() => { + queryEvent.responded = Date.now(); + response.writeHead(status, { "Content-Type": "application/json" }); + response.end(JSON.stringify(body)); + }, delay); +}); +service.listen(0, "127.0.0.1"); +await once(service, "listening"); +const serviceUrl = `http://127.0.0.1:${service.address().port}/v1`; +const browser = await launchBrowser(); +const page = await browser.newPage({ + locale: "en-US", + viewport: { width: 1500, height: 1000 }, +}); +await page.addInitScript(() => { + localStorage.setItem("opendirector.modelSetup.v1", "dismissed"); + localStorage.setItem("opendirector.locale.v1", "en-US"); +}); +page.setDefaultTimeout(15000); +const errors = []; +const openConflicts = []; +const sessionRenewals = new Set(); +const renewalFailures = []; +const screenshots = []; +page.on("pageerror", (error) => errors.push(error.message)); +page.on("response", async (response) => { + if (response.url().endsWith("/open-canvas") && response.status() === 409) + openConflicts.push(await response.text().catch(() => "unavailable")); +}); +page.on("request", (request) => { + if ( + request.method() === "PUT" && + new URL(request.url()).pathname.startsWith("/api/creation/sessions/") + ) + sessionRenewals.add(request); + const session = request.headers()["x-opendirector-session"]; + if (session) headers["X-OpenDirector-Session"] = session; +}); +page.on("requestfinished", async (request) => { + if (!sessionRenewals.has(request)) return; + const response = await request.response(); + if (response?.status() !== 200) renewalFailures.push(response?.status()); + sessionRenewals.delete(request); +}); +page.on("requestfailed", (request) => { + if (sessionRenewals.has(request)) renewalFailures.push(request.failure()); + sessionRenewals.delete(request); +}); +async function reloadAfterRenewal() { + // Advancing the capability clock can also start a session heartbeat. Finish + // that request before pagehide releases the session during this reload. + await until(() => sessionRenewals.size === 0); + assert.deepEqual( + renewalFailures, + [], + "session renewals complete before reload", + ); + await page.reload(); +} +let saved; +try { + const projectId = crypto.randomUUID(); + const projectPath = `/api/creation/projects/${projectId}`; + const command = (type, payload, extra = {}) => + call(`${projectPath}/commands`, { + commandId: crypto.randomUUID(), + projectId, + type, + payload, + ...extra, + }); + await command("project.create", { + name: `Capability checks ${projectId.slice(0, 8)}`, + }); + await page.goto(`${base}/ai-tool/ai-canvas/${projectId}?creation=unified`); + await page.getByRole("button", { name: "Settings", exact: true }).click(); + await page + .getByRole("button", { name: "Model connections", exact: true }) + .click(); + await page + .locator(".cw-model-setup-row") + .filter({ hasText: "Image model" }) + .getByRole("button") + .click(); + await page.getByRole("button", { name: "Add service", exact: true }).click(); + assert.equal( + await page + .getByLabel("Service URL (Base URL)", { exact: true }) + .getAttribute("placeholder"), + "Enter your service's API base URL", + ); + assert.doesNotMatch( + await page.locator(".cw-model-editor").innerText(), + /SSH|127\.0\.0\.1:8200/, + ); + await page + .getByLabel("Service URL (Base URL)", { exact: true }) + .fill(serviceUrl); + await page.getByRole("button", { name: "Add manually", exact: true }).click(); + assert.equal( + await page + .getByLabel("Model ID (the service's served-model-name)", { exact: true }) + .getAttribute("placeholder"), + "Enter the model ID from your service", + ); + await page + .getByLabel("Model ID (the service's served-model-name)", { exact: true }) + .fill(served); + await page + .locator(".cw-model-manual") + .getByRole("button", { name: "Add", exact: true }) + .click(); + const draft = page.locator(".cw-model-editor .cw-model-availability"); + async function presentInBothLanguages(state, expected) { + for (const locale of ["en-US", "zh-CN"]) { + await page.evaluate((next) => { + localStorage.setItem("opendirector.locale.v1", next); + window.dispatchEvent( + new StorageEvent("storage", { + key: "opendirector.locale.v1", + newValue: next, + }), + ); + }, locale); + await until( + async () => + (await page.locator("html").getAttribute("lang")) === locale, + ); + if (expected) { + await draft.getByText(expected[locale], { exact: true }).waitFor(); + assert.equal( + await draft.getByText(expected[locale], { exact: true }).count(), + 1, + `${state}: shared reason appears once in ${locale}`, + ); + } + if (locale === "en-US") + assert.doesNotMatch(await draft.innerText(), /[\u3400-\u9fff]/); + else assert.doesNotMatch(await draft.innerText(), /\b(?:AM|PM)\b/); + assert.equal( + await page + .getByLabel( + locale === "en-US" + ? "Service URL (Base URL)" + : "服务地址（Base URL）", + { exact: true }, + ) + .inputValue(), + serviceUrl, + ); + if (process.env.CREATION_SCREENSHOT) { + const target = process.env.CREATION_SCREENSHOT; + const path = join( + dirname(target), + `${basename(target, extname(target))}-${state}-${locale}.png`, + ); + const option = page + .locator(".cw-model-option") + .filter({ has: page.locator(".cw-model-availability") }); + await option.scrollIntoViewIfNeeded(); + const bounds = await option.evaluate((element) => { + const card = element + .querySelector(".cw-model-availability") + .getBoundingClientRect(); + const label = element.querySelector("label").getBoundingClientRect(); + const scroller = element + .closest(".cw-model-editor-scroll") + .getBoundingClientRect(); + const footer = element + .closest("form") + .querySelector("footer") + .getBoundingClientRect(); + return { + cardTop: card.top, + cardBottom: card.bottom, + labelBottom: label.bottom, + labelWidth: label.width, + cardWidth: card.width, + visibleTop: scroller.top, + visibleBottom: Math.min(scroller.bottom, footer.top), + }; + }); + assert.ok( + bounds.cardTop >= bounds.labelBottom, + `${state}: capability card occupies the row below the model`, + ); + assert.ok( + bounds.labelWidth >= bounds.cardWidth * 0.9, + `${state}: model identity retains the available width`, + ); + assert.ok( + bounds.cardTop >= bounds.visibleTop - 1 && + bounds.cardBottom <= bounds.visibleBottom + 1, + `${state}: the capability card is fully visible above the footer: ${JSON.stringify(bounds)}`, + ); + await option.screenshot({ path }); + screenshots.push(path); + } + } + await page.evaluate(() => { + localStorage.setItem("opendirector.locale.v1", "en-US"); + window.dispatchEvent( + new StorageEvent("storage", { + key: "opendirector.locale.v1", + newValue: "en-US", + }), + ); + }); + await until( + async () => (await page.locator("html").getAttribute("lang")) === "en-US", + ); + } + assert.equal( + await draft.getByText("Not checked", { exact: true }).count(), + 1, + ); + const previousQueries = queries; + await presentInBothLanguages("unchecked", { + "en-US": "Not checked", + "zh-CN": "尚未检查", + }); + await page + .getByLabel("Service URL (Base URL)", { exact: true }) + .fill(serviceUrl + "/"); + await page.waitForTimeout(150); + assert.equal( + queries, + previousQueries, + "editing a form must not query the model server", + ); + await page + .getByLabel("Service URL (Base URL)", { exact: true }) + .fill(serviceUrl); + delay = 350; + const lateDraft = page.waitForResponse((response) => + response.url().endsWith("/model-configs/availability/check"), + ); + await draft + .getByRole("button", { name: /^(Check|Refresh|Retry)$/, exact: true }) + .click(); + await page + .getByLabel("Service URL (Base URL)", { exact: true }) + .fill(serviceUrl + "/changed"); + await lateDraft; + await page.waitForTimeout(50); + assert.equal( + await draft.getByText("Not supported").count(), + 0, + "an old draft response must not describe the edited connection", + ); + await page + .getByLabel("Service URL (Base URL)", { exact: true }) + .fill(serviceUrl); + delay = 0; + await draft + .getByRole("button", { name: /^(Check|Refresh|Retry)$/, exact: true }) + .click(); + await draft.getByText("Not supported").first().waitFor(); + assert.equal( + (await call("/api/model-configs")).some((item) => item.modelId === served), + false, + "a draft check must not save a model", + ); + assert.equal( + await draft.getByText("Not supported", { exact: true }).count(), + 1, + ); + + // The real form keeps draft values while presenting each state and switching language. + await presentInBothLanguages("unsupported", { + "en-US": "Not supported", + "zh-CN": "不支持", + }); + const draftUrl = page.getByLabel("Service URL (Base URL)", { exact: true }); + async function checkDraft(nextMode) { + mode = nextMode; + await draft + .getByRole("button", { name: /^(Check|Refresh|Retry)$/, exact: true }) + .click(); + await until(async () => !(await draft.getByRole("button").isDisabled())); + assert.equal(await draftUrl.inputValue(), serviceUrl); + assert.equal( + (await call("/api/model-configs")).some( + (item) => item.modelId === served, + ), + false, + ); + } + await checkDraft("supported"); + assert.equal( + await draft.getByRole("status").count(), + 0, + "normal state has no aggregate summary", + ); + for (const label of [ + "Text to image", + "Single-image editing", + "Multi-image editing", + ]) + assert.equal(await draft.getByText(label, { exact: true }).count(), 1); + assert.equal(await draft.locator("[data-availability=available]").count(), 3); + await draft.getByText("Input requirements", { exact: true }).click(); + await presentInBothLanguages("normal", null); + assert.equal( + await draft.getByText("Count 1–3", { exact: true }).count(), + 1, + "the same service image limit appears once", + ); + assert.equal( + await draft.getByText("Region input", { exact: true }).count(), + 1, + ); + assert.equal( + await draft.getByText("Service declaration", { exact: true }).count(), + 1, + "the common input-constraint source appears once", + ); + await page.evaluate(() => { + localStorage.setItem("opendirector.locale.v1", "zh-CN"); + window.dispatchEvent( + new StorageEvent("storage", { + key: "opendirector.locale.v1", + newValue: "zh-CN", + }), + ); + }); + await draft.getByText("多图编辑", { exact: true }).waitFor(); + assert.equal(await draft.getByText("选区输入", { exact: true }).count(), 1); + assert.equal(await draft.getByText("1–3 张", { exact: true }).count(), 1); + assert.equal(await draft.getByText("服务声明", { exact: true }).count(), 1); + assert.equal( + await page.getByLabel("服务地址（Base URL）", { exact: true }).inputValue(), + serviceUrl, + ); + await page.evaluate(() => { + localStorage.setItem("opendirector.locale.v1", "en-US"); + window.dispatchEvent( + new StorageEvent("storage", { + key: "opendirector.locale.v1", + newValue: "en-US", + }), + ); + }); + await draft.getByText("Multi-image editing", { exact: true }).waitFor(); + await checkDraft("partial"); + assert.equal(await draft.getByRole("status").count(), 0); + assert.equal( + await draft.getByText("Image count not confirmed", { exact: true }).count(), + 1, + ); + + const draftCheckUrl = "**/api/model-configs/availability/check"; + await presentInBothLanguages("partial", { + "en-US": "Image count not confirmed", + "zh-CN": "多图数量待确认", + }); + // Error observations last two seconds. Hold browser time so both screenshots + // inspect the same response, without issuing another check on a locale change. + await page.clock.install(); + await page.clock.pauseAt(new Date()); + for (const [state, expected] of [ + [ + "legacy", + { + "en-US": "No capability declaration from this service", + "zh-CN": "服务未提供能力声明", + }, + ], + [ + "connection", + { + "en-US": "Cannot connect. Check the service URL.", + "zh-CN": "无法连接服务，请检查地址", + }, + ], + [ + "authentication", + { + "en-US": "Authentication failed. Check the API key.", + "zh-CN": "认证失败，请检查密钥", + }, + ], + ["failure", { "en-US": "Check failed", "zh-CN": "检查失败" }], + ]) { + await checkDraft(state); + await presentInBothLanguages(state, expected); + assert.equal( + await draft + .getByText("Image count not confirmed", { exact: true }) + .count(), + 0, + `${state}: no secondary multi-image reason`, + ); + assert.equal( + await draft.getByText("Not checked", { exact: true }).count(), + 0, + ); + assert.equal( + await draft.locator("li [data-availability=unknown]").count(), + 3, + ); + } + await page.route( + draftCheckUrl, + (route) => + route.fulfill({ + status: 503, + contentType: "application/json", + body: JSON.stringify({ detail: "Fixture check failed" }), + }), + { times: 1 }, + ); + await checkDraft("supported"); + assert.equal(await draft.getByRole("alert").count(), 1); + assert.equal( + await draft.getByText("Not checked", { exact: true }).count(), + 0, + "a failed check does not also say unchecked", + ); + await page.unroute(draftCheckUrl); + await page.clock.resume(); + await page.clock.setSystemTime(new Date()); + delay = 400; + mode = "paused"; + await draft.getByRole("button", { name: "Retry", exact: true }).click(); + assert.equal( + await draft.getByText("Checking…", { exact: true }).count(), + 1, + "checking appears only on the button", + ); + assert.equal(await draft.getByRole("status").count(), 0); + await draft.getByText("Service paused", { exact: true }).waitFor(); + delay = 0; + assert.equal( + await draft.getByText("Service paused", { exact: true }).count(), + 1, + ); + // Clock control lets each language capture the same unexpired observation. + await page.clock.pauseAt(new Date()); + await presentInBothLanguages("paused", { + "en-US": "Service paused", + "zh-CN": "服务已暂停", + }); + await page.clock.fastForward(5100); + await until( + async () => + (await draft.getByText("Service paused", { exact: true }).count()) === 0, + ); + assert.equal( + await draft.getByText("Service paused", { exact: true }).count(), + 0, + ); + assert.equal( + await draft.locator("[data-availability=available]").count(), + 3, + "pause expiry restores the retained declaration", + ); + await page.clock.fastForward(25100); + await draft + .getByText("Results expired. Refresh to check again.", { exact: true }) + .waitFor(); + assert.equal( + await draft + .getByText("Results expired. Refresh to check again.", { exact: true }) + .count(), + 1, + ); + assert.equal(await draft.locator("li[data-stale]").count(), 3); + await presentInBothLanguages("expired", { + "en-US": "Results expired. Refresh to check again.", + "zh-CN": "检查结果已过期，请刷新", + }); + assert.equal(await draftUrl.inputValue(), serviceUrl); + await page.clock.resume(); + await page.clock.setSystemTime(new Date()); + await checkDraft("unsupported"); + assert.equal( + await page + .getByLabel("Automatically check service capabilities", { exact: true }) + .isChecked(), + true, + ); + await page.getByRole("button", { name: "Save service", exact: true }).click(); + await page + .getByRole("button", { name: "Add service", exact: true }) + .waitFor(); + saved = (await call("/api/model-configs")).find( + (item) => item.modelId === served, + ); + assert.ok(saved); + const serviceCard = page + .locator(".cw-model-service") + .filter({ hasText: served }); + await serviceCard.locator("summary").first().click(); + await serviceCard.getByText("Not supported").first().waitFor(); + + // Create a real generation object and prove API rejection has no job/provider side effect. + const specId = crypto.randomUUID(), + itemId = crypto.randomUUID(); + const created = await command("canvas.createGeneration", { + spec: { + id: specId, + projectId, + kind: "image", + purpose: "generation", + prompt: "Preserve this prompt", + modelConfigId: saved.id, + parameters: {}, + }, + item: { + id: itemId, + projectId, + kind: "image", + title: "Capability example", + position: { x: 300, y: 200 }, + binding: { role: "generation", objectRef: { id: specId } }, + }, + references: [], + }); + assert.equal(created.status, "applied"); + let snapshot = await call(projectPath); + const spec = snapshot.generationSpecs.find((item) => item.id === specId); + const callsBefore = (await call("/api/creation-test-health")).calls.length; + const denied = await command( + "generation.submit", + { specId, expectedSpecRevision: spec.revision, originItemId: itemId }, + { + expectedVersions: [ + { objectType: "generation_spec", id: specId, revision: spec.revision }, + { + objectType: "canvas_item", + id: itemId, + revision: snapshot.canvasItems.find((item) => item.id === itemId) + .revision, + }, + ], + }, + ); + assert.equal(denied.status, "rejected"); + assert.equal((await call(projectPath)).jobs.length, 0); + assert.equal( + (await call("/api/creation-test-health")).calls.length, + callsBefore, + ); + + const refreshUrl = "**/api/model-configs/availability"; + await page.route( + refreshUrl, + (route) => + route.fulfill({ + status: 503, + contentType: "application/json", + body: JSON.stringify({ detail: "Fixture service unavailable" }), + }), + { times: 1 }, + ); + await serviceCard + .getByRole("button", { name: /^(Check|Refresh|Retry)$/, exact: true }) + .click(); + await serviceCard.getByRole("alert").waitFor(); + await page.unroute(refreshUrl); + mode = "supported"; + await serviceCard + .getByRole("button", { name: /^(Check|Refresh|Retry)$/, exact: true }) + .click(); + await serviceCard.getByText("Supported").first().waitFor(); + await page + .getByRole("button", { name: "Close settings", exact: true }) + .click(); + await reloadAfterRenewal(); + const node = page.locator(`.react-flow__node[data-id="${itemId}"]`); + await node.click(); + const composer = page.locator(".cw-generation-composer"); + await composer.waitFor(); + assert.match(await composer.innerText(), /Preserve this prompt/); + const generate = composer.locator("button.cw-run-button"); + await generate.waitFor(); + assert.equal(await generate.isEnabled(), true); + await generate.click(); + snapshot = await until(async () => { + const value = await call(projectPath); + return value.jobs.some((job) => job.status === "succeeded") && + value.assets.length > 0 && + value.results.length > 0 + ? value + : null; + }); + assert.equal(snapshot.jobs.length, 1); + assert.equal(snapshot.jobs[0].inputSnapshot.model.modelId, served); + assert.ok(snapshot.assets.length > 0); + assert.deepEqual( + [snapshot.assets[0].metadata.width, snapshot.assets[0].metadata.height], + [320, 240], + ); + assert.equal(snapshot.jobs[0].technicalCheck.status, "passed"); + const decoded = await page.evaluate(async (path) => { + const response = await fetch(path, { + headers: { "X-OpenDirector-Request": "1" }, + }); + if (!response.ok) throw Error(`Media download failed: ${response.status}`); + const bitmap = await createImageBitmap(await response.blob()); + const size = [bitmap.width, bitmap.height]; + bitmap.close(); + return size; + }, `${projectPath}/assets/${snapshot.assets[0].id}/content`); + assert.deepEqual(decoded, [320, 240]); + assert.equal( + (await call("/api/creation-test-health")).calls.length, + callsBefore + 1, + ); + const history = JSON.stringify(snapshot.jobs[0].inputSnapshot); + + // A paused observation expires locally even without another successful response. + mode = "paused"; + await call("/api/model-configs/availability", { + configIds: [saved.id], + manual: true, + }); + await reloadAfterRenewal(); + await page.locator(`.react-flow__node[data-id="${itemId}"]`).click(); + await until( + async () => + await page + .locator(".cw-generation-composer button.cw-run-button") + .isDisabled(), + ); + await page.locator(".cw-generation-composer button.cw-run-button").hover(); + await page.getByText("Service paused").first().waitFor(); + await until( + async () => + await page + .locator(".cw-generation-composer button.cw-run-button") + .isEnabled(), + ); + assert.equal( + JSON.stringify((await call(projectPath)).jobs[0].inputSnapshot), + history, + ); + + // A second browser context exercises the actual Chinese presentation and legacy fallback. + const chinese = await browser.newPage({ + locale: "zh-CN", + viewport: { width: 1500, height: 1000 }, + }); + await chinese.addInitScript(() => { + localStorage.setItem("opendirector.modelSetup.v1", "dismissed"); + localStorage.setItem("opendirector.locale.v1", "zh-CN"); + }); + mode = "legacy"; + const chineseProjectId = crypto.randomUUID(); + await call(`/api/creation/projects/${chineseProjectId}/commands`, { + commandId: crypto.randomUUID(), + projectId: chineseProjectId, + type: "project.create", + payload: { name: `能力检查中文界面 ${chineseProjectId.slice(0, 8)}` }, + }); + await chinese.goto( + `${base}/ai-tool/ai-canvas/${chineseProjectId}?creation=unified`, + ); + await chinese.getByRole("button", { name: "设置", exact: true }).click(); + await chinese.getByRole("button", { name: "模型接入", exact: true }).click(); + const matchesRefresh = (manual) => (response) => { + if ( + !response.url().endsWith("/api/model-configs/availability") || + response.request().method() !== "POST" + ) + return false; + const body = response.request().postDataJSON(); + return ( + body.manual === manual && + body.configIds.includes(saved.id) && + (!manual || body.configIds.length === 1) + ); + }; + // Opening a category checks its saved connections. Let that batch finish + // before testing a manual refresh, which shares the bounded worker queue. + const [categoryRefresh] = await Promise.all([ + chinese.waitForResponse(matchesRefresh(false)), + chinese + .locator(".cw-model-setup-row") + .filter({ hasText: "图片模型" }) + .getByRole("button") + .click(), + ]); + assert.equal(categoryRefresh.status(), 200); + await categoryRefresh.finished(); + const chineseService = chinese + .locator(".cw-model-service") + .filter({ hasText: served }); + await chineseService.locator("summary").first().click(); + await until( + async () => + await chineseService + .getByRole("button", { name: /^(检查|刷新|重试)$/, exact: true }) + .isEnabled(), + ); + const [legacyRefresh] = await Promise.all([ + chinese.waitForResponse(matchesRefresh(true)), + chineseService + .getByRole("button", { name: /^(检查|刷新|重试)$/, exact: true }) + .click(), + ]); + assert.equal(legacyRefresh.status(), 200); + const legacyObservation = (await legacyRefresh.json()).models.find( + (model) => model.configId === saved.id, + ); + assert.equal(legacyObservation?.reasonCode, "protocol_unavailable"); + try { + await chineseService.getByText("服务未提供能力声明").first().waitFor(); + } catch (failure) { + throw new Error( + JSON.stringify({ + observation: legacyObservation, + displayed: await chineseService.innerText(), + browserTime: await chinese.evaluate(() => Date.now()), + }), + { cause: failure }, + ); + } + await until( + async () => + await chineseService + .getByRole("button", { name: /^(检查|刷新|重试)$/, exact: true }) + .isEnabled(), + ); + await chineseService + .locator(".cw-model-availability") + .scrollIntoViewIfNeeded(); + if (process.env.CREATION_SCREENSHOT) + await chinese.screenshot({ + path: process.env.CREATION_SCREENSHOT, + fullPage: true, + }); + assert.equal( + (await call("/api/model-configs")).find((item) => item.id === saved.id) + .modelId, + served, + ); + assert.equal( + JSON.stringify((await call(projectPath)).jobs[0].inputSnapshot), + history, + ); + assert.deepEqual(errors, []); + console.log( + JSON.stringify({ + passed: true, + queries, + openConflicts, + renewalFailures, + screenshots, + checks: [ + "draft isolation", + "stale draft response", + "no query on typing", + "save unsupported configuration", + "zero-job rejection", + "real UI submission and saved media", + "pause expiration", + "immutable history", + "Chinese legacy fallback", + ], + }), + ); + await chinese.close(); +} catch (failure) { + console.error( + JSON.stringify({ + failedPage: page.url(), + pageErrors: errors, + openConflicts, + queryEvents, + visibleText: await page + .locator("body") + .innerText() + .catch(() => "unavailable"), + canvasNodes: await page + .locator(".react-flow__node") + .evaluateAll((nodes) => + nodes.map((node) => node.getAttribute("data-id")), + ) + .catch(() => []), + }), + ); + throw failure; +} finally { + if (saved) await call(`/api/model-configs/${saved.id}`, undefined, "DELETE"); + await browser.close(); + service.closeAllConnections(); + await new Promise((resolve) => service.close(resolve)); +} diff --git a/tests/browser/creation_server.py b/tests/browser/creation_server.py index 63df5066..c4c65dd8 100644 --- a/tests/browser/creation_server.py +++ b/tests/browser/creation_server.py @@ -33,19 +33,22 @@ parser.error('Browser fixtures require an opendirector-creation-* directory directly under the system temp directory') store = LocalProjects(root) configs = ModelConfigs(root) +# These configurations use the in-process adapters below and have no HTTP +# capability service. Keep their checks explicit so they do not occupy the +# bounded query workers used by a browser test's real capability fixture. if not configs.list(): for kind in ('agent', 'image'): configs.save(kind, 'http://127.0.0.1:12345/v1', f'fixture-{kind}', '', - connection_type='self_hosted', protocol='openai', auth_mode='none') + connection_type='self_hosted', protocol='openai', auth_mode='none', capability_auto_check=False) # Test-only adapters let the real preview and tool services exercise owned # assets, model selection and cancellation without contacting a GPU endpoint. if not any(model['kind'] == 'vision' for model in configs.list()): configs.save('vision', 'http://127.0.0.1:12345/v1', 'facebook/sam3', '', - connection_type='self_hosted', protocol='vllm_omni', auth_mode='none') + connection_type='self_hosted', protocol='vllm_omni', auth_mode='none', capability_auto_check=False) if not any(model['modelId'] == 'Qwen/Qwen-Image-2.1' for model in configs.list()): configs.save('image', 'http://127.0.0.1:12345/v1', 'Qwen/Qwen-Image-2.1', '', - connection_type='self_hosted', protocol='openai', auth_mode='none') + connection_type='self_hosted', protocol='openai', auth_mode='none', capability_auto_check=False) selection_gate = asyncio.Event() selection_gate.set() selection_calls, image_edit_calls = [], [] diff --git a/tests/browser/generation_feedback.mjs b/tests/browser/generation_feedback.mjs index b0bcd7d6..08f8c256 100644 --- a/tests/browser/generation_feedback.mjs +++ b/tests/browser/generation_feedback.mjs @@ -71,6 +71,9 @@ async function setup(kind = "image") { locale: "zh-CN", viewport: { width: 1500, height: 1000 }, }); + await page.addInitScript(() => + localStorage.setItem("opendirector.modelSetup.v1", "dismissed"), + ); page.setDefaultTimeout(15000); page.on("pageerror", (e) => errors.push(e.message)); const state = () => call(`/api/creation/projects/${id}`); @@ -397,6 +400,17 @@ try { await submit.click(); await until(async () => (await state()).jobs[0]?.status === "succeeded"); await overlay.waitFor({ state: "hidden" }); + // Presentation changes must not wait for another execution event. + const completedJobs = (await state()).jobs; + await page.getByRole("button", { name: "生成历史", exact: true }).click(); + await page.getByText("当前选用的生成结果", { exact: true }).waitFor(); + await page.getByRole("button", { name: "EN", exact: true }).click(); + await page.getByText("Currently selected generation result", { exact: true }).waitFor(); + await page.getByRole("button", { name: "中文", exact: true }).click(); + await page.getByText("当前选用的生成结果", { exact: true }).waitFor(); + assert.deepEqual((await state()).jobs, completedJobs); + await page.getByRole("button", { name: "生成历史", exact: true }).click(); + console.log("PASS history locale change: existing jobs translate without changing execution records"); await page.getByRole("button", { name: "扩图", exact: true }).click(); const editor = page.getByRole("region", { name: "扩图编辑器", diff --git a/tests/browser/manifest.json b/tests/browser/manifest.json index de411641..c0a3c201 100644 --- a/tests/browser/manifest.json +++ b/tests/browser/manifest.json @@ -111,6 +111,15 @@ "extended" ] }, + { + "file": "capability_discovery.mjs", + "previewSafe": true, + "lanes": [ + "gate", + "production" + ], + "note": "Real application with controllable local capability HTTP service; no model inference." + }, { "file": "composer_duration.mjs", "previewSafe": true, diff --git a/tests/browser/tool_drafts.mjs b/tests/browser/tool_drafts.mjs index f6236698..6799ed16 100644 --- a/tests/browser/tool_drafts.mjs +++ b/tests/browser/tool_drafts.mjs @@ -219,6 +219,40 @@ try { await prompt().inputValue(), "Continue editing the actual output", ); + // Continuing a result retains selection mode, but never reuses the old mask. + await page + .getByText("请先涂抹或识别需要修改的区域", { exact: true }) + .waitFor(); + assert.equal(await page.locator(".iae-generate").isDisabled(), true); + assert.equal((await tasks()).length, 1); + const continuedDrawing = page.locator("canvas.iae-ink-mask"); + await continuedDrawing.waitFor(); + const continuedRect = await continuedDrawing.boundingBox(); + await page.mouse.move( + continuedRect.x + continuedRect.width * 0.55, + continuedRect.y + continuedRect.height * 0.3, + ); + await page.mouse.down(); + await page.mouse.move( + continuedRect.x + continuedRect.width * 0.7, + continuedRect.y + continuedRect.height * 0.55, + { steps: 12 }, + ); + await page.mouse.up(); + snapshot = await until( + state, + (s) => s.editorDocuments.some((d) => d.id === next.id && d.data.selection?.mask), + "saved selection for continued output", + ); + const continued = snapshot.editorDocuments.find((d) => d.id === next.id); + const continuedMaskId = continued.data.selection.mask.assetId; + assert.equal(continued.data.selection.source.assetId, output.id); + assert.notEqual(continuedMaskId, maskId); + await until( + () => page.locator(".iae-generate").isEnabled(), + Boolean, + "continued selection ready", + ); // Lose both acceptance and immediate lookup; reload and explicitly retry. // The browser must resend the original intent, not create another paid task. const sent = []; @@ -250,6 +284,7 @@ try { rows.length === 2 && rows.every((row) => row.status === "succeeded"), "accepted original attempt", ); + assert.equal(sent[0].maskAssetId, continuedMaskId); await open(output.canvasItemId); await prompt().fill("New edits must not replace the unknown request"); await page.locator(".iae-generate").click(); diff --git a/tests/capabilityPresentation.test.ts b/tests/capabilityPresentation.test.ts new file mode 100644 index 00000000..5845392e --- /dev/null +++ b/tests/capabilityPresentation.test.ts @@ -0,0 +1,202 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + availabilityCardDetails, + availabilityNotice, + availabilityRow, +} from "../src/canvas/availabilityPresentation.ts"; +import { assistantToolText } from "../src/canvas/assistantToolText.ts"; +import { toolTaskError } from "../src/canvas/toolTaskText.ts"; +import { toolReasonText } from "../src/canvas/toolReasonText.ts"; +import { setActiveLocale } from "../src/i18n/messages.ts"; +import { availabilityReason } from "../src/backend/availability.ts"; +import type { + ModelAvailability, + OperationAvailability, +} from "../src/contracts/generated.ts"; + +const operation = ( + profile: string, + reasonCode = "supported", +): OperationAvailability => ({ + operation: "image_edit", + profile, + path: "/v1/images/edits", + method: "POST", + output: "image", + status: reasonCode === "supported" ? "available" : "unknown", + reasonCode, + checkedAt: null, + expiresAt: null, + inputDetails: [], +}); +const availability = ( + operations: OperationAvailability[], +): ModelAvailability => ({ + configId: "one", + modelId: "served", + identity: "one", + autoCheck: false, + status: "unknown", + reasonCode: "mixed_operations", + checkedAt: null, + expiresAt: null, + revision: "test", + operations, +}); + +test("normal cards have no aggregate summary; shared expiry is presented once", () => { + setActiveLocale("en-US"); + assert.equal( + availabilityNotice(availability([operation("text_to_image")])), + null, + ); + const stale = operation("image_to_image", "observation_expired"); + const notice = availabilityNotice( + availability([stale, { ...stale, profile: "images_to_image" }]), + ); + assert.equal(notice, "observation_expired"); + assert.deepEqual(availabilityRow(stale, notice), { + status: "unknown", + muted: true, + text: "—", + }); + assert.equal( + availabilityRow(operation("image_to_image", "input_count_unknown"), null) + .text, + "Image count not confirmed", + ); +}); + +test("a shared failed check covers secondary unknown projections", () => { + for (const reason of [ + "not_checked", + "connection_failed", + "authentication_failed", + "protocol_unavailable", + "query_failed", + "observation_expired", + ]) { + const multi = operation("images_to_image", "input_count_unknown"); + const value = { + ...availability([ + operation("text_to_image", reason), + operation("image_to_image", reason), + multi, + ]), + reasonCode: reason, + }; + const notice = availabilityNotice(value); + assert.equal(notice, reason); + assert.deepEqual(availabilityRow(multi, notice), { + status: "unknown", + muted: true, + text: "—", + }); + } + for (const [locale, expected] of [ + [ + "zh-CN", + [ + "检查结果已过期，请刷新", + "无法连接服务，请检查地址", + "认证失败，请检查密钥", + "不支持", + "待确认", + ], + ], + [ + "en-US", + [ + "Results expired. Refresh to check again.", + "Cannot connect. Check the service URL.", + "Authentication failed. Check the API key.", + "Not supported", + "Unknown", + ], + ], + ] as const) { + setActiveLocale(locale); + assert.deepEqual( + [ + "observation_expired", + "connection_failed", + "authentication_failed", + "profile_unsupported", + "mixed_operations", + ].map(availabilityReason), + expected, + ); + } +}); + +test("shared input requirements appear once, differing route limits keep their scope", () => { + const detail = { + name: "image", + source: "service" as const, + support: "supported" as const, + minCount: 1, + maxCount: 3, + min: null, + max: null, + multipleOf: null, + values: null, + }; + const first = { + ...operation("image_to_image"), + inputDetails: [detail, { ...detail, name: "private_unknown_field" }], + }; + const second = { ...operation("images_to_image"), inputDetails: [detail] }; + assert.equal(availabilityCardDetails([first, second]).length, 1); + assert.equal(availabilityCardDetails([first, second])[0].scope, undefined); + assert.equal( + availabilityCardDetails([ + first, + { ...second, inputDetails: [{ ...detail, maxCount: 4 }] }, + ]).length, + 2, + ); + assert.match( + availabilityCardDetails([ + first, + { ...second, inputDetails: [{ ...detail, maxCount: 4 }] }, + ])[0].scope!, + /Single-image/, + ); + setActiveLocale("zh-CN"); + assert.equal(availabilityCardDetails([first])[0].label, "图片"); + assert.equal(availabilityCardDetails([first])[0].source, "服务声明"); +}); + +test("tools and assistant retain concrete capability and selection failures in either language", () => { + const cases = [ + ["该模型不支持文字回复。", "This model does not support text replies."], + ["服务未开放图片编辑。", "The service has not enabled image editing."], + ["最多支持 2 张图片，当前为 3 张。", "Up to 2 images; 3 added."], + ["该工具需要 2 张来源图片。", "This tool requires 2 source images."], + [ + "图片宽度不符合服务要求。", + "Image width does not meet the service requirements.", + ], + [ + "选区与图片的尺寸不一致。", + "The selection and image dimensions must match.", + ], + ["请提供图片或音频。", "Provide images or audio."], + ]; + for (const [zh, en] of cases) { + setActiveLocale("en-US"); + assert.equal( + assistantToolText(zh, undefined, "capability_unavailable"), + en, + ); + assert.equal(toolReasonText(zh), en); + assert.equal(toolTaskError(zh), en); + setActiveLocale("zh-CN"); + assert.equal( + assistantToolText(zh, undefined, "capability_unavailable"), + zh, + ); + assert.equal(toolTaskError(zh), zh); + } +}); diff --git a/tests/conftest.py b/tests/conftest.py index af1661df..a41c259e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,23 @@ configure_runtime() +@pytest.fixture(autouse=True) +def optional_capability_endpoint(monkeypatch): + """Existing provider doubles predate discovery: never query real hosts. + + Capability tests supply their own transport, including the explicit HTTP + transport used by the loopback integration test. + """ + import httpx + from opendirector.integrations.capability_probe import CapabilityProbe + original = CapabilityProbe.__init__ + def initialize(self, *args, **kwargs): + if kwargs.get('transport') is None: + kwargs['transport'] = httpx.MockTransport(lambda request: httpx.Response(404)) + original(self, *args, **kwargs) + monkeypatch.setattr(CapabilityProbe, '__init__', initialize) + + @pytest.fixture def public_cdn(monkeypatch): """Mocked downloads still validate DNS, without depending on public resolvers.""" diff --git a/tests/creationJobs.test.ts b/tests/creationJobs.test.ts index 64fd1554..e09c9fbb 100644 --- a/tests/creationJobs.test.ts +++ b/tests/creationJobs.test.ts @@ -33,6 +33,14 @@ function fixture() { }; } +test("the history projection retains the execution revision used by confirmed recovery", () => { + const { snapshot, active } = fixture(); + const job = { ...active, status: "failed" as const, revision: 17 }; + const projected = creationJobs({ ...snapshot, jobs: [job] })[0]!; + assert.equal(projected.revision, 17); + assert.equal(projected.status, "failed"); +}); + test("a slow old preview cannot delay preparation or cancellation metadata", async () => { const { snapshot, active } = fixture(); const asset = snapshot.assets.find( diff --git a/tests/generationFailureText.test.ts b/tests/generationFailureText.test.ts index 362c9b85..2a79b8e5 100644 --- a/tests/generationFailureText.test.ts +++ b/tests/generationFailureText.test.ts @@ -18,6 +18,33 @@ function failure(patch: Partial): GenerationFailure { }; } +test("unknown 429 includes service load without diagnosing account quota", () => { + for (const message of [ + "模型调用频率或配额受限。", + "模型服务繁忙或请求受限。", + ]) { + const value = failure({ category: "rate_limit", httpStatus: 429, message }); + setActiveLocale("en-US"); + assert.match(generationFailureMessage(value), /busy|overload/i); + setActiveLocale("zh-CN"); + assert.match(generationFailureMessage(value), /服务繁忙/); + } +}); + +test("non-public download DNS explains the cause in both languages", () => { + const value = failure({ + category: "protocol", + stage: "download", + httpStatus: null, + message: "生成结果地址解析为非公网地址，下载已停止。", + }); + setActiveLocale("en-US"); + assert.match(generationFailureMessage(value), /non-public address/); + assert.match(generationFailureMessage(value), /DNS/); + setActiveLocale("zh-CN"); + assert.match(generationFailureMessage(value), /域名解析/); +}); + test("known persisted failures are translated without showing backend Chinese", () => { setActiveLocale("en-US"); assert.equal( @@ -68,7 +95,10 @@ test("new provider messages fall back to category, stage and status", () => { }); test("Kling content-policy rejection remains explicit in both languages", () => { - const value = failure({ providerCode: "1301", message: "供应商判定提示词或参考图片违反内容政策，已拒绝生成。" }); + const value = failure({ + providerCode: "1301", + message: "供应商判定提示词或参考图片违反内容政策，已拒绝生成。", + }); setActiveLocale("en-US"); assert.match(generationFailureMessage(value), /content policy/); setActiveLocale("zh-CN"); @@ -125,11 +155,19 @@ for (const message of [ test("voice_not_found identifies the voice before generic 404 or input failures", () => { try { setActiveLocale("en-US"); - assert.equal(generationFailureMessage(failure({ providerCode: "voice_not_found", httpStatus: 404 })), - "The voice was not found or is unavailable to this account. Check the Voice ID in the MOSS voice library."); + assert.equal( + generationFailureMessage( + failure({ providerCode: "voice_not_found", httpStatus: 404 }), + ), + "The voice was not found or is unavailable to this account. Check the Voice ID in the MOSS voice library.", + ); setActiveLocale("zh-CN"); - assert.equal(generationFailureMessage(failure({ providerCode: "voice_not_found", httpStatus: 404 })), - "该音色不存在或当前账户无权使用，请在 MOSS 音色库核对音色 ID。"); + assert.equal( + generationFailureMessage( + failure({ providerCode: "voice_not_found", httpStatus: 404 }), + ), + "该音色不存在或当前账户无权使用，请在 MOSS 音色库核对音色 ID。", + ); } finally { setActiveLocale("zh-CN"); } @@ -137,19 +175,31 @@ test("voice_not_found identifies the voice before generic 404 or input failures" test("provider moderation verdicts are explained in both languages", () => { const moderated = (message: string) => - failure({ stage: "poll", httpStatus: 200, providerStatus: "failed", providerCode: "1027", message }); + failure({ + stage: "poll", + httpStatus: 200, + providerStatus: "failed", + providerCode: "1027", + message, + }); setActiveLocale("en-US"); assert.match( - generationFailureMessage(moderated("供应商判定输入内容涉及敏感信息，已拒绝生成。")), + generationFailureMessage( + moderated("供应商判定输入内容涉及敏感信息，已拒绝生成。"), + ), /flagged the input as sensitive/, ); assert.match( - generationFailureMessage(moderated("供应商判定生成内容涉及敏感信息，已拒绝交付。")), + generationFailureMessage( + moderated("供应商判定生成内容涉及敏感信息，已拒绝交付。"), + ), /flagged the generated content as sensitive/, ); setActiveLocale("zh-CN"); assert.match( - generationFailureMessage(moderated("供应商判定生成内容涉及敏感信息，已拒绝交付。")), + generationFailureMessage( + moderated("供应商判定生成内容涉及敏感信息，已拒绝交付。"), + ), /生成内容涉及敏感信息/, ); }); diff --git a/tests/jobFailureNotifications.test.ts b/tests/jobFailureNotifications.test.ts new file mode 100644 index 00000000..c111feec --- /dev/null +++ b/tests/jobFailureNotifications.test.ts @@ -0,0 +1,130 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { JobFailureNotifications } from "../src/canvas/workspace/jobFailureNotifications.ts"; + +const failure = (id = "task", createdAt = 101) => ({ + id, + status: "failed" as const, + createdAt, +}); + +test("a synchronous rejection and its failed job projection notify once in either order", () => { + for (const first of ["submission", "observer"]) { + const watch = new JobFailureNotifications(100); + watch.observe({ ...failure(), status: "running" }); + const notices = + first === "submission" + ? [watch.claim("task"), watch.observe(failure())] + : [watch.observe(failure()), watch.claim("task")]; + assert.deepEqual(notices, [true, false]); + assert.equal(watch.observe(failure()), false); + } +}); + +test("an accepted job's background failure is reported once across repeated snapshots", () => { + const watch = new JobFailureNotifications(100); + assert.equal(watch.observe({ ...failure(), status: "running" }), false); + assert.equal(watch.observe(failure()), true); + assert.equal(watch.observe(failure()), false); + assert.equal(watch.observe({ ...failure(), status: "interrupted" }), false); +}); + +test("a render of the prior running snapshot cannot reopen a claimed failure", () => { + const watch = new JobFailureNotifications(100); + watch.observe({ ...failure(), status: "running" }); + assert.equal(watch.claim("task"), true); + assert.equal(watch.observe({ ...failure(), status: "running" }), false); + assert.equal(watch.observe(failure()), false); + // Only running after the failed projection establishes another attempt. + assert.equal(watch.observe({ ...failure(), status: "running" }), false); + assert.equal(watch.observe(failure()), true); +}); + +test("distinct tasks with the same failure each own a notification", () => { + const watch = new JobFailureNotifications(100); + assert.equal(watch.observe(failure("first")), true); + assert.equal(watch.observe(failure("second")), true); + assert.equal(watch.claim("first"), false); + assert.equal(watch.claim("second"), false); +}); + +test("a resumed job can fail again under the same task ID", () => { + const watch = new JobFailureNotifications(100); + assert.equal(watch.observe(failure()), true); + assert.equal(watch.observe({ ...failure(), status: "running" }), false); + assert.equal(watch.observe({ ...failure(), status: "running" }), false); + assert.equal(watch.observe(failure()), true); + assert.equal(watch.claim("task"), false); +}); + +test("reopening does not repeat a recent historical failure", () => { + const previous = new JobFailureNotifications(100); + assert.equal(previous.observe(failure()), true); + const reopened = new JobFailureNotifications(102); + assert.equal(reopened.observe(failure()), false); + assert.equal(reopened.observe(failure()), false); + assert.equal(reopened.observe(failure("new-task", 103)), true); +}); + +test("an older job observed running still reports its later failure", () => { + const watch = new JobFailureNotifications(100); + assert.equal( + watch.observe({ ...failure("old-task", 90), status: "running" }), + false, + ); + assert.equal(watch.observe(failure("old-task", 90)), true); +}); + +test("an unconfirmed submission does not consume a later execution failure", () => { + const watch = new JobFailureNotifications(100); + // An uncertain delivery remains running; only a definitive rejection claims failure. + assert.equal(watch.observe({ ...failure(), status: "running" }), false); + assert.equal(watch.observe({ ...failure(), status: "running" }), false); + assert.equal(watch.observe(failure()), true); +}); + +test("confirmed recovery reports a second failure without an intermediate running snapshot", () => { + const watch = new JobFailureNotifications(100); + const original = { ...failure(), revision: 4 }; + assert.equal(watch.observe(original), true); + watch.recovered("task", 4); + assert.equal(watch.observe(original), false); + assert.equal(watch.observe({ ...original, revision: 6 }), true); + assert.equal(watch.observe({ ...original, revision: 6 }), false); + assert.equal( + watch.observe({ ...original, status: "running", revision: 5 }), + false, + ); + assert.equal(watch.observe({ ...original, revision: 6 }), false); +}); + +test("a recovery receipt cannot repeat a new failure already reported by the observer", () => { + const watch = new JobFailureNotifications(100); + watch.observe({ ...failure(), revision: 4 }); + watch.observe({ ...failure(), status: "running", revision: 5 }); + assert.equal(watch.observe({ ...failure(), revision: 6 }), true); + watch.recovered("task", 4); + assert.equal(watch.observe({ ...failure(), revision: 6 }), false); +}); + +test("a fast failure observed before its recovery receipt is still reported after confirmation", () => { + const watch = new JobFailureNotifications(100); + assert.equal(watch.observe({ ...failure(), revision: 4 }), true); + assert.equal(watch.observe({ ...failure(), revision: 6 }), false); + watch.recovered("task", 4); + assert.equal(watch.observe({ ...failure(), revision: 6 }), true); + assert.equal(watch.observe({ ...failure(), revision: 6 }), false); +}); + +test("revision changes alone do not announce another failure without successful recovery", () => { + const watch = new JobFailureNotifications(100); + assert.equal(watch.observe({ ...failure(), revision: 4 }), true); + assert.equal(watch.observe({ ...failure(), revision: 5 }), false); +}); + +test("a confirmed recovery of historical failure can report new failure after reopening", () => { + const watch = new JobFailureNotifications(200); + assert.equal(watch.observe({ ...failure(), revision: 4 }), false); + watch.recovered("task", 4); + assert.equal(watch.observe({ ...failure(), revision: 6 }), true); +}); diff --git a/tests/modelAvailability.test.ts b/tests/modelAvailability.test.ts new file mode 100644 index 00000000..cb029741 --- /dev/null +++ b/tests/modelAvailability.test.ts @@ -0,0 +1,419 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + availabilityBlocker, + availabilityInputLimit, + availabilityDelay, + availabilityMatchesModel, + availabilityModelKey, + currentAvailability, + failedAvailability, + nextAvailabilityExpiry, + receiveAvailability, + type AvailabilityClock, +} from "../src/backend/availability.ts"; +import type { + ModelAvailability, + OperationAvailability, +} from "../src/contracts/generated.ts"; +import type { ModelConfig } from "../src/backend/rules.ts"; +import { setActiveLocale } from "../src/i18n/messages.ts"; + +const clockAt = (now: number): AvailabilityClock => ({ + wallNow: () => now, + monotonicNow: () => now, +}); + +test("a transport failure permits a matching successful retry without reloading the directory", () => { + const value = observation([operation("text_response")]); + const failed = failedAvailability(value); + const configured = model(failed); + assert.equal( + availabilityBlocker(configured, "text_response", clockAt(2000)), + null, + ); + assert.equal(failed.status, "unknown"); + assert.equal(availabilityMatchesModel(configured, value), true); + assert.equal(value.status, "unavailable"); +}); + +const operation = ( + profile: string, + changes: Partial = {}, +): OperationAvailability => ({ + operation: "chat", + profile, + method: "POST", + path: "/v1/chat/completions", + output: "text", + status: "unavailable", + reasonCode: "execution_paused", + checkedAt: 1000, + expiresAt: 6000, + inputDetails: [], + ...changes, +}); +const observation = ( + operations: OperationAvailability[], +): ModelAvailability => ({ + configId: "saved", + modelId: "served", + identity: "private-identity-digest", + autoCheck: true, + status: "unavailable", + reasonCode: "execution_paused", + checkedAt: 1000, + expiresAt: 6000, + revision: "one", + operations, +}); +const model = (availability?: ModelAvailability): ModelConfig => ({ + id: "saved", + providerId: "service", + kind: "agent", + url: "http://localhost:8100/v1", + modelId: "served", + connectionType: "self_hosted", + protocol: "openai", + authMode: "none", + availability, + inputRules: { + profile: "chat", + allowedKinds: ["text"], + maxImages: 0, + maxVideos: 0, + maxAudios: 0, + videoModes: [], + }, +}); + +test("a paused observation expires independently of a 30-second declaration", () => { + const value = observation([ + operation("text_response"), + operation("image_response", { + reasonCode: "profile_unsupported", + expiresAt: 31000, + }), + ]); + assert.equal( + currentAvailability(value, clockAt(5999)).operations[0].status, + "unavailable", + ); + const expired = currentAvailability(value, clockAt(6000)); + assert.equal(expired.operations[0].status, "unknown"); + assert.equal(expired.operations[1].status, "unavailable"); + assert.equal( + value.operations[0].status, + "unavailable", + "the input observation is immutable", + ); + assert.equal( + currentAvailability(value, clockAt(31000)).operations[1].status, + "unknown", + ); +}); + +test("unknown observations and missing profiles keep manual configuration usable", () => { + assert.equal( + availabilityBlocker(model(), "text_response", clockAt(2000)), + null, + ); + assert.equal( + availabilityBlocker( + model(observation([operation("image_response")])), + "text_response", + clockAt(2000), + ), + null, + ); + assert.equal( + availabilityBlocker( + model(observation([operation("text_response", { status: "unknown" })])), + "text_response", + clockAt(2000), + ), + null, + ); +}); + +test("explicit text denial blocks text without blocking a music profile", () => { + setActiveLocale("en-US"); + const configured = model( + observation([ + operation("text_response", { reasonCode: "profile_unsupported" }), + operation("lyrics_to_music", { + status: "available", + reasonCode: "supported", + }), + ]), + ); + assert.match( + availabilityBlocker(configured, "text_response", clockAt(2000))!, + /Not supported/, + ); + assert.equal( + availabilityBlocker(configured, "lyrics_to_music", clockAt(2000)), + null, + ); + assert.equal( + availabilityBlocker(configured, "text_response", clockAt(6000)), + null, + ); +}); + +test("a failed refreshed observation cannot reuse the earlier paused denial", () => { + const configured = model(observation([operation("text_response")])); + assert.ok(availabilityBlocker(configured, "text_response", clockAt(2000))); + assert.equal( + availabilityBlocker( + { ...configured, availability: undefined }, + "text_response", + clockAt(2000), + ), + null, + ); +}); + +test("expiry scheduling includes each operation and stops after all observations expire", () => { + const configured = model( + observation([ + operation("text_response"), + operation("image_response", { expiresAt: 31000 }), + ]), + ); + assert.equal(nextAvailabilityExpiry([configured], clockAt(2000)), 6000); + assert.equal(nextAvailabilityExpiry([configured], clockAt(6000)), 31000); + assert.equal(nextAvailabilityExpiry([configured], clockAt(31000)), null); +}); + +test("connection changes invalidate observations while observation updates preserve identity", () => { + const configured = model(); + const identity = availabilityModelKey(configured); + for (const patch of [ + { providerId: "other" }, + { modelId: "other" }, + { url: "http://localhost:8200/v1" }, + { proxyMode: "direct" as const }, + { capabilityAutoCheck: false }, + { inputModalities: ["text", "image"] as ("text" | "image")[] }, + { + videoCapabilities: { + outputs: [{ resolution: "720p", durations: [5] }], + defaultResolution: "720p", + defaultDuration: 5, + dimensionStep: 16 as const, + }, + }, + ]) { + assert.notEqual( + availabilityModelKey({ ...configured, ...patch }), + identity, + ); + } + assert.equal( + availabilityModelKey({ ...configured, availability: observation([]) }), + identity, + ); +}); + +test("rejection reasons follow the current language without altering business identity", () => { + const configured = model(observation([operation("text_response")])); + setActiveLocale("zh-CN"); + assert.match( + availabilityBlocker(configured, "text_response", clockAt(2000))!, + /暂停/, + ); + setActiveLocale("en-US"); + assert.match( + availabilityBlocker(configured, "text_response", clockAt(2000))!, + /paused/, + ); + assert.equal(configured.modelId, "served"); +}); + +test("wall-clock rollback and repeated responses do not renew a received pause", () => { + let wall = 1000, + elapsed = 0; + const timing: AvailabilityClock = { + wallNow: () => wall, + monotonicNow: () => elapsed, + }; + const original = receiveAvailability( + observation([operation("text_response")]), + timing, + ); + assert.equal(availabilityDelay(original, timing), 5000); + wall = -60_000; + elapsed = 4999; + const duplicate = receiveAvailability(structuredClone(original), timing); + assert.equal(availabilityDelay(duplicate, timing), 1); + assert.equal(currentAvailability(duplicate, timing).status, "unavailable"); + elapsed = 5000; + assert.equal(currentAvailability(duplicate, timing).status, "unknown"); + assert.equal(availabilityDelay(duplicate, timing), null); + assert.equal( + availabilityBlocker(model(duplicate), "text_response", timing), + null, + ); + assert.equal(original.operations[0].status, "unavailable"); +}); + +test("wall-clock advances do not shorten the already received lifetime", () => { + let wall = 1000, + elapsed = 0; + const timing: AvailabilityClock = { + wallNow: () => wall, + monotonicNow: () => elapsed, + }; + const value = receiveAvailability( + observation([operation("text_response")]), + timing, + ); + wall = 100_000; + elapsed = 1000; + assert.equal(currentAvailability(value, timing).status, "unavailable"); + assert.equal(availabilityDelay(value, timing), 4000); + assert.equal(nextAvailabilityExpiry([model(value)], timing), 104_000); +}); + +test("a matching model ID cannot authorize observations for a changed opaque identity", () => { + const expected = observation([operation("text_response")]); + const configured = model(expected); + assert.equal(availabilityMatchesModel(configured, expected), true); + assert.equal( + availabilityMatchesModel(configured, { + ...expected, + identity: "changed-account-or-url", + }), + false, + ); + assert.equal( + availabilityMatchesModel(configured, { + ...expected, + configId: "other-service", + }), + false, + ); + assert.equal(availabilityMatchesModel(model(), expected), false); + assert.equal( + availabilityMatchesModel(model(), expected, expected.identity), + true, + ); +}); + +test("pause expires before its retained declaration, without renewing either observation", () => { + let now = 1000; + const timing = { wallNow: () => now, monotonicNow: () => now }; + const value = observation([ + operation("image_to_image", { + declaration: { + status: "available", + reasonCode: "supported", + checkedAt: 1000, + expiresAt: 31000, + }, + execution: { + state: "paused", + scope: "deployment", + checkedAt: 1000, + expiresAt: 6000, + }, + inputDetails: [ + { + name: "image", + source: "service", + support: "supported", + minCount: 1, + maxCount: 2, + min: null, + max: null, + multipleOf: null, + values: null, + }, + ], + }), + ]); + receiveAvailability(value, timing); + assert.equal( + availabilityBlocker(model(value), "image_to_image", timing), + "Service paused", + ); + assert.equal( + availabilityInputLimit(model(value), "image_to_image", "image", timing), + 2, + ); + now = 6000; + const replay = receiveAvailability(structuredClone(value), timing); + assert.equal( + currentAvailability(replay, timing).operations[0].status, + "available", + ); + assert.equal( + currentAvailability(replay, timing).operations[0].execution?.state, + "unknown", + ); + assert.equal( + availabilityBlocker(model(replay), "image_to_image", timing), + null, + ); + assert.equal(availabilityDelay(replay, timing), 25000); + now = 31000; + assert.equal( + currentAvailability(replay, timing).operations[0].reasonCode, + "observation_expired", + ); + assert.equal( + availabilityInputLimit(model(replay), "image_to_image", "image", timing), + null, + ); + assert.equal(value.operations[0].status, "unavailable"); +}); + +test("service limits disappear on failure and do not include application constraints", () => { + const detail = { + name: "image", + source: "service" as const, + support: "supported" as const, + minCount: 1, + maxCount: 3, + min: null, + max: null, + multipleOf: null, + values: null, + }; + const value = observation([ + operation("image_to_image", { + status: "available", + reasonCode: "supported", + declaration: { + status: "available", + reasonCode: "supported", + checkedAt: 1000, + expiresAt: 31000, + }, + inputDetails: [detail, { ...detail, source: "application", maxCount: 1 }], + }), + ]); + const timing = clockAt(2000); + assert.equal( + availabilityInputLimit(model(value), "image_to_image", "image", timing), + 3, + ); + assert.equal( + availabilityInputLimit(model(value), "image_to_image", "video", timing), + null, + ); + assert.equal( + availabilityInputLimit( + model(failedAvailability(value)), + "image_to_image", + "image", + timing, + ), + null, + ); + assert.equal( + failedAvailability(value).operations[0].declaration?.status, + "unknown", + ); +}); diff --git a/tests/test_agent_gateway.py b/tests/test_agent_gateway.py index 81587196..3dc8e4b1 100644 --- a/tests/test_agent_gateway.py +++ b/tests/test_agent_gateway.py @@ -7,7 +7,7 @@ from capabilities import requires_posix from opendirector.agent.context import RunContext -from opendirector.agent.gateway import ToolGateway +from opendirector.agent.gateway import ToolGateway, ToolResult from opendirector.agent.tools import AgentTools from opendirector.domain.projects import ProjectError from opendirector.infrastructure.creation_projects import CreationProjects @@ -20,6 +20,24 @@ def anyio_backend(): return 'asyncio' +def test_tool_events_preserve_concrete_capability_error_codes(): + text = '最多支持 4 张图片，当前为 5 张。' + result = ToolResult(value={'error': text, 'code': 'image_input_count'}, error=text) + assert result.event('image.edit')['code'] == 'image_input_count' + receipt = ToolResult(value={'status': 'rejected', 'issues': [ + {'code': 'capability_unavailable', 'message': '该模型不支持文字回复。'}]}, error='该模型不支持文字回复。') + assert receipt.event('generation.submit')['code'] == 'capability_unavailable' + assert receipt.event('generation.submit')['error'] == '该模型不支持文字回复。' + + +def test_tool_event_preserves_nested_import_receipt_issue(): + result = ToolResult(value={'status': 'rejected', 'receipt': {'issues': [ + {'code': 'timeline_font_coverage', 'message': 'Missing glyphs'}]}}, error='Missing glyphs') + assert result.event('timeline.import_package')['code'] == 'timeline_font_coverage' + result.value['receipt']['issues'].append({'code': 'timeline_asset_missing', 'message': 'Missing asset'}) + assert 'code' not in result.event('timeline.import_package') + + @pytest.fixture def gateway(tmp_path): projects = CreationProjects(LocalProjects(tmp_path)) diff --git a/tests/test_assistant_conversations.py b/tests/test_assistant_conversations.py index 2de8681b..3bd72d71 100644 --- a/tests/test_assistant_conversations.py +++ b/tests/test_assistant_conversations.py @@ -35,7 +35,9 @@ async def stream(self, run): monkeypatch.setattr(agent, 'runtime_available', lambda _: True) monkeypatch.setattr(agent, 'skills', empty_skills) from opendirector.infrastructure.model_configs import ModelConfigs - monkeypatch.setattr(ModelConfigs, 'resolve', lambda *args: {'id': str(uuid4()), 'modelId': 'test-model'}) + config = {'id': str(uuid4()), 'modelId': 'test-model', 'kind': 'agent', 'url': 'http://models.test/v1', + 'connectionType': 'api', 'protocol': 'openai', 'authMode': 'none', 'apiKey': ''} + monkeypatch.setattr(ModelConfigs, 'resolve', lambda *args: dict(config)) with TestClient(create_app(LocalProjects(tmp_path)), headers={ 'X-OpenDirector-Request': '1', 'X-OpenDirector-Session': str(uuid4())}) as client: pid = str(uuid4()) diff --git a/tests/test_capability_acceptance.py b/tests/test_capability_acceptance.py new file mode 100644 index 00000000..753f6438 --- /dev/null +++ b/tests/test_capability_acceptance.py @@ -0,0 +1,588 @@ +"""Shared acceptance, HTTP, CLI and Agent tests with isolated model doubles.""" +import asyncio +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import socket +import threading +import time +from unittest.mock import Mock +from uuid import uuid4 + +from fastapi.testclient import TestClient +import httpx +import pytest +from typer.testing import CliRunner +import uvicorn + +from opendirector import cli +from opendirector.agent.tools import AgentTools +from opendirector.application.creation_tool_assets import CreationToolAssets +from opendirector.application.creation_generation import CreationGeneration +from opendirector.application.model_availability import ModelAvailability, requests_for +from opendirector.application.tool_tasks import ToolTasks +from opendirector.domain.creation.commands import parse_command +from opendirector.domain.projects import ProjectError +from opendirector.infrastructure.local_projects import LocalProjects +from opendirector.infrastructure.creation_projects import CreationProjects +from opendirector.infrastructure.model_configs import ModelConfigs +from opendirector.integrations.capability_probe import CapabilityProbe +from opendirector.integrations.providers import GenerationProviders +from opendirector_api.main import create_app +from opendirector_api import agent +from test_creation_canvas_service import project, ver +from test_creation_generation_service import Provider, runtime, configure +from test_model_availability import document, NOW +from test_tool_tasks import png, upload, images_server, video_server, H3, configure as configure_tools +from test_generation_recovery import video +from generation_helpers import command as generation_command, settled + + +def command_for(project, config): + _, project_id, _ = project + target, spec = configure(project, config) + return parse_command(dict(commandId=uuid4(), projectId=project_id, type='generation.submit', + expectedVersions=[ver('canvas_item', target), ver('generation_spec', spec)], + payload=dict(specId=spec.id, expectedSpecRevision=spec.revision, originItemId=target.id))) + + +def test_generation_rejection_creates_no_job_or_provider_call(project): + store, pid, session = project + provider = Provider() + runner, config = runtime(project, provider) + calls = [] + def respond(request): + calls.append(request) + return httpx.Response(200, json=document(model='test-agent', support='unsupported')) + probe = CapabilityProbe(transport=httpx.MockTransport(respond)) + runner.availability = ModelAvailability(runner.configs, probe) + command = command_for(project, config) + async def run(): + try: + receipt = await runner.execute(command, session=session) + assert receipt.status == 'rejected' + assert receipt.issues[0].code == 'capability_unavailable' + assert not store.snapshot(pid).jobs and provider.calls == [] + assert len(calls) == 1 and calls[0].method == 'GET' + finally: + await runner.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def test_missing_required_input_is_rejected_before_job_acceptance(project): + store, pid, session = project + provider = Provider() + runner, config = runtime(project, provider) + declaration = document(model='test-agent') + declaration['models'][0]['operations']['chat']['profiles'][0]['required_inputs'] = ['text', 'audio'] + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=declaration))) + runner.availability = ModelAvailability(runner.configs, probe) + try: + receipt = runner.prepare(command_for(project, config), session=session) + assert receipt.status == 'rejected' and receipt.issues[0].code == 'capability_unavailable' + assert not store.snapshot(pid).jobs and provider.calls == [] + finally: + probe.close() + + +def test_generation_identity_change_during_query_is_rejected_under_commit(project): + store, pid, session = project + runner, config = runtime(project, Provider()) + def respond(request): + # This save must complete while the HTTP query is in progress. + runner.configs.save('agent', 'http://changed.test/v1', 'test-agent', '', config['id'], connection_type='self_hosted') + return httpx.Response(200, json=document(model='test-agent')) + probe = CapabilityProbe(transport=httpx.MockTransport(respond)) + runner.availability = ModelAvailability(runner.configs, probe) + try: + result = runner.prepare(command_for(project, config), session=session) + assert result.status == 'rejected' + assert result.issues[0].code == 'capability_connection_changed' + assert not store.snapshot(pid).jobs + finally: + probe.close() + + +def test_generation_replay_precedes_changed_declaration_and_preserves_snapshot(project): + store, pid, session = project + runner, config = runtime(project, Provider()) + state = {'support': 'supported', 'calls': 0} + def respond(request): + state['calls'] += 1 + return httpx.Response(200, json=document(model='test-agent', support=state['support'])) + probe = CapabilityProbe(transport=httpx.MockTransport(respond)) + runner.availability = ModelAvailability(runner.configs, probe) + try: + command = command_for(project, config) + receipt = runner.prepare(command, session=session) + assert receipt.status == 'accepted' + frozen = store.snapshot(pid).jobs[0].inputSnapshot.model_dump() + state['support'] = 'unsupported' + runner.availability.refresh([config['id']], manual=True) + calls = state['calls'] + replay = runner.prepare(command, session=session) + assert replay.jobIds == receipt.jobIds and state['calls'] == calls + assert store.snapshot(pid).jobs[0].inputSnapshot.model_dump() == frozen + assert 'availability' not in str(frozen) and 'capabilityAutoCheck' not in str(frozen) + finally: + probe.close() + + +def test_tools_reject_new_remote_work_and_keep_local_work_and_replays(project): + store, project_id, session = project + pid = str(project_id) + configs = ModelConfigs(store.local.repository) + configs.save('image', 'http://image.test/v1', 'Tongyi-MAI/Z-Image-Turbo', '', connection_type='self_hosted') + source = upload(store, pid, session, 'source.png', png()) + calls = [] + probe = CapabilityProbe(transport=httpx.MockTransport(lambda request: httpx.Response(200, + json=document('image_edit', 'image_to_image', '/v1/images/edits', 'image', model='Tongyi-MAI/Z-Image-Turbo', support='unsupported')))) + availability = ModelAvailability(configs, probe) + tools = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, + transport=httpx.MockTransport(images_server(calls)), availability=availability) + async def run(): + try: + with pytest.raises(ProjectError) as error: + await tools.submit(pid, {'id': str(uuid4()), 'tool': 'outpaint', 'sourceAssetIds': [source], 'parameters': {}}, session) + assert error.value.code == 'capability_unavailable' + assert not tools.list(pid) and calls == [] + value = {'id': str(uuid4()), 'tool': 'image-hd', 'sourceAssetIds': [source], + 'parameters': {'kind': 'normal', 'resolution': '2K'}} + task = await tools.submit(pid, value, session) + await asyncio.gather(*tools.jobs.values()) + assert tools.get(pid, task['id'])['status'] == 'succeeded' + availability.prepare = Mock(side_effect=AssertionError('replay queried capability')) + replay = await tools.submit(pid, value, session) + assert replay['id'] == task['id'] and calls == [] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def test_tool_edit_uses_its_real_route_and_skips_unsupported_optional_chat(project): + store, project_id, session = project + pid = str(project_id) + configs = ModelConfigs(store.local.repository) + image = configs.save('image', 'http://image.test/v1', 'Tongyi-MAI/Z-Image-Turbo', '', connection_type='self_hosted') + configs.save('agent', 'http://chat.test/v1', 'demo', '', connection_type='self_hosted') + source = upload(store, pid, session, 'source.png', png()) + calls = [] + def respond(request): + value = (document('image_edit', 'image_to_image', '/v1/images/edits', 'image', model=image['modelId']) + if request.url.host == 'image.test' else document(support='unsupported')) + return httpx.Response(200, json=value) + probe = CapabilityProbe(transport=httpx.MockTransport(respond)) + tools = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, + transport=httpx.MockTransport(images_server(calls)), availability=ModelAvailability(configs, probe)) + async def run(): + try: + task = await tools.submit(pid, {'id': str(uuid4()), 'tool': 'outpaint', 'sourceAssetIds': [source], + 'parameters': {'left': 0, 'right': 1, 'top': 0, 'bottom': 0}}, session) + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, task['id']) + assert result['status'] == 'succeeded', result['error'] + assert len(calls) == 1 and 'text.chat' not in result['modelConfigIds'] + assert 'capabilityAutoCheck' not in str(result['modelSnapshots']) + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def deny_new_work(availability, config, declaration, mode): + """Publish real negative evidence before forbidding re-admission calls.""" + operation = next(iter(declaration['models'][0]['operations'].values())) + if mode == 'unsupported': + operation['support'] = 'unsupported' + for profile in operation['profiles']: + profile['support'] = 'unsupported' + else: + operation['execution']['state'] = 'paused' + availability.refresh([config['id']], manual=True) + private = availability.configs.resolve(config['id'], config['kind']) + request = next(item for item in requests_for(private) if item.profile == operation['profiles'][0]['id']) + assert availability.decision(private, request)['status'] == 'unavailable' + for method in ('prepare', 'prepare_many', 'require'): + setattr(availability, method, Mock(side_effect=AssertionError('accepted work must not be readmitted'))) + + +@pytest.mark.parametrize('denial', ['unsupported', 'paused']) +@pytest.mark.parametrize('action', ['remote', 'download', 'cancel']) +def test_accepted_generation_keeps_original_remote_work_after_capability_change(project, video, action, denial): + store, pid, session = project + kind = 'image' if action == 'download' else 'video' + configs = ModelConfigs(store.local.repository) + config = configs.save(kind, 'http://127.0.0.1:12345/v1', 'demo' if kind == 'image' else H3[2], '', + connection_type='self_hosted', protocol='openai' if kind == 'image' else 'vllm_omni') + declaration = document('image_generation' if kind == 'image' else 'video_generation', + 'text_to_image' if kind == 'image' else 'text_to_video', + '/v1/images/generations' if kind == 'image' else '/v1/videos', kind, model=config['modelId']) + queries, calls, state = [], [], {'healthy': False} + probe = CapabilityProbe(transport=httpx.MockTransport(lambda request: ( + queries.append(request) or httpx.Response(200, json=declaration)))) + availability = ModelAvailability(configs, probe) + def handle(request): + calls.append((request.method, request.url.path)) + if request.method == 'POST': + return httpx.Response(200, json={'data': [{'url': config['url'] + '/result.png'}]} if kind == 'image' + else {'id': 'accepted-remote-id', 'status': 'queued'}) + if request.method == 'DELETE': + assert request.url.path == '/v1/videos/accepted-remote-id' + return httpx.Response(200, json={'id': 'accepted-remote-id', 'status': 'cancelled'}) + if not state['healthy']: + raise httpx.ReadTimeout('interrupted read', request=request) + if request.url.path.endswith('/content') or kind == 'image': + return httpx.Response(200, content=png() if kind == 'image' else video) + assert request.url.path == '/v1/videos/accepted-remote-id' + return httpx.Response(200, json={'id': 'accepted-remote-id', 'status': 'completed'}) + async def run(): + first = CreationGeneration(store, configs, availability=availability) + first.provider = GenerationProviders(first.store, httpx.MockTransport(handle), poll_interval=0) + second = None + try: + command = generation_command(project, config, prompt='Keep this accepted request') + accepted = await first.execute(command, session=session) + assert accepted.status == 'accepted' + await settled(first) + initial = store.snapshot(pid).jobs[0] + assert initial.recovery == ('download' if action == 'download' else 'remote') + frozen = initial.inputSnapshot.model_dump() + await first.close() + deny_new_work(availability, config, declaration, denial) + state['healthy'] = True + reopened = CreationProjects(LocalProjects(store.local.repository)) + second = CreationGeneration(reopened, configs, availability=availability) + second.provider = GenerationProviders(second.store, httpx.MockTransport(handle), poll_interval=0) + before = len(calls) + if action == 'cancel': + cancel = parse_command(dict(commandId=uuid4(), projectId=pid, type='job.cancel', + expectedVersions=[ver('job', initial)], payload=dict(jobId=initial.id))) + receipt = await second.execute(cancel, session=session) + assert receipt.status == 'applied' + assert await second.execute(cancel, session=session) == receipt + assert calls[before:] == [('DELETE', '/v1/videos/accepted-remote-id')] + else: + await second.start() + await settled(second) + assert calls[before:] == ([('GET', '/v1/result.png')] if kind == 'image' else + [('GET', '/v1/videos/accepted-remote-id'), ('GET', '/v1/videos/accepted-remote-id/content')]) + final = reopened.snapshot(pid) + assert len(final.jobs) == 1 and final.jobs[0].id == accepted.jobIds[0] + assert final.jobs[0].status == ('cancelled' if action == 'cancel' else 'succeeded') + assert final.jobs[0].externalTaskId == initial.externalTaskId + assert final.jobs[0].inputSnapshot.model_dump() == frozen + assert len(final.results) == (0 if action == 'cancel' else 1) + assert sum(method == 'POST' for method, _ in calls) == 1 and len(queries) == 2 + count = len(calls) + assert second.prepare(command, session=session) == accepted and len(calls) == count + for method in ('prepare', 'prepare_many', 'require'): + getattr(availability, method).assert_not_called() + finally: + await first.close() + if second: + await second.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +@pytest.mark.parametrize('denial', ['unsupported', 'paused']) +def test_remote_tool_frozen_execution_recovery_and_replay_ignore_new_capabilities(project, denial): + store, project_id, session = project + pid = str(project_id) + configs = configure_tools(store, H3) + config = configs.list()[0] + source = upload(store, pid, session, 'poster.png', png((64, 36))) + declaration = document('video_generation', 'image_to_video', '/v1/videos', 'video', model=config['modelId']) + queries, calls = [], [] + probe = CapabilityProbe(transport=httpx.MockTransport(lambda request: ( + queries.append(request) or httpx.Response(200, json=declaration)))) + availability = ModelAvailability(configs, probe) + transport = httpx.MockTransport(video_server(calls, hold=True)) + async def run(): + first = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, + transport=transport, poll_interval=.01, concurrency=1, availability=availability) + second = None + await first.limit.acquire() + try: + value = {'id': str(uuid4()), 'tool': 'dynamic-poster', 'sourceAssetIds': [source], 'parameters': {}} + accepted = await first.submit(pid, value, session) + frozen = json.dumps(accepted['modelSnapshots'], sort_keys=True) + deny_new_work(availability, config, declaration, denial) + first.limit.release() + async with asyncio.timeout(5): + while ('GET', '/v1/videos/video_1') not in calls: + await asyncio.sleep(.01) + await first.close() + second = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, + transport=transport, poll_interval=0, availability=availability) + before = len(calls) + await second.recover() + assert calls[before:] == [('GET', '/v1/videos/video_1')] + recovered = second.get(pid, accepted['id']) + assert recovered['status'] == 'interrupted' + assert recovered['modelCalls'][0]['externalTaskId'] == 'video_1' + assert json.dumps(recovered['modelSnapshots'], sort_keys=True) == frozen + before = len(calls) + replay = await second.submit(pid, value, session) + assert replay['id'] == accepted['id'] and replay['modelCalls'] == recovered['modelCalls'] + assert len(calls) == before and calls.count(('POST', '/v1/videos')) == 1 and len(queries) == 2 + for method in ('prepare', 'prepare_many', 'require'): + getattr(availability, method).assert_not_called() + finally: + await first.close() + if second: + await second.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def test_tool_admission_shares_one_budget_for_image_and_optional_chat(project): + store, project_id, session = project + pid = str(project_id) + configs = ModelConfigs(store.local.repository) + configs.save('image', 'http://image.test/v1', 'Tongyi-MAI/Z-Image-Turbo', '', connection_type='self_hosted') + configs.save('agent', 'http://chat.test/v1', 'demo', '', connection_type='self_hosted') + source = upload(store, pid, session, 'source.png', png()) + finish, queried = threading.Event(), [] + def respond(request): + queried.append(request.url.host) + finish.wait(2) + return httpx.Response(404) + probe = CapabilityProbe(transport=httpx.MockTransport(respond), timeout=.4) + async def run(): + tools = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, + concurrency=1, availability=ModelAvailability(configs, probe)) + await tools.limit.acquire() + try: + started = time.monotonic() + task = await tools.submit(pid, {'id': str(uuid4()), 'tool': 'outpaint', 'sourceAssetIds': [source], 'parameters': {}}, session) + assert time.monotonic() - started < .65 + assert sorted(queried) == ['chat.test', 'image.test'] + assert set(task['modelSnapshots']) == {'image.edit', 'text.chat'} + finally: + finish.set() + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def test_saved_http_api_and_draft_checks_share_parser_but_not_observations(tmp_path): + calls = [] + probe = CapabilityProbe(transport=httpx.MockTransport(lambda request: (calls.append(request) or httpx.Response(200, json=document())))) + headers = {'X-OpenDirector-Request': '1', 'X-OpenDirector-Session': str(uuid4())} + with TestClient(create_app(LocalProjects(tmp_path), capability_probe=probe), headers=headers) as client: + payload = {'kind': 'agent', 'url': 'http://models.test/v1', 'modelId': 'demo', 'apiKey': '', 'connectionType': 'self_hosted'} + saved = client.post('/api/model-configs', json=payload).json() + assert client.get('/api/model-configs/availability').json()['models'][0]['status'] == 'unknown' + assert calls == [] + draft = client.post('/api/model-configs/availability/check', json=payload) + assert draft.status_code == 200, draft.text + assert draft.json()['status'] == 'available' and draft.json()['configId'] is None + assert client.get('/api/model-configs/availability').json()['models'][0]['status'] == 'unknown' + observed = client.post('/api/model-configs/availability', json={'configIds': [saved['id']], 'manual': True}) + assert observed.status_code == 200, observed.text + assert observed.json()['models'][0]['status'] == 'available' + assert client.get('/api/model-configs').json()[0]['availability']['status'] == 'available' + assert len(calls) == 2 + + +@pytest.mark.parametrize('scenario', ['unsupported', 'missing_input', 'changed_before_acceptance', 'native']) +def test_assistant_admission_and_native_runtime_boundary(tmp_path, monkeypatch, scenario): + configs = ModelConfigs(tmp_path) + saved = configs.save('agent', 'http://assistant.test/v1', 'demo', '', connection_type='self_hosted') + calls, turns = [], [] + def respond(request): + calls.append(request) + value = document(support='unsupported' if scenario == 'unsupported' else 'supported') + if scenario == 'missing_input': + value['models'][0]['operations']['chat']['profiles'][0]['required_inputs'] = ['text', 'image'] + return httpx.Response(200, json=value) + probe = CapabilityProbe(transport=httpx.MockTransport(respond)) + async def stream(self, context): + turns.append(context) + yield {'type': 'delta', 'text': 'Fixture response'} + async def skills(*args): + if scenario == 'changed_before_acceptance': + configs.save('agent', 'http://changed.test/v1', 'demo', '', saved['id'], connection_type='self_hosted') + return [] + monkeypatch.setattr(agent.AgentRuntimes, 'stream', stream) + monkeypatch.setattr(agent, 'runtime_available', lambda _: True) + monkeypatch.setattr(agent, 'skills', skills) + headers = {'X-OpenDirector-Request': '1', 'X-OpenDirector-Session': str(uuid4())} + with TestClient(create_app(LocalProjects(tmp_path), capability_probe=probe), headers=headers) as client: + pid = str(uuid4()) + base = f'/api/creation/projects/{pid}' + client.post(base + '/commands', json={'type': 'project.create', 'projectId': pid, + 'commandId': str(uuid4()), 'payload': {'name': 'Capability assistant'}}).raise_for_status() + client.post(base + '/open').raise_for_status() + response = client.post(base + '/assistant/turns', json={'id': str(uuid4()), 'text': 'Hello', + 'modelId': saved['id'], 'runtime': 'codex' if scenario == 'native' else 'default'}) + if scenario in {'unsupported', 'missing_input'}: + assert response.status_code == 409 and turns == [] + assert client.get(base + '/assistant').json()['messages'] == [] + elif scenario == 'changed_before_acceptance': + assert response.status_code == 200 and turns == [] + events = [json.loads(line[6:]) for line in response.text.splitlines() if line.startswith('data: ')] + assert not any(item['type'] == 'start' for item in events) + assert client.get(base + '/assistant').json()['messages'] == [] + else: + assert response.status_code == 200 and len(turns) == 1 and calls == [] + + +def test_loopback_http_discovery_checks_real_route_without_inference(tmp_path): + calls = [] + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + calls.append((self.command, self.path)) + data = json.dumps(document()).encode() + self.send_response(200) + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data) + def log_message(self, *args): + pass + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + configs = ModelConfigs(tmp_path) + configs.save('agent', f'http://127.0.0.1:{server.server_port}/v1', 'demo', '', connection_type='self_hosted') + probe = CapabilityProbe(transport=httpx.HTTPTransport()) + try: + result = ModelAvailability(configs, probe).refresh() + assert result['models'][0]['status'] == 'available' + assert calls == [('GET', '/v1/omni/capabilities')] + finally: + probe.close() + server.shutdown() + server.server_close() + thread.join(2) + + +@pytest.mark.parametrize('live', [False, True]) +def test_cli_availability_uses_shared_read_or_refresh(live, monkeypatch): + client = Mock(live=live, opened=False) + client.request.return_value = client.call.return_value = {'revision': 'test', 'models': []} + monkeypatch.setattr(cli, 'Client', Mock(return_value=client)) + model = str(uuid4()) + result = CliRunner().invoke(cli.app, ['model', 'availability', '--refresh', '--manual', '--model', model]) + assert result.exit_code == 0, result.output + if live: + client.call.assert_called_once_with('model_availability', {'refresh': True, 'manual': True, 'configIds': [model]}) + else: + client.request.assert_called_once_with('POST', '/api/model-configs/availability', json={'manual': True, 'configIds': [model]}) + + +def test_agent_reads_and_refreshes_the_same_availability(project): + store, pid, session = project + runner, config = runtime(project, Provider()) + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=document(model='test-agent')))) + runner.availability = ModelAvailability(runner.configs, probe) + tools = AgentTools(store, runner, runner.configs) + async def run(): + result = await tools.execute(pid, uuid4(), session, 'read', 'model_availability', {}, [], None) + assert result['models'][0]['status'] == 'unknown' + result = await tools.execute(pid, uuid4(), session, 'refresh', 'model_availability', {'refresh': True}, [], None) + assert result['models'][0]['status'] == 'available' + try: + asyncio.run(run()) + finally: + probe.close() + + +@contextmanager +def local_api(app): + """Run the real API on an ephemeral loopback port for the real CLI client.""" + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)) + url = f'http://127.0.0.1:{listener.getsockname()[1]}' + server = uvicorn.Server(uvicorn.Config(app, log_level='warning', access_log=False)) + thread = threading.Thread(target=server.run, kwargs={'sockets': [listener]}, daemon=True) + thread.start() + try: + deadline = time.monotonic() + 10 + while not server.started and thread.is_alive() and time.monotonic() < deadline: + time.sleep(.01) + assert server.started, 'Local API did not start' + yield url + finally: + server.should_exit = True + thread.join(10) + assert not thread.is_alive(), 'Local API did not shut down' + + +@pytest.mark.parametrize('entry', ['cli', 'agent']) +@pytest.mark.parametrize('state', ['supported', 'unsupported', 'paused']) +def test_cli_and_agent_generation_apply_real_admission(project, monkeypatch, entry, state): + store, pid, session = project + provider = Provider() + provider.finish.set() + runner, config = runtime(project, provider) + declaration = document(model='test-agent', support='unsupported' if state == 'unsupported' else 'supported') + if state == 'paused': + declaration['models'][0]['operations']['chat']['execution']['state'] = 'paused' + queries = [] + probe = CapabilityProbe(transport=httpx.MockTransport(lambda request: + queries.append(request) or httpx.Response(200, json=declaration))) + runner.availability = ModelAvailability(runner.configs, probe) + command = command_for(project, config) + arguments = command.model_dump(mode='json', exclude={'commandId', 'projectId'}) + before = store.snapshot(pid) + + async def through_agent(): + tools = AgentTools(store, runner, runner.configs) + try: + value = await tools.execute(pid, uuid4(), session, 'submit', 'execute_command', arguments, [], None) + await settled(runner) + return value + finally: + await runner.close() + + try: + if entry == 'agent': + receipt = asyncio.run(through_agent()) + else: + monkeypatch.delenv('OPENDIRECTOR_AGENT_SOCKET', raising=False) + monkeypatch.delenv('OPENDIRECTOR_AGENT_HOST', raising=False) + app = create_app(store.local, generation_provider=provider, capability_probe=probe) + with local_api(app) as url: + result = CliRunner().invoke(cli.app, ['--url', url, '--project', str(pid), '--session', session, + '--operation-id', str(command.commandId), 'generation', 'submit', '--file', '-'], + input=json.dumps(arguments)) + assert result.exit_code == (0 if state == 'supported' else 1), result.output + receipt = json.loads(result.output) + if state == 'supported': + deadline = time.monotonic() + 5 + while store.snapshot(pid).jobs[0].status not in {'succeeded', 'failed'} and time.monotonic() < deadline: + time.sleep(.01) + after = store.snapshot(pid) + assert len(queries) == 1 and queries[0].method == 'GET' + if state == 'supported': + assert receipt['status'] == 'accepted' + assert len(after.jobs) == len(provider.calls) == 1 + job = after.jobs[0] + assert receipt['jobIds'] == [str(job.id)] and job.status == 'succeeded' + assert job.inputSnapshot.spec.modelConfigId == before.generationSpecs[0].modelConfigId + assert job.inputSnapshot.spec.prompt == before.generationSpecs[0].prompt + else: + assert receipt['status'] == 'rejected' + assert receipt['issues'][0]['code'] == ('capability_paused' if state == 'paused' else 'capability_unavailable') + assert not after.jobs and provider.calls == [] + assert after.generationSpecs == before.generationSpecs + assert after.canvasItems == before.canvasItems + assert after.assets == before.assets and after.results == before.results + finally: + probe.close() diff --git a/tests/test_capability_input_projection.py b/tests/test_capability_input_projection.py new file mode 100644 index 00000000..a80c4a02 --- /dev/null +++ b/tests/test_capability_input_projection.py @@ -0,0 +1,173 @@ +"""Admission follows provider comparison rules and actual media transmission.""" +from copy import deepcopy +from types import SimpleNamespace + +import pytest +from pydantic import ValidationError + +from opendirector.application.model_availability import generation_request, requests_for +from opendirector.domain.omni_capabilities import ParameterConstraint, RequestConstraints +from opendirector.domain.request_constraints import constraint_failure, input_details +from test_model_availability import document, private, service + + +@pytest.mark.parametrize('rule,options,value', [ + ('title', ['English'], 'english'), ('title', ['english'], 'ENGLISH'), + ('lower', ['af_heart'], 'AF_HEART'), ('lower', ['default'], 'DEFAULT'), + ('strip_lower', ['default', ''], ' DEFAULT '), ('strip_lower', ['default', ''], ' '), +]) +def test_enum_comparison_preserves_the_actual_parameter(rule, options, value): + contract = RequestConstraints.model_validate({'parameters': {'voice': { + 'type': 'string', 'enum': options, 'enum_normalization': rule}}}) + parameters = {'voice': value} + assert constraint_failure(contract, None, parameters) is None + assert parameters == {'voice': value} + assert contract.parameters['voice'].enum == options + assert constraint_failure(contract, None, {'voice': 'missing'}) == ( + 'parameter_value_unsupported', {'parameter': 'voice'}) + + +@pytest.mark.parametrize('rule,value', [(None, 'AF_HEART'), ('lower', ' af_heart ')]) +def test_enum_comparison_does_not_add_undeclared_normalization(rule, value): + contract = RequestConstraints.model_validate({'parameters': {'voice': { + 'type': 'string', 'enum': ['af_heart'], 'enum_normalization': rule}}}) + assert constraint_failure(contract, None, {'voice': value}) is not None + + +def test_empty_voice_placeholder_is_accepted_without_becoming_a_displayed_voice(): + contract = RequestConstraints.model_validate({'parameters': {'voice': { + 'type': 'string', 'enum': ['default', ''], 'enum_normalization': 'strip_lower'}}}) + assert constraint_failure(contract, None, {'voice': ' '}) is None + assert input_details(contract)[0]['values'] == ['default'] + + +@pytest.mark.parametrize('constraint', [ + {'type': 'string', 'enum': ['default'], 'enum_normalization': 'future'}, + {'type': 'string', 'enum_normalization': 'lower'}, + {'type': 'integer', 'enum': [1], 'enum_normalization': 'lower'}, + {'type': 'string', 'enum': ['default', 1], 'enum_normalization': 'lower'}, +]) +def test_invalid_enum_normalization_is_not_treated_as_exact(constraint): + with pytest.raises(ValidationError): + ParameterConstraint.model_validate(constraint) + + +def test_unknown_normalization_keeps_optional_discovery_unknown(service): + configs, saved, state, availability = service + state['document']['models'][0]['operations']['chat']['profiles'][0]['routes'][0]['request'] = { + 'parameters': {'language': {'type': 'string', 'enum': ['English'], 'enum_normalization': 'future'}}} + config = private(configs, saved) + availability.prepare(config) + result = availability.model(config) + assert result['status'] == 'unknown' and result['reasonCode'] == 'invalid_response' + + +@pytest.mark.parametrize('voice,rule,options', [ + ('AF_HEART', 'lower', ['af_heart']), (' DEFAULT ', 'strip_lower', ['default', '']), + (' ', 'strip_lower', ['default', '']), (None, 'strip_lower', ['default', '']), +]) +def test_speech_admission_uses_normalization_without_rewriting_voice(service, voice, rule, options): + configs, _, state, availability = service + saved = configs.save('audio', 'http://speech.test/v1', 'demo', '', connection_type='self_hosted') + config = private(configs, saved) + state['document'] = document('speech', 'text_to_speech', '/v1/audio/speech', 'audio') + state['document']['models'][0]['operations']['speech']['profiles'][0]['routes'][0]['request'] = { + 'parameters': {'voice': {'type': 'string', 'enum': options, 'enum_normalization': rule}}} + availability.prepare(config) + spec = SimpleNamespace(kind='audio', parameters=SimpleNamespace(voice=voice, creationType='audio')) + request = generation_request(config, spec, [], parameters={'voice': voice}) + assert availability.decision(config, request)['status'] == 'available' + assert dict(request.parameters)['voice'] == (voice if voice is not None else 'default') + assert spec.parameters.voice == voice + + +def reference(kind, transmission): + return SimpleNamespace(asset=SimpleNamespace(kind=kind), transmission=transmission) + + +@pytest.mark.parametrize('kind', ['audio', 'video']) +def test_text_metadata_does_not_satisfy_required_media_input(service, kind): + configs, saved, state, availability = service + config = private(configs, saved) + state['document']['models'][0]['operations']['chat']['profiles'][0]['required_inputs'] = ['text', kind] + availability.prepare(config) + request = generation_request(config, SimpleNamespace(kind='text'), [reference(kind, 'metadata')]) + assert dict(request.input_counts) == {'text': 1} + assert availability.decision(config, request)['reasonCode'] == 'required_inputs_missing' + + +def test_content_image_counts_survive_metadata_and_context_filtering(service): + configs, saved, _, _ = service + request = generation_request(private(configs, saved), SimpleNamespace(kind='text'), [ + reference('image', 'content'), reference('audio', 'metadata'), + reference('video', 'metadata'), reference('image', 'context_only')]) + assert request.inputs == {'text', 'image'} + assert dict(request.input_counts) == {'text': 1, 'image': 1} + + +@pytest.mark.parametrize('paused', [False, True]) +def test_continuation_does_not_claim_video_upload_but_honors_route_pause(service, paused): + configs, saved, state, availability = service + config = {**private(configs, saved), 'kind': 'video', 'protocol': 'vllm_omni'} + state['document'] = document('video_generation', 'text_to_video', '/v1/videos', 'video') + operation = state['document']['models'][0]['operations']['video_generation'] + upload = deepcopy(operation['profiles'][0]) + upload.update(id='video_to_video', support='unsupported') + operation['profiles'].append(upload) + operation['execution']['state'] = 'paused' if paused else 'unpaused' + availability.prepare(config) + request = generation_request(config, SimpleNamespace(kind='video'), [ + reference('video', 'latent'), reference('image', 'content')]) + assert dict(request.input_counts) == {'text': 1, 'image': 1} + result = availability.decision(config, request) + assert result['declaration']['status'] == 'unknown' + assert result['declaration']['reasonCode'] == 'profile_not_declared' + assert result['reasonCode'] == ('execution_paused' if paused else 'profile_not_declared') + assert result['status'] == ('unavailable' if paused else 'unknown') + state['clock'] += 6 + assert availability.decision(config, request)['status'] == 'unknown' + + +@pytest.mark.parametrize('route,unavailable', [ + ({'method': 'POST', 'path': '/v1/videos', 'transport': 'http', 'exposure': 'disabled'}, True), + ({'method': 'GET', 'path': '/v1/videos', 'transport': 'http', 'exposure': 'disabled'}, False), + ({'method': 'POST', 'path': '/other/videos', 'transport': 'http', 'exposure': 'disabled'}, False), + ({'method': 'POST', 'path': '/v1/videos', 'transport': 'websocket', 'exposure': 'disabled'}, False), + (None, False), +]) +def test_unknown_continuation_profile_still_honors_a_disabled_http_endpoint(service, route, unavailable): + configs, saved, state, availability = service + config = {**private(configs, saved), 'kind': 'video', 'protocol': 'vllm_omni'} + state['document'] = document('video_generation', 'text_to_video', '/v1/videos', 'video') + state['document']['routes'] = [route] if route else [] + availability.prepare(config) + request = generation_request(config, SimpleNamespace(kind='video'), [reference('video', 'latent')]) + result = availability.decision(config, request) + assert result['status'] == ('unavailable' if unavailable else 'unknown') + assert result['reasonCode'] == ('route_disabled' if unavailable else 'profile_not_declared') + + +@pytest.mark.parametrize('continuation', [False, True]) +def test_disabled_endpoint_reason_survives_a_fresh_pause_observation(service, continuation): + configs, saved, state, availability = service + config = {**private(configs, saved), 'kind': 'video', 'protocol': 'vllm_omni'} + state['document'] = document('video_generation', 'text_to_video', '/v1/videos', 'video') + state['document']['routes'][0]['exposure'] = 'disabled' + state['document']['models'][0]['operations']['video_generation']['execution']['state'] = 'paused' + availability.prepare(config) + request = (generation_request(config, SimpleNamespace(kind='video'), [reference('video', 'latent')]) + if continuation else requests_for(config)[0]) + result = availability.decision(config, request) + assert result['reasonCode'] == result['declaration']['reasonCode'] == 'route_disabled' + assert result['execution']['state'] == 'paused' + + +def test_global_endpoint_closure_does_not_require_a_profile_route(service): + configs, saved, state, availability = service + config = {**private(configs, saved), 'kind': 'video', 'protocol': 'vllm_omni'} + state['document'] = document('video_generation', 'text_to_video', '/v1/videos', 'video') + state['document']['models'][0]['operations']['video_generation']['profiles'][0]['routes'] = [] + state['document']['routes'][0]['exposure'] = 'disabled' + availability.prepare(config) + result = availability.decision(config, requests_for(config)[0]) + assert result['status'] == 'unavailable' and result['reasonCode'] == 'route_disabled' diff --git a/tests/test_generation_errors.py b/tests/test_generation_errors.py index 27b5fb05..4fa946f9 100644 --- a/tests/test_generation_errors.py +++ b/tests/test_generation_errors.py @@ -20,6 +20,16 @@ SIGNED_URL = 'https://cdn.test/output?signature=secret-signature' +def test_unknown_429_does_not_diagnose_account_quota(): + from opendirector.integrations.generation_errors import provider_failure + error = provider_failure({'error': {'code': '1305', 'message': 'service overloaded'}}, + stage='submit', http_status=429) + assert error.details.category == 'rate_limit' and error.details.providerCode == '1305' + assert '服务繁忙' in error.details.message and '稍后' in error.details.suggestion + assert '配额受限' not in error.details.message + assert '不会自动重新提交' in str(error) + + @pytest.fixture def project(tmp_path): store, pid, session = new_project(tmp_path) diff --git a/tests/test_image_input_policy.py b/tests/test_image_input_policy.py new file mode 100644 index 00000000..67d83a2f --- /dev/null +++ b/tests/test_image_input_policy.py @@ -0,0 +1,108 @@ +"""The provider receives every accepted image, with distinct mask roles.""" +from io import BytesIO +from types import SimpleNamespace + +from PIL import Image +import pytest + +from opendirector.domain.image_inputs import image_input_policy, native_mask_applicable, validate_image_request +from opendirector.domain.projects import ProjectError + + +def png(mode='RGB', size=(64, 64), color=128): + output = BytesIO() + Image.new(mode, size, color).save(output, format='PNG') + return output.getvalue() + + +def request(**values): + return SimpleNamespace(image=png(), references=(), mask=None, reference=None, width=64, height=64, **values) + + +def policy(maximum=4): + return {'version': 1, 'maxImages': maximum, 'imageBytes': 8 * 1024 * 1024, + 'requestBytes': 48 * 1024 * 1024, 'nativeMask': None} + + +def native_policy(): + return {**policy(), 'nativeMask': {'field': 'mask_image', 'referenceField': 'reference_image', + 'encoding': 'grayscale_white_edit', 'minImages': 1, 'maxImages': 1, 'maxReferenceImages': 1, + 'widthMultiple': 16, 'heightMultiple': 16, 'minWidth': 64, 'minHeight': 64, + 'maxPixels': 1048576, 'sameSize': True}} + + +def test_z_image_accepts_one_image_from_its_existing_edit_adapter(): + config = {'kind': 'image', 'modelId': 'Tongyi-MAI/Z-Image-Turbo', 'protocol': 'openai', 'url': 'http://models.test/v1'} + assert image_input_policy(config)['maxImages'] == 1 + + +@pytest.mark.parametrize('count', [0, 1, 4]) +def test_every_accepted_image_is_counted(count): + value = request() + value.image = png() if count else None + value.references = (png(),) * max(0, count - 1) + validate_image_request(value, policy()) + + +def test_overflow_is_rejected_without_truncating_inputs(): + value = request() + value.references = (png(),) * 4 + with pytest.raises(ProjectError, match='最多支持 4 张图片，当前为 5 张'): + validate_image_request(value, policy()) + assert len(value.references) == 4 + + +def test_mask_and_independent_reference_are_not_ordinary_images(): + value = request() + value.mask, value.reference = png('L'), png() + validate_image_request(value, native_policy()) + value.references = (png(),) + with pytest.raises(ProjectError): + validate_image_request(value, native_policy()) + + +@pytest.mark.parametrize('width,height', [(63, 64), (65, 64), (2048, 1024)]) +def test_native_selection_never_changes_requested_dimensions(width, height): + assert not native_mask_applicable(native_policy(), width, height) + + +def test_missing_geometry_does_not_enable_native_mask(): + value = native_policy() + del value['nativeMask']['widthMultiple'] + assert not native_mask_applicable(value, 64, 64) + + +def test_native_selection_applies_enum_steps_and_both_count_bounds(): + value = native_policy() + value['nativeMask'].update(widthValues=[1024], heightValues=[1024]) + assert not native_mask_applicable(value, 64, 64) + assert native_mask_applicable(value, 1024, 1024) + value['nativeMask']['widthStep'] = 3 + assert not native_mask_applicable(value, 1024, 1024) + value['nativeMask']['widthStep'] = 16 + value['maxImages'] = 0 + assert not native_mask_applicable(value, 1024, 1024) + + +def test_actual_mask_and_source_geometry_are_checked(): + value = request() + value.mask = png('L', (80, 64)) + with pytest.raises(ProjectError): + validate_image_request(value, native_policy()) + + +def test_mask_requires_normalized_grayscale_and_nonempty_selection(): + value = request() + for invalid in (png('RGBA'), png('L', color=0)): + value.mask = invalid + with pytest.raises(ProjectError): + validate_image_request(value, native_policy()) + + +def test_bytes_include_mask_and_independent_reference(): + value = request() + value.mask, value.reference = png('L'), png() + limited = native_policy() + limited['requestBytes'] = len(value.image) + len(value.mask) + with pytest.raises(ProjectError): + validate_image_request(value, limited) diff --git a/tests/test_localization.py b/tests/test_localization.py index 121f8c53..76b9ab8f 100644 --- a/tests/test_localization.py +++ b/tests/test_localization.py @@ -11,6 +11,20 @@ def request(language: str): return SimpleNamespace(headers={"accept-language": language}) +def test_capability_messages_preserve_the_specific_input_failure(): + cases = [ + ('capability_unavailable', '该模型不支持文字回复。', 'This model does not support text replies.'), + ('capability_unavailable', '请添加图片。', 'Add images.'), + ('capability_unavailable', '服务未开放图片编辑。', 'The service has not enabled image editing.'), + ('image_input_count', '最多支持 4 张图片，当前为 5 张。', 'Up to 4 images; 5 added.'), + ('tool_source_count', '该工具需要 2 张来源图片。', 'This tool requires 2 source images.'), + ('image_input_mask', '选区与图片的尺寸不一致。', 'The selection and image dimensions must match.'), + ] + for code, chinese, english in cases: + assert localized_error(request('en-US'), chinese=chinese, status=422, code=code) == english + assert localized_error(request('zh-CN'), chinese=chinese, status=422, code=code) == chinese + + def test_accept_language_uses_the_primary_language() -> None: assert wants_english(request("zh-CN,zh;q=0.9")) is False assert wants_english(request("zh-Hant")) is False diff --git a/tests/test_minimax_image.py b/tests/test_minimax_image.py index 1fa62cb0..7515385e 100644 --- a/tests/test_minimax_image.py +++ b/tests/test_minimax_image.py @@ -9,7 +9,9 @@ from opendirector.domain.capabilities import ImageRequest from opendirector.domain.generation import GenerationError +from opendirector.domain.projects import ProjectError from opendirector.infrastructure.model_configs import ModelConfigs +from opendirector.integrations.generation_capability_parameters import generation_capability_parameters from opendirector.integrations.model_adapters import adapter_for from opendirector.integrations.providers import GenerationProviders @@ -71,6 +73,36 @@ async def run(): assert calls[0][1]['size'] == '2720x1536' expected = [{'image_url': 'data:image/png;base64,' + base64.b64encode(data).decode()} for data in (first, second)] assert calls[0][1]['images'] == calls[2][1]['images'] == expected + adapter = adapter_for(config) + projections = [ + generation_capability_parameters(config, 'image', {'size': '2720x1536'}, has_images=True), + adapter.image_parameters(1024, 1024, has_images=False), + adapter.image_parameters(1024, 1024, has_images=True), + ] + actual = [{key: value for key, value in body.items() if key not in {'model', 'prompt', 'images'}} + for _, body in calls] + checked = [{key: value for key, value in item.items() if key not in {'width', 'height'}} + for item in projections] + assert checked == actual + + +@pytest.mark.parametrize('field', ['mask', 'reference']) +def test_unsupported_auxiliary_image_is_not_silently_dropped(tmp_path, field): + configs = ModelConfigs(tmp_path) + saved = configs.save('image', URL, MODELS[0], 'test-key', protocol='openai') + config = configs.resolve(saved['id'], 'image') + calls = [] + + def respond(request): + calls.append(request.method) + return (httpx.Response(200, content=png('blue')) if request.method == 'GET' + else httpx.Response(200, json={'data': [{'url': RESULT}]})) + + adapter = adapter_for(config, httpx.MockTransport(respond)) + with pytest.raises(ProjectError) as failure: + asyncio.run(adapter.image(config, ImageRequest('Edit', 128, 128, png('red'), **{field: png('blue')}))) + assert failure.value.code == 'image_input_mask' + assert calls == [] def test_submission_timeout_does_not_retry(tmp_path): diff --git a/tests/test_model_availability.py b/tests/test_model_availability.py new file mode 100644 index 00000000..9e2d3930 --- /dev/null +++ b/tests/test_model_availability.py @@ -0,0 +1,595 @@ +"""Optional discovery uses real request contracts and never changes saved intent.""" +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from datetime import datetime, timezone +from types import SimpleNamespace +import asyncio +import json +import threading +import time + +import httpx +import pytest + +from opendirector.application.model_availability import ModelAvailability, generation_request, requests_for, tool_request +from opendirector.application.model_catalog import ModelCatalog +from opendirector.domain.projects import ProjectError +from opendirector.infrastructure.model_configs import ModelConfigs +from opendirector.integrations.capability_probe import CapabilityProbe + + +NOW = 1794225600.0 + + +def document(operation='chat', profile='text_response', path='/v1/chat/completions', output='text', *, model='demo', support='supported'): + return {'object': 'omni.capabilities', 'schema_version': 1, + 'observed_at': datetime.fromtimestamp(NOW, timezone.utc).isoformat(), + 'models': [{'id': model, 'stage_ids': ['0'], 'operations': {operation: { + 'support': support, 'exposure': 'enabled', 'reason_code': 'model_declared', + 'profiles': [{'id': profile, 'support': support, 'outputs': [output], + 'required_inputs': ['text'], 'routes': [{'method': 'POST', 'path': path}]}], + 'execution': {'state': 'unpaused', 'scope': 'deployment', + 'observed_at': datetime.fromtimestamp(NOW, timezone.utc).isoformat()}}}}], + 'routes': [{'transport': 'http', 'method': 'POST', 'path': path, 'exposure': 'enabled'}]} + + +@pytest.fixture +def service(tmp_path): + configs = ModelConfigs(tmp_path) + saved = configs.save('agent', 'http://models.test/v1', 'demo', '', connection_type='self_hosted', auth_mode='none') + state = {'document': document(), 'status': 200, 'calls': [], 'clock': NOW} + + def respond(request): + state['calls'].append(request) + return httpx.Response(state['status'], json=state['document']) + + probe = CapabilityProbe(transport=httpx.MockTransport(respond), clock=lambda: state['clock'], wall_clock=lambda: state['clock']) + availability = ModelAvailability(configs, probe) + yield configs, saved, state, availability + probe.close() + + +def private(configs, saved): + return configs.resolve(saved['id'], saved['kind']) + + +def image_contract(service, *, maximum=4): + configs, _, state, availability = service + saved = configs.save('image', 'http://images.test/v1', 'image-alias', '', connection_type='self_hosted', auth_mode='none') + state['document'] = document('image_edit', 'image_to_image', '/v1/images/edits', 'image', model='image-alias') + profile = state['document']['models'][0]['operations']['image_edit']['profiles'][0] + profile['required_inputs'] = ['text', 'image'] + profile['request'] = {'inputs': {'image': {'support': 'supported', 'max_count': 99}}} + profile['routes'][0]['request'] = {'inputs': {'image': {'support': 'supported', 'min_count': 1, 'max_count': maximum}}} + availability.refresh([saved['id']]) + return private(configs, saved), profile + + +@pytest.mark.parametrize('count,expected', [(0, 'unavailable'), (1, 'available'), (4, 'available'), (5, 'unavailable')]) +def test_actual_image_count_uses_exact_route_constraints(service, count, expected): + _, _, _, availability = service + config, _ = image_contract(service) + request = replace(requests_for(config)[1], inputs=frozenset({'text', 'image'}), input_counts=(('text', 1), ('image', count))) + assert availability.decision(config, request)['status'] == expected + if count == 5: + with pytest.raises(ProjectError, match='最多支持 4 张图片，当前为 5 张'): + availability.require(config, request) + + +def test_missing_route_contract_does_not_apply_model_wide_limit(service): + _, _, _, availability = service + config, profile = image_contract(service, maximum=1) + del profile['routes'][0]['request'] + profile['request']['inputs']['image']['max_count'] = 0 + availability.prepare(config, manual=True) + request = replace(requests_for(config)[1], inputs=frozenset({'text', 'image'}), input_counts=(('image', 2), ('text', 1))) + assert availability.decision(config, request)['status'] == 'available' + assert availability.image_execution_policy(config)['maxImages'] == 8 + + +def test_single_and_multi_image_directory_have_distinct_results(service): + _, _, _, availability = service + config, _ = image_contract(service, maximum=1) + values = {item['profile']: item for item in availability.model(config)['operations']} + assert values['image_to_image']['status'] == 'available' + assert values['images_to_image']['status'] == 'unavailable' + assert values['images_to_image']['reasonCode'] == 'input_count_unsupported' + + +def test_pause_keeps_independently_expiring_declaration(service): + configs, saved, state, availability = service + state['document']['models'][0]['operations']['chat']['execution']['state'] = 'paused' + availability.refresh() + item = availability.model(private(configs, saved))['operations'][0] + assert item['status'] == 'unavailable' + assert item['declaration']['status'] == 'available' + assert item['execution']['expiresAt'] - item['execution']['checkedAt'] == 5000 + assert item['declaration']['expiresAt'] - item['declaration']['checkedAt'] == 30000 + + +def test_explicit_parameter_constraints_do_not_treat_defaults_as_enum(service): + _, _, _, availability = service + config, profile = image_contract(service) + profile['routes'][0]['request']['parameters'] = {'width': {'support': 'supported', 'type': 'integer', 'minimum': 64, 'step': 16}} + availability.prepare(config, manual=True) + request = replace(requests_for(config)[1], inputs=frozenset({'text', 'image'}), parameters=(('width', 80),)) + assert availability.decision(config, request)['status'] == 'available' + with pytest.raises(ProjectError, match='图片宽度不符合服务要求'): + availability.require(config, replace(request, parameters=(('width', 81),))) + + +@pytest.mark.parametrize('adapter', ['openai', 'minimax']) +def test_native_mask_policy_uses_actual_geometry_and_operator_limits(service, adapter): + _, _, _, availability = service + config, profile = image_contract(service) + if adapter == 'minimax': + config.update(url='https://api.minimax.cn/v1', modelId='MiniMax-H3-Image') + service[2]['document']['models'][0]['id'] = config['modelId'] + contract = profile['routes'][0]['request'] + contract['inputs'].update(mask_image={'support': 'supported', 'max_count': 1}, + reference_image={'support': 'supported', 'max_count': 1}) + contract['parameters'] = {'width': {'support': 'supported', 'minimum': 1, 'step': 32}, + 'height': {'support': 'supported', 'minimum': 1, 'maximum': 512}} + contract['mask'] = {'support': 'supported', 'source_role': 'image', 'max_source_count': 1, + 'white': 'edit', 'black': 'preserve', 'binarize_threshold': 0.5, 'requires_same_size': True, + 'geometry': {'state': 'known', 'multiple_of': 16}, + 'source_resize': {'max_pixels': 1048576, 'policy': 'downscale_preserve_aspect_ratio'}} + contract['max_output_pixels'] = 262144 + availability.prepare(config, manual=True) + policy = availability.image_execution_policy(config) + if adapter == 'minimax': + assert policy['maxImages'] == 4 + assert policy['nativeMask'] is None + return + assert policy['maxImages'] == 4 and policy['nativeMask']['maxImages'] == 1 + assert policy['nativeMask']['widthMultiple'] == 32 + assert policy['nativeMask']['maxPixels'] == 262144 + assert policy['nativeMask']['maxHeight'] == 512 + contract['mask']['geometry'] = {'state': 'unknown'} + availability.prepare(config, manual=True) + assert availability.image_execution_policy(config)['nativeMask'] is None + assert availability.decision(config, requests_for(config)[1])['status'] == 'available' + + +@pytest.mark.parametrize('operation,profile,path,output,role', [ + ('chat', 'text_response', '/v1/chat/completions', 'text', 'image'), + ('video_generation', 'text_to_video', '/v1/videos', 'video', 'audio'), + ('speech', 'text_to_speech', '/v1/audio/speech', 'audio', 'reference_audio'), + ('speech', 'lyrics_to_music', '/v1/audio/speech', 'audio', 'voice'), +]) +def test_explicit_role_constraints_are_shared_across_operations(service, operation, profile, path, output, role): + configs, saved, state, availability = service + state['document'] = document(operation, profile, path, output) + item = state['document']['models'][0]['operations'][operation]['profiles'][0] + item['routes'][0]['request'] = {'inputs': {role: {'support': 'unsupported'}}} + availability.refresh() + from opendirector.application.model_availability import RequestOperation + request = RequestOperation(operation, profile, path, output, input_counts=(('text', 1), (role, 1))) + assert availability.decision(private(configs, saved), request)['reasonCode'] == 'input_unsupported' + + +def test_invalid_boolean_count_is_unknown_and_does_not_block(service): + _, _, _, availability = service + config, profile = image_contract(service) + profile['routes'][0]['request']['inputs']['image']['max_count'] = False + availability.prepare(config, manual=True) + assert availability.decision(config, requests_for(config)[1])['reasonCode'] == 'invalid_response' + availability.require(config, requests_for(config)[1]) + + +@pytest.mark.parametrize('voice,expected', [(None, 'default'), ('speaker1', 'speaker1')]) +def test_speech_checks_the_same_explicit_or_default_voice_as_the_adapter(service, voice, expected): + configs, _, state, availability = service + saved = configs.save('audio', 'http://speech.test/v1', 'speech-alias', '', connection_type='self_hosted', auth_mode='none') + config = private(configs, saved) + state['document'] = document('speech', 'text_to_speech', '/v1/audio/speech', 'audio', model='speech-alias') + profile = state['document']['models'][0]['operations']['speech']['profiles'][0] + profile['required_inputs'] = ['text', 'voice'] + profile['routes'][0]['request'] = {'inputs': {'voice': {'support': 'supported', 'min_count': 1}}, + 'parameters': {'voice': {'support': 'supported', 'enum': ['default', 'speaker1']}}} + availability.prepare(config) + spec = SimpleNamespace(kind='audio', parameters=SimpleNamespace(voice=voice, creationType='audio')) + request = generation_request(config, spec, []) + availability.require(config, request) + assert 'voice' in request.inputs and dict(request.input_counts)['voice'] == 1 + assert dict(request.parameters)['voice'] == expected + + +def test_catalog_is_network_free_and_manual_refresh_runs_no_inference(service): + configs, saved, state, availability = service + catalog = ModelCatalog(configs, availability=availability) + before = configs.path.read_bytes() + assert catalog.list()[0]['availability']['status'] == 'unknown' + assert state['calls'] == [] + observed = availability.refresh()['models'][0] + assert observed['status'] == 'available' + assert [(r.method, str(r.url)) for r in state['calls']] == [('GET', 'http://models.test/v1/omni/capabilities')] + assert catalog.list()[0]['availability']['identity'] == observed['identity'] + assert configs.path.read_bytes() == before + + +@pytest.mark.parametrize('status,body', [(404, {}), (401, {}), (500, {}), (200, {'object': 'list'}), + (200, {**document(), 'schema_version': 2}), (200, {**document(), 'models': document()['models'] * 2})]) +def test_legacy_or_invalid_service_stays_unknown_and_preserves_manual_config(service, status, body): + configs, saved, state, availability = service + state.update(status=status, document=body) + before = configs.path.read_bytes() + availability.refresh() + config = private(configs, saved) + availability.require(config, requests_for(config)[0]) + assert availability.model(config)['status'] == 'unknown' + assert configs.path.read_bytes() == before + + +@pytest.mark.parametrize('status,body,reason', [(401, {}, 'authentication_failed'), (403, {}, 'authentication_failed'), + (404, {}, 'protocol_unavailable'), (200, {**document(), 'schema_version': 2}, 'unsupported_version'), + (200, {**document(), 'models': 'invalid'}, 'invalid_response')]) +def test_query_failures_keep_actionable_sanitized_reasons(service, status, body, reason): + configs, saved, state, availability = service + state.update(status=status, document=body) + availability.refresh() + value = availability.model(private(configs, saved)) + assert value['status'] == 'unknown' and value['reasonCode'] == reason + + +def test_connection_failure_is_distinct_from_invalid_document(service): + configs, saved, _, availability = service + def respond(request): + raise httpx.ConnectError('secret response must not escape', request=request) + availability.probe.transport = httpx.MockTransport(respond) + availability.refresh() + value = availability.model(private(configs, saved)) + assert value['reasonCode'] == 'connection_failed' + assert 'secret response' not in str(value) + + +@pytest.mark.parametrize('mutation,reason', [ + (lambda op, doc: (op.update(support='unsupported'), op['profiles'][0].update(support='unsupported')), 'operation_unsupported'), + (lambda op, doc: op.update(exposure='disabled'), 'operation_disabled'), + (lambda op, doc: op['profiles'][0].update(support='unsupported'), 'profile_unsupported'), + (lambda op, doc: op['profiles'][0].update(outputs=['image']), 'output_unsupported'), + (lambda op, doc: doc['routes'][0].update(exposure='disabled'), 'route_disabled'), +]) +def test_explicit_negative_evidence_rejects_exact_request(service, mutation, reason): + configs, saved, state, availability = service + mutation(state['document']['models'][0]['operations']['chat'], state['document']) + availability.refresh() + config = private(configs, saved) + assert availability.model(config)['operations'][0]['reasonCode'] == reason + with pytest.raises(ProjectError) as error: + availability.require(config, requests_for(config)[0]) + assert error.value.code == 'capability_unavailable' + + +@pytest.mark.parametrize('mutation', [ + lambda doc: doc['models'][0].update(id='different-alias'), + lambda doc: doc['models'][0]['operations']['chat']['profiles'][0].update(id='image_response'), + lambda doc: doc['models'][0]['operations']['chat']['profiles'][0]['routes'][0].update(path='/v1/other'), + lambda doc: doc['models'][0]['operations']['chat']['profiles'][0]['routes'][0].update(method='GET'), +]) +def test_no_inference_from_a_different_model_profile_or_route(service, mutation): + configs, saved, state, availability = service + mutation(state['document']) + availability.refresh() + assert availability.model(private(configs, saved))['status'] == 'unknown' + + +def test_independent_pause_and_declaration_expiry_changes_projection_without_network(service): + configs, saved, state, availability = service + execution = state['document']['models'][0]['operations']['chat']['execution'] + execution['state'] = 'paused' + availability.refresh() + initial = availability.list() + assert initial['models'][0]['reasonCode'] == 'execution_paused' + state['clock'] += 5.01 + expired_pause = availability.list() + assert expired_pause['models'][0]['status'] == 'available' + assert expired_pause['revision'] != initial['revision'] + state['clock'] = NOW + 30.01 + assert availability.list()['models'][0]['status'] == 'unknown' + assert len(state['calls']) == 1 + + +@pytest.mark.parametrize('scope,observed', [('worker', NOW), ('deployment', NOW - 10), ('deployment', NOW + 100)]) +def test_pause_scope_and_local_expiry_do_not_depend_on_remote_clock(service, scope, observed): + configs, saved, state, availability = service + state['document']['models'][0]['operations']['chat']['execution'].update(state='paused', scope=scope, + observed_at=datetime.fromtimestamp(observed, timezone.utc).isoformat()) + availability.refresh() + assert availability.list()['models'][0]['status'] == ('available' if scope == 'worker' else 'unavailable') + state['clock'] += 5.01 + assert availability.list()['models'][0]['status'] == 'available' + + +@pytest.mark.parametrize('contradiction', ['operation_profile', 'duplicate_route', 'duplicate_profile_route']) +def test_contradictory_document_is_unknown(service, contradiction): + configs, saved, state, availability = service + doc = state['document'] + if contradiction == 'operation_profile': + doc['models'][0]['operations']['chat']['support'] = 'unsupported' + elif contradiction == 'duplicate_route': + doc['routes'].append({**doc['routes'][0], 'exposure': 'disabled'}) + else: + routes = doc['models'][0]['operations']['chat']['profiles'][0]['routes'] + routes.append({**routes[0], 'exposure': 'disabled'}) + availability.refresh() + assert availability.model(private(configs, saved))['reasonCode'] == 'invalid_response' + + +def test_unknown_profile_outputs_do_not_create_negative_evidence(service): + configs, saved, state, availability = service + state['document']['models'][0]['operations']['chat']['profiles'][0].update(support='unknown', outputs=['image']) + availability.refresh() + assert availability.model(private(configs, saved))['status'] == 'unknown' + + +def test_wall_clock_change_does_not_extend_or_shorten_ttl(service): + configs, saved, state, availability = service + wall = [NOW] + availability.probe.wall_clock = lambda: wall[0] + availability.refresh() + wall[0] += 10000 + assert availability.model(private(configs, saved))['status'] == 'available' + wall[0] -= 20000 + state['clock'] += 31 + assert availability.model(private(configs, saved))['status'] == 'unknown' + + +def test_opt_out_is_persisted_and_manual_check_is_explicit(service): + configs, saved, state, availability = service + configs.save('agent', saved['url'], 'demo', '', saved['id'], connection_type='self_hosted', capability_auto_check=False) + availability.refresh() + assert state['calls'] == [] + with pytest.raises(ProjectError): + availability.refresh(manual=True) + availability.refresh([saved['id']], manual=True) + assert availability.list()['models'][0]['status'] == 'available' + assert len(state['calls']) == 1 + + +def test_shared_service_preference_preserves_omitted_and_explicit_false(tmp_path): + configs = ModelConfigs(tmp_path) + models = configs.save_many('agent', 'http://models.test/v1', ['a', 'b'], '', connection_type='self_hosted') + assert all('capabilityAutoCheck' not in item for item in models) + configs.save('agent', models[0]['url'], 'a', '', models[0]['id'], connection_type='self_hosted', capability_auto_check=False) + assert all(item['capabilityAutoCheck'] is False for item in ModelConfigs(tmp_path).list()) + configs.save_many('agent', models[0]['url'], ['a', 'b'], '', models[0]['id'], connection_type='self_hosted') + assert all(item['capabilityAutoCheck'] is False for item in configs.list()) + stored = json.loads(configs.path.read_text()) + assert all('capabilityAutoCheck' not in item for item in stored['models']) + assert next(iter(stored['services'].values()))['capabilityAutoCheck'] is False + + +def test_disabled_independent_connection_does_not_suppress_enabled_peer(service): + configs, saved, state, availability = service + configs.save('agent', saved['url'], 'demo', '', saved['id'], connection_type='self_hosted', capability_auto_check=False) + configs.save('agent', saved['url'], 'demo', '', connection_type='self_hosted', capability_auto_check=True) + availability.refresh() + assert len(state['calls']) == 1 + + +def test_independent_service_owners_do_not_share_cache_at_the_same_url(service): + configs, saved, state, availability = service + availability.refresh() + other = configs.save('agent', saved['url'], 'demo', '', connection_type='self_hosted') + assert availability.model(private(configs, other))['status'] == 'unknown' + availability.refresh() + assert len(state['calls']) == 2 + + +def test_one_provider_with_multiple_models_shares_one_declaration_query(service): + configs, saved, state, availability = service + configs.delete(saved['id']) + models = configs.save_many('agent', saved['url'], ['demo', 'other'], '', connection_type='self_hosted') + assert models[0]['providerId'] == models[1]['providerId'] + availability.refresh() + assert len(state['calls']) == 1 + assert availability.list()['models'][0]['status'] == 'available' + + +def test_distinct_providers_at_identical_urls_remain_isolated(service): + configs, saved, state, availability = service + configs.delete(saved['id']) + first = configs.save_many('agent', saved['url'], ['demo'], '', connection_type='self_hosted')[0] + second = configs.save_many('agent', saved['url'], ['demo'], '', connection_type='self_hosted')[0] + assert first['providerId'] != second['providerId'] + availability.refresh([first['id']]) + assert availability.model(private(configs, second))['status'] == 'unknown' + availability.refresh([second['id']]) + assert len(state['calls']) == 2 + + +def test_draft_does_not_publish_saved_observation_or_credentials(service): + configs, saved, state, availability = service + catalog = ModelCatalog(configs, availability=availability) + result = asyncio.run(catalog.check_availability('agent', saved['url'], 'demo', '', connection_type='self_hosted')) + assert result['status'] == 'available' and result['configId'] is None + assert availability.list()['models'][0]['status'] == 'unknown' + assert 'apiKey' not in str(result) + + +def test_connection_changes_invalidate_identity_and_late_response(service): + configs, saved, state, availability = service + original = private(configs, saved) + prepared = availability.prepare(original) + changed = configs.save('agent', 'http://other.test/v1', 'demo', '', saved['id'], connection_type='self_hosted') + with pytest.raises(ProjectError) as error: + availability.require(private(configs, changed), requests_for(original)[0], prepared) + assert error.value.code == 'capability_connection_changed' + assert availability.list()['models'][0]['status'] == 'unknown' + + +def test_music_uses_lyrics_profile_on_speech_and_not_generic_audio_generation(service): + configs, _, state, availability = service + saved = configs.save('audio', 'http://music.test/v1', 'demo', '', connection_type='self_hosted', protocol='vllm_omni_music') + state['document'] = document('speech', 'lyrics_to_music', '/v1/audio/speech', 'audio') + state['document']['models'][0]['operations']['speech']['profiles'].append({'id': 'text_to_speech', 'support': 'unsupported'}) + state['document']['models'][0]['operations']['audio_generation'] = {'support': 'unsupported'} + config = private(configs, saved) + availability.prepare(config) + assert availability.model(config)['status'] == 'available' + request = requests_for(config)[0] + assert (request.operation, request.profile, request.path) == ('speech', 'lyrics_to_music', '/v1/audio/speech') + ordinary = {**config, 'protocol': 'openai'} + with pytest.raises(ProjectError): + availability.require(ordinary, requests_for(ordinary)[0]) + + +def test_new_remote_profiles_are_accepted_without_creating_consumers(service): + configs, saved, state, availability = service + state['document']['models'][0]['operations']['speech'] = { + 'support': 'supported', 'profiles': [{'id': 'caption_to_audio', 'support': 'supported', + 'outputs': ['audio'], 'required_inputs': ['caption'], 'routes': [{'method': 'POST', 'path': '/v1/audio/speech'}]}]} + availability.refresh() + value = availability.model(private(configs, saved)) + assert value['status'] == 'available' + assert [item['profile'] for item in value['operations']] == ['text_response'] + + +def test_agent_connection_test_rejects_known_non_text_model_without_inference(service): + from test_model_catalog import Scripted + configs, saved, state, availability = service + state['document']['models'][0]['operations']['chat']['profiles'][0]['support'] = 'unsupported' + tester = Scripted([{'type': 'delta', 'text': 'should not run'}]) + catalog = ModelCatalog(configs, tester, availability) + with pytest.raises(ProjectError) as error: + asyncio.run(catalog.test_agent(saved['url'], 'demo', '', connection_type='self_hosted')) + assert error.value.code == 'capability_unavailable' and tester.calls == [] + + +def test_parallel_callers_share_one_query_and_deadline(tmp_path): + started, finish = threading.Event(), threading.Event() + calls = [] + def respond(request): + calls.append(request) + started.set() + finish.wait(2) + return httpx.Response(200, json=document()) + probe = CapabilityProbe(transport=httpx.MockTransport(respond), timeout=.15) + config = {'url': 'http://models.test/v1', 'connectionType': 'self_hosted'} + try: + with ThreadPoolExecutor(max_workers=6) as pool: + started_at = time.monotonic() + futures = [pool.submit(probe.query, config) for _ in range(6)] + assert started.wait(1) + results = [future.result(timeout=1) for future in futures] + assert time.monotonic() - started_at < 1 + assert all(value.reason == 'query_timeout' for value in results) + assert len(calls) == 1 + finally: + finish.set() + probe.close() + + +def test_cache_is_bounded_and_does_not_persist(tmp_path): + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(404)), capacity=3) + try: + for index in range(7): + probe.query({'url': f'http://service-{index}.test/v1', 'connectionType': 'self_hosted'}) + assert len(probe._cache) == 3 + assert list(tmp_path.iterdir()) == [] + finally: + probe.close() + + +@pytest.mark.parametrize('references,expected', [ + ([('image', 'content')], 'unavailable'), + ([('image', 'content'), ('audio', 'content')], 'available'), + ([('image', 'content'), ('audio', 'context_only')], 'unavailable'), +]) +def test_video_required_audio_is_checked_on_actual_request_not_catalog(service, references, expected): + configs, _, state, availability = service + config = {**configs.resolve(configs.list()[0]['id'], 'agent'), 'kind': 'video', 'protocol': 'vllm_omni'} + state['document'] = document('video_generation', 'image_to_video', '/v1/videos', 'video') + profile = state['document']['models'][0]['operations']['video_generation']['profiles'][0] + profile.update(required_inputs=['text', 'image', 'audio'], outputs=['video', 'audio']) + availability.prepare(config) + catalog_request = next(item for item in requests_for(config) if item.profile == 'image_to_video') + assert availability.decision(config, catalog_request)['status'] == 'available' + refs = [SimpleNamespace(asset=SimpleNamespace(kind=kind), transmission=transmission) for kind, transmission in references] + actual = generation_request(config, SimpleNamespace(kind='video'), refs) + assert actual.profile == 'image_to_video' + value = availability.decision(config, actual) + assert value['status'] == expected + if expected == 'unavailable': + assert value['reasonCode'] == 'required_inputs_missing' + with pytest.raises(ProjectError): + availability.require(config, actual) + else: + availability.require(config, actual) + + +def test_unrecognized_required_input_stays_unknown(service): + configs, saved, state, availability = service + state['document']['models'][0]['operations']['chat']['profiles'][0]['required_inputs'] = ['text', 'future_input'] + config = private(configs, saved) + availability.prepare(config) + request = requests_for(config, inputs={'text'})[0] + value = availability.decision(config, request) + assert value['status'] == 'unknown' and value['reasonCode'] == 'input_contract_unknown' + availability.require(config, request) + + +def test_generation_profiles_and_tool_inputs_follow_the_actual_adapter(service): + configs, saved, _, _ = service + base = private(configs, saved) + def ref(kind): + return SimpleNamespace(asset=SimpleNamespace(kind=kind), transmission='content') + for kind, protocol, refs, profile, inputs in [ + ('image', 'openai', [], 'text_to_image', {'text'}), + ('image', 'openai', [ref('image')], 'image_to_image', {'text', 'image'}), + ('video', 'vllm_omni', [ref('video')], 'video_to_video', {'text', 'video'}), + ('audio', 'vllm_omni_music', [], 'lyrics_to_music', {'text', 'instructions'}), + ('audio', 'vllm_omni_music', [ref('audio')], 'lyrics_to_music', {'text', 'instructions'}), + ]: + spec = SimpleNamespace(kind=kind, parameters=SimpleNamespace(creationType='music')) + request = generation_request({**base, 'kind': kind, 'protocol': protocol}, spec, refs) + assert request.profile == profile and request.inputs == inputs + video = {**base, 'kind': 'video', 'protocol': 'vllm_omni'} + assert tool_request(video, 'video.first_frame').inputs is None + assert tool_request(video, 'video.first_frame', actual=True).inputs == {'text', 'image'} + + +def test_refresh_two_slow_services_uses_one_total_wait_budget(tmp_path): + configs = ModelConfigs(tmp_path) + for name in ['one', 'two']: + configs.save('agent', f'http://{name}.test/v1', 'demo', '', connection_type='self_hosted') + finish, both_started = threading.Event(), threading.Event() + calls, lock = [], threading.Lock() + def respond(request): + with lock: + calls.append(request) + if len(calls) == 2: + both_started.set() + finish.wait(2) + return httpx.Response(200, json=document()) + probe = CapabilityProbe(transport=httpx.MockTransport(respond), timeout=.4) + try: + started = time.monotonic() + result = ModelAvailability(configs, probe).refresh() + assert time.monotonic() - started < .65 + assert both_started.is_set() and len(calls) == 2 + assert all(value['reasonCode'] == 'query_timeout' for value in result['models']) + finally: + finish.set() + probe.close() + + +def test_batch_refresh_keeps_worker_and_queue_limits(tmp_path): + finish = threading.Event() + calls, lock = [], threading.Lock() + def respond(request): + with lock: + calls.append(request) + finish.wait(2) + return httpx.Response(404) + probe = CapabilityProbe(transport=httpx.MockTransport(respond), timeout=.2, capacity=5) + configs = [{'id': str(i), 'url': f'http://service-{i}.test/v1', 'connectionType': 'self_hosted'} for i in range(9)] + try: + result = probe.query_many(configs) + assert len(calls) == 4 and len(probe._pending) == 5 + assert [value.reason for value in result] == ['query_timeout'] * 5 + ['query_busy'] * 4 + assert len(probe._cache) <= 5 + finally: + finish.set() + probe.close() diff --git a/tests/test_model_configs.py b/tests/test_model_configs.py index 46cd5bcf..6d324290 100644 --- a/tests/test_model_configs.py +++ b/tests/test_model_configs.py @@ -33,7 +33,9 @@ def test_four_categories_survive_restart_without_exposing_secrets(api): assert set(response.json()) == {"id", "kind", "url", "modelId", "inputRules", "connectionType", "protocol", "authMode", "proxyMode", "videoAdapter", "videoCapabilities", "inputModalities"} configs.append(response.json()) assert len({item["id"] for item in configs}) == 5 - assert client.get("/api/model-configs").json() == configs + observed = client.get("/api/model-configs").json() + assert all(item['availability']['status'] == 'unknown' for item in observed) + assert [{key: value for key, value in item.items() if key != 'availability'} for item in observed] == configs assert ModelConfigs(store.repository).list() == configs assert not store.root.exists() path = store.state_dir / "models.json" @@ -382,7 +384,8 @@ def test_delete_service_removes_group_atomically(api): other = client.post('/api/model-configs', json=payload(modelId='other')).json() response = client.delete('/api/model-services/' + saved[0]['id']) assert response.status_code == 204 - assert client.get('/api/model-configs').json() == [other] + observed = client.get('/api/model-configs').json() + assert [{key: value for key, value in item.items() if key != 'availability'} for item in observed] == [other] assert json.loads((store.state_dir / 'models.json').read_text())['services'] == {} diff --git a/tests/test_request_capability_acceptance.py b/tests/test_request_capability_acceptance.py new file mode 100644 index 00000000..3a5d1b38 --- /dev/null +++ b/tests/test_request_capability_acceptance.py @@ -0,0 +1,292 @@ +"""Image counts cross real acceptance and serialize through production providers.""" +import asyncio +import base64 +import json +import time +from uuid import uuid4 + +from fastapi.testclient import TestClient +import httpx +import pytest + +from opendirector.agent.tools import AgentTools +from opendirector.application.creation_generation import CreationGeneration +from opendirector.application.model_availability import ModelAvailability +from opendirector.infrastructure.model_configs import ModelConfigs +from opendirector.integrations.capability_probe import CapabilityProbe +from opendirector.integrations.providers import GenerationProviders +from opendirector.integrations.generation_capability_parameters import generation_capability_parameters +from opendirector_api.main import create_app +from generation_helpers import command as generation_command, settled +from test_creation_canvas_service import project +from test_model_availability import document +from test_tool_native_mask import multipart +from test_tool_tasks import png, upload +from test_voice_library import wav + + +def image_runtime(project, maximum, model='served-image-alias', *, declared_model=None, editing=True, sampling_maximum=None): + store, _, _ = project + configs = ModelConfigs(store.local.repository) + config = configs.save('image', 'http://images.test/v1', model, '', + connection_type='self_hosted', protocol='openai', auth_mode='none') + operation, profile_id, path = (('image_edit', 'image_to_image', '/v1/images/edits') if editing + else ('image_generation', 'text_to_image', '/v1/images/generations')) + declaration = document(operation, profile_id, path, 'image', + model=declared_model or model) + profile = declaration['models'][0]['operations'][operation]['profiles'][0] + profile['required_inputs'] = ['text', 'image'] if editing else ['text'] + profile['routes'][0]['request'] = {'inputs': {'image': { + 'support': 'supported', 'min_count': int(editing), 'max_count': maximum}}, + 'parameters': {'response_format': {'support': 'supported', 'enum': ['b64_json']}}} + if sampling_maximum is not None: + profile['routes'][0]['request']['parameters']['num_inference_steps'] = { + 'support': 'supported', 'type': 'integer', 'maximum': sampling_maximum} + queries, posts = [], [] + def discover(request): + queries.append(request) + assert request.method == 'GET' and request.url.path == '/v1/omni/capabilities' + return httpx.Response(200, json=declaration) + def generate(request): + assert request.method == 'POST' and request.url.path == path + posts.append(multipart(request) if editing else (json.loads(request.read()), [])) + return httpx.Response(200, json={'data': [{'b64_json': base64.b64encode(png()).decode()}]}) + probe = CapabilityProbe(transport=httpx.MockTransport(discover)) + runner = CreationGeneration(store, configs, availability=ModelAvailability(configs, probe)) + runner.provider = GenerationProviders(runner.store, httpx.MockTransport(generate), poll_interval=0) + return runner, config, probe, queries, posts + + +@pytest.mark.parametrize('model', ['served-image-alias', 'Tongyi-MAI/Z-Image-Turbo', 'Qwen/Qwen-Image-2.1']) +@pytest.mark.parametrize('editing', [False, True]) +def test_checked_image_parameters_match_serialized_request(project, model, editing): + store, pid, session = project + runner, config, probe, _, posts = image_runtime(project, 4, model, editing=editing) + sources = [upload(store, str(pid), session, 'source.png', png())] if editing else [] + try: + receipt = submit('service', project, runner, probe, generation_command(project, config, references=sources)) + assert receipt['status'] == 'accepted', receipt + job = store.snapshot(pid).jobs[0] + assert job.status == 'succeeded' + wire, _ = posts[0] + checked = generation_capability_parameters(config, 'image', job.adapterParameters, has_images=editing) + assert {key: str(value) for key, value in checked.items() if key not in {'width', 'height'}} == { + key: str(value) for key, value in wire.items() if key not in {'model', 'prompt'}} + assert wire['size'] == f"{checked['width']}x{checked['height']}" + finally: + probe.close() + + +@pytest.mark.parametrize('entry', ['service', 'http', 'agent']) +def test_fixed_image_sampling_is_checked_before_acceptance(project, entry): + store, pid, session = project + runner, config, probe, queries, posts = image_runtime( + project, 0, 'Tongyi-MAI/Z-Image-Turbo', editing=False, sampling_maximum=8) + command = generation_command(project, config) + before = store.snapshot(pid) + try: + receipt = submit(entry, project, runner, probe, command) + assert receipt['status'] == 'rejected', receipt + assert len(queries) == 1 and posts == [] + after = store.snapshot(pid) + assert not after.jobs + assert after.generationSpecs == before.generationSpecs and after.assets == before.assets + finally: + probe.close() + + +@pytest.mark.parametrize('entry', ['service', 'http', 'agent']) +@pytest.mark.parametrize('editing', [False, True]) +def test_minimax_fixed_response_format_is_checked_before_acceptance(project, entry, editing): + store, pid, session = project + configs = ModelConfigs(store.local.repository) + config = configs.save('image', 'https://api.minimax.cn/v1', 'MiniMax-H3-Image', 'test-key', + protocol='openai', capability_auto_check=True) + operation, profile, path = (('image_edit', 'image_to_image', '/v1/images/edits') if editing + else ('image_generation', 'text_to_image', '/v1/images/generations')) + declaration = document(operation, profile, path, 'image', model=config['modelId']) + declaration['models'][0]['operations'][operation]['profiles'][0]['routes'][0]['request'] = { + 'parameters': {'response_format': {'support': 'supported', 'enum': ['b64_json']}}} + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=declaration))) + posts = [] + def generate(request): + posts.append(request) + return httpx.Response(500) + runner = CreationGeneration(store, configs, availability=ModelAvailability(configs, probe)) + runner.provider = GenerationProviders(runner.store, httpx.MockTransport(generate)) + sources = [upload(store, str(pid), session, 'source.png', png((128, 128)))] if editing else [] + command = generation_command(project, config, references=sources) + before = store.snapshot(pid) + try: + receipt = submit(entry, project, runner, probe, command) + assert receipt['status'] == 'rejected', receipt + assert receipt['issues'][0]['code'] == 'capability_unavailable' + after = store.snapshot(pid) + assert not after.jobs and posts == [] + assert after.generationSpecs == before.generationSpecs and after.assets == before.assets + finally: + probe.close() + + +def submit(entry, project, runner, probe, command): + store, pid, session = project + async def asynchronous(): + try: + if entry == 'agent': + arguments = command.model_dump(mode='json', exclude={'commandId', 'projectId'}) + receipt = await AgentTools(store, runner, runner.configs).execute( + pid, uuid4(), session, 'submit', 'execute_command', arguments, [], None) + else: + receipt = (await runner.execute(command, session=session)).model_dump(mode='json') + await settled(runner) + return receipt + finally: + await runner.close() + if entry != 'http': + return asyncio.run(asynchronous()) + app = create_app(store.local, generation_provider=runner.provider, capability_probe=probe) + with TestClient(app, headers={'X-OpenDirector-Request': '1', 'X-OpenDirector-Session': session}) as client: + response = client.post(f'/api/creation/projects/{pid}/commands', json=command.model_dump(mode='json')) + assert response.status_code == 200, response.text + receipt = response.json() + if receipt['status'] == 'accepted': + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + jobs = store.snapshot(pid).jobs + if jobs and all(job.status in {'succeeded', 'failed'} for job in jobs): + break + time.sleep(.01) + assert jobs and all(job.status in {'succeeded', 'failed'} for job in jobs), 'Generation did not finish' + return receipt + + +@pytest.mark.parametrize('voice,normalization,expected', [ + ('AF_HEART', 'lower', 'accepted'), ('DEFAULT', 'strip_lower', 'accepted'), + (None, 'strip_lower', 'accepted'), + ('not-a-speaker', 'lower', 'rejected'), +]) +def test_speech_enum_admission_preserves_the_serialized_voice(project, voice, normalization, expected): + store, pid, session = project + configs = ModelConfigs(store.local.repository) + config = configs.save('audio', 'http://speech.test/v1', 'test-speech', '', + connection_type='self_hosted', protocol='openai', auth_mode='none') + declaration = document('speech', 'text_to_speech', '/v1/audio/speech', 'audio', model=config['modelId']) + declaration['models'][0]['operations']['speech']['profiles'][0]['routes'][0]['request'] = { + 'parameters': {'voice': {'type': 'string', 'enum': ['default', 'af_heart', ''], + 'enum_normalization': normalization}}} + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=declaration))) + posts = [] + def generate(request): + assert request.method == 'POST' and request.url.path == '/v1/audio/speech' + posts.append(json.loads(request.read())) + return httpx.Response(200, content=wav()) + runner = CreationGeneration(store, configs, availability=ModelAvailability(configs, probe)) + runner.provider = GenerationProviders(runner.store, httpx.MockTransport(generate)) + try: + command = generation_command(project, config, parameters={'voice': voice}) + receipt = submit('service', project, runner, probe, command) + assert receipt['status'] == expected, receipt + if expected == 'accepted': + job = store.snapshot(pid).jobs[0] + assert job.status == 'succeeded', job + assert job.inputSnapshot.spec.parameters.voice == voice + assert posts[0]['voice'] == (voice if voice is not None else 'default') + else: + assert not store.snapshot(pid).jobs and not posts + assert receipt['issues'][0]['code'] == 'capability_unavailable' + finally: + probe.close() + + +def test_metadata_audio_cannot_satisfy_a_required_chat_audio_input(project): + store, pid, session = project + configs = ModelConfigs(store.local.repository) + config = configs.save('agent', 'http://chat.test/v1', 'test-chat', '', + connection_type='self_hosted', protocol='openai', auth_mode='none') + declaration = document(model=config['modelId']) + declaration['models'][0]['operations']['chat']['profiles'][0]['required_inputs'] = ['text', 'audio'] + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=declaration))) + posts = [] + def generate(request): + posts.append(request) + return httpx.Response(200, json={'choices': [{'message': {'content': 'Should not run'}}]}) + runner = CreationGeneration(store, configs, availability=ModelAvailability(configs, probe)) + runner.provider = GenerationProviders(runner.store, httpx.MockTransport(generate)) + source = upload(store, str(pid), session, 'reference.wav', wav()) + try: + receipt = submit('service', project, runner, probe, generation_command(project, config, references=[source])) + assert receipt['status'] == 'rejected', receipt + assert receipt['issues'][0]['code'] == 'capability_unavailable' + assert not store.snapshot(pid).jobs and not posts + finally: + probe.close() + + +@pytest.mark.parametrize('entry', ['service', 'http', 'agent']) +@pytest.mark.parametrize('maximum', [1, 4]) +@pytest.mark.parametrize('over_limit', [False, True]) +def test_declared_image_counts_apply_to_every_generation_entry(project, entry, maximum, over_limit): + store, pid, session = project + runner, config, probe, queries, posts = image_runtime(project, maximum) + count = maximum + int(over_limit) + images = [png(color=(index * 30, 0, 255)) for index in range(count)] + sources = [upload(store, str(pid), session, f'{index}.png', data) for index, data in enumerate(images)] + command = generation_command(project, config, references=sources) + before = store.snapshot(pid) + try: + receipt = submit(entry, project, runner, probe, command) + after = store.snapshot(pid) + assert len(queries) == 1 + assert after.references == before.references + if over_limit: + assert receipt['status'] == 'rejected', receipt + assert not after.jobs and posts == [] + assert after.generationSpecs == before.generationSpecs + assert after.canvasItems == before.canvasItems and after.assets == before.assets + assert after.results == before.results + else: + assert receipt['status'] == 'accepted', receipt + assert len(after.jobs) == len(posts) == 1 + assert after.jobs[0].status == 'succeeded', after.jobs[0] + assert after.jobs[0].inputSnapshot.spec.prompt == before.generationSpecs[0].prompt + assert after.jobs[0].inputSnapshot.model.inputRules.maxImages == maximum + assert [str(item.asset.assetId) for item in after.jobs[0].inputSnapshot.references] == sources + fields, files = posts[0] + assert fields['model'] == config['modelId'] + assert [data for _, data in files] == images + declared = generation_capability_parameters(config, 'image', after.jobs[0].adapterParameters, has_images=True) + assert {key: str(value) for key, value in declared.items() if key not in {'width', 'height'}} == { + key: value for key, value in fields.items() if key not in {'model', 'prompt'}} + assert len(after.results) == 1 + finally: + probe.close() + + +def test_unmatched_alias_does_not_apply_another_models_image_limit(project): + store, pid, session = project + runner, config, probe, _, posts = image_runtime(project, 1, model='served-image-alias-v2', declared_model='served-image-alias') + sources = [upload(store, str(pid), session, f'{index}.png', png(color=color)) + for index, color in enumerate(('blue', 'green'))] + try: + receipt = submit('service', project, runner, probe, generation_command(project, config, references=sources)) + assert receipt['status'] == 'accepted', receipt + assert store.snapshot(pid).jobs[0].status == 'succeeded' + assert posts[0][0]['model'] == 'served-image-alias-v2' and len(posts[0][1]) == 2 + finally: + probe.close() + + +def test_zimage_single_image_reaches_the_image_edit_adapter(project): + store, pid, session = project + runner, config, probe, _, posts = image_runtime(project, 1, model='Tongyi-MAI/Z-Image-Turbo') + data = png() + source = upload(store, str(pid), session, 'source.png', data) + try: + receipt = submit('service', project, runner, probe, generation_command(project, config, references=[source])) + assert receipt['status'] == 'accepted', receipt + assert store.snapshot(pid).jobs[0].status == 'succeeded', store.snapshot(pid).jobs[0] + assert len(posts) == 1 and posts[0][0]['model'] == config['modelId'] + assert posts[0][0]['response_format'] == 'b64_json' + assert [data for _, data in posts[0][1]] == [data] + finally: + probe.close() diff --git a/tests/test_result_download_security.py b/tests/test_result_download_security.py index a174da7e..a3ec25b9 100644 --- a/tests/test_result_download_security.py +++ b/tests/test_result_download_security.py @@ -79,6 +79,20 @@ def test_dns_names_and_mixed_answers_cannot_hide_private_addresses(resolve, addr assert len(calls) == 1 +@pytest.mark.parametrize('address', ['198.18.1.38', '2001:2::122']) +def test_fake_ip_dns_rejection_explains_recovery_without_another_request(resolve, address): + resolve[0][:] = [address] + adapter, calls = image_from('https://cdn.test/result.png?signature=private') + with pytest.raises(GenerationError) as error: + generate(adapter) + details = error.value.details + assert (details.stage, details.category) == ('download', 'protocol') + assert '非公网地址' in details.message + assert '域名解析' in details.suggestion + assert 'signature' not in str(error.value) and CONFIG['apiKey'] not in str(error.value) + assert [call.method for call in calls] == ['POST'] + + def test_public_download_pins_checked_ip_and_preserves_host_tls_and_signed_path(resolve): adapter, calls = image_from('https://cdn.test/result.png?signature=keep%2Bme') assert generate(adapter).data == png() diff --git a/tests/test_tool_model_errors.py b/tests/test_tool_model_errors.py new file mode 100644 index 00000000..8f4273c1 --- /dev/null +++ b/tests/test_tool_model_errors.py @@ -0,0 +1,53 @@ +"""Missing tool models retain actionable reasons at the real HTTP boundary.""" +from uuid import uuid4 + +from fastapi.testclient import TestClient +import pytest + +from opendirector.integrations.model_adapters import OpenAIImages, VllmOmniSegmentation +from opendirector_api.main import create_app +from test_tool_tasks import project, png, upload + + +@pytest.mark.parametrize('language', ['zh-CN', 'en-US']) +@pytest.mark.parametrize('operation,capability,chinese,english', [ + ('resize', 'image.edit', '图生图/图片编辑', 'image editing'), + ('selection', 'image.segment', '图像分割', 'image segmentation'), +]) +def test_missing_tool_model_http_names_required_capability_without_creating_work( + project, monkeypatch, language, operation, capability, chinese, english, +): + store, project_id, session = project + source = upload(store, project_id, session, 'source.png', png((64, 48))) + before = store.snapshot(project_id) + calls = [] + + async def unexpected_provider(*args, **kwargs): + calls.append(True) + raise AssertionError('A missing model must not submit a provider request') + + monkeypatch.setattr(OpenAIImages, 'image', unexpected_provider) + monkeypatch.setattr(VllmOmniSegmentation, 'segment', unexpected_provider) + with TestClient(create_app(store.local), headers={ + 'X-OpenDirector-Request': '1', 'X-OpenDirector-Session': session, + 'Accept-Language': language, + }) as client: + path = f'/api/creation/projects/{project_id}' + if operation == 'resize': + response = client.post(path + '/tool-tasks', json={ + 'id': str(uuid4()), 'tool': 'multi-size', 'sourceAssetIds': [source], + 'parameters': {'aspectRatio': 'custom', 'outputWidth': 128, 'outputHeight': 128}, + }) + else: + response = client.post(path + '/image-selections', json={ + 'sourceAssetId': source, 'boxes': [{'x': 10, 'y': 8, 'width': 22, 'height': 24}], + }) + assert response.status_code == 409, response.text + expected = (f'没有可用的「{chinese}」模型，请在“模型接入”中添加。' if language == 'zh-CN' + else f'No {english} model is available. Add one in Model connections.') + assert response.json()['detail'] == expected + assert response.json()['code'] == 'tool_model_missing' + assert response.json()['requiredCapability'] == capability + assert client.get(path + '/tool-tasks').json()['tasks'] == [] + assert store.snapshot(project_id) == before + assert calls == [] diff --git a/tests/test_tool_native_mask.py b/tests/test_tool_native_mask.py new file mode 100644 index 00000000..e9767537 --- /dev/null +++ b/tests/test_tool_native_mask.py @@ -0,0 +1,763 @@ +"""Real tool acceptance, serialized image roles and preserved output pixels.""" +import asyncio +import base64 +from copy import deepcopy +from email import policy as email_policy +from email.parser import BytesParser +from io import BytesIO +from unittest.mock import Mock +from uuid import uuid4 + +import httpx +from PIL import Image +import pytest + +from opendirector.application.creation_tool_assets import CreationToolAssets +from opendirector.application.model_availability import ModelAvailability +from opendirector.application.tool_tasks import ToolTasks +from opendirector.domain.capabilities import ImageRequest +from opendirector.domain.capability_messages import constraint_rejection, english_capability_message +from opendirector.domain.image_inputs import image_input_policy +from opendirector.domain.projects import ProjectError +from opendirector.integrations.capability_probe import CapabilityProbe +from opendirector.integrations.model_adapters import adapter_for +from opendirector.integrations import model_adapters +from opendirector.media import processing as media +from test_tool_tasks import project, configure, upload, png, output_image, output_rgba, SAM3, ZIMAGE, segmentation_server +from test_model_availability import document + + +MODEL = ('image', 'http://image.test/v1', 'test-native-edit') + + +def native_policy(config): + return {**image_input_policy(config), 'nativeMask': { + 'field': 'mask_image', 'referenceField': 'reference_image', 'encoding': 'grayscale_white_edit', + 'sameSize': True, 'minImages': 1, 'maxImages': 1, 'maxReferenceImages': 1, + 'widthMultiple': 16, 'heightMultiple': 16, 'minWidth': 32, 'minHeight': 32, + 'maxPixels': 2048 * 2048}} + + +def multipart(request): + message = BytesParser(policy=email_policy.default).parsebytes( + b'Content-Type: ' + request.headers['content-type'].encode() + b'\r\n\r\n' + request.read()) + fields, files = {}, [] + for part in message.iter_parts(): + name = part.get_param('name', header='content-disposition') + payload = part.get_payload(decode=True) + if part.get_filename(): + files.append((name, payload)) + else: + fields[name] = payload.decode() + return fields, files + + +def image_server(calls): + def respond(request): + assert request.url.path == '/v1/images/edits' + fields, files = multipart(request) + calls.append((fields, files)) + size = tuple(map(int, fields['size'].split('x'))) + return httpx.Response(200, json={'data': [{'b64_json': base64.b64encode(png(size, 'red')).decode()}]}) + return respond + + +def runtime(store, config, configs, calls, selected_policy): + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(404))) + availability = ModelAvailability(configs, probe) + # Only the remote declaration is doubled; actual policy validation, tools, + # persistence, provider adapter and result import all run unchanged. + availability.image_execution_policy = lambda _: deepcopy(selected_policy) + return ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, concurrency=1, + availability=availability, transport=httpx.MockTransport(image_server(calls))), probe + + +def declared_runtime(store, configs, calls, constraints): + config = configs.list()[0] + declaration = document('image_edit', 'image_to_image', '/v1/images/edits', 'image', model=config['modelId']) + profile = declaration['models'][0]['operations']['image_edit']['profiles'][0] + profile['required_inputs'] = ['text', 'image'] + profile['routes'][0]['request'] = constraints + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=declaration))) + tools = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, concurrency=1, + availability=ModelAvailability(configs, probe), transport=httpx.MockTransport(image_server(calls))) + return tools, probe, declaration + + +@pytest.mark.parametrize('constraints', [ + {'max_output_pixels': 512 * 512}, + {'parameters': {'width': {'support': 'supported', 'type': 'integer', 'enum': [64]}}}, +]) +@pytest.mark.parametrize('model', [MODEL, ('image', 'https://api.minimax.cn/v1', 'MiniMax-H3-Image')]) +def test_ordinary_edit_rejects_actual_render_size_before_task_or_post(project, constraints, model): + store, pid, session = project + configs = configure(store, model) + source = upload(store, pid, session, 'source.png', png((64, 64))) + calls = [] + tools, probe, _ = declared_runtime(store, configs, calls, constraints) + async def run(): + try: + with pytest.raises(ProjectError) as failure: + await tools.submit(pid, {'id': str(uuid4()), 'tool': 'view-control', + 'sourceAssetIds': [source], 'parameters': {}}, session) + assert failure.value.code == 'capability_unavailable' + assert tools.list(pid) == [] and calls == [] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def test_ordinary_edit_freezes_actual_render_contract_and_keeps_output_size(project): + store, pid, session = project + configs = configure(store, MODEL) + source = upload(store, pid, session, 'source.png', png((64, 64))) + constraints = {'max_output_pixels': 1024 * 1024, + 'parameters': {'width': {'support': 'supported', 'enum': [1024]}}} + calls = [] + tools, probe, declaration = declared_runtime(store, configs, calls, constraints) + value = {'id': str(uuid4()), 'tool': 'view-control', 'sourceAssetIds': [source], 'parameters': {}} + async def run(): + try: + await tools.limit.acquire() + accepted = await tools.submit(pid, value, session) + frozen = deepcopy(accepted['imageExecutionPolicy']) + assert frozen['requestConstraints']['max_output_pixels'] == 1024 * 1024 + declaration['models'][0]['operations']['image_edit']['profiles'][0]['routes'][0]['request'] = {'max_output_pixels': 1} + tools.availability.refresh([configs.list()[0]['id']]) + tools.availability.prepare = Mock(side_effect=AssertionError('accepted task queried new declaration')) + tools.availability.require = Mock(side_effect=AssertionError('accepted task checked current availability')) + tools.limit.release() + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, accepted['id']) + assert result['status'] == 'succeeded', result['error'] + assert result['imageExecutionPolicy'] == frozen + assert (await tools.submit(pid, value, session))['id'] == accepted['id'] + assert len(calls) == 1 and calls[0][0]['size'] == '1024x1024' + assert output_image(store, pid, result).size == (64, 64) + finally: + if tools.limit.locked(): + tools.limit.release() + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def test_batch_enhancement_checks_each_render_before_accepting_any_source(project): + store, pid, session = project + configs = configure(store, MODEL) + sources = [upload(store, pid, session, f'{index}.png', png(size)) + for index, size in enumerate(((64, 64), (128, 64)))] + calls = [] + tools, probe, _ = declared_runtime(store, configs, calls, {'max_output_pixels': 1024 * 1024}) + async def run(): + try: + with pytest.raises(ProjectError) as failure: + await tools.submit(pid, {'id': str(uuid4()), 'tool': 'image-hd', 'sourceAssetIds': sources, + 'parameters': {'kind': 'pro', 'resolution': '2K'}}, session) + assert failure.value.code == 'capability_unavailable' + assert tools.list(pid) == [] and calls == [] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +@pytest.mark.parametrize('tool,parameters,output_size', [ + ('image-presets', {'preset': 'three-view'}, (2048, 1152)), + ('outpaint', {'right': 1, 'outputWidth': 512, 'outputHeight': 256}, (512, 256)), +]) +def test_reframing_checks_adapter_render_size_and_preserves_requested_output(project, tool, parameters, output_size): + store, pid, session = project + configs = configure(store, MODEL) + source = upload(store, pid, session, 'source.png', png((64, 64))) + calls = [] + constraints = {'parameters': {'width': {'support': 'supported', 'enum': [1536]}, + 'height': {'support': 'supported', 'enum': [1024]}}} + tools, probe, _ = declared_runtime(store, configs, calls, constraints) + async def run(): + try: + task = await tools.submit(pid, {'id': str(uuid4()), 'tool': tool, 'sourceAssetIds': [source], + 'parameters': parameters}, session) + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, task['id']) + assert result['status'] == 'succeeded', result['error'] + assert len(calls) == 1 and calls[0][0]['size'] == '1536x1024' + assert output_image(store, pid, result).size == output_size + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +@pytest.mark.parametrize('legacy', [False, True]) +def test_adapter_checks_frozen_render_limit_without_changing_old_requests(legacy): + config = {'kind': 'image', 'url': MODEL[1], 'modelId': MODEL[2], 'protocol': 'openai'} + policy = {**image_input_policy(config), 'requestConstraints': {'max_output_pixels': 512 * 512}} + calls = [] + adapter = adapter_for(config, httpx.MockTransport(image_server(calls)), image_policy=policy, legacy_image_policy=legacy) + request = ImageRequest('edit', 64, 64, image=png()) + if legacy: + asyncio.run(adapter.image(config, request)) + assert len(calls) == 1 and calls[0][0]['size'] == '1024x1024' + else: + with pytest.raises(ProjectError) as failure: + asyncio.run(adapter.image(config, request)) + assert failure.value.code == 'capability_unavailable' + assert str(failure.value) == '图片尺寸不符合服务要求。' + assert calls == [] + + +@pytest.mark.parametrize('tool,parameters', [ + ('smart-edit', {'editMode': 'selection', 'prompt': 'red'}), + ('inpainting-remove', {}), ('view-control', {}), ('lighting', {}), + ('product-replacement', {}), ('panorama', {}), + ('multi-size', {'outputWidth': 128.7, 'outputHeight': 64.5}), + ('image-presets', {'preset': 'storyboard'}), + ('image-hd', {'kind': 'pro', 'resolution': '2K', 'detailStrength': .12345}), +]) +def test_checked_recipe_parameters_equal_serialized_provider_fields(project, tool, parameters): + store, pid, session = project + configs = configure(store, ZIMAGE) + source = upload(store, pid, session, 'source.png', png((64, 64))) + value = {'id': str(uuid4()), 'tool': tool, 'sourceAssetIds': [source], 'parameters': parameters} + if tool in {'smart-edit', 'inpainting-remove'}: + value['maskAssetId'] = upload(store, pid, session, 'mask.png', png((64, 64), 'white', 'L')) + if tool == 'product-replacement': + value['sourceAssetIds'].append(upload(store, pid, session, 'product.png', png((32, 32), 'green'))) + value['boxes'] = [{'x': 16, 'y': 16, 'width': 32, 'height': 32}] + if tool == 'image-hd': + value['sourceAssetIds'].append(upload(store, pid, session, 'second.png', png((1920, 960)))) + calls, checked = [], [] + tools, probe, _ = declared_runtime(store, configs, calls, {}) + require = tools.availability.require + def observe(config, request, *args, **kwargs): + if request.profile == 'image_to_image': + checked.append(dict(request.parameters)) + return require(config, request, *args, **kwargs) + tools.availability.require = observe + async def run(): + try: + task = await tools.submit(pid, value, session) + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, task['id']) + assert result['status'] == 'succeeded', result['error'] + assert len(calls) == (4 if tool == 'image-presets' else 2 if tool == 'image-hd' else 1) + for index, (fields, _) in enumerate(calls): + actual = {key: val for key, val in fields.items() if key not in {'model', 'prompt'}} + actual['width'], actual['height'] = map(int, actual['size'].split('x')) + if 'strength' in actual: + actual['strength'] = float(actual['strength']) + assert actual == checked[index if tool == 'image-hd' else 0] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +@pytest.mark.parametrize('state', ['no_model', 'paused', 'unsupported']) +def test_same_aspect_resize_needs_no_remote_model_or_admission(project, state): + store, pid, session = project + configs = configure(store, *([] if state == 'no_model' else [MODEL])) + source = upload(store, pid, session, 'source.png', png((64, 64))) + calls = [] + if state == 'no_model': + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(404))) + tools = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, + availability=ModelAvailability(configs, probe), transport=httpx.MockTransport(image_server(calls))) + else: + tools, probe, declaration = declared_runtime(store, configs, calls, {'max_output_pixels': 1}) + operation = declaration['models'][0]['operations']['image_edit'] + if state == 'paused': + operation['execution']['state'] = 'paused' + else: + operation.update(support='unsupported', profiles=[]) + tools.availability.refresh([configs.list()[0]['id']]) + entry = next(item for item in tools.catalog()['tools'] if item['id'] == 'multi-size') + assert entry['available'] and 'reason' not in entry + assert len(entry['models']['image.edit']) == int(state != 'no_model') + if state != 'no_model': + assert entry['models']['image.edit'][0]['availability']['status'] == 'unavailable' + require, prepare = tools.availability.require, tools.availability.prepare_many + tools.availability.require = Mock(side_effect=AssertionError('local resize requested model admission')) + def prepare_empty(items): + assert list(items) == [] + return [] + tools.availability.prepare_many = prepare_empty + value = {'id': str(uuid4()), 'tool': 'multi-size', 'sourceAssetIds': [source], + 'parameters': {'outputWidth': 128, 'outputHeight': 128}} + async def run(): + try: + task = await tools.submit(pid, value, session) + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, task['id']) + assert result['status'] == 'succeeded', result['error'] + assert result['requires'] == [] and result['modelConfigIds'] == {} + assert output_image(store, pid, result).size == (128, 128) + assert calls == [] + assert (await tools.submit(pid, value, session))['id'] == task['id'] + # Opening the panel does not authorize a remote aspect-ratio change. + tools.availability.require, tools.availability.prepare_many = require, prepare + with pytest.raises(ProjectError): + await tools.submit(pid, {**value, 'id': str(uuid4()), + 'parameters': {'outputWidth': 256, 'outputHeight': 128}}, session) + assert len(tools.list(pid)) == 1 and calls == [] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +@pytest.mark.parametrize('reason,details,chinese,english', [ + ('parameter_value_unsupported', {'parameter': 'size'}, '图片尺寸不符合服务要求。', 'Image dimensions do not meet the service requirements.'), + ('parameter_value_unsupported', {'parameter': 'width'}, '图片宽度不符合服务要求。', 'Image width does not meet the service requirements.'), + ('required_inputs_missing', {'input': 'image'}, '请添加图片。', 'Add images.'), +]) +def test_frozen_constraint_rejection_keeps_specific_bilingual_reason(reason, details, chinese, english): + code, message = constraint_rejection(reason, details) + assert code == 'capability_unavailable' and message == chinese + assert english_capability_message(message) == english + + +@pytest.mark.parametrize('allowed', [False, True]) +def test_optional_layer_completion_checks_its_actual_render_parameters(project, allowed): + store, pid, session = project + configs = configure(store, ZIMAGE, SAM3) + source = upload(store, pid, session, 'source.png', png()) + calls, segments = [], [] + constraints = {'parameters': {'width': {'support': 'supported', 'enum': [256 if allowed else 64]}, + 'strength': {'support': 'supported', 'enum': [.85]}}} + tools, probe, _ = declared_runtime(store, configs, calls, constraints) + segment = segmentation_server(segments, {'person': [(0.9, (10, 8, 30, 32))]}) + respond = image_server(calls) + tools.transport = httpx.MockTransport(lambda request: respond(request) if request.url.path == '/v1/images/edits' else segment(request)) + async def run(): + try: + task = await tools.submit(pid, {'id': str(uuid4()), 'tool': 'layer-decompose', 'sourceAssetIds': [source], + 'parameters': {'mode': 'description', 'prompt': 'person'}}, session) + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, task['id']) + assert result['status'] == 'succeeded', result['error'] + assert segments == ['person'] + assert result['backgroundCompletion'] == ('model' if allowed else 'transparent') + assert len(calls) == int(allowed) + if allowed: + assert calls[0][0]['size'] == '256x256' and calls[0][0]['strength'] == '0.850' + else: + assert output_rgba(store, pid, result).getpixel((20, 20))[3] == 0 + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +@pytest.mark.parametrize('tool', ['smart-edit', 'combined', 'inpainting-remove', 'outpaint', 'product-replacement', 'multi-size']) +def test_native_selection_serializes_actual_roles_and_preserves_unselected_pixels(project, tool): + store, pid, session = project + configs = configure(store, MODEL) + config = configs.list()[0] + source = upload(store, pid, session, 'source.png', png((64, 64))) + mask = Image.new('RGBA', (64, 64), (0, 0, 0, 0)) + mask.paste((255, 255, 255, 255), (16, 16, 48, 48)) + mask_id = upload(store, pid, session, 'selection.png', media.png(mask)) + value = {'id': str(uuid4()), 'tool': 'smart-edit' if tool == 'combined' else tool, + 'sourceAssetIds': [source], 'parameters': {}} + if tool in {'smart-edit', 'combined', 'inpainting-remove'}: + value.update(maskAssetId=mask_id, parameters={'editMode': 'combined' if tool == 'combined' else 'selection', 'prompt': 'make it red'}) + if tool == 'combined': + value['annotationAssetId'] = upload(store, pid, session, 'annotation.png', png((64, 64), 'green')) + if tool == 'outpaint': + value['parameters'] = {'right': 1, 'outputWidth': 128, 'outputHeight': 64} + if tool == 'multi-size': + value['parameters'] = {'outputWidth': 128, 'outputHeight': 64} + if tool == 'product-replacement': + value['sourceAssetIds'].append(upload(store, pid, session, 'product.png', png((32, 32), 'green'))) + value['boxes'] = [{'x': 16, 'y': 16, 'width': 32, 'height': 32}] + calls = [] + selected_policy = native_policy(config) + tools, probe = runtime(store, config, configs, calls, selected_policy) + async def run(): + try: + await tools.limit.acquire() + accepted = await tools.submit(pid, value, session) + assert accepted['imageExecution']['mode'] == 'native_mask' + frozen = deepcopy(accepted['imageExecutionPolicy']) + selected_policy['nativeMask'] = None + tools.availability.image_execution_policy = Mock(side_effect=AssertionError('accepted work queried new policy')) + tools.limit.release() + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, accepted['id']) + assert result['status'] == 'succeeded', result['error'] + assert result['imageExecutionPolicy'] == frozen + replay = await tools.submit(pid, value, session) + assert replay['id'] == accepted['id'] and len(calls) == 1 + tools.availability.image_execution_policy.assert_not_called() + return result + finally: + await tools.close() + try: + task = asyncio.run(run()) + finally: + probe.close() + fields, files = calls[0] + assert [name for name, _ in files] == ['image', 'mask_image'] + (['reference_image'] if tool == 'combined' else []) + primary, selection = [Image.open(BytesIO(data)) for _, data in files[:2]] + expected_size = (128, 64) if tool in {'outpaint', 'multi-size'} else (64, 64) + assert primary.size == selection.size == expected_size + assert selection.mode == 'L' and fields['size'] == f'{expected_size[0]}x{expected_size[1]}' + if tool == 'outpaint': + assert selection.getpixel((32, 32)) == 0 and selection.getpixel((96, 32)) == 255 + assert output_image(store, pid, task).getpixel((32, 32)) == (0, 0, 255) + elif tool == 'multi-size': + assert selection.getpixel((64, 32)) == 0 and selection.getpixel((0, 32)) == 255 + assert output_image(store, pid, task).getpixel((64, 32)) == (0, 0, 255) + else: + assert selection.getpixel((0, 0)) == 0 and selection.getpixel((32, 32)) == 255 + assert output_image(store, pid, task).getpixel((0, 0)) == (0, 0, 255) + if tool == 'product-replacement': + assert primary.getpixel((32, 32)) == (0, 128, 0) # Two sources composed into one transmitted image. + + +@pytest.mark.parametrize('reason', ['unknown', 'geometry', 'output_size', 'annotation_only', 'no_reference_role']) +def test_unsuitable_native_selection_chooses_existing_flow_once(project, reason): + store, pid, session = project + configs = configure(store, MODEL) + config = configs.list()[0] + size = (65, 64) if reason == 'geometry' else (64, 64) + source = upload(store, pid, session, 'source.png', png(size)) + mask = upload(store, pid, session, 'selection.png', png(size, 'white', 'L')) + selected_policy = native_policy(config) + value = {'id': str(uuid4()), 'tool': 'smart-edit', 'sourceAssetIds': [source], 'maskAssetId': mask, + 'parameters': {'editMode': 'selection', 'prompt': 'red'}} + if reason == 'unknown': + selected_policy['nativeMask'] = None + if reason == 'annotation_only': + value['parameters']['editMode'] = 'annotation' + if reason == 'output_size': + value.update(tool='outpaint', parameters={'right': 1, 'outputWidth': 256, 'outputHeight': 128}) + if reason == 'no_reference_role': + selected_policy['nativeMask']['referenceField'] = None + value['parameters']['editMode'] = 'combined' + value['annotationAssetId'] = upload(store, pid, session, 'annotation.png', png(size, 'green')) + calls = [] + tools, probe = runtime(store, config, configs, calls, selected_policy) + async def run(): + try: + task = await tools.submit(pid, value, session) + assert task['imageExecution']['mode'] == 'legacy' + await asyncio.gather(*tools.jobs.values()) + assert tools.get(pid, task['id'])['status'] == 'succeeded' + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + assert len(calls) == 1 and all(name == 'image' for name, _ in calls[0][1]) + + +@pytest.mark.parametrize('model,count', [('compatible-image', 9), ('Qwen/Qwen-Image-2.1', 5), ('gpt-image-1', 17)]) +def test_over_limit_adapter_inputs_are_rejected_without_slicing_or_http(model, count): + config = {'kind': 'image', 'url': MODEL[1], 'modelId': model, 'protocol': 'openai'} + calls = [] + adapter = adapter_for(config, httpx.MockTransport(image_server(calls))) + request = ImageRequest('edit', 64, 64, image=png(), references=(png(),) * (count - 1)) + with pytest.raises(ProjectError) as failure: + asyncio.run(adapter.image(config, request)) + assert failure.value.code == 'image_input_count' + assert len(request.references) == count - 1 and calls == [] + + +def test_compatible_adapter_sends_every_image_allowed_by_the_shared_policy(): + config = {'kind': 'image', 'url': MODEL[1], 'modelId': 'compatible-image', 'protocol': 'openai'} + calls = [] + adapter = adapter_for(config, httpx.MockTransport(image_server(calls))) + images = [png(color=color) for color in ('blue', 'green', 'red')] + asyncio.run(adapter.image(config, ImageRequest('edit', 64, 64, image=images[0], references=tuple(images[1:])))) + assert [data for _, data in calls[0][1]] == images + + +@pytest.mark.parametrize('model,count,sent', [('compatible-image', 9, 1), ('Qwen/Qwen-Image-2.1', 6, 4), ('gpt-image-1', 18, 16)]) +def test_legacy_adapter_reconstructs_the_accepted_request_without_new_limits(model, count, sent): + config = {'kind': 'image', 'url': MODEL[1], 'modelId': model, 'protocol': 'openai'} + calls = [] + # New byte limits must not be retroactively applied to accepted old work. + policy = {**image_input_policy(config), 'imageBytes': 1, 'requestBytes': 1} + adapter = adapter_for(config, httpx.MockTransport(image_server(calls)), image_policy=policy, legacy_image_policy=True) + request = ImageRequest('edit', 64, 64, image=png(), references=(png(),) * (count - 1)) + asyncio.run(adapter.image(config, request)) + assert len(calls) == 1 and len(calls[0][1]) == sent + assert len(request.references) == count - 1 + + +@pytest.mark.parametrize('tool,count', [('outpaint', 2), ('product-replacement', 3)]) +def test_extra_recipe_sources_are_rejected_before_acceptance(project, tool, count): + store, pid, session = project + configs = configure(store, MODEL) + config = configs.list()[0] + sources = [upload(store, pid, session, f'{index}.png', png()) for index in range(count)] + calls = [] + tools, probe = runtime(store, config, configs, calls, native_policy(config)) + async def run(): + try: + with pytest.raises(ProjectError) as failure: + await tools.submit(pid, {'id': str(uuid4()), 'tool': tool, 'sourceAssetIds': sources, 'parameters': {}}, session) + assert failure.value.code == 'tool_source_count' + assert tools.list(pid) == [] and calls == [] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def test_native_provider_failure_does_not_retry_with_another_edit_mode(project): + store, pid, session = project + configs = configure(store, MODEL) + config = configs.list()[0] + source = upload(store, pid, session, 'source.png', png((64, 64))) + mask = upload(store, pid, session, 'selection.png', png((64, 64), 'white', 'L')) + calls = [] + tools, probe = runtime(store, config, configs, calls, native_policy(config)) + def reject(request): + calls.append(multipart(request)) + return httpx.Response(400, json={'error': {'message': 'Rejected selection'}}) + tools.transport = httpx.MockTransport(reject) + async def run(): + try: + task = await tools.submit(pid, {'id': str(uuid4()), 'tool': 'smart-edit', 'sourceAssetIds': [source], + 'maskAssetId': mask, 'parameters': {'editMode': 'selection', 'prompt': 'red'}}, session) + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, task['id']) + assert result['status'] == 'failed' and result['imageExecution']['mode'] == 'native_mask' + assert len(result['modelCalls']) == len(calls) == 1 + assert [name for name, _ in calls[0][1]] == ['image', 'mask_image'] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def test_empty_selection_rejects_before_creating_a_task(project): + store, pid, session = project + configs = configure(store, MODEL) + config = configs.list()[0] + source = upload(store, pid, session, 'source.png', png((64, 64))) + mask = upload(store, pid, session, 'selection.png', png((64, 64), 'black', 'L')) + calls = [] + tools, probe = runtime(store, config, configs, calls, native_policy(config)) + async def run(): + try: + with pytest.raises(ProjectError): + await tools.submit(pid, {'id': str(uuid4()), 'tool': 'inpainting-remove', + 'sourceAssetIds': [source], 'maskAssetId': mask, 'parameters': {}}, session) + assert tools.list(pid) == [] and calls == [] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +@pytest.mark.parametrize('maximum', [0, 1]) +def test_tool_admission_uses_declared_roles_and_limits_from_the_actual_route(project, maximum): + store, pid, session = project + configs = configure(store, MODEL) + source = upload(store, pid, session, 'source.png', png((64, 64))) + mask = upload(store, pid, session, 'selection.png', png((64, 64), 'white', 'L')) + annotation = upload(store, pid, session, 'annotation.png', png((64, 64), 'green')) + declaration = document('image_edit', 'image_to_image', '/v1/images/edits', 'image', model=MODEL[2]) + profile = declaration['models'][0]['operations']['image_edit']['profiles'][0] + profile['required_inputs'] = ['text', 'image'] + profile['routes'][0]['request'] = { + 'inputs': {'image': {'support': 'supported', 'max_count': maximum}, + 'mask_image': {'support': 'supported', 'max_count': 1}, + 'reference_image': {'support': 'supported', 'max_count': 1}}, + 'mask': {'support': 'supported', 'source_role': 'image', 'max_source_count': 1, + 'white': 'edit', 'black': 'preserve', 'binarize_threshold': 0.5, + 'requires_same_size': True, 'geometry': {'state': 'known', 'multiple_of': 16}, + 'source_resize': {'max_pixels': 2048 * 2048, 'policy': 'downscale_preserve_aspect_ratio'}}} + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=declaration))) + calls = [] + tools = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, + availability=ModelAvailability(configs, probe), transport=httpx.MockTransport(image_server(calls))) + value = {'id': str(uuid4()), 'tool': 'smart-edit', 'sourceAssetIds': [source], + 'maskAssetId': mask, 'annotationAssetId': annotation, + 'parameters': {'editMode': 'combined', 'prompt': 'red'}} + async def run(): + try: + if not maximum: + with pytest.raises(ProjectError) as error: + await tools.submit(pid, value, session) + assert error.value.code == 'image_input_count' + assert tools.list(pid) == [] and calls == [] + return + task = await tools.submit(pid, value, session) + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, task['id']) + assert result['status'] == 'succeeded', result['error'] + assert result['imageExecution']['roles'] == ['image', 'mask', 'reference'] + assert result['imageExecutionPolicy']['maxImages'] == 1 + assert [name for name, _ in calls[0][1]] == ['image', 'mask_image', 'reference_image'] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +def test_local_upscale_preserves_multiple_recipe_sources(project): + store, pid, session = project + configs = configure(store, MODEL) + sources = [upload(store, pid, session, f'{color}.png', png((64, 64), color)) for color in ('blue', 'green')] + calls = [] + tools, probe = runtime(store, configs.list()[0], configs, calls, native_policy(configs.list()[0])) + async def run(): + try: + task = await tools.submit(pid, {'id': str(uuid4()), 'tool': 'image-hd', 'sourceAssetIds': sources, + 'parameters': {'kind': 'normal', 'resolution': '2K'}}, session) + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, task['id']) + assert result['status'] == 'succeeded' and len(result['outputs']) == 2 + assert result['sourceAssetIds'] == sources and calls == [] + assert [output_image(store, pid, result, index).getpixel((10, 10)) for index in range(2)] == [(0, 0, 255), (0, 128, 0)] + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + + +@pytest.mark.parametrize('state,reason,message', [ + ('paused', 'execution_paused', '服务已暂停，请恢复后重试。'), + ('unsupported', 'operation_unsupported', '该模型不支持图片编辑。'), + ('disabled', 'route_disabled', '服务未开放图片编辑。'), +]) +def test_tool_catalog_keeps_the_specific_cached_unavailability_reason(project, state, reason, message): + store, _, _ = project + configs = configure(store, MODEL) + config = configs.list()[0] + declaration = document('image_edit', 'image_to_image', '/v1/images/edits', 'image', model=MODEL[2], + support='unsupported' if state == 'unsupported' else 'supported') + operation = declaration['models'][0]['operations']['image_edit'] + if state == 'paused': + operation['execution']['state'] = 'paused' + elif state == 'disabled': + operation['profiles'][0]['routes'][0]['exposure'] = 'disabled' + calls = [] + probe = CapabilityProbe(transport=httpx.MockTransport(lambda request: calls.append(request) or httpx.Response(200, json=declaration))) + availability = ModelAvailability(configs, probe) + tools = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, availability=availability) + try: + availability.refresh([config['id']]) + tools.availability.prepare = Mock(side_effect=AssertionError('catalog queried the service')) + entry = next(item for item in tools.catalog()['tools'] if item['id'] == 'outpaint') + assert not entry['available'] and entry['reasonCode'] == reason and entry['reason'] == message + assert len(calls) == 1 + finally: + probe.close() + + +def test_accepted_legacy_tool_keeps_single_annotation_input_without_native_mask(project): + store, pid, session = project + configs = configure(store, MODEL) + config = configs.list()[0] + source = upload(store, pid, session, 'source.png', png((64, 64))) + annotation = upload(store, pid, session, 'annotation.png', png((64, 64), 'green')) + mask = upload(store, pid, session, 'selection.png', png((64, 64), 'white', 'L')) + value = {'id': str(uuid4()), 'tool': 'smart-edit', 'sourceAssetIds': [source], + 'maskAssetId': mask, 'annotationAssetId': annotation, + 'parameters': {'editMode': 'combined', 'prompt': 'red'}} + calls = [] + tools, probe = runtime(store, config, configs, calls, native_policy(config)) + async def run(): + try: + await tools.limit.acquire() + accepted = await tools.submit(pid, value, session) + # This is the persisted shape written by the previous version. + with tools.store.transaction(): + legacy = tools.get(pid, accepted['id']) + legacy.pop('imageExecutionPolicy') + legacy.pop('imageExecution') + tools._write(tools._directory(pid), legacy) + tools.availability.image_execution_policy = Mock(side_effect=AssertionError('legacy task checked new policy')) + tools.limit.release() + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, accepted['id']) + assert result['status'] == 'succeeded', result['error'] + replay = await tools.submit(pid, value, session) + assert replay['id'] == result['id'] and len(calls) == 1 + assert 'imageExecutionPolicy' not in result + tools.availability.image_execution_policy.assert_not_called() + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() + assert [name for name, _ in calls[0][1]] == ['image'] + assert Image.open(BytesIO(calls[0][1][0][1])).getpixel((32, 32)) == (0, 128, 0) + + +@pytest.mark.parametrize('legacy', [False, True]) +def test_cutout_freezes_its_image_policy_and_preserves_accepted_legacy_validation(project, monkeypatch, legacy): + store, pid, session = project + configs = configure(store, ('image', MODEL[1], 'Qwen/Qwen-Image-2.1')) + config = configs.list()[0] + source = upload(store, pid, session, 'source.png', png((64, 64))) + calls = [] + selected_policy = native_policy(config) + tools, probe = runtime(store, config, configs, calls, selected_policy) + value = {'id': str(uuid4()), 'tool': 'cutout', 'sourceAssetIds': [source], 'parameters': {}} + async def run(): + try: + await tools.limit.acquire() + accepted = await tools.submit(pid, value, session) + assert accepted['requires'] == ['image.cutout'] + assert accepted['imageExecutionPolicy']['nativeMask'] is None + if legacy: + with tools.store.transaction(): + saved = tools.get(pid, accepted['id']) + saved.pop('imageExecutionPolicy') + saved.pop('imageExecution') + tools._write(tools._directory(pid), saved) + validate = Mock(side_effect=AssertionError('old cutout acquired new validation rules')) + monkeypatch.setattr(model_adapters, 'validate_image_request', validate) + selected_policy['imageBytes'] = 1 + tools.availability.image_execution_policy = Mock(side_effect=AssertionError('accepted cutout checked current policy')) + tools.limit.release() + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, accepted['id']) + assert result['status'] == 'succeeded', result['error'] + assert (await tools.submit(pid, value, session))['id'] == accepted['id'] + assert len(calls) == 1 and [name for name, _ in calls[0][1]] == ['image'] + assert ('imageExecutionPolicy' not in result) if legacy else result['imageExecutionPolicy']['imageBytes'] > 1 + if legacy: + validate.assert_not_called() + tools.availability.image_execution_policy.assert_not_called() + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() diff --git a/tests/test_tool_optional_image.py b/tests/test_tool_optional_image.py new file mode 100644 index 00000000..62c9f868 --- /dev/null +++ b/tests/test_tool_optional_image.py @@ -0,0 +1,79 @@ +"""Optional background completion fixes its model choice before segmentation.""" +import asyncio +from copy import deepcopy +from uuid import uuid4 + +import httpx +import pytest + +from opendirector.application.creation_tool_assets import CreationToolAssets +from opendirector.application.model_availability import ModelAvailability +from opendirector.application.tool_tasks import ToolTasks +from opendirector.domain.projects import ProjectError +from opendirector.integrations.capability_probe import CapabilityProbe +from test_model_availability import document +from test_tool_tasks import project, configure, upload, png, SAM3, segmentation_server, output_rgba + + +@pytest.mark.parametrize('state', ['paused', 'unsupported', 'added', 'changed', 'legacy']) +def test_layer_background_completion_never_selects_a_later_model_for_new_work(project, state): + store, pid, session = project + configs = configure(store, SAM3) + model = None + if state in {'paused', 'unsupported', 'changed'}: + model = configs.save('image', 'http://image.test/v1', 'image-before', '', connection_type='self_hosted', protocol='openai') + declaration = document('image_edit', 'image_to_image', '/v1/images/edits', 'image', model='image-before', + support='unsupported' if state == 'unsupported' else 'supported') + if state == 'paused': + declaration['models'][0]['operations']['image_edit']['execution']['state'] = 'paused' + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=declaration))) + source = upload(store, pid, session, 'source.png', png()) + mask = upload(store, pid, session, 'mask.png', png(color='white', mode='L')) + segments, edits = [], [] + tools = ToolTasks(store.local, configs, CreationToolAssets(store), projects=store, concurrency=1, + availability=ModelAvailability(configs, probe), + transport=httpx.MockTransport(segmentation_server(segments, {'person': [(0.9, (10, 8, 30, 32))]}, edits=edits))) + value = {'id': str(uuid4()), 'tool': 'layer-decompose', 'sourceAssetIds': [source], + 'parameters': {'mode': 'description', 'prompt': 'person'}} + async def run(): + try: + if state in {'paused', 'unsupported'}: + with pytest.raises(ProjectError) as error: + await tools.submit(pid, {'id': str(uuid4()), 'tool': 'inpainting-remove', + 'sourceAssetIds': [source], 'maskAssetId': mask, 'parameters': {}}, session) + assert error.value.code == ('capability_paused' if state == 'paused' else 'capability_unavailable') + assert not tools.list(pid) and not segments and not edits + await tools.limit.acquire() + accepted = await tools.submit(pid, value, session) + expected = 'model' if state == 'changed' else 'transparent' + assert accepted['backgroundCompletion'] == expected + frozen = deepcopy(accepted['modelSnapshots']) + if state in {'added', 'legacy'}: + if state == 'legacy': + with tools.store.transaction(): + old = tools.get(pid, accepted['id']) + old.pop('backgroundCompletion') + tools._write(tools._directory(pid), old) + configs.save('image', 'http://later.test/v1', 'image-later', '', connection_type='self_hosted', protocol='openai') + elif state == 'changed': + configs.save('image', 'http://later.test/v1', 'image-later', '', model['id'], + connection_type='self_hosted', protocol='openai') + tools.limit.release() + await asyncio.gather(*tools.jobs.values()) + result = tools.get(pid, accepted['id']) + assert result['status'] == 'succeeded', result['error'] + assert result['modelSnapshots'] == frozen and len(result['outputs']) == 2 + assert (await tools.submit(pid, value, session))['id'] == accepted['id'] + assert segments == ['person'] + if state == 'legacy': + assert len(edits) == 1 and 'backgroundCompletion' not in result + assert output_rgba(store, pid, result).getpixel((20, 20))[3] == 255 + else: + assert not edits and output_rgba(store, pid, result).getpixel((20, 20))[3] == 0 + assert result['backgroundCompletion'] == expected + finally: + await tools.close() + try: + asyncio.run(run()) + finally: + probe.close() diff --git a/tests/test_tool_tasks.py b/tests/test_tool_tasks.py index dfb00c4a..77704f5d 100644 --- a/tests/test_tool_tasks.py +++ b/tests/test_tool_tasks.py @@ -505,7 +505,7 @@ def test_qwen_image_edits_with_condition_images_on_its_grid_without_strength(): calls = [] adapter.transport = httpx.MockTransport(images_server(calls)) result = asyncio.run(adapter.image(config, ImageRequest(prompt='换成黄色雨衣', width=1000, height=600, image=png(), strength=0.7, - references=(png(color='red'),) * 5))) + references=(png(color='red'),) * 3))) body = calls[0] assert result.suffix == 'png' and body.count(b'name="image"') == 4 and b'name="strength"' not in body assert b'name="size"\r\n\r\n992x608' in body diff --git a/tests/test_vllm_omni_h3.py b/tests/test_vllm_omni_h3.py index 776d6a15..cd2465b3 100644 --- a/tests/test_vllm_omni_h3.py +++ b/tests/test_vllm_omni_h3.py @@ -9,19 +9,24 @@ from uuid import uuid4 import httpx +from fastapi.testclient import TestClient from PIL import Image import pytest from opendirector.application.creation_generation import CreationGeneration from opendirector.application.creation_tool_assets import CreationToolAssets -from generation_helpers import new_project, prepared as prepare_command, tasks +from opendirector.application.model_availability import ModelAvailability +from generation_helpers import new_project, command as generation_command, prepared as prepare_command, tasks from opendirector.domain.input_rules import input_rules from opendirector.domain.projects import ProjectError from opendirector.infrastructure.local_projects import LocalProjects from opendirector.infrastructure.model_configs import ModelConfigs from opendirector.integrations.providers import GenerationProviders +from opendirector.integrations.capability_probe import CapabilityProbe from opendirector.integrations.vllm_omni import VllmOmniGeneration, h3_frames from opendirector.integrations.volcengine import VolcengineGeneration +from test_model_availability import document +from opendirector_api.main import create_app H3 = {'protocol': 'vllm_omni', 'kind': 'video', 'url': 'http://gpu-box:8000/v1', 'modelId': 'MiniMaxAI/MiniMax-H3', 'apiKey': 'token'} @@ -140,6 +145,11 @@ def test_canvas_continues_a_previous_h3_clip_by_its_job_id(project, tmp_path): server = FakeOmni(clip.read_bytes()) configs = ModelConfigs(store.local.repository) config = configs.save('video', H3['url'], H3['modelId'], 'token', connection_type='self_hosted', protocol='vllm_omni', auth_mode='bearer') + declaration = document('video_generation', 'text_to_video', '/v1/videos', 'video', model=H3['modelId']) + operation = declaration['models'][0]['operations']['video_generation'] + operation['profiles'].append({'id': 'video_to_video', 'support': 'unsupported'}) + probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=declaration))) + availability = ModelAvailability(configs, probe) async def generate(inputs): parameters = dict(videoMode=inputs['videoMode']) @@ -151,12 +161,40 @@ async def generate(inputs): async def run(): nonlocal manager - manager = CreationGeneration(store, configs) + manager = CreationGeneration(store, configs, availability=availability) manager.provider = GenerationProviders(manager.store, httpx.MockTransport(server), poll_interval=0) first = await generate({'prompt': '灯塔在黄昏亮起', 'referenceIds': [], 'videoMode': 'first_frame'}) assert first['status'] == 'succeeded' and first['externalTaskId'] == 'video_gen_1', (first['error'], first['errorDetails']) assert json.loads(server.posts[0][0]['extra_params']) == {'task': 't2va', 'aspect_ratio': '16:9'} + # The public command API cannot accept a cached continuation on a + # globally disabled endpoint, even without a declared input profile. + declaration['routes'][0]['exposure'] = 'disabled' + command = generation_command((store, value['id'], session), config, + prompt='关闭接口时接续', parameters={'videoMode': 'continuation'}, references=[first['assetId']]) + api_probe = CapabilityProbe(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=declaration))) + try: + app = create_app(store.local, generation_provider=manager.provider, capability_probe=api_probe) + with TestClient(app, headers={'X-OpenDirector-Request': '1', 'X-OpenDirector-Session': session}) as client: + response = client.post(f"/api/creation/projects/{value['id']}/commands", json=command.model_dump(mode='json')) + assert response.status_code == 200, response.text + receipt = response.json() + assert receipt['status'] == 'rejected', receipt + assert receipt['issues'][0]['code'] == 'capability_unavailable' + assert len(store.snapshot(value['id']).jobs) == len(server.posts) == 1 + finally: + api_probe.close() + declaration['routes'][0]['exposure'] = 'enabled' + + # A fresh deployment pause still prevents a continuation on /videos. + operation['execution']['state'] = 'paused' + availability.refresh([config['id']], manual=True) + with pytest.raises(ProjectError, match='服务已暂停'): + await generate({'prompt': '暂停期间接续', 'referenceIds': [first['assetId']], 'videoMode': 'continuation'}) + assert len(store.snapshot(value['id']).jobs) == len(server.posts) == 1 + operation['execution']['state'] = 'unpaused' + availability.refresh([config['id']], manual=True) + second = await generate({'prompt': '光束扫过海面', 'referenceIds': [first['assetId']], 'videoMode': 'continuation'}) assert second['status'] == 'succeeded' and second['videoMode'] == 'continuation', (second['error'], second['errorDetails']) fields, files = server.posts[1] @@ -179,7 +217,10 @@ async def run(): await manager.close() manager = None - asyncio.run(run()) + try: + asyncio.run(run()) + finally: + probe.close() def test_other_models_keep_rejecting_video_references(project): diff --git a/tests/test_vllm_omni_image.py b/tests/test_vllm_omni_image.py index ad9b89a1..a4e439b0 100644 --- a/tests/test_vllm_omni_image.py +++ b/tests/test_vllm_omni_image.py @@ -2,6 +2,8 @@ import base64 import io import json +from email.parser import BytesParser +from email.policy import default from uuid import uuid4 import httpx @@ -27,9 +29,9 @@ def png(): return out.getvalue() -def test_z_image_profile_is_text_only_and_routes_to_its_adapter(): +def test_z_image_profile_accepts_one_image_and_routes_to_its_adapter(): rules = input_rules(ZIMAGE) - assert rules['profile'] == 'vllm-omni-image' and rules['allowedKinds'] == ['text'] and rules['maxImages'] == 0 + assert rules['profile'] == 'vllm-omni-image' and rules['allowedKinds'] == ['text', 'image'] and rules['maxImages'] == 1 assert input_rules({**ZIMAGE, 'url': 'https://example.test/omni/v1', 'modelId': 'z-image-turbo'})['profile'] == 'vllm-omni-image' assert input_rules({**ZIMAGE, 'kind': 'video'})['profile'] == 'compatible' assert input_rules({**ZIMAGE, 'modelId': 'Tongyi-MAI/Z-Image'})['profile'] == 'compatible' @@ -50,8 +52,18 @@ def project(tmp_path): def server(posts, reply, authorization='Bearer token'): def handle(request): assert request.headers.get('authorization') == authorization - assert request.method == 'POST' and request.url.path == '/v1/images/generations' - posts.append(json.loads(request.read())) + assert request.method == 'POST' + if request.url.path == '/v1/images/edits': + message = BytesParser(policy=default).parsebytes( + ('Content-Type: ' + request.headers['content-type'] + '\r\n\r\n').encode() + request.read()) + parts = list(message.iter_parts()) + fields = {part.get_param('name', header='content-disposition'): part.get_payload(decode=True).decode() + for part in parts if part.get_filename() is None} + fields['images'] = [part.get_payload(decode=True) for part in parts if part.get_filename() is not None] + posts.append(fields) + else: + assert request.url.path == '/v1/images/generations' + posts.append(json.loads(request.read())) return reply return handle @@ -141,11 +153,24 @@ async def run(): assert (job.execution.numInferenceSteps, job.execution.guidanceScale) == (accepted.get('num_inference_steps'), accepted.get('guidance_scale')) -def test_reference_images_are_rejected_before_any_request(project): +def test_one_reference_is_sent_to_the_edit_route_without_losing_its_bytes(project): store, value, session, _ = project picture = CreationToolAssets(store).import_file(value['id'], str(uuid4()), '角色.png', png(), session) - with pytest.raises(ProjectError, match='参考素材类型'): - run_image(project, httpx.Response(500), [picture['id']]) + reply = httpx.Response(200, json={'data': [{'b64_json': base64.b64encode(png()).decode()}]}) + task, posts = run_image(project, reply, [picture['id']]) + assert task['status'] == 'succeeded' + assert len(posts) == 1 and posts[0]['images'] == [png()] + assert not {'num_inference_steps', 'guidance_scale'} & posts[0].keys() + job = store.snapshot(value['id']).jobs[-1] + assert job.execution.numInferenceSteps is None and job.execution.guidanceScale is None + + +def test_extra_reference_is_rejected_before_any_request(project): + store, value, session, _ = project + pictures = [CreationToolAssets(store).import_file(value['id'], str(uuid4()), f'{index}.png', png(), session) + for index in range(2)] + with pytest.raises(ProjectError, match='最多支持 1 张图片，当前为 2 张'): + run_image(project, httpx.Response(500), [item['id'] for item in pictures]) @pytest.mark.parametrize('reply', [ diff --git a/tests/toolReasonText.test.ts b/tests/toolReasonText.test.ts index c25fcdbb..195a277b 100644 --- a/tests/toolReasonText.test.ts +++ b/tests/toolReasonText.test.ts @@ -14,6 +14,8 @@ test("tool availability reasons are translated without leaking capability labels "Connect an image segmentation model to detect selections automatically; you can still draw one manually.", ); assert.equal(toolReasonText("connect an image model"), "connect an image model"); + assert.equal(toolReasonText("新增的暂停说明", undefined, "execution_paused"), "Service paused"); + assert.equal(toolReasonText("服务未开放图片编辑。", undefined, "operation_disabled"), "The service has not enabled image editing."); setActiveLocale("zh-CN"); assert.equal( toolReasonText("需要接入具备「图像编辑」能力的模型。"),