diff --git a/apps/api/src/opendirector_api/localization.py b/apps/api/src/opendirector_api/localization.py index 0586213f..aeb4fbc0 100644 --- a/apps/api/src/opendirector_api/localization.py +++ b/apps/api/src/opendirector_api/localization.py @@ -43,9 +43,14 @@ 'voice_provider_protocol': 'The voice service returned an invalid response or pagination did not finish. Check its API version.', 'voice_provider_error': 'The voice service request failed. Check the connection and account permissions.', 'voice_provider_timeout': 'The voice service timed out. Check its status and try again.', + 'voice_name_taken': 'This account already has a voice with that name. Choose another name.', + 'voice_provider_quota': 'The voice service account has insufficient balance or resource packages. Top up and try again.', "moss_speech_text": "Enter the text to read aloud.", "moss_speech_voice": "Enter a Voice ID available to your MOSS account.", "moss_speech_reference": "MOSS speech generation does not accept references. Enter speech text and a Voice ID.", + "zhipu_speech_text": "Enter the text to read aloud, at most 1,024 characters per GLM-TTS request.", + "zhipu_speech_voice": "Select a voice available to your Zhipu account.", + "zhipu_speech_reference": "GLM-TTS does not accept references. Enter speech text and select a voice.", "moss_prompt_limit": "MOSS requires a nonempty prompt of at most 4,000 characters, including text references.", "moss_image_size": "MOSS image dimensions must be 256–2048, multiples of 16, and no more than 2,097,152 pixels in total.", "moss_image_reference": "MOSS image generation does not accept image references or editing inputs.", diff --git a/docs/development/validation.md b/docs/development/validation.md index 3708a6bd..451ced37 100644 --- a/docs/development/validation.md +++ b/docs/development/validation.md @@ -249,6 +249,16 @@ CREATION_API_URL=http://127.0.0.1:8019 CREATION_WEB_URL=http://127.0.0.1:5179 no Python 用例使用模拟 HTTP 服务及真实 PNG、MP3,检查请求字段、音色、执行快照、文件探测、工程重开和下载恢复不重复生成。通用错误分类和安全提示由现有错误测试负责。浏览器用例检查两种语言下的提供商及型号列表、获取模型失败的具体原因、音色必填、保存重开和实际音频播放;播放器使用隔离服务的 WAV 替身,不代表 MOSS 已生成语音。真实验收另需有效 MOSS 凭证和音色 ID,并保留实际音频、模型标识及播放器截图。 +智谱 BigModel GLM-TTS 接入使用以下检查: + +```sh +.venv/bin/python -m pytest tests/test_zhipu.py tests/test_voice_library.py tests/test_generation_errors.py -q +# 真实服务验收(会产生费用):在隔离的 OPENDIRECTOR_ROOT 上启动 opendirector serve 后运行 +OPENDIRECTOR_URL=http://127.0.0.1:18238 ZHIPU_API_KEY=... ZHIPU_SAMPLE=/path/to/sample.wav node tests/browser/zhipu_real.mjs +``` + +Python 用例使用按真实服务响应录制的模拟 HTTP,检查 `/audio/speech` 只请求非流式 WAV 并按 RIFF/WAVE 字节验收(服务默认返回裸 PCM,且部分错误以 HTTP 200 + JSON 返回)、`1113` 余额不足归为额度而非限流、音色库只列出文档中的 7 个预置音色加 `PRIVATE` 复刻音色、复刻前的真实音频校验(mp3/wav、3–30 秒、10 MB)、复刻成功或失败后都删除上传的样本文件、`POST /voice/delete` 删除及重名/不存在的错误映射、试听只从公网 HTTPS 下载且不带凭据,以及统一生成服务的音色预检与执行快照。真实验收需要已充值且开通 GLM-TTS 的 BigModel 账户;保留预置音色与复刻音色生成的实际音频、复刻到删除的完整流程截图,并确认账户中没有遗留的 `voice-clone-input` 文件。 + ### Voice library / 音色库 diff --git a/docs/guides/user-guide.md b/docs/guides/user-guide.md index d8a7346b..17c818bd 100644 --- a/docs/guides/user-guide.md +++ b/docs/guides/user-guide.md @@ -93,6 +93,10 @@ On MOSS Open Platform, select the stable `moss-tts-1.5-flash` or `moss-tts-1.0-p Choose **Create voice** to upload a local recording, name it, and review its playback, format, size and duration before submitting. vLLM-Omni requires the consent recording ID; adapters such as Drama3 also require the recording transcript. MOSS-TTS-Nano needs a registered voice with reference audio: a listed `default` voice is insufficient. This upload registers the voice with the selected service without adding a canvas reference node. +On Zhipu BigModel Open Platform, select `glm-tts`. The library lists the seven documented system voices and the voices cloned on that account. To clone a voice, upload a 3–30 second MP3 or WAV recording of at most 10 MB; the recording transcript is optional but improves similarity. OpenDirector deletes the uploaded sample from the account after registration. Each request reads at most 1,024 characters, and references are not accepted. Cloned voices can be deleted from the library. + +The same provider can also be added as a conversation Agent: click **Fetch models** to read the account's GLM models (for example `glm-4.5-flash` or `glm-5.3-flash`) and select the ones to enable. + Selected voices are saved in the project; favorites are saved as browser preferences. Both are scoped to the configured service and account. Compatible models sharing a service can retain the selection; another service or account requires its own registered voice. Use a voice’s secondary menu to copy its provider ID or delete an uploaded voice where supported. Deletion requires confirmation; MOSS deletion is unavailable until its API supports it. Service errors are shown without credentials; an unsupported model keeps its existing voice controls. Agents use the same validation through `voice_list`, `voice_create`, and `voice_delete`. The [business CLI](cli.md) also exposes `voice list`, `voice create`, and `voice delete`. diff --git a/docs/guides/user-guide.zh-CN.md b/docs/guides/user-guide.zh-CN.md index 5328fcd7..3f947e08 100644 --- a/docs/guides/user-guide.zh-CN.md +++ b/docs/guides/user-guide.zh-CN.md @@ -93,6 +93,10 @@ HTTP 400。 选择**创建音色**,上传本地录音、填写名称,并在提交前检查试听、格式、大小和时长。vLLM-Omni 需要填写授权录音 ID;Drama3 等适配器还必须填写录音逐字稿。MOSS-TTS-Nano 需要带有参考音频的已注册音色,列表中的 `default` 不能代替录音。上传会直接在当前服务注册音色,不创建画布参考音频节点。 +在智谱 BigModel 开放平台上选择 `glm-tts`。音色库列出文档中的 7 个系统音色和该账户下已复刻的音色。复刻音色时,上传 3–30 秒、不超过 10 MB 的 MP3 或 WAV 录音;录音逐字稿可选,填写后相似度更高。注册完成后,OpenDirector 会从账户中删除上传的样本文件。每次最多朗读 1,024 个字符,不支持参考素材。复刻音色可以在音色库中删除。 + +该提供商也可以作为对话 Agent 添加:点击“获取模型”读取账户可用的 GLM 模型(例如 `glm-4.5-flash`、`glm-5.3-flash`),勾选需要启用的模型即可。 + 所选音色随工程保存,收藏作为浏览器偏好保存,均按服务连接和账户隔离。同一连接下的兼容模型可以保留选择;其他服务或账户必须单独注册音色。音色的更多操作菜单可复制服务端 ID,或在服务支持时删除上传音色。删除前需要确认;MOSS 尚未开放删除操作。服务错误不会显示凭据;不支持音色库的模型保留原有音色控件。 Agent 通过 `voice_list`、`voice_create`、`voice_delete` 使用同一套校验;[业务 CLI](cli.md) 也提供 `voice list`、`voice create`、`voice delete`。 diff --git a/packages/core/src/opendirector/application/creation_generation.py b/packages/core/src/opendirector/application/creation_generation.py index c94b6d23..59967e3c 100644 --- a/packages/core/src/opendirector/application/creation_generation.py +++ b/packages/core/src/opendirector/application/creation_generation.py @@ -212,6 +212,9 @@ def _parameters(self, spec, config, node, records): params.update(prompt_enhance=False, watermark=not self.settings.get()['removeWatermark']) if spec.kind == 'audio' and input_rules(config)['profile'] == 'moss-speech': params = dict(voice=intent['voice'], response_format='mp3', delivery_method='url', stream=False) + if spec.kind == 'audio' and input_rules(config)['profile'] == 'zhipu-speech': + params = dict(voice=intent['voice'], response_format='wav', stream=False, + watermark_enabled=not self.settings.get()['removeWatermark']) return params @staticmethod @@ -229,6 +232,8 @@ def _execution(kind, parameters, config): if kind == 'audio': if input_rules(config)['profile'] == 'moss-speech': return dict(kind=kind, voice=parameters['voice'], format='mp3') + if input_rules(config)['profile'] == 'zhipu-speech': + return dict(kind=kind, voice=parameters['voice'], format='wav', sampleRate=24000) return dict(kind=kind, voice=parameters.get('voice') or ('default' if config['connectionType'] == 'self_hosted' else 'alloy') if config['protocol'] == 'openai' else None, speaker=parameters.get('speaker') if config['protocol'] != 'openai' else None, format='mp3', sampleRate=24000 if config['protocol'] != 'openai' else None) diff --git a/packages/core/src/opendirector/application/voice_library.py b/packages/core/src/opendirector/application/voice_library.py index ed567186..cc7078dc 100644 --- a/packages/core/src/opendirector/application/voice_library.py +++ b/packages/core/src/opendirector/application/voice_library.py @@ -1,5 +1,7 @@ """Shared voice registry operations for HTTP, Agent and generation acceptance.""" import json +import socket +from ipaddress import ip_address from pathlib import Path import tempfile from time import monotonic @@ -9,6 +11,7 @@ from opendirector.domain.projects import ProjectError from opendirector.domain.voices import VOICE_CREATE_TEXT_LIMITS, voice_error, voice_library_capabilities, valid_voice_name, voice_incompatibility +from opendirector.domain.zhipu_speech import ZHIPU_CLONE_MODEL, ZHIPU_CLONE_PREVIEW_TEXT, ZHIPU_PRESET_VOICES from opendirector.integrations.http_settings import auth_headers, trust_environment from opendirector.integrations.volcengine import endpoint from opendirector.media.probe import probe_file @@ -19,9 +22,29 @@ _CURRENT_CONNECTION = object() +def _public_https(url): + """Pin an account listing's CDN preview to a checked public HTTPS address, keeping TLS identity.""" + parsed = urlsplit(url) + host = (parsed.hostname or '').rstrip('.').lower() + if parsed.scheme != 'https' or not host or parsed.username or parsed.password or host == 'localhost' or host.endswith('.localhost'): + raise voice_error('voice_provider_error', 502) + try: + answers = [answer[4][0] for answer in socket.getaddrinfo(host, parsed.port or 443, type=socket.SOCK_STREAM)] + except OSError: + raise voice_error('voice_provider_error', 502) from None + for answer in answers: + address = ip_address(answer) + address = getattr(address, 'ipv4_mapped', None) or address + if not address.is_global or address.is_multicast or address.is_reserved: + raise voice_error('voice_provider_error', 502) + if not answers: raise voice_error('voice_provider_error', 502) + target = httpx.URL(url) + return target.copy_with(host=answers[0]), {'Host': target.netloc.decode('ascii'), 'Connection': 'close'}, {'sni_hostname': target.host} + + class VoiceLibrary: - def __init__(self, configs, transport=None): - self.configs, self.transport = configs, transport + def __init__(self, configs, transport=None, resolve_public=_public_https): + self.configs, self.transport, self.resolve_public = configs, transport, resolve_public def _config(self, model_config_id): config = dict(self.configs.resolve(str(model_config_id), 'audio')) @@ -29,26 +52,23 @@ def _config(self, model_config_id): if capabilities is None: raise voice_error('voice_library_unsupported', 409) return config, capabilities - def _request(self, config, method, suffix='', **kwargs): + def _request(self, config, method, suffix='', *, path='/audio/voices', **kwargs): try: # Sync service runs in worker threads, including job preparation. with httpx.Client(transport=self.transport, trust_env=trust_environment(config['url'], config=config), timeout=httpx.Timeout(120, connect=15), follow_redirects=False) as client: - with client.stream(method, endpoint(config['url'], '/audio/voices') + suffix, + with client.stream(method, endpoint(config['url'], path) + suffix, headers=auth_headers(config), **kwargs) as response: data = bytearray() for chunk in response.iter_bytes(): data.extend(chunk) if len(data) > 2 * 1024 * 1024: raise voice_error('voice_provider_protocol', 502) if not response.is_success: - # Retain useful upstream detail; redact the configured credential - # before it reaches either the HTTP or Agent presentation boundary. - detail = self._provider_detail(data, config) - error = voice_error('voice_provider_error', response.status_code if 400 <= response.status_code < 500 else 502) - if detail: error.args = (str(error) + ' ' + detail,) - raise error + raise self._provider_failure(data, config, response.status_code) value = json.loads(data) if data else {} if not isinstance(value, dict): raise voice_error('voice_provider_protocol', 502) + # Zhipu reports some failures, such as an unsupported method, with HTTP 200. + if isinstance(value.get('error'), dict): raise self._provider_failure(data, config, 502) return value except httpx.TimeoutException: raise voice_error('voice_provider_timeout', 504) from None @@ -57,6 +77,20 @@ def _request(self, config, method, suffix='', **kwargs): except (ValueError, TypeError): raise voice_error('voice_provider_protocol', 502) from None + def _provider_failure(self, data, config, status): + # Retain useful upstream detail; redact the configured credential + # before it reaches either the HTTP or Agent presentation boundary. + detail = self._provider_detail(data, config) + try: error = json.loads(data).get('error') or {} + except (ValueError, AttributeError): error = {} + code = str(error.get('code', '')) if isinstance(error, dict) else '' + if code == '1113': return voice_error('voice_provider_quota', 402) + if code == '1214' and detail == '音色名称已存在': return voice_error('voice_name_taken', 409) + if code == '1214' and detail.startswith('音色不存在'): return voice_error('voice_not_found', 404) + error = voice_error('voice_provider_error', status if 400 <= status < 500 else 502) + if detail: error.args = (str(error) + ' ' + detail,) + return error + @staticmethod def _provider_detail(data, config): try: @@ -153,6 +187,22 @@ def _moss_page(self, config, capabilities, limit, after): raise voice_error('voice_provider_protocol', 502) return [self._record(item, capabilities, source='uploaded', config=config) for item in items], cursor if more else None + def _zhipu_page(self, config, capabilities): + # The documented system voices are always available; the account's + # OFFICIAL listing includes entries synthesis rejects, so it is not read. + records = [self._record({'id': voice}, capabilities, source='builtin', has_audio=False, config=config) + for voice in ZHIPU_PRESET_VOICES] + value = self._request(config, 'GET', path='/voice/list', params={'voiceType': 'PRIVATE'}) + items = value.get('voice_list') + if not isinstance(items, list) or len(items) > 10000: raise voice_error('voice_provider_protocol', 502) + for item in items: + if not isinstance(item, dict) or item.get('voice_type') != 'PRIVATE': + raise voice_error('voice_provider_protocol', 502) + records.append(self._record({'id': item.get('voice'), 'name': item.get('voice_name'), + 'preview_url': item.get('download_url'), 'created_at': item.get('create_time')}, + capabilities, source='uploaded', has_audio=True, config=config)) + return records, None + def _assert_current(self, config, capabilities): current, rules = self._config(config['id']) if rules != capabilities or current['modelId'] != config['modelId']: @@ -169,6 +219,8 @@ def _pages(self, config, capabilities, limit, after=None): self._assert_current(config, capabilities) if capabilities['provider'] == 'moss': records, cursor = self._moss_page(config, capabilities, limit, after) + elif capabilities['provider'] == 'zhipu': + records, cursor = self._zhipu_page(config, capabilities) else: records, cursor = self._vllm_page(config, capabilities) for record, _ in records: @@ -244,6 +296,8 @@ def create(self, model_config_id, data, *, filename, name, consent=None, ref_tex seconds = media['durationSeconds'] if (capabilities['minSeconds'] is not None and seconds < capabilities['minSeconds']) or (capabilities['maxSeconds'] is not None and seconds > capabilities['maxSeconds']): raise voice_error('voice_duration') fields = {'name': name} + if capabilities['provider'] == 'zhipu': + return self._zhipu_create(config, capabilities, data, extension, name, text) if capabilities['provider'] == 'vllm_omni': fields['consent'] = consent.strip() if text: fields['ref_text'] = text @@ -256,6 +310,30 @@ def create(self, model_config_id, data, *, filename, name, consent=None, ref_tex record, _ = self._record(item, capabilities, source='uploaded', has_audio=True, config=config) return record + def _zhipu_create(self, config, capabilities, data, extension, name, text): + # Upload the sample, register the clone, then delete the uploaded sample. + # The registered voice keeps working without it; the generated preview + # file remains because the listing's download URL serves it. + mime = {'wav': 'audio/wav', 'mp3': 'audio/mpeg'}[extension] + uploaded = self._request(config, 'POST', path='/files', data={'purpose': 'voice-clone-input'}, + files={'file': ('sample.' + extension, data, mime)}) + file_id = uploaded.get('id') + if not isinstance(file_id, str) or not 0 < len(file_id) <= 512: raise voice_error('voice_provider_protocol', 502) + try: + payload = dict(model=ZHIPU_CLONE_MODEL, voice_name=name, input=ZHIPU_CLONE_PREVIEW_TEXT, file_id=file_id) + if text: payload['text'] = text + value = self._request(config, 'POST', path='/voice/clone', json=payload) + finally: + try: self._request(config, 'DELETE', '/' + quote(file_id, safe=''), path='/files') + except ProjectError: pass + voice = value.get('voice') + if not valid_voice_name(voice): raise voice_error('voice_provider_protocol', 502) + # Read back the registered entry for its preview locator and stable fields. + record, _ = self._find(config, capabilities, voice) + if record is None: + record, _ = self._record({'id': voice, 'name': name}, capabilities, source='uploaded', has_audio=True, config=config) + return record + def preview(self, model_config_id, voice_id, connection_id): config, capabilities = self._config(model_config_id) if connection_id != capabilities['connectionId']: raise voice_error('voice_connection_changed', 409) @@ -264,15 +342,24 @@ def preview(self, model_config_id, voice_id, connection_id): parsed, configured = urlsplit(url), urlsplit(config['url']) def origin(value): return value.scheme, value.hostname, value.port or (443 if value.scheme == 'https' else 80) headers = auth_headers(config) if origin(parsed) == origin(configured) else {} + target, extensions = url, {} + if capabilities['provider'] == 'zhipu': + # The listing hands out a third-party CDN URL: no credentials, public addresses only. + target, headers, extensions = self.resolve_public(url) try: with httpx.Client(transport=self.transport, trust_env=trust_environment(url, config=config), timeout=30, follow_redirects=False) as client: - with client.stream('GET', url, headers=headers) as response: + with client.stream('GET', target, headers=headers, extensions=extensions) as response: mime = response.headers.get('content-type', '').split(';')[0].lower() - if not response.is_success or not mime.startswith('audio/'): raise voice_error('voice_provider_error', 502) + # Zhipu's CDN labels WAV previews application/octet-stream; check the bytes instead. + sniff = capabilities['provider'] == 'zhipu' + if not response.is_success or not (mime.startswith('audio/') or sniff): raise voice_error('voice_provider_error', 502) data = bytearray() for chunk in response.iter_bytes(): data.extend(chunk) if len(data) > 10 * 1024 * 1024: raise voice_error('voice_size', 413) + if sniff: + if not (data[:4] == b'RIFF' and data[8:12] == b'WAVE'): raise voice_error('voice_provider_error', 502) + mime = 'audio/wav' self._assert_current(config, capabilities) return bytes(data), mime except httpx.TimeoutException: raise voice_error('voice_provider_timeout', 504) from None @@ -288,6 +375,10 @@ def delete(self, model_config_id, voice_id, connection_id): if not record['canDelete']: raise voice_error('voice_delete_unsupported', 409) config, current = self._config(model_config_id) if current['connectionId'] != connection_id: raise voice_error('voice_connection_changed', 409) + if current['provider'] == 'zhipu': + value = self._request(config, 'POST', path='/voice/delete', json={'voice': voice_id}) + if value.get('voice') != voice_id: raise voice_error('voice_provider_protocol', 502) + return value = self._request(config, 'DELETE', '/' + quote(voice_id, safe='')) if value.get('success') is not True: raise voice_error('voice_provider_protocol', 502) diff --git a/packages/core/src/opendirector/domain/creation/commands.py b/packages/core/src/opendirector/domain/creation/commands.py index 7ca6b9a4..46317167 100644 --- a/packages/core/src/opendirector/domain/creation/commands.py +++ b/packages/core/src/opendirector/domain/creation/commands.py @@ -401,11 +401,13 @@ def command_fingerprint(command: Command) -> str: 'capability_unsupported', 'preview_expired', 'storage_unavailable', 'submission_unknown', 'provider_error', 'moss_speech_text', 'moss_speech_voice', 'moss_speech_reference', 'moss_prompt_limit', 'moss_image_size', 'moss_image_reference', 'moss_layout_reference', + 'zhipu_speech_text', 'zhipu_speech_voice', 'zhipu_speech_reference', 'voice_library_unsupported', 'voice_connection_changed', 'voice_reference_required', 'voice_transcript_required', 'voice_transcript_too_long', 'voice_description_too_long', 'voice_not_found', 'voice_delete_unsupported', 'voice_consent_required', 'voice_name_invalid', 'voice_audio_invalid', 'voice_format', - 'voice_size', 'voice_duration', 'voice_provider_protocol', 'voice_provider_error', 'voice_provider_timeout', 'voice_model_unsupported'] + 'voice_size', 'voice_duration', 'voice_provider_protocol', 'voice_provider_error', 'voice_provider_timeout', 'voice_model_unsupported', + 'voice_name_taken', 'voice_provider_quota'] class ContractIssue(Contract): diff --git a/packages/core/src/opendirector/domain/creation/generation.py b/packages/core/src/opendirector/domain/creation/generation.py index 974f43e0..29196cd1 100644 --- a/packages/core/src/opendirector/domain/creation/generation.py +++ b/packages/core/src/opendirector/domain/creation/generation.py @@ -369,6 +369,10 @@ def validate_generation(spec: GenerationSpec, model: ModelConfigPublic, referenc from opendirector.domain.moss_speech import validate_moss_speech_input text = spec.prompt if spec.promptContent is None else ''.join(part.text for part in spec.promptContent if part.type == 'text') validate_moss_speech_input(text, spec.parameters.voice) + elif rules.profile == 'zhipu-speech': + from opendirector.domain.zhipu_speech import validate_zhipu_speech_input + text = spec.prompt if spec.promptContent is None else ''.join(part.text for part in spec.promptContent if part.type == 'text') + validate_zhipu_speech_input(text, spec.parameters.voice) elif spec.kind == 'audio' and rules.voiceRequired and not spec.parameters.voice: raise ValueError('请填写服务音色 ID。') if len(references) > MAX_REFERENCES: diff --git a/packages/core/src/opendirector/domain/creation/models.py b/packages/core/src/opendirector/domain/creation/models.py index 937a2d0f..7aa2da25 100644 --- a/packages/core/src/opendirector/domain/creation/models.py +++ b/packages/core/src/opendirector/domain/creation/models.py @@ -29,7 +29,7 @@ class ImageOutput(Contract): class VoiceLibraryCapabilities(Contract): - provider: Literal['vllm_omni', 'moss'] = Field(description='Voice registry protocol.') + provider: Literal['vllm_omni', 'moss', 'zhipu'] = Field(description='Voice registry protocol.') connectionId: str = Field(min_length=1, max_length=200, description='Opaque service, endpoint and account identity.') requiresTranscript: bool = Field(description='Registered voice must have a reference transcript.') requiresReferenceAudio: bool = Field(description='Registered voice must have actual reference audio.') diff --git a/packages/core/src/opendirector/domain/input_rules.py b/packages/core/src/opendirector/domain/input_rules.py index 3ffbcf22..0ccc9a86 100644 --- a/packages/core/src/opendirector/domain/input_rules.py +++ b/packages/core/src/opendirector/domain/input_rules.py @@ -6,6 +6,7 @@ from .moss_image import MOSS_IMAGE_PROMPT_CHARACTERS, is_moss_image from .moss_speech import is_moss_speech from .kling_models import KLING_MODELS, KLING_RATIOS, kling_capabilities +from .zhipu_speech import ZHIPU_SPEECH_CHARACTERS, is_zhipu_speech VideoMode = Literal['first_frame', 'reference', 'continuation'] @@ -26,6 +27,10 @@ def _adapter_rules(config): rules.update(profile='moss-speech', allowedKinds=[], maxImages=0, voiceRequired=True, referencesAllowed=False) return rules + if is_zhipu_speech(config): + rules.update(profile='zhipu-speech', allowedKinds=[], maxImages=0, voiceRequired=True, + referencesAllowed=False, maxPromptCharacters=ZHIPU_SPEECH_CHARACTERS) + return rules if kind == 'vision' or config.get('protocol') == 'vllm_omni_segmentation': # Shared image segmentation / video tracking service, never a generator. rules.update(profile='vllm-omni-segmentation', allowedKinds=[], maxImages=0, videoModes=[]) diff --git a/packages/core/src/opendirector/domain/voices.py b/packages/core/src/opendirector/domain/voices.py index 791bcd53..6d3398a6 100644 --- a/packages/core/src/opendirector/domain/voices.py +++ b/packages/core/src/opendirector/domain/voices.py @@ -5,6 +5,7 @@ from urllib.parse import urlsplit, urlunsplit from .moss_speech import is_moss_speech +from .zhipu_speech import is_zhipu_speech from .projects import ProjectError # Character limits and the corresponding error for each upload metadata field. @@ -32,6 +33,8 @@ 'voice_provider_protocol': '音色服务返回格式无效或分页未结束,请检查服务版本和接口。', 'voice_provider_error': '音色服务请求失败,请检查服务配置和账户权限。', 'voice_provider_timeout': '音色服务请求超时,请检查服务状态后重试。', + 'voice_name_taken': '该账户已有同名音色,请换一个名称。', + 'voice_provider_quota': '音色服务账户余额或资源包不足,请充值后重试。', } @@ -43,6 +46,11 @@ def voice_library_capabilities(config): if config.get('kind') != 'audio' or config.get('protocol') != 'openai': return None name = config.get('modelId', '').rsplit('/', 1)[-1].lower().replace('_', '-') + if is_zhipu_speech(config): + return _scoped(config, 'zhipu', dict( + requiresTranscript=False, requiresReferenceAudio=False, supportsUploadedVoices=True, + referenceEmbeddingDimension=None, canDelete=True, maxBytes=10 * 1024 * 1024, + formats=['wav', 'mp3'], minSeconds=3, maxSeconds=30)) moss = is_moss_speech(config) cosyvoice = name.startswith(('cosyvoice3', 'fun-cosyvoice3')) fish_speech = name.startswith('fish-speech') or name in {'s2-pro', 'drama-3', 'drama-3-int4'} @@ -53,15 +61,6 @@ def voice_library_capabilities(config): if not moss and not (config.get('connectionType') == 'self_hosted' and known): return None provider = 'moss' if moss else 'vllm_omni' - parsed = urlsplit(config['url']) - host = parsed.hostname.lower() - if ':' in host: host = '[' + host + ']' - port = parsed.port - if port and (parsed.scheme, port) not in {('https', 443), ('http', 80)}: host += ':' + str(port) - endpoint = urlunsplit((parsed.scheme.lower(), host, parsed.path.rstrip('/'), '', '')) - account = hashlib.sha256((config.get('apiKey', '') if config.get('authMode') != 'none' else '').encode()).hexdigest() - identity = [provider, config.get('providerId') or config.get('id'), endpoint, config.get('authMode', 'none'), account] - connection = hashlib.sha256(json.dumps(identity, separators=(',', ':')).encode()).hexdigest() # Upstream vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py: # validate() requires reference audio for standard/realtime/local/Nano. # Local-Transformer ref_text is optional continuation conditioning; Drama3 @@ -80,11 +79,24 @@ def voice_library_capabilities(config): qwen_base = name.startswith('qwen3-tts') and name.endswith('-base') dimension = (1024 if '0.6b' in name else 2048 if '1.7b' in name else None) if qwen_base else None reference = reference or qwen_base - return dict(provider=provider, connectionId=connection, requiresTranscript=transcript, + return _scoped(config, provider, dict(requiresTranscript=transcript, requiresReferenceAudio=reference, supportsUploadedVoices=uploaded, referenceEmbeddingDimension=dimension, canDelete=not moss, maxBytes=10 * 1024 * 1024, formats=['wav', 'mp3', 'm4a', 'flac'] if moss else ['wav', 'mp3', 'flac', 'ogg', 'aac', 'webm', 'm4a', 'mp4'], - minSeconds=None if moss else 1, maxSeconds=30) + minSeconds=None if moss else 1, maxSeconds=30)) + + +def _scoped(config, provider, capabilities): + parsed = urlsplit(config['url']) + host = parsed.hostname.lower() + if ':' in host: host = '[' + host + ']' + port = parsed.port + if port and (parsed.scheme, port) not in {('https', 443), ('http', 80)}: host += ':' + str(port) + endpoint = urlunsplit((parsed.scheme.lower(), host, parsed.path.rstrip('/'), '', '')) + account = hashlib.sha256((config.get('apiKey', '') if config.get('authMode') != 'none' else '').encode()).hexdigest() + identity = [provider, config.get('providerId') or config.get('id'), endpoint, config.get('authMode', 'none'), account] + connection = hashlib.sha256(json.dumps(identity, separators=(',', ':')).encode()).hexdigest() + return dict(provider=provider, connectionId=connection, **capabilities) def valid_voice_name(value): diff --git a/packages/core/src/opendirector/domain/zhipu_speech.py b/packages/core/src/opendirector/domain/zhipu_speech.py new file mode 100644 index 00000000..07a64921 --- /dev/null +++ b/packages/core/src/opendirector/domain/zhipu_speech.py @@ -0,0 +1,36 @@ +"""Model identity and input rules for Zhipu BigModel GLM-TTS speech synthesis.""" +import re +from urllib.parse import urlsplit + +from .projects import ProjectError + + +ZHIPU_SPEECH_MODELS = frozenset({'glm-tts'}) +ZHIPU_CLONE_MODEL = 'glm-tts-clone' +ZHIPU_SPEECH_CHARACTERS = 1024 +# The documented system voices. The account's OFFICIAL list also contains +# entries the synthesis endpoint rejects, so it is not used as a catalog. +ZHIPU_PRESET_VOICES = ('tongtong', 'chuichui', 'xiaochen', 'jam', 'kazi', 'douji', 'luodo') +ZHIPU_DEFAULT_VOICE = 'tongtong' +# The clone endpoint requires preview text; the preview is fixed product copy. +ZHIPU_CLONE_PREVIEW_TEXT = '你好,这是我的声音。Hello, this is my voice.' + + +def is_zhipu_api(config): + url = urlsplit(config.get('url', '')) + return (config.get('protocol') == 'openai' and config.get('connectionType', 'api') == 'api' + and url.scheme == 'https' and url.hostname == 'open.bigmodel.cn' and url.port in {None, 443} + and url.path.rstrip('/') == '/api/paas/v4') + + +def is_zhipu_speech(config): + return config.get('kind') == 'audio' and is_zhipu_api(config) and config.get('modelId') in ZHIPU_SPEECH_MODELS + + +def validate_zhipu_speech_input(text, voice): + if not isinstance(text, str) or not text.strip(): + raise ProjectError('请填写需要朗读的文本。', 422, code='zhipu_speech_text') + if len(text) > ZHIPU_SPEECH_CHARACTERS: + raise ProjectError(f'GLM-TTS 单次朗读文本不能超过 {ZHIPU_SPEECH_CHARACTERS} 个字符,请拆分后生成。', 422, code='zhipu_speech_text') + if not isinstance(voice, str) or not re.fullmatch(r'[A-Za-z0-9_.-]{1,64}', voice): + raise ProjectError('请选择智谱账户可用的音色。', 422, code='zhipu_speech_voice') diff --git a/packages/core/src/opendirector/integrations/generation_errors.py b/packages/core/src/opendirector/integrations/generation_errors.py index a5a73cb1..84879b1d 100644 --- a/packages/core/src/opendirector/integrations/generation_errors.py +++ b/packages/core/src/opendirector/integrations/generation_errors.py @@ -80,6 +80,10 @@ def provider_failure(value, *, stage, headers=None, http_status=None, secrets=() category = 'input' elif code == '55000000': category = 'temporary' + diagnostic_text = error.get('message', '') + if code == '1113' and isinstance(diagnostic_text, str) and ('余额' in diagnostic_text or '资源包' in diagnostic_text): + # Zhipu BigModel reports an empty balance as HTTP 429; retrying cannot help. + category = 'quota' message, suggestion = _REASONS[category] if provider == 'kling': kling_categories = {'1000': 'authentication', '1001': 'authentication', '1002': 'authentication', @@ -101,7 +105,7 @@ def provider_failure(value, *, stage, headers=None, http_status=None, secrets=() elif re.search(r'(?:not support|unsupported|not supported)', diagnostic) and re.search(r'(?:reference_(?:image|video|audio)|image_url|video_url|audio_url|input_audio)', diagnostic): category, message = 'reference', '供应商拒绝了参考类型或输入组合。' suggestion = '请核对参考类型和生成模式,或选择其他已适配模型;当前输入和已有结果保留。' - if name == 'voice_not_found': + if name == 'voice_not_found' or (code == '1214' and diagnostic.startswith('音色不存在')): category, message = 'input', '所选音色不存在或不可用。' suggestion = '请检查音色 ID 是否属于当前账户可用的音色。' elif name == 'inputimagesensitivecontentdetected.privacyinformation': diff --git a/packages/core/src/opendirector/integrations/model_adapters.py b/packages/core/src/opendirector/integrations/model_adapters.py index a95d4ba2..b0b1c18d 100644 --- a/packages/core/src/opendirector/integrations/model_adapters.py +++ b/packages/core/src/opendirector/integrations/model_adapters.py @@ -21,6 +21,7 @@ from opendirector.domain.input_rules import input_rules from opendirector.domain.moss_image import validate_moss_image_prompt, validate_moss_image_size from opendirector.domain.moss_speech import validate_moss_speech_input +from opendirector.domain.zhipu_speech import validate_zhipu_speech_input from opendirector.domain.projects import ProjectError from opendirector.integrations.generation_errors import checked, protocol_failure, provider_failure, transport_failure from opendirector.integrations.http_settings import MODEL_HTTP_HOOKS, auth_headers, trust_environment @@ -210,6 +211,41 @@ async def download(self, config, url): return data +class ZhipuSpeech(ModelAdapter): + """Zhipu BigModel GLM-TTS on /audio/speech. + + The service only offers WAV or raw PCM, and its streaming mode returns PCM + chunks over SSE, so a non-streaming WAV request is the only complete + container. A 200 response is accepted only when the body is a RIFF/WAVE file: + the service also reports some errors as JSON with HTTP 200. + """ + + capabilities = frozenset({'audio.speech'}) + + async def speech(self, config, request: SpeechRequest, *, result_ready=None, watermark=True) -> MediaResult: + validate_zhipu_speech_input(request.text, request.voice) + payload = dict(model=config['modelId'], input=request.text, voice=request.voice, + response_format='wav', stream=False, watermark_enabled=watermark) + secrets = (config.get('apiKey', ''),) + + async def submit(): + async with _client(self, config) as client: + async with client.stream('POST', endpoint(config['url'], '/audio/speech'), + headers=auth_headers(config), json=payload) as response: + return await bounded(response, stage='submit', secrets=secrets) + + data = await _call('submit', submit) + if not (data[:4] == b'RIFF' and data[8:12] == b'WAVE'): + try: + value = json.loads(data) + except (ValueError, UnicodeError): + value = None + if isinstance(value, dict) and value.get('error'): + raise provider_failure(value, stage='submit', http_status=200, secrets=secrets) + raise protocol_failure('submit') + return MediaResult(bytes(data), 'wav') + + class OpenAIImages(ModelAdapter): """OpenAI Images API: /images/generations and multipart /images/edits. @@ -674,6 +710,8 @@ def adapter_for(config, transport=None, poll_interval=3) -> ModelAdapter | None: return MossImages(transport, poll_interval) if profile == 'moss-speech': return MossSpeech(transport, poll_interval) + if profile == 'zhipu-speech': + return ZhipuSpeech(transport, poll_interval) if kind == 'image' and profile == 'vllm-omni-image': return OpenAIImages(transport, poll_interval, free_size=True, strength=True) if kind == 'image' and profile == 'vllm-omni-qwen-image': diff --git a/packages/core/src/opendirector/integrations/providers.py b/packages/core/src/opendirector/integrations/providers.py index b6385dd5..ec7af728 100644 --- a/packages/core/src/opendirector/integrations/providers.py +++ b/packages/core/src/opendirector/integrations/providers.py @@ -18,7 +18,7 @@ from opendirector.integrations.kling_video import KlingVideoGeneration from opendirector.integrations.vllm_omni import VllmOmniGeneration from opendirector.integrations.vllm_omni_image import VllmOmniImageGeneration -from opendirector.integrations.model_adapters import MossImages, MossSpeech, OpenAISpeech, VllmOmniMusic, VllmOmniScore +from opendirector.integrations.model_adapters import MossImages, MossSpeech, OpenAISpeech, VllmOmniMusic, VllmOmniScore, ZhipuSpeech from opendirector.domain.capabilities import ImageRequest, MusicRequest, ScoreRequest, SpeechRequest from opendirector.integrations.model_adapters import OpenAISpeech, VllmOmniMusic, VllmOmniScore, image_parts from opendirector.domain.capabilities import MusicRequest, ScoreRequest, SpeechRequest @@ -36,6 +36,7 @@ def __init__(self, store, transport=None, poll_interval=3, retry_delays=(0.5, 1. self.vllm_omni_image = VllmOmniImageGeneration(store, transport, poll_interval) self.moss_image = MossImages(transport, poll_interval) self.moss_speech = MossSpeech(transport, poll_interval) + self.zhipu_speech = ZhipuSpeech(transport, poll_interval) def _profile_adapter(self, config): if config and config.get('protocol') == 'kling_video': @@ -122,6 +123,11 @@ async def generate(self, config, task, submitted, *, result_ready=None): result = await self.moss_speech.speech(config, SpeechRequest(prepared['prompt'], task['parameters'].get('voice')), result_ready=result_ready) return result.suffix, result.data + if input_rules(config)['profile'] == 'zhipu-speech': + prepared = await asyncio.to_thread(generation_references, self.store, task, config) + result = await self.zhipu_speech.speech(config, SpeechRequest(prepared['prompt'], task['parameters'].get('voice')), + watermark=task['parameters'].get('watermark_enabled', True)) + return result.suffix, result.data if config.get('protocol') in {None, 'volcengine'} or task['kind'] == 'text': return await super().generate(config, task, submitted, result_ready=result_ready) kind, stage = task['kind'], 'submit' diff --git a/packages/core/src/opendirector/integrations/reference_inputs.py b/packages/core/src/opendirector/integrations/reference_inputs.py index f1a9d102..dacd1e49 100644 --- a/packages/core/src/opendirector/integrations/reference_inputs.py +++ b/packages/core/src/opendirector/integrations/reference_inputs.py @@ -107,6 +107,8 @@ def generation_references(store, task, config, *, complete=True, overrides=None) return result if kind == 'audio': if rules.get('referencesAllowed') is False and task['referenceIds']: + if rules['profile'] == 'zhipu-speech': + raise ProjectError('GLM-TTS 不支持参考素材,请直接填写朗读文本并选择音色。', 422, code='zhipu_speech_reference') raise ProjectError('MOSS 语音生成不支持参考素材,请直接填写朗读文本和音色 ID。', 422, code='moss_speech_reference') # Historical context remains recorded; it is never appended to speech. result['manifest'] = [{'assetId': meta['id'], 'fingerprint': meta['fingerprint'], 'kind': source_kind, 'transmission': 'context_only'} diff --git a/src/backend/rules.ts b/src/backend/rules.ts index 378e4ceb..729249b2 100644 --- a/src/backend/rules.ts +++ b/src/backend/rules.ts @@ -456,6 +456,12 @@ export function serviceGenerationSettings( !next.voiceName ) next.voiceName = "2890a3a4-0012-4997-a545-8b045fe83bbe"; + if ( + next.mode === "audio" && + config.inputRules.profile === "zhipu-speech" && + !next.voiceName + ) + next.voiceName = "tongtong"; if ( !generateAudioOptions({ published: config.inputRules.generateAudioOptions ?? null, diff --git a/src/canvas/AssistantPanel.tsx b/src/canvas/AssistantPanel.tsx index 3efd0d22..7b75226a 100644 --- a/src/canvas/AssistantPanel.tsx +++ b/src/canvas/AssistantPanel.tsx @@ -34,6 +34,7 @@ import { import { assetOptionId, referenceCandidates } from "./referenceCandidates.ts"; import { type MentionOption } from "./referenceMentions"; import { useAssistantChat } from "./useAssistantChat"; +import { needsAgentModel } from "./assistantReadiness.ts"; import type { AssistantLayout } from "./useAssistantLayout"; import { usePendingCanvasReferences } from "./usePendingCanvasReferences.ts"; import { useProjectMentionAssets } from "./useProjectMentionAssets"; @@ -138,6 +139,8 @@ export function AssistantPanel({ chat.adding || checkingReferences || (chat.sending && !chat.started); + const currentRuntime = chat.runtimes.find((item) => item.id === (value.runtime ?? "default")); + const missingAgentModel = needsAgentModel(chat.loaded, currentRuntime, !!chat.agent); const referenceDisabled = !chat.loaded || chat.adding || checkingReferences || chat.sending; const sessionDisabledReason = chat.changing @@ -761,6 +764,13 @@ export function AssistantPanel({ t("assistant.panel.checkingEngine") ) : chat.adding ? ( t("assistant.panel.addingReferences") + ) : missingAgentModel ? ( + <> + {t("assistant.agent.noAgentModels")}{" · "} + + {t("assistant.agent.configureModel")} + + > ) : ( `${chat.runtimes.find((item) => item.id === (value.runtime ?? "default"))?.label ?? t("assistant.panel.chooseAgent")}` )} diff --git a/src/canvas/CreateVoiceForm.tsx b/src/canvas/CreateVoiceForm.tsx index 6d90effa..cba91dda 100644 --- a/src/canvas/CreateVoiceForm.tsx +++ b/src/canvas/CreateVoiceForm.tsx @@ -178,7 +178,7 @@ export function CreateVoiceForm({ disabled={busy} onChange={(event) => setName(event.target.value)} /> - {isVllm && ( + {(isVllm || capabilities.provider === "zhipu") && ( <> {t( @@ -199,6 +199,10 @@ export function CreateVoiceForm({ {capabilities.requiresTranscript && ( {t("voiceLibrary.transcriptRequired")} )} + > + )} + {isVllm && ( + <> {t("voiceLibrary.description")} diff --git a/src/canvas/assistantReadiness.ts b/src/canvas/assistantReadiness.ts new file mode 100644 index 00000000..473cbeed --- /dev/null +++ b/src/canvas/assistantReadiness.ts @@ -0,0 +1,12 @@ +import type { RuntimeInfo } from "../backend/chat.ts"; + +/** A settings-backed engine is ready but has no conversation model to run. + * The send button stays disabled, so the panel explains why and links to + * the model settings instead of failing silently. */ +export function needsAgentModel( + loaded: boolean, + runtime: Pick | undefined, + hasAgent: boolean, +): boolean { + return loaded && runtime?.status === "ready" && runtime.modelSource === "settings" && !hasAgent; +} diff --git a/src/canvas/generationFailureText.ts b/src/canvas/generationFailureText.ts index 23246b64..025c9da5 100644 --- a/src/canvas/generationFailureText.ts +++ b/src/canvas/generationFailureText.ts @@ -12,6 +12,7 @@ import { defaultTranslate } from "./assistantTranslate.ts"; const BY_MESSAGE: Record = { "供应商判定提示词或参考图片违反内容政策,已拒绝生成。": "errors.generationFailure.policy", "生成服务拒绝了输入内容或参数。": "errors.generationFailure.input", + "所选音色不存在或不可用。": "errors.generationFailure.voiceNotFound", "生成服务拒绝了参考素材。": "errors.generationFailure.reference", "生成服务鉴权失败。": "errors.generationFailure.authentication", "当前账户没有调用该模型的权限。": "errors.generationFailure.permission", diff --git a/src/canvas/providerPresets.ts b/src/canvas/providerPresets.ts index 4845cc42..906378a6 100644 --- a/src/canvas/providerPresets.ts +++ b/src/canvas/providerPresets.ts @@ -8,6 +8,7 @@ const EN_NAMES: Record = { "MiniMax 官方视频": "MiniMax official video", "可灵官方视频": "Kling official video", "MOSS 开放平台": "MOSS Open Platform", + "智谱 BigModel 开放平台": "Zhipu BigModel Open Platform", "火山方舟 / Agent Plan": "Volcano Ark / Agent Plan", "vLLM-Omni 图片接口": "vLLM-Omni image API", "vLLM-Omni 语音合成接口": "vLLM-Omni speech synthesis API", @@ -138,6 +139,18 @@ export const PROVIDER_PRESETS: ProviderPreset[] = [ }, }, + { + id: "zhipu", + logoOrganization: "zai-org", + name: "智谱 BigModel 开放平台", + kinds: ["agent", "audio"], + url: "https://open.bigmodel.cn/api/paas/v4", + protocol: "openai", + connectionType: "api", + authMode: "bearer", + modelsByKind: { audio: ["glm-tts"] }, + }, + { id: "ark", logoOrganization: "ByteDance-Seed", diff --git a/src/contracts/creation.schema.json b/src/contracts/creation.schema.json index 7664beb5..7106e211 100644 --- a/src/contracts/creation.schema.json +++ b/src/contracts/creation.schema.json @@ -3484,6 +3484,9 @@ "moss_image_size", "moss_image_reference", "moss_layout_reference", + "zhipu_speech_text", + "zhipu_speech_voice", + "zhipu_speech_reference", "voice_library_unsupported", "voice_connection_changed", "voice_reference_required", @@ -3501,7 +3504,9 @@ "voice_provider_protocol", "voice_provider_error", "voice_provider_timeout", - "voice_model_unsupported" + "voice_model_unsupported", + "voice_name_taken", + "voice_provider_quota" ], "title": "Code", "type": "string" @@ -14356,7 +14361,8 @@ "description": "Voice registry protocol.", "enum": [ "vllm_omni", - "moss" + "moss", + "zhipu" ], "title": "Provider", "type": "string" @@ -18619,6 +18625,9 @@ "moss_image_size", "moss_image_reference", "moss_layout_reference", + "zhipu_speech_text", + "zhipu_speech_voice", + "zhipu_speech_reference", "voice_library_unsupported", "voice_connection_changed", "voice_reference_required", @@ -18636,7 +18645,9 @@ "voice_provider_protocol", "voice_provider_error", "voice_provider_timeout", - "voice_model_unsupported" + "voice_model_unsupported", + "voice_name_taken", + "voice_provider_quota" ], "title": "Code", "type": "string" @@ -29945,7 +29956,8 @@ "description": "Voice registry protocol.", "enum": [ "vllm_omni", - "moss" + "moss", + "zhipu" ], "title": "Provider", "type": "string" diff --git a/src/contracts/generated.ts b/src/contracts/generated.ts index eae4201c..574b7c26 100644 --- a/src/contracts/generated.ts +++ b/src/contracts/generated.ts @@ -387,7 +387,7 @@ export interface ContractInput { } export interface ContractIssueInput extends ContractInput { - readonly code: "timeline_package_invalid" | "timeline_package_too_large" | "timeline_revision_missing" | "timeline_import_missing" | "timeline_import_conflict" | "timeline_registration_rejected" | "timeline_font_too_large" | "timeline_font_unavailable" | "timeline_font_unsupported" | "timeline_font_invalid" | "timeline_font_fingerprint" | "timeline_font_coverage" | "timeline_font_total_bytes" | "timeline_font_unverified" | "timeline_font_mime" | "timeline_bundled_font_fingerprint" | "timeline_bundled_font_unavailable" | "invalid_input" | "revision_conflict" | "role_restricted" | "missing_object" | "reference_invalid" | "kling_model" | "kling_mode" | "kling_image" | "kling_frames" | "kling_prompt" | "kling_aspect" | "kling_output" | "kling_cancel" | "capability_unsupported" | "preview_expired" | "storage_unavailable" | "submission_unknown" | "provider_error" | "moss_speech_text" | "moss_speech_voice" | "moss_speech_reference" | "moss_prompt_limit" | "moss_image_size" | "moss_image_reference" | "moss_layout_reference" | "voice_library_unsupported" | "voice_connection_changed" | "voice_reference_required" | "voice_transcript_required" | "voice_transcript_too_long" | "voice_description_too_long" | "voice_not_found" | "voice_delete_unsupported" | "voice_consent_required" | "voice_name_invalid" | "voice_audio_invalid" | "voice_format" | "voice_size" | "voice_duration" | "voice_provider_protocol" | "voice_provider_error" | "voice_provider_timeout" | "voice_model_unsupported"; + readonly code: "timeline_package_invalid" | "timeline_package_too_large" | "timeline_revision_missing" | "timeline_import_missing" | "timeline_import_conflict" | "timeline_registration_rejected" | "timeline_font_too_large" | "timeline_font_unavailable" | "timeline_font_unsupported" | "timeline_font_invalid" | "timeline_font_fingerprint" | "timeline_font_coverage" | "timeline_font_total_bytes" | "timeline_font_unverified" | "timeline_font_mime" | "timeline_bundled_font_fingerprint" | "timeline_bundled_font_unavailable" | "invalid_input" | "revision_conflict" | "role_restricted" | "missing_object" | "reference_invalid" | "kling_model" | "kling_mode" | "kling_image" | "kling_frames" | "kling_prompt" | "kling_aspect" | "kling_output" | "kling_cancel" | "capability_unsupported" | "preview_expired" | "storage_unavailable" | "submission_unknown" | "provider_error" | "moss_speech_text" | "moss_speech_voice" | "moss_speech_reference" | "moss_prompt_limit" | "moss_image_size" | "moss_image_reference" | "moss_layout_reference" | "zhipu_speech_text" | "zhipu_speech_voice" | "zhipu_speech_reference" | "voice_library_unsupported" | "voice_connection_changed" | "voice_reference_required" | "voice_transcript_required" | "voice_transcript_too_long" | "voice_description_too_long" | "voice_not_found" | "voice_delete_unsupported" | "voice_consent_required" | "voice_name_invalid" | "voice_audio_invalid" | "voice_format" | "voice_size" | "voice_duration" | "voice_provider_protocol" | "voice_provider_error" | "voice_provider_timeout" | "voice_model_unsupported" | "voice_name_taken" | "voice_provider_quota"; readonly message: string; readonly objectId?: string | null; readonly field?: string | null; @@ -1527,7 +1527,7 @@ export interface ViewportInput extends PointInput { } export interface VoiceLibraryCapabilitiesInput extends ContractInput { - readonly provider: "vllm_omni" | "moss"; + readonly provider: "vllm_omni" | "moss" | "zhipu"; readonly connectionId: string; readonly requiresTranscript: boolean; readonly requiresReferenceAudio: boolean; @@ -1957,7 +1957,7 @@ export interface Contract { } export interface ContractIssue extends Contract { - readonly code: "timeline_package_invalid" | "timeline_package_too_large" | "timeline_revision_missing" | "timeline_import_missing" | "timeline_import_conflict" | "timeline_registration_rejected" | "timeline_font_too_large" | "timeline_font_unavailable" | "timeline_font_unsupported" | "timeline_font_invalid" | "timeline_font_fingerprint" | "timeline_font_coverage" | "timeline_font_total_bytes" | "timeline_font_unverified" | "timeline_font_mime" | "timeline_bundled_font_fingerprint" | "timeline_bundled_font_unavailable" | "invalid_input" | "revision_conflict" | "role_restricted" | "missing_object" | "reference_invalid" | "kling_model" | "kling_mode" | "kling_image" | "kling_frames" | "kling_prompt" | "kling_aspect" | "kling_output" | "kling_cancel" | "capability_unsupported" | "preview_expired" | "storage_unavailable" | "submission_unknown" | "provider_error" | "moss_speech_text" | "moss_speech_voice" | "moss_speech_reference" | "moss_prompt_limit" | "moss_image_size" | "moss_image_reference" | "moss_layout_reference" | "voice_library_unsupported" | "voice_connection_changed" | "voice_reference_required" | "voice_transcript_required" | "voice_transcript_too_long" | "voice_description_too_long" | "voice_not_found" | "voice_delete_unsupported" | "voice_consent_required" | "voice_name_invalid" | "voice_audio_invalid" | "voice_format" | "voice_size" | "voice_duration" | "voice_provider_protocol" | "voice_provider_error" | "voice_provider_timeout" | "voice_model_unsupported"; + readonly code: "timeline_package_invalid" | "timeline_package_too_large" | "timeline_revision_missing" | "timeline_import_missing" | "timeline_import_conflict" | "timeline_registration_rejected" | "timeline_font_too_large" | "timeline_font_unavailable" | "timeline_font_unsupported" | "timeline_font_invalid" | "timeline_font_fingerprint" | "timeline_font_coverage" | "timeline_font_total_bytes" | "timeline_font_unverified" | "timeline_font_mime" | "timeline_bundled_font_fingerprint" | "timeline_bundled_font_unavailable" | "invalid_input" | "revision_conflict" | "role_restricted" | "missing_object" | "reference_invalid" | "kling_model" | "kling_mode" | "kling_image" | "kling_frames" | "kling_prompt" | "kling_aspect" | "kling_output" | "kling_cancel" | "capability_unsupported" | "preview_expired" | "storage_unavailable" | "submission_unknown" | "provider_error" | "moss_speech_text" | "moss_speech_voice" | "moss_speech_reference" | "moss_prompt_limit" | "moss_image_size" | "moss_image_reference" | "moss_layout_reference" | "zhipu_speech_text" | "zhipu_speech_voice" | "zhipu_speech_reference" | "voice_library_unsupported" | "voice_connection_changed" | "voice_reference_required" | "voice_transcript_required" | "voice_transcript_too_long" | "voice_description_too_long" | "voice_not_found" | "voice_delete_unsupported" | "voice_consent_required" | "voice_name_invalid" | "voice_audio_invalid" | "voice_format" | "voice_size" | "voice_duration" | "voice_provider_protocol" | "voice_provider_error" | "voice_provider_timeout" | "voice_model_unsupported" | "voice_name_taken" | "voice_provider_quota"; readonly message: string; readonly objectId: string | null; readonly field: string | null; @@ -3096,7 +3096,7 @@ export interface Viewport extends Point { } export interface VoiceLibraryCapabilities extends Contract { - readonly provider: "vllm_omni" | "moss"; + readonly provider: "vllm_omni" | "moss" | "zhipu"; readonly connectionId: string; readonly requiresTranscript: boolean; readonly requiresReferenceAudio: boolean; diff --git a/src/i18n/messages/en-US/errors.ts b/src/i18n/messages/en-US/errors.ts index 9b43de92..98df5b59 100644 --- a/src/i18n/messages/en-US/errors.ts +++ b/src/i18n/messages/en-US/errors.ts @@ -139,7 +139,7 @@ export const errors = { "Enter a voice id supported by the service (letters, numbers, dots, underscores, or hyphens).", "errors.rules.voiceRequired": "Enter a valid Voice ID from the service's voice library before generating.", "errors.rules.referencesUnsupported": "This model does not support references. Disconnect the references before generating.", - "errors.generationFailure.voiceNotFound": "The voice was not found or is unavailable to this account. Check the Voice ID in the MOSS voice library.", + "errors.generationFailure.voiceNotFound": "The voice was not found or is unavailable to this account. Select an available voice in the voice library.", "errors.rules.audioDisable": "The video adapter does not support disabling generated audio.", "errors.rules.duration": diff --git a/src/i18n/messages/zh-CN/errors.ts b/src/i18n/messages/zh-CN/errors.ts index 77a9cc13..4c145768 100644 --- a/src/i18n/messages/zh-CN/errors.ts +++ b/src/i18n/messages/zh-CN/errors.ts @@ -99,7 +99,7 @@ export const errors = { "errors.rules.voiceId": "请输入服务支持的音色标识(字母、数字、点、下划线或连字符)。", "errors.rules.voiceRequired": "请先填写服务音色库中有效的音色 ID,再开始生成。", "errors.rules.referencesUnsupported": "当前模型不支持参考素材,请断开参考后再生成。", - "errors.generationFailure.voiceNotFound": "该音色不存在或当前账户无权使用,请在 MOSS 音色库核对音色 ID。", + "errors.generationFailure.voiceNotFound": "该音色不存在或当前账户无权使用,请在音色库重新选择可用音色。", "errors.rules.audioDisable": "当前视频适配器不支持关闭生成音频。", "errors.rules.duration": "当前模型不支持所选时长,请重新选择。", "errors.rules.resolution": "当前模型或参考模式不支持所选分辨率,请重新选择。", diff --git a/tests/assistantReadiness.test.ts b/tests/assistantReadiness.test.ts new file mode 100644 index 00000000..c8775884 --- /dev/null +++ b/tests/assistantReadiness.test.ts @@ -0,0 +1,14 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { needsAgentModel } from "../src/canvas/assistantReadiness.ts"; + +const ready = { status: "ready", modelSource: "settings" } as const; + +test("explains a disabled send only for a ready settings engine without a conversation model", () => { + assert.equal(needsAgentModel(true, ready, false), true); + assert.equal(needsAgentModel(true, ready, true), false); + assert.equal(needsAgentModel(false, ready, false), false); + assert.equal(needsAgentModel(true, { ...ready, status: "checking" }, false), false); + assert.equal(needsAgentModel(true, { ...ready, modelSource: "cli" }, false), false); + assert.equal(needsAgentModel(true, undefined, false), false); +}); diff --git a/tests/browser/manifest.json b/tests/browser/manifest.json index a6e8c1ab..b9cb9d50 100644 --- a/tests/browser/manifest.json +++ b/tests/browser/manifest.json @@ -468,6 +468,11 @@ "gate" ], "note": "Runs with voice_library_server.py in the Release checks voice library phase." + }, + { + "file": "zhipu_real.mjs", + "helper": true, + "note": "Manual acceptance against the real Zhipu BigModel service; needs ZHIPU_API_KEY and incurs charges, so no lane runs it." } ] } diff --git a/tests/browser/zhipu_real.mjs b/tests/browser/zhipu_real.mjs new file mode 100644 index 00000000..4bd488d4 --- /dev/null +++ b/tests/browser/zhipu_real.mjs @@ -0,0 +1,134 @@ +/** Real Zhipu GLM-TTS evidence: preset narration and the clone lifecycle through the UI. + * Requires a running OpenDirector (OPENDIRECTOR_URL) on an isolated OPENDIRECTOR_ROOT, + * ZHIPU_API_KEY for a funded BigModel account, and ZHIPU_SAMPLE pointing to a 3–30 s WAV. + * Contacts the real service and incurs charges; it deletes the voice it creates. + */ +import assert from "node:assert/strict"; +import { mkdir, readFile } from "node:fs/promises"; +import { launchBrowser } from "./engine.mjs"; + +const base = process.env.OPENDIRECTOR_URL ?? "http://127.0.0.1:18238"; +const key = process.env.ZHIPU_API_KEY; +const sample = process.env.ZHIPU_SAMPLE; +const output = process.env.ZHIPU_SCREENSHOTS ?? "/tmp/opendirector-zhipu-evidence"; +assert.ok(key && sample, "Set ZHIPU_API_KEY and ZHIPU_SAMPLE"); +const headers = { "X-OpenDirector-Request": "1", "Content-Type": "application/json" }; +async function call(path, body) { + const response = await fetch(base + path, { headers, ...(body ? { method: "POST", body: JSON.stringify(body) } : {}) }); + assert.ok(response.ok, await response.clone().text()); + return response.json(); +} +async function until(check, attempts = 900) { + for (let i = 0; i < attempts; i++) { + const value = await check(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw Error("Expected state was not observed"); +} +const copy = { + "zh-CN": { tools: "画布创作工具", add: "添加音频", model: "生成模型", prompt: "朗读文本", library: "音色库", + mine: "我的音色", create: "创建音色", name: "音色名称", recording: "参考录音", transcript: "录音逐字稿(选填)", + generate: "提交生成", saved: "已保存到本机", delete: "删除音色", + presetText: "这是 OpenDirector 使用智谱预置音色彤彤生成的旁白。", cloneText: "这是使用刚刚复刻的音色生成的旁白。" }, + "en-US": { tools: "Canvas creation tools", add: "Add Audio", model: "Generation model", prompt: "Text to read", + library: "Voice library", mine: "My voices", create: "Create voice", name: "Voice name", recording: "Reference recording", + transcript: "Recording transcript (optional)", generate: "Generate", saved: "Saved on this device", delete: "Delete voice", + presetText: "OpenDirector narration with the Zhipu preset voice Tongtong.", cloneText: "Narration with the voice that was just cloned." }, +}; +const config = await call("/api/model-configs", { kind: "audio", url: "https://open.bigmodel.cn/api/paas/v4", modelId: "glm-tts", + apiKey: key, connectionType: "api", protocol: "openai", authMode: "bearer" }); +assert.equal(config.inputRules.profile, "zhipu-speech"); +const recording = await readFile(sample); +await mkdir(output, { recursive: true }); +const browser = await launchBrowser(); +const errors = []; +try { + for (const [locale, L] of Object.entries(copy)) { + const page = await browser.newPage({ viewport: { width: 1500, height: 1000 } }); + page.setDefaultTimeout(60000); + page.on("pageerror", (error) => errors.push(error.message)); + page.on("request", (request) => { + if (/bigmodel\.cn|ufileos\.com/.test(request.url())) errors.push("Browser contacted provider directly"); + }); + const id = crypto.randomUUID(); + await call(`/api/creation/projects/${id}/commands`, { commandId: crypto.randomUUID(), projectId: id, + type: "project.create", payload: { name: `GLM-TTS ${locale} ${id.slice(0, 8)}` } }); + const state = () => call(`/api/creation/projects/${id}`); + const params = async () => (await state()).generationSpecs[0]?.parameters; + await page.addInitScript((locale) => { + localStorage.setItem("opendirector.modelSetup.v1", "dismissed"); + localStorage.setItem("opendirector.locale.v1", locale); + }, locale); + const dialog = () => page.getByRole("dialog", { name: L.library, exact: true }); + const open = () => page.getByRole("button", { name: L.library, exact: true }).click(); + const generate = async (text, count) => { + await page.getByRole("textbox", { name: L.prompt, exact: true }).fill(text); + await until(async () => (await params())?.voice); + await page.getByRole("button", { name: L.generate, exact: true }).click(); + const job = await until(async () => { + const jobs = (await state()).jobs; + return jobs.length === count && ["succeeded", "failed"].includes(jobs.at(-1).status) && jobs.at(-1); + }); + assert.equal(job.status, "succeeded", JSON.stringify(job)); + return job; + }; + await page.goto(`${base}/ai-tool/ai-canvas/${id}?creation=unified`); + await page.getByText(L.saved, { exact: true }).waitFor(); + await page.getByRole("toolbar", { name: L.tools }).getByRole("button", { name: L.add, exact: true }).click(); + await page.getByRole("combobox", { name: L.model, exact: true }).click(); + await page.getByRole("listbox", { name: L.model, exact: true }).locator(`[data-option-id="config:${config.id}"]`).click(); + await until(async () => (await state()).generationSpecs[0]?.modelConfigId === config.id); + + // Preset voice: the library lists the documented system voices. + await open(); + await dialog().getByRole("button", { name: "tongtong", exact: true }).waitFor(); + for (const voice of ["chuichui", "xiaochen", "jam", "kazi", "douji", "luodo"]) + await dialog().getByRole("button", { name: voice, exact: true }).waitFor(); + await page.screenshot({ path: `${output}/${locale}-1-preset-library.png`, fullPage: true }); + await dialog().getByRole("button", { name: "tongtong", exact: true }).click(); + await until(async () => (await params()).voice === "tongtong"); + const preset = await generate(L.presetText, 1); + assert.equal(preset.execution.voice, "tongtong"); + assert.equal(preset.execution.format, "wav"); + await page.locator("audio").first().waitFor({ state: "attached" }); + await page.screenshot({ path: `${output}/${locale}-2-preset-result.png`, fullPage: true }); + + // Clone: upload, create and select the new voice. + await open(); + await dialog().getByRole("tab", { name: L.mine, exact: true }).click(); + await dialog().getByRole("button", { name: L.create, exact: true }).click(); + const name = `od238-${locale}-${id.slice(0, 6)}`; + await dialog().getByLabel(L.recording, { exact: true }).setInputFiles({ name: "sample.wav", mimeType: "audio/wav", buffer: recording }); + await dialog().getByLabel(L.name, { exact: true }).fill(name); + await dialog().getByLabel(L.transcript, { exact: true }).fill(process.env.ZHIPU_SAMPLE_TEXT ?? ""); + await dialog().getByText(/sample\.wav/).waitFor(); + await page.screenshot({ path: `${output}/${locale}-3-clone-form.png`, fullPage: true }); + await dialog().getByRole("button", { name: L.create, exact: true }).click(); + const selection = await until(async () => (await params())?.voiceSelection?.name === name && (await params()).voiceSelection); + assert.equal(selection.source, "uploaded"); + await until(async () => (await dialog().count()) === 0); + const cloned = await generate(L.cloneText, 2); + assert.equal(cloned.execution.voice, selection.voiceId); + await page.screenshot({ path: `${output}/${locale}-4-clone-result.png`, fullPage: true }); + + // Delete the cloned voice from the library. + await open(); + await dialog().getByRole("tab", { name: L.mine, exact: true }).click(); + const entry = dialog().locator(".cw-voice-entry").filter({ has: page.getByRole("button", { name, exact: true }) }); + await entry.waitFor(); + await entry.locator("summary").click(); + await entry.getByRole("button", { name: L.delete, exact: true }).click(); + await page.screenshot({ path: `${output}/${locale}-5-delete-confirm.png`, fullPage: true }); + await dialog().getByRole("alertdialog").getByRole("button", { name: L.delete, exact: true }).click(); + await until(async () => (await entry.count()) === 0); + const listed = await call(`/api/voices?modelConfigId=${config.id}`); + assert.ok(!listed.voices.some((voice) => voice.voiceId === selection.voiceId)); + await page.screenshot({ path: `${output}/${locale}-6-after-delete.png`, fullPage: true }); + console.log(locale, "preset", preset.id, "clone", selection.voiceId, "deleted"); + await page.close(); + } + assert.deepEqual(errors, []); +} finally { + await browser.close(); +} diff --git a/tests/generationFailureText.test.ts b/tests/generationFailureText.test.ts index 362c9b85..61f5f9f6 100644 --- a/tests/generationFailureText.test.ts +++ b/tests/generationFailureText.test.ts @@ -125,11 +125,15 @@ for (const message of [ test("voice_not_found identifies the voice before generic 404 or input failures", () => { try { setActiveLocale("en-US"); - assert.equal(generationFailureMessage(failure({ providerCode: "voice_not_found", httpStatus: 404 })), - "The voice was not found or is unavailable to this account. Check the Voice ID in the MOSS voice library."); + const english = "The voice was not found or is unavailable to this account. Select an available voice in the voice library."; + const chinese = "该音色不存在或当前账户无权使用,请在音色库重新选择可用音色。"; + // MOSS reports a voice_not_found code; Zhipu reports code 1214 with this backend message. + const zhipu = failure({ providerCode: "1214", httpStatus: 400, category: "input", message: "所选音色不存在或不可用。" }); + assert.equal(generationFailureMessage(failure({ providerCode: "voice_not_found", httpStatus: 404 })), english); + assert.equal(generationFailureMessage(zhipu), english); setActiveLocale("zh-CN"); - assert.equal(generationFailureMessage(failure({ providerCode: "voice_not_found", httpStatus: 404 })), - "该音色不存在或当前账户无权使用,请在 MOSS 音色库核对音色 ID。"); + assert.equal(generationFailureMessage(failure({ providerCode: "voice_not_found", httpStatus: 404 })), chinese); + assert.equal(generationFailureMessage(zhipu), chinese); } finally { setActiveLocale("zh-CN"); } diff --git a/tests/providerPresets.test.ts b/tests/providerPresets.test.ts index 77c00b4a..8dcbae2f 100644 --- a/tests/providerPresets.test.ts +++ b/tests/providerPresets.test.ts @@ -116,3 +116,30 @@ test("Kling Flash descriptions distinguish first-frame capabilities in both lang assert.doesNotMatch(modelDescription(model)!, /尾帧/); } finally { setActiveLocale(previous); } }); + +test("Zhipu BigModel preset offers GLM-TTS on the official API in both languages", () => { + const preset = PROVIDER_PRESETS.find((item) => item.id === "zhipu"); + assert.ok(preset); + assert.equal(preset.url, "https://open.bigmodel.cn/api/paas/v4"); + assert.equal(preset.protocol, "openai"); + assert.equal(preset.connectionType, "api"); + assert.equal(preset.authMode, "bearer"); + assert.deepEqual(preset.kinds, ["agent", "audio"]); + // Conversation models come from the service's /models list; speech stays fixed. + assert.deepEqual(preset.modelsByKind, { audio: ["glm-tts"] }); + assert.ok(providerOptions("agent").some((item) => item.id === "zhipu")); + for (const kind of ["image", "video", "vision"] as const) + assert.ok(!providerOptions(kind).some((item) => item.id === "zhipu")); + const model: ModelConfig = { ...preset, id: "saved", kind: "audio", modelId: "glm-tts", url: `${preset.url}/`, + inputRules: { profile: "zhipu-speech", allowedKinds: [], maxImages: 0, maxVideos: 0, maxAudios: 0, videoModes: [] } }; + assert.equal(providerPresetFor(model), "zhipu"); + const previous = getActiveLocale(); + try { + setActiveLocale("zh-CN"); + assert.equal(providerOptions("audio").find((item) => item.id === "zhipu")?.name, "智谱 BigModel 开放平台"); + setActiveLocale("en-US"); + assert.equal(providerOptions("audio").find((item) => item.id === "zhipu")?.name, "Zhipu BigModel Open Platform"); + } finally { + setActiveLocale(previous); + } +}); diff --git a/tests/test_zhipu.py b/tests/test_zhipu.py new file mode 100644 index 00000000..9f118b61 --- /dev/null +++ b/tests/test_zhipu.py @@ -0,0 +1,335 @@ +"""Zhipu BigModel GLM-TTS wire contracts, voice registry and shared generation acceptance. + +Provider HTTP is mocked from responses recorded against the real service. +""" +import io +import json +import wave +from uuid import uuid4 + +import httpx +import pytest + +from opendirector.application.voice_library import VoiceLibrary +from opendirector.domain.capabilities import SpeechRequest +from opendirector.domain.generation import GenerationError +from opendirector.domain.input_rules import input_rules +from opendirector.domain.projects import ProjectError +from opendirector.domain.zhipu_speech import ZHIPU_PRESET_VOICES, is_zhipu_speech +from opendirector.integrations.model_adapters import OpenAISpeech, ZhipuSpeech, adapter_for +from opendirector.integrations.generation_errors import provider_failure +from test_creation_generation_service import anyio_backend # noqa: F401 + + +SECRET = 'zhipu-private-test-key' +BASE = 'https://open.bigmodel.cn/api/paas/v4' +CDN = 'https://bigmodel-us3-prod-voice-clone.cn-wlcb.ufileos.com/out.wav?ufileattname=a.wav' +CLONED = 'b7f05169-b935-522f-9b0a-b49336783d42' + + +def config(**patch): + return {**dict(id=str(uuid4()), kind='audio', url=BASE, modelId='glm-tts', protocol='openai', + connectionType='api', authMode='bearer', apiKey=SECRET), **patch} + + +def wav(seconds=5.0, rate=24000): + stream = io.BytesIO() + with wave.open(stream, 'wb') as audio: + audio.setnchannels(1); audio.setsampwidth(2); audio.setframerate(rate) + audio.writeframes(b'\0\0' * int(rate * seconds)) + return stream.getvalue() + + +class Configs: + def __init__(self, value): self.value = value + def resolve(self, identity, kind): + assert identity == self.value['id'] and kind == 'audio' + return self.value + + +def public(url): + target = httpx.URL(url) + return target, {'Host': target.netloc.decode('ascii')}, {} + + +def library(value, handle, resolve_public=public): + return VoiceLibrary(Configs(value), transport=httpx.MockTransport(handle), resolve_public=resolve_public) + + +def listing(*private): + return httpx.Response(200, json={'voice_list': [ + {'voice': voice, 'voice_name': name, 'voice_type': 'PRIVATE', 'download_url': CDN, + 'create_time': '2026-11-10 10:05:39'} for voice, name in private]}) + + +def test_profile_requires_the_official_endpoint_and_model(): + value = config() + assert is_zhipu_speech(value) + rules = input_rules(value) + assert rules['profile'] == 'zhipu-speech' and rules['voiceRequired'] and rules['referencesAllowed'] is False + assert rules['maxPromptCharacters'] == 1024 + library_rules = rules['voiceLibrary'] + assert library_rules['provider'] == 'zhipu' and library_rules['canDelete'] + assert library_rules['formats'] == ['wav', 'mp3'] and library_rules['maxBytes'] == 10 * 1024 * 1024 + assert not library_rules['requiresTranscript'] and not library_rules['requiresReferenceAudio'] + assert SECRET not in json.dumps(rules) + assert isinstance(adapter_for(value), ZhipuSpeech) + for patch in ({'url': 'https://open.bigmodel.cn/api/paas/v3'}, {'url': 'http://open.bigmodel.cn/api/paas/v4'}, + {'url': 'https://bigmodel.example/api/paas/v4'}, {'modelId': 'glm-tts-clone'}, + {'connectionType': 'self_hosted'}, {'kind': 'image'}): + other = config(**patch) + assert not is_zhipu_speech(other) + assert input_rules(other)['profile'] != 'zhipu-speech' + # A self-hosted GLM-TTS keeps the vLLM-Omni voice registry and plain speech adapter. + local = config(url='http://gpu.test/v1', connectionType='self_hosted') + assert input_rules(local)['voiceLibrary']['provider'] == 'vllm_omni' + assert isinstance(adapter_for(local), OpenAISpeech) + + +@pytest.mark.anyio +async def test_speech_sends_wav_request_and_accepts_only_a_wave_body(): + calls = [] + def handle(request): + calls.append(request) + assert str(request.url) == BASE + '/audio/speech' + assert request.headers['authorization'] == f'Bearer {SECRET}' + assert json.loads(request.content) == dict(model='glm-tts', input='你好', voice='tongtong', + response_format='wav', stream=False, watermark_enabled=False) + return httpx.Response(200, content=wav(.2), headers={'content-type': 'audio/wav; charset=UTF-8'}) + result = await ZhipuSpeech(httpx.MockTransport(handle)).speech(config(), SpeechRequest('你好', 'tongtong'), watermark=False) + assert result.suffix == 'wav' and result.data[:4] == b'RIFF' + assert len(calls) == 1 + + +@pytest.mark.anyio +@pytest.mark.parametrize('reply', [ + httpx.Response(200, content=b'\x00\x00+\x00\xf5\x00', headers={'content-type': 'audio/pcm'}), + httpx.Response(200, content=b'', headers={'content-type': 'audio/wav'}), + httpx.Response(200, content=b'data: {"choices":[]}\n\n', headers={'content-type': 'text/event-stream'}), +]) +async def test_non_wave_success_is_a_protocol_failure(reply): + with pytest.raises(GenerationError) as caught: + await ZhipuSpeech(httpx.MockTransport(lambda _: reply)).speech(config(), SpeechRequest('你好', 'tongtong')) + assert caught.value.details.category == 'submission_unknown' or caught.value.details.category == 'protocol' + + +@pytest.mark.anyio +async def test_json_error_inside_a_success_status_is_a_provider_failure(): + reply = httpx.Response(200, json={'error': {'code': '500', 'message': "Request method 'DELETE' not supported"}}) + with pytest.raises(GenerationError) as caught: + await ZhipuSpeech(httpx.MockTransport(lambda _: reply)).speech(config(), SpeechRequest('你好', 'tongtong')) + assert caught.value.details.providerCode == '500' + + +@pytest.mark.anyio +@pytest.mark.parametrize('status,body,category', [ + (429, {'error': {'code': '1113', 'message': '余额不足或无可用资源包,请充值。'}}, 'quota'), + (400, {'error': {'code': '1214', 'message': '音色不存在'}}, 'input'), + (401, {'error': {'code': '1000', 'message': '身份验证失败。'}}, 'authentication'), +]) +async def test_provider_errors_are_classified(status, body, category): + with pytest.raises(GenerationError) as caught: + await ZhipuSpeech(httpx.MockTransport(lambda _: httpx.Response(status, json=body))).speech( + config(), SpeechRequest('你好', 'tongtong')) + assert caught.value.details.category == category + assert SECRET not in str(caught.value) + + +def test_quota_is_not_a_rate_limit_and_missing_voice_is_readable(): + quota = provider_failure({'error': {'code': '1113', 'message': '余额不足或无可用资源包,请充值。'}}, stage='submit', http_status=429) + assert quota.details.category == 'quota' + missing = provider_failure({'error': {'code': '1214', 'message': '音色不存在'}}, stage='submit', http_status=400) + assert missing.details.message == '所选音色不存在或不可用。' + + +@pytest.mark.anyio +@pytest.mark.parametrize('text,voice,code', [ + ('', 'tongtong', 'zhipu_speech_text'), ('字' * 1025, 'tongtong', 'zhipu_speech_text'), + ('你好', None, 'zhipu_speech_voice'), ('你好', '音色', 'zhipu_speech_voice'), +]) +async def test_invalid_input_never_reaches_the_provider(text, voice, code): + adapter = ZhipuSpeech(httpx.MockTransport(lambda _: pytest.fail('invalid input reached provider'))) + with pytest.raises(ProjectError) as error: + await adapter.speech(config(), SpeechRequest(text, voice)) + assert error.value.code == code + + +def test_list_offers_documented_presets_and_private_clones_only(): + value = config() + def handle(request): + assert request.method == 'GET' and request.url.path == '/api/paas/v4/voice/list' + assert request.url.params['voiceType'] == 'PRIVATE' + return listing((CLONED, '旁白 测试/一')) + result = library(value, handle).list(value['id']) + voices = {voice['voiceId']: voice for voice in result['voices']} + assert list(voices)[:len(ZHIPU_PRESET_VOICES)] == list(ZHIPU_PRESET_VOICES) + assert all(voices[name]['source'] == 'builtin' and not voices[name]['canDelete'] for name in ZHIPU_PRESET_VOICES) + clone = voices[CLONED] + assert clone['name'] == '旁白 测试/一' and clone['source'] == 'uploaded' and clone['canDelete'] and clone['compatible'] + assert clone['previewUrl'].startswith('/api/voices/' + CLONED + '/preview?') + assert result['nextCursor'] is None + + +@pytest.mark.parametrize('body', [{'voice_list': [{'voice': 'x', 'voice_type': 'OFFICIAL'}]}, {'voice_list': None}, {}]) +def test_unexpected_listing_is_rejected(body): + value = config() + with pytest.raises(ProjectError) as error: + library(value, lambda _: httpx.Response(200, json=body)).list(value['id']) + assert error.value.code == 'voice_provider_protocol' + + +def test_create_uploads_clones_cleans_up_and_reads_back(): + value = config() + calls = [] + def handle(request): + calls.append((request.method, request.url.path)) + if request.url.path.endswith('/files') and request.method == 'POST': + body = request.read() + assert b'name="purpose"\r\n\r\nvoice-clone-input' in body and b'RIFF' in body + return httpx.Response(200, json={'id': 'in-1.wav', 'object': 'file', 'purpose': 'voice-clone-input'}) + if request.url.path.endswith('/voice/clone'): + payload = json.loads(request.content) + assert payload['model'] == 'glm-tts-clone' and payload['voice_name'] == 'Alice' + assert payload['file_id'] == 'in-1.wav' and payload['input'].strip() and payload['text'] == '你好' + return httpx.Response(200, json={'voice': CLONED, 'file_id': 'out-1.wav', 'file_purpose': 'voice-clone-output'}) + if request.method == 'DELETE': + assert request.url.path.endswith('/files/in-1.wav') + return httpx.Response(200, json={'id': 'in-1.wav', 'deleted': True}) + return listing((CLONED, 'Alice')) + record = library(value, handle).create(value['id'], wav(), filename='sample.wav', name='Alice', ref_text='你好') + assert record['voiceId'] == CLONED and record['source'] == 'uploaded' and record['previewUrl'] + assert calls[:3] == [('POST', '/api/paas/v4/files'), ('POST', '/api/paas/v4/voice/clone'), + ('DELETE', '/api/paas/v4/files/in-1.wav')] + + +@pytest.mark.parametrize('message,code,status', [ + ('音色名称已存在', 'voice_name_taken', 409), + ('API 调用参数有误,请检查文档。no audio segment found', 'voice_provider_error', 400), +]) +def test_failed_clone_still_deletes_the_uploaded_sample(message, code, status): + value = config() + deleted = [] + def handle(request): + if request.url.path.endswith('/files') and request.method == 'POST': + return httpx.Response(200, json={'id': 'in-2.wav'}) + if request.method == 'DELETE': + deleted.append(request.url.path) + return httpx.Response(200, json={'deleted': True}) + return httpx.Response(400, json={'error': {'code': '1214' if '名称' in message else '1210', 'message': message}}) + with pytest.raises(ProjectError) as error: + library(value, handle).create(value['id'], wav(), filename='sample.wav', name='Alice') + assert error.value.code == code and error.value.status == status + assert deleted == ['/api/paas/v4/files/in-2.wav'] + + +@pytest.mark.parametrize('data,filename,code', [ + (wav(1), 'sample.wav', 'voice_duration'), (wav(31), 'sample.wav', 'voice_duration'), + (wav(), 'sample.m4a', 'voice_format'), (b'not audio', 'sample.wav', 'voice_audio_invalid'), +]) +def test_invalid_recording_never_reaches_the_provider(data, filename, code): + value = config() + service = library(value, lambda _: pytest.fail('invalid upload reached provider')) + with pytest.raises(ProjectError) as error: + service.create(value['id'], data, filename=filename, name='Alice') + assert error.value.code == code + + +def test_quota_during_registry_calls_is_reported_as_quota(): + value = config() + reply = httpx.Response(429, json={'error': {'code': '1113', 'message': '余额不足或无可用资源包,请充值。'}}) + with pytest.raises(ProjectError) as error: + library(value, lambda _: reply).list(value['id']) + assert error.value.code == 'voice_provider_quota' + + +def test_delete_posts_the_voice_and_rejects_presets_and_unknown_voices(): + value = config() + scope = input_rules(value)['voiceLibrary']['connectionId'] + posted = [] + def handle(request): + if request.url.path.endswith('/voice/delete'): + posted.append(json.loads(request.content)) + return httpx.Response(200, json={'voice': CLONED, 'update_time': '2026-11-10 10:05:57'}) + return listing((CLONED, 'Alice')) + service = library(value, handle) + service.delete(value['id'], CLONED, scope) + assert posted == [{'voice': CLONED}] + for voice, code in [('tongtong', 'voice_delete_unsupported'), ('missing', 'voice_not_found')]: + with pytest.raises(ProjectError) as error: service.delete(value['id'], voice, scope) + assert error.value.code == code + with pytest.raises(ProjectError) as error: service.delete(value['id'], CLONED, 'other') + assert error.value.code == 'voice_connection_changed' + assert len(posted) == 1 + + +def test_delete_error_in_success_envelope_is_not_reported_as_deleted(): + value = config() + scope = input_rules(value)['voiceLibrary']['connectionId'] + def handle(request): + if request.url.path.endswith('/voice/delete'): + return httpx.Response(200, json={'error': {'code': '500', 'message': 'not supported'}}) + return listing((CLONED, 'Alice')) + with pytest.raises(ProjectError) as error: + library(value, handle).delete(value['id'], CLONED, scope) + assert error.value.code == 'voice_provider_error' + + +def test_preview_downloads_cdn_wave_without_credentials(): + value = config() + scope = input_rules(value)['voiceLibrary']['connectionId'] + def handle(request): + if request.url.host == 'open.bigmodel.cn': + return listing((CLONED, 'Alice')) + assert 'authorization' not in request.headers + return httpx.Response(200, content=wav(.2), headers={'content-type': 'application/octet-stream'}) + data, mime = library(value, handle).preview(value['id'], CLONED, scope) + assert mime == 'audio/wav' and data[:4] == b'RIFF' + + +def test_preview_rejects_non_wave_and_private_destinations(): + value = config() + scope = input_rules(value)['voiceLibrary']['connectionId'] + def handle(request): + if request.url.host == 'open.bigmodel.cn': + return listing((CLONED, 'Alice')) + return httpx.Response(200, content=b'', headers={'content-type': 'application/octet-stream'}) + with pytest.raises(ProjectError) as error: + library(value, handle).preview(value['id'], CLONED, scope) + assert error.value.code == 'voice_provider_error' + def private(request): + if request.url.host == 'open.bigmodel.cn': + return httpx.Response(200, json={'voice_list': [{'voice': CLONED, 'voice_name': 'A', 'voice_type': 'PRIVATE', + 'download_url': 'https://127.0.0.1/x.wav'}]}) + pytest.fail('private preview destination was contacted') + from opendirector.application.voice_library import _public_https + with pytest.raises(ProjectError) as error: + library(value, private, resolve_public=_public_https).preview(value['id'], CLONED, scope) + assert error.value.code == 'voice_provider_error' + + +def test_generation_snapshot_and_preflight(tmp_path): + from generation_helpers import command, new_project + from opendirector.application.creation_generation import CreationGeneration + from opendirector.infrastructure.model_configs import ModelConfigs + project = new_project(tmp_path) + configs = ModelConfigs(tmp_path) + saved = configs.save('audio', BASE, 'glm-tts', SECRET, protocol='openai') + assert saved['inputRules']['profile'] == 'zhipu-speech' + runner = CreationGeneration(project[0], configs) + runner.voices = library(configs.resolve(saved['id'], 'audio'), lambda _: listing((CLONED, 'Alice'))) + scope = saved['inputRules']['voiceLibrary']['connectionId'] + deleted = dict(connectionId=scope, voiceId='missing', name='Gone', source='uploaded', hasReferenceAudio=True) + for parameters, code in [({'voice': None}, 'zhipu_speech_voice'), + ({'voice': 'missing', 'voiceSelection': deleted}, 'voice_not_found'), + ({'voice': CLONED, 'voiceSelection': {**deleted, 'voiceId': CLONED, 'connectionId': 'stale'}}, 'voice_connection_changed')]: + request = command(project, saved, prompt='Read this', parameters=parameters) + receipt = runner.prepare(request, session=project[2]) + assert receipt.status == 'rejected' and receipt.issues[0].code == code, receipt + assert project[0].snapshot(project[1]).jobs == () + selection = {**deleted, 'voiceId': CLONED, 'name': 'Alice'} + request = command(project, saved, prompt='Read this', parameters={'voice': CLONED, 'voiceSelection': selection}) + receipt = runner.prepare(request, session=project[2]) + assert receipt.status != 'rejected', receipt + job = project[0].snapshot(project[1]).jobs[-1] + execution = job.execution.model_dump() if hasattr(job.execution, 'model_dump') else job.execution + assert execution['voice'] == CLONED and execution['format'] == 'wav'
{t("voiceLibrary.transcriptRequired")}