diff --git a/apps/api/src/opendirector_api/localization.py b/apps/api/src/opendirector_api/localization.py index 0586213f..aeb4fbc0 100644 --- a/apps/api/src/opendirector_api/localization.py +++ b/apps/api/src/opendirector_api/localization.py @@ -43,9 +43,14 @@ 'voice_provider_protocol': 'The voice service returned an invalid response or pagination did not finish. Check its API version.', 'voice_provider_error': 'The voice service request failed. Check the connection and account permissions.', 'voice_provider_timeout': 'The voice service timed out. Check its status and try again.', + 'voice_name_taken': 'This account already has a voice with that name. Choose another name.', + 'voice_provider_quota': 'The voice service account has insufficient balance or resource packages. Top up and try again.', "moss_speech_text": "Enter the text to read aloud.", "moss_speech_voice": "Enter a Voice ID available to your MOSS account.", "moss_speech_reference": "MOSS speech generation does not accept references. Enter speech text and a Voice ID.", + "zhipu_speech_text": "Enter the text to read aloud, at most 1,024 characters per GLM-TTS request.", + "zhipu_speech_voice": "Select a voice available to your Zhipu account.", + "zhipu_speech_reference": "GLM-TTS does not accept references. Enter speech text and select a voice.", "moss_prompt_limit": "MOSS requires a nonempty prompt of at most 4,000 characters, including text references.", "moss_image_size": "MOSS image dimensions must be 256–2048, multiples of 16, and no more than 2,097,152 pixels in total.", "moss_image_reference": "MOSS image generation does not accept image references or editing inputs.", diff --git a/docs/development/validation.md b/docs/development/validation.md index 3708a6bd..451ced37 100644 --- a/docs/development/validation.md +++ b/docs/development/validation.md @@ -249,6 +249,16 @@ CREATION_API_URL=http://127.0.0.1:8019 CREATION_WEB_URL=http://127.0.0.1:5179 no Python 用例使用模拟 HTTP 服务及真实 PNG、MP3,检查请求字段、音色、执行快照、文件探测、工程重开和下载恢复不重复生成。通用错误分类和安全提示由现有错误测试负责。浏览器用例检查两种语言下的提供商及型号列表、获取模型失败的具体原因、音色必填、保存重开和实际音频播放;播放器使用隔离服务的 WAV 替身,不代表 MOSS 已生成语音。真实验收另需有效 MOSS 凭证和音色 ID,并保留实际音频、模型标识及播放器截图。 +智谱 BigModel GLM-TTS 接入使用以下检查: + +```sh +.venv/bin/python -m pytest tests/test_zhipu.py tests/test_voice_library.py tests/test_generation_errors.py -q +# 真实服务验收(会产生费用):在隔离的 OPENDIRECTOR_ROOT 上启动 opendirector serve 后运行 +OPENDIRECTOR_URL=http://127.0.0.1:18238 ZHIPU_API_KEY=... ZHIPU_SAMPLE=/path/to/sample.wav node tests/browser/zhipu_real.mjs +``` + +Python 用例使用按真实服务响应录制的模拟 HTTP,检查 `/audio/speech` 只请求非流式 WAV 并按 RIFF/WAVE 字节验收(服务默认返回裸 PCM,且部分错误以 HTTP 200 + JSON 返回)、`1113` 余额不足归为额度而非限流、音色库只列出文档中的 7 个预置音色加 `PRIVATE` 复刻音色、复刻前的真实音频校验(mp3/wav、3–30 秒、10 MB)、复刻成功或失败后都删除上传的样本文件、`POST /voice/delete` 删除及重名/不存在的错误映射、试听只从公网 HTTPS 下载且不带凭据,以及统一生成服务的音色预检与执行快照。真实验收需要已充值且开通 GLM-TTS 的 BigModel 账户;保留预置音色与复刻音色生成的实际音频、复刻到删除的完整流程截图,并确认账户中没有遗留的 `voice-clone-input` 文件。 + ### Voice library / 音色库 diff --git a/docs/guides/user-guide.md b/docs/guides/user-guide.md index d8a7346b..17c818bd 100644 --- a/docs/guides/user-guide.md +++ b/docs/guides/user-guide.md @@ -93,6 +93,10 @@ On MOSS Open Platform, select the stable `moss-tts-1.5-flash` or `moss-tts-1.0-p Choose **Create voice** to upload a local recording, name it, and review its playback, format, size and duration before submitting. vLLM-Omni requires the consent recording ID; adapters such as Drama3 also require the recording transcript. MOSS-TTS-Nano needs a registered voice with reference audio: a listed `default` voice is insufficient. This upload registers the voice with the selected service without adding a canvas reference node. +On Zhipu BigModel Open Platform, select `glm-tts`. The library lists the seven documented system voices and the voices cloned on that account. To clone a voice, upload a 3–30 second MP3 or WAV recording of at most 10 MB; the recording transcript is optional but improves similarity. OpenDirector deletes the uploaded sample from the account after registration. Each request reads at most 1,024 characters, and references are not accepted. Cloned voices can be deleted from the library. + +The same provider can also be added as a conversation Agent: click **Fetch models** to read the account's GLM models (for example `glm-4.5-flash` or `glm-5.3-flash`) and select the ones to enable. + Selected voices are saved in the project; favorites are saved as browser preferences. Both are scoped to the configured service and account. Compatible models sharing a service can retain the selection; another service or account requires its own registered voice. Use a voice’s secondary menu to copy its provider ID or delete an uploaded voice where supported. Deletion requires confirmation; MOSS deletion is unavailable until its API supports it. Service errors are shown without credentials; an unsupported model keeps its existing voice controls. Agents use the same validation through `voice_list`, `voice_create`, and `voice_delete`. The [business CLI](cli.md) also exposes `voice list`, `voice create`, and `voice delete`. diff --git a/docs/guides/user-guide.zh-CN.md b/docs/guides/user-guide.zh-CN.md index 5328fcd7..3f947e08 100644 --- a/docs/guides/user-guide.zh-CN.md +++ b/docs/guides/user-guide.zh-CN.md @@ -93,6 +93,10 @@ HTTP 400。 选择**创建音色**,上传本地录音、填写名称,并在提交前检查试听、格式、大小和时长。vLLM-Omni 需要填写授权录音 ID;Drama3 等适配器还必须填写录音逐字稿。MOSS-TTS-Nano 需要带有参考音频的已注册音色,列表中的 `default` 不能代替录音。上传会直接在当前服务注册音色,不创建画布参考音频节点。 +在智谱 BigModel 开放平台上选择 `glm-tts`。音色库列出文档中的 7 个系统音色和该账户下已复刻的音色。复刻音色时,上传 3–30 秒、不超过 10 MB 的 MP3 或 WAV 录音;录音逐字稿可选,填写后相似度更高。注册完成后,OpenDirector 会从账户中删除上传的样本文件。每次最多朗读 1,024 个字符,不支持参考素材。复刻音色可以在音色库中删除。 + +该提供商也可以作为对话 Agent 添加:点击“获取模型”读取账户可用的 GLM 模型(例如 `glm-4.5-flash`、`glm-5.3-flash`),勾选需要启用的模型即可。 + 所选音色随工程保存,收藏作为浏览器偏好保存,均按服务连接和账户隔离。同一连接下的兼容模型可以保留选择;其他服务或账户必须单独注册音色。音色的更多操作菜单可复制服务端 ID,或在服务支持时删除上传音色。删除前需要确认;MOSS 尚未开放删除操作。服务错误不会显示凭据;不支持音色库的模型保留原有音色控件。 Agent 通过 `voice_list`、`voice_create`、`voice_delete` 使用同一套校验;[业务 CLI](cli.md) 也提供 `voice list`、`voice create`、`voice delete`。 diff --git a/packages/core/src/opendirector/application/creation_generation.py b/packages/core/src/opendirector/application/creation_generation.py index c94b6d23..59967e3c 100644 --- a/packages/core/src/opendirector/application/creation_generation.py +++ b/packages/core/src/opendirector/application/creation_generation.py @@ -212,6 +212,9 @@ def _parameters(self, spec, config, node, records): params.update(prompt_enhance=False, watermark=not self.settings.get()['removeWatermark']) if spec.kind == 'audio' and input_rules(config)['profile'] == 'moss-speech': params = dict(voice=intent['voice'], response_format='mp3', delivery_method='url', stream=False) + if spec.kind == 'audio' and input_rules(config)['profile'] == 'zhipu-speech': + params = dict(voice=intent['voice'], response_format='wav', stream=False, + watermark_enabled=not self.settings.get()['removeWatermark']) return params @staticmethod @@ -229,6 +232,8 @@ def _execution(kind, parameters, config): if kind == 'audio': if input_rules(config)['profile'] == 'moss-speech': return dict(kind=kind, voice=parameters['voice'], format='mp3') + if input_rules(config)['profile'] == 'zhipu-speech': + return dict(kind=kind, voice=parameters['voice'], format='wav', sampleRate=24000) return dict(kind=kind, voice=parameters.get('voice') or ('default' if config['connectionType'] == 'self_hosted' else 'alloy') if config['protocol'] == 'openai' else None, speaker=parameters.get('speaker') if config['protocol'] != 'openai' else None, format='mp3', sampleRate=24000 if config['protocol'] != 'openai' else None) diff --git a/packages/core/src/opendirector/application/voice_library.py b/packages/core/src/opendirector/application/voice_library.py index ed567186..cc7078dc 100644 --- a/packages/core/src/opendirector/application/voice_library.py +++ b/packages/core/src/opendirector/application/voice_library.py @@ -1,5 +1,7 @@ """Shared voice registry operations for HTTP, Agent and generation acceptance.""" import json +import socket +from ipaddress import ip_address from pathlib import Path import tempfile from time import monotonic @@ -9,6 +11,7 @@ from opendirector.domain.projects import ProjectError from opendirector.domain.voices import VOICE_CREATE_TEXT_LIMITS, voice_error, voice_library_capabilities, valid_voice_name, voice_incompatibility +from opendirector.domain.zhipu_speech import ZHIPU_CLONE_MODEL, ZHIPU_CLONE_PREVIEW_TEXT, ZHIPU_PRESET_VOICES from opendirector.integrations.http_settings import auth_headers, trust_environment from opendirector.integrations.volcengine import endpoint from opendirector.media.probe import probe_file @@ -19,9 +22,29 @@ _CURRENT_CONNECTION = object() +def _public_https(url): + """Pin an account listing's CDN preview to a checked public HTTPS address, keeping TLS identity.""" + parsed = urlsplit(url) + host = (parsed.hostname or '').rstrip('.').lower() + if parsed.scheme != 'https' or not host or parsed.username or parsed.password or host == 'localhost' or host.endswith('.localhost'): + raise voice_error('voice_provider_error', 502) + try: + answers = [answer[4][0] for answer in socket.getaddrinfo(host, parsed.port or 443, type=socket.SOCK_STREAM)] + except OSError: + raise voice_error('voice_provider_error', 502) from None + for answer in answers: + address = ip_address(answer) + address = getattr(address, 'ipv4_mapped', None) or address + if not address.is_global or address.is_multicast or address.is_reserved: + raise voice_error('voice_provider_error', 502) + if not answers: raise voice_error('voice_provider_error', 502) + target = httpx.URL(url) + return target.copy_with(host=answers[0]), {'Host': target.netloc.decode('ascii'), 'Connection': 'close'}, {'sni_hostname': target.host} + + class VoiceLibrary: - def __init__(self, configs, transport=None): - self.configs, self.transport = configs, transport + def __init__(self, configs, transport=None, resolve_public=_public_https): + self.configs, self.transport, self.resolve_public = configs, transport, resolve_public def _config(self, model_config_id): config = dict(self.configs.resolve(str(model_config_id), 'audio')) @@ -29,26 +52,23 @@ def _config(self, model_config_id): if capabilities is None: raise voice_error('voice_library_unsupported', 409) return config, capabilities - def _request(self, config, method, suffix='', **kwargs): + def _request(self, config, method, suffix='', *, path='/audio/voices', **kwargs): try: # Sync service runs in worker threads, including job preparation. with httpx.Client(transport=self.transport, trust_env=trust_environment(config['url'], config=config), timeout=httpx.Timeout(120, connect=15), follow_redirects=False) as client: - with client.stream(method, endpoint(config['url'], '/audio/voices') + suffix, + with client.stream(method, endpoint(config['url'], path) + suffix, headers=auth_headers(config), **kwargs) as response: data = bytearray() for chunk in response.iter_bytes(): data.extend(chunk) if len(data) > 2 * 1024 * 1024: raise voice_error('voice_provider_protocol', 502) if not response.is_success: - # Retain useful upstream detail; redact the configured credential - # before it reaches either the HTTP or Agent presentation boundary. - detail = self._provider_detail(data, config) - error = voice_error('voice_provider_error', response.status_code if 400 <= response.status_code < 500 else 502) - if detail: error.args = (str(error) + ' ' + detail,) - raise error + raise self._provider_failure(data, config, response.status_code) value = json.loads(data) if data else {} if not isinstance(value, dict): raise voice_error('voice_provider_protocol', 502) + # Zhipu reports some failures, such as an unsupported method, with HTTP 200. + if isinstance(value.get('error'), dict): raise self._provider_failure(data, config, 502) return value except httpx.TimeoutException: raise voice_error('voice_provider_timeout', 504) from None @@ -57,6 +77,20 @@ def _request(self, config, method, suffix='', **kwargs): except (ValueError, TypeError): raise voice_error('voice_provider_protocol', 502) from None + def _provider_failure(self, data, config, status): + # Retain useful upstream detail; redact the configured credential + # before it reaches either the HTTP or Agent presentation boundary. + detail = self._provider_detail(data, config) + try: error = json.loads(data).get('error') or {} + except (ValueError, AttributeError): error = {} + code = str(error.get('code', '')) if isinstance(error, dict) else '' + if code == '1113': return voice_error('voice_provider_quota', 402) + if code == '1214' and detail == '音色名称已存在': return voice_error('voice_name_taken', 409) + if code == '1214' and detail.startswith('音色不存在'): return voice_error('voice_not_found', 404) + error = voice_error('voice_provider_error', status if 400 <= status < 500 else 502) + if detail: error.args = (str(error) + ' ' + detail,) + return error + @staticmethod def _provider_detail(data, config): try: @@ -153,6 +187,22 @@ def _moss_page(self, config, capabilities, limit, after): raise voice_error('voice_provider_protocol', 502) return [self._record(item, capabilities, source='uploaded', config=config) for item in items], cursor if more else None + def _zhipu_page(self, config, capabilities): + # The documented system voices are always available; the account's + # OFFICIAL listing includes entries synthesis rejects, so it is not read. + records = [self._record({'id': voice}, capabilities, source='builtin', has_audio=False, config=config) + for voice in ZHIPU_PRESET_VOICES] + value = self._request(config, 'GET', path='/voice/list', params={'voiceType': 'PRIVATE'}) + items = value.get('voice_list') + if not isinstance(items, list) or len(items) > 10000: raise voice_error('voice_provider_protocol', 502) + for item in items: + if not isinstance(item, dict) or item.get('voice_type') != 'PRIVATE': + raise voice_error('voice_provider_protocol', 502) + records.append(self._record({'id': item.get('voice'), 'name': item.get('voice_name'), + 'preview_url': item.get('download_url'), 'created_at': item.get('create_time')}, + capabilities, source='uploaded', has_audio=True, config=config)) + return records, None + def _assert_current(self, config, capabilities): current, rules = self._config(config['id']) if rules != capabilities or current['modelId'] != config['modelId']: @@ -169,6 +219,8 @@ def _pages(self, config, capabilities, limit, after=None): self._assert_current(config, capabilities) if capabilities['provider'] == 'moss': records, cursor = self._moss_page(config, capabilities, limit, after) + elif capabilities['provider'] == 'zhipu': + records, cursor = self._zhipu_page(config, capabilities) else: records, cursor = self._vllm_page(config, capabilities) for record, _ in records: @@ -244,6 +296,8 @@ def create(self, model_config_id, data, *, filename, name, consent=None, ref_tex seconds = media['durationSeconds'] if (capabilities['minSeconds'] is not None and seconds < capabilities['minSeconds']) or (capabilities['maxSeconds'] is not None and seconds > capabilities['maxSeconds']): raise voice_error('voice_duration') fields = {'name': name} + if capabilities['provider'] == 'zhipu': + return self._zhipu_create(config, capabilities, data, extension, name, text) if capabilities['provider'] == 'vllm_omni': fields['consent'] = consent.strip() if text: fields['ref_text'] = text @@ -256,6 +310,30 @@ def create(self, model_config_id, data, *, filename, name, consent=None, ref_tex record, _ = self._record(item, capabilities, source='uploaded', has_audio=True, config=config) return record + def _zhipu_create(self, config, capabilities, data, extension, name, text): + # Upload the sample, register the clone, then delete the uploaded sample. + # The registered voice keeps working without it; the generated preview + # file remains because the listing's download URL serves it. + mime = {'wav': 'audio/wav', 'mp3': 'audio/mpeg'}[extension] + uploaded = self._request(config, 'POST', path='/files', data={'purpose': 'voice-clone-input'}, + files={'file': ('sample.' + extension, data, mime)}) + file_id = uploaded.get('id') + if not isinstance(file_id, str) or not 0 < len(file_id) <= 512: raise voice_error('voice_provider_protocol', 502) + try: + payload = dict(model=ZHIPU_CLONE_MODEL, voice_name=name, input=ZHIPU_CLONE_PREVIEW_TEXT, file_id=file_id) + if text: payload['text'] = text + value = self._request(config, 'POST', path='/voice/clone', json=payload) + finally: + try: self._request(config, 'DELETE', '/' + quote(file_id, safe=''), path='/files') + except ProjectError: pass + voice = value.get('voice') + if not valid_voice_name(voice): raise voice_error('voice_provider_protocol', 502) + # Read back the registered entry for its preview locator and stable fields. + record, _ = self._find(config, capabilities, voice) + if record is None: + record, _ = self._record({'id': voice, 'name': name}, capabilities, source='uploaded', has_audio=True, config=config) + return record + def preview(self, model_config_id, voice_id, connection_id): config, capabilities = self._config(model_config_id) if connection_id != capabilities['connectionId']: raise voice_error('voice_connection_changed', 409) @@ -264,15 +342,24 @@ def preview(self, model_config_id, voice_id, connection_id): parsed, configured = urlsplit(url), urlsplit(config['url']) def origin(value): return value.scheme, value.hostname, value.port or (443 if value.scheme == 'https' else 80) headers = auth_headers(config) if origin(parsed) == origin(configured) else {} + target, extensions = url, {} + if capabilities['provider'] == 'zhipu': + # The listing hands out a third-party CDN URL: no credentials, public addresses only. + target, headers, extensions = self.resolve_public(url) try: with httpx.Client(transport=self.transport, trust_env=trust_environment(url, config=config), timeout=30, follow_redirects=False) as client: - with client.stream('GET', url, headers=headers) as response: + with client.stream('GET', target, headers=headers, extensions=extensions) as response: mime = response.headers.get('content-type', '').split(';')[0].lower() - if not response.is_success or not mime.startswith('audio/'): raise voice_error('voice_provider_error', 502) + # Zhipu's CDN labels WAV previews application/octet-stream; check the bytes instead. + sniff = capabilities['provider'] == 'zhipu' + if not response.is_success or not (mime.startswith('audio/') or sniff): raise voice_error('voice_provider_error', 502) data = bytearray() for chunk in response.iter_bytes(): data.extend(chunk) if len(data) > 10 * 1024 * 1024: raise voice_error('voice_size', 413) + if sniff: + if not (data[:4] == b'RIFF' and data[8:12] == b'WAVE'): raise voice_error('voice_provider_error', 502) + mime = 'audio/wav' self._assert_current(config, capabilities) return bytes(data), mime except httpx.TimeoutException: raise voice_error('voice_provider_timeout', 504) from None @@ -288,6 +375,10 @@ def delete(self, model_config_id, voice_id, connection_id): if not record['canDelete']: raise voice_error('voice_delete_unsupported', 409) config, current = self._config(model_config_id) if current['connectionId'] != connection_id: raise voice_error('voice_connection_changed', 409) + if current['provider'] == 'zhipu': + value = self._request(config, 'POST', path='/voice/delete', json={'voice': voice_id}) + if value.get('voice') != voice_id: raise voice_error('voice_provider_protocol', 502) + return value = self._request(config, 'DELETE', '/' + quote(voice_id, safe='')) if value.get('success') is not True: raise voice_error('voice_provider_protocol', 502) diff --git a/packages/core/src/opendirector/domain/creation/commands.py b/packages/core/src/opendirector/domain/creation/commands.py index 7ca6b9a4..46317167 100644 --- a/packages/core/src/opendirector/domain/creation/commands.py +++ b/packages/core/src/opendirector/domain/creation/commands.py @@ -401,11 +401,13 @@ def command_fingerprint(command: Command) -> str: 'capability_unsupported', 'preview_expired', 'storage_unavailable', 'submission_unknown', 'provider_error', 'moss_speech_text', 'moss_speech_voice', 'moss_speech_reference', 'moss_prompt_limit', 'moss_image_size', 'moss_image_reference', 'moss_layout_reference', + 'zhipu_speech_text', 'zhipu_speech_voice', 'zhipu_speech_reference', 'voice_library_unsupported', 'voice_connection_changed', 'voice_reference_required', 'voice_transcript_required', 'voice_transcript_too_long', 'voice_description_too_long', 'voice_not_found', 'voice_delete_unsupported', 'voice_consent_required', 'voice_name_invalid', 'voice_audio_invalid', 'voice_format', - 'voice_size', 'voice_duration', 'voice_provider_protocol', 'voice_provider_error', 'voice_provider_timeout', 'voice_model_unsupported'] + 'voice_size', 'voice_duration', 'voice_provider_protocol', 'voice_provider_error', 'voice_provider_timeout', 'voice_model_unsupported', + 'voice_name_taken', 'voice_provider_quota'] class ContractIssue(Contract): diff --git a/packages/core/src/opendirector/domain/creation/generation.py b/packages/core/src/opendirector/domain/creation/generation.py index 974f43e0..29196cd1 100644 --- a/packages/core/src/opendirector/domain/creation/generation.py +++ b/packages/core/src/opendirector/domain/creation/generation.py @@ -369,6 +369,10 @@ def validate_generation(spec: GenerationSpec, model: ModelConfigPublic, referenc from opendirector.domain.moss_speech import validate_moss_speech_input text = spec.prompt if spec.promptContent is None else ''.join(part.text for part in spec.promptContent if part.type == 'text') validate_moss_speech_input(text, spec.parameters.voice) + elif rules.profile == 'zhipu-speech': + from opendirector.domain.zhipu_speech import validate_zhipu_speech_input + text = spec.prompt if spec.promptContent is None else ''.join(part.text for part in spec.promptContent if part.type == 'text') + validate_zhipu_speech_input(text, spec.parameters.voice) elif spec.kind == 'audio' and rules.voiceRequired and not spec.parameters.voice: raise ValueError('请填写服务音色 ID。') if len(references) > MAX_REFERENCES: diff --git a/packages/core/src/opendirector/domain/creation/models.py b/packages/core/src/opendirector/domain/creation/models.py index 937a2d0f..7aa2da25 100644 --- a/packages/core/src/opendirector/domain/creation/models.py +++ b/packages/core/src/opendirector/domain/creation/models.py @@ -29,7 +29,7 @@ class ImageOutput(Contract): class VoiceLibraryCapabilities(Contract): - provider: Literal['vllm_omni', 'moss'] = Field(description='Voice registry protocol.') + provider: Literal['vllm_omni', 'moss', 'zhipu'] = Field(description='Voice registry protocol.') connectionId: str = Field(min_length=1, max_length=200, description='Opaque service, endpoint and account identity.') requiresTranscript: bool = Field(description='Registered voice must have a reference transcript.') requiresReferenceAudio: bool = Field(description='Registered voice must have actual reference audio.') diff --git a/packages/core/src/opendirector/domain/input_rules.py b/packages/core/src/opendirector/domain/input_rules.py index 3ffbcf22..0ccc9a86 100644 --- a/packages/core/src/opendirector/domain/input_rules.py +++ b/packages/core/src/opendirector/domain/input_rules.py @@ -6,6 +6,7 @@ from .moss_image import MOSS_IMAGE_PROMPT_CHARACTERS, is_moss_image from .moss_speech import is_moss_speech from .kling_models import KLING_MODELS, KLING_RATIOS, kling_capabilities +from .zhipu_speech import ZHIPU_SPEECH_CHARACTERS, is_zhipu_speech VideoMode = Literal['first_frame', 'reference', 'continuation'] @@ -26,6 +27,10 @@ def _adapter_rules(config): rules.update(profile='moss-speech', allowedKinds=[], maxImages=0, voiceRequired=True, referencesAllowed=False) return rules + if is_zhipu_speech(config): + rules.update(profile='zhipu-speech', allowedKinds=[], maxImages=0, voiceRequired=True, + referencesAllowed=False, maxPromptCharacters=ZHIPU_SPEECH_CHARACTERS) + return rules if kind == 'vision' or config.get('protocol') == 'vllm_omni_segmentation': # Shared image segmentation / video tracking service, never a generator. rules.update(profile='vllm-omni-segmentation', allowedKinds=[], maxImages=0, videoModes=[]) diff --git a/packages/core/src/opendirector/domain/voices.py b/packages/core/src/opendirector/domain/voices.py index 791bcd53..6d3398a6 100644 --- a/packages/core/src/opendirector/domain/voices.py +++ b/packages/core/src/opendirector/domain/voices.py @@ -5,6 +5,7 @@ from urllib.parse import urlsplit, urlunsplit from .moss_speech import is_moss_speech +from .zhipu_speech import is_zhipu_speech from .projects import ProjectError # Character limits and the corresponding error for each upload metadata field. @@ -32,6 +33,8 @@ 'voice_provider_protocol': '音色服务返回格式无效或分页未结束,请检查服务版本和接口。', 'voice_provider_error': '音色服务请求失败,请检查服务配置和账户权限。', 'voice_provider_timeout': '音色服务请求超时,请检查服务状态后重试。', + 'voice_name_taken': '该账户已有同名音色,请换一个名称。', + 'voice_provider_quota': '音色服务账户余额或资源包不足,请充值后重试。', } @@ -43,6 +46,11 @@ def voice_library_capabilities(config): if config.get('kind') != 'audio' or config.get('protocol') != 'openai': return None name = config.get('modelId', '').rsplit('/', 1)[-1].lower().replace('_', '-') + if is_zhipu_speech(config): + return _scoped(config, 'zhipu', dict( + requiresTranscript=False, requiresReferenceAudio=False, supportsUploadedVoices=True, + referenceEmbeddingDimension=None, canDelete=True, maxBytes=10 * 1024 * 1024, + formats=['wav', 'mp3'], minSeconds=3, maxSeconds=30)) moss = is_moss_speech(config) cosyvoice = name.startswith(('cosyvoice3', 'fun-cosyvoice3')) fish_speech = name.startswith('fish-speech') or name in {'s2-pro', 'drama-3', 'drama-3-int4'} @@ -53,15 +61,6 @@ def voice_library_capabilities(config): if not moss and not (config.get('connectionType') == 'self_hosted' and known): return None provider = 'moss' if moss else 'vllm_omni' - parsed = urlsplit(config['url']) - host = parsed.hostname.lower() - if ':' in host: host = '[' + host + ']' - port = parsed.port - if port and (parsed.scheme, port) not in {('https', 443), ('http', 80)}: host += ':' + str(port) - endpoint = urlunsplit((parsed.scheme.lower(), host, parsed.path.rstrip('/'), '', '')) - account = hashlib.sha256((config.get('apiKey', '') if config.get('authMode') != 'none' else '').encode()).hexdigest() - identity = [provider, config.get('providerId') or config.get('id'), endpoint, config.get('authMode', 'none'), account] - connection = hashlib.sha256(json.dumps(identity, separators=(',', ':')).encode()).hexdigest() # Upstream vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py: # validate() requires reference audio for standard/realtime/local/Nano. # Local-Transformer ref_text is optional continuation conditioning; Drama3 @@ -80,11 +79,24 @@ def voice_library_capabilities(config): qwen_base = name.startswith('qwen3-tts') and name.endswith('-base') dimension = (1024 if '0.6b' in name else 2048 if '1.7b' in name else None) if qwen_base else None reference = reference or qwen_base - return dict(provider=provider, connectionId=connection, requiresTranscript=transcript, + return _scoped(config, provider, dict(requiresTranscript=transcript, requiresReferenceAudio=reference, supportsUploadedVoices=uploaded, referenceEmbeddingDimension=dimension, canDelete=not moss, maxBytes=10 * 1024 * 1024, formats=['wav', 'mp3', 'm4a', 'flac'] if moss else ['wav', 'mp3', 'flac', 'ogg', 'aac', 'webm', 'm4a', 'mp4'], - minSeconds=None if moss else 1, maxSeconds=30) + minSeconds=None if moss else 1, maxSeconds=30)) + + +def _scoped(config, provider, capabilities): + parsed = urlsplit(config['url']) + host = parsed.hostname.lower() + if ':' in host: host = '[' + host + ']' + port = parsed.port + if port and (parsed.scheme, port) not in {('https', 443), ('http', 80)}: host += ':' + str(port) + endpoint = urlunsplit((parsed.scheme.lower(), host, parsed.path.rstrip('/'), '', '')) + account = hashlib.sha256((config.get('apiKey', '') if config.get('authMode') != 'none' else '').encode()).hexdigest() + identity = [provider, config.get('providerId') or config.get('id'), endpoint, config.get('authMode', 'none'), account] + connection = hashlib.sha256(json.dumps(identity, separators=(',', ':')).encode()).hexdigest() + return dict(provider=provider, connectionId=connection, **capabilities) def valid_voice_name(value): diff --git a/packages/core/src/opendirector/domain/zhipu_speech.py b/packages/core/src/opendirector/domain/zhipu_speech.py new file mode 100644 index 00000000..07a64921 --- /dev/null +++ b/packages/core/src/opendirector/domain/zhipu_speech.py @@ -0,0 +1,36 @@ +"""Model identity and input rules for Zhipu BigModel GLM-TTS speech synthesis.""" +import re +from urllib.parse import urlsplit + +from .projects import ProjectError + + +ZHIPU_SPEECH_MODELS = frozenset({'glm-tts'}) +ZHIPU_CLONE_MODEL = 'glm-tts-clone' +ZHIPU_SPEECH_CHARACTERS = 1024 +# The documented system voices. The account's OFFICIAL list also contains +# entries the synthesis endpoint rejects, so it is not used as a catalog. +ZHIPU_PRESET_VOICES = ('tongtong', 'chuichui', 'xiaochen', 'jam', 'kazi', 'douji', 'luodo') +ZHIPU_DEFAULT_VOICE = 'tongtong' +# The clone endpoint requires preview text; the preview is fixed product copy. +ZHIPU_CLONE_PREVIEW_TEXT = '你好,这是我的声音。Hello, this is my voice.' + + +def is_zhipu_api(config): + url = urlsplit(config.get('url', '')) + return (config.get('protocol') == 'openai' and config.get('connectionType', 'api') == 'api' + and url.scheme == 'https' and url.hostname == 'open.bigmodel.cn' and url.port in {None, 443} + and url.path.rstrip('/') == '/api/paas/v4') + + +def is_zhipu_speech(config): + return config.get('kind') == 'audio' and is_zhipu_api(config) and config.get('modelId') in ZHIPU_SPEECH_MODELS + + +def validate_zhipu_speech_input(text, voice): + if not isinstance(text, str) or not text.strip(): + raise ProjectError('请填写需要朗读的文本。', 422, code='zhipu_speech_text') + if len(text) > ZHIPU_SPEECH_CHARACTERS: + raise ProjectError(f'GLM-TTS 单次朗读文本不能超过 {ZHIPU_SPEECH_CHARACTERS} 个字符,请拆分后生成。', 422, code='zhipu_speech_text') + if not isinstance(voice, str) or not re.fullmatch(r'[A-Za-z0-9_.-]{1,64}', voice): + raise ProjectError('请选择智谱账户可用的音色。', 422, code='zhipu_speech_voice') diff --git a/packages/core/src/opendirector/integrations/generation_errors.py b/packages/core/src/opendirector/integrations/generation_errors.py index a5a73cb1..84879b1d 100644 --- a/packages/core/src/opendirector/integrations/generation_errors.py +++ b/packages/core/src/opendirector/integrations/generation_errors.py @@ -80,6 +80,10 @@ def provider_failure(value, *, stage, headers=None, http_status=None, secrets=() category = 'input' elif code == '55000000': category = 'temporary' + diagnostic_text = error.get('message', '') + if code == '1113' and isinstance(diagnostic_text, str) and ('余额' in diagnostic_text or '资源包' in diagnostic_text): + # Zhipu BigModel reports an empty balance as HTTP 429; retrying cannot help. + category = 'quota' message, suggestion = _REASONS[category] if provider == 'kling': kling_categories = {'1000': 'authentication', '1001': 'authentication', '1002': 'authentication', @@ -101,7 +105,7 @@ def provider_failure(value, *, stage, headers=None, http_status=None, secrets=() elif re.search(r'(?:not support|unsupported|not supported)', diagnostic) and re.search(r'(?:reference_(?:image|video|audio)|image_url|video_url|audio_url|input_audio)', diagnostic): category, message = 'reference', '供应商拒绝了参考类型或输入组合。' suggestion = '请核对参考类型和生成模式,或选择其他已适配模型;当前输入和已有结果保留。' - if name == 'voice_not_found': + if name == 'voice_not_found' or (code == '1214' and diagnostic.startswith('音色不存在')): category, message = 'input', '所选音色不存在或不可用。' suggestion = '请检查音色 ID 是否属于当前账户可用的音色。' elif name == 'inputimagesensitivecontentdetected.privacyinformation': diff --git a/packages/core/src/opendirector/integrations/model_adapters.py b/packages/core/src/opendirector/integrations/model_adapters.py index a95d4ba2..b0b1c18d 100644 --- a/packages/core/src/opendirector/integrations/model_adapters.py +++ b/packages/core/src/opendirector/integrations/model_adapters.py @@ -21,6 +21,7 @@ from opendirector.domain.input_rules import input_rules from opendirector.domain.moss_image import validate_moss_image_prompt, validate_moss_image_size from opendirector.domain.moss_speech import validate_moss_speech_input +from opendirector.domain.zhipu_speech import validate_zhipu_speech_input from opendirector.domain.projects import ProjectError from opendirector.integrations.generation_errors import checked, protocol_failure, provider_failure, transport_failure from opendirector.integrations.http_settings import MODEL_HTTP_HOOKS, auth_headers, trust_environment @@ -210,6 +211,41 @@ async def download(self, config, url): return data +class ZhipuSpeech(ModelAdapter): + """Zhipu BigModel GLM-TTS on /audio/speech. + + The service only offers WAV or raw PCM, and its streaming mode returns PCM + chunks over SSE, so a non-streaming WAV request is the only complete + container. A 200 response is accepted only when the body is a RIFF/WAVE file: + the service also reports some errors as JSON with HTTP 200. + """ + + capabilities = frozenset({'audio.speech'}) + + async def speech(self, config, request: SpeechRequest, *, result_ready=None, watermark=True) -> MediaResult: + validate_zhipu_speech_input(request.text, request.voice) + payload = dict(model=config['modelId'], input=request.text, voice=request.voice, + response_format='wav', stream=False, watermark_enabled=watermark) + secrets = (config.get('apiKey', ''),) + + async def submit(): + async with _client(self, config) as client: + async with client.stream('POST', endpoint(config['url'], '/audio/speech'), + headers=auth_headers(config), json=payload) as response: + return await bounded(response, stage='submit', secrets=secrets) + + data = await _call('submit', submit) + if not (data[:4] == b'RIFF' and data[8:12] == b'WAVE'): + try: + value = json.loads(data) + except (ValueError, UnicodeError): + value = None + if isinstance(value, dict) and value.get('error'): + raise provider_failure(value, stage='submit', http_status=200, secrets=secrets) + raise protocol_failure('submit') + return MediaResult(bytes(data), 'wav') + + class OpenAIImages(ModelAdapter): """OpenAI Images API: /images/generations and multipart /images/edits. @@ -674,6 +710,8 @@ def adapter_for(config, transport=None, poll_interval=3) -> ModelAdapter | None: return MossImages(transport, poll_interval) if profile == 'moss-speech': return MossSpeech(transport, poll_interval) + if profile == 'zhipu-speech': + return ZhipuSpeech(transport, poll_interval) if kind == 'image' and profile == 'vllm-omni-image': return OpenAIImages(transport, poll_interval, free_size=True, strength=True) if kind == 'image' and profile == 'vllm-omni-qwen-image': diff --git a/packages/core/src/opendirector/integrations/providers.py b/packages/core/src/opendirector/integrations/providers.py index b6385dd5..ec7af728 100644 --- a/packages/core/src/opendirector/integrations/providers.py +++ b/packages/core/src/opendirector/integrations/providers.py @@ -18,7 +18,7 @@ from opendirector.integrations.kling_video import KlingVideoGeneration from opendirector.integrations.vllm_omni import VllmOmniGeneration from opendirector.integrations.vllm_omni_image import VllmOmniImageGeneration -from opendirector.integrations.model_adapters import MossImages, MossSpeech, OpenAISpeech, VllmOmniMusic, VllmOmniScore +from opendirector.integrations.model_adapters import MossImages, MossSpeech, OpenAISpeech, VllmOmniMusic, VllmOmniScore, ZhipuSpeech from opendirector.domain.capabilities import ImageRequest, MusicRequest, ScoreRequest, SpeechRequest from opendirector.integrations.model_adapters import OpenAISpeech, VllmOmniMusic, VllmOmniScore, image_parts from opendirector.domain.capabilities import MusicRequest, ScoreRequest, SpeechRequest @@ -36,6 +36,7 @@ def __init__(self, store, transport=None, poll_interval=3, retry_delays=(0.5, 1. self.vllm_omni_image = VllmOmniImageGeneration(store, transport, poll_interval) self.moss_image = MossImages(transport, poll_interval) self.moss_speech = MossSpeech(transport, poll_interval) + self.zhipu_speech = ZhipuSpeech(transport, poll_interval) def _profile_adapter(self, config): if config and config.get('protocol') == 'kling_video': @@ -122,6 +123,11 @@ async def generate(self, config, task, submitted, *, result_ready=None): result = await self.moss_speech.speech(config, SpeechRequest(prepared['prompt'], task['parameters'].get('voice')), result_ready=result_ready) return result.suffix, result.data + if input_rules(config)['profile'] == 'zhipu-speech': + prepared = await asyncio.to_thread(generation_references, self.store, task, config) + result = await self.zhipu_speech.speech(config, SpeechRequest(prepared['prompt'], task['parameters'].get('voice')), + watermark=task['parameters'].get('watermark_enabled', True)) + return result.suffix, result.data if config.get('protocol') in {None, 'volcengine'} or task['kind'] == 'text': return await super().generate(config, task, submitted, result_ready=result_ready) kind, stage = task['kind'], 'submit' diff --git a/packages/core/src/opendirector/integrations/reference_inputs.py b/packages/core/src/opendirector/integrations/reference_inputs.py index f1a9d102..dacd1e49 100644 --- a/packages/core/src/opendirector/integrations/reference_inputs.py +++ b/packages/core/src/opendirector/integrations/reference_inputs.py @@ -107,6 +107,8 @@ def generation_references(store, task, config, *, complete=True, overrides=None) return result if kind == 'audio': if rules.get('referencesAllowed') is False and task['referenceIds']: + if rules['profile'] == 'zhipu-speech': + raise ProjectError('GLM-TTS 不支持参考素材,请直接填写朗读文本并选择音色。', 422, code='zhipu_speech_reference') raise ProjectError('MOSS 语音生成不支持参考素材,请直接填写朗读文本和音色 ID。', 422, code='moss_speech_reference') # Historical context remains recorded; it is never appended to speech. result['manifest'] = [{'assetId': meta['id'], 'fingerprint': meta['fingerprint'], 'kind': source_kind, 'transmission': 'context_only'} diff --git a/src/backend/rules.ts b/src/backend/rules.ts index 378e4ceb..729249b2 100644 --- a/src/backend/rules.ts +++ b/src/backend/rules.ts @@ -456,6 +456,12 @@ export function serviceGenerationSettings( !next.voiceName ) next.voiceName = "2890a3a4-0012-4997-a545-8b045fe83bbe"; + if ( + next.mode === "audio" && + config.inputRules.profile === "zhipu-speech" && + !next.voiceName + ) + next.voiceName = "tongtong"; if ( !generateAudioOptions({ published: config.inputRules.generateAudioOptions ?? null, diff --git a/src/canvas/AssistantPanel.tsx b/src/canvas/AssistantPanel.tsx index 3efd0d22..7b75226a 100644 --- a/src/canvas/AssistantPanel.tsx +++ b/src/canvas/AssistantPanel.tsx @@ -34,6 +34,7 @@ import { import { assetOptionId, referenceCandidates } from "./referenceCandidates.ts"; import { type MentionOption } from "./referenceMentions"; import { useAssistantChat } from "./useAssistantChat"; +import { needsAgentModel } from "./assistantReadiness.ts"; import type { AssistantLayout } from "./useAssistantLayout"; import { usePendingCanvasReferences } from "./usePendingCanvasReferences.ts"; import { useProjectMentionAssets } from "./useProjectMentionAssets"; @@ -138,6 +139,8 @@ export function AssistantPanel({ chat.adding || checkingReferences || (chat.sending && !chat.started); + const currentRuntime = chat.runtimes.find((item) => item.id === (value.runtime ?? "default")); + const missingAgentModel = needsAgentModel(chat.loaded, currentRuntime, !!chat.agent); const referenceDisabled = !chat.loaded || chat.adding || checkingReferences || chat.sending; const sessionDisabledReason = chat.changing @@ -761,6 +764,13 @@ export function AssistantPanel({ t("assistant.panel.checkingEngine") ) : chat.adding ? ( t("assistant.panel.addingReferences") + ) : missingAgentModel ? ( + <> + {t("assistant.agent.noAgentModels")}{" · "} + + ) : ( `${chat.runtimes.find((item) => item.id === (value.runtime ?? "default"))?.label ?? t("assistant.panel.chooseAgent")}` )} diff --git a/src/canvas/CreateVoiceForm.tsx b/src/canvas/CreateVoiceForm.tsx index 6d90effa..cba91dda 100644 --- a/src/canvas/CreateVoiceForm.tsx +++ b/src/canvas/CreateVoiceForm.tsx @@ -178,7 +178,7 @@ export function CreateVoiceForm({ disabled={busy} onChange={(event) => setName(event.target.value)} /> - {isVllm && ( + {(isVllm || capabilities.provider === "zhipu") && ( <>