From f163c9835be9930f3b95119bebb431bd16a737f2 Mon Sep 17 00:00:00 2001 From: nwalker Date: Tue, 10 Nov 2026 20:32:09 -0800 Subject: [PATCH 1/2] fix(ios): send input through dtuhidd on Xcode 27 simulators CoreSimulator 1155.4 (Xcode 27) routes guest HID through dtuhidd. The guest accepts legacy Indigo messages from SimulatorKit's HID client and drops them, so taps, Home, hardware buttons and keys reported success and did nothing. When the loaded CoreSimulator is 1155.4 or later, touches (single, edge and two-finger), Home, the App Switcher, hardware buttons and keyboard usages go to dtuhidd's digitizer service as XPC dictionaries instead. Older toolchains keep the Indigo path unchanged. Fixes #90 --- packages/server/build.rs | 1 + packages/server/native/DFDTUHIDClient.h | 49 +++ packages/server/native/DFDTUHIDClient.m | 296 ++++++++++++++++++ .../native/DFPrivateSimulatorDisplayBridge.m | 193 +++++++++++- 4 files changed, 530 insertions(+), 9 deletions(-) create mode 100644 packages/server/native/DFDTUHIDClient.h create mode 100644 packages/server/native/DFDTUHIDClient.m diff --git a/packages/server/build.rs b/packages/server/build.rs index 7597517f..9dbafbb6 100644 --- a/packages/server/build.rs +++ b/packages/server/build.rs @@ -38,6 +38,7 @@ fn main() { let files = [ camera.join("SimDeckCameraService.m"), cli.join("DFPrivateSimulatorDisplayBridge.m"), + cli.join("DFDTUHIDClient.m"), cli.join("XCWH264Encoder.m"), cli.join("XCWProcessRunner.m"), cli.join("XCWPrivateSimulatorBooter.m"), diff --git a/packages/server/native/DFDTUHIDClient.h b/packages/server/native/DFDTUHIDClient.h new file mode 100644 index 00000000..eea6a2a4 --- /dev/null +++ b/packages/server/native/DFDTUHIDClient.h @@ -0,0 +1,49 @@ +#import + +NS_ASSUME_NONNULL_BEGIN + +typedef NS_ENUM(NSInteger, DFDTUHIDTouchPhase) { + DFDTUHIDTouchPhaseDown, + DFDTUHIDTouchPhaseUp, +}; + +/// Input for simulators whose guest routes HID through `dtuhidd` (CoreSimulator 1155.4, Xcode 27, and later). +/// Those guests accept legacy Indigo messages from SimulatorKit's HID client and then drop them, so touches, buttons +/// and keys must go to `dtuhidd`'s digitizer service as plain XPC dictionaries instead. +@interface DFDTUHIDClient : NSObject + +/// Whether the CoreSimulator loaded in this process routes guest input through `dtuhidd`. ++ (BOOL)isRequiredByLoadedCoreSimulator; + +/// Connects to the device's digitizer service and waits until `dtuhidd` answers, retrying while it starts. +- (nullable instancetype)initWithDevice:(id)device error:(NSError **)error; + +/// One contact at a normalized top-left point in 0...1. The first down starts the contact, later downs move it. +- (BOOL)sendTouchPhase:(DFDTUHIDTouchPhase)phase + x:(double)x + y:(double)y + edge:(uint32_t)edge + error:(NSError **)error; + +/// Two contacts at normalized points, tracked separately from the single contact. +- (BOOL)sendTwoFingerTouchPhase:(DFDTUHIDTouchPhase)phase + x1:(double)x1 + y1:(double)y1 + x2:(double)x2 + y2:(double)y2 + error:(NSError **)error; + +/// A hardware button identified by its HID usage page and usage (Home is consumer 0x0c / 0x40). +- (BOOL)sendButtonUsagePage:(uint32_t)usagePage usage:(uint32_t)usage down:(BOOL)down error:(NSError **)error; + +/// A USB HID keyboard usage. +- (BOOL)sendKeyboardUsage:(uint32_t)usage down:(BOOL)down error:(NSError **)error; + +/// Whether the connection is still usable; `dtuhidd` invalidates it when the simulator shuts down. +@property (nonatomic, readonly, getter=isValid) BOOL valid; + +- (void)disconnect; + +@end + +NS_ASSUME_NONNULL_END diff --git a/packages/server/native/DFDTUHIDClient.m b/packages/server/native/DFDTUHIDClient.m new file mode 100644 index 00000000..681a2505 --- /dev/null +++ b/packages/server/native/DFDTUHIDClient.m @@ -0,0 +1,296 @@ +#import "DFDTUHIDClient.h" + +#import +#import +#import +#import + +static NSString *const DFDTUHIDErrorDomain = @"SimDeck.DTUHID"; +static NSString *const DFDTUHIDDigitizerService = @"com.apple.coredevice.feature.remote.hid.digitizer"; +static NSString *const DFDTUHIDFirstCoreSimulatorVersion = @"1155.4"; + +// dtuhidd decodes these payload enums from uint64. HIDButtonState is 1-based; 0 is rejected. +static const uint64_t DFDTUHIDContactStart = 0; +static const uint64_t DFDTUHIDContactPosition = 1; +static const uint64_t DFDTUHIDContactEnd = 2; +static const uint64_t DFDTUHIDButtonDown = 1; +static const uint64_t DFDTUHIDButtonUp = 2; + +// dtuhidd is demand-launched and aborts if it starts before the guest has a display, after which launchd throttles +// its respawn while the service lookup keeps succeeding. Only an answered request proves a live daemon. +static const int64_t DFDTUHIDLivenessTimeoutNs = 3 * NSEC_PER_SEC; +static const useconds_t DFDTUHIDLivenessBackoffUs = 3 * 1000 * 1000; +static const NSUInteger DFDTUHIDLivenessAttempts = 4; +static const useconds_t DFDTUHIDDeviceOpenTailUs = 200 * 1000; +static const int64_t DFDTUHIDSendBarrierTimeoutNs = 2 * NSEC_PER_SEC; + +typedef xpc_object_t (*DFXPCEndpointCreateMachPort4SimFn)(mach_port_t port, uint64_t a, uint64_t b); +typedef xpc_connection_t (*DFXPCConnectionCreateFromEndpointFn)(xpc_object_t endpoint); +typedef void (*DFXPCConnectionEnableSim2Host4SimFn)(xpc_connection_t connection); + +static NSError *DFDTUHIDError(NSString *message) { + return [NSError errorWithDomain:DFDTUHIDErrorDomain code:1 userInfo:@{ NSLocalizedDescriptionKey: message }]; +} + +static xpc_object_t DFDTUHIDEnvelope(const char *messageType, BOOL isBarrier, xpc_object_t payload) { + xpc_object_t message = xpc_dictionary_create(NULL, NULL, 0); + xpc_dictionary_set_string(message, "messageType", messageType); + xpc_dictionary_set_bool(message, "isBarrier", isBarrier); + xpc_dictionary_set_string(message, "featureIdentifier", DFDTUHIDDigitizerService.UTF8String); + xpc_dictionary_set_value(message, "payload", payload); + return message; +} + +static xpc_object_t DFDTUHIDPoint(double x, double y) { + xpc_object_t point = xpc_dictionary_create(NULL, NULL, 0); + xpc_dictionary_set_double(point, "x", fmax(0.0, fmin(1.0, x))); + xpc_dictionary_set_double(point, "y", fmax(0.0, fmin(1.0, y))); + return point; +} + +static xpc_object_t DFDTUHIDKeyboardPayload(uint64_t usage, uint64_t state) { + xpc_object_t payload = xpc_dictionary_create(NULL, NULL, 0); + xpc_dictionary_set_uint64(payload, "usageCode", usage); + xpc_dictionary_set_uint64(payload, "state", state); + return payload; +} + +@implementation DFDTUHIDClient { + xpc_connection_t _connection; + dispatch_queue_t _queue; + // Guarded by _queue. + BOOL _invalidated; + NSUInteger _interruptions; + BOOL _contactDown; + NSUInteger _contactInterruptions; + BOOL _twoFingerDown; + NSUInteger _twoFingerInterruptions; +} + ++ (BOOL)isRequiredByLoadedCoreSimulator { + Class simDeviceClass = NSClassFromString(@"SimDevice"); + if (simDeviceClass == Nil) { + return NO; + } + // CoreSimulator is a system framework the Xcode installer replaces, so the loaded bundle decides, not the + // selected Xcode. + NSString *version = [NSBundle bundleForClass:simDeviceClass].infoDictionary[@"CFBundleVersion"]; + if (![version isKindOfClass:[NSString class]] || version.length == 0) { + return NO; + } + return [version compare:DFDTUHIDFirstCoreSimulatorVersion options:NSNumericSearch] != NSOrderedAscending; +} + +- (nullable instancetype)initWithDevice:(id)device error:(NSError **)error { + self = [super init]; + if (self == nil) { + return nil; + } + _queue = dispatch_queue_create("SimDeck.DTUHID", DISPATCH_QUEUE_SERIAL); + + NSError *lastError = nil; + for (NSUInteger attempt = 1; attempt <= DFDTUHIDLivenessAttempts; attempt++) { + xpc_connection_t connection = [self connectToDevice:device error:&lastError]; + if (connection != nil && [self confirmLiveness:connection error:&lastError]) { + _connection = connection; + usleep(DFDTUHIDDeviceOpenTailUs); + return self; + } + if (connection != nil) { + xpc_connection_cancel(connection); + } + if (attempt < DFDTUHIDLivenessAttempts) { + usleep(DFDTUHIDLivenessBackoffUs); + } + } + if (error != NULL) { + *error = lastError ?: DFDTUHIDError(@"dtuhidd did not answer."); + } + return nil; +} + +- (nullable xpc_connection_t)connectToDevice:(id)device error:(NSError **)error { + DFXPCEndpointCreateMachPort4SimFn endpointFromPort = + (DFXPCEndpointCreateMachPort4SimFn)dlsym(RTLD_DEFAULT, "xpc_endpoint_create_mach_port_4sim"); + DFXPCConnectionCreateFromEndpointFn connectionFromEndpoint = + (DFXPCConnectionCreateFromEndpointFn)dlsym(RTLD_DEFAULT, "xpc_connection_create_from_endpoint"); + DFXPCConnectionEnableSim2Host4SimFn enableSim2Host = + (DFXPCConnectionEnableSim2Host4SimFn)dlsym(RTLD_DEFAULT, "xpc_connection_enable_sim2host_4sim"); + if (endpointFromPort == NULL || connectionFromEndpoint == NULL || enableSim2Host == NULL) { + if (error != NULL) { + *error = DFDTUHIDError(@"libxpc does not export the simulator endpoint functions dtuhidd needs."); + } + return nil; + } + + NSError *lookupError = nil; + mach_port_t port = ((mach_port_t(*)(id, SEL, NSString *, NSError **))objc_msgSend)( + device, sel_registerName("lookup:error:"), DFDTUHIDDigitizerService, &lookupError); + if (port == MACH_PORT_NULL) { + if (error != NULL) { + *error = DFDTUHIDError([NSString stringWithFormat:@"The simulator does not vend %@: %@", + DFDTUHIDDigitizerService, + lookupError.localizedDescription ?: @"lookup failed"]); + } + return nil; + } + + // The endpoint takes the lookup's send right. + xpc_object_t endpoint = endpointFromPort(port, 0, 0); + xpc_connection_t connection = endpoint != nil ? connectionFromEndpoint(endpoint) : nil; + if (connection == nil) { + if (error != NULL) { + *error = DFDTUHIDError(@"Could not create an XPC connection to dtuhidd."); + } + return nil; + } + // Without this the guest sees the peer but never a payload. + enableSim2Host(connection); + + __weak DFDTUHIDClient *weakSelf = self; + xpc_connection_set_target_queue(connection, _queue); + xpc_connection_set_event_handler(connection, ^(xpc_object_t event) { + DFDTUHIDClient *strongSelf = weakSelf; + if (strongSelf == nil || xpc_get_type(event) != XPC_TYPE_ERROR) { + return; + } + if (event == XPC_ERROR_CONNECTION_INTERRUPTED) { + strongSelf->_interruptions += 1; + } else if (event == XPC_ERROR_CONNECTION_INVALID) { + strongSelf->_invalidated = YES; + } + }); + xpc_connection_resume(connection); + return connection; +} + +/// A barrier carrying keyboard usage 0 ("no event"), so the daemon answers without the guest seeing a key. +- (BOOL)confirmLiveness:(xpc_connection_t)connection error:(NSError **)error { + xpc_object_t probe = DFDTUHIDEnvelope("IndigoKeyboardButtonEvent", YES, DFDTUHIDKeyboardPayload(0, DFDTUHIDButtonUp)); + dispatch_semaphore_t answered = dispatch_semaphore_create(0); + __block BOOL alive = NO; + xpc_connection_send_message_with_reply(connection, probe, _queue, ^(xpc_object_t reply) { + alive = xpc_get_type(reply) == XPC_TYPE_DICTIONARY; + dispatch_semaphore_signal(answered); + }); + if (dispatch_semaphore_wait(answered, dispatch_time(DISPATCH_TIME_NOW, DFDTUHIDLivenessTimeoutNs)) != 0 || !alive) { + if (error != NULL) { + *error = DFDTUHIDError(@"dtuhidd did not answer its liveness probe."); + } + return NO; + } + return YES; +} + +- (BOOL)isValid { + __block BOOL valid = NO; + dispatch_sync(_queue, ^{ + valid = self->_connection != nil && !self->_invalidated; + }); + return valid; +} + +- (void)disconnect { + dispatch_sync(_queue, ^{ + if (self->_connection != nil) { + xpc_connection_cancel(self->_connection); + self->_connection = nil; + } + self->_invalidated = YES; + }); +} + +- (void)dealloc { + if (_connection != nil) { + xpc_connection_cancel(_connection); + } +} + +/// Sends `message` and waits until XPC has handed it to the guest, so a later event never overtakes it. +- (BOOL)send:(xpc_object_t)message error:(NSError **)error { + __block xpc_connection_t connection = nil; + dispatch_sync(_queue, ^{ + if (self->_connection != nil && !self->_invalidated) { + connection = self->_connection; + xpc_connection_send_message(connection, message); + } + }); + if (connection == nil) { + if (error != NULL) { + *error = DFDTUHIDError(@"The dtuhidd connection is closed."); + } + return NO; + } + // The barrier block runs on _queue, so this wait must not happen on it. + dispatch_semaphore_t sent = dispatch_semaphore_create(0); + xpc_connection_send_barrier(connection, ^{ + dispatch_semaphore_signal(sent); + }); + if (dispatch_semaphore_wait(sent, dispatch_time(DISPATCH_TIME_NOW, DFDTUHIDSendBarrierTimeoutNs)) != 0) { + if (error != NULL) { + *error = DFDTUHIDError(@"Timed out sending to dtuhidd."); + } + return NO; + } + return YES; +} + +/// Maps down/up onto dtuhidd's start/position/end. dtuhidd forgets a contact when the connection is interrupted, so a +/// down that would continue a contact from before an interruption starts a new one instead. +- (uint64_t)contactEventForPhase:(DFDTUHIDTouchPhase)phase twoFinger:(BOOL)twoFinger { + __block uint64_t eventType = DFDTUHIDContactEnd; + dispatch_sync(_queue, ^{ + BOOL *down = twoFinger ? &self->_twoFingerDown : &self->_contactDown; + NSUInteger *startedAt = twoFinger ? &self->_twoFingerInterruptions : &self->_contactInterruptions; + if (phase == DFDTUHIDTouchPhaseUp) { + *down = NO; + eventType = DFDTUHIDContactEnd; + } else if (*down && *startedAt == self->_interruptions) { + eventType = DFDTUHIDContactPosition; + } else { + *down = YES; + *startedAt = self->_interruptions; + eventType = DFDTUHIDContactStart; + } + }); + return eventType; +} + +- (BOOL)sendTouchPhase:(DFDTUHIDTouchPhase)phase x:(double)x y:(double)y edge:(uint32_t)edge error:(NSError **)error { + xpc_object_t payload = xpc_dictionary_create(NULL, NULL, 0); + xpc_dictionary_set_value(payload, "pointOne", DFDTUHIDPoint(x, y)); + xpc_dictionary_set_uint64(payload, "eventType", [self contactEventForPhase:phase twoFinger:NO]); + xpc_dictionary_set_uint64(payload, "edge", edge); + xpc_dictionary_set_uint64(payload, "target", 0); + return [self send:DFDTUHIDEnvelope("IndigoDigitizerEvent", NO, payload) error:error]; +} + +- (BOOL)sendTwoFingerTouchPhase:(DFDTUHIDTouchPhase)phase + x1:(double)x1 + y1:(double)y1 + x2:(double)x2 + y2:(double)y2 + error:(NSError **)error { + xpc_object_t payload = xpc_dictionary_create(NULL, NULL, 0); + xpc_dictionary_set_value(payload, "pointOne", DFDTUHIDPoint(x1, y1)); + xpc_dictionary_set_value(payload, "pointTwo", DFDTUHIDPoint(x2, y2)); + xpc_dictionary_set_uint64(payload, "eventType", [self contactEventForPhase:phase twoFinger:YES]); + xpc_dictionary_set_uint64(payload, "edge", 0); + xpc_dictionary_set_uint64(payload, "target", 0); + return [self send:DFDTUHIDEnvelope("IndigoDigitizerEvent", NO, payload) error:error]; +} + +- (BOOL)sendButtonUsagePage:(uint32_t)usagePage usage:(uint32_t)usage down:(BOOL)down error:(NSError **)error { + xpc_object_t payload = xpc_dictionary_create(NULL, NULL, 0); + xpc_dictionary_set_uint64(payload, "usagePage", usagePage); + xpc_dictionary_set_uint64(payload, "usageCode", usage); + xpc_dictionary_set_uint64(payload, "state", down ? DFDTUHIDButtonDown : DFDTUHIDButtonUp); + return [self send:DFDTUHIDEnvelope("IndigoButtonEvent", NO, payload) error:error]; +} + +- (BOOL)sendKeyboardUsage:(uint32_t)usage down:(BOOL)down error:(NSError **)error { + xpc_object_t payload = DFDTUHIDKeyboardPayload(usage, down ? DFDTUHIDButtonDown : DFDTUHIDButtonUp); + return [self send:DFDTUHIDEnvelope("IndigoKeyboardButtonEvent", NO, payload) error:error]; +} + +@end diff --git a/packages/server/native/DFPrivateSimulatorDisplayBridge.m b/packages/server/native/DFPrivateSimulatorDisplayBridge.m index 83855db7..5fcba5a4 100644 --- a/packages/server/native/DFPrivateSimulatorDisplayBridge.m +++ b/packages/server/native/DFPrivateSimulatorDisplayBridge.m @@ -1,4 +1,5 @@ #import "DFPrivateSimulatorDisplayBridge.h" +#import "DFDTUHIDClient.h" #import #import @@ -2755,6 +2756,8 @@ @implementation DFPrivateSimulatorDisplayBridge { id _activeScreen; id _rawScreen; id _hidClient; + BOOL _usesDTUHID; + DFDTUHIDClient *_dtuhidClient; id _digitizerInputView; dispatch_queue_t _callbackQueue; dispatch_queue_t _delegateFrameQueue; @@ -3026,6 +3029,11 @@ - (nullable instancetype)initWithUDID:(NSString *)udid _inputFamily = DFSimulatorInputFamilyForCoreSimulatorDevice(_device); DFLog(@"Resolved simulator input family for %@: %s", udid, DFSimulatorInputFamilyName(_inputFamily)); + _usesDTUHID = [DFDTUHIDClient isRequiredByLoadedCoreSimulator]; + if (_usesDTUHID) { + DFLog(@"CoreSimulator routes guest input through dtuhidd; sending touches, buttons and keys there for %@", udid); + } + NSError *hidClientError = nil; _hidClient = DFCreateSimulatorKitHIDClientForDevice(_device, &hidClientError); if (_hidClient != nil) { @@ -3558,6 +3566,56 @@ - (nullable CVPixelBufferRef)copyPixelBuffer { return buttons; } +#pragma mark - dtuhidd input + +/// dtuhidd's edge values, which differ from the legacy Indigo edge constants above. +static uint32_t DFDTUHIDEdgeForPrivateTouchEdge(DFPrivateSimulatorTouchEdge edge) { + switch (edge) { + case DFPrivateSimulatorTouchEdgeTop: + return 1; + case DFPrivateSimulatorTouchEdgeLeft: + return 2; + case DFPrivateSimulatorTouchEdgeBottom: + return 3; + case DFPrivateSimulatorTouchEdgeRight: + return 4; + case DFPrivateSimulatorTouchEdgeNone: + return 0; + } + return 0; +} + +/// Runs on the callback queue. Connects on first use because dtuhidd only answers once the guest has a display, and +/// reconnects after the simulator drops the connection. +- (nullable DFDTUHIDClient *)dtuhidClient:(NSError **)error { + if (_dtuhidClient != nil && _dtuhidClient.isValid) { + return _dtuhidClient; + } + [_dtuhidClient disconnect]; + _dtuhidClient = [[DFDTUHIDClient alloc] initWithDevice:_device error:error]; + return _dtuhidClient; +} + +- (BOOL)dtuhidPressButtonUsagePage:(uint32_t)usagePage + usage:(uint32_t)usage + holdSeconds:(NSTimeInterval)holdSeconds + error:(NSError **)error { + DFDTUHIDClient *client = [self dtuhidClient:error]; + if (client == nil || ![client sendButtonUsagePage:usagePage usage:usage down:YES error:error]) { + return NO; + } + [NSThread sleepForTimeInterval:holdSeconds]; + return [client sendButtonUsagePage:usagePage usage:usage down:NO error:error]; +} + +- (BOOL)sendKeyboardUsage:(uint16_t)usage down:(BOOL)down error:(NSError **)error { + if (_usesDTUHID) { + DFDTUHIDClient *client = [self dtuhidClient:error]; + return client != nil && [client sendKeyboardUsage:usage down:down error:error]; + } + return DFSendSingleKeyboardEvent(_hidClient, usage, down, error); +} + - (BOOL)pressChromeButtonWithIdentifier:(NSString *)identifier error:(NSError * _Nullable __autoreleasing *)error { __block BOOL success = NO; __block NSError *dispatchError = nil; @@ -3619,6 +3677,17 @@ - (BOOL)pressHomeButton:(NSError * _Nullable __autoreleasing *)error { __block NSError *dispatchError = nil; dispatch_block_t work = ^{ + if (self->_usesDTUHID) { + NSError *dtuhidError = nil; + if ([self dtuhidPressButtonUsagePage:DFConsumerControlUsagePage usage:0x40 holdSeconds:0.08 error:&dtuhidError]) { + DFLog(@"Home dispatched via dtuhidd"); + success = YES; + } else { + dispatchError = dtuhidError; + } + return; + } + // Prefer Simulator.app's own chrome affordance when available (iPhone 8 / SE // with the Touch ID bezel). This goes through the same code path as clicking // the modeled Home button in the Simulator window — guaranteed not to reboot. @@ -3705,6 +3774,22 @@ - (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error { __block NSError *dispatchError = nil; dispatch_block_t work = ^{ + if (self->_usesDTUHID) { + NSError *dtuhidError = nil; + BOOL pressed = [self dtuhidPressButtonUsagePage:DFConsumerControlUsagePage usage:0x40 holdSeconds:0.06 error:&dtuhidError]; + if (pressed) { + [NSThread sleepForTimeInterval:0.06]; + pressed = [self dtuhidPressButtonUsagePage:DFConsumerControlUsagePage usage:0x40 holdSeconds:0.06 error:&dtuhidError]; + } + if (pressed) { + DFLog(@"App Switcher dispatched via dtuhidd"); + success = YES; + } else { + dispatchError = dtuhidError; + } + return; + } + NSError *hidError = nil; if (DFOpenAppSwitcherViaHIDClient(self->_hidClient, &hidError)) { success = YES; @@ -3839,6 +3924,11 @@ - (BOOL)sendHardwareButtonNamed:(NSString *)buttonName __block BOOL success = NO; __block NSError *dispatchError = nil; dispatch_block_t work = ^{ + if (self->_usesDTUHID) { + DFDTUHIDClient *client = [self dtuhidClient:&dispatchError]; + success = client != nil && [client sendButtonUsagePage:DFConsumerControlUsagePage usage:0x40 down:pressed error:&dispatchError]; + return; + } if (self->_hidClient == nil) { dispatchError = DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @@ -3914,6 +4004,36 @@ - (BOOL)sendHardwareButtonNamed:(NSString *)buttonName __block NSError *dispatchError = nil; dispatch_block_t work = ^{ + if (self->_usesDTUHID) { + // dtuhidd takes buttons only as HID usages; the legacy button codes have no equivalent. + static NSDictionary *> *dtuhidButtons; + static dispatch_once_t dtuhidOnce; + dispatch_once(&dtuhidOnce, ^{ + NSMutableDictionary *> *buttons = [arbitraryButtons mutableCopy]; + buttons[@"lock"] = @[ @(DFConsumerControlUsagePage), @0x30 ]; + buttons[@"side"] = @[ @(DFConsumerControlUsagePage), @0x30 ]; + buttons[@"side-button"] = @[ @(DFConsumerControlUsagePage), @0x30 ]; + buttons[@"siri"] = @[ @(DFConsumerControlUsagePage), @0xCF ]; + dtuhidButtons = buttons; + }); + NSArray *dtuhidUsage = dtuhidButtons[normalizedName]; + if (!hasExplicitUsage && dtuhidUsage.count < 2) { + dispatchError = DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + [NSString stringWithFormat:@"Hardware button `%@` has no HID usage dtuhidd accepts.", buttonName ?: @""] + ); + return; + } + uint32_t page = hasExplicitUsage ? usagePage.unsignedIntValue : dtuhidUsage[0].unsignedIntValue; + uint32_t usageValue = hasExplicitUsage ? usage.unsignedIntValue : dtuhidUsage[1].unsignedIntValue; + DFDTUHIDClient *client = [self dtuhidClient:&dispatchError]; + if (client == nil || ![client sendButtonUsagePage:page usage:usageValue down:pressed error:&dispatchError]) { + return; + } + DFLog(@"Sending dtuhidd button `%@` page=0x%x usage=0x%x %@", buttonName, page, usageValue, pressed ? @"down" : @"up"); + success = YES; + return; + } if (self->_hidClient == nil) { dispatchError = DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @@ -4037,6 +4157,9 @@ - (BOOL)rotateLeft:(NSError * _Nullable __autoreleasing *)error { - (void)disconnect { dispatch_block_t work = ^{ + [self->_dtuhidClient disconnect]; + self->_dtuhidClient = nil; + if (self->_screenAdapter != nil && self->_screenAdapterCallbackUUID != nil) { ((void(*)(id, SEL, id))objc_msgSend)(self->_screenAdapter, sel_registerName("unregisterScreenAdapterCallbacksWithUUID:"), self->_screenAdapterCallbackUUID); } @@ -4207,6 +4330,24 @@ - (BOOL)sendTouchAtNormalizedX:(double)normalizedX } uint32_t target = DFIndigoDigitizerTarget; + if (self->_usesDTUHID) { + DFDTUHIDClient *client = [self dtuhidClient:&dispatchError]; + DFDTUHIDTouchPhase dtuhidPhase = + (phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved) + ? DFDTUHIDTouchPhaseDown + : DFDTUHIDTouchPhaseUp; + if (client == nil || ![client sendTouchPhase:dtuhidPhase x:clampedX y:clampedY edge:0 error:&dispatchError]) { + return; + } + self->_lastTouchPoint = point; + self->_hasLastTouchPoint = dtuhidPhase == DFDTUHIDTouchPhaseDown; + if (!self->_hasLastTouchPoint) { + self->_lastTouchPoint = CGPointZero; + } + success = YES; + return; + } + if (hidClient == nil) { dispatchError = DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @@ -4290,7 +4431,7 @@ - (BOOL)sendEdgeTouchAtNormalizedX:(double)normalizedX dispatchError = DFTVDirectTouchUnsupportedError(); return; } - if (self->_hidClient == nil) { + if (self->_hidClient == nil && !self->_usesDTUHID) { dispatchError = DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @"SimulatorKit did not provide a headless HID client for this simulator." @@ -4309,6 +4450,24 @@ - (BOOL)sendEdgeTouchAtNormalizedX:(double)normalizedX clampedY * fmax(displaySize.height - 1.0, 1.0) ); + if (self->_usesDTUHID) { + DFDTUHIDClient *client = [self dtuhidClient:&dispatchError]; + DFDTUHIDTouchPhase dtuhidPhase = + (phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved) + ? DFDTUHIDTouchPhaseDown + : DFDTUHIDTouchPhaseUp; + if (client == nil || ![client sendTouchPhase:dtuhidPhase x:clampedX y:clampedY edge:DFDTUHIDEdgeForPrivateTouchEdge(edge) error:&dispatchError]) { + return; + } + self->_lastTouchPoint = point; + self->_hasLastTouchPoint = dtuhidPhase == DFDTUHIDTouchPhaseDown; + if (!self->_hasLastTouchPoint) { + self->_lastTouchPoint = CGPointZero; + } + success = YES; + return; + } + uint32_t target = DFIndigoDigitizerTarget; IndigoHIDEdge hidEdge = DFIndigoHIDEdgeForPrivateTouchEdge(edge); BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved; @@ -4384,7 +4543,7 @@ - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1 dispatchError = DFTVDirectTouchUnsupportedError(); return; } - if (self->_hidClient == nil) { + if (self->_hidClient == nil && !self->_usesDTUHID) { dispatchError = DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @"SimulatorKit did not provide a headless HID client for this simulator." @@ -4415,6 +4574,22 @@ - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1 break; } + if (self->_usesDTUHID) { + DFDTUHIDClient *client = [self dtuhidClient:&dispatchError]; + DFDTUHIDTouchPhase dtuhidPhase = + (phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved) + ? DFDTUHIDTouchPhaseDown + : DFDTUHIDTouchPhaseUp; + if (client == nil || ![client sendTwoFingerTouchPhase:dtuhidPhase x1:x1 y1:y1 x2:x2 y2:y2 error:&dispatchError]) { + return; + } + if (phase != DFPrivateSimulatorTouchPhaseMoved) { + DFLog(@"Sending %@ dtuhidd multi-touch p1=(%.4f, %.4f) p2=(%.4f, %.4f)", phaseLabel, x1, y1, x2, y2); + } + success = YES; + return; + } + uint32_t target = DFIndigoDigitizerTarget; IndigoHIDMessage *message = DFCreateIndigoMultiTouchMessageDirect(CGPointMake(x1, y1), CGPointMake(x2, y2), @@ -4572,7 +4747,7 @@ - (BOOL)sendKeyCode:(uint16_t)keyCode __block NSError *dispatchError = nil; dispatch_block_t work = ^{ - if (self->_hidClient == nil) { + if (self->_hidClient == nil && !self->_usesDTUHID) { dispatchError = DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @"SimulatorKit did not provide a headless HID client for keyboard input." @@ -4602,24 +4777,24 @@ - (BOOL)sendKeyCode:(uint16_t)keyCode NSError *messageError = nil; for (NSNumber *modifierKeyCode in modifierKeyCodes) { - if (!DFSendSingleKeyboardEvent(self->_hidClient, modifierKeyCode.unsignedShortValue, YES, &messageError)) { + if (![self sendKeyboardUsage:modifierKeyCode.unsignedShortValue down:YES error:&messageError]) { dispatchError = messageError; return; } } - if (!DFSendSingleKeyboardEvent(self->_hidClient, keyCode, YES, &messageError)) { + if (![self sendKeyboardUsage:keyCode down:YES error:&messageError]) { dispatchError = messageError; return; } - if (!DFSendSingleKeyboardEvent(self->_hidClient, keyCode, NO, &messageError)) { + if (![self sendKeyboardUsage:keyCode down:NO error:&messageError]) { dispatchError = messageError; return; } for (NSNumber *modifierKeyCode in [modifierKeyCodes reverseObjectEnumerator]) { - if (!DFSendSingleKeyboardEvent(self->_hidClient, modifierKeyCode.unsignedShortValue, NO, &messageError)) { + if (![self sendKeyboardUsage:modifierKeyCode.unsignedShortValue down:NO error:&messageError]) { dispatchError = messageError; return; } @@ -4649,7 +4824,7 @@ - (BOOL)sendKeyCode:(uint16_t)keyCode __block NSError *dispatchError = nil; dispatch_block_t work = ^{ - if (self->_hidClient == nil) { + if (self->_hidClient == nil && !self->_usesDTUHID) { dispatchError = DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @"SimulatorKit did not provide a headless HID client for keyboard input." @@ -4658,7 +4833,7 @@ - (BOOL)sendKeyCode:(uint16_t)keyCode } NSError *messageError = nil; - if (!DFSendSingleKeyboardEvent(self->_hidClient, keyCode, down, &messageError)) { + if (![self sendKeyboardUsage:keyCode down:down error:&messageError]) { dispatchError = messageError; return; } From 3327f00c11be4fb9960f18cd1e99689b83c1ba22 Mon Sep 17 00:00:00 2001 From: nwalker Date: Tue, 10 Nov 2026 20:35:59 -0800 Subject: [PATCH 2/2] docs: describe the dtuhidd input path --- AGENTS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index a8facf46..9d80b4ef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,6 +29,8 @@ The native side should own anything that depends on macOS frameworks, `xcrun sim Wraps `xcrun simctl` for discovery, lifecycle management, app launching, URL opening, appearance toggles, and simulator log capture. - `packages/server/native/DFPrivateSimulatorDisplayBridge.*` Owns headless private display frames plus HID-based touch and keyboard injection. +- `packages/server/native/DFDTUHIDClient.*` + Sends touches, buttons, and keys to the guest `dtuhidd` digitizer service on CoreSimulator 1155.4 (Xcode 27) and later. - `packages/server/native/XCWAccessibilityBridge.*` Owns private CoreSimulator accessibility snapshots through `AccessibilityPlatformTranslation`. - `packages/server/native/XCWPrivateSimulatorSession.*` @@ -82,6 +84,8 @@ CoreSimulator service contexts resolve the active developer directory from `DEVE Accessibility recovery may use simulator launchctl UIKit application state plus hit-tested translations to recover candidate foreground pids; the returned tree must still be rooted at tokenized `AXPTranslator` application objects, because `translationApplicationObjectForPid:` can omit the bridge delegate token after private display lifecycle changes. Full-tree snapshots merge those recovered roots with the private frontmost application translation. Shallow snapshots with `maxDepth <= 2` use the tokenized frontmost application translation directly when it is available, and only run the expensive recovery sweep if frontmost lookup fails, so agent-oriented describe loops avoid launchctl and hit-test recovery overhead. Interactive-only snapshots also prune non-actionable native AX leaves during Objective-C serialization before the Rust-side compacting pass; keep this native pruning conservative so selector taps still retain actionable rows plus their ancestors. When multiple candidate application roots are discovered, serialize all of them in preferred order: non-extension app roots first, then largest translated roots, with `.appex`/PlugIns processes de-prioritized so SpringBoard and Safari app roots stay primary while widgets and WebContent roots remain debuggable. Widget renderer extension roots may report local frames; normalize those roots and children against matching SpringBoard widget placeholder frames before returning the snapshot. Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, mute, Apple Watch digital crown, Watch side button, and Watch left-side button dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony/vendor HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. Apple Watch Digital Crown rotation dispatches through `IndigoHIDMessageForDigitalCrownEvent` when SimulatorKit exposes it, with `IndigoHIDMessageForScrollEvent(..., target=0x34)` as the fallback. Browser mouse/trackpad wheel input over the device screen sends the normalized screen point with the scroll delta, moves the SimulatorKit pointer target there, then dispatches native scroll packets through `IndigoHIDMessageForScrollEvent(..., target=0x2)` with a digitizer-target fallback instead of synthesizing touch drags. tvOS simulators do not support direct screen touch; browser/API tap maps to Enter, swipe maps to arrow keys, and the native bridge rejects tvOS touch packets before they reach guest `SimulatorHID`. watchOS/tvOS skip dynamic pointer/mouse service warm-up because those guest runtimes abort on unsupported virtual services. Apple TV and Apple Watch simulators are fixed-orientation devices, so client and server rotation paths must not expose or dispatch device rotation for those families. On macOS/Xcode 27-era CoreSimulator profiles, `mainScreenWidth`, `mainScreenHeight`, and `mainScreenScale` may be absent from `profile.plist`; DeviceKit chrome rendering must read `capabilities.plist` `ScreenDimensionsCapability` or the primary `displays` entry before falling back to the framebuffer mask PDF. If none of those sources produce usable display geometry, the chrome profile must fail instead of returning a tiny synthetic bezel that hides the stream. +From CoreSimulator 1155.4 (Xcode 27) the guest routes HID through `dtuhidd` and drops legacy Indigo input: SimulatorKit's HID client accepts the messages and the guest ignores them, so success from `sendWithMessage:` proves nothing. When the loaded CoreSimulator bundle (not the selected Xcode) is 1155.4 or newer, the display bridge sends touches, edge and two-finger touches, Home, the App Switcher, hardware buttons, and keyboard usages through `DFDTUHIDClient` instead: XPC dictionaries (`messageType`, `isBarrier`, `featureIdentifier`, `payload`) on `com.apple.coredevice.feature.remote.hid.digitizer`, reached by `-[SimDevice lookup:error:]`, `xpc_endpoint_create_mach_port_4sim`, and `xpc_connection_enable_sim2host_4sim`. The client connects lazily and confirms a live daemon with a barrier request before use, because `dtuhidd` can abort during boot while its service lookup still succeeds. Buttons go only as HID usages there; legacy button codes such as Apple Pay and the software-keyboard toggle have no `dtuhidd` equivalent. Scroll and Digital Crown still use the Indigo path. + Two-point multi-touch dispatch prefers the current SimulatorKit/Indigo packet constructor and falls back to SimDeck's manual Indigo packet adapter. On Xcode 26 SimulatorKit, the constructor expects pixel-space points and stable two-finger movement requires sending `LeftMouseDown` for both `began` and `moved`, then `LeftMouseUp` for `ended`/`cancelled`; using `LeftMouseDragged` for multi-touch moves only advances one contact in UIKit. Do not coalesce multi-touch move packets in the WebSocket or WebRTC control paths, because gesture recognizers need the intermediate two-contact samples. WebKit inspection uses the simulator `webinspectord` Unix socket named `com.apple.webinspectord_sim.socket` and WebKit's binary-plist Remote Inspector selectors. It lists only WebKit content that the runtime exposes as inspectable. For app-owned `WKWebView` on iOS 16.4 and newer, the app must set `isInspectable = true`.