CameraCapture.mm 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861
  1. #if !PLATFORM_TVOS && UNITY_USES_WEBCAM
  2. #include "CameraCapture.h"
  3. #include "AVCapture.h"
  4. #include "CMVideoSampling.h"
  5. #include "CVTextureCache.h"
  6. #import <CoreVideo/CoreVideo.h>
  7. #include <cmath>
  8. static NSMutableArray<CameraCaptureController*> *activeColorAndDepthCameraControllers = nil;
  9. @implementation CameraCaptureController
  10. {
  11. AVCaptureDevice* _captureDevice;
  12. AVCaptureSession* _captureSession;
  13. AVCaptureDeviceInput* _captureInput;
  14. AVCaptureVideoDataOutput* _captureOutput;
  15. #if !PLATFORM_VISIONOS
  16. AVCaptureDepthDataOutput* _captureDepthOutput;
  17. AVCaptureDataOutputSynchronizer* _captureSynchronizer;
  18. #endif
  19. @public bool _isDepth;
  20. uint8_t* _pixelBufferCopy;
  21. CMVideoSampling _cmVideoSampling;
  22. NSString* _preset;
  23. CGPoint _focusPoint;
  24. #if !PLATFORM_VISIONOS
  25. AVCaptureFocusMode _focusMode;
  26. #endif
  27. @public void* _userData;
  28. @public size_t _width, _height;
  29. }
  30. - (bool)initCapture:(AVCaptureDevice*)device
  31. {
  32. if (UnityGetAVCapturePermission(avVideoCapture) == avCapturePermissionDenied)
  33. return false;
  34. self.captureDevice = device;
  35. #if PLATFORM_VISIONOS
  36. self.captureInput = [[AVCaptureDeviceInput alloc] initWithDevice: device error:nil];
  37. #else
  38. self.captureInput = [AVCaptureDeviceInput deviceInputWithDevice: device error: nil];
  39. #endif
  40. self.captureOutput = [[AVCaptureVideoDataOutput alloc] init];
  41. if (self.captureOutput == nil || self.captureInput == nil)
  42. return false;
  43. self.captureOutput.alwaysDiscardsLateVideoFrames = YES;
  44. NSDictionary* options = @{ (NSString*)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA) };
  45. [self.captureOutput setVideoSettings: options];
  46. CMVideoSampling_Initialize(&self->_cmVideoSampling);
  47. _width = _height = 0;
  48. _focusPoint = CGPointMake(0.5, 0.5); // default focus point is center
  49. #if !PLATFORM_VISIONOS
  50. _focusMode = AVCaptureFocusModeContinuousAutoFocus;
  51. #endif
  52. _pixelBufferCopy = nullptr;
  53. return true;
  54. }
  55. - (void)setCaptureFPS:(float)fps
  56. {
  57. if ([self.captureDevice lockForConfiguration: nil])
  58. {
  59. if (self.captureDevice.activeFormat)
  60. {
  61. fps = [self pickAvailableFrameRate: fps];
  62. self.captureDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps);
  63. self.captureDevice.activeVideoMaxFrameDuration = CMTimeMake(1, fps);
  64. }
  65. else
  66. {
  67. #if !PLATFORM_VISIONOS
  68. // In some corner cases (seeing this on iPod iOS 6.1.5) activeFormat is null.
  69. #pragma clang diagnostic push
  70. #pragma clang diagnostic ignored "-Wdeprecated-declarations"
  71. self.captureOutput.minFrameDuration = CMTimeMake(1, fps);
  72. #pragma clang diagnostic pop
  73. #endif
  74. }
  75. [self.captureDevice unlockForConfiguration];
  76. }
  77. }
  78. - (bool)initCapture:(AVCaptureDevice*)device preset:(NSString*)preset fps:(float)fps
  79. {
  80. if (![self initCapture: device])
  81. return false;
  82. self.captureSession = [[AVCaptureSession alloc] init];
  83. if ([self.captureSession canAddInput: self.captureInput])
  84. [self.captureSession addInput: self.captureInput];
  85. else
  86. return false;
  87. if ([self.captureSession canAddOutput: self.captureOutput])
  88. [self.captureSession addOutput: self.captureOutput];
  89. else
  90. return false;
  91. // queue on main thread to simplify gles life
  92. [self.captureOutput setSampleBufferDelegate: self queue: dispatch_get_main_queue()];
  93. self->_preset = preset;
  94. #if !PLATFORM_VISIONOS
  95. [self.captureSession setSessionPreset: preset];
  96. #endif
  97. [self setCaptureFPS: fps];
  98. return true;
  99. }
  100. - (void)captureOutput:(AVCaptureOutput*)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection*)connection
  101. {
  102. intptr_t tex = (intptr_t)CMVideoSampling_SampleBuffer(&self->_cmVideoSampling, sampleBuffer, &_width, &_height);
  103. UnityDidCaptureVideoFrame(tex, self->_userData);
  104. }
  105. - (void)capturePixelBufferToMemBuffer:(uint8_t*)dst
  106. {
  107. CVPixelBufferRef pbuf = (CVPixelBufferRef)self->_cmVideoSampling.cvImageBuffer;
  108. const size_t srcRowSize = CVPixelBufferGetBytesPerRow(pbuf);
  109. const size_t bufSize = srcRowSize * self->_height;
  110. if (self->_pixelBufferCopy == nullptr)
  111. {
  112. self->_pixelBufferCopy = (uint8_t*)::malloc(bufSize);
  113. }
  114. // while not the best way memory-wise, we want to minimize stalling
  115. CVPixelBufferLockBaseAddress(pbuf, kCVPixelBufferLock_ReadOnly);
  116. {
  117. ::memcpy(self->_pixelBufferCopy, CVPixelBufferGetBaseAddress(pbuf), bufSize);
  118. }
  119. CVPixelBufferUnlockBaseAddress(pbuf, kCVPixelBufferLock_ReadOnly);
  120. OSType pixelFormat = CVPixelBufferGetPixelFormatType(pbuf);
  121. size_t bpp = 0;
  122. switch (pixelFormat)
  123. {
  124. case kCVPixelFormatType_32BGRA:
  125. bpp = 4;
  126. break;
  127. case kCVPixelFormatType_DepthFloat16:
  128. bpp = 2;
  129. break;
  130. default:
  131. assert(false);
  132. break;
  133. }
  134. const size_t dstRowSize = self->_width * bpp;
  135. uint8_t* src = self->_pixelBufferCopy + (self->_height - 1) * srcRowSize;
  136. for (size_t i = 0; i < self->_height; ++i)
  137. {
  138. ::memcpy(dst, src, dstRowSize);
  139. dst += dstRowSize;
  140. src -= srcRowSize;
  141. }
  142. }
  143. - (int)isCVTextureFlipped
  144. {
  145. return IsCVTextureFlipped(self->_cmVideoSampling.cvTextureCacheTexture);
  146. }
  147. #if !PLATFORM_VISIONOS
  148. + (BOOL)focusPointSupported:(AVCaptureDevice*)captureDevice withFocusMode:(AVCaptureFocusMode)focusMode
  149. {
  150. return captureDevice.focusPointOfInterestSupported && [captureDevice isFocusModeSupported: focusMode];
  151. }
  152. #endif
  153. - (int)setFocusPointWithX:(float)x Y:(float)y
  154. {
  155. #if !PLATFORM_VISIONOS
  156. if (x < 0 || x > 1 || y < 0 || y > 1)
  157. {
  158. _focusPoint = CGPointMake(0.5, 0.5); // default value for iOS
  159. _focusMode = AVCaptureFocusModeContinuousAutoFocus;
  160. }
  161. else
  162. {
  163. _focusPoint = CGPointMake(x, 1.0 - y);
  164. _focusMode = AVCaptureFocusModeAutoFocus;
  165. }
  166. #endif
  167. return [self setFocusPoint];
  168. }
  169. - (int)setFocusPoint
  170. {
  171. #if !PLATFORM_VISIONOS
  172. if (self.captureDevice != nil && [CameraCaptureController focusPointSupported: self.captureDevice withFocusMode: _focusMode])
  173. {
  174. if ([self.captureDevice lockForConfiguration: nil])
  175. {
  176. self.captureDevice.focusPointOfInterest = _focusPoint;
  177. self.captureDevice.focusMode = _focusMode;
  178. [self.captureDevice unlockForConfiguration];
  179. return 1;
  180. }
  181. }
  182. #endif
  183. return 0;
  184. }
  185. + (NSMutableArray<CameraCaptureController*>*)getActiveColorAndDepthCameraControllers
  186. {
  187. if (activeColorAndDepthCameraControllers == nil)
  188. {
  189. activeColorAndDepthCameraControllers = [[NSMutableArray alloc] init];
  190. }
  191. return activeColorAndDepthCameraControllers;
  192. }
  193. + (void)addColorAndDepthCameraController:(CameraCaptureController*)controller
  194. {
  195. CameraCaptureController* prevController = [self findColorAndDepthCameraController: controller.captureDevice isDepth: controller->_isDepth];
  196. if (prevController != nil)
  197. [prevController pause];
  198. CameraCaptureController* otherController = [self findColorAndDepthCameraController: controller.captureDevice isDepth: !controller->_isDepth];
  199. if (otherController != nil)
  200. {
  201. [otherController.captureSession stopRunning];
  202. [otherController clearColorAndDepthCameraCaptureSession];
  203. }
  204. [[self getActiveColorAndDepthCameraControllers] addObject: controller];
  205. }
  206. + (void)removeColorAndDepthCameraController:(CameraCaptureController*)controller
  207. {
  208. [[self getActiveColorAndDepthCameraControllers] removeObject: controller];
  209. CameraCaptureController* otherController = [self findColorAndDepthCameraController: controller.captureDevice isDepth: !controller->_isDepth];
  210. if (otherController != nil)
  211. {
  212. [otherController initColorAndDepthCameraCaptureSession];
  213. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{
  214. [otherController.captureSession startRunning];
  215. });
  216. }
  217. }
  218. + (void)clearColorAndDepthCameraControllers
  219. {
  220. NSMutableArray<CameraCaptureController*>* activeColorAndDepthCameraControllers = [self getActiveColorAndDepthCameraControllers];
  221. for (CameraCaptureController *controller in activeColorAndDepthCameraControllers)
  222. {
  223. if (controller.captureSession != nil)
  224. {
  225. [controller.captureSession stopRunning];
  226. [controller clearColorAndDepthCameraCaptureSession];
  227. }
  228. }
  229. [activeColorAndDepthCameraControllers removeAllObjects];
  230. }
  231. + (CameraCaptureController*)findColorAndDepthCameraController:(AVCaptureDevice*)device isDepth:(bool)isDepth
  232. {
  233. for (CameraCaptureController *controller in [self getActiveColorAndDepthCameraControllers])
  234. {
  235. if (controller.captureDevice == device && controller->_isDepth == isDepth)
  236. return controller;
  237. }
  238. return nil;
  239. }
  240. - (bool)initColorAndDepthCameraCapture:(AVCaptureDevice*)device preset:(NSString*)preset fps:(float)fps isDepth:(bool)isDepth
  241. {
  242. if (![self initCapture: device])
  243. return false;
  244. #if !PLATFORM_VISIONOS
  245. self.captureDepthOutput = [[AVCaptureDepthDataOutput alloc] init];
  246. if (self.captureDepthOutput == nil)
  247. return false;
  248. self.captureDepthOutput.filteringEnabled = YES; // getting filtered depth data to avoid invalid values
  249. self.captureDepthOutput.alwaysDiscardsLateDepthData = YES;
  250. #endif
  251. self->_preset = preset;
  252. [self initColorAndDepthCameraCaptureSession];
  253. [self setCaptureFPS: fps];
  254. #if !PLATFORM_VISIONOS
  255. NSArray<AVCaptureOutput*> *outputs = [NSArray arrayWithObjects: self.captureOutput, self.captureDepthOutput, nil];
  256. self.captureSynchronizer = [[AVCaptureDataOutputSynchronizer alloc] initWithDataOutputs: outputs];
  257. // queue on main thread to simplify gles life
  258. [self.captureSynchronizer setDelegate: self queue: dispatch_get_main_queue()];
  259. #endif
  260. _isDepth = isDepth;
  261. return true;
  262. }
  263. - (void)initColorAndDepthCameraCaptureSession
  264. {
  265. self.captureSession = [[AVCaptureSession alloc] init];
  266. #if !PLATFORM_VISIONOS
  267. [self.captureSession setSessionPreset: self->_preset];
  268. #endif
  269. [self.captureSession addInput: self.captureInput];
  270. [self.captureSession addOutput: self.captureOutput];
  271. #if !PLATFORM_VISIONOS
  272. [self.captureSession addOutput: self.captureDepthOutput];
  273. #endif
  274. }
  275. - (void)clearColorAndDepthCameraCaptureSession
  276. {
  277. [self.captureSession removeInput: self.captureInput];
  278. [self.captureSession removeOutput: self.captureOutput];
  279. #if !PLATFORM_VISIONOS
  280. [self.captureSession removeOutput: self.captureDepthOutput];
  281. #endif
  282. self.captureSession = nil;
  283. }
  284. #if !PLATFORM_VISIONOS
  285. - (void)dataOutputSynchronizer:(AVCaptureDataOutputSynchronizer *)synchronizer didOutputSynchronizedDataCollection:(AVCaptureSynchronizedDataCollection *)synchronizedDataCollection
  286. {
  287. AVCaptureSynchronizedSampleBufferData *sampleData = (AVCaptureSynchronizedSampleBufferData*)[synchronizedDataCollection synchronizedDataForCaptureOutput: self.captureOutput];
  288. if (CMSampleBufferGetImageBuffer(sampleData.sampleBuffer) != nil)
  289. {
  290. CameraCaptureController* colorController = !self->_isDepth ? self : [CameraCaptureController findColorAndDepthCameraController: self.captureDevice isDepth: false];
  291. if (colorController != nil)
  292. {
  293. intptr_t tex = (intptr_t)CMVideoSampling_SampleBuffer(&colorController->_cmVideoSampling, sampleData.sampleBuffer, &(colorController->_width), &(colorController->_height));
  294. UnityDidCaptureVideoFrame(tex, colorController->_userData);
  295. }
  296. }
  297. AVCaptureSynchronizedDepthData *depthData = (AVCaptureSynchronizedDepthData*)[synchronizedDataCollection synchronizedDataForCaptureOutput: self.captureDepthOutput];
  298. if (depthData.depthData.depthDataMap != nil)
  299. {
  300. CameraCaptureController* depthController = self->_isDepth ? self : [CameraCaptureController findColorAndDepthCameraController: self.captureDevice isDepth: true];
  301. if (depthController != nil)
  302. {
  303. intptr_t tex = (intptr_t)CMVideoSampling_ImageBuffer(&depthController->_cmVideoSampling, [depthData.depthData depthDataByConvertingToDepthDataType: kCVPixelFormatType_DepthFloat16].depthDataMap, &(depthController->_width), &(depthController->_height));
  304. UnityDidCaptureVideoFrame(tex, depthController->_userData);
  305. }
  306. }
  307. }
  308. #endif
  309. - (void)start
  310. {
  311. #if !PLATFORM_VISIONOS
  312. if (self.captureDepthOutput != nil)
  313. {
  314. [CameraCaptureController addColorAndDepthCameraController: self];
  315. }
  316. else
  317. #endif
  318. {
  319. [CameraCaptureController clearColorAndDepthCameraControllers];
  320. }
  321. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{
  322. [self.captureSession startRunning];
  323. });
  324. }
  325. - (void)pause
  326. {
  327. [self.captureSession stopRunning];
  328. #if !PLATFORM_VISIONOS
  329. if (self.captureDepthOutput != nil)
  330. #endif
  331. {
  332. [CameraCaptureController removeColorAndDepthCameraController: self];
  333. }
  334. }
  335. - (void)stop
  336. {
  337. [self.captureSession stopRunning];
  338. [self.captureSession removeInput: self.captureInput];
  339. [self.captureSession removeOutput: self.captureOutput];
  340. self.captureInput = nil;
  341. self.captureOutput = nil;
  342. #if !PLATFORM_VISIONOS
  343. if (self.captureDepthOutput != nil)
  344. {
  345. self.captureSynchronizer = nil;
  346. [self.captureSession removeOutput: self.captureDepthOutput];
  347. self.captureDepthOutput = nil;
  348. [CameraCaptureController removeColorAndDepthCameraController: self];
  349. }
  350. #else
  351. [CameraCaptureController removeColorAndDepthCameraController: self];
  352. #endif
  353. self.captureDevice = nil;
  354. self.captureSession = nil;
  355. if (self->_pixelBufferCopy != nullptr)
  356. {
  357. ::free(self->_pixelBufferCopy);
  358. self->_pixelBufferCopy = nullptr;
  359. }
  360. CMVideoSampling_Uninitialize(&self->_cmVideoSampling);
  361. }
  362. - (float)pickAvailableFrameRate:(float)fps
  363. {
  364. AVFrameRateRange* bestRange = nil;
  365. float minDiff = INFINITY;
  366. float epsilon = 0.1;
  367. fps = fps > epsilon ? fps : 24;
  368. for (AVFrameRateRange* rate in self.captureDevice.activeFormat.videoSupportedFrameRateRanges)
  369. {
  370. if (fps + epsilon > rate.minFrameRate && fps - epsilon < rate.maxFrameRate)
  371. return fps;
  372. else
  373. {
  374. float diff = ::fmin(::fabs(fps - rate.minFrameRate), ::fabs(fps - rate.maxFrameRate));
  375. if (diff < minDiff)
  376. {
  377. minDiff = diff;
  378. bestRange = rate;
  379. }
  380. }
  381. }
  382. return fps > bestRange.maxFrameRate ? bestRange.maxFrameRate : bestRange.minFrameRate;
  383. }
  384. @synthesize captureDevice = _captureDevice;
  385. @synthesize captureSession = _captureSession;
  386. @synthesize captureOutput = _captureOutput;
  387. @synthesize captureInput = _captureInput;
  388. #if !PLATFORM_VISIONOS
  389. @synthesize captureDepthOutput = _captureDepthOutput;
  390. @synthesize captureSynchronizer = _captureSynchronizer;
  391. #endif
  392. @end
  393. #if !PLATFORM_VISIONOS
  394. // Preset for getting depth data with max resolution available
  395. static NSString* const depthCaptureSessionPreset = AVCaptureSessionPresetPhoto;
  396. #endif
  397. static NSMutableArray<CameraCaptureDevice*> *videoCaptureDevices = nil;
  398. @implementation CameraCaptureDevice
  399. {
  400. @public AVCaptureDevice* _device;
  401. @public int _frontFacing;
  402. @public int _autoFocusPointSupported;
  403. @public WebCamKind _kind;
  404. @public NSMutableArray<NSValue*>* _resolutions;
  405. NSMutableArray<NSString*>* _resPresets;
  406. }
  407. - (bool)isColorAndDepthCaptureDevice
  408. {
  409. #if !PLATFORM_VISIONOS
  410. for (AVCaptureDeviceFormat *format in [self->_device formats])
  411. {
  412. if ([format supportedDepthDataFormats].count > 0)
  413. return true;
  414. }
  415. #endif
  416. return false;
  417. }
  418. - (WebCamKind)getKind
  419. {
  420. #if !PLATFORM_VISIONOS
  421. AVCaptureDeviceType type = _device.deviceType;
  422. if ([type isEqualToString: AVCaptureDeviceTypeBuiltInWideAngleCamera])
  423. return kWebCamWideAngle;
  424. if ([type isEqualToString: AVCaptureDeviceTypeBuiltInTelephotoCamera])
  425. return kWebCamTelephoto;
  426. if ([type isEqualToString: AVCaptureDeviceTypeBuiltInDualCamera] && [self isColorAndDepthCaptureDevice])
  427. return kWebCamColorAndDepth;
  428. if (@available(iOS 13, *))
  429. {
  430. if ([type isEqualToString: AVCaptureDeviceTypeBuiltInUltraWideCamera])
  431. return kWebCamUltraWideAngle;
  432. if ([type isEqualToString: AVCaptureDeviceTypeBuiltInDualWideCamera])
  433. return kWebCamWideAngle;
  434. if ([type isEqualToString: AVCaptureDeviceTypeBuiltInTripleCamera])
  435. return kWebCamUltraWideAngle;
  436. }
  437. else
  438. {
  439. // Only works if device language is English; this was original impl, keeping to not regress
  440. if ([self->_device.localizedName containsString: @"Ultra Wide"])
  441. return kWebCamUltraWideAngle;
  442. }
  443. #if defined(__IPHONE_17_0) || defined(__TVOS_17_0)
  444. if (@available(iOS 17.0, *))
  445. {
  446. if ([type isEqualToString: AVCaptureDeviceTypeContinuityCamera])
  447. return kWebCamWideAngle;
  448. }
  449. #endif
  450. #endif
  451. #if PLATFORM_IOS
  452. #ifdef __IPHONE_15_4
  453. if (@available(iOS 15.4, *))
  454. {
  455. if ([type isEqualToString: AVCaptureDeviceTypeBuiltInLiDARDepthCamera])
  456. return kWebCamColorAndDepth;
  457. }
  458. #endif
  459. if ([type isEqualToString: AVCaptureDeviceTypeBuiltInTrueDepthCamera] && [self isColorAndDepthCaptureDevice])
  460. return kWebCamColorAndDepth;
  461. #endif
  462. return kWebCamWideAngle;
  463. }
  464. - (void)fillCaptureDeviceResolutions
  465. {
  466. #if PLATFORM_VISIONOS
  467. const int count = 0;
  468. #else
  469. static NSString* preset[] =
  470. {
  471. AVCaptureSessionPresetLow, // usually 192x144
  472. AVCaptureSessionPreset352x288,
  473. AVCaptureSessionPresetMedium, // usually 480x320
  474. AVCaptureSessionPreset640x480,
  475. AVCaptureSessionPreset1280x720,
  476. AVCaptureSessionPreset1920x1080, // usually the same as AVCaptureSessionPresetHigh
  477. AVCaptureSessionPreset3840x2160,
  478. };
  479. const int count = sizeof(preset) / sizeof(preset[0]);
  480. #endif
  481. self->_resolutions = [NSMutableArray arrayWithCapacity: count];
  482. self->_resPresets = [NSMutableArray arrayWithCapacity: count];
  483. #if PLATFORM_VISIONOS
  484. AVCaptureInput* captureInput = [[AVCaptureDeviceInput alloc] initWithDevice: self->_device error: nil];
  485. #else
  486. AVCaptureInput* captureInput = [AVCaptureDeviceInput deviceInputWithDevice: self->_device error: nil];
  487. #endif
  488. //Don't attempt to setup an AVCaptureSession if the user has explicitly denied permission to use the camera.
  489. if (captureInput != nil)
  490. {
  491. AVCaptureSession* captureSession = [[AVCaptureSession alloc] init];
  492. if ([captureSession canAddInput: captureInput])
  493. [captureSession addInput: captureInput];
  494. #if !PLATFORM_VISIONOS
  495. if (self->_kind == kWebCamColorAndDepth)
  496. {
  497. AVCaptureDepthDataOutput* captureDepthOutput = [[AVCaptureDepthDataOutput alloc] init];
  498. if ([captureSession canSetSessionPreset: depthCaptureSessionPreset])
  499. {
  500. [captureSession setSessionPreset: AVCaptureSessionPresetPhoto];
  501. [captureSession addOutput: captureDepthOutput];
  502. CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(self->_device.activeDepthDataFormat.formatDescription); // for ColorAndDepth camera return depth buffer resolution
  503. [self->_resolutions addObject: [NSValue valueWithCGSize: CGSizeMake(dim.width, dim.height)]];
  504. [self->_resPresets addObject: AVCaptureSessionPresetPhoto];
  505. }
  506. }
  507. else
  508. {
  509. for (int i = 0; i < count; ++i)
  510. {
  511. if ([captureSession canSetSessionPreset: preset[i]])
  512. {
  513. [captureSession setSessionPreset: preset[i]];
  514. CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(self->_device.activeFormat.formatDescription);
  515. [self->_resolutions addObject: [NSValue valueWithCGSize: CGSizeMake(dim.width, dim.height)]];
  516. [self->_resPresets addObject: preset[i]];
  517. }
  518. }
  519. }
  520. #endif
  521. }
  522. }
  523. - (NSString*)pickPresetFromWidth:(int)w height:(int)h
  524. {
  525. #if !PLATFORM_VISIONOS
  526. if (self->_kind == kWebCamColorAndDepth)
  527. {
  528. return depthCaptureSessionPreset;
  529. }
  530. #endif
  531. int requestedWidth = w > 0 ? w : 640;
  532. int requestedHeight = h > 0 ? h : 480;
  533. if (requestedHeight > requestedWidth) // hardware camera frame is landscape oriented
  534. std::swap(requestedWidth, requestedHeight);
  535. NSInteger ret = -1;
  536. double bestMatch = std::numeric_limits<double>::max();
  537. for (NSInteger i = 0, n = [_resolutions count]; i < n; ++i)
  538. {
  539. double width = [self->_resolutions[i] CGSizeValue].width;
  540. double height = [self->_resolutions[i] CGSizeValue].height;
  541. double match = std::abs(std::log(requestedWidth / width)) + std::abs(std::log(requestedHeight / height));
  542. if (match < bestMatch)
  543. {
  544. ret = i;
  545. bestMatch = match;
  546. }
  547. }
  548. #if PLATFORM_VISIONOS
  549. return nil;
  550. #else
  551. NSAssert(ret != -1, @"Cannot pick capture preset");
  552. return ret != -1 ? self->_resPresets[ret] : AVCaptureSessionPresetHigh;
  553. #endif
  554. }
  555. - (CameraCaptureDevice*)initWithDevice:(AVCaptureDevice*)device
  556. {
  557. self->_device = device;
  558. self->_frontFacing = device.position == AVCaptureDevicePositionFront ? 1 : 0;
  559. #if PLATFORM_VISIONOS
  560. self->_autoFocusPointSupported = 0;
  561. #else
  562. self->_autoFocusPointSupported = [CameraCaptureController focusPointSupported: device withFocusMode: AVCaptureFocusModeAutoFocus] ? 1 : 0;
  563. #endif
  564. self->_kind = [self getKind];
  565. [self fillCaptureDeviceResolutions];
  566. return self;
  567. }
  568. - (bool)initCaptureForController:(CameraCaptureController*)controller width:(int)w height:(int)h fps:(float)fps isDepth:(bool)isDepth
  569. {
  570. bool initResult = false;
  571. NSString *preset = [self pickPresetFromWidth: w height: h];
  572. if ([self isColorAndDepthCaptureDevice])
  573. {
  574. initResult = [controller initColorAndDepthCameraCapture: self->_device preset: preset fps: fps isDepth: isDepth];
  575. }
  576. else
  577. {
  578. assert(!isDepth);
  579. initResult = [controller initCapture: self->_device preset: preset fps: fps];
  580. }
  581. return initResult;
  582. }
  583. + (bool)initialized
  584. {
  585. return videoCaptureDevices != nil;
  586. }
  587. + (void)createCameraCaptureDevicesArray
  588. {
  589. videoCaptureDevices = [NSMutableArray arrayWithCapacity: 2];
  590. }
  591. + (void)addCameraCaptureDevice:(AVCaptureDevice*)device
  592. {
  593. [videoCaptureDevices addObject: [[CameraCaptureDevice alloc] initWithDevice: device]];
  594. }
  595. @end
  596. extern "C" void UnityEnumVideoCaptureDevices(void* udata, void(*callback)(void* udata, const char* name, int frontFacing, int autoFocusPointSupported, int kind, const int* resolutions, int resCount))
  597. {
  598. if (![CameraCaptureDevice initialized])
  599. {
  600. [CameraCaptureDevice createCameraCaptureDevicesArray];
  601. #if PLATFORM_VISIONOS
  602. AVCaptureDevice* systemPreferredCamera = [AVCaptureDevice systemPreferredCamera];
  603. if (systemPreferredCamera != nil)
  604. [CameraCaptureDevice addCameraCaptureDevice: systemPreferredCamera];
  605. #else
  606. NSMutableArray<AVCaptureDeviceType>* captureDevices = [NSMutableArray arrayWithObjects: AVCaptureDeviceTypeBuiltInWideAngleCamera, AVCaptureDeviceTypeBuiltInTelephotoCamera, nil];
  607. [captureDevices addObject: AVCaptureDeviceTypeBuiltInDualCamera];
  608. [captureDevices addObject: AVCaptureDeviceTypeBuiltInTrueDepthCamera];
  609. if (UnityiOS130orNewer())
  610. {
  611. [captureDevices addObject: AVCaptureDeviceTypeBuiltInUltraWideCamera];
  612. [captureDevices addObject: AVCaptureDeviceTypeBuiltInDualWideCamera];
  613. [captureDevices addObject: AVCaptureDeviceTypeBuiltInTripleCamera];
  614. }
  615. AVCaptureDeviceDiscoverySession *captureDeviceDiscoverySession = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes: captureDevices mediaType: AVMediaTypeVideo position: AVCaptureDevicePositionUnspecified];
  616. for (AVCaptureDevice* device in [captureDeviceDiscoverySession devices])
  617. {
  618. [CameraCaptureDevice addCameraCaptureDevice: device];
  619. }
  620. #endif
  621. }
  622. // we should not provide camera devices information while access has not been granted
  623. // but we need to try to enumerate camera devices anyway to trigger permission request dialog
  624. if ([AVCaptureDevice authorizationStatusForMediaType: AVMediaTypeVideo] != AVAuthorizationStatusAuthorized)
  625. return;
  626. for (CameraCaptureDevice *cameraCaptureDevice in videoCaptureDevices)
  627. {
  628. int resCount = (int)[cameraCaptureDevice->_resolutions count];
  629. int *resolutions = new int[resCount * 2];
  630. for (int i = 0; i < resCount; ++i)
  631. {
  632. resolutions[i * 2] = (int)[cameraCaptureDevice->_resolutions[i] CGSizeValue].width;
  633. resolutions[i * 2 + 1] = (int)[cameraCaptureDevice->_resolutions[i] CGSizeValue].height;
  634. }
  635. NSString* localizedName = cameraCaptureDevice->_device.localizedName;
  636. const char* deviceName = localizedName != nil ? localizedName.UTF8String : "";
  637. callback(udata, deviceName, cameraCaptureDevice->_frontFacing, cameraCaptureDevice->_autoFocusPointSupported, cameraCaptureDevice->_kind, resolutions, resCount);
  638. delete[] resolutions;
  639. }
  640. }
  641. extern "C" void* UnityInitCameraCapture(int deviceIndex, int w, int h, int fps, int isDepth, void* udata)
  642. {
  643. if (videoCaptureDevices != nil && deviceIndex < videoCaptureDevices.count)
  644. {
  645. CameraCaptureController* controller = [CameraCaptureController alloc];
  646. bool initResult = [videoCaptureDevices[deviceIndex] initCaptureForController: controller width: w height: h fps: (float)fps isDepth: (isDepth != 0)];
  647. if (initResult)
  648. {
  649. controller->_userData = udata;
  650. return (__bridge_retained void*)controller;
  651. }
  652. controller = nil;
  653. }
  654. return 0;
  655. }
  656. extern "C" void UnityStartCameraCapture(void* capture)
  657. {
  658. [(__bridge CameraCaptureController*)capture start];
  659. }
  660. extern "C" void UnityPauseCameraCapture(void* capture)
  661. {
  662. [(__bridge CameraCaptureController*)capture pause];
  663. }
  664. extern "C" void UnityStopCameraCapture(void* capture)
  665. {
  666. CameraCaptureController* controller = (__bridge_transfer CameraCaptureController*)capture;
  667. [controller stop];
  668. controller = nil;
  669. }
  670. extern "C" void UnityCameraCaptureExtents(void* capture, int* w, int* h)
  671. {
  672. CameraCaptureController* controller = (__bridge CameraCaptureController*)capture;
  673. if (controller == nil)
  674. return;
  675. *w = (int)controller->_width;
  676. *h = (int)controller->_height;
  677. }
  678. extern "C" void UnityCameraCaptureReadToMemory(void* capture, void* dst_, int w, int h)
  679. {
  680. CameraCaptureController* controller = (__bridge CameraCaptureController*)capture;
  681. if (controller == nil)
  682. return;
  683. assert(w == controller->_width && h == controller->_height);
  684. [controller capturePixelBufferToMemBuffer: (uint8_t*)dst_];
  685. }
  686. extern "C" int UnityCameraCaptureVideoRotationDeg(void* capture)
  687. {
  688. CameraCaptureController* controller = (__bridge CameraCaptureController*)capture;
  689. if (controller == nil)
  690. return 0;
  691. // all cams are landscape.
  692. switch (UnityCurrentOrientation())
  693. {
  694. case portrait: return 90;
  695. case portraitUpsideDown: return 270;
  696. case landscapeLeft: return controller.captureDevice.position == AVCaptureDevicePositionFront ? 180 : 0;
  697. case landscapeRight: return controller.captureDevice.position == AVCaptureDevicePositionFront ? 0 : 180;
  698. default: assert(false && "bad orientation returned from UnityCurrentOrientation()"); break;
  699. }
  700. return 0;
  701. }
  702. extern "C" int UnityCameraCaptureVerticallyMirrored(void* capture)
  703. {
  704. CameraCaptureController* controller = (__bridge CameraCaptureController*)capture;
  705. if (controller == nil)
  706. return 0;
  707. return [controller isCVTextureFlipped];
  708. }
  709. extern "C" int UnityCameraCaptureSetAutoFocusPoint(void* capture, float x, float y)
  710. {
  711. CameraCaptureController* controller = (__bridge CameraCaptureController*)capture;
  712. if (controller == nil)
  713. return 0;
  714. return [controller setFocusPointWithX: x Y: y];
  715. }
  716. #else
  717. // Stubs for when UNITY_USES_WEBCAM is not defined
  718. extern "C" void UnityEnumVideoCaptureDevices(void* udata, void(*callback)(void* udata, const char* name, int frontFacing, int autoFocusPointSupported, int kind, const int* resolutions, int resCount))
  719. {
  720. }
  721. extern "C" void* UnityInitCameraCapture(int deviceIndex, int w, int h, int fps, int isDepth, void* udata)
  722. {
  723. return 0;
  724. }
  725. extern "C" void UnityStartCameraCapture(void* capture)
  726. {
  727. }
  728. extern "C" void UnityPauseCameraCapture(void* capture)
  729. {
  730. }
  731. extern "C" void UnityStopCameraCapture(void* capture)
  732. {
  733. }
  734. extern "C" void UnityCameraCaptureExtents(void* capture, int* w, int* h)
  735. {
  736. }
  737. extern "C" void UnityCameraCaptureReadToMemory(void* capture, void* dst_, int w, int h)
  738. {
  739. }
  740. extern "C" int UnityCameraCaptureVideoRotationDeg(void* capture)
  741. {
  742. return 0;
  743. }
  744. extern "C" int UnityCameraCaptureVerticallyMirrored(void* capture)
  745. {
  746. return 0;
  747. }
  748. extern "C" int UnityCameraCaptureSetAutoFocusPoint(void* capture, float x, float y)
  749. {
  750. return 0;
  751. }
  752. #endif