From a97f288a562c0bed7cbd8b3568adb40d1042c9c9 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Fri, 30 Jan 2026 16:31:53 +0000 Subject: [PATCH 1/2] Add composite view capture support for Apple Vision Pro Implements continuous real-time composite view capture using ReplayKit, allowing capture of both physical passthrough and digital content. Key changes: - Added com.apple.developer.screen-capture.composite-passthrough entitlement - Created CompositeViewCapture.swift with ReplayKit-based continuous capture - Extended EnterpriseCameraAccessManager.cs with UseCompositeCapture toggle - Added composite texture handling with Metal pipeline (mirrors physical camera) - Updated ConfigureXcodeSettings.cs with NSPhotoLibraryUsageDescription Features: - Toggle between physical camera and composite view capture - Real-time texture updates via getCompositeTexture() - No audio capture (microphone disabled) - Public API: GetCompositeTexture2D() Resolves #16 Co-authored-by: Masahiro Yamaguchi --- .../FilesToAdd/Entitlements.entitlements | 2 + .../PrePostProcess/ConfigureXcodeSettings.cs | 1 + .../Scripts/CompositeViewCapture.swift | 238 ++++++++++++++++++ .../Scripts/EnterpriseCameraAccessManager.cs | 127 +++++++++- 4 files changed, 358 insertions(+), 10 deletions(-) create mode 100644 Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift diff --git a/Packages/com.styly.enterprise-camera-access/Editor/FilesToAdd/Entitlements.entitlements b/Packages/com.styly.enterprise-camera-access/Editor/FilesToAdd/Entitlements.entitlements index 136cfe4..bc96323 100644 --- a/Packages/com.styly.enterprise-camera-access/Editor/FilesToAdd/Entitlements.entitlements +++ b/Packages/com.styly.enterprise-camera-access/Editor/FilesToAdd/Entitlements.entitlements @@ -4,5 +4,7 @@ com.apple.developer.arkit.main-camera-access.allow + com.apple.developer.screen-capture.composite-passthrough + diff --git a/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs b/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs index 1085053..d8dc700 100644 --- a/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs +++ b/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs @@ -34,6 +34,7 @@ public static void OnPostProcessBuild(BuildTarget buildTarget, string pathToBuil string EntitlementsXmlAbsolutePath = Path.Combine(GetCurrentPackageAbsolutePath(), EntitlementsXmlPath); AddEntitlementsFile(EntitlementsXmlAbsolutePath, pathToBuiltProject); AddKeyValueToPlist("NSEnterpriseMCAMUsageDescription", "This app capture images from the main camera", pathToBuiltProject); + AddKeyValueToPlist("NSPhotoLibraryUsageDescription", "This app captures composite view screenshots", pathToBuiltProject); SetMinimumDeploymentVersion("XROS_DEPLOYMENT_TARGET", "2.0", pathToBuiltProject); } } diff --git a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift new file mode 100644 index 0000000..6f8cc78 --- /dev/null +++ b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift @@ -0,0 +1,238 @@ +import ReplayKit +import ARKit +import AVFoundation +import MetalKit +import Accelerate +import Foundation + +// Composite View Capture using ReplayKit +// This captures both the physical passthrough and digital content in real-time + +var compositeCurrentTexture: MTLTexture? +let compositeMtlDevice: MTLDevice = MTLCreateSystemDefaultDevice()! +var compositeTextureCache: CVMetalTextureCache! = nil +var compositeCommandQueue: MTLCommandQueue! +var compositePointer: UnsafeMutableRawPointer! = nil +var isCompositeRunning: Bool = false + +@_cdecl("startCompositeCapture") +public func startCompositeCapture() { + print("############ START COMPOSITE CAPTURE ############") + isCompositeRunning = true + + let screenRecorder = RPScreenRecorder.shared() + + // Disable microphone to avoid audio capture (as per requirements) + screenRecorder.isMicrophoneEnabled = false + screenRecorder.isCameraEnabled = false + + // Start capture with handler for each frame + screenRecorder.startCapture(handler: { (sampleBuffer, bufferType, error) in + if let error = error { + print("Composite capture error: \(error.localizedDescription)") + return + } + + // Only process video frames + guard bufferType == .video else { return } + guard isCompositeRunning else { return } + + // Get the pixel buffer from the sample buffer + guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { + print("Failed to get pixel buffer from sample buffer") + return + } + + createCompositeTexture(pixelBuffer) + }, completionHandler: { error in + if let error = error { + print("Failed to start composite capture: \(error.localizedDescription)") + isCompositeRunning = false + } else { + print("Composite capture started successfully") + } + }) +} + +@_cdecl("stopCompositeCapture") +public func stopCompositeCapture() { + print("############ STOP COMPOSITE CAPTURE ##############") + isCompositeRunning = false + + let screenRecorder = RPScreenRecorder.shared() + screenRecorder.stopCapture { error in + if let error = error { + print("Failed to stop composite capture: \(error.localizedDescription)") + } else { + print("Composite capture stopped successfully") + } + } +} + +@_cdecl("getCompositeTexture") +public func getCompositeTexture() -> UnsafeMutableRawPointer? { + return compositePointer +} + +@_cdecl("isCompositeAvailable") +public func isCompositeAvailable() -> Bool { + return RPScreenRecorder.shared().isAvailable +} + +private func createCompositeTexture(_ pixelBuffer: CVPixelBuffer) { + guard let pixelBufferBGRA: CVPixelBuffer = try? pixelBuffer.toCompositeBGRA() else { return } + let width = CVPixelBufferGetWidth(pixelBufferBGRA) + let height = CVPixelBufferGetHeight(pixelBufferBGRA) + var cvTexture: CVMetalTexture? + + if compositeTextureCache == nil { + CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, compositeMtlDevice, nil, &compositeTextureCache) + } + + _ = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + compositeTextureCache, + pixelBufferBGRA, + nil, + .bgra8Unorm_srgb, + width, + height, + 0, + &cvTexture) + guard let imageTexture = cvTexture else { return } + let texture: MTLTexture = CVMetalTextureGetTexture(imageTexture)! + + if compositeCurrentTexture == nil { + let texdescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: texture.pixelFormat, + width: texture.width, + height: texture.height, + mipmapped: false) + texdescriptor.usage = .unknown + compositeCurrentTexture = compositeMtlDevice.makeTexture(descriptor: texdescriptor) + } + + if compositeCommandQueue == nil { + compositeCommandQueue = compositeMtlDevice.makeCommandQueue() + } + + let commandBuffer = compositeCommandQueue.makeCommandBuffer()! + let blitEncoder = commandBuffer.makeBlitCommandEncoder()! + blitEncoder.copy(from: texture, + sourceSlice: 0, sourceLevel: 0, + sourceOrigin: MTLOrigin(x: 0, y: 0, z: 0), + sourceSize: MTLSizeMake(texture.width, texture.height, texture.depth), + to: compositeCurrentTexture!, destinationSlice: 0, destinationLevel: 0, + destinationOrigin: MTLOrigin(x: 0, y: 0, z: 0)) + blitEncoder.endEncoding() + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + + if compositePointer == nil { + compositePointer = Unmanaged.passUnretained(compositeCurrentTexture!).toOpaque() + } +} + +extension CVPixelBuffer { + public func toCompositeBGRA() throws -> CVPixelBuffer? { + let pixelBuffer = self + let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer) + + // ReplayKit typically provides BGRA format already, but handle YUV if needed + if pixelFormat == kCVPixelFormatType_32BGRA { + return pixelBuffer + } + + guard pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange else { return pixelBuffer } + + let yImage: CompositeVImage = pixelBuffer.withComposite({ CompositeVImage(pixelBuffer: $0, plane: 0) })! + let cbcrImage: CompositeVImage = pixelBuffer.withComposite({ CompositeVImage(pixelBuffer: $0, plane: 1) })! + let outPixelBuffer = CVPixelBuffer.makeComposite(width: yImage.width, height: yImage.height, format: kCVPixelFormatType_32BGRA)! + var argbImage = outPixelBuffer.withComposite({ CompositeVImage(pixelBuffer: $0) })! + try argbImage.drawComposite(yBuffer: yImage.buffer, cbcrBuffer: cbcrImage.buffer) + argbImage.permuteComposite(channelMap: [3, 2, 1, 0]) + return outPixelBuffer + } +} + +struct CompositeVImage { + let width: Int + let height: Int + let bytesPerRow: Int + var buffer: vImage_Buffer + + init?(pixelBuffer: CVPixelBuffer, plane: Int) { + guard let rawBuffer = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, plane) else { return nil } + self.width = CVPixelBufferGetWidthOfPlane(pixelBuffer, plane) + self.height = CVPixelBufferGetHeightOfPlane(pixelBuffer, plane) + self.bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, plane) + self.buffer = vImage_Buffer( + data: UnsafeMutableRawPointer(mutating: rawBuffer), + height: vImagePixelCount(height), + width: vImagePixelCount(width), + rowBytes: bytesPerRow) + } + + init?(pixelBuffer: CVPixelBuffer) { + guard let rawBuffer = CVPixelBufferGetBaseAddress(pixelBuffer) else { return nil } + self.width = CVPixelBufferGetWidth(pixelBuffer) + self.height = CVPixelBufferGetHeight(pixelBuffer) + self.bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer) + self.buffer = vImage_Buffer( + data: UnsafeMutableRawPointer(mutating: rawBuffer), + height: vImagePixelCount(height), + width: vImagePixelCount(width), + rowBytes: bytesPerRow) + } + + mutating func drawComposite(yBuffer: vImage_Buffer, cbcrBuffer: vImage_Buffer) throws { + try buffer.drawComposite(yBuffer: yBuffer, cbcrBuffer: cbcrBuffer) + } + + mutating func permuteComposite(channelMap: [UInt8]) { + buffer.permuteComposite(channelMap: channelMap) + } +} + +extension CVPixelBuffer { + func withComposite(_ closure: ((_ pixelBuffer: CVPixelBuffer) -> T)) -> T { + CVPixelBufferLockBaseAddress(self, .readOnly) + let result = closure(self) + CVPixelBufferUnlockBaseAddress(self, .readOnly) + return result + } + + static func makeComposite(width: Int, height: Int, format: OSType) -> CVPixelBuffer? { + var pixelBuffer: CVPixelBuffer? = nil + CVPixelBufferCreate(kCFAllocatorDefault, + width, + height, + format, + [String(kCVPixelBufferIOSurfacePropertiesKey): [ + "IOSurfaceOpenGLESFBOCompatibility": true, + "IOSurfaceOpenGLESTextureCompatibility": true, + "IOSurfaceCoreAnimationCompatibility": true, + ]] as CFDictionary, + &pixelBuffer) + return pixelBuffer + } +} + +extension vImage_Buffer { + mutating func drawComposite(yBuffer: vImage_Buffer, cbcrBuffer: vImage_Buffer) throws { + var yBuffer = yBuffer + var cbcrBuffer = cbcrBuffer + var conversionMatrix: vImage_YpCbCrToARGB = { + var pixelRange = vImage_YpCbCrPixelRange(Yp_bias: 0, CbCr_bias: 128, YpRangeMax: 255, CbCrRangeMax: 255, YpMax: 255, YpMin: 1, CbCrMax: 255, CbCrMin: 0) + var matrix = vImage_YpCbCrToARGB() + vImageConvert_YpCbCrToARGB_GenerateConversion(kvImage_YpCbCrToARGBMatrix_ITU_R_709_2, &pixelRange, &matrix, kvImage420Yp8_CbCr8, kvImageARGB8888, UInt32(kvImageNoFlags)) + return matrix + }() + let error = vImageConvert_420Yp8_CbCr8ToARGB8888(&yBuffer, &cbcrBuffer, &self, &conversionMatrix, nil, 255, UInt32(kvImageNoFlags)) + if error != kvImageNoError { + fatalError() + } + } + + mutating func permuteComposite(channelMap: [UInt8]) { + vImagePermuteChannels_ARGB8888(&self, &self, channelMap, 0) + } +} diff --git a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs index cefadef..037d803 100644 --- a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs +++ b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs @@ -19,6 +19,9 @@ public class EnterpriseCameraAccessManager : MonoBehaviour [Tooltip("WebCam will be used for Editor mode or Smartphone. Default camera is used if this field is empty.")] public string WebCamDeviceName = ""; + [Tooltip("Enable composite view capture (passthrough + digital content) instead of physical camera only")] + public bool UseCompositeCapture = false; + private WebCamTexture webCamTexture; private Texture2D tmpTexture = null; private string tempBase64String = null; @@ -30,6 +33,14 @@ public class EnterpriseCameraAccessManager : MonoBehaviour private IntPtr _texturePtr; private int _width = 1920; private int _height = 1080; + + // Composite capture + private bool _hasSetCompositeTexture = false; + private Texture2D _compositeTexture; + private RenderTexture _compositeRenderTexture; + private IntPtr _compositeTexturePtr; + private int _compositeWidth = 1920; + private int _compositeHeight = 1080; #endif #if USE_PICOXR && UNITY_ANDROID && !UNITY_EDITOR @@ -71,11 +82,24 @@ void Start() #endif #if UNITY_VISIONOS && !UNITY_EDITOR - _renderTexture = new RenderTexture(_width, _height, 1, RenderTextureFormat.ARGB32); - _renderTexture.enableRandomWrite = true; - _renderTexture.Create(); - PreviewMaterial.mainTexture = _renderTexture; - startCapture(); + if (UseCompositeCapture) + { + // Initialize composite capture + _compositeRenderTexture = new RenderTexture(_compositeWidth, _compositeHeight, 1, RenderTextureFormat.ARGB32); + _compositeRenderTexture.enableRandomWrite = true; + _compositeRenderTexture.Create(); + PreviewMaterial.mainTexture = _compositeRenderTexture; + startCompositeCapture(); + } + else + { + // Initialize physical camera capture + _renderTexture = new RenderTexture(_width, _height, 1, RenderTextureFormat.ARGB32); + _renderTexture.enableRandomWrite = true; + _renderTexture.Create(); + PreviewMaterial.mainTexture = _renderTexture; + startCapture(); + } return; #endif @@ -98,7 +122,14 @@ void OnDisable() #endif #if UNITY_VISIONOS && !UNITY_EDITOR - stopCapture(); + if (UseCompositeCapture) + { + stopCompositeCapture(); + } + else + { + stopCapture(); + } return; #endif if (webCamTexture != null) { webCamTexture.Stop(); } @@ -138,13 +169,27 @@ IEnumerator RequestCameraPermission_Android() void Update() { #if UNITY_VISIONOS && !UNITY_EDITOR - if (_hasSetTexture) + if (UseCompositeCapture) { - UpdateTexture(); + if (_hasSetCompositeTexture) + { + UpdateCompositeTexture(); + } + else + { + TryGetCompositeTexture(); + } } else { - TryGetTexture(); + if (_hasSetTexture) + { + UpdateTexture(); + } + else + { + TryGetTexture(); + } } #else // Apply WebCamTexture to material @@ -260,10 +305,57 @@ private void UpdateTexture() // スケールとオフセットを使用して上下反転を行う Vector2 scale = new Vector2(1, -1); Vector2 offset = new Vector2(0, 1); - + Graphics.Blit(_texture, _renderTexture, scale, offset); Unity.PolySpatial.PolySpatialObjectUtils.MarkDirty(_renderTexture); } + + private void TryGetCompositeTexture() + { + IntPtr texturePtr = getCompositeTexture(); + if (texturePtr == IntPtr.Zero) return; + + _compositeTexturePtr = texturePtr; + + if (_compositeTexture != null) + { + UnityEngine.Object.Destroy(_compositeTexture); + } + + _compositeTexture = Texture2D.CreateExternalTexture(_compositeWidth, _compositeHeight, TextureFormat.BGRA32, false, false, _compositeTexturePtr); + _compositeTexture.UpdateExternalTexture(_compositeTexturePtr); + + // スケールとオフセットを使用して上下反転を行う + Vector2 scale = new Vector2(1, -1); + Vector2 offset = new Vector2(0, 1); + + Graphics.Blit(_compositeTexture, _compositeRenderTexture, scale, offset); + PreviewMaterial.mainTexture = _compositeRenderTexture; + + _hasSetCompositeTexture = true; + } + + private void UpdateCompositeTexture() + { + // スケールとオフセットを使用して上下反転を行う + Vector2 scale = new Vector2(1, -1); + Vector2 offset = new Vector2(0, 1); + + Graphics.Blit(_compositeTexture, _compositeRenderTexture, scale, offset); + Unity.PolySpatial.PolySpatialObjectUtils.MarkDirty(_compositeRenderTexture); + } + + /// + /// Get composite view texture (passthrough + digital content) + /// + public Texture2D GetCompositeTexture2D() + { +#if UNITY_VISIONOS && !UNITY_EDITOR + return _compositeTexture; +#else + return null; +#endif + } #endif #if UNITY_VISIONOS && !UNITY_EDITOR @@ -277,9 +369,24 @@ private void UpdateTexture() static extern void stopCapture(); [DllImport("__Internal")] static extern IntPtr getTexture(); + [DllImport("__Internal")] + static extern void startCompositeCapture(); + [DllImport("__Internal")] + static extern void stopCompositeCapture(); + [DllImport("__Internal")] + static extern IntPtr getCompositeTexture(); + [DllImport("__Internal")] + static extern bool isCompositeAvailable(); #else static void SetNativeCallbackOfCameraAccess(CallbackDelegate callback) { } static void StartVisionProMainCameraCapture() { } + static void startCapture() { } + static void stopCapture() { } + static IntPtr getTexture() { return IntPtr.Zero; } + static void startCompositeCapture() { } + static void stopCompositeCapture() { } + static IntPtr getCompositeTexture() { return IntPtr.Zero; } + static bool isCompositeAvailable() { return false; } #endif From 04968f03a51c107463eb248e58043a73bd6eaa7a Mon Sep 17 00:00:00 2001 From: Masahiro Yamaguchi Date: Sat, 31 Jan 2026 08:46:58 -0800 Subject: [PATCH 2/2] Address PR review feedback for composite view capture - Remove unused ARKit import - Use explicit .shaderRead Metal texture usage instead of .unknown - Replace force unwraps with guard statements for better error handling - Replace fatalError() with logging for graceful error recovery - Add memory cleanup in OnDisable for textures and render textures - Optimize TryGetCompositeTexture to only recreate texture when pointer changes - Add UpdateExternalTexture call in UpdateCompositeTexture for proper refresh - Add IsCompositeCaptureAvailable() public API method - Add XML documentation for GetCompositeTexture2D - Expand tooltip to explain composite vs physical capture trade-offs - Remove unnecessary NSPhotoLibraryUsageDescription plist entry - Log unsupported pixel formats instead of silently returning Co-Authored-By: Claude Opus 4.5 --- .../PrePostProcess/ConfigureXcodeSettings.cs | 3 +- .../Scripts/CompositeViewCapture.swift | 25 ++++--- .../Scripts/EnterpriseCameraAccessManager.cs | 66 ++++++++++++++++--- 3 files changed, 73 insertions(+), 21 deletions(-) diff --git a/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs b/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs index d8dc700..00c3298 100644 --- a/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs +++ b/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs @@ -34,7 +34,6 @@ public static void OnPostProcessBuild(BuildTarget buildTarget, string pathToBuil string EntitlementsXmlAbsolutePath = Path.Combine(GetCurrentPackageAbsolutePath(), EntitlementsXmlPath); AddEntitlementsFile(EntitlementsXmlAbsolutePath, pathToBuiltProject); AddKeyValueToPlist("NSEnterpriseMCAMUsageDescription", "This app capture images from the main camera", pathToBuiltProject); - AddKeyValueToPlist("NSPhotoLibraryUsageDescription", "This app captures composite view screenshots", pathToBuiltProject); SetMinimumDeploymentVersion("XROS_DEPLOYMENT_TARGET", "2.0", pathToBuiltProject); } } @@ -126,7 +125,7 @@ static void SetMinimumDeploymentVersion(string DepleymentTargetString, string ve } /// - /// Add key/value pair to info.plist + /// Add key/value pair to info.plist /// static void AddKeyValueToPlist(string key, string value, string pathToBuiltProject) { diff --git a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift index 6f8cc78..2094b02 100644 --- a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift +++ b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift @@ -1,5 +1,4 @@ import ReplayKit -import ARKit import AVFoundation import MetalKit import Accelerate @@ -89,7 +88,7 @@ private func createCompositeTexture(_ pixelBuffer: CVPixelBuffer) { CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, compositeMtlDevice, nil, &compositeTextureCache) } - _ = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + let textureStatus = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, compositeTextureCache, pixelBufferBGRA, nil, @@ -98,15 +97,16 @@ private func createCompositeTexture(_ pixelBuffer: CVPixelBuffer) { height, 0, &cvTexture) - guard let imageTexture = cvTexture else { return } - let texture: MTLTexture = CVMetalTextureGetTexture(imageTexture)! + guard textureStatus == kCVReturnSuccess, + let imageTexture = cvTexture, + let texture = CVMetalTextureGetTexture(imageTexture) else { return } if compositeCurrentTexture == nil { let texdescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: texture.pixelFormat, width: texture.width, height: texture.height, mipmapped: false) - texdescriptor.usage = .unknown + texdescriptor.usage = [.shaderRead] compositeCurrentTexture = compositeMtlDevice.makeTexture(descriptor: texdescriptor) } @@ -114,13 +114,14 @@ private func createCompositeTexture(_ pixelBuffer: CVPixelBuffer) { compositeCommandQueue = compositeMtlDevice.makeCommandQueue() } - let commandBuffer = compositeCommandQueue.makeCommandBuffer()! - let blitEncoder = commandBuffer.makeBlitCommandEncoder()! + guard let commandBuffer = compositeCommandQueue.makeCommandBuffer(), + let blitEncoder = commandBuffer.makeBlitCommandEncoder(), + let destTexture = compositeCurrentTexture else { return } blitEncoder.copy(from: texture, sourceSlice: 0, sourceLevel: 0, sourceOrigin: MTLOrigin(x: 0, y: 0, z: 0), sourceSize: MTLSizeMake(texture.width, texture.height, texture.depth), - to: compositeCurrentTexture!, destinationSlice: 0, destinationLevel: 0, + to: destTexture, destinationSlice: 0, destinationLevel: 0, destinationOrigin: MTLOrigin(x: 0, y: 0, z: 0)) blitEncoder.endEncoding() commandBuffer.commit() @@ -141,7 +142,10 @@ extension CVPixelBuffer { return pixelBuffer } - guard pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange else { return pixelBuffer } + guard pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange else { + print("toCompositeBGRA: Unsupported pixel format: \(pixelFormat). Expected BGRA or 420YpCbCr8BiPlanarFullRange.") + return nil + } let yImage: CompositeVImage = pixelBuffer.withComposite({ CompositeVImage(pixelBuffer: $0, plane: 0) })! let cbcrImage: CompositeVImage = pixelBuffer.withComposite({ CompositeVImage(pixelBuffer: $0, plane: 1) })! @@ -228,7 +232,8 @@ extension vImage_Buffer { }() let error = vImageConvert_420Yp8_CbCr8ToARGB8888(&yBuffer, &cbcrBuffer, &self, &conversionMatrix, nil, 255, UInt32(kvImageNoFlags)) if error != kvImageNoError { - fatalError() + print("vImage conversion error: \(error)") + return } } diff --git a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs index 037d803..60ddb95 100644 --- a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs +++ b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs @@ -19,7 +19,7 @@ public class EnterpriseCameraAccessManager : MonoBehaviour [Tooltip("WebCam will be used for Editor mode or Smartphone. Default camera is used if this field is empty.")] public string WebCamDeviceName = ""; - [Tooltip("Enable composite view capture (passthrough + digital content) instead of physical camera only")] + [Tooltip("Enable composite view capture (passthrough + digital content) instead of physical camera only. Note: (1) Composite capture may have performance implications, (2) Physical camera only captures the real world without digital content, (3) Composite requires screen-capture entitlement and has privacy implications.")] public bool UseCompositeCapture = false; private WebCamTexture webCamTexture; @@ -125,10 +125,36 @@ void OnDisable() if (UseCompositeCapture) { stopCompositeCapture(); + + if (_compositeRenderTexture != null) + { + _compositeRenderTexture.Release(); + Destroy(_compositeRenderTexture); + _compositeRenderTexture = null; + } + + if (_compositeTexture != null) + { + Destroy(_compositeTexture); + _compositeTexture = null; + } } else { stopCapture(); + + if (_renderTexture != null) + { + _renderTexture.Release(); + Destroy(_renderTexture); + _renderTexture = null; + } + + if (_texture != null) + { + Destroy(_texture); + _texture = null; + } } return; #endif @@ -288,12 +314,12 @@ private void TryGetTexture() _texture = Texture2D.CreateExternalTexture(_width, _height, TextureFormat.BGRA32, false, false, _texturePtr); _texture.UpdateExternalTexture(_texturePtr); - + // スケールとオフセットを使用して上下反転を行う // scale.y を -1 にすることで上下反転、offset.y を 1 にすることで位置を調整 Vector2 scale = new Vector2(1, -1); Vector2 offset = new Vector2(0, 1); - + Graphics.Blit(_texture, _renderTexture, scale, offset); PreviewMaterial.mainTexture = _renderTexture; @@ -315,14 +341,19 @@ private void TryGetCompositeTexture() IntPtr texturePtr = getCompositeTexture(); if (texturePtr == IntPtr.Zero) return; - _compositeTexturePtr = texturePtr; - - if (_compositeTexture != null) + // Only recreate the external texture if the underlying pointer has changed + if (_compositeTexture == null || texturePtr != _compositeTexturePtr) { - UnityEngine.Object.Destroy(_compositeTexture); + _compositeTexturePtr = texturePtr; + + if (_compositeTexture != null) + { + UnityEngine.Object.Destroy(_compositeTexture); + } + + _compositeTexture = Texture2D.CreateExternalTexture(_compositeWidth, _compositeHeight, TextureFormat.BGRA32, false, false, _compositeTexturePtr); } - _compositeTexture = Texture2D.CreateExternalTexture(_compositeWidth, _compositeHeight, TextureFormat.BGRA32, false, false, _compositeTexturePtr); _compositeTexture.UpdateExternalTexture(_compositeTexturePtr); // スケールとオフセットを使用して上下反転を行う @@ -337,6 +368,9 @@ private void TryGetCompositeTexture() private void UpdateCompositeTexture() { + // Refresh the texture data from native side + _compositeTexture.UpdateExternalTexture(_compositeTexturePtr); + // スケールとオフセットを使用して上下反転を行う Vector2 scale = new Vector2(1, -1); Vector2 offset = new Vector2(0, 1); @@ -346,8 +380,12 @@ private void UpdateCompositeTexture() } /// - /// Get composite view texture (passthrough + digital content) + /// Get composite view texture (passthrough + digital content). + /// Returns null if composite capture is not available on the current platform, + /// or if the capture has not been initialized yet. Check IsCompositeCaptureAvailable() + /// before calling this method. /// + /// The composite texture, or null if not available or not initialized. public Texture2D GetCompositeTexture2D() { #if UNITY_VISIONOS && !UNITY_EDITOR @@ -356,6 +394,16 @@ public Texture2D GetCompositeTexture2D() return null; #endif } + + /// + /// Indicates whether composite capture is available on the current platform. + /// Call this before using composite capture features such as GetCompositeTexture2D(). + /// + /// True if composite capture is available, false otherwise. + public bool IsCompositeCaptureAvailable() + { + return isCompositeAvailable(); + } #endif #if UNITY_VISIONOS && !UNITY_EDITOR