diff --git a/Packages/com.styly.enterprise-camera-access/Editor/FilesToAdd/Entitlements.entitlements b/Packages/com.styly.enterprise-camera-access/Editor/FilesToAdd/Entitlements.entitlements
index 136cfe4..bc96323 100644
--- a/Packages/com.styly.enterprise-camera-access/Editor/FilesToAdd/Entitlements.entitlements
+++ b/Packages/com.styly.enterprise-camera-access/Editor/FilesToAdd/Entitlements.entitlements
@@ -4,5 +4,7 @@
com.apple.developer.arkit.main-camera-access.allow
+ com.apple.developer.screen-capture.composite-passthrough
+
diff --git a/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs b/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs
index 1085053..00c3298 100644
--- a/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs
+++ b/Packages/com.styly.enterprise-camera-access/Editor/PrePostProcess/ConfigureXcodeSettings.cs
@@ -125,7 +125,7 @@ static void SetMinimumDeploymentVersion(string DepleymentTargetString, string ve
}
///
- /// Add key/value pair to info.plist
+ /// Add key/value pair to info.plist
///
static void AddKeyValueToPlist(string key, string value, string pathToBuiltProject)
{
diff --git a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift
new file mode 100644
index 0000000..2094b02
--- /dev/null
+++ b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/CompositeViewCapture.swift
@@ -0,0 +1,243 @@
+import ReplayKit
+import AVFoundation
+import MetalKit
+import Accelerate
+import Foundation
+
+// Composite View Capture using ReplayKit
+// This captures both the physical passthrough and digital content in real-time
+
+var compositeCurrentTexture: MTLTexture?
+let compositeMtlDevice: MTLDevice = MTLCreateSystemDefaultDevice()!
+var compositeTextureCache: CVMetalTextureCache! = nil
+var compositeCommandQueue: MTLCommandQueue!
+var compositePointer: UnsafeMutableRawPointer! = nil
+var isCompositeRunning: Bool = false
+
+@_cdecl("startCompositeCapture")
+public func startCompositeCapture() {
+ print("############ START COMPOSITE CAPTURE ############")
+ isCompositeRunning = true
+
+ let screenRecorder = RPScreenRecorder.shared()
+
+ // Disable microphone to avoid audio capture (as per requirements)
+ screenRecorder.isMicrophoneEnabled = false
+ screenRecorder.isCameraEnabled = false
+
+ // Start capture with handler for each frame
+ screenRecorder.startCapture(handler: { (sampleBuffer, bufferType, error) in
+ if let error = error {
+ print("Composite capture error: \(error.localizedDescription)")
+ return
+ }
+
+ // Only process video frames
+ guard bufferType == .video else { return }
+ guard isCompositeRunning else { return }
+
+ // Get the pixel buffer from the sample buffer
+ guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
+ print("Failed to get pixel buffer from sample buffer")
+ return
+ }
+
+ createCompositeTexture(pixelBuffer)
+ }, completionHandler: { error in
+ if let error = error {
+ print("Failed to start composite capture: \(error.localizedDescription)")
+ isCompositeRunning = false
+ } else {
+ print("Composite capture started successfully")
+ }
+ })
+}
+
+@_cdecl("stopCompositeCapture")
+public func stopCompositeCapture() {
+ print("############ STOP COMPOSITE CAPTURE ##############")
+ isCompositeRunning = false
+
+ let screenRecorder = RPScreenRecorder.shared()
+ screenRecorder.stopCapture { error in
+ if let error = error {
+ print("Failed to stop composite capture: \(error.localizedDescription)")
+ } else {
+ print("Composite capture stopped successfully")
+ }
+ }
+}
+
+@_cdecl("getCompositeTexture")
+public func getCompositeTexture() -> UnsafeMutableRawPointer? {
+ return compositePointer
+}
+
+@_cdecl("isCompositeAvailable")
+public func isCompositeAvailable() -> Bool {
+ return RPScreenRecorder.shared().isAvailable
+}
+
+private func createCompositeTexture(_ pixelBuffer: CVPixelBuffer) {
+ guard let pixelBufferBGRA: CVPixelBuffer = try? pixelBuffer.toCompositeBGRA() else { return }
+ let width = CVPixelBufferGetWidth(pixelBufferBGRA)
+ let height = CVPixelBufferGetHeight(pixelBufferBGRA)
+ var cvTexture: CVMetalTexture?
+
+ if compositeTextureCache == nil {
+ CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, compositeMtlDevice, nil, &compositeTextureCache)
+ }
+
+ let textureStatus = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
+ compositeTextureCache,
+ pixelBufferBGRA,
+ nil,
+ .bgra8Unorm_srgb,
+ width,
+ height,
+ 0,
+ &cvTexture)
+ guard textureStatus == kCVReturnSuccess,
+ let imageTexture = cvTexture,
+ let texture = CVMetalTextureGetTexture(imageTexture) else { return }
+
+ if compositeCurrentTexture == nil {
+ let texdescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: texture.pixelFormat,
+ width: texture.width,
+ height: texture.height,
+ mipmapped: false)
+ texdescriptor.usage = [.shaderRead]
+ compositeCurrentTexture = compositeMtlDevice.makeTexture(descriptor: texdescriptor)
+ }
+
+ if compositeCommandQueue == nil {
+ compositeCommandQueue = compositeMtlDevice.makeCommandQueue()
+ }
+
+ guard let commandBuffer = compositeCommandQueue.makeCommandBuffer(),
+ let blitEncoder = commandBuffer.makeBlitCommandEncoder(),
+ let destTexture = compositeCurrentTexture else { return }
+ blitEncoder.copy(from: texture,
+ sourceSlice: 0, sourceLevel: 0,
+ sourceOrigin: MTLOrigin(x: 0, y: 0, z: 0),
+ sourceSize: MTLSizeMake(texture.width, texture.height, texture.depth),
+ to: destTexture, destinationSlice: 0, destinationLevel: 0,
+ destinationOrigin: MTLOrigin(x: 0, y: 0, z: 0))
+ blitEncoder.endEncoding()
+ commandBuffer.commit()
+ commandBuffer.waitUntilCompleted()
+
+ if compositePointer == nil {
+ compositePointer = Unmanaged.passUnretained(compositeCurrentTexture!).toOpaque()
+ }
+}
+
+extension CVPixelBuffer {
+ public func toCompositeBGRA() throws -> CVPixelBuffer? {
+ let pixelBuffer = self
+ let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
+
+ // ReplayKit typically provides BGRA format already, but handle YUV if needed
+ if pixelFormat == kCVPixelFormatType_32BGRA {
+ return pixelBuffer
+ }
+
+ guard pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange else {
+ print("toCompositeBGRA: Unsupported pixel format: \(pixelFormat). Expected BGRA or 420YpCbCr8BiPlanarFullRange.")
+ return nil
+ }
+
+ let yImage: CompositeVImage = pixelBuffer.withComposite({ CompositeVImage(pixelBuffer: $0, plane: 0) })!
+ let cbcrImage: CompositeVImage = pixelBuffer.withComposite({ CompositeVImage(pixelBuffer: $0, plane: 1) })!
+ let outPixelBuffer = CVPixelBuffer.makeComposite(width: yImage.width, height: yImage.height, format: kCVPixelFormatType_32BGRA)!
+ var argbImage = outPixelBuffer.withComposite({ CompositeVImage(pixelBuffer: $0) })!
+ try argbImage.drawComposite(yBuffer: yImage.buffer, cbcrBuffer: cbcrImage.buffer)
+ argbImage.permuteComposite(channelMap: [3, 2, 1, 0])
+ return outPixelBuffer
+ }
+}
+
+struct CompositeVImage {
+ let width: Int
+ let height: Int
+ let bytesPerRow: Int
+ var buffer: vImage_Buffer
+
+ init?(pixelBuffer: CVPixelBuffer, plane: Int) {
+ guard let rawBuffer = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, plane) else { return nil }
+ self.width = CVPixelBufferGetWidthOfPlane(pixelBuffer, plane)
+ self.height = CVPixelBufferGetHeightOfPlane(pixelBuffer, plane)
+ self.bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, plane)
+ self.buffer = vImage_Buffer(
+ data: UnsafeMutableRawPointer(mutating: rawBuffer),
+ height: vImagePixelCount(height),
+ width: vImagePixelCount(width),
+ rowBytes: bytesPerRow)
+ }
+
+ init?(pixelBuffer: CVPixelBuffer) {
+ guard let rawBuffer = CVPixelBufferGetBaseAddress(pixelBuffer) else { return nil }
+ self.width = CVPixelBufferGetWidth(pixelBuffer)
+ self.height = CVPixelBufferGetHeight(pixelBuffer)
+ self.bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
+ self.buffer = vImage_Buffer(
+ data: UnsafeMutableRawPointer(mutating: rawBuffer),
+ height: vImagePixelCount(height),
+ width: vImagePixelCount(width),
+ rowBytes: bytesPerRow)
+ }
+
+ mutating func drawComposite(yBuffer: vImage_Buffer, cbcrBuffer: vImage_Buffer) throws {
+ try buffer.drawComposite(yBuffer: yBuffer, cbcrBuffer: cbcrBuffer)
+ }
+
+ mutating func permuteComposite(channelMap: [UInt8]) {
+ buffer.permuteComposite(channelMap: channelMap)
+ }
+}
+
+extension CVPixelBuffer {
+ func withComposite(_ closure: ((_ pixelBuffer: CVPixelBuffer) -> T)) -> T {
+ CVPixelBufferLockBaseAddress(self, .readOnly)
+ let result = closure(self)
+ CVPixelBufferUnlockBaseAddress(self, .readOnly)
+ return result
+ }
+
+ static func makeComposite(width: Int, height: Int, format: OSType) -> CVPixelBuffer? {
+ var pixelBuffer: CVPixelBuffer? = nil
+ CVPixelBufferCreate(kCFAllocatorDefault,
+ width,
+ height,
+ format,
+ [String(kCVPixelBufferIOSurfacePropertiesKey): [
+ "IOSurfaceOpenGLESFBOCompatibility": true,
+ "IOSurfaceOpenGLESTextureCompatibility": true,
+ "IOSurfaceCoreAnimationCompatibility": true,
+ ]] as CFDictionary,
+ &pixelBuffer)
+ return pixelBuffer
+ }
+}
+
+extension vImage_Buffer {
+ mutating func drawComposite(yBuffer: vImage_Buffer, cbcrBuffer: vImage_Buffer) throws {
+ var yBuffer = yBuffer
+ var cbcrBuffer = cbcrBuffer
+ var conversionMatrix: vImage_YpCbCrToARGB = {
+ var pixelRange = vImage_YpCbCrPixelRange(Yp_bias: 0, CbCr_bias: 128, YpRangeMax: 255, CbCrRangeMax: 255, YpMax: 255, YpMin: 1, CbCrMax: 255, CbCrMin: 0)
+ var matrix = vImage_YpCbCrToARGB()
+ vImageConvert_YpCbCrToARGB_GenerateConversion(kvImage_YpCbCrToARGBMatrix_ITU_R_709_2, &pixelRange, &matrix, kvImage420Yp8_CbCr8, kvImageARGB8888, UInt32(kvImageNoFlags))
+ return matrix
+ }()
+ let error = vImageConvert_420Yp8_CbCr8ToARGB8888(&yBuffer, &cbcrBuffer, &self, &conversionMatrix, nil, 255, UInt32(kvImageNoFlags))
+ if error != kvImageNoError {
+ print("vImage conversion error: \(error)")
+ return
+ }
+ }
+
+ mutating func permuteComposite(channelMap: [UInt8]) {
+ vImagePermuteChannels_ARGB8888(&self, &self, channelMap, 0)
+ }
+}
diff --git a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs
index cefadef..60ddb95 100644
--- a/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs
+++ b/Packages/com.styly.enterprise-camera-access/Runtime/Scripts/EnterpriseCameraAccessManager.cs
@@ -19,6 +19,9 @@ public class EnterpriseCameraAccessManager : MonoBehaviour
[Tooltip("WebCam will be used for Editor mode or Smartphone. Default camera is used if this field is empty.")]
public string WebCamDeviceName = "";
+ [Tooltip("Enable composite view capture (passthrough + digital content) instead of physical camera only. Note: (1) Composite capture may have performance implications, (2) Physical camera only captures the real world without digital content, (3) Composite requires screen-capture entitlement and has privacy implications.")]
+ public bool UseCompositeCapture = false;
+
private WebCamTexture webCamTexture;
private Texture2D tmpTexture = null;
private string tempBase64String = null;
@@ -30,6 +33,14 @@ public class EnterpriseCameraAccessManager : MonoBehaviour
private IntPtr _texturePtr;
private int _width = 1920;
private int _height = 1080;
+
+ // Composite capture
+ private bool _hasSetCompositeTexture = false;
+ private Texture2D _compositeTexture;
+ private RenderTexture _compositeRenderTexture;
+ private IntPtr _compositeTexturePtr;
+ private int _compositeWidth = 1920;
+ private int _compositeHeight = 1080;
#endif
#if USE_PICOXR && UNITY_ANDROID && !UNITY_EDITOR
@@ -71,11 +82,24 @@ void Start()
#endif
#if UNITY_VISIONOS && !UNITY_EDITOR
- _renderTexture = new RenderTexture(_width, _height, 1, RenderTextureFormat.ARGB32);
- _renderTexture.enableRandomWrite = true;
- _renderTexture.Create();
- PreviewMaterial.mainTexture = _renderTexture;
- startCapture();
+ if (UseCompositeCapture)
+ {
+ // Initialize composite capture
+ _compositeRenderTexture = new RenderTexture(_compositeWidth, _compositeHeight, 1, RenderTextureFormat.ARGB32);
+ _compositeRenderTexture.enableRandomWrite = true;
+ _compositeRenderTexture.Create();
+ PreviewMaterial.mainTexture = _compositeRenderTexture;
+ startCompositeCapture();
+ }
+ else
+ {
+ // Initialize physical camera capture
+ _renderTexture = new RenderTexture(_width, _height, 1, RenderTextureFormat.ARGB32);
+ _renderTexture.enableRandomWrite = true;
+ _renderTexture.Create();
+ PreviewMaterial.mainTexture = _renderTexture;
+ startCapture();
+ }
return;
#endif
@@ -98,7 +122,40 @@ void OnDisable()
#endif
#if UNITY_VISIONOS && !UNITY_EDITOR
- stopCapture();
+ if (UseCompositeCapture)
+ {
+ stopCompositeCapture();
+
+ if (_compositeRenderTexture != null)
+ {
+ _compositeRenderTexture.Release();
+ Destroy(_compositeRenderTexture);
+ _compositeRenderTexture = null;
+ }
+
+ if (_compositeTexture != null)
+ {
+ Destroy(_compositeTexture);
+ _compositeTexture = null;
+ }
+ }
+ else
+ {
+ stopCapture();
+
+ if (_renderTexture != null)
+ {
+ _renderTexture.Release();
+ Destroy(_renderTexture);
+ _renderTexture = null;
+ }
+
+ if (_texture != null)
+ {
+ Destroy(_texture);
+ _texture = null;
+ }
+ }
return;
#endif
if (webCamTexture != null) { webCamTexture.Stop(); }
@@ -138,13 +195,27 @@ IEnumerator RequestCameraPermission_Android()
void Update()
{
#if UNITY_VISIONOS && !UNITY_EDITOR
- if (_hasSetTexture)
+ if (UseCompositeCapture)
{
- UpdateTexture();
+ if (_hasSetCompositeTexture)
+ {
+ UpdateCompositeTexture();
+ }
+ else
+ {
+ TryGetCompositeTexture();
+ }
}
else
{
- TryGetTexture();
+ if (_hasSetTexture)
+ {
+ UpdateTexture();
+ }
+ else
+ {
+ TryGetTexture();
+ }
}
#else
// Apply WebCamTexture to material
@@ -243,12 +314,12 @@ private void TryGetTexture()
_texture = Texture2D.CreateExternalTexture(_width, _height, TextureFormat.BGRA32, false, false, _texturePtr);
_texture.UpdateExternalTexture(_texturePtr);
-
+
// スケールとオフセットを使用して上下反転を行う
// scale.y を -1 にすることで上下反転、offset.y を 1 にすることで位置を調整
Vector2 scale = new Vector2(1, -1);
Vector2 offset = new Vector2(0, 1);
-
+
Graphics.Blit(_texture, _renderTexture, scale, offset);
PreviewMaterial.mainTexture = _renderTexture;
@@ -260,10 +331,79 @@ private void UpdateTexture()
// スケールとオフセットを使用して上下反転を行う
Vector2 scale = new Vector2(1, -1);
Vector2 offset = new Vector2(0, 1);
-
+
Graphics.Blit(_texture, _renderTexture, scale, offset);
Unity.PolySpatial.PolySpatialObjectUtils.MarkDirty(_renderTexture);
}
+
+ private void TryGetCompositeTexture()
+ {
+ IntPtr texturePtr = getCompositeTexture();
+ if (texturePtr == IntPtr.Zero) return;
+
+ // Only recreate the external texture if the underlying pointer has changed
+ if (_compositeTexture == null || texturePtr != _compositeTexturePtr)
+ {
+ _compositeTexturePtr = texturePtr;
+
+ if (_compositeTexture != null)
+ {
+ UnityEngine.Object.Destroy(_compositeTexture);
+ }
+
+ _compositeTexture = Texture2D.CreateExternalTexture(_compositeWidth, _compositeHeight, TextureFormat.BGRA32, false, false, _compositeTexturePtr);
+ }
+
+ _compositeTexture.UpdateExternalTexture(_compositeTexturePtr);
+
+ // スケールとオフセットを使用して上下反転を行う
+ Vector2 scale = new Vector2(1, -1);
+ Vector2 offset = new Vector2(0, 1);
+
+ Graphics.Blit(_compositeTexture, _compositeRenderTexture, scale, offset);
+ PreviewMaterial.mainTexture = _compositeRenderTexture;
+
+ _hasSetCompositeTexture = true;
+ }
+
+ private void UpdateCompositeTexture()
+ {
+ // Refresh the texture data from native side
+ _compositeTexture.UpdateExternalTexture(_compositeTexturePtr);
+
+ // スケールとオフセットを使用して上下反転を行う
+ Vector2 scale = new Vector2(1, -1);
+ Vector2 offset = new Vector2(0, 1);
+
+ Graphics.Blit(_compositeTexture, _compositeRenderTexture, scale, offset);
+ Unity.PolySpatial.PolySpatialObjectUtils.MarkDirty(_compositeRenderTexture);
+ }
+
+ ///
+ /// Get composite view texture (passthrough + digital content).
+ /// Returns null if composite capture is not available on the current platform,
+ /// or if the capture has not been initialized yet. Check IsCompositeCaptureAvailable()
+ /// before calling this method.
+ ///
+ /// The composite texture, or null if not available or not initialized.
+ public Texture2D GetCompositeTexture2D()
+ {
+#if UNITY_VISIONOS && !UNITY_EDITOR
+ return _compositeTexture;
+#else
+ return null;
+#endif
+ }
+
+ ///
+ /// Indicates whether composite capture is available on the current platform.
+ /// Call this before using composite capture features such as GetCompositeTexture2D().
+ ///
+ /// True if composite capture is available, false otherwise.
+ public bool IsCompositeCaptureAvailable()
+ {
+ return isCompositeAvailable();
+ }
#endif
#if UNITY_VISIONOS && !UNITY_EDITOR
@@ -277,9 +417,24 @@ private void UpdateTexture()
static extern void stopCapture();
[DllImport("__Internal")]
static extern IntPtr getTexture();
+ [DllImport("__Internal")]
+ static extern void startCompositeCapture();
+ [DllImport("__Internal")]
+ static extern void stopCompositeCapture();
+ [DllImport("__Internal")]
+ static extern IntPtr getCompositeTexture();
+ [DllImport("__Internal")]
+ static extern bool isCompositeAvailable();
#else
static void SetNativeCallbackOfCameraAccess(CallbackDelegate callback) { }
static void StartVisionProMainCameraCapture() { }
+ static void startCapture() { }
+ static void stopCapture() { }
+ static IntPtr getTexture() { return IntPtr.Zero; }
+ static void startCompositeCapture() { }
+ static void stopCompositeCapture() { }
+ static IntPtr getCompositeTexture() { return IntPtr.Zero; }
+ static bool isCompositeAvailable() { return false; }
#endif