How to scan a barcode with all available back cameras of an iphone at the same time?

Question

What I want: I want to scan for barcodes with all available back cameras of an iPhone at the same time. So the user don't have to care about the distance of the barcode so much.

What I tried:

1. I created an AVCaptureSession for each camera, and used one camera for the videoPreviewLayer so that the user can see a live video of one camera. The problem here is that if I start more than one AVCaptureSession, the live video will freeze.

   func setupCaptureSessions() {
       let devices = [ AVCaptureDevice.DiscoverySession(deviceTypes: codeScanningDevicesTypes, mediaType: .video, position: .back).devices.first!,
                       AVCaptureDevice.DiscoverySession(deviceTypes: codeScanningDevicesTypes, mediaType: .video, position: .back).devices.last! ]
   
       for device in devices {
           let captureSession = AVCaptureSession()
           captureSession.beginConfiguration()
   
           guard let deviceInput = try? AVCaptureDeviceInput(device: device) else {
               print("Failed to create device input for camera")
               return
           }
   
           guard captureSession.canAddInput(deviceInput) else {
               print("Failed to add device input to capture session")
               return
           }
   
           captureSession.addInput(deviceInput)
   
           let metadataOutput = AVCaptureMetadataOutput()
           if captureSession.canAddOutput(metadataOutput) {
               captureSession.addOutput(metadataOutput)
   
               metadataOutput.setMetadataObjectsDelegate(self, queue: DispatchQueue.main)
               metadataOutput.metadataObjectTypes = metadataOutput.availableMetadataObjectTypes
           }
   
           captureSession.commitConfiguration()
           captureSessions.append(captureSession)
       }
   }
   
   func startScanning() {
   
   
       // Video Konfiguration
       let videoOutput = AVCaptureVideoDataOutput()
       videoOutput.alwaysDiscardsLateVideoFrames = true
   
       guard let captureSession = captureSessions.first else { return }
   
       if captureSession.canAddOutput(videoOutput) {
           videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "video_ouput_queue"))
           captureSession.addOutput(videoOutput)
       }
   
   
       videoPreviewLayer.session = captureSession
       videoPreviewLayer.videoGravity = .resizeAspectFill
       videoPreviewLayer.connection?.videoOrientation = videoOrientationFromCurrentDeviceOrientation()
   
       for captureSession in captureSessions {
           captureSession.startRunning()
       }
   }
   

2. Another try was to use an AVCaptureMultiCamSession and just to create a bunch of AVCaptureDeviceInput and at them to the captureSession. The problem here is that just one camera was really scanning for barcodes:

   let codeScanningDevicesTypes: [AVCaptureDevice.DeviceType] = [
       .builtInUltraWideCamera,
       .builtInDualWideCamera,
       .builtInWideAngleCamera,
       .builtInTripleCamera,
       .builtInDualCamera,
       .builtInTelephotoCamera,
   ]
   
   private let allDetactableCodeTypes = [
       AVMetadataObject.ObjectType.code39,
       .code39Mod43,
       .code93,
       .code128,
       .ean8,
       .ean13,
       .interleaved2of5,
       .itf14,
       .upce,
       .aztec,
       .dataMatrix,
       .pdf417,
       .qr
   ]
   
   private func setupCaptureSession() {
   
       defer {
           captureDevice?.unlockForConfiguration()
           captureSession.commitConfiguration()
       }
   
       guard let device = captureDevice else {
           return
       }
   
       let backCameras:[AVCaptureDevice] = AVCaptureDevice.DiscoverySession(deviceTypes: codeScanningDevicesTypes, mediaType: .video, position: .back).devices
   
       captureSession.beginConfiguration()
   
       // Inputs:
   
       backCameras.forEach { device in
   
           guard let deviceInput = try? AVCaptureDeviceInput(device: device) else {
               return
           }
   
           if captureSession.canAddInput(deviceInput) {
               captureSession.addInput(deviceInput)
           }
   
           do {
               try configureDevice(device: device)
           } catch {
               let error = ImageScannerControllerError.inputDevice
               delegate?.captureSessionManager(self, didFailWithError: error)
               return
           }
       }
   
   
       // Outputs:
       let metadataOutput = AVCaptureMetadataOutput()
   
       if captureSession.canAddOutput(metadataOutput) {
           captureSession.addOutput(metadataOutput)
           metadataOutput.setMetadataObjectsDelegate(self, queue: DispatchQueue.main)
           metadataOutput.metadataObjectTypes = allDetactableCodeTypes
       }
   
       let videoOutput = AVCaptureVideoDataOutput()
       videoOutput.alwaysDiscardsLateVideoFrames = true
   
       if captureSession.canAddOutput(videoOutput) {
           videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "video_ouput_queue"))
           captureSession.addOutput(videoOutput)
       }
   
       videoPreviewLayer.session = captureSession
       videoPreviewLayer.videoGravity = .resizeAspectFill
       videoPreviewLayer.connection?.videoOrientation = videoOrientationFromCurrentDeviceOrientation()
   }
   

So the question now, how is it possible to scan for barcodes with all available cameras at the same time?

Answer 1

I could find a solution for myself, maybe it can help someone with the same Problem:

    private let videoPreviewLayerView: UIView
    private let videoPreviewLayer: AVCaptureVideoPreviewLayer
    
  private lazy var dualVideoSession: AVCaptureMultiCamSession = {
      AVCaptureMultiCamSession()
    }()
    let dualVideoSessionOutputQueue = DispatchQueue(label: "dual video session data output queue")

     private func configurCodeScanningDevice() {
    
        dualVideoSession.beginConfiguration()
        defer {
          dualVideoSession.commitConfiguration()
            dualVideoSession.startRunning()
        }
        
        videoPreviewLayer.setSessionWithNoConnection(dualVideoSession)
        videoPreviewLayer.videoGravity = .resizeAspectFill
        
        let ultraWideCamera = AVCaptureDevice.default(.builtInUltraWideCamera, for: .video, position: .back)
        let wideCamera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back)
        let cameras = [wideCamera, ultraWideCamera].compactMap{ $0 }
        
        guard !cameras.isEmpty else { return }
        
        for (index, camera) in cameras.enumerated() {
          
          guard let deviceInput = createInputAndAddToSession(device: camera),
                let videoInputPort = createVideoPort(deviceInput: deviceInput, device: camera),
                let videoDataOuput = createVideoDataOutputAndAddToSession(),
                configureDevice(device: camera) == true
          else {
            continue
          }
          
          _ = createConnection(inputPort: videoInputPort, videoDataOutput: videoDataOuput)
          
          if index == 0 {
            createLayerConnection(videoInputPort: videoInputPort)
          }
          createMetadataOutput()
        }
        
        // For Debugging:
    //    printAllSessionConnections()
      }
      
      private func printAllSessionConnections() {
            dualVideoSession.connections.forEach { connection in
              print(connection)
              print("\n")
            }
      }
      
      private func createMetadataOutput() {
        // NOTE: The order is important: addOutput must be called before setMetadataObjectTypes
        captureMetadataOutput.availableMetadataObjectTypes
        let captureMetadataOutput = AVCaptureMetadataOutput()
        dualVideoSession.addOutput(captureMetadataOutput)
        captureMetadataOutput.setMetadataObjectsDelegate(self, queue: DispatchQueue.main)
        captureMetadataOutput.metadataObjectTypes = codeDetectionConfig.detectableCodeTypes.isEmpty ? allDetactableCodeTypes : codeDetectionConfig.detectableCodeTypes
      }
      
      private func createLayerConnection(videoInputPort: AVCaptureInput.Port) {
    
        let frontLayerConnection = AVCaptureConnection(inputPort: videoInputPort, videoPreviewLayer: videoPreviewLayer)
        guard dualVideoSession.canAddConnection(frontLayerConnection) else {
            print("no connection to front layer")
            return
        }
        dualVideoSession.addConnection(frontLayerConnection)
        frontLayerConnection.automaticallyAdjustsVideoMirroring = false
        frontLayerConnection.isVideoMirrored = false
        frontLayerConnection.videoOrientation = videoOrientationFromCurrentDeviceOrientation()
      }
      
      private func createInputAndAddToSession(device: AVCaptureDevice) -> AVCaptureDeviceInput? {
        guard let deviceInput = try? AVCaptureDeviceInput(device: device),
                dualVideoSession.canAddInput(deviceInput) else {
            print("no camera input")
            return nil
        }
        dualVideoSession.addInputWithNoConnections(deviceInput)
        return deviceInput
      }
      
      private func createVideoPort(deviceInput: AVCaptureDeviceInput, device: AVCaptureDevice) -> AVCaptureInput.Port? {
        // search front video port for dual video session
        guard let videoPort = deviceInput.ports(for: .video, sourceDeviceType: device.deviceType, sourceDevicePosition: device.position).first else {
            print("no front camera device input's video port")
            return nil
        }
        
        return videoPort
      }
      
      private func createVideoDataOutputAndAddToSession() -> AVCaptureVideoDataOutput? {
        let videoDataOutput = AVCaptureVideoDataOutput()
        guard dualVideoSession.canAddOutput(videoDataOutput) else {
            print("no back camera output")
            return nil
        }
        dualVideoSession.addOutputWithNoConnections(videoDataOutput)
        videoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)]
        videoDataOutput.setSampleBufferDelegate(self, queue: dualVideoSessionOutputQueue)
        
        return videoDataOutput
      }
      
      private func createConnection(inputPort: AVCaptureInput.Port, videoDataOutput: AVCaptureOutput) -> Bool {
        // connect front output to dual video session
        let frontOutputConnection = AVCaptureConnection(inputPorts: [inputPort], output: videoDataOutput)
        guard dualVideoSession.canAddConnection(frontOutputConnection) else {
            print("no connection to the front video output")
            return false
        }
        dualVideoSession.addConnection(frontOutputConnection)
        frontOutputConnection.videoOrientation = .portrait
        frontOutputConnection.automaticallyAdjustsVideoMirroring = false
        frontOutputConnection.isVideoMirrored = true
        
        return true
      }