SENSiOSCameraDelegate Class Reference

#import <SENSiOSCameraDelegate.h>

Inherits NSObject, and <AVCaptureVideoDataOutputSampleBufferDelegate>.

Instance Methods
(BOOL)	- startCamera:withWidth:andHeight:autoFocusState:videoStabilizationState:intrinsicsState:
(BOOL)	- stopCamera
(SENSCaptureProps)	- retrieveCaptureProperties
(id)	- init [implementation]
(void)	- captureOutput:didOutputSampleBuffer:fromConnection: [implementation]
(void)	- videoCameraStarted: [implementation]
(void)	- printActiveFormat [implementation]
(void)	- checkPermission [implementation]

Properties
std::function< void(unsigned char *, int, int, matrix_float3x3 *)>	updateCB
std::function< void(bool)>	permissionCB

Private Attributes
AVCaptureSession *	_captureSession
AVCaptureDevice *	_camera
AVCaptureDeviceInput *	_cameraInput
AVCaptureVideoDataOutput *	_videoOutput
BOOL	_cameraIntrinsicsDelivery

Detailed Description

Definition at line 11 of file SENSiOSCameraDelegate.h.

Method Documentation

captureOutput:didOutputSampleBuffer:fromConnection:

- (void) captureOutput: (AVCaptureOutput*)  captureOutput didOutputSampleBuffer: (CMSampleBufferRef)  sampleBuffer fromConnection: (AVCaptureConnection*)  connection

implementation

Definition at line 19 of file SENSiOSCameraDelegate.mm.

                     :(AVCaptureOutput*)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection*)connection
{
    // Check if this is the output we are expecting:
    if (captureOutput == _videoOutput)
    {
        // If it's a video frame, copy and process it
        CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
 
        CVReturn ret = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
        if (ret != kCVReturnSuccess)
            return;
 
        int            imgWidth  = (int)CVPixelBufferGetWidth(pixelBuffer);
        int            imgHeight = (int)CVPixelBufferGetHeight(pixelBuffer);
        unsigned char* data      = (unsigned char*)CVPixelBufferGetBaseAddress(pixelBuffer);
 
        if (!data)
        {
            NSLog(@"No pixel buffer data");
            return;
        }
 
        matrix_float3x3* camMatrix = nil;
        if (_cameraIntrinsicsDelivery)
        {
            CFTypeRef cameraIntrinsicData = CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_CameraIntrinsicMatrix, nil);
            if (cameraIntrinsicData != nil)
            {
                CFDataRef cfdr = (CFDataRef)(cameraIntrinsicData);
                (CFDataGetBytePtr(cfdr));
                camMatrix = (matrix_float3x3*)(CFDataGetBytePtr(cfdr));
            }
        }
 
        if (_updateCB)
        {
            _updateCB(data, imgWidth, imgHeight, camMatrix);
        }
 
        CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
    }
}

checkPermission

- (void) checkPermission

implementation

Definition at line 19 of file SENSiOSCameraDelegate.mm.

{
    NSString*             mediaType  = AVMediaTypeVideo;
    AVAuthorizationStatus authStatus = [AVCaptureDevice authorizationStatusForMediaType:mediaType];
    if (authStatus == AVAuthorizationStatusAuthorized)
    {
        if (_permissionCB)
            _permissionCB(true);
    }
    else if (authStatus == AVAuthorizationStatusDenied)
    {
        if (_permissionCB)
            _permissionCB(false);
    }
    else if (authStatus == AVAuthorizationStatusRestricted)
    {
        // restricted, normally won't happen
    }
    else if (authStatus == AVAuthorizationStatusNotDetermined)
    {
        // not determined?!
        [AVCaptureDevice requestAccessForMediaType:mediaType
                                 completionHandler:^(BOOL granted) {
                                   if (granted)
                                   {
                                       NSLog(@"Granted access to %@", mediaType);
                                       if (_permissionCB)
                                           _permissionCB(true);
                                   }
                                   else
                                   {
                                       NSLog(@"Not granted access to %@", mediaType);
                                       if (_permissionCB)
                                           _permissionCB(false);
                                   }
                                 }];
    }
}

init

- (id) init

implementation

Definition at line 19 of file SENSiOSCameraDelegate.mm.

{
    //Initialize the parent class(es) up the hierarchy and create self:
    self = [super init];
 
    //Initialize members (not necessary with ARC)
    _captureSession = nil;
    _camera         = nil;
    _cameraInput    = nil;
    _videoOutput    = nil;
 
    _cameraIntrinsicsDelivery = NO;
 
    if (self)
    {
        [self checkPermission];
    }
 
    return self;
}

printActiveFormat

- (void) printActiveFormat

implementation

Definition at line 19 of file SENSiOSCameraDelegate.mm.

{
    if (_camera == nil)
        return;
 
    AVCaptureDeviceFormat* currFormat = [_camera activeFormat];
    CMFormatDescriptionRef formatDesc = [currFormat formatDescription];
    if (formatDesc)
    {
        CMVideoDimensions dims = CMVideoFormatDescriptionGetDimensions(formatDesc);
        printf("active format: w %d h %d", dims.width, dims.height);
    }
}

retrieveCaptureProperties

- (SENSCaptureProps) retrieveCaptureProperties

Definition at line 19 of file SENSiOSCameraDelegate.mm.

{
    SENSCaptureProps characsVec;
 
    // specifying AVMediaTypeVideo will ensure we only get a list of cameras, no microphones
    NSArray* devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
 
    for (AVCaptureDevice* device in devices)
    {
        //device id
        std::string      deviceId = [[device uniqueID] UTF8String];
        SENSCameraFacing facing;
        //facing
        if (AVCaptureDevicePositionFront == [device position])
            facing = SENSCameraFacing::FRONT;
        else if (AVCaptureDevicePositionBack == [device position])
            facing = SENSCameraFacing::BACK;
        else
            facing = SENSCameraFacing::UNKNOWN;
 
        SENSCameraDeviceProps characs(deviceId, facing);
 
        NSArray<AVCaptureDeviceFormat*>* deviceFormats = [device formats];
        for (AVCaptureDeviceFormat* format in deviceFormats)
        {
            CMFormatDescriptionRef formatDesc = [format formatDescription];
            if (formatDesc)
            {
                //todo: for all formats setup a video output and check for intrinsics and
 
                CMVideoDimensions dims = CMVideoFormatDescriptionGetDimensions(formatDesc);
                int               w    = dims.width;
                int               h    = dims.height;
                //printf("dims: w %d h %d\n", w, h);
                if (!characs.contains({w, h}))
                {
                    //calculate focal length in pixel from horizontal field of view
                    float horizFovDeg    = [format videoFieldOfView];
                    float focalLengthPix = SENS::calcFocalLengthPixFromFOVDeg(horizFovDeg, w);
 
                    characs.add(w, h, focalLengthPix);
                }
            }
        }
 
        characsVec.push_back(characs);
    }
    return characsVec;
}

startCamera:withWidth:andHeight:autoFocusState:videoStabilizationState:intrinsicsState:

- (BOOL) startCamera:		(NSString*)	deviceId
withWidth:		(int)	width
andHeight:		(int)	height
autoFocusState:		(BOOL)	autoFocusEnabled
videoStabilizationState:		(BOOL)	videoStabilizationEnabled
intrinsicsState:		(BOOL)	provideIntrinsics

Definition at line 19 of file SENSiOSCameraDelegate.mm.

                   :(NSString*)deviceId
                withWidth:(int)width
                andHeight:(int)height
           autoFocusState:(BOOL)autoFocusEnabled
  videoStabilizationState:(BOOL)videoStabilizationEnabled
          intrinsicsState:(BOOL)provideIntrinsics
{
    // Make sure we initialize our camera pointer:
    _camera = nil;
 
    // specifying AVMediaTypeVideo will ensure we only get a list of cameras, no microphones
    NSArray* devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
 
    for (AVCaptureDevice* device in devices)
    {
        if ([device uniqueID] == deviceId)
        {
            _camera = device;
            break;
        }
    }
 
    if (_camera == nil)
        return NO;
 
    //We set the format directly on the device to get all available configurations (not all are available on AVCaptureSession)
    //see: https://developer.apple.com/documentation/avfoundation/avcapturedevice?language=objc
    //see: https://stackoverflow.com/questions/36689578/avfoundation-capturing-video-with-custom-resolution/40097057
    //find best corresponding format
    AVCaptureDeviceFormat* bestFormat = nil;
    AVFrameRateRange*      bestRange  = nil;
 
    for (AVCaptureDeviceFormat* format in [_camera formats])
    {
        CMFormatDescriptionRef formatDesc = [format formatDescription];
        if (formatDesc)
        {
            CMVideoDimensions dims = CMVideoFormatDescriptionGetDimensions(formatDesc);
            if (dims.width == width && dims.height == height)
            {
                //std::cout << "w " << dims.width << " h " << dims.height << std::endl;
                for (AVFrameRateRange* range in [format videoSupportedFrameRateRanges])
                {
                    //std::cout << "min fps " << range.minFrameRate << " max fps " << range.maxFrameRate << std::endl;
                    if (bestRange == nil || bestRange.maxFrameRate < range.maxFrameRate)
                    {
                        bestFormat = format;
                        bestRange  = range;
                    }
                }
            }
        }
    }
 
    // Set a frame rate for the camera:
    // We first need to lock the camera, so no one else can mess with its configuration:
    if ([_camera lockForConfiguration:nil])
    {
        // Set the device's active format.
        [_camera setActiveFormat:bestFormat];
        //[self printActiveFormat];
 
        // Set the device's min/max frame duration.
        CMTime duration = [bestRange minFrameDuration];
        [_camera setActiveVideoMinFrameDuration:duration];
        [_camera setActiveVideoMaxFrameDuration:duration];
 
        //parameterize focus mode
        if (autoFocusEnabled)
            [_camera setFocusMode:AVCaptureFocusModeContinuousAutoFocus];
        else
            [_camera setFocusModeLockedWithLensPosition:1.0 completionHandler:nil];
 
        //we set geometric distortion correction to no for now because we dont know how it influences the camera intrinsics
        if (@available(iOS 13.0, *))
        {
            if ([_camera isGeometricDistortionCorrectionSupported])
            {
                [_camera setGeometricDistortionCorrectionEnabled:NO];
            }
        }
 
        //Make sure we have a capture session
        if (nil == _captureSession)
        {
            _captureSession = [[AVCaptureSession alloc] init];
        }
 
        //disable that video device active format is overwritten
        [_captureSession setAutomaticallyConfiguresCaptureDeviceForWideColor:NO];
        //again: setting session present to AVCaptureSessionPresetInputPriority enabled that the previously set format on the video device is not overwritten
        if ([_captureSession canSetSessionPreset:AVCaptureSessionPresetInputPriority])
            [_captureSession setSessionPreset:AVCaptureSessionPresetInputPriority];
 
        // Plug camera and capture sesiossion together:
        {
            // Request a camera input from the camera
            NSError* error = nil;
            _cameraInput   = [AVCaptureDeviceInput deviceInputWithDevice:_camera
                                                                 error:&error];
            // Check if we've got any errors
            if (nil != error)
                return NO;
 
            // We've got the input from the camera, now attach it to the capture session:
            if ([_captureSession canAddInput:_cameraInput])
                [_captureSession addInput:_cameraInput];
            else
                return NO;
        }
 
        // Add the video output:
        {
            // Create the video data output
            _videoOutput = [[AVCaptureVideoDataOutput alloc] init];
 
            // Create a queue for capturing video frames
            dispatch_queue_t captureQueue = dispatch_queue_create("captureQueue", DISPATCH_QUEUE_SERIAL);
 
            // Use the AVCaptureVideoDataOutputSampleBufferDelegate capabilities of CameraDelegate
            [_videoOutput setSampleBufferDelegate:self queue:captureQueue];
 
            // Set up the video output:
            // Do we care about missing frames?
            [_videoOutput setAlwaysDiscardsLateVideoFrames:YES];
 
            // We want the frames in some BGR format
            NSNumber* framePixelFormat = [NSNumber numberWithInt:kCVPixelFormatType_32BGRA];
            _videoOutput.videoSettings = [NSDictionary dictionaryWithObject:framePixelFormat
                                                                     forKey:(id)kCVPixelBufferPixelFormatTypeKey];
 
            // Add the video data output to the capture session
            if ([_captureSession canAddOutput:_videoOutput])
                [_captureSession addOutput:_videoOutput];
            else
                return NO;
 
            //parameterize video stabilization: if available and should be disabled we disable software video stabilization
            AVCaptureConnection* capCon = [_videoOutput connectionWithMediaType:AVMediaTypeVideo];
            if (capCon != nil)
            {
                //we have to turn video stabilization mode off if we want intrinsics!!
                if ([capCon isVideoStabilizationSupported])
                {
                    if (videoStabilizationEnabled)
                        [capCon setPreferredVideoStabilizationMode:AVCaptureVideoStabilizationModeAuto];
                    else
                        [capCon setPreferredVideoStabilizationMode:AVCaptureVideoStabilizationModeOff];
                }
 
                if (provideIntrinsics && [capCon isCameraIntrinsicMatrixDeliverySupported])
                {
                    [capCon setCameraIntrinsicMatrixDeliveryEnabled:YES];
                    _cameraIntrinsicsDelivery = YES;
                }
            }
        }
 
        // Set up a callback, so we are notified when the camera actually starts
        [[NSNotificationCenter defaultCenter] addObserver:self
                                                 selector:@selector(videoCameraStarted:)
                                                     name:AVCaptureSessionDidStartRunningNotification
                                                   object:_captureSession];
 
        // Start the captureing
        [_captureSession startRunning];
        //[self printActiveFormat];
 
        [_camera unlockForConfiguration];
    }
 
    // Note: Returning true from this function only means that setting up went OK.
    // It doesn't mean that the camera has started yet.
    // We get notified about the camera having started in the videoCameraStarted() callback.
    return YES;
}

stopCamera

- (BOOL) stopCamera

Definition at line 19 of file SENSiOSCameraDelegate.mm.

{
    if (nil != _captureSession)
    {
        [_captureSession stopRunning];
        _captureSession = nil;
        _camera         = nil;
        _cameraInput    = nil;
        _videoOutput    = nil;
    }
    return true;
}

videoCameraStarted:

- (void) videoCameraStarted: (NSNotification*)  note

implementation

Definition at line 19 of file SENSiOSCameraDelegate.mm.

                          :(NSNotification*)note
{
    //Note: not used at the moment
    // This callback has done its job, now disconnect it
    [[NSNotificationCenter defaultCenter] removeObserver:self
                                                    name:AVCaptureSessionDidStartRunningNotification
                                                  object:_captureSession];
}

Member Data Documentation

_camera

- (AVCaptureDevice*) _camera

private

Definition at line 15 of file SENSiOSCameraDelegate.mm.

_cameraInput

- (AVCaptureDeviceInput*) _cameraInput

private

Definition at line 16 of file SENSiOSCameraDelegate.mm.

_cameraIntrinsicsDelivery

- (BOOL) _cameraIntrinsicsDelivery

private

Definition at line 19 of file SENSiOSCameraDelegate.mm.

_captureSession

- (AVCaptureSession*) _captureSession

private

Definition at line 14 of file SENSiOSCameraDelegate.mm.

_videoOutput

- (AVCaptureVideoDataOutput*) _videoOutput

private

Definition at line 17 of file SENSiOSCameraDelegate.mm.

Property Documentation

permissionCB

- (function<void(bool)>) SENSiOSCameraDelegate:

readwritenonatomicassign

Definition at line 24 of file SENSiOSCameraDelegate.h.

updateCB

- (function<void(unsigned char*, int, int, matrix_float3x3*)>) SENSiOSCameraDelegate:

readwritenonatomicassign

Definition at line 23 of file SENSiOSCameraDelegate.h.

---

The documentation for this class was generated from the following files:

• SENSiOSCameraDelegate.h
• SENSiOSCameraDelegate.mm