builtin-programs/camera/macos.folk

# camera/macos.folk --
#
#     Hardware interface with webcams on macOS (AVFoundation).

if {$::tcl_platform(os) ne "darwin"} { return }

set enumCc [C]
$enumCc cflags -x objective-c
$enumCc cflags -Wno-deprecated-declarations
$enumCc endcflags -framework AVFoundation -framework Foundation -framework CoreMedia

$enumCc code {
    #import <AVFoundation/AVFoundation.h>
    #import <CoreMedia/CoreMedia.h>
    #import <Foundation/Foundation.h>
}

$enumCc proc listDevices {} Jim_Obj* {
    Jim_Obj* resultList = Jim_NewListObj(interp, NULL, 0);
    @autoreleasepool {
        NSArray* devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
        for (AVCaptureDevice* device in devices) {
            // Group formats by (width, height), collecting unique max frame rates.
            // dimOrder preserves first-seen ordering (AVFoundation lists largest first).
            NSMutableDictionary* fpsSetByDim = [NSMutableDictionary dictionary];
            NSMutableArray* dimOrder = [NSMutableArray array];

            for (AVCaptureDeviceFormat* fmt in device.formats) {
                CMVideoDimensions dims =
                    CMVideoFormatDescriptionGetDimensions(fmt.formatDescription);
                NSString* key =
                    [NSString stringWithFormat:@"%d %d", dims.width, dims.height];
                NSMutableSet* fpsSet = fpsSetByDim[key];
                if (!fpsSet) {
                    fpsSet = [NSMutableSet set];
                    fpsSetByDim[key] = fpsSet;
                    [dimOrder addObject:key];
                }
                for (AVFrameRateRange* range in fmt.videoSupportedFrameRateRanges) {
                    [fpsSet addObject:@(range.maxFrameRate)];
                }
            }

            // Build resolutions list (fps sorted descending).
            Jim_Obj* resList = Jim_NewListObj(interp, NULL, 0);
            for (NSString* key in dimOrder) {
                int w, h;
                sscanf([key UTF8String], "%d %d", &w, &h);
                NSArray* sortedFps =
                    [[[fpsSetByDim[key] allObjects]
                        sortedArrayUsingSelector:@selector(compare:)] reversedArray];
                Jim_Obj* fpsList = Jim_NewListObj(interp, NULL, 0);
                for (NSNumber* fps in sortedFps) {
                    Jim_ListAppendElement(interp, fpsList,
                        Jim_NewDoubleObj(interp, [fps doubleValue]));
                }
                int fpsLen;
                const char* fpsStr = Jim_GetString(fpsList, &fpsLen);
                Jim_Obj* resDict = Jim_ObjPrintf(
                    "width %d height %d framerates {%s}", w, h, fpsStr);
                Jim_ListAppendElement(interp, resList, resDict);
            }

            int resLen;
            const char* resStr = Jim_GetString(resList, &resLen);
            Jim_Obj* devDict = Jim_ObjPrintf(
                "uniqueID {%s} name {%s} resolutions {%s}",
                [device.uniqueID UTF8String],
                [device.localizedName UTF8String],
                resStr);
            Jim_ListAppendElement(interp, resultList, devDict);
        }
    }
    return resultList;
}

$enumCc cflags -Wall
set enumLib [$enumCc compile]

foreach device [$enumLib listDevices] {
    set uniqueID [dict get $device uniqueID]
    set name     [dict get $device name]
    set resolutions [dict get $device resolutions]

    # We capture as BGRA and compress to JPEG, so advertise MJPG so the
    # setup UI shows resolutions and frame-rate choices.
    set formats [list [list fourcc MJPG description {Motion-JPEG} \
                            resolutions $resolutions]]

    Claim $::thisNode has camera "avf:$uniqueID" with card $name formats $formats
}

When the image library is /imageLib/ &\
     the jpeg library is /jpegLib/ {

set camc [C]
$camc extend $imageLib
$camc cflags -x objective-c
$camc cflags -Wno-deprecated-declarations
$camc endcflags -framework AVFoundation -framework Foundation -framework CoreMedia -framework CoreVideo -lturbojpeg

$camc include <stdlib.h>
$camc include <string.h>
$camc include <pthread.h>
$camc include <stdint.h>
$camc include <stdbool.h>

$camc struct Jpeg {
    uint8_t* start;
    size_t length;
}

$camc code {
    #undef EXTERN
    #include <turbojpeg.h>

    #import <AVFoundation/AVFoundation.h>
    #import <CoreMedia/CoreMedia.h>
    #import <CoreVideo/CoreVideo.h>
    #import <Foundation/Foundation.h>

    typedef struct Camera Camera;
    struct Camera {
        AVCaptureSession* session;
        id delegate;
        uint32_t requestedWidth;
        uint32_t requestedHeight;
        pthread_mutex_t mutex;
        pthread_cond_t cond;
        uint8_t* frameData;
        uint32_t frameWidth;
        uint32_t frameHeight;
        uint32_t frameBytesPerRow;
        bool hasFrame;
        bool closed;
    };

    @interface FolkCameraDelegate : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate>
    @property (nonatomic, assign) Camera* cam;
    @end

    @implementation FolkCameraDelegate
    - (void)captureOutput:(AVCaptureOutput *)output
          didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
                 fromConnection:(AVCaptureConnection *)connection {
        Camera* cam = self.cam;
        CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
        CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);

        size_t width = CVPixelBufferGetWidth(imageBuffer);
        size_t height = CVPixelBufferGetHeight(imageBuffer);
        size_t bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
        uint8_t* data = (uint8_t*)CVPixelBufferGetBaseAddress(imageBuffer);
        size_t len = bytesPerRow * height;

        pthread_mutex_lock(&cam->mutex);
        if (!cam->closed) {
            if (cam->frameData) free(cam->frameData);
            cam->frameData = malloc(len);
            memcpy(cam->frameData, data, len);
            cam->frameWidth = (uint32_t)width;
            cam->frameHeight = (uint32_t)height;
            cam->frameBytesPerRow = (uint32_t)bytesPerRow;
            cam->hasFrame = true;
            pthread_cond_signal(&cam->cond);
        }
        pthread_mutex_unlock(&cam->mutex);

        CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
    }
    @end

    static Jpeg bgra_to_jpeg(uint8_t* data, uint32_t width, uint32_t height, uint32_t bytesPerRow) {
        tjhandle handle = tjInitCompress();
        if (!handle) {
            FOLK_ERROR("camera/macos: Failed to init JPEG compressor");
        }
        unsigned char* jpegBuf = NULL;
        unsigned long jpegSize = 0;
        int ret = tjCompress2(handle, data, width, bytesPerRow, height,
                              TJPF_BGRA, &jpegBuf, &jpegSize,
                              TJSAMP_420, 85, TJFLAG_FASTDCT);
        tjDestroy(handle);
        if (ret != 0) {
            FOLK_ERROR("camera/macos: JPEG compression failed: %s", tjGetErrorStr());
        }
        uint8_t* outBuf = malloc(jpegSize);
        memcpy(outBuf, jpegBuf, jpegSize);
        tjFree(jpegBuf);
        return (Jpeg){ .start = outBuf, .length = jpegSize };
    }
}

$camc proc cameraOpen {char* deviceID int width int height} Camera* {
    Camera* cam = calloc(1, sizeof(Camera));
    cam->requestedWidth = width;
    cam->requestedHeight = height;
    pthread_mutex_init(&cam->mutex, NULL);
    pthread_cond_init(&cam->cond, NULL);

    @autoreleasepool {
        // Check and request camera permission.
        AVAuthorizationStatus authStatus =
            [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo];
        if (authStatus == AVAuthorizationStatusNotDetermined) {
            dispatch_semaphore_t sem = dispatch_semaphore_create(0);
            [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo
                                    completionHandler:^(BOOL granted) {
                dispatch_semaphore_signal(sem);
            }];
            dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER);
            dispatch_release(sem);
            authStatus = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo];
        }
        if (authStatus != AVAuthorizationStatusAuthorized) {
            pthread_mutex_destroy(&cam->mutex);
            pthread_cond_destroy(&cam->cond);
            free(cam);
            FOLK_ERROR("camera/macos: Camera not authorized. Grant access in System Settings > Privacy > Camera.");
        }

        // Find device by uniqueID or localizedName.
        NSString* deviceIDStr = [NSString stringWithUTF8String:deviceID];
        AVCaptureDevice* device = nil;
        NSArray* devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
        for (AVCaptureDevice* d in devices) {
            if ([d.uniqueID isEqualToString:deviceIDStr] ||
                [d.localizedName isEqualToString:deviceIDStr]) {
                device = d;
                break;
            }
        }
        if (device == nil) {
            device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
        }
        if (device == nil) {
            pthread_mutex_destroy(&cam->mutex);
            pthread_cond_destroy(&cam->cond);
            free(cam);
            FOLK_ERROR("camera/macos: No camera found for '%s'", deviceID);
        }

        NSError* error = nil;
        AVCaptureDeviceInput* input =
            [AVCaptureDeviceInput deviceInputWithDevice:device error:&error];
        if (input == nil) {
            pthread_mutex_destroy(&cam->mutex);
            pthread_cond_destroy(&cam->cond);
            free(cam);
            FOLK_ERROR("camera/macos: Failed to create input: %s",
                       [[error localizedDescription] UTF8String]);
        }

        cam->session = [[AVCaptureSession alloc] init];
        if (width <= 640 && height <= 480) {
            cam->session.sessionPreset = AVCaptureSessionPreset640x480;
        } else if (width <= 1280 && height <= 720) {
            cam->session.sessionPreset = AVCaptureSessionPreset1280x720;
        } else {
            cam->session.sessionPreset = AVCaptureSessionPreset1920x1080;
        }

        if (![cam->session canAddInput:input]) {
            [cam->session release];
            pthread_mutex_destroy(&cam->mutex);
            pthread_cond_destroy(&cam->cond);
            free(cam);
            FOLK_ERROR("camera/macos: Cannot add input to session");
        }
        [cam->session addInput:input];

        AVCaptureVideoDataOutput* output = [[AVCaptureVideoDataOutput alloc] init];
        output.videoSettings = @{
            (NSString*)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)
        };
        output.alwaysDiscardsLateVideoFrames = YES;

        FolkCameraDelegate* delegate = [[FolkCameraDelegate alloc] init];
        delegate.cam = cam;
        cam->delegate = delegate;

        dispatch_queue_t queue =
            dispatch_queue_create("folk.camera.macos", DISPATCH_QUEUE_SERIAL);
        [output setSampleBufferDelegate:delegate queue:queue];
        dispatch_release(queue);

        if (![cam->session canAddOutput:output]) {
            [cam->session release];
            [delegate release];
            [output release];
            pthread_mutex_destroy(&cam->mutex);
            pthread_cond_destroy(&cam->cond);
            free(cam);
            FOLK_ERROR("camera/macos: Cannot add output to session");
        }
        [cam->session addOutput:output];
        [output release];
    }

    printf("camera/macos: Opened '%s'\n", deviceID);
    return cam;
}

$camc proc cameraStart {Camera* cam} void {
    [cam->session startRunning];
}

$camc proc cameraClose {Camera* cam} void {
    pthread_mutex_lock(&cam->mutex);
    cam->closed = true;
    pthread_cond_broadcast(&cam->cond);
    pthread_mutex_unlock(&cam->mutex);

    [cam->session stopRunning];
    [cam->session release];
    cam->session = nil;
    [cam->delegate release];
    cam->delegate = nil;

    pthread_mutex_lock(&cam->mutex);
    if (cam->frameData) {
        free(cam->frameData);
        cam->frameData = NULL;
    }
    pthread_mutex_unlock(&cam->mutex);

    pthread_mutex_destroy(&cam->mutex);
    pthread_cond_destroy(&cam->cond);
    free(cam);
}

$camc proc cameraFrameJpeg {Camera* cam} Jpeg {
    pthread_mutex_lock(&cam->mutex);
    while (!cam->hasFrame && !cam->closed) {
        pthread_cond_wait(&cam->cond, &cam->mutex);
    }
    if (cam->closed) {
        pthread_mutex_unlock(&cam->mutex);
        FOLK_ERROR("camera/macos: Camera closed");
    }

    uint8_t* frameData = cam->frameData;
    uint32_t frameWidth = cam->frameWidth;
    uint32_t frameHeight = cam->frameHeight;
    uint32_t frameBytesPerRow = cam->frameBytesPerRow;
    cam->frameData = NULL;
    cam->hasFrame = false;
    pthread_mutex_unlock(&cam->mutex);

    Jpeg jpeg = bgra_to_jpeg(frameData, frameWidth, frameHeight, frameBytesPerRow);
    free(frameData);
    return jpeg;
}

$camc proc jpegFree {Jpeg jpeg} void {
    free(jpeg.start);
}

$camc cflags -Wall
set camLib [$camc compile]

When /someone/ wishes $::thisNode uses camera /camera/ with /...options/ {
    if {![string match "avf:*" $camera]} { return }

    set deviceID [string range $camera 4 end]
    set width [dict get $options width]
    set height [dict get $options height]

    if {[dict exists $options crops]} {
        set crops [dict get $options crops]
    }

    set camObj [$camLib cameraOpen $deviceID $width $height]
    puts "camera/macos: Loaded $camera -> $camObj"
    $camLib cameraStart $camObj
    # skip 5 frames for camera warmup
    for {set i 0} {$i < 5} {incr i} {
        set f [$camLib cameraFrameJpeg $camObj]
        $camLib jpegFree $f
    }

    try -signal {
        if {[info exists crops]} {
            for {set i 0} {$i < [llength $crops]} {incr i} {
                set cropPath [list $camera $i]
                set cropWidth [dict get [lindex $crops $i] width]
                set cropHeight [dict get [lindex $crops $i] height]

                Claim camera $cropPath has width $cropWidth height $cropHeight

                When camera $cropPath has jpeg frame /jpeg/ at timestamp /ts/ {
                    set grayImage [$jpegLib jpegDecompressGray $jpeg \
                                       $cropWidth $cropHeight \
                                       [expr {int($ts * 1000)}]]
                    set rgbImage [$jpegLib jpegDecompressRGB $jpeg \
                                      $cropWidth $cropHeight \
                                      [expr {int($ts * 1000) + 1}]]
                    Hold! -key [list camera $cropPath gray-frame] \
                        Claim camera $cropPath has gray frame $grayImage at timestamp $ts \
                        -destructor [list $imageLib imageFree $grayImage]
                    Hold! -key [list camera $cropPath frame] \
                        Claim camera $cropPath has frame $rgbImage at timestamp $ts \
                        -destructor [list $imageLib imageFree $rgbImage]
                }
            }
        } else {
            Claim camera $camera has width $width height $height

            When camera $camera has jpeg frame /jpeg/ at timestamp /ts/ {
                set grayImage [$jpegLib jpegDecompressGray $jpeg \
                                   $width $height \
                                   [expr {int($ts * 1000)}]]
                set rgbImage [$jpegLib jpegDecompressRGB $jpeg \
                                  $width $height \
                                  [expr {int($ts * 1000) + 1}]]
                Hold! -key [list camera $camera gray-frame] \
                    Claim camera $camera has gray frame $grayImage at timestamp $ts \
                    -destructor [list $imageLib imageFree $grayImage]
                Hold! -key [list camera $camera frame] \
                    Claim camera $camera has frame $rgbImage at timestamp $ts \
                    -destructor [list $imageLib imageFree $rgbImage]
            }
        }

        while true {
            tracy zoneBegin

            set ms [clock milliseconds]
            set jpeg [$camLib cameraFrameJpeg $camObj]
            set timestamp [expr {$ms / 1000.0}]
            tracy zoneName "camera/macos: $timestamp"

            if {[info exists crops]} {
                for {set i 0} {$i < [llength $crops]} {incr i} {
                    set crop [lindex $crops $i]
                    set cropPath [list $camera $i]
                    set croppedJpeg [$jpegLib jpegSubimage $jpeg \
                                         [dict get $crop x] \
                                         [dict get $crop y] \
                                         [dict get $crop width] \
                                         [dict get $crop height]]
                    Hold! -key [list camera $cropPath jpeg] \
                        Claim camera $cropPath has jpeg frame $croppedJpeg at timestamp $timestamp \
                        -destructor [list $camLib jpegFree $croppedJpeg]
                }
            }
            Hold! -key [list camera $camera jpeg] \
                Claim camera $camera has jpeg frame $jpeg at timestamp $timestamp \
                -destructor [list $camLib jpegFree $jpeg]

            tracy zoneEnd
        }
    } on error e {
        puts stderr "camera/macos: Error $e"
        Hold! -key [list camera-error $camera] \
            Claim camera $camera failed with error $e
    } on signal sig {
        puts stderr "camera/macos: Signal $sig"
    } finally {
        puts "camera/macos: Close $camObj"
        $camLib cameraClose $camObj
    }
}

}