Connecting a Camera to the jPC

The jPC is equipped with a CSI port for solutions that may require computer vision. The jPC administration program demonstrates one such use case with a QR code reader.

The jPC's CSI port is configured by default for the imx708 image sensor and tested with the Raspberry Pi Camera 3 module. Other camera sensors are possible, but have not yet been tested.

CAREFULLY connect the camera module to the CSI port.

Use GStreamer to obtain a camera feed for your applications.

The following GStreamer command will show a real-time camera feed on the local display.

gst-launch-1.0 libcamerasrc ! video/x-raw,width=800,height=480,framerate=30/1,format=NV12 \
! videoconvert ! queue ! fpsdisplaysink video-sink=waylandsink

The following code uses the ZBar library to scan camera frames produced by GStreamer for QR codes. The GUI is created using GirCore.

#:package GirCore.GLib-2.0@0.6.3
#:package GirCore.GdkPixbuf-2.0@0.6.3
#:package GirCore.Gtk-4.0@0.6.3
 
using GdkPixbuf;
using Gio;
using Gtk;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading.Channels;
using Task = System.Threading.Tasks.Task;
 
public class Program
{
    const int Width = 480;
    const int Height = 320;
    const int FrameSize = Width * Height * 3;
 
    static IntPtr _scanner;
    static Channel<byte[]> _previewChannel = Channel.CreateUnbounded<byte[]>();
    static Channel<byte[]> _scanChannel = Channel.CreateUnbounded<byte[]>();
 
    public static int Main(string[] args)
    {
        var app = Gtk.Application.New("org.example.QrDemo", ApplicationFlags.FlagsNone);
        app.OnActivate += (gioApp, _) => CreateGUI((Gtk.Application)gioApp);
 
        var status = app.RunWithSynchronizationContext(args);
        return status;
    }
 
    static void CreateGUI(Gtk.Application app)
    {
        var window = ApplicationWindow.New(app);
        window.Title = "GTK + ZBar QR Demo";
        window.SetDefaultSize(Width, Height + 80);
        window.SetResizable(false);
 
        var vbox = Box.New(Orientation.Vertical, 4);
 
        var picture = Picture.New();
        picture.SetSizeRequest(Width, Height);
        picture.SetHalign(Align.Center);
 
        var label = Label.New("Waiting for QR...");
 
        vbox.Append(picture);
        vbox.Append(label);
 
        window.Child = vbox;
 
        window.Present();
 
        _scanner = zbar_image_scanner_create();
 
        StreamFramesAndScan(picture, label);
    }
 
    private static void StreamFramesAndScan(Picture picture, Label label)
    {
        var psi = new ProcessStartInfo
        {
            FileName = "gst-launch-1.0",
            Arguments =
                "libcamerasrc ! " +
                $"video/x-raw,width={Width},height={Height},framerate=30/1,format=NV12 ! " +
                "videoconvert ! " +
                $"video/x-raw,format=RGB ! " +
                "multipartmux boundary=frame ! " +
                "fdsink fd=1",
            UseShellExecute = false,
            RedirectStandardOutput = true,
            RedirectStandardError = false,
            CreateNoWindow = true
        };
 
        if (Process.Start(psi) is not Process process)
        {
            GLib.Functions.IdleAdd(0, () => { label.SetText("Failed to start GStreamer"); return false; });
            return;
        }
 
        _previewChannel = Channel.CreateUnbounded<byte[]>();
        _scanChannel = Channel.CreateUnbounded<byte[]>();
 
        var stream = process.StandardOutput.BaseStream;
        var cts = new CancellationTokenSource();
 
        ParseFrames(stream, process, label, cts);
        GeneratePreview(picture, cts);
        ScanForQRCode(label, cts);
    }
 
    static void ParseFrames(Stream stream, Process process, Label label, CancellationTokenSource cts)
    {
        Task.Run(async () =>
        {
            try
            {
                var previewWriter = _previewChannel.Writer;
                var scanWriter = _scanChannel.Writer;
 
                while (true)
                {
                    // Skip to frame boundary "\r\n\r\n"
                    while (stream.ReadByte() != '\r') { }
                    if (stream.ReadByte() != '\n'
                        || stream.ReadByte() != '\r'
                        || stream.ReadByte() != '\n')
                    {
                        continue;
                    }
 
                    var frame = new byte[FrameSize];
                    stream.ReadExactly(frame);
 
                    if (frame.Length == FrameSize)
                    {
                        await previewWriter.WriteAsync(frame, cts.Token);
                        await scanWriter.WriteAsync(frame, cts.Token);
                    }
                }
            }
            finally
            {
                cts.Cancel();
                _previewChannel.Writer.TryComplete();
                _scanChannel.Writer.TryComplete();
            }
        });
    }
 
    static void GeneratePreview(Picture picture, CancellationTokenSource cts)
    {
        Task.Run(async () =>
        {
            try
            {
                byte[]? lastFrame = null;
                await foreach (var frame in _previewChannel.Reader.ReadAllAsync(cts.Token))
                {
                    if (ReferenceEquals(frame, lastFrame)) continue;
                    lastFrame = frame;
 
                    GLib.Functions.IdleAdd(0, () =>
                    {
                        try
                        {
                            var pix = Pixbuf.NewFromBytes(GLib.Bytes.New(frame), Colorspace.Rgb, false, 8, Width, Height, Width * 3);
                            picture.SetPixbuf(pix);
                        }
                        catch { }
                        return false;
                    });
                }
            }
            catch (OperationCanceledException) { }
        });
    }
 
    static void ScanForQRCode(Label label, CancellationTokenSource cts)
    {
        Task.Run(async () =>
        {
            try
            {
                byte[]? lastFrame = null;
                await foreach (var frame in _scanChannel.Reader.ReadAllAsync(cts.Token))
                {
                    if (ReferenceEquals(frame, lastFrame)) continue;
                    lastFrame = frame;
 
                    var text = ScanQrFromRgb(frame);
                    if (!string.IsNullOrEmpty(text))
                        GLib.Functions.IdleAdd(0, () => { label.SetText($"QR Code: {text}"); return false; });
                }
            }
            catch (OperationCanceledException) { }
        });
    }
 
    readonly static uint FourCC = 'Y' | ((uint)'8' << 8) | ((uint)'0' << 16) | ((uint)'0' << 24);
 
    private static string? ScanQrFromRgb(ReadOnlySpan<byte> rgb)
    {
        // Convert RGB to Grayscale
        var gray = new byte[Width * Height];
        for (int i = 0; i < rgb.Length; i += 3)
        {
            int y = (int)((rgb[i] * 0.299) + (rgb[i + 1] * 0.587) + (rgb[i + 2] * 0.114));
            gray[i / 3] = (byte)(y < 0 ? 0 : y > 255 ? 255 : y);
        }
 
        IntPtr image = zbar_image_create();
        zbar_image_set_format(image, FourCC);
        zbar_image_set_size(image, Width, Height);
 
        IntPtr dataPtr = Marshal.AllocHGlobal(gray.Length);
        try
        {
            Marshal.Copy(gray, 0, dataPtr, gray.Length);
            zbar_image_set_data(image, dataPtr, (UIntPtr)gray.Length, IntPtr.Zero);
 
            if (zbar_scan_image(_scanner, image) <= 0)
            {
                zbar_image_destroy(image);
                return null;
            }
 
            IntPtr symbol = zbar_image_first_symbol(image);
            string? result = symbol != IntPtr.Zero ? Marshal.PtrToStringAnsi(zbar_symbol_get_data(symbol)) : null;
            zbar_image_destroy(image);
            return result;
        }
        finally
        {
            Marshal.FreeHGlobal(dataPtr);
        }
    }
 
    const string ZBarLib = "libzbar.so.0";
    [DllImport(ZBarLib)]
    private static extern IntPtr zbar_image_scanner_create();
 
    [DllImport(ZBarLib)]
    private static extern IntPtr zbar_image_create();
 
    [DllImport(ZBarLib)]
    private static extern void zbar_image_destroy(IntPtr image);
 
    [DllImport(ZBarLib)]
    private static extern void zbar_image_set_format(IntPtr image, uint fourcc);
 
    [DllImport(ZBarLib)]
    private static extern void zbar_image_set_size(IntPtr image, uint width, uint height);
 
    [DllImport(ZBarLib)]
    private static extern void zbar_image_set_data(IntPtr image, IntPtr data, UIntPtr length, IntPtr cleanup);
 
    [DllImport(ZBarLib)]
    private static extern int zbar_scan_image(IntPtr scanner, IntPtr image);
 
    [DllImport(ZBarLib)]
    private static extern IntPtr zbar_image_first_symbol(IntPtr image);
 
    [DllImport(ZBarLib)]
    private static extern IntPtr zbar_symbol_get_data(IntPtr symbol);
}