ML.Net YoloV5 + Camera + GPIO on ARM64 Raspberry PI

This project builds on my ML.Net YoloV5 + Camera on ARM64 Raspberry PI post and adds support for turning a Light Emitting Diode(LED) on if the label of any object detected in an image is in the PredictionLabelsOfInterest list.

{
  "ApplicationSettings": {
    "ImageTimerDue": "0.00:00:15",
    "ImageTimerPeriod": "0.00:00:30",

    "CameraUrl": "...",
    "CameraUserName": "..",
    "CameraUserPassword": "...",

    "LedPinNumer": 5,

    "InputImageFilenameLocal": "InputLatest.jpg",
    "OutputImageFilenameLocal": "OutputLatest.jpg",

    "ProcessWaitForExit": 10000,

    "YoloV5ModelPath": "Assets/yolov5/yolov5s.onnx",

    "PredicitionScoreThreshold": 0.5,

    "PredictionLabelsOfInterest": [
      "bicycle",
      "person",
      "bench"
    ]
  }
}

The test-rig has consists of a Unv ADZK-10 Security Camera, Power over Ethernet(PoE) module, D-Link 8 port switch, Raspberry PI 8G 4b with a Seeedstudio Grove-Base Hat for Raspberry Pi, and Grove-Blue LED Button.

Test-rig configuration

class Program
{
	private static Model.ApplicationSettings _applicationSettings;
	private static bool _cameraBusy = false;
	private static YoloScorer<YoloCocoP5Model> _scorer = null;
#if GPIO_SUPPORT
	private static GpioController _gpiocontroller;
#endif

	static async Task Main(string[] args)
	{
		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} YoloV5ObjectDetectionCamera starting");

		try
		{
			// load the app settings into configuration
			var configuration = new ConfigurationBuilder()
				 .AddJsonFile("appsettings.json", false, true)
				 .Build();

			_applicationSettings = configuration.GetSection("ApplicationSettings").Get<Model.ApplicationSettings>();

#if GPIO_SUPPORT
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} GPIO setup start");

			_gpiocontroller = new GpioController(PinNumberingScheme.Logical);

			_gpiocontroller.OpenPin(_applicationSettings.ButtonPinNumer, PinMode.InputPullDown);

			_gpiocontroller.OpenPin(_applicationSettings.LedPinNumer, PinMode.Output);
			_gpiocontroller.Write(_applicationSettings.LedPinNumer, PinValue.Low);

			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} GPIO setup done");
#endif

			_scorer = new YoloScorer<YoloCocoP5Model>(_applicationSettings.YoloV5ModelPath);

			Timer imageUpdatetimer = new Timer(ImageUpdateTimerCallback, null, _applicationSettings.ImageImageTimerDue, _applicationSettings.ImageTimerPeriod);

			Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} press <ctrl^c> to exit");
			Console.WriteLine();

			try
			{
				await Task.Delay(Timeout.Infinite);
			}
			catch (TaskCanceledException)
			{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Application shutown requested");
			}
		}
		catch (Exception ex)
		{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Application shutown failure {ex.Message}", ex);
		}
	}

	private static void ImageUpdateTimerCallback(object state)
	{
		DateTime requestAtUtc = DateTime.UtcNow;

		// Just incase - stop code being called while photo already in progress
		if (_cameraBusy)
		{
			return;
		}
		_cameraBusy = true;

		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image processing start");

		try
		{
#if SECURITY_CAMERA
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Security Camera Image download start");

			NetworkCredential networkCredential = new NetworkCredential()
			{
				UserName = _applicationSettings.CameraUserName,
				Password = _applicationSettings.CameraUserPassword,
			};

			using (WebClient client = new WebClient())
			{
				client.Credentials = networkCredential;

				client.DownloadFile(_applicationSettings.CameraUrl, _applicationSettings.InputImageFilenameLocal);
			}
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Security Camera Image download done");
#endif

#if RASPBERRY_PI_CAMERA
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Raspberry PI Image capture start");

			using (Process process = new Process())
			{
				process.StartInfo.FileName = @"libcamera-jpeg";
				process.StartInfo.Arguments = $"-o {_applicationSettings.InputImageFilenameLocal} --nopreview -t1 --rotation 180";
				process.StartInfo.RedirectStandardError = true;

				process.Start();

				if (!process.WaitForExit(_applicationSettings.ProcessWaitForExit) || (process.ExitCode != 0))
				{
					Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image update failure {process.ExitCode}");
				}
			}

			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Raspberry PI Image capture done");
#endif

			List<YoloPrediction> predictions;

			// Process the image on local file system
			using (Image image = Image.FromFile(_applicationSettings.InputImageFilenameLocal))
			{
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} YoloV5 inferencing start");
				predictions = _scorer.Predict(image);
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} YoloV5 inferencing done");

#if OUTPUT_IMAGE_MARKUP
				using (Graphics graphics = Graphics.FromImage(image))
				{
					Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image markup start");

					foreach (var prediction in predictions) // iterate predictions to draw results
					{
						double score = Math.Round(prediction.Score, 2);

						graphics.DrawRectangles(new Pen(prediction.Label.Color, 1), new[] { prediction.Rectangle });

						var (x, y) = (prediction.Rectangle.X - 3, prediction.Rectangle.Y - 23);

						graphics.DrawString($"{prediction.Label.Name} ({score})", new Font("Consolas", 16, GraphicsUnit.Pixel), new SolidBrush(prediction.Label.Color), new PointF(x, y));
					}

					image.Save(_applicationSettings.OutputImageFilenameLocal);

					Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image markup done");
				}
#endif
			}

#if PREDICTION_CLASSES
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image classes start");
			foreach (var prediction in predictions)
			{
				Console.WriteLine($"  Name:{prediction.Label.Name} Score:{prediction.Score:f2} Valid:{prediction.Score > _applicationSettings.PredicitionScoreThreshold}");
			}
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image classes done");
#endif

#if PREDICTION_CLASSES_OF_INTEREST
			IEnumerable<string> predictionsOfInterest= predictions.Where(p=>p.Score > _applicationSettings.PredicitionScoreThreshold).Select(c => c.Label.Name).Intersect(_applicationSettings.PredictionLabelsOfInterest, StringComparer.OrdinalIgnoreCase);

			if (predictionsOfInterest.Any())
			{
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Camera image comtains {String.Join(",", predictionsOfInterest)}");
			}

   #if GPIO_SUPPORT
		   if (predictionsOfInterest.Any())
			{
				_gpiocontroller.Write(_applicationSettings.LedPinNumer, PinValue.High);
			}
			else
			{
				_gpiocontroller.Write(_applicationSettings.LedPinNumer, PinValue.Low);
			}
	#endif
#endif
		}
		catch (Exception ex)
		{
			Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Camera image download, upload or post procesing failed {ex.Message}");
		}
		finally
		{
			_cameraBusy = false;
		}

		TimeSpan duration = DateTime.UtcNow - requestAtUtc;

		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image processing done {duration.TotalSeconds:f2} sec");
		Console.WriteLine();
	}
}

The name of the digital output pin, input image, output image and yoloV5 model file names are configured in the appsettings.json file.

Mountain bike leaning against garage
YoloV5 based application console

The 22-01-31 06:52 “person” detection is me moving the mountain bike into position.

Marked up image of my mountain bike leaning against the garage

Summary

Once the YoloV5s model was loaded, inferencing was taking roughly 1.45 seconds. The application is starting to get a bit “nasty” so for the next version I’ll need to do some refactoring.

ML.Net YoloV5 + Camera on ARM64 Raspberry PI

Building on my previous post I modified the code to support capturing images with a security camera(Unv ADZK-10) or a Raspberry PI Camera V2.

namespace devMobile.IoT.MachineLearning.ObjectDetectionCamera
{
	class Program
	{
		private static Model.ApplicationSettings _applicationSettings;
		private static YoloScorer<YoloCocoP5Model> _scorer = null;
		private static bool _cameraBusy = false;

		static async Task Main(string[] args)
		{
			Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} ObjectDetectionCamera starting");

			try
			{
				// load the app settings into configuration
				var configuration = new ConfigurationBuilder()
					 .AddJsonFile("appsettings.json", false, true)
					 .Build();

				_applicationSettings = configuration.GetSection("ApplicationSettings").Get<Model.ApplicationSettings>();

				_scorer = new YoloScorer<YoloCocoP5Model>(_applicationSettings.YoloV5ModelPath);

				Timer imageUpdatetimer = new Timer(ImageUpdateTimerCallback, null, _applicationSettings.ImageImageTimerDue, _applicationSettings.ImageTimerPeriod);

				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} press <ctrl^c> to exit");

				try
				{
					await Task.Delay(Timeout.Infinite);
				}
				catch (TaskCanceledException)
				{
					Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Application shutown requested");
				}
			}
			catch (Exception ex)
			{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Application shutown failure {ex.Message}", ex);
			}
		}

		private static async void ImageUpdateTimerCallback(object state)
		{
			DateTime requestAtUtc = DateTime.UtcNow;

			// Just incase - stop code being called while photo already in progress
			if (_cameraBusy)
			{
				return;
			}
			_cameraBusy = true;

			try
			{
#if SECURITY_CAMERA
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Security Camera Image download start");

				NetworkCredential networkCredential = new NetworkCredential()
				{
					UserName = _applicationSettings.CameraUserName,
					Password = _applicationSettings.CameraUserPassword,
				};

				using (WebClient client = new WebClient())
				{
					client.Credentials = networkCredential;

					client.DownloadFile(_applicationSettings.CameraUrl, _applicationSettings.InputImageFilenameLocal);
				}
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Security Camera Image download done");
#endif

#if RASPBERRY_PI_CAMERA
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Raspberry PI Image capture start");

				using (Process process = new Process())
				{
					process.StartInfo.FileName = @"libcamera-jpeg";
					process.StartInfo.Arguments = $"-o {_applicationSettings.InputImageFilenameLocal} --nopreview -t1 --rotation 180";
					process.StartInfo.RedirectStandardError = true;

					process.Start();

					if (!process.WaitForExit(_applicationSettings.ProcessWaitForExit) || (process.ExitCode != 0))
					{
						Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image update failure {process.ExitCode}");
					}
				}

				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Raspberry PI Image capture done");
#endif

				// Process the image on local file system
				using (Image image = Image.FromFile(_applicationSettings.InputImageFilenameLocal))
				{
					Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} YoloV5 inferencing start");
					System.Collections.Generic.List<YoloPrediction> predictions = _scorer.Predict(image);
					Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} YoloV5 inferencing done");

					using (Graphics graphics = Graphics.FromImage(image))
					{
						foreach (var prediction in predictions) // iterate predictions to draw results
						{
							double score = Math.Round(prediction.Score, 2);

							graphics.DrawRectangles(new Pen(prediction.Label.Color, 1), new[] { prediction.Rectangle });

							var (x, y) = (prediction.Rectangle.X - 3, prediction.Rectangle.Y - 23);

							graphics.DrawString($"{prediction.Label.Name} ({score})", new Font("Consolas", 16, GraphicsUnit.Pixel), new SolidBrush(prediction.Label.Color), new PointF(x, y));

							Console.WriteLine($"  {prediction.Label.Name} {score:f1}");

						}

						image.Save(_applicationSettings.OutputImageFilenameLocal);
					}
				}
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} YoloV5 inferencing done");
			}
			catch (Exception ex)
			{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Camera image download, upload or post procesing failed {ex.Message}");
			}
			finally
			{
				_cameraBusy = false;
			}

			TimeSpan duration = DateTime.UtcNow - requestAtUtc;

			Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image Update done {duration.TotalSeconds:f1} sec");
		}
	}
}

The name of the input image, output image and yoloV5 model file names are configured in the appsettings.json file.

Raspberry PI Camera V2 Results

Raspberry PI Camera V2 source image
ObjectDectionCamera application running on my RaspberryPI4
Raspberry PI Camera V2 image with MBRs

Security camera Results

Security Camera source image
ObjectDetectionCamera application running on RaspberryPI 8G 4B
Security Camera image with MBRs

Summary

The RaspberryPI Camera V2 images were 3280×2464 2.04M and the security camera images were 1920 x1080 464K so there was a significant quality and size difference.

When I ran the YoloV5s model application on my development box (Intel(R) Core(TM) i7-8700T CPU @ 2.40GHz) a security camera image took less than a second to process.

ObjectDetectionCamera application running on my development box

On the RaspberryPI V4b 8G the Raspberry PI Camera V2 images took roughly 1.52 seconds and security camera images roughly 1.47 seconds.

I was “standing on the shoulders of giants” the Mentalstack code just worked. With a pretrained yoloV5 model, the ML.Net Open Neural Network Exchange(ONNX) plumbing I had a working solution in a couple of hours.

libcamera-jpeg on Raspberry Pi OS Bullseye Duration

The image capture process was taking about 5 seconds which a bit longer than I was expecting.

libcamera-jpeg -o rotated.jpg --rotation 180

The libcamera-jpeg program has a lot of command line parameters.

pi@raspberrypi4a:~ $ libcamera-jpeg --help
Valid options are:
  -h [ --help ] [=arg(=1)] (=0)         Print this help message
  --version [=arg(=1)] (=0)             Displays the build version number
  -v [ --verbose ] [=arg(=1)] (=0)      Output extra debug and diagnostics
  -c [ --config ] [=arg(=config.txt)]   Read the options from a file. If no filename is specified, default to
                                        config.txt. In case of duplicate options, the ones provided on the command line
                                        will be used. Note that the config file must only contain the long form
                                        options.
  --info-text arg (=#%frame (%fps fps) exp %exp ag %ag dg %dg)
                                        Sets the information string on the titlebar. Available values:
                                        %frame (frame number)
                                        %fps (framerate)
                                        %exp (shutter speed)
                                        %ag (analogue gain)
                                        %dg (digital gain)
                                        %rg (red colour gain)
                                        %bg (blue colour gain)
                                        %focus (focus FoM value)
                                        %aelock (AE locked status)
  --width arg (=0)                      Set the output image width (0 = use default value)
  --height arg (=0)                     Set the output image height (0 = use default value)
  -t [ --timeout ] arg (=5000)          Time (in ms) for which program runs
  -o [ --output ] arg                   Set the output file name
  --post-process-file arg               Set the file name for configuring the post-processing
  --rawfull [=arg(=1)] (=0)             Force use of full resolution raw frames
  -n [ --nopreview ] [=arg(=1)] (=0)    Do not show a preview window
  -p [ --preview ] arg (=0,0,0,0)       Set the preview window dimensions, given as x,y,width,height e.g. 0,0,640,480
  -f [ --fullscreen ] [=arg(=1)] (=0)   Use a fullscreen preview window
  --qt-preview [=arg(=1)] (=0)          Use Qt-based preview window (WARNING: causes heavy CPU load, fullscreen not
                                        supported)
  --hflip [=arg(=1)] (=0)               Request a horizontal flip transform
  --vflip [=arg(=1)] (=0)               Request a vertical flip transform
  --rotation arg (=0)                   Request an image rotation, 0 or 180
  --roi arg (=0,0,0,0)                  Set region of interest (digital zoom) e.g. 0.25,0.25,0.5,0.5
  --shutter arg (=0)                    Set a fixed shutter speed
  --analoggain arg (=0)                 Set a fixed gain value (synonym for 'gain' option)
  --gain arg                            Set a fixed gain value
  --metering arg (=centre)              Set the metering mode (centre, spot, average, custom)
  --exposure arg (=normal)              Set the exposure mode (normal, sport)
  --ev arg (=0)                         Set the EV exposure compensation, where 0 = no change
  --awb arg (=auto)                     Set the AWB mode (auto, incandescent, tungsten, fluorescent, indoor, daylight,
                                        cloudy, custom)
  --awbgains arg (=0,0)                 Set explict red and blue gains (disable the automatic AWB algorithm)
  --flush [=arg(=1)] (=0)               Flush output data as soon as possible
  --wrap arg (=0)                       When writing multiple output files, reset the counter when it reaches this
                                        number
  --brightness arg (=0)                 Adjust the brightness of the output images, in the range -1.0 to 1.0
  --contrast arg (=1)                   Adjust the contrast of the output image, where 1.0 = normal contrast
  --saturation arg (=1)                 Adjust the colour saturation of the output, where 1.0 = normal and 0.0 =
                                        greyscale
  --sharpness arg (=1)                  Adjust the sharpness of the output image, where 1.0 = normal sharpening
  --framerate arg (=30)                 Set the fixed framerate for preview and video modes
  --denoise arg (=auto)                 Sets the Denoise operating mode: auto, off, cdn_off, cdn_fast, cdn_hq
  --viewfinder-width arg (=0)           Width of viewfinder frames from the camera (distinct from the preview window
                                        size
  --viewfinder-height arg (=0)          Height of viewfinder frames from the camera (distinct from the preview window
                                        size)
  --tuning-file arg (=-)                Name of camera tuning file to use, omit this option for libcamera default
                                        behaviour
  --lores-width arg (=0)                Width of low resolution frames (use 0 to omit low resolution stream
  --lores-height arg (=0)               Height of low resolution frames (use 0 to omit low resolution stream
  -q [ --quality ] arg (=93)            Set the JPEG quality parameter
  -x [ --exif ] arg                     Add these extra EXIF tags to the output file
  --timelapse arg (=0)                  Time interval (in ms) between timelapse captures
  --framestart arg (=0)                 Initial frame counter value for timelapse captures
  --datetime [=arg(=1)] (=0)            Use date format for output file names
  --timestamp [=arg(=1)] (=0)           Use system timestamps for output file names
  --restart arg (=0)                    Set JPEG restart interval
  -k [ --keypress ] [=arg(=1)] (=0)     Perform capture when ENTER pressed
  -s [ --signal ] [=arg(=1)] (=0)       Perform capture when signal received
  --thumb arg (=320:240:70)             Set thumbnail parameters as width:height:quality
  -e [ --encoding ] arg (=jpg)          Set the desired output encoding, either jpg, png, rgb, bmp or yuv420
  -r [ --raw ] [=arg(=1)] (=0)          Also save raw file in DNG format
  --latest arg                          Create a symbolic link with this name to most recent saved file
  --immediate [=arg(=1)] (=0)           Perform first capture immediately, with no preview phase
pi@raspberrypi4a:~ $

My libcamera-jpeg application is run “headless” so I tried turning off the image preview functionality.

libcamera-jpeg -o rotatednopreview.jpg --nopreview

When I ran libcamera-jpeg in a console windows or my application this didn’t appear to make any noticeable difference.

libcamera-jpeg run from the command line with –nopreview

libcamera-jpeg run by my application with –nopreview

I then had another look at the libcamera-jpeg command line parameters to see if any looked useful for reducing the time that it took to take a save an image and this one caught my attention.

I had assumed the delay was related to how long the preview window was displayed.

libcamera-jpeg run from the command line with –nopreview –t1

I modified the application (V5) then ran it from the command line and the time reduced to less than a second.

private static void ImageUpdateTimerCallback(object state)
{
	try
	{
		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image update start");

		// Just incase - stop code being called while photo already in progress
		if (_cameraBusy)
		{
			return;
		}

		Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image capture start");

		using (Process process = new Process())
		{
			process.StartInfo.FileName = @"libcamera-jpeg";
			// V1 it works
			//process.StartInfo.Arguments = $"-o {_applicationSettings.ImageFilenameLocal}";
			// V3a Image right way up
			//process.StartInfo.Arguments = $"-o {_applicationSettings.ImageFilenameLocal} --vflip --hflip";
			// V3b Image right way up
			//process.StartInfo.Arguments = $"-o {_applicationSettings.ImageFilenameLocal} --rotation 180";
			// V4 Image no preview
			//process.StartInfo.Arguments = $"-o {_applicationSettings.ImageFilenameLocal} --rotation 180 --nopreview";
			// V5 Image no preview, no timeout
			process.StartInfo.Arguments = $"-o {_applicationSettings.ImageFilenameLocal} --nopreview -t1 --rotation 180";
			//process.StartInfo.RedirectStandardOutput = true;
			// V2 No diagnostics
			process.StartInfo.RedirectStandardError = true;
			//process.StartInfo.UseShellExecute = false;
			//process.StartInfo.CreateNoWindow = true; 

			process.Start();

			if (!process.WaitForExit(10000) || (process.ExitCode != 0))
			{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image update failure {process.ExitCode}");
			}
		}

		Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image capture done");
	}
	catch (Exception ex)
	{
		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image update error {ex.Message}");
	}
	finally
	{
		_cameraBusy = false;
	}
}
libcamera-jpeg run by my application with –nopreview -t1

The image capture process now takes less that a second which is much better (but not a lot less than retrieving an image from one of my security cameras).

libcamera on Raspberry Pi OS Bullseye

This is a “note to self” post about using libcamera(replacement for raspistill) on my Raspberry PI 4 Model B to capture an image from my Raspberry Pi Camera Module 2 with an application built with .NET Core.

I wanted one of my ML.Net demos to use the Raspberry PI Camera rather than a security camera (so it was more portable) but it took a bit more work than I expected.

Version 1 used Process.Start to launch the libcamera-jpeg application with a command line to store an image to the local file system.

libcamera-jpeg -o latest.jpg
libcamera-jpeg with diagnostic information displayed

There was a lot of diagnostic information which I didn’t want displayed so after reading many stackoverflow posts (lots of different approaches none of which worked in my scenario), then some trial and error I found that I only had to enable RedirectStandardError.

libcamera-jpeg without diagnostic information displayed

At this point there was a lot less noise but the image was upside down.

Inverted picture of my 30th anniversary Mini Cooper in the backyard

I then added a vertical flip to the command line parameters

libcamera-jpeg -o latest.jpg --vflip
My 30th anniversary Mini Cooper in the backyard

The image was backwards so I added a horizontal flip to the commandline parameters

libcamera-jpeg -o latest.jpg --vflip --hflip

or

libcamera-jpeg -o latest.jpg --rotation 180
My 30th anniversary Mini Cooper in the backyard with the correct orientation

The libcamera code is in a Timer callback so I added the _cameraBusy boolean flag to stop reentrancy problems.

private static void ImageUpdateTimerCallback(object state)
{
	try
	{
		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image update start");

		// Just incase - stop code being called while photo already in progress
		if (_cameraBusy)
		{
			return;
		}

		Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image capture start");

		using (Process process = new Process())
		{
			process.StartInfo.FileName = @"libcamera-jpeg";
			// V1 it works
			//process.StartInfo.Arguments = $"-o {_applicationSettings.ImageFilenameLocal}";
			// V3 Image right way up
			//process.StartInfo.Arguments = $"-o {_applicationSettings.ImageFilenameLocal} --vflip";
			// V3 Image right way round
			process.StartInfo.Arguments = $"-o {_applicationSettings.ImageFilenameLocal} --vflip --hflip";
			//process.StartInfo.RedirectStandardOutput = true;
			// V2 No diagnostics
			process.StartInfo.RedirectStandardError = true;
			//process.StartInfo.UseShellExecute = false;
			//process.StartInfo.CreateNoWindow = true; 

			process.Start();

			if (!process.WaitForExit(10000) || (process.ExitCode != 0))
			{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image update failure {process.ExitCode}");
			}
		}

		Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image capture done");
	}
	catch (Exception ex)
	{
		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image update error {ex.Message}");
	}
	finally
	{
		_cameraBusy = false;
	}
}

This was the simplest way I could get an image onto the local file system without lots of dependencies on third party libraries. The image capture process takes about 5 seconds which a bit longer than I was expecting.

ML.Net ONNX Object Detection Sample refactoring

I use CustomVision.AI to tag images, then train, test and tune models for my projects. I wanted to be able to export a model for use on a embedded device with minimal manual steps so the Object Detection-ASP.NET Core Web & WPF Desktop Sample in the dotnet/machine-learning-samples looked like a reasonable place to get some “inspiration”.

Extracting the ObjectDetection-Onnx code from the zip file

I updated the OnnxObjectDectection library, OnnxObjectDetectionApp, and OnnxObjectDetectionWeb project Nugets to the latest versions then “smoke tested” the desktop and web applications.

Updated OnnxObjectDetection project NuGets
Updated OnnxObjectDetectionApp project NuGets

The desktop application used the OpenCvSharp3-AnyCPU NuGet which had been deprecated so I upgraded to OpenCvSharp4-Windows NuGet which required a couple of small code modification.

private async Task CaptureCamera(CancellationToken token)
{
   if (capture == null)
      capture = new VideoCapture();

   capture.Open(0,apiPreference:VideoCaptureAPIs.DSHOW);

   if (capture.IsOpened())
   {
      while (!token.IsCancellationRequested)
      {
         using MemoryStream memoryStream = capture.RetrieveMat().Flip(FlipMode.Y).ToMemoryStream();

         await Application.Current.Dispatcher.InvokeAsync(() =>
         {
            var imageSource = new BitmapImage();

            imageSource.BeginInit();
            imageSource.CacheOption = BitmapCacheOption.OnLoad;
            imageSource.StreamSource = memoryStream;
            imageSource.EndInit();

            WebCamImage.Source = imageSource;
         });

         var bitmapImage = new Bitmap(memoryStream);

         await ParseWebCamFrame(bitmapImage, token);
      }

   capture.Release();
   }
}

I ran the OnnxObjectDetectionApp and the provided TinyYolo2_model.onnx model using my webcam.

TinyYolo2_model identifying me as a “person”
Updated OnnxObjectDetectionWeb project NuGets

I ran the OnnxObjectDetectionWeb with the provided TinyYolo2_model.onnx model and a photograph of a car I used to own.

TinyYolo2_model correctly identifying my Lotus 7 as a car.

I have a simple CustomVision.AI demo project for counting toy farm animals which I used to test my modifications.

Quick test of the ToyCowCounter model in CustomVision.ai portal

I exported The ToyCowCounter in ONNX format

Toy Cow Counter Exporting in ONNX format

I copied the exported file to the OnnxModels folder, and then in the Visual Studio 2019 solution explorer configured the file properties “Build Action-Content” and “Copy To Output Directory-Copy if newer”.

When I restarted the OnnxObjectDetectionApp the application would detect my toy cows with a reasonable accuracy.

ToyCowCounter model identifying a cow

The accuracy of the ToyCowCounter model wasn’t great as it had been trained with a limited dataset collected with a different camera and a plain backdrop.