ML.Net YoloV5 + Camera on ARM64 Raspberry PI

Building on my previous post I modified the code to support capturing images with a security camera(Unv ADZK-10) or a Raspberry PI Camera V2.

namespace devMobile.IoT.MachineLearning.ObjectDetectionCamera
{
	class Program
	{
		private static Model.ApplicationSettings _applicationSettings;
		private static YoloScorer<YoloCocoP5Model> _scorer = null;
		private static bool _cameraBusy = false;

		static async Task Main(string[] args)
		{
			Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} ObjectDetectionCamera starting");

			try
			{
				// load the app settings into configuration
				var configuration = new ConfigurationBuilder()
					 .AddJsonFile("appsettings.json", false, true)
					 .Build();

				_applicationSettings = configuration.GetSection("ApplicationSettings").Get<Model.ApplicationSettings>();

				_scorer = new YoloScorer<YoloCocoP5Model>(_applicationSettings.YoloV5ModelPath);

				Timer imageUpdatetimer = new Timer(ImageUpdateTimerCallback, null, _applicationSettings.ImageImageTimerDue, _applicationSettings.ImageTimerPeriod);

				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} press <ctrl^c> to exit");

				try
				{
					await Task.Delay(Timeout.Infinite);
				}
				catch (TaskCanceledException)
				{
					Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Application shutown requested");
				}
			}
			catch (Exception ex)
			{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Application shutown failure {ex.Message}", ex);
			}
		}

		private static async void ImageUpdateTimerCallback(object state)
		{
			DateTime requestAtUtc = DateTime.UtcNow;

			// Just incase - stop code being called while photo already in progress
			if (_cameraBusy)
			{
				return;
			}
			_cameraBusy = true;

			try
			{
#if SECURITY_CAMERA
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Security Camera Image download start");

				NetworkCredential networkCredential = new NetworkCredential()
				{
					UserName = _applicationSettings.CameraUserName,
					Password = _applicationSettings.CameraUserPassword,
				};

				using (WebClient client = new WebClient())
				{
					client.Credentials = networkCredential;

					client.DownloadFile(_applicationSettings.CameraUrl, _applicationSettings.InputImageFilenameLocal);
				}
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Security Camera Image download done");
#endif

#if RASPBERRY_PI_CAMERA
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Raspberry PI Image capture start");

				using (Process process = new Process())
				{
					process.StartInfo.FileName = @"libcamera-jpeg";
					process.StartInfo.Arguments = $"-o {_applicationSettings.InputImageFilenameLocal} --nopreview -t1 --rotation 180";
					process.StartInfo.RedirectStandardError = true;

					process.Start();

					if (!process.WaitForExit(_applicationSettings.ProcessWaitForExit) || (process.ExitCode != 0))
					{
						Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image update failure {process.ExitCode}");
					}
				}

				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Raspberry PI Image capture done");
#endif

				// Process the image on local file system
				using (Image image = Image.FromFile(_applicationSettings.InputImageFilenameLocal))
				{
					Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} YoloV5 inferencing start");
					System.Collections.Generic.List<YoloPrediction> predictions = _scorer.Predict(image);
					Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} YoloV5 inferencing done");

					using (Graphics graphics = Graphics.FromImage(image))
					{
						foreach (var prediction in predictions) // iterate predictions to draw results
						{
							double score = Math.Round(prediction.Score, 2);

							graphics.DrawRectangles(new Pen(prediction.Label.Color, 1), new[] { prediction.Rectangle });

							var (x, y) = (prediction.Rectangle.X - 3, prediction.Rectangle.Y - 23);

							graphics.DrawString($"{prediction.Label.Name} ({score})", new Font("Consolas", 16, GraphicsUnit.Pixel), new SolidBrush(prediction.Label.Color), new PointF(x, y));

							Console.WriteLine($"  {prediction.Label.Name} {score:f1}");

						}

						image.Save(_applicationSettings.OutputImageFilenameLocal);
					}
				}
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} YoloV5 inferencing done");
			}
			catch (Exception ex)
			{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Camera image download, upload or post procesing failed {ex.Message}");
			}
			finally
			{
				_cameraBusy = false;
			}

			TimeSpan duration = DateTime.UtcNow - requestAtUtc;

			Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image Update done {duration.TotalSeconds:f1} sec");
		}
	}
}

The name of the input image, output image and yoloV5 model file names are configured in the appsettings.json file.

Raspberry PI Camera V2 Results

Raspberry PI Camera V2 source image
ObjectDectionCamera application running on my RaspberryPI4
Raspberry PI Camera V2 image with MBRs

Security camera Results

Security Camera source image
ObjectDetectionCamera application running on RaspberryPI 8G 4B
Security Camera image with MBRs

Summary

The RaspberryPI Camera V2 images were 3280×2464 2.04M and the security camera images were 1920 x1080 464K so there was a significant quality and size difference.

When I ran the YoloV5s model application on my development box (Intel(R) Core(TM) i7-8700T CPU @ 2.40GHz) a security camera image took less than a second to process.

ObjectDetectionCamera application running on my development box

On the RaspberryPI V4b 8G the Raspberry PI Camera V2 images took roughly 1.52 seconds and security camera images roughly 1.47 seconds.

I was “standing on the shoulders of giants” the Mentalstack code just worked. With a pretrained yoloV5 model, the ML.Net Open Neural Network Exchange(ONNX) plumbing I had a working solution in a couple of hours.