Smartish Edge Camera – Azure IoT Updateable Properties (not persisted)

This post builds on my Smartish Edge Camera -Azure IoT Direct Methods post adding two updateable properties for the image capture and processing timer the due and period values. The two properties can be updated together or independently but the values are not persisted.

When I was searching for answers I found this code in many posts and articles but it didn’t really cover my scenario.

private static async Task OnDesiredPropertyChanged(TwinCollection desiredProperties, 
  object userContext)
{
   Console.WriteLine("desired property chPleange:");
   Console.WriteLine(JsonConvert.SerializeObject(desiredProperties));
   Console.WriteLine("Sending current time as reported property");
   TwinCollection reportedProperties = new TwinCollection
   {
       ["DateTimeLastDesiredPropertyChangeReceived"] = DateTime.Now
   };

    await Client.UpdateReportedPropertiesAsync(reportedProperties).ConfigureAwait(false);
}

When AZURE_DEVICE_PROPERTIES is defined in the SmartEdgeCameraAzureIoTService project properties the device reports a number of properties on startup and SetDesiredPropertyUpdateCallbackAsync is used to configure the method called whenever the client receives a state update(desired or reported) from the Azure IoT Hub.

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
	_logger.LogInformation("Azure IoT Smart Edge Camera Service starting");

	try
	{
#if AZURE_IOT_HUB_CONNECTION
		_deviceClient = await AzureIoTHubConnection();
#endif
#if AZURE_IOT_HUB_DPS_CONNECTION
		_deviceClient = await AzureIoTHubDpsConnection();
#endif

#if AZURE_DEVICE_PROPERTIES
		_logger.LogTrace("ReportedPropeties upload start");

		TwinCollection reportedProperties = new TwinCollection();

		reportedProperties["OSVersion"] = Environment.OSVersion.VersionString;
		reportedProperties["MachineName"] = Environment.MachineName;
		reportedProperties["ApplicationVersion"] = Assembly.GetAssembly(typeof(Program)).GetName().Version;
		reportedProperties["ImageTimerDue"] = _applicationSettings.ImageTimerDue;
		reportedProperties["ImageTimerPeriod"] = _applicationSettings.ImageTimerPeriod;
		reportedProperties["YoloV5ModelPath"] = _applicationSettings.YoloV5ModelPath;

		reportedProperties["PredictionScoreThreshold"] = _applicationSettings.PredictionScoreThreshold;
		reportedProperties["PredictionLabelsOfInterest"] = _applicationSettings.PredictionLabelsOfInterest;
		reportedProperties["PredictionLabelsMinimum"] = _applicationSettings.PredictionLabelsMinimum;

		await _deviceClient.UpdateReportedPropertiesAsync(reportedProperties, stoppingToken);

		_logger.LogTrace("ReportedPropeties upload done");
#endif

		_logger.LogTrace("YoloV5 model setup start");
		_scorer = new YoloScorer<YoloCocoP5Model>(_applicationSettings.YoloV5ModelPath);
		_logger.LogTrace("YoloV5 model setup done");

		_ImageUpdatetimer = new Timer(ImageUpdateTimerCallback, null, _applicationSettings.ImageTimerDue, _applicationSettings.ImageTimerPeriod);

		await _deviceClient.SetMethodHandlerAsync("ImageTimerStart", ImageTimerStartHandler, null);
		await _deviceClient.SetMethodHandlerAsync("ImageTimerStop", ImageTimerStopHandler, null);
		await _deviceClient.SetMethodDefaultHandlerAsync(DefaultHandler, null);

		await _deviceClient.SetDesiredPropertyUpdateCallbackAsync(OnDesiredPropertyChangedAsync, null);

		try
		{
			await Task.Delay(Timeout.Infinite, stoppingToken);
		}
		catch (TaskCanceledException)
		{
			_logger.LogInformation("Application shutown requested");
		}
	}
	catch (Exception ex)
	{
		_logger.LogError(ex, "Application startup failure");
	}
	finally
	{
		_deviceClient?.Dispose();
	}

	_logger.LogInformation("Azure IoT Smart Edge Camera Service shutdown");
}

// Lots of other code here

private async Task OnDesiredPropertyChangedAsync(TwinCollection desiredProperties, object userContext)
{
	TwinCollection reportedProperties = new TwinCollection();

	_logger.LogInformation("OnDesiredPropertyChanged handler");

	// NB- This approach does not save the ImageTimerDue or ImageTimerPeriod, a stop/start with return to appsettings.json configuration values. If only
	// one parameter specified other is default from appsettings.json. If timer settings changed I think they won't take
	// effect until next time Timer fires.

	try
	{
		// Check to see if either of ImageTimerDue or ImageTimerPeriod has changed
		if (!desiredProperties.Contains("ImageTimerDue") && !desiredProperties.Contains("ImageTimerPeriod"))
		{
			_logger.LogInformation("OnDesiredPropertyChanged neither ImageTimerDue or ImageTimerPeriod present");
			return;
		}

		TimeSpan imageTimerDue = _applicationSettings.ImageTimerDue;

		// Check that format of ImageTimerDue valid if present
		if (desiredProperties.Contains("ImageTimerDue"))
		{
			if (TimeSpan.TryParse(desiredProperties["ImageTimerDue"].Value, out imageTimerDue))
			{
				reportedProperties["ImageTimerDue"] = imageTimerDue;
			}
			else
			{
				_logger.LogInformation("OnDesiredPropertyChanged ImageTimerDue invalid");
				return;
			}
		}

		TimeSpan imageTimerPeriod = _applicationSettings.ImageTimerPeriod;

		// Check that format of ImageTimerPeriod valid if present
		if (desiredProperties.Contains("ImageTimerPeriod"))
		{
			if (TimeSpan.TryParse(desiredProperties["ImageTimerPeriod"].Value, out imageTimerPeriod))
			{
				reportedProperties["ImageTimerPeriod"] = imageTimerPeriod;
			}
			else
			{
				_logger.LogInformation("OnDesiredPropertyChanged ImageTimerPeriod invalid");
				return;
			}
		}

		_logger.LogInformation("Desired Due:{0} Period:{1}", imageTimerDue, imageTimerPeriod);

		if (!_ImageUpdatetimer.Change(imageTimerDue, imageTimerPeriod))
		{
			_logger.LogInformation("Desired Due:{0} Period:{1} failed", imageTimerDue, imageTimerPeriod);
		}

		await _deviceClient.UpdateReportedPropertiesAsync(reportedProperties);
	}
	catch (Exception ex)
	{
		_logger.LogError(ex, "OnDesiredPropertyChangedAsync handler failed");
	}
}

The TwinCollection desiredProperties is checked for ImageTimerDue and ImageTimerPeriod properties and if either of these are present and valid the Timer.Change method is called.

The AzureMLMetSmartEdgeCamera supports both Azure IoT Hub and Azure IoT Central so I have included images from Azure IoT Explorer and my Azure IoT Central Templates.

SmartEdge Camera Device Twin properties in Azure IoT Explorer

When I modified, then saved the Azure IoT Hub Device Twin desired properties JavaScript Object Notation(JSON) in Azure IoT Hub Explorer the method configured with SetDesiredPropertyUpdateCallbackAsync was invoked on the device.

In Azure IoT Central I added two Capabilities to the device template, the time properties ImageTimerDue, and ImageTimerPeriod.

Azure IoT Central SmartEdgeCamera Device template capabilities

I added a View to the template so the two properties could be changed (I didn’t configure either as required)

Azure IoT Central SmartEdgeCamera Device Default view designer

In the “Device Properties”, “Operation Tab” when I changed the ImageTimerDue and/or ImageTimerPeriod there was visual feedback that there was an update in progress.

Azure IoT Central SmartEdgeCamera Device Properties update start

Then on the device the SmartEdgeCameraAzureIoTService the method configured with SetDesiredPropertyUpdateCallbackAsync was invoked on the device.

SmartEdge Camera Console application displaying updated properties

Once the properties have been updated on the device the UpdateReportedPropertiesAsync method is called

Then a message with the updated property values from the device was visible in the telemetry

Azure IoT Central SmartEdgeCamera Device Properties update done

Then finally the “Operation Tab” displayed a visual confirmation that the value(s) had been updated.

Smartish Edge Camera – Azure IoT Direct Methods

This post builds on my Smartish Edge Camera – Azure IoT Image-Upload post adding two Direct Methods for Starting and Stopping the image capture and processing timer. The AzureMLMetSmartEdgeCamera supports both Azure IoT Hub and Azure IoT Central connectivity.

Azure IoT Explorer invoking a Direct Method

BEWARE – The Direct Method names are case sensitive which regularly trips me up when I use Azure IoT Explorer. If the Direct Method name is unknown a default handler is called, the issue logged and a Hyper Text Transfer Protocol(HTTP) Not Implemented(501) error returned

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
	_logger.LogInformation("Azure IoT Smart Edge Camera Service starting");

	try
	{
#if AZURE_IOT_HUB_CONNECTION
		_deviceClient = await AzureIoTHubConnection();
#endif
#if AZURE_IOT_HUB_DPS_CONNECTION
		_deviceClient = await AzureIoTHubDpsConnection();
#endif

...
		_logger.LogTrace("YoloV5 model setup start");
		_scorer = new YoloScorer<YoloCocoP5Model>(_applicationSettings.YoloV5ModelPath);
		_logger.LogTrace("YoloV5 model setup done");

		_ImageUpdatetimer = new Timer(ImageUpdateTimerCallback, null, _applicationSettings.ImageTimerDue, _applicationSettings.ImageTimerPeriod);

		await _deviceClient.SetMethodHandlerAsync("ImageTimerStart", ImageTimerStartHandler, null);
		await _deviceClient.SetMethodHandlerAsync("ImageTimerStop", ImageTimerStopHandler, null);
		await _deviceClient.SetMethodDefaultHandlerAsync(DefaultHandler, null);
...
		try
		{
			await Task.Delay(Timeout.Infinite, stoppingToken);
		}
		catch (TaskCanceledException)
		{
			_logger.LogInformation("Application shutown requested");
		}
	}
	catch (Exception ex)
	{
		_logger.LogError(ex, "Application startup failure");
	}
	finally
	{
		_deviceClient?.Dispose();
	}

	_logger.LogInformation("Azure IoT Smart Edge Camera Service shutdown");
}

private async Task<MethodResponse> ImageTimerStartHandler(MethodRequest methodRequest, object userContext)
{
	_logger.LogInformation("ImageUpdatetimer Start Due:{0} Period:{1}", _applicationSettings.ImageTimerDue, _applicationSettings.ImageTimerPeriod);

	_ImageUpdatetimer.Change(_applicationSettings.ImageTimerDue, _applicationSettings.ImageTimerPeriod);

	return new MethodResponse((short)HttpStatusCode.OK);
}

private async Task<MethodResponse> ImageTimerStopHandler(MethodRequest methodRequest, object userContext)
{
	_logger.LogInformation("ImageUpdatetimer Stop");

	_ImageUpdatetimer.Change(Timeout.Infinite, Timeout.Infinite);

	return new MethodResponse((short)HttpStatusCode.OK);
}

private async Task<MethodResponse> DefaultHandler(MethodRequest methodRequest, object userContext)
{
	_logger.LogInformation("Direct Method default handler Name:{0}", methodRequest.Name);

	return new MethodResponse((short)HttpStatusCode.NotFound);
}

I created an Azure IoT Central Template with two command capabilities. (For more detail see my post TTI V3 Connector Azure IoT Central Cloud to Device(C2D)).

Azure IoT Central Template Direct Method configuration
Azure IoT Central Template Direct Method invocation
Azure Smart Edge Camera console application Start Direct Method call

Initially, I had one long post which covered Direct Methods, Readonly Properties and Updateable Properties but it got too long so I split it into three.

Smartish Edge Camera – Azure IoT Image Upload

This post builds on my Smartish Edge Camera – Azure Storage Service, Azure IoT Hub, and Azure IoT Central projects adding optional camera and marked-up image upload to Azure Blob Storage for Azure IoT Hubs and Azure IoT Central.

Azure IoT Hub – File upload storage account configuration
Azure IoT Central – File upload storage account configuration

The “new improved” process of uploading files to an Azure IoT Hub and Azure IoT Central is surprisingly complex to use and make robust(I think the initial approach with DeviceClient.UploadToBlobAsync which is now “deprecated” was easier to use).

public async Task UploadImage(List<YoloPrediction> predictions, string filepath, string blobpath)
{
	var fileUploadSasUriRequest = new FileUploadSasUriRequest()
	{
		BlobName = blobpath 
	};

	FileUploadSasUriResponse sasUri = await _deviceClient.GetFileUploadSasUriAsync(fileUploadSasUriRequest);

	var blockBlobClient = new BlockBlobClient(sasUri.GetBlobUri());

	var fileUploadCompletionNotification = new FileUploadCompletionNotification()
	{
		// Mandatory. Must be the same value as the correlation id returned in the sas uri response
		CorrelationId = sasUri.CorrelationId,

		IsSuccess = true
	};

	try
	{
		using (FileStream fileStream = File.OpenRead(filepath))
		{
			Response<BlobContentInfo> response = await blockBlobClient.UploadAsync(fileStream); //, blobUploadOptions);

			fileUploadCompletionNotification.StatusCode = response.GetRawResponse().Status;

			if (fileUploadCompletionNotification.StatusCode != ((int)HttpStatusCode.Created))
			{
				fileUploadCompletionNotification.IsSuccess = false;

				fileUploadCompletionNotification.StatusDescription = response.GetRawResponse().ReasonPhrase;
			}
		}
	}
	catch (RequestFailedException ex)
	{
		fileUploadCompletionNotification.StatusCode = ex.Status;

		fileUploadCompletionNotification.IsSuccess = false;

		fileUploadCompletionNotification.StatusDescription = ex.Message;
	}
	finally
	{
		await _deviceClient.CompleteFileUploadAsync(fileUploadCompletionNotification);
	}
}

If there is an object with a label in the PredictionLabelsOfInterest list, the camera and marked-up images can (configured with ImageCameraUpload & ImageMarkedupUpload) be uploaded to an Azure Storage Blob container associated with an Azure IoT Hub/ Azure IoT Central instance.

{
  "Logging": {
    "LogLevel": {
      "Default": "Information",
      "Microsoft": "Warning",
      "Microsoft.Hosting.Lifetime": "Information"
    }
  },

  "Application": {
    "DeviceID": "",
    "ImageTimerDue": "0.00:00:15",
    "ImageTimerPeriod": "0.00:00:30",

    "ImageCameraFilepath": "ImageCamera.jpg",
    "ImageMarkedUpFilepath": "ImageMarkedup.jpg",

    "ImageCameraUpload": false,
    "ImageMarkedupUpload": true,

    "ImageUploadFilepath": "ImageMarkedup.jpg",

    "YoloV5ModelPath": "YoloV5/yolov5s.onnx",

    "PredictionScoreThreshold": 0.7,
    "PredictionLabelsOfInterest": [
      "bicycle",
      "person"
    ],

    "PredictionLabelsMinimum": [
      "bicycle",
      "car",
      "person"
    ],

    "ImageCameraFilenameFormat": "{0:yyyyMMdd}/{0:HHmmss}.jpg"
  },

  "SecurityCamera": {
    "CameraUrl": "",
    "CameraUserName": "",
    "CameraUserPassword": ""
  },

  "RaspberryPICamera": {
    "ProcessWaitForExit": 1000,
    "Rotation": 180
  },

  "AzureIoTHub": {
    "ConnectionString": ""
  },

  "AzureIoTHubDPS": {
    "GlobalDeviceEndpoint": "global.azure-devices-provisioning.net",
    "IDScope": "",
    "GroupEnrollmentKey": ""
  },

  "AzureStorage": {
    "ImageCameraFilenameFormat": "{0:yyyyMMdd}/camera/{0:HHmmss}.jpg",
    "ImageMarkedUpFilenameFormat": "{0:yyyyMMdd}/markedup/{0:HHmmss}.jpg"
  }
}

The Blob’s path is prefixed with the device id (My Azure Storage Service created an Azure Blob Storage container for each device).

Azure IoT Central SmartEdge Camera devices

The format of the Azure Storage Blob path is configurable(ImageCameraFilenameFormat & ImageMarkedUpFilenameFormat + Universal Coordinated Time(UTC)) so images can be grouped.

Configurable Blob paths in Azure Storage Explorer

After creating a new Azure IoT Hub uploads started failing with an exception and there weren’t a lot of useful search results (April 2022). I found error this was caused by missing or incorrect Azure Storage Account configuration.

Azure IoT Hub Upload application failure logging
{"Message":"{\"errorCode\":400022,\"trackingId\":\"1175af36ec884cc4a54978f77b877a01-G:0-TimeStamp:04/12/2022 10:19:04\",\"message\":\"BadRequest\",\"timestampUtc\":\"2022-04-12T10:19:04.5925999Z\"}","ExceptionMessage":""}

   at Microsoft.Azure.Devices.Client.Transport.HttpClientHelper.<ExecuteAsync>d__23.MoveNext()
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at Microsoft.Azure.Devices.Client.Transport.HttpClientHelper.<PostAsync>d__19`2.MoveNext()
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at System.Runtime.CompilerServices.ConfiguredTaskAwaitable`1.ConfiguredTaskAwaiter.GetResult()
   at Microsoft.Azure.Devices.Client.Transport.HttpTransportHandler.<GetFileUploadSasUriAsync>d__15.MoveNext()
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at System.Runtime.CompilerServices.TaskAwaiter`1.GetResult()
   at devMobile.IoT.MachineLearning.SmartEdgeCameraAzureIoTService.Worker.<UploadImage>d__14.MoveNext() in C:\Users\BrynLewis\source\repos\AzureMLNetSmartEdgeCamera\SmartEdgeCameraAzureIoTService\Worker.cs:line 430
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
   at devMobile.IoT.MachineLearning.SmartEdgeCameraAzureIoTService.Worker.<ImageUpdateTimerCallback>d__10.MoveNext() in C:\Users\BrynLewis\source\repos\AzureMLNetSmartEdgeCamera\SmartEdgeCameraAzureIoTService\Worker.cs:line 268

While testing the application I noticed an “unexpected” object detected in my backyard…

Unexpected object detection diagnostic logging
Unexpected object detection results marked-up image

The mentalstack/yolov5-net and NuGet have been incredibly useful and MentalStack team have done a marvelous job building and supporting this project. For this project my test-rig consisted of a Unv ADZK-10 Security Camera, Power over Ethernet(PoE) and my HP Prodesk 400G4 DM (i7-8700T).

Smartish Edge Camera – Azure IoT Central

This post builds on Smartish Edge Camera – Azure Hub Part 1 using the Azure IoT Hub Device Provisioning Service(DPS) to connect to Azure IoT Central.

The list of object classes is in the YoloCocoP5Model.cs file in the mentalstack/yolov5-net repository.

public override List<YoloLabel> Labels { get; set; } = new List<YoloLabel>()
{
    new YoloLabel { Id = 1, Name = "person" },
    new YoloLabel { Id = 2, Name = "bicycle" },
    new YoloLabel { Id = 3, Name = "car" },
    new YoloLabel { Id = 4, Name = "motorcycle" },
    new YoloLabel { Id = 5, Name = "airplane" },
    new YoloLabel { Id = 6, Name = "bus" },
    new YoloLabel { Id = 7, Name = "train" },
    new YoloLabel { Id = 8, Name = "truck" },
    new YoloLabel { Id = 9, Name = "boat" },
    new YoloLabel { Id = 10, Name = "traffic light" },
    new YoloLabel { Id = 11, Name = "fire hydrant" },
    new YoloLabel { Id = 12, Name = "stop sign" },
    new YoloLabel { Id = 13, Name = "parking meter" },
    new YoloLabel { Id = 14, Name = "bench" },
    new YoloLabel { Id = 15, Name = "bird" },
    new YoloLabel { Id = 16, Name = "cat" },
    new YoloLabel { Id = 17, Name = "dog" },
    new YoloLabel { Id = 18, Name = "horse" },
    new YoloLabel { Id = 19, Name = "sheep" },
    new YoloLabel { Id = 20, Name = "cow" },
    new YoloLabel { Id = 21, Name = "elephant" },
    new YoloLabel { Id = 22, Name = "bear" },
    new YoloLabel { Id = 23, Name = "zebra" },
    new YoloLabel { Id = 24, Name = "giraffe" },
    new YoloLabel { Id = 25, Name = "backpack" },
    new YoloLabel { Id = 26, Name = "umbrella" },
    new YoloLabel { Id = 27, Name = "handbag" },
    new YoloLabel { Id = 28, Name = "tie" },
    new YoloLabel { Id = 29, Name = "suitcase" },
    new YoloLabel { Id = 30, Name = "frisbee" },
    new YoloLabel { Id = 31, Name = "skis" },
    new YoloLabel { Id = 32, Name = "snowboard" },
    new YoloLabel { Id = 33, Name = "sports ball" },
    new YoloLabel { Id = 34, Name = "kite" },
    new YoloLabel { Id = 35, Name = "baseball bat" },
    new YoloLabel { Id = 36, Name = "baseball glove" },
    new YoloLabel { Id = 37, Name = "skateboard" },
    new YoloLabel { Id = 38, Name = "surfboard" },
    new YoloLabel { Id = 39, Name = "tennis racket" },
    new YoloLabel { Id = 40, Name = "bottle" },
    new YoloLabel { Id = 41, Name = "wine glass" },
    new YoloLabel { Id = 42, Name = "cup" },
    new YoloLabel { Id = 43, Name = "fork" },
    new YoloLabel { Id = 44, Name = "knife" },
    new YoloLabel { Id = 45, Name = "spoon" },
    new YoloLabel { Id = 46, Name = "bowl" },
    new YoloLabel { Id = 47, Name = "banana" },
    new YoloLabel { Id = 48, Name = "apple" },
    new YoloLabel { Id = 49, Name = "sandwich" },
    new YoloLabel { Id = 50, Name = "orange" },
    new YoloLabel { Id = 51, Name = "broccoli" },
    new YoloLabel { Id = 52, Name = "carrot" },
    new YoloLabel { Id = 53, Name = "hot dog" },
    new YoloLabel { Id = 54, Name = "pizza" },
    new YoloLabel { Id = 55, Name = "donut" },
    new YoloLabel { Id = 56, Name = "cake" },
    new YoloLabel { Id = 57, Name = "chair" },
    new YoloLabel { Id = 58, Name = "couch" },
    new YoloLabel { Id = 59, Name = "potted plant" },
    new YoloLabel { Id = 60, Name = "bed" },
    new YoloLabel { Id = 61, Name = "dining table" },
    new YoloLabel { Id = 62, Name = "toilet" },
    new YoloLabel { Id = 63, Name = "tv" },
    new YoloLabel { Id = 64, Name = "laptop" },
    new YoloLabel { Id = 65, Name = "mouse" },
    new YoloLabel { Id = 66, Name = "remote" },
    new YoloLabel { Id = 67, Name = "keyboard" },
    new YoloLabel { Id = 68, Name = "cell phone" },
    new YoloLabel { Id = 69, Name = "microwave" },
    new YoloLabel { Id = 70, Name = "oven" },
    new YoloLabel { Id = 71, Name = "toaster" },
    new YoloLabel { Id = 72, Name = "sink" },
    new YoloLabel { Id = 73, Name = "refrigerator" },
    new YoloLabel { Id = 74, Name = "book" },
    new YoloLabel { Id = 75, Name = "clock" },
    new YoloLabel { Id = 76, Name = "vase" },
    new YoloLabel { Id = 77, Name = "scissors" },
    new YoloLabel { Id = 78, Name = "teddy bear" },
    new YoloLabel { Id = 79, Name = "hair drier" },
    new YoloLabel { Id = 80, Name = "toothbrush" }
};

Some of the label choices seem a bit arbitrary(frisbee, surfboard) and American(fire hydrant, baseball bat, baseball glove) It was quite tedious configuring the 80 labels in my Azure IoT Central template.

Azure IoT Central Template with all the YoloV5 labels configured

If there is an object with a label in the PredictionLabelsOfInterest list, a tally of each of the different object classes in the image is sent to an Azure IoT Hub/ Azure IoT Central.

"Application": {
  "DeviceID": "",
  "ImageTimerDue": "0.00:00:15",
  "ImageTimerPeriod": "0.00:00:30",

  "ImageCameraFilepath": "ImageCamera.jpg",
  "ImageMarkedUpFilepath": "ImageMarkedup.jpg",

  "YoloV5ModelPath": "YoloV5/yolov5s.onnx",

  "PredictionScoreThreshold": 0.7,
  "PredictionLabelsOfInterest": [
    "bicycle",
    "person"
  ],
  "PredictionLabelsMinimum": [
    "bicycle",
    "car",
    "person"
  ]
}
My backyard just after the car left (the dry patch in shingle on the right)
Smartish Edge Camera Service console just after car left
Smartish Edge Camera Azure IoT Central graphs showing missing data points

After the You Only Look Once(YOLOV5)+ML.Net+Open Neural Network Exchange(ONNX) plumbing has loaded a timer with a configurable due time and period is started.

private async void ImageUpdateTimerCallback(object state)
{
	DateTime requestAtUtc = DateTime.UtcNow;

	// Just incase - stop code being called while photo already in progress
	if (_cameraBusy)
	{
		return;
	}
	_cameraBusy = true;

	_logger.LogInformation("Image processing start");

	try
	{
#if CAMERA_RASPBERRY_PI
		RaspberryPIImageCapture();
#endif
#if CAMERA_SECURITY
		SecurityCameraImageCapture();
#endif
		List<YoloPrediction> predictions;

		using (Image image = Image.FromFile(_applicationSettings.ImageCameraFilepath))
		{
			_logger.LogTrace("Prediction start");
			predictions = _scorer.Predict(image);
			_logger.LogTrace("Prediction done");

			OutputImageMarkup(image, predictions, _applicationSettings.ImageMarkedUpFilepath);
		}

		if (_logger.IsEnabled(LogLevel.Trace))
		{
			_logger.LogTrace("Predictions {0}", predictions.Select(p => new { p.Label.Name, p.Score }));
		}

		var predictionsValid = predictions.Where(p => p.Score >= _applicationSettings.PredictionScoreThreshold).Select(p => p.Label.Name);

		// Count up the number of each class detected in the image
		var predictionsTally = predictionsValid.GroupBy(p => p)
				.Select(p => new
				{
					Label = p.Key,
					Count = p.Count()
				});

		if (_logger.IsEnabled(LogLevel.Information))
		{
			_logger.LogInformation("Predictions tally before {0}", predictionsTally.ToList());
		}

		// Add in any missing counts the cloudy side is expecting
		if (_applicationSettings.PredictionLabelsMinimum != null)
		{
			foreach( String label in _applicationSettings.PredictionLabelsMinimum)
			{
				if (!predictionsTally.Any(c=>c.Label == label ))
				{
					predictionsTally = predictionsTally.Append(new {Label = label, Count = 0 });
				}
			}
		}

		if (_logger.IsEnabled(LogLevel.Information))
		{
			_logger.LogInformation("Predictions tally after {0}", predictionsTally.ToList());
		}

		if ((_applicationSettings.PredictionLabelsOfInterest == null) || (predictionsValid.Select(c => c).Intersect(_applicationSettings.PredictionLabelsOfInterest, StringComparer.OrdinalIgnoreCase).Any()))
		{
			JObject telemetryDataPoint = new JObject();

			foreach (var predictionTally in predictionsTally)
			{
				telemetryDataPoint.Add(predictionTally.Label, predictionTally.Count);
			}

			using (Message message = new Message(Encoding.ASCII.GetBytes(JsonConvert.SerializeObject(telemetryDataPoint))))
			{
				message.Properties.Add("iothub-creation-time-utc", requestAtUtc.ToString("s", CultureInfo.InvariantCulture));

				await _deviceClient.SendEventAsync(message);
			}
		}
	}
	catch (Exception ex)
	{
		_logger.LogError(ex, "Camera image download, post processing, or telemetry failed");
	}
	finally
	{
		_cameraBusy = false;
	}

	TimeSpan duration = DateTime.UtcNow - requestAtUtc;

	_logger.LogInformation("Image processing done {0:f2} sec", duration.TotalSeconds);
}

Using some Language Integrated Query (LINQ) code any predictions with a score < PredictionScoreThreshold are discarded. A count of the instances of each class is generated with some more LINQ code.

The PredictionLabelsMinimum(optional) is then used to add additional labels with a count of 0 to PredictionsTally so there are no missing datapoints. This is specifically for Azure IoT Central Dashboard so the graph lines are continuous.

Smartish Edge Camera Service console just after put bike in-front of the garage

If any of the list of valid predictions labels is in the PredictionLabelsOfInterest list (if the PredictionLabelsOfInterest is empty any label is a label of interest) the list of prediction class counts is used to populate a Newtonsoft JObject which is serialised to generate a Java Script Object Notation(JSON) Azure IoT Hub message payload.

The “automagic” graph scaling can be sub-optimal

The mentalstack/yolov5-net and NuGet have been incredibly useful and MentalStack team have done a marvelous job building and supporting this project.

The test-rig consisted of a Unv ADZK-10 Security Camera, Power over Ethernet(PoE) and my HP Prodesk 400G4 DM (i7-8700T).

Smartish Edge Camera – Azure IoT Hub

The SmartEdgeCameraAzureIoTService application uses the same You Only Look Once(YOLOV5) + ML.Net + Open Neural Network Exchange(ONNX) plumbing as the SmartEdgeCameraAzureStorageService.

If there is an object with a label in the PredictionLabelsOfInterest list, a tally of each of the different object classes is sent to an Azure IoT Hub.

"Application": {
  "DeviceID": "",
  "ImageTimerDue": "0.00:00:15",
  "ImageTimerPeriod": "0.00:00:30",

  "ImageCameraFilepath": "ImageCamera.jpg",

  "YoloV5ModelPath": "YoloV5/yolov5s.onnx",

  "PredicitionScoreThreshold": 0.7,
  "PredictionLabelsOfInterest": [
    "person"
  ],
}

The Azure IoT hub can configured via a Shared Access Signature(SAS) device policy connection string or the Azure IoT Hub Device Provisioning Service(DPS)

Cars and bicycles in my backyard with no object(s) of interest
SmartEdgeCameraAzureIoTService no object(s) of interest
Cars and bicycles in my backyard with one object of interest
SmartEdgeCameraAzureIoTService one object of interest
Azure IoT Explorer Telemetry with one object of interest

After the You Only Look Once(YOLOV5)+ML.Net+Open Neural Network Exchange(ONNX) plumbing has loaded a timer with a configurable due time and period is started. Using some Language Integrated Query (LINQ) code any predictions with a score < PredictionScoreThreshold are discarded, then the list of predictions is checked to see if there are any in the PredictionLabelsOfInterest. If there are any matching predictions a count of the instances of each class is generated with more LINQ code.

private async void ImageUpdateTimerCallback(object state)
{
	DateTime requestAtUtc = DateTime.UtcNow;

	// Just incase - stop code being called while photo already in progress
	if (_cameraBusy)
	{
		return;
	}
	_cameraBusy = true;

	_logger.LogInformation("Image processing start");

	try
	{
#if CAMERA_RASPBERRY_PI
		RaspberryPIImageCapture();
#endif
#if CAMERA_SECURITY
		SecurityCameraImageCapture();
#endif
		List<YoloPrediction> predictions;

		using (Image image = Image.FromFile(_applicationSettings.ImageCameraFilepath))
		{
			_logger.LogTrace("Prediction start");
			predictions = _scorer.Predict(image);
			_logger.LogTrace("Prediction done");
		}

		if (_logger.IsEnabled(LogLevel.Trace))
		{
			_logger.LogTrace("Predictions {0}", predictions.Select(p => new { p.Label.Name, p.Score }));
		}

		var predictionsOfInterest = predictions.Where(p => p.Score > _applicationSettings.PredicitionScoreThreshold)
										.Select(c => c.Label.Name)
										.Intersect(_applicationSettings.PredictionLabelsOfInterest, StringComparer.OrdinalIgnoreCase);

		if (predictionsOfInterest.Any())
		{
			if (_logger.IsEnabled(LogLevel.Trace))
			{
				_logger.LogTrace("Predictions of interest {0}", predictionsOfInterest.ToList());
			}

			var predictionsTally = predictions.GroupBy(p => p.Label.Name)
									.Select(p => new
									{
										Label = p.Key,
										Count = p.Count()
									});

			if (_logger.IsEnabled(LogLevel.Information))
			{
				_logger.LogInformation("Predictions tally {0}", predictionsTally.ToList());
			}

			JObject telemetryDataPoint = new JObject();

			foreach (var predictionTally in predictionsTally)
			{
				telemetryDataPoint.Add(predictionTally.Label, predictionTally.Count);
			}

			using (Message message = new Message(Encoding.ASCII.GetBytes(JsonConvert.SerializeObject(telemetryDataPoint))))
			{
				message.Properties.Add("iothub-creation-time-utc", requestAtUtc.ToString("s", CultureInfo.InvariantCulture));

				await _deviceClient.SendEventAsync(message);
			}
		}
	}
	catch (Exception ex)
	{
		_logger.LogError(ex, "Camera image download, post processing, telemetry failed");
	}
	finally
	{
		_cameraBusy = false;
	}

	TimeSpan duration = DateTime.UtcNow - requestAtUtc;

	_logger.LogInformation("Image processing done {0:f2} sec", duration.TotalSeconds);
}

The list of prediction class counts is used to populate a Newtonsoft JObject which serialised to generate a Java Script Object Notation(JSON) payload for an Azure IoT Hub message.

The test-rig consisted of a Unv ADZK-10 Security Camera, Power over Ethernet(PoE) and my HP Prodesk 400G4 DM (i7-8700T)

Smartish Edge Camera – Azure Storage Service

The AzureIoTSmartEdgeCameraService was a useful proof of concept(PoC) but the codebase was starting to get unwieldy so it has been split into the SmartEdgeCameraAzureStorageService and SmartEdgeCameraAzureIoTService.

The initial ML.Net +You only look once V5(YoloV5) project uploaded raw (effectively a time lapse camera) and marked-up (with searchable tags) images to Azure Storage. But, after using it in a “real” project I found…

  • The time-lapse functionality which continually uploaded images wasn’t that useful. I have another standalone application which has that functionality.
  • If an object with a label in the “PredictionLabelsOfInterest” and a score greater than PredicitionScoreThreshold was detected it was useful to have the option to upload the camera and/or marked-up (including objects below the threshold) image(s).
  • Having both camera and marked-up images tagged so they were searchable with an application like Azure Storage Explorer was very useful.
Security Camera Image
Security Camera image with bounding boxes around all detected objects
Azure Storage Explorer filter for images containing 1 person

After the You Only Look Once(YOLOV5)+ML.Net+Open Neural Network Exchange(ONNX) plumbing has loaded a timer with a configurable due time and period was started.

private async void ImageUpdateTimerCallback(object state)
{
	DateTime requestAtUtc = DateTime.UtcNow;

	// Just incase - stop code being called while photo already in progress
	if (_cameraBusy)
	{
		return;
	}
	_cameraBusy = true;

	_logger.LogInformation("Image processing start");

	try
	{
#if CAMERA_RASPBERRY_PI
		RaspberryPIImageCapture();
#endif
#if CAMERA_SECURITY
		SecurityCameraImageCapture();
#endif
		List<YoloPrediction> predictions;

		using (Image image = Image.FromFile(_applicationSettings.ImageCameraFilepath))
		{
			_logger.LogTrace("Prediction start");
			predictions = _scorer.Predict(image);
			_logger.LogTrace("Prediction done");

			OutputImageMarkup(image, predictions, _applicationSettings.ImageMarkedUpFilepath);
		}

		if (_logger.IsEnabled(LogLevel.Trace))
		{
			_logger.LogTrace("Predictions {0}", predictions.Select(p => new { p.Label.Name, p.Score }));
		}

		var predictionsOfInterest = predictions.Where(p => p.Score > _applicationSettings.PredicitionScoreThreshold).Select(c => c.Label.Name).Intersect(_applicationSettings.PredictionLabelsOfInterest, StringComparer.OrdinalIgnoreCase);
		if (_logger.IsEnabled(LogLevel.Trace))
		{
			_logger.LogTrace("Predictions of interest {0}", predictionsOfInterest.ToList());
		}

		var predictionsTally = predictions.Where(p => p.Score >= _applicationSettings.PredicitionScoreThreshold)
									.GroupBy(p => p.Label.Name)
									.Select(p => new
									{
										Label = p.Key,
										Count = p.Count()
									});

		if (predictionsOfInterest.Any())
		{
			BlobUploadOptions blobUploadOptions = new BlobUploadOptions()
			{
				Tags = new Dictionary<string, string>()
			};

			foreach (var predicition in predictionsTally)
			{
				blobUploadOptions.Tags.Add(predicition.Label, predicition.Count.ToString());
			}

			if (_applicationSettings.ImageCameraUpload)
			{
				_logger.LogTrace("Image camera upload start");

				string imageFilenameCloud = string.Format(_azureStorageSettings.ImageCameraFilenameFormat, requestAtUtc);

				await _imagecontainerClient.GetBlobClient(imageFilenameCloud).UploadAsync(_applicationSettings.ImageCameraFilepath, blobUploadOptions);

				_logger.LogTrace("Image camera upload done");
			}

			if (_applicationSettings.ImageMarkedupUpload)
			{
				_logger.LogTrace("Image marked-up upload start");

				string imageFilenameCloud = string.Format(_azureStorageSettings.ImageMarkedUpFilenameFormat, requestAtUtc);

				await _imagecontainerClient.GetBlobClient(imageFilenameCloud).UploadAsync(_applicationSettings.ImageMarkedUpFilepath, blobUploadOptions);

				_logger.LogTrace("Image marked-up upload done");
			}
		}

		if (_logger.IsEnabled(LogLevel.Information))
		{
			_logger.LogInformation("Predictions tally {0}", predictionsTally.ToList());
		}
	}
	catch (Exception ex)
	{
		_logger.LogError(ex, "Camera image download, post procesing, image upload, or telemetry failed");
	}
	finally
	{
		_cameraBusy = false;
	}

	TimeSpan duration = DateTime.UtcNow - requestAtUtc;

	_logger.LogInformation("Image processing done {0:f2} sec", duration.TotalSeconds);
}

The test-rig consisted of a Unv ADZK-10 Security Camera, Power over Ethernet(PoE) module, D-Link Switch and a Raspberry Pi 4B 8G, or ASUS PE100A, or my HP Prodesk 400G4 DM (i7-8700T)

Security Camera Image download times

Excluding the first download it takes on average 0.16 secs to download a security camera image with my network setup.

Development PC image download and processing console

The HP Prodesk 400G4 DM (i7-8700T) took on average 1.16 seconds to download an image from the camera, run the model, and upload the two images to Azure Storage

Raspberry PI 4B image download and processing console

The Raspberry Pi 4B 8G took on average 2.18 seconds to download an image from the camera, run the model, then upload the two images to Azure Storage

ASUS PE100A image download an processing console

The ASUS PE100A took on average 3.79 seconds to download an image from the camera, run the model, then upload the two images to Azure Storage.

Smartish Edge Camera – Azure Storage Image Tags

This ML.Net +You only look once V5(YoloV5) + RaspberryPI 4B project uploads raw camera and marked up (with searchable tags) images to Azure Storage.

Raspberry PI 4 B backyard test rig

My backyard test-rig consists of a Unv ADZK-10 Security Camera, Power over Ethernet(PoE) module, D-Link Switch and a Raspberry Pi 4B 8G.

{
   ...

  "Application": {
    "DeviceId": "edgecamera",
...
    "PredicitionScoreThreshold": 0.7,
    "PredictionLabelsOfInterest": [
      "bicycle",
      "person",
      "car"
    ],
    "OutputImageMarkup": true
  },
...
  "AzureStorage": {
    "ConnectionString": "FhisIsNotTheConnectionStringYouAreLookingFor",
    "ImageCameraFilenameFormat": "{0:yyyyMMdd}/camera/{0:HHmmss}.jpg",
    "ImageMarkedUpFilenameFormat": "{0:yyyyMMdd}/markedup/{0:HHmmss}.jpg"
  }
}

After the You Only Look Once(YOLOV5)+ML.Net+Open Neural Network Exchange(ONNX) plumbing has loaded a timer with a configurable due time and period is started.

private async void ImageUpdateTimerCallback(object state)
{
	DateTime requestAtUtc = DateTime.UtcNow;

	// Just incase - stop code being called while photo already in progress
	if (_cameraBusy)
	{
		return;
	}
	_cameraBusy = true;

	_logger.LogInformation("Image processing start");

	try
	{
#if CAMERA_RASPBERRY_PI
		RaspberryPIImageCapture();
#endif
#if CAMERA_SECURITY
		SecurityCameraImageCapture();
#endif
		if (_applicationSettings.ImageCameraUpload)
		{
			_logger.LogTrace("Image camera upload start");

			string imageFilenameCloud = string.Format(_azureStorageSettings.ImageCameraFilenameFormat, requestAtUtc);

			await _imagecontainerClient.GetBlobClient(imageFilenameCloud).UploadAsync(_applicationSettings.ImageCameraFilepath, true);

			_logger.LogTrace("Image camera upload done");
		}

		List<YoloPrediction> predictions;

		using (Image image = Image.FromFile(_applicationSettings.ImageCameraFilepath))
		{
			_logger.LogTrace("Prediction start");
			predictions = _scorer.Predict(image);
			_logger.LogTrace("Prediction done");

			OutputImageMarkup(image, predictions, _applicationSettings.ImageMarkedUpFilepath);
		}

		if (_logger.IsEnabled(LogLevel.Trace))
		{
			_logger.LogTrace("Predictions {0}", predictions.Select(p => new { p.Label.Name, p.Score }));
		}

		var predictionsOfInterest = predictions.Where(p => p.Score > _applicationSettings.PredicitionScoreThreshold).Select(c => c.Label.Name).Intersect(_applicationSettings.PredictionLabelsOfInterest, StringComparer.OrdinalIgnoreCase);
		if (_logger.IsEnabled(LogLevel.Trace))
		{
			_logger.LogTrace("Predictions of interest {0}", predictionsOfInterest.ToList());
		}

		var predictionsTally = predictions.Where(p => p.Score >= _applicationSettings.PredicitionScoreThreshold)
									.GroupBy(p => p.Label.Name)
									.Select(p => new
									{
										Label = p.Key,
										Count = p.Count()
									});

		if (_applicationSettings.ImageMarkedupUpload && predictionsOfInterest.Any())
		{
			_logger.LogTrace("Image marked-up upload start");

			string imageFilenameCloud = string.Format(_azureStorageSettings.ImageMarkedUpFilenameFormat, requestAtUtc);

			BlobUploadOptions blobUploadOptions = new BlobUploadOptions()
			{
				Tags = new Dictionary<string, string>()
			};

			foreach (var predicition in predictionsTally)
			{
				blobUploadOptions.Tags.Add(predicition.Label, predicition.Count.ToString());
			}

			BlobClient blobClient = _imagecontainerClient.GetBlobClient(imageFilenameCloud);

			await blobClient.UploadAsync(_applicationSettings.ImageMarkedUpFilepath, blobUploadOptions);

			_logger.LogTrace("Image marked-up upload done");
		}

		if (_logger.IsEnabled(LogLevel.Information))
		{
			_logger.LogInformation("Predictions tally {0}", predictionsTally.ToList());
		}
	}
	catch (Exception ex)
	{
		_logger.LogError(ex, "Camera image download, post procesing, image upload, or telemetry failed");
	}
	finally
	{
		_cameraBusy = false;
	}

	TimeSpan duration = DateTime.UtcNow - requestAtUtc;

	_logger.LogInformation("Image processing done {0:f2} sec", duration.TotalSeconds);
}
RaspberryPI 4B console application output

A marked up image is uploaded to Azure Storage if any of the objects detected (with a score greater than PredicitionScoreThreshold) is in the PredictionLabelsOfInterest list.

Single bicycle
Two bicycles
Three bicycles
Three bicycles with person in the foreground
Two bicycles with a person and dog in the foreground

I have added Tags to the images so they can be filtered with tools like Azure Storage Explorer.

All the camera images
All the marked up images with more than one bicycle
All the marked up images with more than two bicycles
All the marked up images with people and bicycles

Smartish Edge Camera – Azure Storage basics

This project is another reworked version of on my ML.Net YoloV5 + Camera + GPIO on ARM64 Raspberry PI which supports only the uploading of camera and marked up images to Azure Storage.

My backyard test-rig consists of a Unv IPC675LFW Pan Tilt Zoom(PTZ) Security Camera, Power over Ethernet(PoE) module, and a Raspberry Pi 4B 8G.

Raspberry PI 4 B backyard test rig

The application can be compiled with Raspberry PI V2 Camera or Unv Security Camera (The security camera configuration may work for other cameras/vendors).

The appsetings.json file has configuration options for the Azure Storage Account, DeviceID (Used for the Azure Blob storage container name), the list of object classes of interest (based on the YoloV5 image classes) , and the image blob storage file names (used to “bucket” images).

{
  "Logging": {
    "LogLevel": {
      "Default": "Information",
      "Microsoft": "Warning",
      "Microsoft.Hosting.Lifetime": "Information"
    }
  },

  "Application": {
    "DeviceId": "edgecamera",

    "ImageTimerDue": "0.00:00:15",
    "ImageTimerPeriod": "0.00:00:30",

    "ImageCameraFilepath": "ImageCamera.jpg",
    "ImageMarkedUpFilepath": "ImageMarkedup.jpg",

    "ImageCameraUpload": true,
    "ImageMarkedupUpload": true,

    "YoloV5ModelPath": "YoloV5/yolov5s.onnx",

    "PredicitionScoreThreshold": 0.7,
    "PredictionLabelsOfInterest": [
      "bicycle",
      "person",
      "car"
    ],
    "OutputImageMarkup": true
  },

  "SecurityCamera": {
    "CameraUrl": "",
    "CameraUserName": "",
    "CameraUserPassword": ""
  },

  "RaspberryPICamera": {
    "ProcessWaitForExit": 1000,
    "Rotation": 180
  },

  "AzureStorage": {
    "ConnectionString": "FhisIsNotTheConnectionStringYouAreLookingFor",
    "ImageCameraFilenameFormat": "{0:yyyyMMdd}/camera/{0:HHmmss}.jpg",
    "ImageMarkedUpFilenameFormat": "{0:yyyyMMdd}/markedup/{0:HHmmss}.jpg"
  }
}

Part of this refactor was injecting(DI) the logging and configuration dependencies.

public class Program
{
	public static void Main(string[] args)
	{
		CreateHostBuilder(args).Build().Run();
	}

	public static IHostBuilder CreateHostBuilder(string[] args) =>
		 Host.CreateDefaultBuilder(args)
			.ConfigureServices((hostContext, services) =>
			{
				services.Configure<ApplicationSettings>(hostContext.Configuration.GetSection("Application"));
				services.Configure<SecurityCameraSettings>(hostContext.Configuration.GetSection("SecurityCamera"));
				services.Configure<RaspberryPICameraSettings>(hostContext.Configuration.GetSection("RaspberryPICamera"));
				services.Configure<AzureStorageSettings>(hostContext.Configuration.GetSection("AzureStorage"));
			})
			.ConfigureLogging(logging =>
			{
				logging.ClearProviders();
				logging.AddSimpleConsole(c => c.TimestampFormat = "[HH:mm:ss.ff]");
			})
			.UseSystemd()
			.ConfigureServices((hostContext, services) =>
			{
			  services.AddHostedService<Worker>();
			});
		}
	}
}

After the You Only Look Once(YOLOV5)+ML.Net+Open Neural Network Exchange(ONNX) plumbing has loaded a timer with a configurable due time and period is started.

private async void ImageUpdateTimerCallback(object state)
{
	DateTime requestAtUtc = DateTime.UtcNow;

	// Just incase - stop code being called while photo already in progress
	if (_cameraBusy)
	{
		return;
	}
	_cameraBusy = true;

	_logger.LogInformation("Image processing start");

	try
	{
#if CAMERA_RASPBERRY_PI
		RaspberryPIImageCapture();
#endif
#if CAMERA_SECURITY
		SecurityCameraImageCapture();
#endif
		if (_applicationSettings.ImageCameraUpload)
		{
					await AzureStorageImageUpload(requestAtUtc, _applicationSettings.ImageCameraFilepath, 
 azureStorageSettings.ImageCameraFilenameFormat);
		}

		List<YoloPrediction> predictions;

		using (Image image = Image.FromFile(_applicationSettings.ImageCameraFilepath))
		{
			_logger.LogTrace("Prediction start");
			predictions = _scorer.Predict(image);
			_logger.LogTrace("Prediction done");

			OutputImageMarkup(image, predictions, _applicationSettings.ImageMarkedUpFilepath);
		}

		if (_logger.IsEnabled(LogLevel.Trace))
		{
			_logger.LogTrace("Predictions {0}", predictions.Select(p => new { p.Label.Name, p.Score }));
		}

		var predictionsOfInterest = predictions.Where(p => p.Score > _applicationSettings.PredicitionScoreThreshold).Select(c => c.Label.Name).Intersect(_applicationSettings.PredictionLabelsOfInterest, StringComparer.OrdinalIgnoreCase);
		if (_logger.IsEnabled(LogLevel.Trace))
		{
			_logger.LogTrace("Predictions of interest {0}", predictionsOfInterest.ToList());
		}

		if (_applicationSettings.ImageMarkedupUpload && predictionsOfInterest.Any())
		{
			await AzureStorageImageUpload(requestAtUtc, _applicationSettings.ImageMarkedUpFilepath, _azureStorageSettings.ImageMarkedUpFilenameFormat);
		}

		var predictionsTally = predictions.Where(p => p.Score >= _applicationSettings.PredicitionScoreThreshold)
									.GroupBy(p => p.Label.Name)
									.Select(p => new
									{
										Label = p.Key,
										Count = p.Count()
									});

		if (_logger.IsEnabled(LogLevel.Information))
		{
			_logger.LogInformation("Predictions tally {0}", predictionsTally.ToList());
		}
	}
	catch (Exception ex)
	{
		_logger.LogError(ex, "Camera image download, post procesing, image upload, or telemetry failed");
	}
	finally
	{
		_cameraBusy = false;
	}

	TimeSpan duration = DateTime.UtcNow - requestAtUtc;

	_logger.LogInformation("Image processing done {0:f2} sec", duration.TotalSeconds);
}

In the ImageUpdateTimerCallback method a camera image is captured (by my Raspberry Pi Camera Module 2 or IPC675LFW Security Camera) and written to the local file system.

Raspberry PI4B console displaying image processing and uploading

The MentalStack YoloV5 model ML.Net support library processes the camera image on the local filesystem. The prediction output (can be inspected with Netron) is parsed generating list of objects that have been detected, their Minimum Bounding Rectangle(MBR) and class.

Image from security camera
Azure IoT Storage Explorer displaying list of camera images

The list of predictions is post processed with a Language Integrated Query(LINQ) which filters out predictions with a score below a configurable threshold(PredicitionScoreThreshold) and returns a count of each class. If this list intersects with the configurable PredictionLabelsOfInterest a marked up image is uploaded to Azure Storage.

Image from security camera marked up with Minimum Bounding Boxes(MBRs)
Azure IoT Storage Explorer displaying list of marked up camera images

The current implementation is quite limited, the camera image upload, object detection and image upload if there are objects of interest is implemented in a single timer callback. I’m considering implementing two timers one for the uploading of camera images (time lapse camera) and the other for running the object detection process and uploading marked up images.

Marked up images are uploaded if any of the objects detected (with a score greater than PredicitionScoreThreshold) is in the PredictionLabelsOfInterest. I’m considering adding a PredicitionScoreThreshold and minimum count for individual prediction classes, and optionally marked up image upload only when the list of objects detected has changed.

Azure Smartish Edge Camera – The basics

This project builds on my ML.Net YoloV5 + Camera + GPIO on ARM64 Raspberry PI with the addition of basic support for Azure IoT Hubs, the Azure IoT Hub Device Provisioning Service(DPS), and Azure IoT Central.

My backyard test-rig has consists of a Unv ADZK-10 Security Camera, Power over Ethernet(PoE) module, and an ASUS PE100A.

Backyard test-rig

The application can be compiled with support for Azure IoT Connection strings or the Device Provisioning Service(DPS). The appsetings.json file has configuration options for Azure IoT Hub connection string or DPS Global Device Endpoint+ScopeID+Group Enrollment key.

{
  "ApplicationSettings": {
    "DeviceId": "NotTheEdgeCamera",

    "ImageTimerDue": "0.00:00:15",
    "ImageTimerPeriod": "0.00:00:30",

    "CameraUrl": "http://10.0.0.55:85/images/snapshot.jpg",
    "CameraUserName": ",,,",
    "CameraUserPassword": "...",

    "ButtonPinNumer": 6,
    "LedPinNumer": 5,

    "InputImageFilenameLocal": "InputLatest.jpg",
    "OutputImageFilenameLocal": "OutputLatest.jpg",

    "ProcessWaitForExit": 10000,

    "YoloV5ModelPath": "Assets/YoloV5/yolov5s.onnx",

    "PredicitionScoreThreshold": 0.5,

    "AzureIoTHubConnectionString": "...",

    "GlobalDeviceEndpoint": "global.azure-devices-provisioning.net",
    "AzureIoTHubDpsIDScope": "...",
    "AzureIoTHubDpsGroupEnrollmentKey": "..."
  }
}

After the You Only Look Once(YOLOV5)+ML.Net+Open Neural Network Exchange(ONNX) plumbing has loaded a timer with a configurable due time and period is started.

private static async void ImageUpdateTimerCallback(object state)
{
	DateTime requestAtUtc = DateTime.UtcNow;

	// Just incase - stop code being called while photo already in progress
	if (_cameraBusy)
	{
		return;
	}
	_cameraBusy = true;

	Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image processing start");

	try
	{
#if SECURITY_CAMERA
		Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Security Camera Image download start");
		SecurityCameraImageCapture();
		Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Security Camera Image download done");
#endif

#if RASPBERRY_PI_CAMERA
		Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Raspberry PI Image capture start");
		RaspberryPICameraImageCapture();
		Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Raspberry PI Image capture done");
#endif

		List<YoloPrediction> predictions;

		// Process the image on local file system
		using (Image image = Image.FromFile(_applicationSettings.InputImageFilenameLocal))
		{
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} YoloV5 inferencing start");
			predictions = _scorer.Predict(image);
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} YoloV5 inferencing done");

#if OUTPUT_IMAGE_MARKUP
			using (Graphics graphics = Graphics.FromImage(image))
			{
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image markup start");

				foreach (var prediction in predictions) // iterate predictions to draw results
				{
					double score = Math.Round(prediction.Score, 2);

					graphics.DrawRectangles(new Pen(prediction.Label.Color, 1), new[] { prediction.Rectangle });

					var (x, y) = (prediction.Rectangle.X - 3, prediction.Rectangle.Y - 23);

					graphics.DrawString($"{prediction.Label.Name} ({score})", new Font("Consolas", 16, GraphicsUnit.Pixel), new SolidBrush(prediction.Label.Color), new PointF(x, y));
				}

				image.Save(_applicationSettings.OutputImageFilenameLocal);

				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image markup done");
			}
#endif
		}

#if AZURE_IOT_HUB_CONNECTION || AZURE_IOT_HUB_DPS_CONNECTION
		await AzureIoTHubTelemetry(requestAtUtc, predictions);
#endif
	}
	catch (Exception ex)
	{
		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Camera image download, post procesing, image upload, or telemetry failed {ex.Message}");
	}
	finally
	{
		_cameraBusy = false;
	}

	TimeSpan duration = DateTime.UtcNow - requestAtUtc;

	Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image processing done {duration.TotalSeconds:f2} sec");
	Console.WriteLine();
}

In the ImageUpdateTimerCallback method a camera image is captured (Raspberry Pi Camera Module 2 or Unv ADZK-10 Security Camera) and written to the local file system.

SSH Connection to Azure PE100 running Smartish Camera application

The YoloV5 model ML.Net support library then loads the image and processes the prediction output (can be inspected with Netron) generating list of objects that have been detected, their Minimum Bounding Rectangle(MBR) and class.

public static async Task AzureIoTHubTelemetry(DateTime requestAtUtc, List<YoloPrediction> predictions)
{
	JObject telemetryDataPoint = new JObject();

	foreach (var predictionTally in predictions.Where(p => p.Score >= _applicationSettings.PredicitionScoreThreshold).GroupBy(p => p.Label.Name)
					.Select(p => new
					{
						Label = p.Key,
						Count = p.Count()
					}))
	{
		Console.WriteLine("  {0} {1}", predictionTally.Label, predictionTally.Count);

		telemetryDataPoint.Add(predictionTally.Label, predictionTally.Count);
	}

	try
	{
		using (Message message = new Message(Encoding.ASCII.GetBytes(JsonConvert.SerializeObject(telemetryDataPoint))))
		{
			message.Properties.Add("iothub-creation-time-utc", requestAtUtc.ToString("s", CultureInfo.InvariantCulture));

			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} AzureIoTHubClient SendEventAsync prediction information start");
			await _deviceClient.SendEventAsync(message);
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} AzureIoTHubClient SendEventAsync prediction information finish");
		}
	}
	catch (Exception ex)
	{
		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} AzureIoTHubClient SendEventAsync cow counting failed {ex.Message}");
	}
}

The list of predictions is post processed with a Language Integrated Query(LINQ) which filters out predictions with a score below a configurable threshold and returns a count of each class.

My backyard from the deck

The aggregated YoloV5 prediction results are then uploaded to an Azure IoT Hub or Azure IoT Central

Azure IoT Explorer Displaying message payloads from the Smartish Edge Camera
Azure IoT Central displaying message payloads from the Smartish Edge Camera

ML.Net YoloV5 + Camera + GPIO on ARM64 Raspberry PI

This project builds on my ML.Net YoloV5 + Camera on ARM64 Raspberry PI post and adds support for turning a Light Emitting Diode(LED) on if the label of any object detected in an image is in the PredictionLabelsOfInterest list.

{
  "ApplicationSettings": {
    "ImageTimerDue": "0.00:00:15",
    "ImageTimerPeriod": "0.00:00:30",

    "CameraUrl": "...",
    "CameraUserName": "..",
    "CameraUserPassword": "...",

    "LedPinNumer": 5,

    "InputImageFilenameLocal": "InputLatest.jpg",
    "OutputImageFilenameLocal": "OutputLatest.jpg",

    "ProcessWaitForExit": 10000,

    "YoloV5ModelPath": "Assets/yolov5/yolov5s.onnx",

    "PredicitionScoreThreshold": 0.5,

    "PredictionLabelsOfInterest": [
      "bicycle",
      "person",
      "bench"
    ]
  }
}

The test-rig has consists of a Unv ADZK-10 Security Camera, Power over Ethernet(PoE) module, D-Link 8 port switch, Raspberry PI 8G 4b with a Seeedstudio Grove-Base Hat for Raspberry Pi, and Grove-Blue LED Button.

Test-rig configuration

class Program
{
	private static Model.ApplicationSettings _applicationSettings;
	private static bool _cameraBusy = false;
	private static YoloScorer<YoloCocoP5Model> _scorer = null;
#if GPIO_SUPPORT
	private static GpioController _gpiocontroller;
#endif

	static async Task Main(string[] args)
	{
		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} YoloV5ObjectDetectionCamera starting");

		try
		{
			// load the app settings into configuration
			var configuration = new ConfigurationBuilder()
				 .AddJsonFile("appsettings.json", false, true)
				 .Build();

			_applicationSettings = configuration.GetSection("ApplicationSettings").Get<Model.ApplicationSettings>();

#if GPIO_SUPPORT
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} GPIO setup start");

			_gpiocontroller = new GpioController(PinNumberingScheme.Logical);

			_gpiocontroller.OpenPin(_applicationSettings.ButtonPinNumer, PinMode.InputPullDown);

			_gpiocontroller.OpenPin(_applicationSettings.LedPinNumer, PinMode.Output);
			_gpiocontroller.Write(_applicationSettings.LedPinNumer, PinValue.Low);

			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} GPIO setup done");
#endif

			_scorer = new YoloScorer<YoloCocoP5Model>(_applicationSettings.YoloV5ModelPath);

			Timer imageUpdatetimer = new Timer(ImageUpdateTimerCallback, null, _applicationSettings.ImageImageTimerDue, _applicationSettings.ImageTimerPeriod);

			Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} press <ctrl^c> to exit");
			Console.WriteLine();

			try
			{
				await Task.Delay(Timeout.Infinite);
			}
			catch (TaskCanceledException)
			{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Application shutown requested");
			}
		}
		catch (Exception ex)
		{
				Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Application shutown failure {ex.Message}", ex);
		}
	}

	private static void ImageUpdateTimerCallback(object state)
	{
		DateTime requestAtUtc = DateTime.UtcNow;

		// Just incase - stop code being called while photo already in progress
		if (_cameraBusy)
		{
			return;
		}
		_cameraBusy = true;

		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image processing start");

		try
		{
#if SECURITY_CAMERA
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Security Camera Image download start");

			NetworkCredential networkCredential = new NetworkCredential()
			{
				UserName = _applicationSettings.CameraUserName,
				Password = _applicationSettings.CameraUserPassword,
			};

			using (WebClient client = new WebClient())
			{
				client.Credentials = networkCredential;

				client.DownloadFile(_applicationSettings.CameraUrl, _applicationSettings.InputImageFilenameLocal);
			}
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Security Camera Image download done");
#endif

#if RASPBERRY_PI_CAMERA
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Raspberry PI Image capture start");

			using (Process process = new Process())
			{
				process.StartInfo.FileName = @"libcamera-jpeg";
				process.StartInfo.Arguments = $"-o {_applicationSettings.InputImageFilenameLocal} --nopreview -t1 --rotation 180";
				process.StartInfo.RedirectStandardError = true;

				process.Start();

				if (!process.WaitForExit(_applicationSettings.ProcessWaitForExit) || (process.ExitCode != 0))
				{
					Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image update failure {process.ExitCode}");
				}
			}

			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Raspberry PI Image capture done");
#endif

			List<YoloPrediction> predictions;

			// Process the image on local file system
			using (Image image = Image.FromFile(_applicationSettings.InputImageFilenameLocal))
			{
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} YoloV5 inferencing start");
				predictions = _scorer.Predict(image);
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} YoloV5 inferencing done");

#if OUTPUT_IMAGE_MARKUP
				using (Graphics graphics = Graphics.FromImage(image))
				{
					Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image markup start");

					foreach (var prediction in predictions) // iterate predictions to draw results
					{
						double score = Math.Round(prediction.Score, 2);

						graphics.DrawRectangles(new Pen(prediction.Label.Color, 1), new[] { prediction.Rectangle });

						var (x, y) = (prediction.Rectangle.X - 3, prediction.Rectangle.Y - 23);

						graphics.DrawString($"{prediction.Label.Name} ({score})", new Font("Consolas", 16, GraphicsUnit.Pixel), new SolidBrush(prediction.Label.Color), new PointF(x, y));
					}

					image.Save(_applicationSettings.OutputImageFilenameLocal);

					Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image markup done");
				}
#endif
			}

#if PREDICTION_CLASSES
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image classes start");
			foreach (var prediction in predictions)
			{
				Console.WriteLine($"  Name:{prediction.Label.Name} Score:{prediction.Score:f2} Valid:{prediction.Score > _applicationSettings.PredicitionScoreThreshold}");
			}
			Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss:fff} Image classes done");
#endif

#if PREDICTION_CLASSES_OF_INTEREST
			IEnumerable<string> predictionsOfInterest= predictions.Where(p=>p.Score > _applicationSettings.PredicitionScoreThreshold).Select(c => c.Label.Name).Intersect(_applicationSettings.PredictionLabelsOfInterest, StringComparer.OrdinalIgnoreCase);

			if (predictionsOfInterest.Any())
			{
				Console.WriteLine($" {DateTime.UtcNow:yy-MM-dd HH:mm:ss} Camera image comtains {String.Join(",", predictionsOfInterest)}");
			}

   #if GPIO_SUPPORT
		   if (predictionsOfInterest.Any())
			{
				_gpiocontroller.Write(_applicationSettings.LedPinNumer, PinValue.High);
			}
			else
			{
				_gpiocontroller.Write(_applicationSettings.LedPinNumer, PinValue.Low);
			}
	#endif
#endif
		}
		catch (Exception ex)
		{
			Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Camera image download, upload or post procesing failed {ex.Message}");
		}
		finally
		{
			_cameraBusy = false;
		}

		TimeSpan duration = DateTime.UtcNow - requestAtUtc;

		Console.WriteLine($"{DateTime.UtcNow:yy-MM-dd HH:mm:ss} Image processing done {duration.TotalSeconds:f2} sec");
		Console.WriteLine();
	}
}

The name of the digital output pin, input image, output image and yoloV5 model file names are configured in the appsettings.json file.

Mountain bike leaning against garage
YoloV5 based application console

The 22-01-31 06:52 “person” detection is me moving the mountain bike into position.

Marked up image of my mountain bike leaning against the garage

Summary

Once the YoloV5s model was loaded, inferencing was taking roughly 1.45 seconds. The application is starting to get a bit “nasty” so for the next version I’ll need to do some refactoring.