Azure IoT Hub MQTT LoRa Field Gateway

Back in April I started working on an MQTT LoRa Field gateway which was going to support a selection of different Software as a service(SaaS) Internet of Things IoT) platforms.

I now have a working Azure IoT Hub plug-in (Azure IoT Central support as planned as well) with the first iteration focused on Device to Cloud (D2C) messaging. In a future iteration I will add Cloud to Device messaging(C2D).

My applications use a lightweight, easy to implemented protocol which is intended for hobbyist and educational use rather than commercial applications (I have been working on a more secure version as yet another side project)

I have a number of sample Arduino with Dragino LoRa Shield for Arduino, MakerFabs Maduino, Dragino LoRa Mini Dev, M2M Low power Node and Netduino with Elecrow LoRa RFM95 Shield etc. clients. These work with both my platform specific (Adafruit.IO, Azure IoT Hub/Central) gateways and protocol specific field gateways.

Azure IoT Hub Device Explorer Data Display

When the application is first started it creates a minimal configuration file which should be downloaded, the missing information filled out, then uploaded using the File explorer in the Windows device portal.

{
  "MQTTUserName": "YourIoTHubHub.azure-devices.net/MQTTLoRa915MHz/api-version=2018-06-30",
  "MQTTPassword": "SharedAccessSignature sr=YourIoTHubHub.azure-devices.net%2Fdevices%2FMQTTLoRa915MHz&sig=123456789012345678901234567890123456789012345%3D&se=1574673583",
  "MQTTClientID": "MQTTLoRa915MHz",
  "MQTTServer": "YourIoTHubHub.azure-devices.net",
  "Address": "LoRaIoT1",
  "Frequency": 915000000.0,
  "MessageHandlerAssembly": "Mqtt.IoTCore.FieldGateway.LoRa.AzureIoTHub",
  "PlatformSpecificConfiguration": ""
}

The application logs debugging information to the Windows 10 IoT Core ETW logging Microsoft-Windows-Diagnostics-LoggingChannel

MQTT LoRa Gateway with Azure IoT Hub plug-in

The message handler uploads all values in an inbound messages in one MQTT message.

namespace devMobile.Mqtt.IoTCore.FieldGateway
{
   using System;
   using System.Diagnostics;
   using System.Text;
   using Windows.Foundation.Diagnostics;

   using devMobile.IoT.Rfm9x;
   using MQTTnet;
   using MQTTnet.Client;
   using Newtonsoft.Json.Linq;
   using Newtonsoft.Json;

   public class MessageHandler : IMessageHandler
   {
      private LoggingChannel Logging { get; set; }
      private IMqttClient MqttClient { get; set; }
      private Rfm9XDevice Rfm9XDevice { get; set; }
      private string PlatformSpecificConfiguration { get; set; }

      void IMessageHandler.Initialise(LoggingChannel logging, IMqttClient mqttClient, Rfm9XDevice rfm9XDevice, string platformSpecificConfiguration)
      {
         LoggingFields processInitialiseLoggingFields = new LoggingFields();

         this.Logging = logging;
         this.MqttClient = mqttClient;
         this.Rfm9XDevice = rfm9XDevice;
         this.PlatformSpecificConfiguration = platformSpecificConfiguration;
      }

      async void IMessageHandler.Rfm9XOnReceive(Rfm9XDevice.OnDataReceivedEventArgs e)
      {
         LoggingFields processReceiveLoggingFields = new LoggingFields();
         char[] sensorReadingSeparators = { ',' };
         char[] sensorIdAndValueSeparators = { ' ' };

         processReceiveLoggingFields.AddString("PacketSNR", e.PacketSnr.ToString("F1"));
         processReceiveLoggingFields.AddInt32("PacketRSSI", e.PacketRssi);
         processReceiveLoggingFields.AddInt32("RSSI", e.Rssi);

         string addressBcdText = BitConverter.ToString(e.Address);
         processReceiveLoggingFields.AddInt32("DeviceAddressLength", e.Address.Length);
         processReceiveLoggingFields.AddString("DeviceAddressBCD", addressBcdText);

         string messageText;
         try
         {
            messageText = UTF8Encoding.UTF8.GetString(e.Data);
            processReceiveLoggingFields.AddString("MessageText", messageText);
         }
         catch (Exception ex)
         {
            processReceiveLoggingFields.AddString("Exception", ex.ToString());
            this.Logging.LogEvent("PayloadProcess failure converting payload to text", processReceiveLoggingFields, LoggingLevel.Warning);
            return;
         }

         // Chop up the CSV text
         string[] sensorReadings = messageText.Split(sensorReadingSeparators, StringSplitOptions.RemoveEmptyEntries);
         if (sensorReadings.Length < 1)
         {
            this.Logging.LogEvent("PayloadProcess payload contains no sensor readings", processReceiveLoggingFields, LoggingLevel.Warning);
            return;
         }

         JObject payloadJObject = new JObject();

         JObject feeds = new JObject();

         // Chop up each sensor read into an ID & value
         foreach (string sensorReading in sensorReadings)
         {
            string[] sensorIdAndValue = sensorReading.Split(sensorIdAndValueSeparators, StringSplitOptions.RemoveEmptyEntries);

            // Check that there is an id & value
            if (sensorIdAndValue.Length != 2)
            {
               this.Logging.LogEvent("PayloadProcess payload invalid format", processReceiveLoggingFields, LoggingLevel.Warning);
               return;
            }

            string sensorId = string.Concat(addressBcdText, sensorIdAndValue[0]);
            string value = sensorIdAndValue[1];

            feeds.Add(sensorId.ToLower(), value);
         }
         payloadJObject.Add("feeds", feeds);

         string topic = $"devices/{MqttClient.Options.ClientId}/messages/events/";

         try
         {
            var message = new MqttApplicationMessageBuilder()
               .WithTopic(topic)
               .WithPayload(JsonConvert.SerializeObject(payloadJObject))
               .WithAtLeastOnceQoS()
               .Build();
            Debug.WriteLine(" {0:HH:mm:ss} MQTT Client PublishAsync start", DateTime.UtcNow);
            await MqttClient.PublishAsync(message);
            Debug.WriteLine(" {0:HH:mm:ss} MQTT Client PublishAsync finish", DateTime.UtcNow);

            this.Logging.LogEvent("PublishAsync Azure IoTHub payload", processReceiveLoggingFields, LoggingLevel.Information);
         }
         catch (Exception ex)
         {
            processReceiveLoggingFields.AddString("Exception", ex.ToString());
            this.Logging.LogEvent("PublishAsync Azure IoTHub payload", processReceiveLoggingFields, LoggingLevel.Error);
         }
      }

      void IMessageHandler.MqttApplicationMessageReceived(MqttApplicationMessageReceivedEventArgs e)
      {
         LoggingFields processReceiveLoggingFields = new LoggingFields();

         processReceiveLoggingFields.AddString("ClientId", e.ClientId);
#if DEBUG
         processReceiveLoggingFields.AddString("Payload", e.ApplicationMessage.ConvertPayloadToString());
#endif
         processReceiveLoggingFields.AddString("QualityOfServiceLevel", e.ApplicationMessage.QualityOfServiceLevel.ToString());
         processReceiveLoggingFields.AddBoolean("Retain", e.ApplicationMessage.Retain);
         processReceiveLoggingFields.AddString("Topic", e.ApplicationMessage.Topic);

         this.Logging.LogEvent("MqttApplicationMessageReceived topic not processed", processReceiveLoggingFields, LoggingLevel.Error);
      }

      void IMessageHandler.Rfm9xOnTransmit(Rfm9XDevice.OnDataTransmitedEventArgs e)
      {
      }
   }
}

The formatting of the username and generation of password are password are a bit awkward and will be fixed in a future refactoring. Along with regenerating the SAS connection token just before it is due to expire.

Azure IoT Hub with MQTTnet

As I’m testing my Message Queue Telemetry Transport(MQTT) LoRa gateway I’m building a proof of concept(PoC) .Net core console application for each IoT platform I would like to support.

This PoC was to confirm that my device could connect to the Microsoft Azure IoT Hub MQTT API then format topics and payloads correctly.

Azure IoT Hub MQTT Console Client

I had tried with a couple of different MQTT libraries from micro controllers and embedded devices without success. With the benefit of hindsight (plus this article) I think I had the SAS key format wrong.

The Azure IoT Hub MQTT broker requires only a server name (fully resolved CName), device ID and SAS Key.

   class Program
   {
      private static IMqttClient mqttClient = null;
      private static IMqttClientOptions mqttOptions = null;
      private static string server;
      private static string username;
      private static string password;
      private static string clientId;
      private static string topicD2C;
      private static string topicC2D;

      static void Main(string[] args)
      {
         MqttFactory factory = new MqttFactory();
         mqttClient = factory.CreateMqttClient();

         if (args.Length != 3)
         {
            Console.WriteLine("[AzureIoTHubHostName] [deviceID] [SASKey]");
            Console.WriteLine("Press <enter> to exit");
            Console.ReadLine();
            return;
         }

         server = args[0];
         clientId = args[1];
         sasKey= args[2];

         username = $"{server}/{clientId}/api-version=2018-06-30";
         topicD2C = $"devices/{clientId}/messages/events/";
         topicC2D = $"devices/{clientId}/messages/devicebound/#";

         Console.WriteLine($"MQTT Server:{server} Username:{username} ClientID:{clientId}");

         mqttOptions = new MqttClientOptionsBuilder()
            .WithTcpServer(server, 8883)
            .WithCredentials(username, sasKey)
            .WithClientId(clientId)
            .WithTls()
            .Build();

         mqttClient.UseDisconnectedHandler(new MqttClientDisconnectedHandlerDelegate(e => MqttClient_Disconnected(e)));
         mqttClient.UseApplicationMessageReceivedHandler(new MqttApplicationMessageReceivedHandlerDelegate(e => MqttClient_ApplicationMessageReceived(e)));
         mqttClient.ConnectAsync(mqttOptions).Wait();

         mqttClient.SubscribeAsync(topicC2D, MQTTnet.Protocol.MqttQualityOfServiceLevel.AtLeastOnce).GetAwaiter().GetResult();

         while (true)
         {
            JObject payloadJObject = new JObject();

            payloadJObject.Add("OfficeTemperature", "22." + DateTime.UtcNow.Millisecond.ToString());
            payloadJObject.Add("OfficeHumidity", (DateTime.UtcNow.Second + 40).ToString());

            string payload = JsonConvert.SerializeObject(payloadJObject);
            Console.WriteLine($"Topic:{topicD2C} Payload:{payload}");

            var message = new MqttApplicationMessageBuilder()
               .WithTopic(topicD2C)
               .WithPayload(payload)
               .WithAtLeastOnceQoS()
            .Build();

            Console.WriteLine("PublishAsync start");
            mqttClient.PublishAsync(message).Wait();
            Console.WriteLine("PublishAsync finish");

            Thread.Sleep(30100);
         }
      }

      private static void MqttClient_ApplicationMessageReceived(MqttApplicationMessageReceivedEventArgs e)
      {
         Console.WriteLine($"ClientId:{e.ClientId} Topic:{e.ApplicationMessage.Topic} Payload:{e.ApplicationMessage.ConvertPayloadToString()}");
      }

      private static async void MqttClient_Disconnected(MqttClientDisconnectedEventArgs e)
      {
         Debug.WriteLine("Disconnected");
         await Task.Delay(TimeSpan.FromSeconds(5));

         try
         {
            await mqttClient.ConnectAsync(mqttOptions);
         }
         catch (Exception ex)
         {
            Debug.WriteLine("Reconnect failed {0}", ex.Message);
         }
      }
   }

Overall the initial configuration went smoothly after I figured out the required Quality of Service (QoS) settings, and the SAS Key format.

Using the approach described in the Microsoft documentation I manually generated the SAS Key.(In my Netduino samples I have code for generating a SAS Key in my HTTPS Azure IoT Hub Client)

Azure Device Explorer Device Management
Azure Device Explorer SAS Key Generator

Once I had the configuration correct I could see telemetry from the device and send it messages.

Azure Device Explorer Data View

In a future post I will upload data to the Azure IoT Central for display. Then explore using a “module” attached to a device which maybe useful for my field gateway.

STM32 Blue Pill LoRaWAN node

A few weeks ago I ordered an STM32 Blue Pill LoRaWAN node from the M2M Shop on Tindie for evaluation. I have bought a few M2M client devices including a Low power LoRaWan Node Model A328, and Low power LoRaWan Node Model B1284 for projects and they have worked well. This one looked interesting as I had never used a maple like device before.

Bill of materials (Prices as at July 2019)

  • STM32 Blue Pill LoRaWAN node USD21
  • Grove – Temperature&Humidity Sensor USD11.5
  • Grove – 4 pin Female Jumper to Grove 4 pin Conversion Cable USD3.90

The two sockets on the main board aren’t Grove compatible so I used the 4 pin female to Grove 4 pin conversion cable to connect the temperature and humidity sensor.

STM32 Blue Pill LoRaWAN node test rig

I used a modified version of my Arduino client code which worked after I got the pin reset pin sorted and the female sockets in the right order.

/*
  Copyright ® 2019 July devMobile Software, All Rights Reserved

  THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY
  KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
  PURPOSE.
  
  Adapted from LoRa Duplex communication with Sync Word

  Sends temperature & humidity data from Seeedstudio 

  https://www.seeedstudio.com/Grove-Temperature-Humidity-Sensor-High-Accuracy-Min-p-1921.html

  To my Windows 10 IoT Core RFM 9X library

  https://blog.devmobile.co.nz/2018/09/03/rfm9x-iotcore-payload-addressing/
*/
#include <itoa.h>     
#include <SPI.h>     
#include <LoRa.h>

#include <TH02_dev.h>

#define DEBUG
//#define DEBUG_TELEMETRY
//#define DEBUG_LORA

// LoRa field gateway configuration (these settings must match your field gateway)
const char DeviceAddress[] = {"BLUEPILL"};

// Azure IoT Hub FieldGateway
const char FieldGatewayAddress[] = {"LoRaIoT1"}; 
const float FieldGatewayFrequency =  915000000.0;
const byte FieldGatewaySyncWord = 0x12 ;

// Bluepill hardware configuration
const int ChipSelectPin = PA4;
const int InterruptPin = PA0;
const int ResetPin = -1;

// LoRa radio payload configuration
const byte SensorIdValueSeperator = ' ' ;
const byte SensorReadingSeperator = ',' ;
const byte PayloadSizeMaximum = 64 ;
byte payload[PayloadSizeMaximum];
byte payloadLength = 0 ;

const int LoopDelaySeconds = 300 ;

// Sensor configuration
const char SensorIdTemperature[] = {"t"};
const char SensorIdHumidity[] = {"h"};


void setup()
{
  Serial.begin(9600);
#ifdef DEBUG
  while (!Serial);
#endif
  Serial.println("Setup called");

  Serial.println("LoRa setup start");

  // override the default chip select and reset pins
  LoRa.setPins(ChipSelectPin, ResetPin, InterruptPin);
  if (!LoRa.begin(FieldGatewayFrequency))
  {
    Serial.println("LoRa begin failed");
    while (true); // Drop into endless loop requiring restart
  }

  // Need to do this so field gateways pays attention to messsages from this device
  LoRa.enableCrc();
  LoRa.setSyncWord(FieldGatewaySyncWord);

#ifdef DEBUG_LORA
  LoRa.dumpRegisters(Serial);
#endif
  Serial.println("LoRa setup done.");

  PayloadHeader((byte*)FieldGatewayAddress, strlen(FieldGatewayAddress), (byte*)DeviceAddress, strlen(DeviceAddress));

 // Configure the Seeedstudio TH02 temperature & humidity sensor
  Serial.println("TH02 setup");
  TH02.begin();
  delay(100);
  Serial.println("TH02 Setup done");  

  Serial.println("Setup done");
}

void loop() {
  // read the value from the sensor:
  double temperature = TH02.ReadTemperature();
  double humidity = TH02.ReadHumidity();

  Serial.print("Humidity: ");
  Serial.print(humidity, 0);
  Serial.print(" %\t");
  Serial.print("Temperature: ");
  Serial.print(temperature, 1);
  Serial.println(" *C");

  PayloadReset();

  PayloadAdd(SensorIdHumidity, humidity, 0) ;
  PayloadAdd(SensorIdTemperature, temperature, 1) ;

  LoRa.beginPacket();
  LoRa.write(payload, payloadLength);
  LoRa.endPacket();

  Serial.println("Loop done");

  delay(LoopDelaySeconds * 1000);
}


void PayloadHeader( byte *to, byte toAddressLength, byte *from, byte fromAddressLength)
{
  byte addressesLength = toAddressLength + fromAddressLength ;

#ifdef DEBUG_TELEMETRY
  Serial.println("PayloadHeader- ");
  Serial.print( "To Address len:");
  Serial.print( toAddressLength );
  Serial.print( " From Address len:");
  Serial.print( fromAddressLength );
  Serial.print( " Addresses length:");
  Serial.print( addressesLength );
  Serial.println( );
#endif

  payloadLength = 0 ;

  // prepare the payload header with "To" Address length (top nibble) and "From" address length (bottom nibble)
  payload[payloadLength] = (toAddressLength << 4) | fromAddressLength ;
  payloadLength += 1;

  // Copy the "To" address into payload
  memcpy(&payload[payloadLength], to, toAddressLength);
  payloadLength += toAddressLength ;

  // Copy the "From" into payload
  memcpy(&payload[payloadLength], from, fromAddressLength);
  payloadLength += fromAddressLength ;
}


void PayloadAdd( const char *sensorId, float value, byte decimalPlaces)
{
  byte sensorIdLength = strlen( sensorId ) ;

#ifdef DEBUG_TELEMETRY
  Serial.println("PayloadAdd-float ");
  Serial.print( "SensorId:");
  Serial.print( sensorId );
  Serial.print( " sensorIdLen:");
  Serial.print( sensorIdLength );
  Serial.print( " Value:");
  Serial.print( value, decimalPlaces );
  Serial.print( " payloadLength:");
  Serial.print( payloadLength);
#endif

  memcpy( &payload[payloadLength], sensorId,  sensorIdLength) ;
  payloadLength += sensorIdLength ;
  payload[ payloadLength] = SensorIdValueSeperator;
  payloadLength += 1 ;
  payloadLength += strlen( dtostrf(value, -1, decimalPlaces, (char *)&payload[payloadLength]));
  payload[ payloadLength] = SensorReadingSeperator;
  payloadLength += 1 ;

#ifdef DEBUG_TELEMETRY
  Serial.print( " payloadLength:");
  Serial.print( payloadLength);
  Serial.println( );
#endif
}


void PayloadAdd( const char *sensorId, int value )
{
  byte sensorIdLength = strlen( sensorId ) ;

#ifdef DEBUG_TELEMETRY
  Serial.println("PayloadAdd-int ");
  Serial.print( "SensorId:");
  Serial.print( sensorId );
  Serial.print( " sensorIdLen:");
  Serial.print( sensorIdLength );
  Serial.print( " Value:");
  Serial.print( value );
  Serial.print( " payloadLength:");
  Serial.print( payloadLength);
#endif

  memcpy( &payload[payloadLength], sensorId,  sensorIdLength) ;
  payloadLength += sensorIdLength ;
  payload[ payloadLength] = SensorIdValueSeperator;
  payloadLength += 1 ;
  payloadLength += strlen( itoa( value, (char *)&payload[payloadLength], 10));
  payload[ payloadLength] = SensorReadingSeperator;
  payloadLength += 1 ;

#ifdef DEBUG_TELEMETRY
  Serial.print( " payloadLength:");
  Serial.print( payloadLength);
  Serial.println( );
#endif
}

void PayloadAdd( const char *sensorId, unsigned int value )
{
  byte sensorIdLength = strlen( sensorId ) ;

#ifdef DEBUG_TELEMETRY
  Serial.println("PayloadAdd-unsigned int ");
  Serial.print( "SensorId:");
  Serial.print( sensorId );
  Serial.print( " sensorIdLen:");
  Serial.print( sensorIdLength );
  Serial.print( " Value:");
  Serial.print( value );
  Serial.print( " payloadLength:");
  Serial.print( payloadLength);
#endif

  memcpy( &payload[payloadLength], sensorId,  sensorIdLength) ;
  payloadLength += sensorIdLength ;
  payload[ payloadLength] = SensorIdValueSeperator;
  payloadLength += 1 ;
  payloadLength += strlen( utoa( value, (char *)&payload[payloadLength], 10));
  payload[ payloadLength] = SensorReadingSeperator;
  payloadLength += 1 ;

#ifdef DEBUG_TELEMETRY
  Serial.print( " payloadLength:");
  Serial.print( payloadLength);
  Serial.println( );
#endif
}


void PayloadReset()
{
  byte fromAddressLength = payload[0] & 0xf ;
  byte toAddressLength = payload[0] >> 4 ;
  byte addressesLength = toAddressLength + fromAddressLength ;

  payloadLength = addressesLength + 1;

#ifdef DEBUG_TELEMETRY
  Serial.println("PayloadReset- ");
  Serial.print( "To Address len:");
  Serial.print( toAddressLength );
  Serial.print( " From Address len:");
  Serial.print( fromAddressLength );
  Serial.print( " Addresses length:");
  Serial.print( addressesLength );
  Serial.println( );
#endif
}

To get the application to compile I also had to include itoa.h rather than stdlib.h.

maple_loader v0.1
Resetting to bootloader via DTR pulse
[Reset via USB Serial Failed! Did you select the right serial port?]
Searching for DFU device [1EAF:0003]...
Assuming the board is in perpetual bootloader mode and continuing to attempt dfu programming...

dfu-util - (C) 2007-2008 by OpenMoko Inc.

Initially I had some problems deploying my software because I hadn’t followed the instructions and run the installation batch file.

14:03:56.946 -> Setup called
14:03:56.946 -> LoRa setup start
14:03:56.946 -> LoRa setup done.
14:03:56.946 -> TH02 setup
14:03:57.046 -> TH02 Setup done
14:03:57.046 -> Setup done
14:03:57.115 -> Humidity: 76 %	Temperature: 18.9 *C
14:03:57.182 -> Loop done
14:08:57.226 -> Humidity: 74 %	Temperature: 18.7 *C
14:08:57.295 -> Loop done
14:13:57.360 -> Humidity: 76 %	Temperature: 18.3 *C
14:13:57.430 -> Loop done
14:18:57.475 -> Humidity: 74 %	Temperature: 18.2 *C
14:18:57.544 -> Loop done
14:23:57.593 -> Humidity: 70 %	Temperature: 17.8 *C
14:23:57.662 -> Loop done
14:28:57.733 -> Humidity: 71 %	Temperature: 17.8 *C
14:28:57.802 -> Loop done
14:33:57.883 -> Humidity: 73 %	Temperature: 17.9 *C
14:33:57.952 -> Loop done
14:38:57.997 -> Humidity: 73 %	Temperature: 18.0 *C
14:38:58.066 -> Loop done
14:43:58.138 -> Humidity: 73 %	Temperature: 18.1 *C
14:43:58.208 -> Loop done
14:48:58.262 -> Humidity: 73 %	Temperature: 18.3 *C
14:48:58.331 -> Loop done
14:53:58.374 -> Humidity: 73 %	Temperature: 18.2 *C
14:53:58.444 -> Loop done
14:58:58.509 -> Humidity: 73 %	Temperature: 18.3 *C
14:58:58.578 -> Loop done
15:03:58.624 -> Humidity: 65 %	Temperature: 16.5 *C
15:03:58.694 -> Loop done
15:08:58.766 -> Humidity: 71 %	Temperature: 18.8 *C
15:08:58.836 -> Loop done
15:13:58.893 -> Humidity: 75 %	Temperature: 19.1 *C
15:13:58.963 -> Loop done

I configured the device to upload to my Azure IoT Hub/Azure IoT Central gateway and after getting the device name configuration right it has been running reliably for a couple of days

Azure IoT Central Temperature and humidity

The device was sitting outside on the deck and rapid increase in temperature is me bringing it inside.

Windows 10 IoT Core TPM SAS Token Expiry

This is for people who were searching for why the SAS token issued by the TPM on their Windows 10 IoT Core device is expiring much quicker than expected or might have noticed that something isn’t quite right with the “validity” period. (as at early May 2019). If you want to “follow along at home” the code I used is available on GitHub.

I found the SAS key was expiring in roughly 5 minutes and the validity period in the configuration didn’t appear to have any effect on how long the SAS token was valid.

10:04:16 Application started
...
10:04:27 SAS token needs renewing
10:04:30 SAS token renewed 
 10:04:30.984 AzureIoTHubClient SendEventAsync starting
 10:04:36.709 AzureIoTHubClient SendEventAsync starting
The thread 0x1464 has exited with code 0 (0x0).
 10:04:37.808 AzureIoTHubClient SendEventAsync finished
 10:04:37.808 AzureIoTHubClient SendEventAsync finished
The thread 0xb88 has exited with code 0 (0x0).
The thread 0x1208 has exited with code 0 (0x0).
The thread 0x448 has exited with code 0 (0x0).
The thread 0x540 has exited with code 0 (0x0).
 10:04:46.763 AzureIoTHubClient SendEventAsync starting
 10:04:47.051 AzureIoTHubClient SendEventAsync finished
The thread 0x10d8 has exited with code 0 (0x0).
The thread 0x6e0 has exited with code 0 (0x0).
The thread 0xf7c has exited with code 0 (0x0).
 10:04:56.808 AzureIoTHubClient SendEventAsync starting
 10:04:57.103 AzureIoTHubClient SendEventAsync finished
The thread 0xb8c has exited with code 0 (0x0).
The thread 0xc60 has exited with code 0 (0x0).
 10:05:06.784 AzureIoTHubClient SendEventAsync starting
 10:05:07.057 AzureIoTHubClient SendEventAsync finished
...
The thread 0x4f4 has exited with code 0 (0x0).
The thread 0xe10 has exited with code 0 (0x0).
The thread 0x3c8 has exited with code 0 (0x0).
 10:09:06.773 AzureIoTHubClient SendEventAsync starting
 10:09:07.044 AzureIoTHubClient SendEventAsync finished
The thread 0xf70 has exited with code 0 (0x0).
The thread 0x1214 has exited with code 0 (0x0).
 10:09:16.819 AzureIoTHubClient SendEventAsync starting
 10:09:17.104 AzureIoTHubClient SendEventAsync finished
The thread 0x1358 has exited with code 0 (0x0).
The thread 0x400 has exited with code 0 (0x0).
 10:09:26.802 AzureIoTHubClient SendEventAsync starting
 10:09:27.064 AzureIoTHubClient SendEventAsync finished
The thread 0x920 has exited with code 0 (0x0).
The thread 0x1684 has exited with code 0 (0x0).
The thread 0x4ec has exited with code 0 (0x0).
 10:09:36.759 AzureIoTHubClient SendEventAsync starting
'backgroundTaskHost.exe' (CoreCLR: CoreCLR_UWP_Domain): Loaded 'C:\Data\Programs\WindowsApps\Microsoft.NET.CoreFramework.Debug.2.2_2.2.27505.2_arm__8wekyb3d8bbwe\System.Net.Requests.dll'. Skipped loading symbols. Module is optimized and the debugger option 'Just My Code' is enabled.
'backgroundTaskHost.exe' (CoreCLR: CoreCLR_UWP_Domain): Loaded 'C:\Data\Programs\WindowsApps\Microsoft.NET.CoreFramework.Debug.2.2_2.2.27505.2_arm__8wekyb3d8bbwe\System.Net.WebSockets.dll'. Skipped loading symbols. Module is optimized and the debugger option 'Just My Code' is enabled.
Sending payload to AzureIoTHub failed:CONNECT failed: RefusedNotAuthorized

I went and looked at the NuGet package details and it seemed a bit old.

I have the RedGate Reflector plugin installed on my development box so I quickly disassembled the Microsoft.Devices.TPM assembly to see what was going on. The Reflector code is pretty readable and it wouldn’t take much “refactoring” to get it looking like “human” generated code.

public string GetSASToken(uint validity = 0xe10)
{
    string deviceId = this.GetDeviceId();
    string hostName = this.GetHostName();
    long num = (DateTime.get_Now().ToUniversalTime().ToFileTime() / 0x98_9680L) - 0x2_b610_9100L;
    string str3 = "";
    if ((hostName.Length > 0) && (deviceId.Length > 0))
    {
        object[] objArray1 = new object[] { hostName, "/devices/", deviceId, "\n", (long) num };
        byte[] bytes = new UTF8Encoding().GetBytes(string.Concat((object[]) objArray1));
        byte[] buffer2 = this.SignHmac(bytes);
        if (buffer2.Length != 0)
        {
            string str5 = this.AzureUrlEncode(Convert.ToBase64String(buffer2));
            object[] objArray2 = new object[] { "SharedAccessSignature sr=", hostName, "/devices/", deviceId, "&sig=", str5, "&se=", (long) num };
            str3 = string.Concat((object[]) objArray2);
        }
    }
    return str3;
}

The validity parameter appears to not used. Below is the current code from the Azure IoT CSharp SDK on GitHub repository and they are different, the validity is used.

public string GetSASToken(uint validity = 3600)
{
   const long WINDOWS_TICKS_PER_SEC = 10000000;
   const long EPOCH_DIFFERNECE = 11644473600;
   string deviceId = GetDeviceId();
   string hostName = GetHostName();
   long expirationTime = (DateTime.Now.ToUniversalTime().ToFileTime() / WINDOWS_TICKS_PER_SEC) - EPOCH_DIFFERNECE;
   expirationTime += validity;
   string sasToken = "";
   if ((hostName.Length > 0) && (deviceId.Length > 0))
   {
      // Encode the message to sign with the TPM
      UTF8Encoding utf8 = new UTF8Encoding();
      string tokenContent = hostName + "/devices/" + deviceId + "\n" + expirationTime;
      Byte[] encodedBytes = utf8.GetBytes(tokenContent);

      // Sign the message
      Byte[] hmac = SignHmac(encodedBytes);

      // if we got a signature foramt it
      if (hmac.Length > 0)
      {
         // Encode the output and assemble the connection string
         string hmacString = AzureUrlEncode(System.Convert.ToBase64String(hmac));
         sasToken = "SharedAccessSignature sr=" + hostName + "/devices/" + deviceId + "&sig=" + hmacString + "&se=" + expirationTime;
         }
   }
   return sasToken;
}

I went back and look at the Github history and it looks like a patch was applied after the NuGet packages were released in May 2016.

If you read from the TPM and get nothing make sure you’re using the right TPM slot number and have “System Management” checked in the capabilities tab of the application manifest.

I’m still not certain the validity is being applied correctly and will dig into in a future post.

Azure IOT Hub nRF24L01 Windows 10 IoT Core Field Gateway with BorosRF2

A couple of BorosRF2 Dual nRF24L01 Hats arrived earlier in the week. After some testing with my nRF24L01 Test application I have added compile-time configuration options for the two nRF24L01 sockets to my Azure IoT Hub nRF24L01 Field Gateway.

Boros RF2 with Dual nRF24L01 devices
public sealed class StartupTask : IBackgroundTask
{
   private const string ConfigurationFilename = "config.json";

   private const byte MessageHeaderPosition = 0;
   private const byte MessageHeaderLength = 1;

   // nRF24 Hardware interface configuration
#if CEECH_NRF24L01P_SHIELD
   private const byte RF24ModuleChipEnablePin = 25;
   private const byte RF24ModuleChipSelectPin = 0;
   private const byte RF24ModuleInterruptPin = 17;
#endif

#if BOROS_RF2_SHIELD_RADIO_0
   private const byte RF24ModuleChipEnablePin = 24;
   private const byte RF24ModuleChipSelectPin = 0;
   private const byte RF24ModuleInterruptPin = 27;
#endif

#if BOROS_RF2_SHIELD_RADIO_1
   private const byte RF24ModuleChipEnablePin = 25;
   private const byte RF24ModuleChipSelectPin = 1;
   private const byte RF24ModuleInterruptPin = 22;
#endif

private readonly LoggingChannel logging = new LoggingChannel("devMobile Azure IotHub nRF24L01 Field Gateway", null, new Guid("4bd2826e-54a1-4ba9-bf63-92b73ea1ac4a"));
private readonly RF24 rf24 = new RF24();

This version supports one nRF24L01 device socket active at a time.

Enabling both nRF24L01 device sockets broke outbound message routing in a prototype branch with cloud to device(C2D) messaging support. This functionality is part of an Over The Air(OTA) device provisioning implementation I’m working o.

Windows 10 IoT Core Time-Lapse Camera Azure IoT Hub Storage Revisited

In my previous post the application uploaded images to an Azure storage account associated with an Azure IoT Hub based on configuration file settings. The application didn’t use any of the Azure IoT Hub device management functionality like device twins and direct methods.

Time-lapse camera setup

In this version only the Azure IoT hub connection string and protocol to use are stored in the JSON configuration file.

{
  "AzureIoTHubConnectionString": "",
  "TransportType": "Mqtt",
} 

On startup the application uploads a selection of properties to the Azure IoT Hub to assist with support, fault finding etc.

// This is from the OS 
reportedProperties["Timezone"] = TimeZoneSettings.CurrentTimeZoneDisplayName;
reportedProperties["OSVersion"] = Environment.OSVersion.VersionString;
reportedProperties["MachineName"] = Environment.MachineName;
reportedProperties["ApplicationDisplayName"] = package.DisplayName;
reportedProperties["ApplicationName"] = packageId.Name;
reportedProperties["ApplicationVersion"] = string.Format($"{version.Major}.{version.Minor}.{version.Build}.{version.Revision}");

// Unique identifier from the hardware
SystemIdentificationInfo systemIdentificationInfo = SystemIdentification.GetSystemIdForPublisher();
using (DataReader reader = DataReader.FromBuffer(systemIdentificationInfo.Id))
{
   byte[] bytes = new byte[systemIdentificationInfo.Id.Length];
   reader.ReadBytes(bytes);
   reportedProperties["SystemId"] = BitConverter.ToString(bytes);
}

Azure Portal Device Properties

The Azure Storage file and folder name formats along with the image capture due and update periods are configured in the DeviceTwin properties. Initially I had some problems with the dynamic property types so had to .ToString and then Timespan.TryParse the periods.

Twin deviceTwin= azureIoTHubClient.GetTwinAsync().Result;

if (!deviceTwin.Properties.Desired.Contains("AzureImageFilenameLatestFormat"))
{
   this.logging.LogMessage("DeviceTwin.Properties AzureImageFilenameLatestFormat setting missing", LoggingLevel.Warning);
   return;
}
…
if (!deviceTwin.Properties.Desired.Contains("ImageUpdateDue") || !TimeSpan.TryParse(deviceTwin.Properties.Desired["ImageUpdateDue"].Value.ToString(), out imageUpdateDue))
{
   this.logging.LogMessage("DeviceTwin.Properties ImageUpdateDue setting missing or invalid format", LoggingLevel.Warning);
   return;
}
Azure Portal Device Settings

The application also supports two commands “ImageCapture’ and “DeviceReboot”. For testing I used Azure Device Explorer

After running the installer (available from GitHub) the application will create a default configuration file in

\User Folders\LocalAppData\PhotoTimerTriggerAzureIoTHubStorage-uwp_1.2.0.0_arm__nmn3tag1rpsaw\LocalState\

Which can be downloaded, modified then uploaded using the portal file explorer application. If you want to make the application run on device start-up the radio button below needs to be selected.

Windows 10 IoT Core Time-Lapse Camera Azure IoT Hub Storage

After building a couple of time lapse camera applications for Windows 10 IoT Core I built a version which uploads the images to the Azure storage account associated with an Azure IoT Hub.

I really wanted to be able to do a time-lapse video of a storm coming up the Canterbury Plains to Christchurch and combine it with the wind direction, windspeed, temperature and humidity data from my weather station which uploads data to Azure through my Azure IoT Hub LoRa field gateway.

Time-lapse camera setup

The application captures images with a configurable period after configurable start-up delay. The Azure storage root folder name is based on the device name in the Azure IoT Hub connection string. The folder(s) where the historic images are stored are configurable and the images can optionally be in monthly, daily, hourly etc. folders. The current image is stored in the root folder for the device and it’s name is configurable.

{
  "AzureIoTHubConnectionString": "",
  "TransportType": "Mqtt",
  "AzureImageFilenameFormatLatest": "latest.jpg",
  "AzureImageFilenameFormatHistory": "{0:yyMMdd}/{0:yyMMddHHmmss}.jpg",
  "ImageUpdateDueSeconds": 30,
  "ImageUpdatePeriodSeconds": 300
} 

With the above setup I have a folder for each device in the historic fiolder and the most recent image i.e. “latest.jpg” in the root folder. The file and folder names are assembled with a parameterised string.format . The parameter {0} is the current UTC time

Pay attention to your folder/file name formatting, I was tripped up by

  • mm – minutes vs. MM – months
  • hh – 12 hour clock vs. HH -24 hour clock

With 12 images every hour

The application logs events on start-up and every time a picture is taken

After running the installer (available from GitHub) the application will create a default configuration file in

User Folders\LocalAppData\PhotoTimerTriggerAzureIoTHubStorage-uwp_1.0.0.0_arm__nmn3tag1rpsaw\LocalState\

Which can be downloaded, modified then uploaded using the portal file explorer application. If you want to make the application run on device start-up the radio button below needs to be selected.

/*
    Copyright ® 2019 March devMobile Software, All Rights Reserved
 
    MIT License

…
*/
namespace devMobile.Windows10IotCore.IoT.PhotoTimerTriggerAzureIoTHubStorage
{
	using System;
	using System.IO;
	using System.Diagnostics;
	using System.Threading;

	using Microsoft.Azure.Devices.Client;
	using Microsoft.Extensions.Configuration;

	using Windows.ApplicationModel;
	using Windows.ApplicationModel.Background;
	using Windows.Foundation.Diagnostics;
	using Windows.Media.Capture;
	using Windows.Media.MediaProperties;
	using Windows.Storage;
	using Windows.System;
	
	public sealed class StartupTask : IBackgroundTask
	{
		private BackgroundTaskDeferral backgroundTaskDeferral = null;
		private readonly LoggingChannel logging = new LoggingChannel("devMobile Photo Timer Azure IoT Hub Storage", null, new Guid("4bd2826e-54a1-4ba9-bf63-92b73ea1ac4a"));
		private DeviceClient azureIoTHubClient = null;
		private const string ConfigurationFilename = "appsettings.json";
		private Timer ImageUpdatetimer;
		private MediaCapture mediaCapture;
		private string azureIoTHubConnectionString;
		private TransportType transportType;
		private string azureStorageimageFilenameLatestFormat;
		private string azureStorageImageFilenameHistoryFormat;
		private const string ImageFilenameLocal = "latest.jpg";
		private volatile bool cameraBusy = false;

		public void Run(IBackgroundTaskInstance taskInstance)
		{
			StorageFolder localFolder = ApplicationData.Current.LocalFolder;
			int imageUpdateDueSeconds;
			int imageUpdatePeriodSeconds;

			this.logging.LogEvent("Application starting");

			// Log the Application build, OS version information etc.
			LoggingFields startupInformation = new LoggingFields();
			startupInformation.AddString("Timezone", TimeZoneSettings.CurrentTimeZoneDisplayName);
			startupInformation.AddString("OSVersion", Environment.OSVersion.VersionString);
			startupInformation.AddString("MachineName", Environment.MachineName);

			// This is from the application manifest 
			Package package = Package.Current;
			PackageId packageId = package.Id;
			PackageVersion version = packageId.Version;
			startupInformation.AddString("ApplicationVersion", string.Format($"{version.Major}.{version.Minor}.{version.Build}.{version.Revision}"));

			try
			{
				// see if the configuration file is present if not copy minimal sample one from application directory
				if (localFolder.TryGetItemAsync(ConfigurationFilename).AsTask().Result == null)
				{
					StorageFile templateConfigurationfile = Package.Current.InstalledLocation.GetFileAsync(ConfigurationFilename).AsTask().Result;
					templateConfigurationfile.CopyAsync(localFolder, ConfigurationFilename).AsTask();

					this.logging.LogMessage("JSON configuration file missing, templated created", LoggingLevel.Warning);
					return;
				}

				IConfiguration configuration = new ConfigurationBuilder().AddJsonFile(Path.Combine(localFolder.Path, ConfigurationFilename), false, true).Build();

				azureIoTHubConnectionString = configuration.GetSection("AzureIoTHubConnectionString").Value;
				startupInformation.AddString("AzureIoTHubConnectionString", azureIoTHubConnectionString);

				transportType = (TransportType)Enum.Parse( typeof(TransportType), configuration.GetSection("TransportType").Value);
				startupInformation.AddString("TransportType", transportType.ToString());

				azureStorageimageFilenameLatestFormat = configuration.GetSection("AzureImageFilenameFormatLatest").Value;
				startupInformation.AddString("ImageFilenameLatestFormat", azureStorageimageFilenameLatestFormat);

				azureStorageImageFilenameHistoryFormat = configuration.GetSection("AzureImageFilenameFormatHistory").Value;
				startupInformation.AddString("ImageFilenameHistoryFormat", azureStorageImageFilenameHistoryFormat);

				imageUpdateDueSeconds = int.Parse(configuration.GetSection("ImageUpdateDueSeconds").Value);
				startupInformation.AddInt32("ImageUpdateDueSeconds", imageUpdateDueSeconds);

				imageUpdatePeriodSeconds = int.Parse(configuration.GetSection("ImageUpdatePeriodSeconds").Value);
				startupInformation.AddInt32("ImageUpdatePeriodSeconds", imageUpdatePeriodSeconds);
			}
			catch (Exception ex)
			{
				this.logging.LogMessage("JSON configuration file load or settings retrieval failed " + ex.Message, LoggingLevel.Error);
				return;
			}

			try
			{
				azureIoTHubClient = DeviceClient.CreateFromConnectionString(azureIoTHubConnectionString, transportType);
			}
			catch (Exception ex)
			{
				this.logging.LogMessage("AzureIOT Hub connection failed " + ex.Message, LoggingLevel.Error);
				return;
			}

			try
			{
				mediaCapture = new MediaCapture();
				mediaCapture.InitializeAsync().AsTask().Wait();
			}
			catch (Exception ex)
			{
				this.logging.LogMessage("Camera configuration failed " + ex.Message, LoggingLevel.Error);
				return;
			}

			ImageUpdatetimer = new Timer(ImageUpdateTimerCallback, null, new TimeSpan(0, 0, imageUpdateDueSeconds), new TimeSpan(0, 0, imageUpdatePeriodSeconds));

			this.logging.LogEvent("Application started", startupInformation);

			//enable task to continue running in background
			backgroundTaskDeferral = taskInstance.GetDeferral();
		}

		private async void ImageUpdateTimerCallback(object state)
		{
			DateTime currentTime = DateTime.UtcNow;
			Debug.WriteLine($"{DateTime.UtcNow.ToLongTimeString()} Timer triggered");

			// Just incase - stop code being called while photo already in progress
			if (cameraBusy)
			{
				return;
			}
			cameraBusy = true;

			try
			{
				using (Windows.Storage.Streams.InMemoryRandomAccessStream captureStream = new Windows.Storage.Streams.InMemoryRandomAccessStream())
				{
					await mediaCapture.CapturePhotoToStreamAsync(ImageEncodingProperties.CreateJpeg(), captureStream);
					await captureStream.FlushAsync();
#if DEBUG
					IStorageFile photoFile = await KnownFolders.PicturesLibrary.CreateFileAsync(ImageFilenameLocal, CreationCollisionOption.ReplaceExisting);
					ImageEncodingProperties imageProperties = ImageEncodingProperties.CreateJpeg();
					await mediaCapture.CapturePhotoToStorageFileAsync(imageProperties, photoFile);
#endif

					string azureFilenameLatest = string.Format(azureStorageimageFilenameLatestFormat, currentTime);
					string azureFilenameHistory = string.Format(azureStorageImageFilenameHistoryFormat, currentTime);

					LoggingFields imageInformation = new LoggingFields();
					imageInformation.AddDateTime("TakenAtUTC", currentTime);
#if DEBUG
					imageInformation.AddString("LocalFilename", photoFile.Path);
#endif
					imageInformation.AddString("AzureFilenameLatest", azureFilenameLatest);
					imageInformation.AddString("AzureFilenameHistory", azureFilenameHistory);
					this.logging.LogEvent("Saving image(s) to Azure storage", imageInformation);

					// Update the latest image in storage
					if (!string.IsNullOrWhiteSpace(azureFilenameLatest))
					{
						captureStream.Seek(0);
						Debug.WriteLine("AzureIoT Hub latest image upload start");
						await azureIoTHubClient.UploadToBlobAsync(azureFilenameLatest, captureStream.AsStreamForRead());
						Debug.WriteLine("AzureIoT Hub latest image upload done");
					}

					// Upload the historic image to storage
					if (!string.IsNullOrWhiteSpace(azureFilenameHistory))
					{
						captureStream.Seek(0);
						Debug.WriteLine("AzureIoT Hub historic image upload start");
						await azureIoTHubClient.UploadToBlobAsync(azureFilenameHistory, captureStream.AsStreamForRead());
						Debug.WriteLine("AzureIoT Hub historic image upload done");
					}
				}
			}
			catch (Exception ex)
			{
				this.logging.LogMessage("Camera photo save or AzureIoTHub storage upload failed " + ex.Message, LoggingLevel.Error);
			}
			finally
			{
				cameraBusy = false;
			}
		}
	}
}

The images in Azure Storage could then be assembled into a video using a tool like Time Lapse Creator or processed with Azure Custom Vision Service.