Tag Archives: Cayenne

The Things Network HTTP Integration Part3

Posted on by

When Serialisation goes bad- payload_fields

This is the third in a series of posts about building an HTTP Integration for a The Things Network(TTN) application.

In part 1 & part 2 I had been ignoring the payload_fields property of the Payload class. The documentation indicates that payload_fields property is populated when an uplink message is Decoded.

There is a built in decoder for Cayenne Low Power Payload(LPP) which looked like the simplest option to start with.

TTN Application integration payload decoder selection

I modified the Seeeduino LoRaWAN Over The Air Activation(OTAA) sample application and added the CayenneLPP library from Electronic Cats.

#include <LoRaWan.h>
#include <CayenneLPP.h> 

CayenneLPP lpp(64);
char buffer[256];

void setup(void)
{
    SerialUSB.begin(9600);
    while(!SerialUSB);

    lora.init();

    memset(buffer, 0, 256);
    lora.getVersion(buffer, 256, 1);
    SerialUSB.print("Ver:");
    SerialUSB.print(buffer); 
 
    memset(buffer, 0, 256);
    lora.getId(buffer, 256, 1);
    SerialUSB.print("ID:");
    SerialUSB.println(buffer);

    lora.setKey(NULL, NULL, "12345678901234567890123456789012");
    lora.setId(NULL, "1234567890123456", "1234567890123456");

    lora.setPort(10);
        
    lora.setDeciveMode(LWOTAA);
    lora.setDataRate(DR0, AS923);

    lora.setDutyCycle(false);
    lora.setJoinDutyCycle(false);
 
    lora.setPower(14);

    while(!lora.setOTAAJoin(JOIN, 10));
}
 
void loop(void)
{   
    bool result = false;

    lpp.reset ();

    // Original LPPv1 data types only these work
    // https://www.thethingsnetwork.org/docs/devices/arduino/api/cayennelpp.html
    // https://loranow.com/cayennelpp/
    //
    lpp.addAnalogInput( 0, 0.01234) ;
    lpp.addAnalogOutput( 0, 0.56789);
    lpp.addDigitalInput(0, false);    
    lpp.addDigitalInput(1, true);    
    
    lpp.addGPS (1, -43.5309, 172.6371, 6.192);
    lpp.addAccelerometer(0, 0.0, 0.0, 1.0);
    lpp.addGyrometer(1, 0.0,0.0,0.0);
    
    lpp.addLuminosity(0, 0);    // Pitch black
    lpp.addLuminosity(1, 8000); // 40w fluro   
    lpp.addPresence(0, 0);
    lpp.addPresence(1, 1);

    lpp.addBarometricPressure(0,0.0);
    lpp.addBarometricPressure(0,1013.25);
    lpp.addRelativeHumidity (0, 50.0);
    lpp.addRelativeHumidity (1, 60.0);
    lpp.addTemperature (0, -273.00);
    lpp.addTemperature (1, 0.0);
    lpp.addTemperature (2, 100.0);

    // Additional data types don't think any of these worked
    //lpp.addUnixTime(1, millis()); 
    //lpp.addGenericSensor(1, 1.23456);
    //lpp.addVoltage(1, 4.5);
    //lpp.addCurrent(0, 1.0);
    //lpp.addFrequency (1, 50); 
    //lpp.addPercentage(1, 50);
    //lpp.addAltitude(1, 20.5);
    //lpp.addPower(1, 1500);
    //lpp.addDistance(1, 120.0);
    //lpp.addEnergy(1, 2.345);
    //lpp.addDirection(1, -98.76);
    //lpp.addSwitch(0, 1);
    //lpp.addConcentration(0, 10);
    //lpp.addColour(1, 255, 255, 255);

    uint8_t *lppBuffer = lpp.getBuffer();
    uint8_t lppLen = lpp.getSize();

    SerialUSB.print("Length is: ");
    SerialUSB.println(lppLen);

    // Dump buffer content for debugging
    PrintHexBuffer (lppBuffer, lppLen);    

    //result = lora.transferPacket("Hello World!", 10);
    result = lora.transferPacket(lppBuffer, lppLen);

    if(result)
    {
        short length;
        short rssi;
 
        memset(buffer, 0, sizeof(buffer));
        length = lora.receivePacket(buffer, 256, &rssi);
 
        if(length)
        {
            SerialUSB.print("Length is: ");
            SerialUSB.println(length);
            SerialUSB.print("RSSI is: ");
            SerialUSB.println(rssi);
            SerialUSB.print("Data is: ");
            for(unsigned char i = 0; i < length; i ++)
            {
                SerialUSB.print("0x");
                SerialUSB.print(buffer[i], HEX);
                SerialUSB.print(" ");
            }
            SerialUSB.println();
        }
    }
    delay( 30000);
}

void PrintHexBuffer( uint8_t *buffer, uint8_t size )
{

    for( uint8_t i = 0; i < size; i++ )
    {
        if(buffer[i] < 0x10)
        {
            Serial.print('0');
        }
        SerialUSB.print( buffer[i], HEX );
        Serial.print(" ");
    }
    SerialUSB.println( );
}

I then copied and saved to files the payloads from the Azure Application Insights events generated when an uplink messages were processed.

{
   "app_id": "rak811wisnodetest",
   "dev_id": "seeeduinolorawan4",
   "hardware_serial": "1234567890123456",
   "port": 10,
   "counter": 1,
   "is_retry": true,
   "payload_raw": "AWcBEAFlAGQBAAEBAgAyAYgAqYgGIxgBJuw=",
   "payload_fields": {
      "analog_in_1": 0.5,
      "digital_in_1": 1,
      "gps_1": {
         "altitude": 755,
         "latitude": 4.34,
         "longitude": 40.22
      },
      "luminosity_1": 100,
      "temperature_1": 27.2
   },
   "metadata": {
      "time": "2020-08-28T10:41:04.496594225Z",
      "frequency": 923.4,
      "modulation": "LORA",
      "data_rate": "SF12BW125",
      "coding_rate": "4/5",
      "gateways": [
         {
            "gtw_id": "eui-b827ebfffe6c279d",
            "timestamp": 3971612260,
            "time": "2020-08-28T10:41:03.313471Z",
            "channel": 1,
            "rssi": -53,
            "snr": 11.2,
            "rf_chain": 0,
            "latitude": -43.49885,
            "longitude": 172.60095,
            "altitude": 25
         }
      ]
   },
   "downlink_url": "https://integrations.thethingsnetwork.org/ttn-eu/api/v2/down/rak811wisnodetest/azure-webapi-endpoint?key=ttn-account-v2.12345678901234567_12345_1234567-dduo"
}

I used JSON2Csharp to generate C# classes which would deserialise the above uplink message. 

// Third version of classes for unpacking HTTP payload 
public class Gps1V3
{
   public int altitude { get; set; }
   public double latitude { get; set; }
   public double longitude { get; set; }
}

public class PayloadFieldsV3
{
   public double analog_in_1 { get; set; }
   public int digital_in_1 { get; set; }
   public Gps1V3 gps_1 { get; set; }
   public int luminosity_1 { get; set; }
   public double temperature_1 { get; set; }
}

public class GatewayV3 
{
   public string gtw_id { get; set; }
   public ulong timestamp { get; set; }
   public DateTime time { get; set; }
   public int channel { get; set; }
   public int rssi { get; set; }
   public double snr { get; set; }
   public int rf_chain { get; set; }
   public double latitude { get; set; }
   public double longitude { get; set; }
   public int altitude { get; set; }
}

public class MetadataV3
{
   public string time { get; set; }
   public double frequency { get; set; }
   public string modulation { get; set; }
   public string data_rate { get; set; }
   public string coding_rate { get; set; }
   public List<GatewayV3> gateways { get; set; }
}

public class PayloadV3
{
   public string app_id { get; set; }
   public string dev_id { get; set; }
   public string hardware_serial { get; set; }
   public int port { get; set; }
   public int counter { get; set; }
   public bool is_retry { get; set; }
   public string payload_raw { get; set; }
   public PayloadFieldsV3 payload_fields { get; set; }
   public MetadataV3 metadata { get; set; }
   public string downlink_url { get; set; }
}

I added a new to controller to my application which used the generated classes to deserialise the body of the POST from the TTN Application Integration.

[Route("[controller]")]
[ApiController]
public class ClassSerialisationV3Fields : ControllerBase
{
   private static readonly ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);

   public string Index()
   {
      return "move along nothing to see";
   }

   [HttpPost]
   public IActionResult Post([FromBody] PayloadV3 payload)
   {
      // Check that the post data is good
      if (!this.ModelState.IsValid)
      {
         log.WarnFormat("ClassSerialisationV3Fields validation failed {0}", this.ModelState.Messages());

         return this.BadRequest(this.ModelState);
      }

      log.Info($"DevEUI:{payload.hardware_serial} Payload Base64:{payload.payload_raw} analog_in_1:{payload.payload_fields.analog_in_1} digital_in_1:{payload.payload_fields.digital_in_1} gps_1:{payload.payload_fields.gps_1.latitude},{payload.payload_fields.gps_1.longitude},{payload.payload_fields.gps_1.altitude} luminosity_1:{payload.payload_fields.luminosity_1} temperature_1:{payload.payload_fields.temperature_1}");

      return this.Ok();
   }
}

I then updated the TTN application integration to send messages to my new endpoint. In the body of the Application Insights events I could see the devEUI, port, and the payload fields had been extracted from the message. 

DevEUI:1234567890123456 Payload Base64:AWcBEAFlAGQBAAEBAgAyAYgAqYgGIxgBJuw= analog_in_1:0.5 digital_in_1:1 gps_1:4.34,40.22,755 luminosity_1:100 temperature_1:27.2

This arrangement was pretty nasty and sort of worked but in the “real world” would not have been viable. I would need to generate lots of custom classes for each application taking into account the channel numbers (e,g, analog_in_1,analog_in_2) and datatypes used.

I also explored which datatypes were supported by the TTN decoder, after some experimentation (Aug 2019) it looks like only the LPPV1 ones are.

  • AnalogInput
  • AnalogOutput
  • DigitalInput
  • DigitalOutput
  • GPS
  • Accelerometer
  • Gyrometer
  • Luminosity
  • Presence
  • BarometricPressure
  • RelativeHumidity
  • Temperature

What I need is a more flexible way to stored and decode payload_fields property..

myDevices Cayenne with MQTTnet

Posted on by

As I’m testing my Message Queue Telemetry Transport(MQTT) LoRa gateway I’m building a proof of concept(PoC) .Net core console application for each IoT platform I would like to support.

This PoC was to confirm that I could connect to the myDevices Cayenne MQTT API and format the topics and payloads correctly. The myDevices team have built many platform specific libraries that wrap the MQTT platform APIs to make integration for first timers easier (which is great). Though, as an experienced Bring Your Own Device(BYOD) client developer, I did find myself looking at the C/C++ code to figure out how to implement parts of my .Net test client.

The myDevices screen designer had “widgets” which generated commands for devices so I extended the test client implementation to see this worked.

The MQTT broker, username, password, client ID, channel number and optional subscription channel number are command line options. 

class Program
{
	private static IMqttClient mqttClient = null;
	private static IMqttClientOptions mqttOptions = null;
	private static string server;
	private static string username;
	private static string password;
	private static string clientId;
	private static string channelData;
	private static string channelSubscribe;

	static void Main(string[] args)
	{
		MqttFactory factory = new MqttFactory();
		mqttClient = factory.CreateMqttClient();

		if ((args.Length != 5) && (args.Length != 6))
		{
			Console.WriteLine("[MQTT Server] [UserName] [Password] [ClientID] [Channel]");
			Console.WriteLine("[MQTT Server] [UserName] [Password] [ClientID] [ChannelData] [ChannelSubscribe]");
			Console.WriteLine("Press <enter> to exit");
			Console.ReadLine();
			return;
		}

		server = args[0];
		username = args[1];
		password = args[2];
		clientId = args[3];
		channelData = args[4];

		if (args.Length == 5)
		{
			Console.WriteLine($"MQTT Server:{server} Username:{username} ClientID:{clientId} ChannelData:{channelData}");
		}

		if (args.Length == 6)
		{
			channelSubscribe = args[5];
			Console.WriteLine($"MQTT Server:{server} Username:{username} ClientID:{clientId} ChannelData:{channelData} ChannelSubscribe:{channelSubscribe}");
		}

		mqttOptions = new MqttClientOptionsBuilder()
			.WithTcpServer(server)
			.WithCredentials(username, password)
			.WithClientId(clientId)
			.WithTls()
			.Build();

		mqttClient.ConnectAsync(mqttOptions).Wait();

		if (args.Length == 6)
		{
			string topic = $"v1/{username}/things/{clientId}/cmd/{channelSubscribe}";

			Console.WriteLine($"Subscribe Topic:{topic}");
			mqttClient.SubscribeAsync(topic).Wait();
			// mqttClient.SubscribeAsync(topic, global::MQTTnet.Protocol.MqttQualityOfServiceLevel.AtLeastOnce).Wait(); 
			// Thought this might help with subscription but it didn't, looks like ACK might be broken in MQTTnet
			mqttClient.ApplicationMessageReceived += MqttClient_ApplicationMessageReceived;
		}
		mqttClient.Disconnected += MqttClient_Disconnected;

		string topicTemperatureData = $"v1/{username}/things/{clientId}/data/{channelData}";

		Console.WriteLine();

		while (true)
		{
			string value = "22." + DateTime.UtcNow.Millisecond.ToString();
			Console.WriteLine($"Publish Topic {topicTemperatureData}  Value {value}");

			var message = new MqttApplicationMessageBuilder()
				.WithTopic(topicTemperatureData)
				.WithPayload(value)
				.WithQualityOfServiceLevel(global::MQTTnet.Protocol.MqttQualityOfServiceLevel.AtLeastOnce)
				//.WithQualityOfServiceLevel(MQTTnet.Protocol.MqttQualityOfServiceLevel.ExactlyOnce) // Causes publish to hang
				.WithRetainFlag()
				.Build();

			Console.WriteLine("PublishAsync start");

			mqttClient.PublishAsync(message).Wait();
			Console.WriteLine("PublishAsync finish");
			Console.WriteLine();

			Thread.Sleep(30100);
		}
	}

	private static void MqttClient_ApplicationMessageReceived(object sender, MqttApplicationMessageReceivedEventArgs e)
	{
		Console.WriteLine($"ApplicationMessageReceived ClientId:{e.ClientId} Topic:{e.ApplicationMessage.Topic} Qos:{e.ApplicationMessage.QualityOfServiceLevel} Payload:{e.ApplicationMessage.ConvertPayloadToString()}");
		Console.WriteLine();
	}

	private static async void MqttClient_Disconnected(object sender, MqttClientDisconnectedEventArgs e)
	{
		Debug.WriteLine("Disconnected");
		await Task.Delay(TimeSpan.FromSeconds(5));

		try
		{
			await mqttClient.ConnectAsync(mqttOptions);
		}
		catch (Exception ex)
		{
			Debug.WriteLine("Reconnect failed {0}", ex.Message);
		}
	}
}

For this PoC I used the MQTTnet package which is available via NuGet. It appeared to be reasonably well supported and has had recent updates. There did appear to be some issues with myDevices Cayenne default quality of service (QoS) and the default QoS used by MQTTnet connections and also the acknowledgement of the receipt of published messages.

myDevices Cayenne .Net Core 2 client
Cayenne UI with graph, button and value widgets

Overall the initial configuration went ok, I found the dragging of widgets onto the overview screen had some issues (maybe the caching of control settings (I found my self refreshing the whole page every so often) and I couldn’t save a custom widget icon at all.

I put a button widget on the overview screen and associated it with a channel publication. The client received a message when the button was pressed

myDevices .Net Core 2 client displaying a received command message

But the button widget was disabled until the overview screen was manually refreshed.

Cayenne UI after button press

I need to revisit how I confirm that the actuator has been set to the desired value and the command completed.

Overall the myDevices Cayenne experience (April 2019) was a bit flaky with basic functionality like the saving of custom widget icons broken, updates of the real-time data viewer didn’t occur or were delayed, and there were other configuration screen update issues.