When Serialisation goes bad- payload_fields
This is the third in a series of posts about building an HTTP Integration for a The Things Network(TTN) application.
In part 1 & part 2 I had been ignoring the payload_fields property of the Payload class. The documentation indicates that payload_fields property is populated when an uplink message is Decoded.
There is a built in decoder for Cayenne Low Power Payload(LPP) which looked like the simplest option to start with.

I modified the Seeeduino LoRaWAN Over The Air Activation(OTAA) sample application and added the CayenneLPP library from Electronic Cats.
#include <LoRaWan.h> #include <CayenneLPP.h> CayenneLPP lpp(64); char buffer[256]; void setup(void) { SerialUSB.begin(9600); while(!SerialUSB); lora.init(); memset(buffer, 0, 256); lora.getVersion(buffer, 256, 1); SerialUSB.print("Ver:"); SerialUSB.print(buffer); memset(buffer, 0, 256); lora.getId(buffer, 256, 1); SerialUSB.print("ID:"); SerialUSB.println(buffer); lora.setKey(NULL, NULL, "12345678901234567890123456789012"); lora.setId(NULL, "1234567890123456", "1234567890123456"); lora.setPort(10); lora.setDeciveMode(LWOTAA); lora.setDataRate(DR0, AS923); lora.setDutyCycle(false); lora.setJoinDutyCycle(false); lora.setPower(14); while(!lora.setOTAAJoin(JOIN, 10)); } void loop(void) { bool result = false; lpp.reset (); // Original LPPv1 data types only these work // https://www.thethingsnetwork.org/docs/devices/arduino/api/cayennelpp.html // https://loranow.com/cayennelpp/ // lpp.addAnalogInput( 0, 0.01234) ; lpp.addAnalogOutput( 0, 0.56789); lpp.addDigitalInput(0, false); lpp.addDigitalInput(1, true); lpp.addGPS (1, -43.5309, 172.6371, 6.192); lpp.addAccelerometer(0, 0.0, 0.0, 1.0); lpp.addGyrometer(1, 0.0,0.0,0.0); lpp.addLuminosity(0, 0); // Pitch black lpp.addLuminosity(1, 8000); // 40w fluro lpp.addPresence(0, 0); lpp.addPresence(1, 1); lpp.addBarometricPressure(0,0.0); lpp.addBarometricPressure(0,1013.25); lpp.addRelativeHumidity (0, 50.0); lpp.addRelativeHumidity (1, 60.0); lpp.addTemperature (0, -273.00); lpp.addTemperature (1, 0.0); lpp.addTemperature (2, 100.0); // Additional data types don't think any of these worked //lpp.addUnixTime(1, millis()); //lpp.addGenericSensor(1, 1.23456); //lpp.addVoltage(1, 4.5); //lpp.addCurrent(0, 1.0); //lpp.addFrequency (1, 50); //lpp.addPercentage(1, 50); //lpp.addAltitude(1, 20.5); //lpp.addPower(1, 1500); //lpp.addDistance(1, 120.0); //lpp.addEnergy(1, 2.345); //lpp.addDirection(1, -98.76); //lpp.addSwitch(0, 1); //lpp.addConcentration(0, 10); //lpp.addColour(1, 255, 255, 255); uint8_t *lppBuffer = lpp.getBuffer(); uint8_t lppLen = lpp.getSize(); SerialUSB.print("Length is: "); SerialUSB.println(lppLen); // Dump buffer content for debugging PrintHexBuffer (lppBuffer, lppLen); //result = lora.transferPacket("Hello World!", 10); result = lora.transferPacket(lppBuffer, lppLen); if(result) { short length; short rssi; memset(buffer, 0, sizeof(buffer)); length = lora.receivePacket(buffer, 256, &rssi); if(length) { SerialUSB.print("Length is: "); SerialUSB.println(length); SerialUSB.print("RSSI is: "); SerialUSB.println(rssi); SerialUSB.print("Data is: "); for(unsigned char i = 0; i < length; i ++) { SerialUSB.print("0x"); SerialUSB.print(buffer[i], HEX); SerialUSB.print(" "); } SerialUSB.println(); } } delay( 30000); } void PrintHexBuffer( uint8_t *buffer, uint8_t size ) { for( uint8_t i = 0; i < size; i++ ) { if(buffer[i] < 0x10) { Serial.print('0'); } SerialUSB.print( buffer[i], HEX ); Serial.print(" "); } SerialUSB.println( ); }
I then copied and saved to files the payloads from the Azure Application Insights events generated when an uplink messages were processed.
{ "app_id": "rak811wisnodetest", "dev_id": "seeeduinolorawan4", "hardware_serial": "1234567890123456", "port": 10, "counter": 1, "is_retry": true, "payload_raw": "AWcBEAFlAGQBAAEBAgAyAYgAqYgGIxgBJuw=", "payload_fields": { "analog_in_1": 0.5, "digital_in_1": 1, "gps_1": { "altitude": 755, "latitude": 4.34, "longitude": 40.22 }, "luminosity_1": 100, "temperature_1": 27.2 }, "metadata": { "time": "2020-08-28T10:41:04.496594225Z", "frequency": 923.4, "modulation": "LORA", "data_rate": "SF12BW125", "coding_rate": "4/5", "gateways": [ { "gtw_id": "eui-b827ebfffe6c279d", "timestamp": 3971612260, "time": "2020-08-28T10:41:03.313471Z", "channel": 1, "rssi": -53, "snr": 11.2, "rf_chain": 0, "latitude": -43.49885, "longitude": 172.60095, "altitude": 25 } ] }, "downlink_url": "https://integrations.thethingsnetwork.org/ttn-eu/api/v2/down/rak811wisnodetest/azure-webapi-endpoint?key=ttn-account-v2.12345678901234567_12345_1234567-dduo" }
I used JSON2Csharp to generate C# classes which would deserialise the above uplink message.
// Third version of classes for unpacking HTTP payload public class Gps1V3 { public int altitude { get; set; } public double latitude { get; set; } public double longitude { get; set; } } public class PayloadFieldsV3 { public double analog_in_1 { get; set; } public int digital_in_1 { get; set; } public Gps1V3 gps_1 { get; set; } public int luminosity_1 { get; set; } public double temperature_1 { get; set; } } public class GatewayV3 { public string gtw_id { get; set; } public ulong timestamp { get; set; } public DateTime time { get; set; } public int channel { get; set; } public int rssi { get; set; } public double snr { get; set; } public int rf_chain { get; set; } public double latitude { get; set; } public double longitude { get; set; } public int altitude { get; set; } } public class MetadataV3 { public string time { get; set; } public double frequency { get; set; } public string modulation { get; set; } public string data_rate { get; set; } public string coding_rate { get; set; } public List<GatewayV3> gateways { get; set; } } public class PayloadV3 { public string app_id { get; set; } public string dev_id { get; set; } public string hardware_serial { get; set; } public int port { get; set; } public int counter { get; set; } public bool is_retry { get; set; } public string payload_raw { get; set; } public PayloadFieldsV3 payload_fields { get; set; } public MetadataV3 metadata { get; set; } public string downlink_url { get; set; } }
I added a new to controller to my application which used the generated classes to deserialise the body of the POST from the TTN Application Integration.
[Route("[controller]")] [ApiController] public class ClassSerialisationV3Fields : ControllerBase { private static readonly ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); public string Index() { return "move along nothing to see"; } [HttpPost] public IActionResult Post([FromBody] PayloadV3 payload) { // Check that the post data is good if (!this.ModelState.IsValid) { log.WarnFormat("ClassSerialisationV3Fields validation failed {0}", this.ModelState.Messages()); return this.BadRequest(this.ModelState); } log.Info($"DevEUI:{payload.hardware_serial} Payload Base64:{payload.payload_raw} analog_in_1:{payload.payload_fields.analog_in_1} digital_in_1:{payload.payload_fields.digital_in_1} gps_1:{payload.payload_fields.gps_1.latitude},{payload.payload_fields.gps_1.longitude},{payload.payload_fields.gps_1.altitude} luminosity_1:{payload.payload_fields.luminosity_1} temperature_1:{payload.payload_fields.temperature_1}"); return this.Ok(); } }
I then updated the TTN application integration to send messages to my new endpoint. In the body of the Application Insights events I could see the devEUI, port, and the payload fields had been extracted from the message.
DevEUI:1234567890123456 Payload Base64:AWcBEAFlAGQBAAEBAgAyAYgAqYgGIxgBJuw= analog_in_1:0.5 digital_in_1:1 gps_1:4.34,40.22,755 luminosity_1:100 temperature_1:27.2
This arrangement was pretty nasty and sort of worked but in the “real world” would not have been viable. I would need to generate lots of custom classes for each application taking into account the channel numbers (e,g, analog_in_1,analog_in_2) and datatypes used.
I also explored which datatypes were supported by the TTN decoder, after some experimentation (Aug 2019) it looks like only the LPPV1 ones are.
- AnalogInput
- AnalogOutput
- DigitalInput
- DigitalOutput
- GPS
- Accelerometer
- Gyrometer
- Luminosity
- Presence
- BarometricPressure
- RelativeHumidity
- Temperature
What I need is a more flexible way to stored and decode payload_fields property..