Out stupiding myself

This is the forth in a series of posts about building an HTTP Integration for a The Things Network(TTN) application.

Unpacking the payload_fields property was causing me some issues. I tried many different approaches but they all failed.

public class PayloadV4
{
   public string app_id { get; set; }
   public string dev_id { get; set; }
   public string hardware_serial { get; set; }
   public int port { get; set; }
   public int counter { get; set; }
   public bool is_retry { get; set; }
   public string payload_raw { get; set; }
   //public JsonObject payload_fields { get; set; }
   //public JObject payload_fields { get; set; }
   //public JToken payload_fields { get; set; }
   //public JContainer payload_fields { get; set; }
   //public dynamic payload_fields { get; set; }
   public Object payload_fields { get; set; }
   public MetadataV4 metadata { get; set; }
   public string downlink_url { get; set; }
}

I tried using the excellent JsonSubTypes library to build a polymorphic convertor, which failed.

...
public class PolymorphicJsonConverter : JsonConverter
{
   public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
   {
      JObject item = JObject.Load(reader);
      var type = item["type"].Value<string>();

      if (type == "PayloadFieldDigitalInput")
      {
         return item.ToObject<PayloadFieldDigitalInput>();
      }
      else if (type == "PayloadFieldDigitalInput")
      {
         return item.ToObject<PayloadFieldDigitalOutput>();
      }
      else if (type == "PayloadFieldAnalogInput")
      {
         return item.ToObject<PayloadFieldDigitalOutput>();
      }
      else if (type == "PayloadFieldAnalogInput")
      {
         return item.ToObject<PayloadFieldDigitalOutput>();
      }
      else
      {
         return null;
      }
    }
...
}

It was about this point I figured that I was down a very deep rabbit hole and I should just embrace my “stupid”.

I realised I shouldn’t unpack the payload as the number of generated classes required and the complexity of other approaches was going to rapidly get out of hand. Using an Object and recursively traversing its contents with System.Text.Json looked like a viable approach.

public class GatewayV4 
{
   public string gtw_id { get; set; }
   public ulong timestamp { get; set; }
   public DateTime time { get; set; }
   public int channel { get; set; }
   public int rssi { get; set; }
   public double snr { get; set; }
   public int rf_chain { get; set; }
   public double latitude { get; set; }
   public double longitude { get; set; }
   public int altitude { get; set; }
}

public class MetadataV4
{
   public string time { get; set; }
   public double frequency { get; set; }
   public string modulation { get; set; }
   public string data_rate { get; set; }
   public string coding_rate { get; set; }
   public List<GatewayV4> gateways { get; set; }
}

public class PayloadV4
{
   public string app_id { get; set; }
   public string dev_id { get; set; }
   public string hardware_serial { get; set; }
   public int port { get; set; }
   public int counter { get; set; }
   public bool is_retry { get; set; }
   public string payload_raw { get; set; }
   // finally settled on an Object
   public Object payload_fields { get; set; }
   public MetadataV4 metadata { get; set; }
   public string downlink_url { get; set; }
}

So, I added yet another new to controller to my application to deserialise the body of the POST from the TTN Application Integration.

[Route("[controller]")]
[ApiController]
public class ClassSerialisationV4Fields : ControllerBase
{
   private static readonly ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);

   public string Index()
   {
      return "move along nothing to see";
   }

   [HttpPost]
   public IActionResult Post([FromBody] PayloadV4 payload)
   {
      string payloadFieldsUnpacked = string.Empty;
         
      // Check that the post data is good
      if (!this.ModelState.IsValid)
      {
         log.WarnFormat("ClassSerialisationV4Fields validation failed {0}", this.ModelState.Messages());

         return this.BadRequest(this.ModelState);
      }

      JsonElement jsonElement = (JsonElement)payload.payload_fields;
      foreach (var property in jsonElement.EnumerateObject())
      {
         // Special handling for nested properties
         if (property.Name.StartsWith("gps_") || property.Name.StartsWith("accelerometer_") || property.Name.StartsWith("gyrometer_"))
         {
            payloadFieldsUnpacked += $"Property Name:{property.Name}\r\n";
            JsonElement gpsElement = (JsonElement)property.Value;
            foreach (var gpsProperty in gpsElement.EnumerateObject())
            {
               payloadFieldsUnpacked += $" Property Name:{gpsProperty.Name} Property Value:{gpsProperty.Value}\r\n";
            }
         }
         else
         {
            payloadFieldsUnpacked += $"Property Name:{property.Name} Property Value:{property.Value}\r\n";
         }
      }

      log.Info(payloadFieldsUnpacked);

      return this.Ok();
   }
}

In the body of the events in Azure Application Insights I could see messages and the format looked fine for simple payloads

Property Name:analog_in_1 Property Value:0.5
Property Name:digital_in_1 Property Value:1
Property Name:gps_1
 Property Name:altitude Property Value:755
 Property Name:latitude Property Value:4.34
 Property Name:longitude Property Value:40.22
Property Name:luminosity_1 Property Value:100
Property Name:temperature_1 Property Value:27.2

Then with payloads with lots of nested fields.

Property Name:accelerometer_0
 Property Name:x Property Value:0
 Property Name:y Property Value:0
 Property Name:z Property Value:1
Property Name:gps_1
 Property Name:alt Property Value:6.19
 Property Name:lat Property Value:-43.5309
 Property Name:lon Property Value:172.6371
Property Name:gyrometer_1
 Property Name:x Property Value:0
 Property Name:y Property Value:0
 Property Name:z Property Value:0

To make it faster to debug the unpacking of messages I built another “nasty” console application.

class Program
{
   static void Main(string[] args)
   {
      try
      {
         using (StreamReader r = new StreamReader(args[0]))
         {
            Payload payload = JsonConvert.DeserializeObject<Payload>(File.ReadAllText(args[0]));

            JObject payloadFields = (JObject)payload.payload_fields;

            foreach (JProperty child in payloadFields.Children())
            {
               EnumerateChildren(0, child);
            }
         }
      }
      catch (Exception ex)
      {
         Console.WriteLine(ex.Message);
      }

      Console.WriteLine();
      Console.WriteLine("Press <enter> to exit");

      Console.ReadLine();
   }

   static void EnumerateChildren(int indent, JToken token)
   {
      string prepend = string.Empty.PadLeft(indent);

      if (token is JProperty)
         if (token.First is JValue)
         {
            JProperty property = (JProperty)token;
            Console.WriteLine($"{prepend} Name:{property.Name} Value:{property.Value}");
         }
         else
         {
            JProperty property = (JProperty)token;
            Console.WriteLine($"{prepend}Name:{property.Name}");
            indent = indent + 3;
         }

      foreach (JToken token2 in token.Children())
      {
         EnumerateChildren(indent, token2);
      }
   }
}

The application displayed the structure of the object nicely indented so it was easier to visualise.

This application was largely re-written as part of the next post as a result of a System.Text.Json limitation.