Sometimes there is no easy way to build a “list of lists” using the contents of multiple database tables. I have run into this problem a few times especially when building webby services which query the database of a “legacy” (aka. production) system.
Retrieving a list of StockGroups and their StockItems from the World Wide Importers database was one of the better “real world” examples I could come up with.
SQL Server Management Studio Diagram showing relationships of tables
There is a fair bit of duplication (StockGroupID, StockGroupName) in the results set
SQL Server Management Studio StockItems-StockItemStockGroups-StockGroups query and results
There were 442 rows in the results set and 227 StockItems in the database so I ordered the query results by StockItemID and confirmed that there were many StockItems in several StockGroups.
public class StockItemListDtoV1
{
public int Id { get; set; }
public string Name { get; set; }
public decimal RecommendedRetailPrice { get; set; }
public decimal TaxRate { get; set; }
}
public class StockGroupStockItemsListDto
{
StockGroupStockItemsListDto()
{
StockItems = new List<StockItemListDto>();
}
public int StockGroupID { get; set; }
public string StockGroupName { get; set; }
public List<StockItemListDto> StockItems { get; set; }
}
My initial version uses a Generic List for a StockGroup’s StockItems which is most probably not a good idea.
[Route("api/[controller]")]
[ApiController]
public class InvoiceQuerySplitOnController : ControllerBase
{
private readonly string connectionString;
private readonly ILogger<InvoiceQuerySplitOnController> logger;
public InvoiceQuerySplitOnController(IConfiguration configuration, ILogger<InvoiceQuerySplitOnController> logger)
{
this.connectionString = configuration.GetConnectionString("WorldWideImportersDatabase");
this.logger = logger;
}
[HttpGet]
public async Task<ActionResult<IAsyncEnumerable<StockGroupStockItemsListDto>>> Get()
{
IEnumerable<StockGroupStockItemsListDto> response = null;
try
{
using (SqlConnection db = new SqlConnection(this.connectionString))
{
var stockGroups = await db.QueryAsync<StockGroupStockItemsListDto, StockItemListDto, StockGroupStockItemsListDto>(
sql: @"SELECT [StockGroups].[StockGroupID] as 'StockGroupID'" +
",[StockGroups].[StockGroupName]" +
",[StockItems].StockItemID as 'ID'" +
",[StockItems].StockItemName as 'Name'" +
",[StockItems].TaxRate" +
",[StockItems].RecommendedRetailPrice " +
"FROM [Warehouse].[StockGroups] " +
"INNER JOIN[Warehouse].[StockItemStockGroups] ON ([StockGroups].[StockGroupID] = [StockItemStockGroups].[StockGroupID])" +
"INNER JOIN[Warehouse].[StockItems] ON ([Warehouse].[StockItemStockGroups].[StockItemID] = [StockItems].[StockItemID])",
(stockGroup, stockItem) =>
{
// Not certain I think using a List<> here is a good idea...
stockGroup.StockItems.Add(stockItem);
return stockGroup;
},
splitOn: "ID",
commandType: CommandType.Text);
response = stockGroups.GroupBy(p => p.StockGroupID).Select(g =>
{
var groupedStockGroup = g.First();
groupedStockGroup.StockItems = g.Select(p => p.StockItems.Single()).ToList();
return groupedStockGroup;
});
}
}
catch (SqlException ex)
{
logger.LogError(ex, "Retrieving S, Invoice Lines or Stock Item Transactions");
return this.StatusCode(StatusCodes.Status500InternalServerError);
}
return this.Ok(response);
}
The MultiMapper syntax always trips me up and it usually takes a couple of attempts to get it to work.
List of StockGroups with StockItems
I have extended my DapperTransient module adding WithRetry versions of the 14 MultiMapper methods.
My current “day job” is building applications for managing portfolios of foreign currency instruments. A portfolio can contain many different types of instrument (Forwards, Options, Swaps etc.). One of the “optimisations” we use is retrieving all the different types of instruments in a portfolio with one stored procedure call.
SQL Server Management Studio Dependency viewer
The closest scenario I could come up with using the World Wide Importers database was retrieving a summary of all the information associated with an Invoice for display on a single screen.
CREATE PROCEDURE [Sales].[InvoiceSummaryGetV1](@InvoiceID as int)
AS
BEGIN
SELECT [InvoiceID]
-- ,[CustomerID]
-- ,[BillToCustomerID]
,[OrderID]
,[Invoices].[DeliveryMethodID]
,[DeliveryMethodName]
-- ,[ContactPersonID]
-- ,[AccountsPersonID]
,[SalespersonPersonID] as SalesPersonID
,[SalesPerson].[PreferredName] as SalesPersonName
-- ,[PackedByPersonID]
,[InvoiceDate]
,[CustomerPurchaseOrderNumber]
,[IsCreditNote]
,[CreditNoteReason]
,[Comments]
,[DeliveryInstructions]
-- ,[InternalComments]
-- ,[TotalDryItems]
-- ,[TotalChillerItems]
,[DeliveryRun]
,[RunPosition] as DeliveryRunPosition
,[ReturnedDeliveryData] as DeliveryData
,[ConfirmedDeliveryTime] as DeliveredAt
,[ConfirmedReceivedBy] as DeliveredTo
-- ,[LastEditedBy]
-- ,[LastEditedWhen]
FROM [Sales].[Invoices]
INNER JOIN [Application].[People] as SalesPerson ON (Invoices.[SalespersonPersonID] = [SalesPerson].[PersonID])
INNER JOIN [Application].[DeliveryMethods] as DeliveryMethod ON (Invoices.[DeliveryMethodID] = DeliveryMethod.[DeliveryMethodID])
WHERE ([Invoices].[InvoiceID] = @InvoiceID)
SELECT [InvoiceLineID]
,[InvoiceID]
,[StockItemID]
,[Description] as StockItemDescription
,[InvoiceLines].[PackageTypeID]
,[PackageType].[PackageTypeName]
,[Quantity]
,[UnitPrice]
,[TaxRate]
,[TaxAmount]
-- ,[LineProfit]
,[ExtendedPrice]
-- ,[LastEditedBy]
-- ,[LastEditedWhen]
FROM [Sales].[InvoiceLines]
INNER JOIN [Warehouse].[PackageTypes] as PackageType ON ([PackageType].[PackageTypeID] = [InvoiceLines].[PackageTypeID])
WHERE ([InvoiceLines].[InvoiceID] = @InvoiceID)
SELECT [StockItemTransactionID]
,[StockItemTransactions].[StockItemID]
,StockItem.[StockItemName] as StockItemName
,[StockItemTransactions].[TransactionTypeID]
,[TransactionType].[TransactionTypeName]
-- ,[CustomerID]
-- ,[InvoiceID]
-- ,[SupplierID]
-- ,[PurchaseOrderID]
,[TransactionOccurredWhen] as TransactionAt
,[Quantity]
-- ,[LastEditedBy]
-- ,[LastEditedWhen]
FROM [Warehouse].[StockItemTransactions]
INNER JOIN [Warehouse].[StockItems] as StockItem ON ([StockItemTransactions].StockItemID = [StockItem].StockItemID)
INNER JOIN [Application].[TransactionTypes] as TransactionType ON ([StockItemTransactions].[TransactionTypeID] = TransactionType.[TransactionTypeID])
WHERE ([StockItemTransactions].[InvoiceID] = @InvoiceID)
END
The stored procedure returns 3 recordsets, a “summary” of the Order, a summary of the associated OrderLines and a summary of the associated StockItemTransactions.
public async Task<ActionResult<Model.InvoiceSummaryGetDtoV1>>Get([Range(1, int.MaxValue, ErrorMessage = "Invoice id must greater than 0")] int id)
{
Model.InvoiceSummaryGetDtoV1 response = null;
try
{
using (SqlConnection db = new SqlConnection(this.connectionString))
{
var invoiceSummary = await db.QueryMultipleWithRetryAsync("[Sales].[InvoiceSummaryGetV1]", param: new { InvoiceId = id }, commandType: CommandType.StoredProcedure);
response = await invoiceSummary.ReadSingleOrDefaultWithRetryAsync<Model.InvoiceSummaryGetDtoV1>();
if (response == default)
{
logger.LogInformation("Invoice:{0} not found", id);
return this.NotFound($"Invoice:{id} not found");
}
response.InvoiceLines = (await invoiceSummary.ReadWithRetryAsync<Model.InvoiceLineSummaryListDtoV1>()).ToArray();
response.StockItemTransactions = (await invoiceSummary.ReadWithRetryAsync<Model.StockItemTransactionSummaryListDtoV1>()).ToArray();
}
}
catch (SqlException ex)
{
logger.LogError(ex, "Retrieving Invoice, Invoice Lines or Stock Item Transactions");
return this.StatusCode(StatusCodes.Status500InternalServerError);
}
return this.Ok(response);
}
I started again, but kept the first section as it covers one of the simplest possible approaches to caching using the [ResponseCache] attribute and VaryByQueryKeys.
[HttpGet("Response")]
[ResponseCache(Duration = StockItemsListResponseCacheDuration)]
public async Task<ActionResult<IAsyncEnumerable<Model.StockItemListDtoV1>>> GetResponse()
{
IEnumerable<Model.StockItemListDtoV1> response = null;
logger.LogInformation("Response cache load");
try
{
response = await dapper.QueryAsync<Model.StockItemListDtoV1>(sql: @"SELECT [StockItemID] as ""ID"", [StockItemName] as ""Name"", [RecommendedRetailPrice], [TaxRate] FROM [Warehouse].[StockItems]", commandType: CommandType.Text);
}
catch (SqlException ex)
{
logger.LogError(ex, "Retrieving list of StockItems");
return this.StatusCode(StatusCodes.Status500InternalServerError);
}
return this.Ok(response);
}
[HttpGet("ResponseVarying")]
[ResponseCache(Duration = StockItemsListResponseCacheDuration, VaryByQueryKeys = new string[] { "id" })]
public async Task<ActionResult<Model.StockItemGetDtoV1>> Get([FromQuery(Name = "id"), Range(1, int.MaxValue, ErrorMessage = "Stock item id must greater than 0")] int id)
{
Model.StockItemGetDtoV1 response = null;
logger.LogInformation("Response cache varying load id:{0}", id);
try
{
response = await dapper.QuerySingleOrDefaultAsync<Model.StockItemGetDtoV1>(sql: "[Warehouse].[StockItemsStockItemLookupV1]", param: new { stockItemId = id }, commandType: CommandType.StoredProcedure);
if (response == default)
{
logger.LogInformation("StockItem:{0} not found", id);
return this.NotFound($"StockItem:{id} not found");
}
}
catch (SqlException ex)
{
logger.LogError(ex, "Looking up StockItem with Id:{0}", id);
return this.StatusCode(StatusCodes.Status500InternalServerError);
}
return this.Ok(response);
}
All the browsers appeared to respect the cache control headers but Firefox was the only one which did not initiate a new request when I pressed return in the Uniform Resource Locator(URL) field.
[HttpGet("DapperMemory")]
public async Task<ActionResult<IAsyncEnumerable<Model.StockItemListDtoV1>>> GetDapper()
{
List<Model.StockItemListDtoV1> response;
logger.LogInformation("Dapper cache load");
try
{
response = await dapper.QueryAsync<Model.StockItemListDtoV1>(
sql: @"SELECT [StockItemID] as ""ID"", [StockItemName] as ""Name"", [RecommendedRetailPrice], [TaxRate] FROM [Warehouse].[StockItems]",
commandType: CommandType.Text,
enableCache: true,
cacheExpire: TimeSpan.Parse(this.Configuration.GetValue<string>("DapperCachingDuration"))
);
}
catch (SqlException ex)
{
logger.LogError(ex, "Retrieving list of StockItems");
return this.StatusCode(StatusCodes.Status500InternalServerError);
}
return this.Ok(response);
}
[HttpGet("DapperMemoryVarying")]
public async Task<ActionResult<Model.StockItemGetDtoV1>> GetDapperVarying([FromQuery(Name = "id"), Range(1, int.MaxValue, ErrorMessage = "Stock item id must greater than 0")] int id)
{
Model.StockItemGetDtoV1 response = null;
logger.LogInformation("Dapper cache varying load id:{0}", id);
try
{
response = await dapper.QuerySingleOrDefaultAsync<Model.StockItemGetDtoV1>(
sql: "[Warehouse].[StockItemsStockItemLookupV1]",
param: new { stockItemId = id },
commandType: CommandType.StoredProcedure,
cacheKey: $"StockItem:{id}",
enableCache: true,
cacheExpire: TimeSpan.Parse(this.Configuration.GetValue<string>("DapperCachingDuration"))
);
if (response == default)
{
logger.LogInformation("StockItem:{0} not found", id);
return this.NotFound($"StockItem:{id} not found");
}
}
catch (SqlException ex)
{
logger.LogError(ex, "Looking up StockItem with Id:{0}", id);
return this.StatusCode(StatusCodes.Status500InternalServerError);
}
return this.Ok(response);
}
Both the Dapper.Extensions In-Memory and Redis cache reduced the number of database requests to the bare minimum. In a larger application the formatting of the cacheKey (cacheKey: “StockItems” & cacheKey: $”StockItem:{id}”) would be important to stop database query result collisions.
SQL Server Profiler displaying the list and single record requests.
Memurai running as a Windows Service on my development machine
When the Web API project was restarted the contents in-memory cache were lost. The Redis cache contents survive a restart and can be access from multiple clients.
It’s not uncommon for SQL Azure servers and databases to suffer from “transient failures”. In application logs I have seen these occur during scale up/down events, periods where my application’s performance has been temporarily impacted (but its throughput has not changed), which I assume has been some load balancing going on in the background and when network connectivity has been a bit flakey.
Now I’m using The Polly Project which builds on the concepts of TOPAZ but has been thoroughly re-engineered with lots of extensibility, an active community and modern codebase. Inspired by Ben Hyrman and several other developers I have built a minimalist wrapper for the Dapper Async methods which detects transient errors using the same approach as the Entity Framework Core library.
I did think about retry functionality for async methods which returned object/dynamic but have only implemented strongly typed ones for the initial version.
[HttpGet]
public async Task<ActionResult<IAsyncEnumerable<Model.StockItemListDtoV1>>> Get()
{
IEnumerable<Model.StockItemListDtoV1> response = null;
try
{
using (SqlConnection db = new SqlConnection(this.connectionString))
{
response = await db.QueryWithRetryAsync<Model.StockItemListDtoV1>(sql: @"SELECT [StockItemID] as ""ID"", [StockItemName] as ""Name"", [RecommendedRetailPrice], [TaxRate] FROM [Warehouse].[StockItems]", commandType: CommandType.Text);
}
}
catch (SqlException ex)
{
logger.LogError(ex, "Retrieving list of StockItems");
return this.StatusCode(StatusCodes.Status500InternalServerError);
}
return this.Ok(response);
}
I have struggled to get reproduceable transient failures without pausing execution in the Visual Studio debugger and tinkering with variables or scaling up/down my databases (limit to how often this can be done) or unplugging the network cable at the wrong time.
My first couple of attempts at an Azure Queue Trigger Function which could do retries when an uplink message couldn’t be processed immediately(I didn’t want to throw an exception as this was just a transient issue) didn’t work. I wanted to return the uplink message to the Azure Storage Queue with the initial visibility set to a couple of seconds without throwing an exception.
I tried decorating the method with an Azure Storage Queueoutput binding but finally settled on the approach below. I can insert a single message into the storage queue and the application would start looping every minute.
public static class UplinkMessageProcessor
{
const string RunTag = "Processor001";
static int ConcurrentThreadCount = 0;
static int MessagesProcessed = 0;
[FunctionName("UplinkMessageProcessor")]
public static void Run(
[QueueTrigger("%UplinkQueueName%", Connection = "AzureStorageConnectionString")]
CloudQueueMessage cloudQueueMessage,
IBinder binder, ILogger log)
{
try
{
Interlocked.Increment(ref ConcurrentThreadCount);
Interlocked.Increment(ref MessagesProcessed);
log.LogInformation($"{MessagesProcessed} {RunTag} Threads:{ConcurrentThreadCount}");
CloudQueue outputQueue = binder.Bind<CloudQueue>(new QueueAttribute("%UplinkQueueName%"));
CloudQueueMessage message = new CloudQueueMessage(cloudQueueMessage.AsString);
outputQueue.AddMessage(message, initialVisibilityDelay: new TimeSpan(0, 1, 0));
Thread.Sleep(2000);
Interlocked.Decrement(ref ConcurrentThreadCount);
}
catch (Exception ex)
{
log.LogError(ex, "Processing of Uplink message failed");
throw;
}
}
}
I used the binder.bind method to get the CloudQueue and CloudQueueMessage details so I could insert a hidden messages back into the queue.
The version of Azure Storage queue libraries used by the function bindings (Sep 2020) may cause some compile time warnings if you select the wrong NuGet package.
Hopefully this has enough keywords that someone trying todo the same thing finds it.
I have used the Enterprise library Blocks (which in different forms have been around since 2005) in quite a few projects. Individually the components are pretty good (not always best of breed) but they are well integrated and when used in the way which they were intended to be used work well.
I have just upgraded a client application to Visual Studio 2015 + .Net 4.5 + Enterprise Library V6 and some of the steps were not immediately obvious so hopefully this saves someone else some time. I have sample code for Azure Cloud Service Web and Worker roles.
For both web and worker roles I added the Azure Diagnostics listener to the listener config section of the enterprise library logging settings.
For the worker role I configured the exception and logging blocks in the worker role startup
public override bool OnStart()
{
// Set the maximum number of concurrent connections
ServicePointManager.DefaultConnectionLimit = 12;
...
LogWriterFactory logWriterFactory = new LogWriterFactory();
LogWriter logWriter = logWriterFactory.Create();
Logger.SetLogWriter(logWriter);
...
return result;
}
Then in the webrole webapi2 API controllers you can use embedded SQL or call stored procedures with retries. (This sample code uses the Northwind database and default retry configuration)
public IEnumerable&amp;lt;ProductDto&amp;gt; Get()
{
var products = new List&amp;lt;ProductDto&amp;gt;();
WebApiApplication.exManager.Process(() =&amp;gt;
{
Database db = new DatabaseProviderFactory().Create(&amp;quot;NorthwindInstance&amp;quot;);
RetryPolicy retry = new RetryPolicy&amp;lt;SqlDatabaseTransientErrorDetectionStrategy&amp;gt;(RetryStrategy.DefaultExponential);
var productAccessor = db.CreateSqlStringAccessor(
&amp;quot;SELECT [ProductID],[ProductName],[QuantityPerUnit],[UnitPrice],[UnitsInStock],[Discontinued] FROM Products&amp;quot;,
MapBuilder&amp;lt;ProductDto&amp;gt;
.MapAllProperties()
.Map(p =&amp;gt; p.ID).ToColumn(&amp;quot;ProductID&amp;quot;)
.Map(p =&amp;gt; p.Name).ToColumn(&amp;quot;ProductName&amp;quot;)
.Map(p =&amp;gt; p.QuantityPerUnit).ToColumn(&amp;quot;QuantityPerUnit&amp;quot;)
.Map(p =&amp;gt; p.UnitPrice).ToColumn(&amp;quot;UnitPrice&amp;quot;)
.Map(p =&amp;gt; p.UnitsInStock).ToColumn(&amp;quot;UnitsInStock&amp;quot;)
.Map(p =&amp;gt; p.Discontinued).ToColumn(&amp;quot;Discontinued&amp;quot;)
.Build());
products = retry.ExecuteAction(() =&amp;gt;
{
return productAccessor.Execute().ToList();
});
}, &amp;quot;ProductService&amp;quot;);
return products;
}
public ProductDto Get(int id)
{
ProductDto productDto = null;
WebApiApplication.exManager.Process(() =&amp;gt;
{
Database db = new DatabaseProviderFactory().Create(&amp;quot;NorthwindInstance&amp;quot;);
var productAccessor = db.CreateSqlStringAccessor(
&amp;quot;SELECT [ProductID],[ProductName],[QuantityPerUnit],[UnitPrice],[UnitsInStock],[Discontinued] FROM Products WHERE [ProductID]=@ProductID&amp;quot;,
new ProdductGetByProductIdParameterMapper(db),
MapBuilder&amp;amp;lt;ProductDto&amp;amp;gt;
.MapAllProperties()
.Map(p =&amp;gt; p.ID).ToColumn(&amp;quot;ProductID&amp;quot;)
.Map(p =&amp;gt; p.Name).ToColumn(&amp;quot;ProductName&amp;quot;)
.Map(p =&amp;gt; p.QuantityPerUnit).ToColumn(&amp;quot;QuantityPerUnit&amp;quot;)
.Map(p =&amp;gt; p.UnitPrice).ToColumn(&amp;quot;UnitPrice&amp;quot;)
.Map(p =&amp;gt; p.UnitsInStock).ToColumn(&amp;quot;UnitsInStock&amp;quot;)
.Map(p =&amp;gt; p.Discontinued).ToColumn(&amp;quot;Discontinued&amp;quot;)
.Build());
productDto = productAccessor.Execute(id).SingleOrDefault();
}, &amp;quot;ProductService&amp;quot;);
return productDto;
}