/// <summary>
/// Main workflow entry point
/// </summary>
/// <param name="subscription">Pass here valid subscription key</param>
/// <param name="name">Pass here name for subscription, will be used to store cached file.</param>
/// <returns></returns>
public async Task DoWork(string subscription, string name)
{
_logger.Information("Checking API subscription {Name}", name);
// Define main place holder for subscription information block from API.
SubscriptionModel sub;
try
{
// Download subscription information from API
sub = JsonConvert.DeserializeObject <SubscriptionModel>(
await ApiRequestClient.GetString($"{ApiBase}Subscriptions?subscription={subscription}"));
}
catch (Exception e)
{
_logger.Fatal(e, "Error while reading subscription for {Name}", name);
return;
}
// Check if subscription is missing.
if (sub == null)
{
_logger.Error("Subscription for {Name} is empty, check key value", name);
return;
}
// Check if subscription has any data sets.
if (sub.DataSets == null || sub.DataSets.Count == 0)
{
_logger.Warning("Subscription {Name} has no data sets, exiting", name);
return;
}
// Check cached json to check if subscription has new data.
if (!await NeedToProcessDataSet(name, sub))
{
_logger.Information("No changes in data for {Name}, exiting", name);
return;
}
// Start processing subscription data.
_logger.Information("Loaded subscription {Name} data, will process {NumOfSets} data sets",
name, sub.DataSets.Count);
// Loop through all data sets on the subscription.
foreach (var dataSet in sub.DataSets)
{
var fileName = Path.Combine(_basePath, $"{dataSet.DataSourceKey}.json");
await DownloadDataSet(subscription, dataSet.DataSourceKey, dataSet.NumberOfPoints, fileName);
}
// Replace or save subscription block from API to cached file, this will be used to check if there
// are any updates in future requests.
try
{
await File.WriteAllTextAsync(GetSubscriptionCachedFileName(name),
JsonConvert.SerializeObject(sub));
}
catch (Exception e)
{
_logger.Error(e, "Writing subscription cache file caused exception");
}
_logger.Information("Completed all tasks, exiting");
}
/// <summary>
/// Download and save data set into JSON text file.
/// Please note:
/// This is simplistic text processing approach that does not include JSON validation
/// When creating a tool that will later convert or move data to another format, please change the method
/// to use DTO conversion and validate data context.
/// </summary>
/// <param name="subscription">Subscription key</param>
/// <param name="key">Data set key</param>
/// <param name="rows">How many rows data set has</param>
/// <param name="fileName">File name where json will be saved</param>
/// <returns></returns>
private async Task DownloadDataSet(string subscription, int key, int rows, string fileName)
{
_logger.Information("Downloading data set {Id} to {File}", key, fileName);
// Set remaining rows count
var remaining = rows;
// Set start of download page, we load from 0
var skip = 0;
// We will collect JSON strings into string builder, this is simplistic approach and we do not validate data.
var data = new StringBuilder();
// Since saved json will contain array of points, append start of array.
data.Append('[');
// Loop to page through data segments.
while (remaining > 0)
{
_logger.Information("Remaining rows: {Rows}", remaining);
// build API request url
var url = $"{ApiBase}Data?subscription={subscription}&key={key}&numberOfItems={DownloadPageSize}&skip={skip}";
// shift loop variables
remaining -= DownloadPageSize;
skip += DownloadPageSize;
// read json
var json = await ApiRequestClient.GetString(url);
// Check if response has any data
if (string.IsNullOrWhiteSpace(json))
{
_logger.Warning("Reading data at point {Skip} returned empty result, ignoring", skip);
continue;
}
// Small sanity check to make sure response is array like thing.
if (!json.Contains("[") || !json.Contains("]"))
{
continue;
}
// returned json is list, remove list symbols to convert them into block, then append
json = json.Remove(json.IndexOf('['), 1);
json = json.Remove(json.LastIndexOf(']'), 1);
data.AppendLine(json);
if (remaining > 0)
{
data.Append(',');
}
}
// append array closing symbol.
data.Append(']');
// write string builder memory stream into final string.
var finalJson = data.ToString();
// check if json ends in a way that will cause error
if (finalJson.EndsWith(",]"))
{
// remove trailing ,
finalJson = finalJson.Remove(finalJson.LastIndexOf(','), 1);
}
// write json to file
await File.WriteAllTextAsync(fileName, finalJson);
_logger.Information("Completed saving {File}", fileName);
}
