private static void StoreLastSyncKey(SyncData syncData)
{
var memsource = new MemorySource <SyncData>();
memsource.DisableLogging = true;
memsource.DataAsList.Add(syncData);
var syncdest = new JsonDestination <SyncData>("LastSyncId.json");
syncdest.DisableLogging = true;
memsource.LinkTo(syncdest);
Network.Execute(memsource);
}
public static async Task Run([TimerTrigger("0 */1 * * * *"
//, RunOnStartup=true) //only for testing purposes
)] TimerInfo myTimer, ILogger log)
{
Logging.LogInstance = log;
string sqlConnectionString = Environment.GetEnvironmentVariable("SqlServerConnectionString", EnvironmentVariableTarget.Process);
string storageConnString = Environment.GetEnvironmentVariable("AzureWebJobsStorage", EnvironmentVariableTarget.Process);
SqlConnectionManager conn = new SqlConnectionManager(sqlConnectionString);
if (!DemoHelper.WasInitialized)
{
containerName = DemoHelper.PrepareForDemo(storageConnString, conn);
}
SyncData syncDataLastRun = ReadLastSyncKey();
var parameter = new[] { new QueryParameter()
{
Name = "syncId", Value = syncDataLastRun.SyncId
} };
var dbSource = new DbSource <Order>()
{
ConnectionManager = conn,
Sql = $"SELECT Id, Number, Details, Date FROM Orders WHERE Id > @syncId ORDER BY Date",
SqlParameter = parameter
};
var jsonDest = new JsonDestination <Order>();
jsonDest.ResourceType = ResourceType.AzureBlob;
jsonDest.AzureBlobStorage.ConnectionString = storageConnString;
jsonDest.AzureBlobStorage.ContainerName = containerName;
var currentDate = new DateTime(1900, 1, 1);
jsonDest.HasNextUri = (_, order) => {
if (order.Date.Date > currentDate.Date)
{
currentDate = order.Date;
return(true);
}
return(false);
};
jsonDest.GetNextUri = (_, order) => "OrderData_" + order.Date.ToString("yyyy-MM-dd") + ".json";
var multicast = new Multicast <Order>();
var aggregation = new Aggregation <Order, SyncData>();
aggregation.AggregateColumns = new[] {
new AggregateColumn()
{
InputValuePropName = "Id",
AggregatedValuePropName = "SyncId",
AggregationMethod = AggregationMethod.Max
}
};
var syncMemoryDest = new MemoryDestination <SyncData>();
/*
* |---> jsonDest ("OrderData_2020-01-01.json", "OrderData_2020-01-02.json", ..)
* |
* dbSource --> multicast
* |
* |---> aggregation --> syncMemoryDest (1st run: SyncId = 5, 2nd run: SyncId = 7)
*/
dbSource.LinkTo(multicast);
multicast.LinkTo(jsonDest);
multicast.LinkTo(aggregation);
aggregation.LinkTo(syncMemoryDest);
Network.Execute(dbSource);
if (syncMemoryDest.Data.Count > 0)
{
SyncData syncDataThisRun = syncMemoryDest.Data.First();
StoreLastSyncKey(syncDataThisRun);
}
}