/// <inheritdoc />
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(statisticsArguments))
{
destination.DataContext.LogMessage = logMessage;
logMessage($"Connected to destination database '{destination.Name}' ({destination.ConnectionString})");
using (ICounterRepository source = RepositoryFactory.CreateCounterRepository(counterRepoArguments))
{
source.LogMessage = logMessage;
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
Dictionary <CounterReport, List <CounterRecord> > preRecordsReports = source.AvailableReports.ToDictionary(report => report, report => source.RequestRecords(runDate, report).ToList());
if (!preRecordsReports.Any(x => x.Value.Count > 0))
{
logMessage("No Records for this date");
return;
}
if (localJsonArguments != null)
{
foreach (var reportRecord in preRecordsReports)
{
using (IDirectoryRepository directoryRepo = RepositoryFactory.CreateDirectoryRepository(localJsonArguments))
{
using (Stream stream = directoryRepo.CreateFile($"{source.Name} {runDate:yyyy-MM-dd} {reportRecord.Key}.json", FileCreationMode.Overwrite))
{
JsonSerializer serializer = JsonSerializer.Create();
using (JsonTextWriter jsonTextWriter = new JsonTextWriter(new StreamWriter(stream)))
{
serializer.Serialize(jsonTextWriter, reportRecord.Value);
}
}
}
}
}
var preRecords = preRecordsReports.SelectMany(x => x.Value).ToList();
//Add Metrics of vendorRecords to single vendor record
foreach (var recordsByRunDate in preRecords.GroupBy(x => x.RunDate))
{
if (recordsByRunDate.Any(x => x.ItemType == ItemType.Vendor))
{
foreach (var vendorRecord in recordsByRunDate.Where(x => x.ItemType == ItemType.Vendor))
{
vendorRecord.Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Proprietary, IdentifierValue = source.Name
} };
}
}
else
{
preRecords.Add(new CounterRecord {
ItemName = source.Name, ItemType = ItemType.Vendor, RunDate = recordsByRunDate.Select(y => y.RunDate).First(), Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Proprietary, IdentifierValue = source.Name
} }, Metrics = new CounterMetric[] { }
});
}
}
if (preRecords.Any(x => x.ItemType == ItemType.Database))
{
foreach (CounterRecord record in preRecords)
{
if (record.ItemType != ItemType.Database)
{
continue;
}
record.ItemPlatform = source.Name;
record.Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Database, IdentifierValue = record.ItemName
} };
}
}
else
{
preRecords.Add(new CounterRecord {
ItemName = source.Name, ItemType = ItemType.Database, ItemPlatform = source.Name, RunDate = runDate, Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Database, IdentifierValue = source.Name
} }, Metrics = new CounterMetric[] { }
});
}
var where = preRecords.Where(x => x.ItemType == ItemType.Vendor);
Console.WriteLine(where.Count());
var records = preRecords
.Where(x => !string.IsNullOrEmpty(x.ItemName))
.Select(SanitizeIdentifiers)
.GroupBy(x => x, new CounterRecordComparer())
.Select(AggregateDuplicates)
.Select((r, i) => new { Index = i, Record = r });
var counterRecords = records.GroupBy(x => x.Record.RunDate).ToList();
logMessage($"{preRecords.Count + 1} Total Records");
logMessage($"{recordswithNoIdentifiers} Records with no Identifiers");
logMessage($"{counterRecords.Sum(x => x.Count())} Unique Records");
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
destination.DataContext.Connection.Close();
destination.DataContext.Dispose();
destination.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
foreach (var counterGroup in counterRecords)
{
destination.DataContext.BulkImportCounterTransactions(
counterGroup
.ToDataReader(r => new object[] { null, null, null, r.Index, r.Record.ItemName, r.Record.ItemPlatform, r.Record.ItemType, r.Record.RunDate }),
counterGroup
.SelectMany(r => r.Record.Identifiers.Bind(a => a.Select(i => new { r.Index, Identifier = i, r.Record.ItemType })))
.ToDataReader(i => new object[] { i.Index, i.Identifier.IdentifierType, i.Identifier.IdentifierValue, i.ItemType }),
counterGroup
.SelectMany(r => r.Record.Metrics.Bind(a => a.Select(m => new { r.Index, Metric = m })))
.ToDataReader(m => new object[] { m.Index, m.Metric.MetricType, m.Metric.MetricValue }));
//Add Record of report being run
using (var harvester = RepositoryFactory.CreateHarvesterRepository(harvesterArguments))
{
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name);
foreach (CounterReport report in preRecordsReports.Keys)
{
if (!harvester.DataContext.CounterOperationRecords.Any(x => x.OperationID == OperationID && x.RunDate == runDate && x.Report == report.ToString()))
{
harvester.DataContext.CounterOperationRecords.InsertOnSubmit(new CounterOperationRecord
{
OperationID = OperationID,
RepositoryID = repository.ID,
RunDate = runDate,
Report = report.ToString(),
ExecutedDate = DateTime.Now
});
}
else
{
harvester.DataContext.CounterOperationRecords.First(x => x.OperationID == OperationID && x.RunDate == runDate && x.Report == report.ToString()).ExecutedDate = DateTime.Now;
}
}
harvester.DataContext.SubmitChanges();
}
}
}
}
}