/// <inheritdoc />
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(statisticsArguments))
{
destination.DataContext.LogMessage = logMessage;
logMessage($"Connected to destination database '{destination.Name}' ({destination.ConnectionString})");
using (ICounterRepository source = RepositoryFactory.CreateCounterRepository(counterRepoArguments))
{
source.LogMessage = logMessage;
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
Dictionary <CounterReport, List <CounterRecord> > preRecordsReports = source.AvailableReports.ToDictionary(report => report, report => source.RequestRecords(runDate, report).ToList());
if (!preRecordsReports.Any(x => x.Value.Count > 0))
{
logMessage("No Records for this date");
return;
}
if (localJsonArguments != null)
{
foreach (var reportRecord in preRecordsReports)
{
using (IDirectoryRepository directoryRepo = RepositoryFactory.CreateDirectoryRepository(localJsonArguments))
{
using (Stream stream = directoryRepo.CreateFile($"{source.Name} {runDate:yyyy-MM-dd} {reportRecord.Key}.json", FileCreationMode.Overwrite))
{
JsonSerializer serializer = JsonSerializer.Create();
using (JsonTextWriter jsonTextWriter = new JsonTextWriter(new StreamWriter(stream)))
{
serializer.Serialize(jsonTextWriter, reportRecord.Value);
}
}
}
}
}
var preRecords = preRecordsReports.SelectMany(x => x.Value).ToList();
//Add Metrics of vendorRecords to single vendor record
foreach (var recordsByRunDate in preRecords.GroupBy(x => x.RunDate))
{
if (recordsByRunDate.Any(x => x.ItemType == ItemType.Vendor))
{
foreach (var vendorRecord in recordsByRunDate.Where(x => x.ItemType == ItemType.Vendor))
{
vendorRecord.Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Proprietary, IdentifierValue = source.Name
} };
}
}
else
{
preRecords.Add(new CounterRecord {
ItemName = source.Name, ItemType = ItemType.Vendor, RunDate = recordsByRunDate.Select(y => y.RunDate).First(), Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Proprietary, IdentifierValue = source.Name
} }, Metrics = new CounterMetric[] { }
});
}
}
if (preRecords.Any(x => x.ItemType == ItemType.Database))
{
foreach (CounterRecord record in preRecords)
{
if (record.ItemType != ItemType.Database)
{
continue;
}
record.ItemPlatform = source.Name;
record.Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Database, IdentifierValue = record.ItemName
} };
}
}
else
{
preRecords.Add(new CounterRecord {
ItemName = source.Name, ItemType = ItemType.Database, ItemPlatform = source.Name, RunDate = runDate, Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Database, IdentifierValue = source.Name
} }, Metrics = new CounterMetric[] { }
});
}
var where = preRecords.Where(x => x.ItemType == ItemType.Vendor);
Console.WriteLine(where.Count());
var records = preRecords
.Where(x => !string.IsNullOrEmpty(x.ItemName))
.Select(SanitizeIdentifiers)
.GroupBy(x => x, new CounterRecordComparer())
.Select(AggregateDuplicates)
.Select((r, i) => new { Index = i, Record = r });
var counterRecords = records.GroupBy(x => x.Record.RunDate).ToList();
logMessage($"{preRecords.Count + 1} Total Records");
logMessage($"{recordswithNoIdentifiers} Records with no Identifiers");
logMessage($"{counterRecords.Sum(x => x.Count())} Unique Records");
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
destination.DataContext.Connection.Close();
destination.DataContext.Dispose();
destination.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
foreach (var counterGroup in counterRecords)
{
destination.DataContext.BulkImportCounterTransactions(
counterGroup
.ToDataReader(r => new object[] { null, null, null, r.Index, r.Record.ItemName, r.Record.ItemPlatform, r.Record.ItemType, r.Record.RunDate }),
counterGroup
.SelectMany(r => r.Record.Identifiers.Bind(a => a.Select(i => new { r.Index, Identifier = i, r.Record.ItemType })))
.ToDataReader(i => new object[] { i.Index, i.Identifier.IdentifierType, i.Identifier.IdentifierValue, i.ItemType }),
counterGroup
.SelectMany(r => r.Record.Metrics.Bind(a => a.Select(m => new { r.Index, Metric = m })))
.ToDataReader(m => new object[] { m.Index, m.Metric.MetricType, m.Metric.MetricValue }));
//Add Record of report being run
using (var harvester = RepositoryFactory.CreateHarvesterRepository(harvesterArguments))
{
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name);
foreach (CounterReport report in preRecordsReports.Keys)
{
if (!harvester.DataContext.CounterOperationRecords.Any(x => x.OperationID == OperationID && x.RunDate == runDate && x.Report == report.ToString()))
{
harvester.DataContext.CounterOperationRecords.InsertOnSubmit(new CounterOperationRecord
{
OperationID = OperationID,
RepositoryID = repository.ID,
RunDate = runDate,
Report = report.ToString(),
ExecutedDate = DateTime.Now
});
}
else
{
harvester.DataContext.CounterOperationRecords.First(x => x.OperationID == OperationID && x.RunDate == runDate && x.Report == report.ToString()).ExecutedDate = DateTime.Now;
}
}
harvester.DataContext.SubmitChanges();
}
}
}
}
}
public override void Execute(DateTime runDate, Action <string> logMessage, System.Threading.CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(_logDirectoryArgs))
{
logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})");
Regex filePattern = new Regex(_arguments.FilePattern);
foreach (DirectoryObjectMetadata file in source.ListFiles().Where(x => x.Path.Contains(".zip")))
{
string tempZipDirectoryPath = file.Path.Replace(".zip", "");
string tempZipDirectoryName = file.Name.Replace(".zip", "");
ZipFile.ExtractToDirectory(file.Path, tempZipDirectoryPath);
source.DeleteFile(file.Path);
foreach (String unzippedfile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => filePattern.IsMatch(x)))
{
string filename = unzippedfile.Split(new[] { "\\" }, StringSplitOptions.None).Last();
List <String> currentFiles = source.ListFiles().Select(x => x.Name).ToList();
if (!currentFiles.Contains(filename))
{
source.MoveFile(unzippedfile, filename);
}
}
foreach (String gzipFile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => x.Contains(".gz")))
{
string fileNameConcat = tempZipDirectoryName + ".log";
List <String> currentFiles = destination.ListFiles().Select(x => x.Name).ToList();
if (!currentFiles.Contains(fileNameConcat))
{
using (GZipStream gzipStream = new GZipStream(source.OpenFile(gzipFile), CompressionMode.Decompress))
{
using (Stream unzippedDestination = destination.CreateFile(fileNameConcat, Repository.Directory.FileCreationMode.ThrowIfFileExists))
{
gzipStream.CopyTo(unzippedDestination);
}
}
}
}
source.DeleteDirectory(tempZipDirectoryPath);
}
}
List <String> modified = new List <String>();
Int32 newCount = 0;
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
{
logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
IEnumerable <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/");
Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.Operation.Name == Name && d.Repository.Name == source.Name).ToDictionary(d => d.FilePath);
Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name);
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
foreach (DirectoryObjectMetadata file in sourceFiles)
{
if (!dictionary.ContainsKey(file.Path))
{
modified.Add(file.Name);
newCount++;
harvester.DataContext.DirectoryRecords.InsertOnSubmit(new DirectoryRecord
{
OperationID = OperationID,
RepositoryID = repository.ID,
FilePath = file.Path,
FileModifiedDate = file.ModifiedDate,
CreationDate = DateTime.Now,
ModifiedDate = DateTime.Now
});
}
else
{
DirectoryRecord element = dictionary[file.Path];
if (file.ModifiedDate > element.FileModifiedDate)
{
modified.Add(file.Name);
element.FileModifiedDate = file.ModifiedDate;
element.ModifiedDate = DateTime.Now;
}
}
}
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
harvester.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
harvester.DataContext.SubmitChanges();
}
logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");
if (modified.Count == 0)
{
return;
}
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_statisticsArgs))
{
logMessage($"Connected to database '{destination.Name}' ({destination.ConnectionString})");
StreamParser <EZProxyAudit> Parser = new StreamParser <EZProxyAudit>();
List <EZProxyAudit> records = modified.Select(file =>
{
logMessage($"Processing '{file}':");
int lineNumber = 0;
return(Parser.ParseStream(source.OpenFile(file)).Select(x => new EZProxyAudit
{
DateTime = x.DateTime,
Event = x.Event,
IP = x.IP,
Other = x.Other,
Session = x.Session,
Username = x.Username,
LineNumber = lineNumber++,
}));
}).SelectMany(x => x).ToList();
logMessage($"Records Found: {records.Count}");
destination.DataContext.BulkImportEZProxyAudit(records.ToDataReader(r => new object[] { r.DateTime, r.Event, r.IP, r.Username, r.Session, r.Other, r.LineNumber }));
}
}
}
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(sourceDirectory))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(destinationDirectory))
{
logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})");
Regex filePattern = new Regex(arguments.FilePattern, RegexOptions.IgnoreCase);
List <DirectoryObjectMetadata> destinationFiles = destination.ListFiles("/").ToList();
int newCount = 0, modified = 0;
foreach (DirectoryObjectMetadata file in source.ListFiles("/").Where(x => filePattern.IsMatch(x.Name)))
{
try
{
if (!destinationFiles.Select(x => x.Name).Contains(file.Name))
{
logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}");
using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite))
{
using (Stream sourceStream = source.OpenFile(file.Name))
{
sourceStream.CopyTo(destStream);
}
}
destination.CopyFile("TMP" + file.Name, file.Name);
destination.DeleteFile("TMP" + file.Name);
logMessage($"Added {file.Name} from {source.Name} to {destination.Name}");
newCount++;
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
destination.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
}
else
{
DirectoryObjectMetadata destinationFile = destinationFiles.Find(x => x.Name == file.Name);
if (file.ModifiedDate > destinationFile.ModifiedDate)
{
logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}");
using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite))
{
using (Stream sourceStream = source.OpenFile(file.Name))
{
sourceStream.CopyTo(destStream);
}
}
destination.DeleteFile(file.Name);
destination.MoveFile("TMP" + file.Name, file.Name);
logMessage($"Updated {file.Name} from {source.Name} to {destination.Name}");
modified++;
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
destination.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
}
}
}
catch (Exception)
{
destination.DeleteFile("TMP" + file.Name);
throw;
}
}
logMessage($"Discovered {newCount} new files and {modified} updated files).");
}
}
}
