/// <inheritdoc /> public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken) { using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(statisticsArguments)) { destination.DataContext.LogMessage = logMessage; logMessage($"Connected to destination database '{destination.Name}' ({destination.ConnectionString})"); using (ICounterRepository source = RepositoryFactory.CreateCounterRepository(counterRepoArguments)) { source.LogMessage = logMessage; logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})"); Dictionary <CounterReport, List <CounterRecord> > preRecordsReports = source.AvailableReports.ToDictionary(report => report, report => source.RequestRecords(runDate, report).ToList()); if (!preRecordsReports.Any(x => x.Value.Count > 0)) { logMessage("No Records for this date"); return; } if (localJsonArguments != null) { foreach (var reportRecord in preRecordsReports) { using (IDirectoryRepository directoryRepo = RepositoryFactory.CreateDirectoryRepository(localJsonArguments)) { using (Stream stream = directoryRepo.CreateFile($"{source.Name} {runDate:yyyy-MM-dd} {reportRecord.Key}.json", FileCreationMode.Overwrite)) { JsonSerializer serializer = JsonSerializer.Create(); using (JsonTextWriter jsonTextWriter = new JsonTextWriter(new StreamWriter(stream))) { serializer.Serialize(jsonTextWriter, reportRecord.Value); } } } } } var preRecords = preRecordsReports.SelectMany(x => x.Value).ToList(); //Add Metrics of vendorRecords to single vendor record foreach (var recordsByRunDate in preRecords.GroupBy(x => x.RunDate)) { if (recordsByRunDate.Any(x => x.ItemType == ItemType.Vendor)) { foreach (var vendorRecord in recordsByRunDate.Where(x => x.ItemType == ItemType.Vendor)) { vendorRecord.Identifiers = new[] { new CounterIdentifier { IdentifierType = IdentifierType.Proprietary, IdentifierValue = source.Name } }; } } else { preRecords.Add(new CounterRecord { ItemName = source.Name, ItemType = ItemType.Vendor, RunDate = recordsByRunDate.Select(y => y.RunDate).First(), Identifiers = new[] { new CounterIdentifier { IdentifierType = IdentifierType.Proprietary, IdentifierValue = source.Name } }, Metrics = new CounterMetric[] { } }); } } if (preRecords.Any(x => x.ItemType == ItemType.Database)) { foreach (CounterRecord record in preRecords) { if (record.ItemType != ItemType.Database) { continue; } record.ItemPlatform = source.Name; record.Identifiers = new[] { new CounterIdentifier { IdentifierType = IdentifierType.Database, IdentifierValue = record.ItemName } }; } } else { preRecords.Add(new CounterRecord { ItemName = source.Name, ItemType = ItemType.Database, ItemPlatform = source.Name, RunDate = runDate, Identifiers = new[] { new CounterIdentifier { IdentifierType = IdentifierType.Database, IdentifierValue = source.Name } }, Metrics = new CounterMetric[] { } }); } var where = preRecords.Where(x => x.ItemType == ItemType.Vendor); Console.WriteLine(where.Count()); var records = preRecords .Where(x => !string.IsNullOrEmpty(x.ItemName)) .Select(SanitizeIdentifiers) .GroupBy(x => x, new CounterRecordComparer()) .Select(AggregateDuplicates) .Select((r, i) => new { Index = i, Record = r }); var counterRecords = records.GroupBy(x => x.Record.RunDate).ToList(); logMessage($"{preRecords.Count + 1} Total Records"); logMessage($"{recordswithNoIdentifiers} Records with no Identifiers"); logMessage($"{counterRecords.Sum(x => x.Count())} Unique Records"); if (cancellationToken.IsCancellationRequested) { source.Dispose(); destination.DataContext.Connection.Close(); destination.DataContext.Dispose(); destination.Dispose(); cancellationToken.ThrowIfCancellationRequested(); } foreach (var counterGroup in counterRecords) { destination.DataContext.BulkImportCounterTransactions( counterGroup .ToDataReader(r => new object[] { null, null, null, r.Index, r.Record.ItemName, r.Record.ItemPlatform, r.Record.ItemType, r.Record.RunDate }), counterGroup .SelectMany(r => r.Record.Identifiers.Bind(a => a.Select(i => new { r.Index, Identifier = i, r.Record.ItemType }))) .ToDataReader(i => new object[] { i.Index, i.Identifier.IdentifierType, i.Identifier.IdentifierValue, i.ItemType }), counterGroup .SelectMany(r => r.Record.Metrics.Bind(a => a.Select(m => new { r.Index, Metric = m }))) .ToDataReader(m => new object[] { m.Index, m.Metric.MetricType, m.Metric.MetricValue })); //Add Record of report being run using (var harvester = RepositoryFactory.CreateHarvesterRepository(harvesterArguments)) { if (OperationID == 0) { logMessage("Warning: OperationID was not set properly. Correcting this."); OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID; } Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name); foreach (CounterReport report in preRecordsReports.Keys) { if (!harvester.DataContext.CounterOperationRecords.Any(x => x.OperationID == OperationID && x.RunDate == runDate && x.Report == report.ToString())) { harvester.DataContext.CounterOperationRecords.InsertOnSubmit(new CounterOperationRecord { OperationID = OperationID, RepositoryID = repository.ID, RunDate = runDate, Report = report.ToString(), ExecutedDate = DateTime.Now }); } else { harvester.DataContext.CounterOperationRecords.First(x => x.OperationID == OperationID && x.RunDate == runDate && x.Report == report.ToString()).ExecutedDate = DateTime.Now; } } harvester.DataContext.SubmitChanges(); } } } } }
public override void Execute(DateTime runDate, Action <string> logMessage, System.Threading.CancellationToken cancellationToken) { using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs)) { logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})"); using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(_logDirectoryArgs)) { logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})"); Regex filePattern = new Regex(_arguments.FilePattern); foreach (DirectoryObjectMetadata file in source.ListFiles().Where(x => x.Path.Contains(".zip"))) { string tempZipDirectoryPath = file.Path.Replace(".zip", ""); string tempZipDirectoryName = file.Name.Replace(".zip", ""); ZipFile.ExtractToDirectory(file.Path, tempZipDirectoryPath); source.DeleteFile(file.Path); foreach (String unzippedfile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => filePattern.IsMatch(x))) { string filename = unzippedfile.Split(new[] { "\\" }, StringSplitOptions.None).Last(); List <String> currentFiles = source.ListFiles().Select(x => x.Name).ToList(); if (!currentFiles.Contains(filename)) { source.MoveFile(unzippedfile, filename); } } foreach (String gzipFile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => x.Contains(".gz"))) { string fileNameConcat = tempZipDirectoryName + ".log"; List <String> currentFiles = destination.ListFiles().Select(x => x.Name).ToList(); if (!currentFiles.Contains(fileNameConcat)) { using (GZipStream gzipStream = new GZipStream(source.OpenFile(gzipFile), CompressionMode.Decompress)) { using (Stream unzippedDestination = destination.CreateFile(fileNameConcat, Repository.Directory.FileCreationMode.ThrowIfFileExists)) { gzipStream.CopyTo(unzippedDestination); } } } } source.DeleteDirectory(tempZipDirectoryPath); } } List <String> modified = new List <String>(); Int32 newCount = 0; using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs)) { logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})"); IEnumerable <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/"); Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.Operation.Name == Name && d.Repository.Name == source.Name).ToDictionary(d => d.FilePath); Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name); if (OperationID == 0) { logMessage("Warning: OperationID was not set properly. Correcting this."); OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID; } foreach (DirectoryObjectMetadata file in sourceFiles) { if (!dictionary.ContainsKey(file.Path)) { modified.Add(file.Name); newCount++; harvester.DataContext.DirectoryRecords.InsertOnSubmit(new DirectoryRecord { OperationID = OperationID, RepositoryID = repository.ID, FilePath = file.Path, FileModifiedDate = file.ModifiedDate, CreationDate = DateTime.Now, ModifiedDate = DateTime.Now }); } else { DirectoryRecord element = dictionary[file.Path]; if (file.ModifiedDate > element.FileModifiedDate) { modified.Add(file.Name); element.FileModifiedDate = file.ModifiedDate; element.ModifiedDate = DateTime.Now; } } } if (cancellationToken.IsCancellationRequested) { source.Dispose(); harvester.Dispose(); cancellationToken.ThrowIfCancellationRequested(); } harvester.DataContext.SubmitChanges(); } logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated)."); if (modified.Count == 0) { return; } using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_statisticsArgs)) { logMessage($"Connected to database '{destination.Name}' ({destination.ConnectionString})"); StreamParser <EZProxyAudit> Parser = new StreamParser <EZProxyAudit>(); List <EZProxyAudit> records = modified.Select(file => { logMessage($"Processing '{file}':"); int lineNumber = 0; return(Parser.ParseStream(source.OpenFile(file)).Select(x => new EZProxyAudit { DateTime = x.DateTime, Event = x.Event, IP = x.IP, Other = x.Other, Session = x.Session, Username = x.Username, LineNumber = lineNumber++, })); }).SelectMany(x => x).ToList(); logMessage($"Records Found: {records.Count}"); destination.DataContext.BulkImportEZProxyAudit(records.ToDataReader(r => new object[] { r.DateTime, r.Event, r.IP, r.Username, r.Session, r.Other, r.LineNumber })); } } }
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken) { using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(sourceDirectory)) { logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})"); using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(destinationDirectory)) { logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})"); Regex filePattern = new Regex(arguments.FilePattern, RegexOptions.IgnoreCase); List <DirectoryObjectMetadata> destinationFiles = destination.ListFiles("/").ToList(); int newCount = 0, modified = 0; foreach (DirectoryObjectMetadata file in source.ListFiles("/").Where(x => filePattern.IsMatch(x.Name))) { try { if (!destinationFiles.Select(x => x.Name).Contains(file.Name)) { logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}"); using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite)) { using (Stream sourceStream = source.OpenFile(file.Name)) { sourceStream.CopyTo(destStream); } } destination.CopyFile("TMP" + file.Name, file.Name); destination.DeleteFile("TMP" + file.Name); logMessage($"Added {file.Name} from {source.Name} to {destination.Name}"); newCount++; if (cancellationToken.IsCancellationRequested) { source.Dispose(); destination.Dispose(); cancellationToken.ThrowIfCancellationRequested(); } } else { DirectoryObjectMetadata destinationFile = destinationFiles.Find(x => x.Name == file.Name); if (file.ModifiedDate > destinationFile.ModifiedDate) { logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}"); using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite)) { using (Stream sourceStream = source.OpenFile(file.Name)) { sourceStream.CopyTo(destStream); } } destination.DeleteFile(file.Name); destination.MoveFile("TMP" + file.Name, file.Name); logMessage($"Updated {file.Name} from {source.Name} to {destination.Name}"); modified++; if (cancellationToken.IsCancellationRequested) { source.Dispose(); destination.Dispose(); cancellationToken.ThrowIfCancellationRequested(); } } } } catch (Exception) { destination.DeleteFile("TMP" + file.Name); throw; } } logMessage($"Discovered {newCount} new files and {modified} updated files)."); } } }