public IEnumerable <CounterRecord> RequestRecords(DateTime runDate, CounterReport report)
{
if (_arguments.JsonRepository != null)
{
using (IDirectoryRepository localJsonRepo = RepositoryFactory.CreateDirectoryRepository(_arguments.JsonRepository))
{
List <DirectoryObjectMetadata> jsonFiles = localJsonRepo.ListFiles().ToList();
if (jsonFiles.Any(x => x.Name == $"{Name} {runDate:yyyy-MM-dd} {report}.json"))
{
DirectoryObjectMetadata directoryObjectMetadata = jsonFiles.First(x => x.Name == $"{Name} {runDate:yyyy-MM-dd} {report}.json");
using (Stream openFile = localJsonRepo.OpenFile(directoryObjectMetadata.Name))
{
string jsonString = new StreamReader(openFile).ReadToEnd();
foreach (var record in JsonConvert.DeserializeObject <IEnumerable <CounterRecord> >(jsonString))
{
yield return(record);
}
}
yield break;
}
}
}
LogMessage($"Requesting {report.ToString()}");
var reportItems = GetResponse(runDate, report.ToString());
if (reportItems == null)
{
yield break;
}
foreach (IReportItem reportItem in reportItems)
{
if (reportItem.ItemDataType == Sushi.DataType.Platform && string.IsNullOrWhiteSpace(reportItem.ItemName))
{
reportItem.ItemName = reportItem.ItemPublisher;
}
yield return(new CounterRecord
{
ItemName = reportItem.ItemName,
ItemPlatform = reportItem.ItemPlatform,
ItemType = Convert(reportItem.ItemDataType),
RunDate = runDate,
Identifiers = reportItem.ItemIdentifier.Bind(a => a.Select(i => Convert(i, reportItem.ItemPlatform, reportItem.ItemDataType)).Where(i => i != null).ToArray(), new CounterIdentifier[] { }),
Metrics = reportItem.ItemPerformance.SelectMany(m => m.Instance).Select(i => new CounterMetric {
MetricType = Convert(i.MetricType), MetricValue = Int32.Parse(i.Count)
}).ToArray()
});
}
}
private IEnumerable <EZProxyLog> ParseFile(IDirectoryRepository source, string file)
{
Stream stream = source.OpenFile(file);
StreamReader reader = new StreamReader(stream);
string line;
int index = 0;
while ((line = reader.ReadLine()) != null)
{
index++;
bool overall = overallExpression.IsMatch(line);
if (!overall && !legacyExpression.IsMatch(line))
{
LogMessage($" ERROR on Line {index} Unknown log format found: {line}");
continue;
//throw new InvalidDataException("ERROR Unknown log format found: " + line);
}
Match match = overall ? overallExpression.Match(line) : legacyExpression.Match(line);
GroupCollection groups = match.Groups;
if (overall)
{
yield return(new EZProxyLog
{
IP = groups["IP"].Value,
Username = ParseUsername(groups["Username"].Value),
DateTime = ParseDate(groups["DateTime"].Value),
Request = groups["Request"].Value,
HTTPCode = Int16.Parse(groups["HTTPCode"].Value),
BytesTransferred = Int32.Parse(groups["BytesTransferred"].Value),
Referer = ParseNull(groups["Referer"].Value),
UserAgent = ParseNull(groups["UserAgent"].Value),
});
}
else
{
yield return(new EZProxyLog
{
IP = groups["IP"].Value,
Username = ParseUsername(groups["Username"].Value),
DateTime = ParseDate(groups["DateTime"].Value),
Request = groups["Request"].Value,
HTTPCode = Int16.Parse(groups["HTTPCode"].Value),
BytesTransferred = Int32.Parse(groups["BytesTransferred"].Value),
});
}
}
}
public override void Execute(DateTime runDate, Action <string> logMessage, System.Threading.CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(_logDirectoryArgs))
{
logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})");
Regex filePattern = new Regex(_arguments.FilePattern);
foreach (DirectoryObjectMetadata file in source.ListFiles().Where(x => x.Path.Contains(".zip")))
{
string tempZipDirectoryPath = file.Path.Replace(".zip", "");
string tempZipDirectoryName = file.Name.Replace(".zip", "");
ZipFile.ExtractToDirectory(file.Path, tempZipDirectoryPath);
source.DeleteFile(file.Path);
foreach (String unzippedfile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => filePattern.IsMatch(x)))
{
string filename = unzippedfile.Split(new[] { "\\" }, StringSplitOptions.None).Last();
List <String> currentFiles = source.ListFiles().Select(x => x.Name).ToList();
if (!currentFiles.Contains(filename))
{
source.MoveFile(unzippedfile, filename);
}
}
foreach (String gzipFile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => x.Contains(".gz")))
{
string fileNameConcat = tempZipDirectoryName + ".log";
List <String> currentFiles = destination.ListFiles().Select(x => x.Name).ToList();
if (!currentFiles.Contains(fileNameConcat))
{
using (GZipStream gzipStream = new GZipStream(source.OpenFile(gzipFile), CompressionMode.Decompress))
{
using (Stream unzippedDestination = destination.CreateFile(fileNameConcat, Repository.Directory.FileCreationMode.ThrowIfFileExists))
{
gzipStream.CopyTo(unzippedDestination);
}
}
}
}
source.DeleteDirectory(tempZipDirectoryPath);
}
}
List <String> modified = new List <String>();
Int32 newCount = 0;
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
{
logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
IEnumerable <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/");
Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.Operation.Name == Name && d.Repository.Name == source.Name).ToDictionary(d => d.FilePath);
Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name);
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
foreach (DirectoryObjectMetadata file in sourceFiles)
{
if (!dictionary.ContainsKey(file.Path))
{
modified.Add(file.Name);
newCount++;
harvester.DataContext.DirectoryRecords.InsertOnSubmit(new DirectoryRecord
{
OperationID = OperationID,
RepositoryID = repository.ID,
FilePath = file.Path,
FileModifiedDate = file.ModifiedDate,
CreationDate = DateTime.Now,
ModifiedDate = DateTime.Now
});
}
else
{
DirectoryRecord element = dictionary[file.Path];
if (file.ModifiedDate > element.FileModifiedDate)
{
modified.Add(file.Name);
element.FileModifiedDate = file.ModifiedDate;
element.ModifiedDate = DateTime.Now;
}
}
}
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
harvester.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
harvester.DataContext.SubmitChanges();
}
logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");
if (modified.Count == 0)
{
return;
}
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_statisticsArgs))
{
logMessage($"Connected to database '{destination.Name}' ({destination.ConnectionString})");
StreamParser <EZProxyAudit> Parser = new StreamParser <EZProxyAudit>();
List <EZProxyAudit> records = modified.Select(file =>
{
logMessage($"Processing '{file}':");
int lineNumber = 0;
return(Parser.ParseStream(source.OpenFile(file)).Select(x => new EZProxyAudit
{
DateTime = x.DateTime,
Event = x.Event,
IP = x.IP,
Other = x.Other,
Session = x.Session,
Username = x.Username,
LineNumber = lineNumber++,
}));
}).SelectMany(x => x).ToList();
logMessage($"Records Found: {records.Count}");
destination.DataContext.BulkImportEZProxyAudit(records.ToDataReader(r => new object[] { r.DateTime, r.Event, r.IP, r.Username, r.Session, r.Other, r.LineNumber }));
}
}
}
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
List <String> modified = new List <String>();
Int32 newCount = 0;
List <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/").Where(y => y.Name.ToLower().StartsWith("daily_transaction_export_for_vamp") && (y.Name.Contains(".txt") || y.Name.Contains(".csv"))).ToList();
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
{
logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.OperationID == OperationID && d.RepositoryID == harvester.DataContext.Repositories.First(y => y.Name == source.Name).ID).ToDictionary(d => d.FilePath);
foreach (DirectoryObjectMetadata file in sourceFiles)
{
if (!dictionary.ContainsKey(file.Path))
{
modified.Add(file.Name);
newCount++;
}
else
{
DirectoryRecord element = dictionary[file.Path];
if (file.ModifiedDate > element.FileModifiedDate)
{
modified.Add(file.Name);
}
}
}
}
logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");
if (modified.Count == 0 && newCount == 0)
{
logMessage("No Records to be processed.");
return;
}
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_databaseArgs))
{
logMessage($"Connected to destination database '{destination.Name}' ({destination.ConnectionString})");
var records = modified.Select(file =>
{
logMessage($"Processing '{file}':");
Stream inputStream = source.OpenFile(file);
CountingEnumerable <WmsTransactionRecord> counter = new CountingEnumerable <WmsTransactionRecord>(ReadLines(inputStream, GetFileDate(file)));
return(Tuple.Create(counter, (Action)(() =>
{
inputStream.Dispose();
logMessage($"\t{counter.Count} records processed.");
})));
}).SelectMany(i => i).Select(wms => new
{
wms.EventType,
wms.InstitutionName,
ItemBarcode = wms.ItemBarcode == "N/A" ? null : wms.ItemBarcode,
UserBarcode = wms.UserBarcode == "N/A" ? null : wms.UserBarcode,
wms.LoanCheckedOutDate,
wms.LoanDueDate,
wms.RecordDate
}).ToList();
logMessage($"Processed {records.Count} records");
try
{
destination.DataContext.BulkImportTransactions(records.ToDataReader(x => new object[] { x.ItemBarcode, x.UserBarcode, x.LoanDueDate, x.LoanCheckedOutDate, runDate, x.EventType, x.InstitutionName, x.RecordDate }));
UpdateHarvesterRecord(logMessage, sourceFiles, source.Name, _harvesterArgs);
}
catch (Exception ex)
{
Console.WriteLine(ex);
throw;
}
finally
{
destination.DataContext.Connection.Close();
destination.DataContext.Dispose();
}
}
}
}
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
Regex filePattern = new Regex(@"^((ITU[.]Item_inventories[.]\d{8})|(ITU[.]Circulation_Item_Inventories[.]\d{8}))[.]txt([.]zip)?$", RegexOptions.IgnoreCase);
var sourceFilesBeforeZip = source.ListFiles().Where(y => filePattern.IsMatch(y.Name)).ToArray();
foreach (DirectoryObjectMetadata zipFile in sourceFilesBeforeZip.Where(x => x.Path.Contains(".txt.zip")))
{
if (sourceFilesBeforeZip.All(x => x.Name != zipFile.Name.Replace(".zip", "")))
{
ZipFile.ExtractToDirectory(zipFile.Path, Path.GetDirectoryName(zipFile.Path.Replace(".txt.zip", "")));
}
//source.DeleteFile(zipFile.Path);
}
List <String> modified = new List <String>();
Int32 newCount = 0;
List <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/").Where(y => filePattern.IsMatch(y.Name) && !y.Name.EndsWith(".zip")).ToList();
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
{
logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
Entities.Repository repository = harvester.DataContext.Repositories.First(y => y.Name == source.Name);
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.OperationID == OperationID && d.RepositoryID == repository.ID).ToDictionary(d => d.FilePath);
foreach (DirectoryObjectMetadata file in sourceFiles)
{
if (!dictionary.ContainsKey(file.Path))
{
modified.Add(file.Name);
newCount++;
}
else
{
DirectoryRecord element = dictionary[file.Path];
if (file.ModifiedDate > element.FileModifiedDate)
{
modified.Add(file.Name);
}
}
}
}
if (modified.Count == 0 && newCount == 0)
{
logMessage("No Records to be processed.");
return;
}
logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_databaseArgs))
{
logMessage($"Connected to destination database '{destination.Name}' ({destination.ConnectionString})");
bool exceptionHandled = false;
List <DirectoryObjectMetadata> successfulFiles = new List <DirectoryObjectMetadata>();
StreamParser <WmsInventoryRecord> Parser = new StreamParser <WmsInventoryRecord>();
StreamParser <WmsInventoryRecordEdge> ParserEdge = new StreamParser <WmsInventoryRecordEdge>();
StreamParser <WmsInventoryRecordDiff> ParserDiff = new StreamParser <WmsInventoryRecordDiff>();
StreamParser <WmsInventoryRecord2018> Parser2018 = new StreamParser <WmsInventoryRecord2018>();
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
var completedFiles = modified.Select <string, (string file, DateTime RunDate, IEnumerable <IWmsInventoryRecord> records)>(file =>
{
logMessage($"Processing '{file}':");
Stream inputStream = source.OpenFile(file);
DateTime fileRunDate = GetFileDate(file);
string headers = ExtractHeader(inputStream);
//logMessage(headers);
if (IsWmsInventoryRecord(headers))
{
return(file, fileRunDate, Parser.ParseStream(inputStream));
}
if (IsWmsInventoryRecord2018(headers))
{
return(file, fileRunDate, Parser2018.ParseStream(inputStream));
}
if (IsWmsInventoryRecordDiff(headers))
{
return(file, fileRunDate, ParserDiff.ParseStream(inputStream));
}
if (IsWmsInventoryRecordEdge(headers))
{
return(file, fileRunDate, ParserEdge.ParseStream(inputStream));
}
throw new InvalidDataException($"Header format not recognized: '{headers}'");
}).Select(wmsFileRecord =>
{
var parsed = wmsFileRecord.records.Select(wms => new InventoryRecord
{
OclcNumber = wms.OclcNumber,
Title = ParseTitle(wms.Title),
Author = wms.Author,
MaterialFormat = wms.MaterialFormat.ToString(),
Barcode = wms.Barcode,
Cost = wms.Cost,
LastInventoriedDate = wms.LastInventoriedDate,
DeletedDate = wms.ItemDeletedDate,
ItemType = wms.ItemType.ToString(),
CallNumber = wms.CallNumber,
ShelvingLocation = wms.ShelvingLocation?.ToString(),
CurrentStatus = wms.CurrentStatus?.ToString(),
Description = wms.Description,
RunDate = wmsFileRecord.RunDate,
Anomalous = AnomalousBarcode(wms.Barcode),
}).Where(y => y.Title != null || y.Barcode != null).GroupBy(x => new { x.OclcNumber, x.Barcode, x.RunDate }).Select(x => x.First()).ToArray();
logMessage($"Records Found: {parsed.Length}");
if (cancellationToken.IsCancellationRequested)
{
logMessage("Operation was cancelled");
exceptionHandled = true;
return(null);
}
if (parsed.Length <= 0)
{
logMessage("Failed to parse properly and return any meaningful records. This might mean that non of the parsed records had a Title or Barcode.");
exceptionHandled = true;
return(null);
}
try
{
destination.DataContext.BulkImportInventory(
parsed.ToDataReader(r => new object[] { r.OclcNumber, r.Title, r.MaterialFormat, r.Author, r.Barcode, r.Cost, r.LastInventoriedDate, r.DeletedDate, r.ItemType, r.CallNumber, r.ShelvingLocation, r.CurrentStatus, r.Description, r.RunDate, r.Anomalous }));
}
catch (SqlException ex)
{
logMessage(ex.Message);
if (ex.InnerException != null)
{
logMessage(ex.InnerException.ToString());
}
logMessage("Sql Server was most likely put into an unusable state after this exception and thus the whole operation was canceled.");
exceptionHandled = true;
}
return(sourceFiles.First(x => x.Name == wmsFileRecord.file));
}).Where(x => x != null);
foreach (var success in completedFiles)
{
successfulFiles.Add(success);
}
UpdateHarvesterRecord(logMessage, successfulFiles, source.Name, _harvesterArgs);
if (exceptionHandled)
{
destination.DataContext.Connection.Close();
destination.DataContext.Dispose();
throw new Exception("An Exception was encountered. At least one file failed");
}
}
}
}
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(sourceDirectory))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(destinationDirectory))
{
logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})");
Regex filePattern = new Regex(arguments.FilePattern, RegexOptions.IgnoreCase);
List <DirectoryObjectMetadata> destinationFiles = destination.ListFiles("/").ToList();
int newCount = 0, modified = 0;
foreach (DirectoryObjectMetadata file in source.ListFiles("/").Where(x => filePattern.IsMatch(x.Name)))
{
try
{
if (!destinationFiles.Select(x => x.Name).Contains(file.Name))
{
logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}");
using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite))
{
using (Stream sourceStream = source.OpenFile(file.Name))
{
sourceStream.CopyTo(destStream);
}
}
destination.CopyFile("TMP" + file.Name, file.Name);
destination.DeleteFile("TMP" + file.Name);
logMessage($"Added {file.Name} from {source.Name} to {destination.Name}");
newCount++;
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
destination.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
}
else
{
DirectoryObjectMetadata destinationFile = destinationFiles.Find(x => x.Name == file.Name);
if (file.ModifiedDate > destinationFile.ModifiedDate)
{
logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}");
using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite))
{
using (Stream sourceStream = source.OpenFile(file.Name))
{
sourceStream.CopyTo(destStream);
}
}
destination.DeleteFile(file.Name);
destination.MoveFile("TMP" + file.Name, file.Name);
logMessage($"Updated {file.Name} from {source.Name} to {destination.Name}");
modified++;
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
destination.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
}
}
}
catch (Exception)
{
destination.DeleteFile("TMP" + file.Name);
throw;
}
}
logMessage($"Discovered {newCount} new files and {modified} updated files).");
}
}
}
