public DirectoryService(IDirectoryRepository directoryRepository, IQueueService queueService, IQueueRepository queueRepository, ILogger logger)
{
_directoryRepository = directoryRepository;
_queueService = queueService;
_queueRepository = queueRepository;
_logger = logger;
}
public DirectoriesController(IWebHostEnvironment webHostEnvironment,
IDirectoryRepository directoryRepository,
ITranslator translator)
{
_webHostEnvironment = webHostEnvironment;
_directoryRepository = directoryRepository;
_translator = translator;
}
public EndpointResolverService(IDirectoryService directoryService, IDefaultEndpointConfiguration defaultEndpointConfiguration, IUrlBuilder urlBuilder, IDirectoryRepository directoryRepository, ILogger logger)
{
_directoryService = directoryService;
_defaultEndpointConfiguration = defaultEndpointConfiguration;
_urlBuilder = urlBuilder;
_directoryRepository = directoryRepository;
_logger = logger;
}
public void Init()
{
var connectionFactory = new OrmLiteConnectionFactory(":memory:", SqliteDialect.Provider);
_directoryRepository = new OrmLiteDirectoryRepository(connectionFactory);
_directoryRepository.InitSchema();
_directoryRepository.RegisterDirectory(OrmLiteDirectoryRepositoryTest.Directory1);
_directoryRepository.RegisterDirectory(OrmLiteDirectoryRepositoryTest.Directory2);
}
public ConvertTextureCommandHandler(
IDirectoryRepository directoryRepository,
ILogger logger,
ITextureService textureService)
{
_directoryRepository = directoryRepository;
_logger = logger;
_textureService = textureService;
}
private static Mock <IServiceBase> MockAuthService(IDirectoryRepository repository, MockHttpRequest request = null)
{
request = request ?? new MockHttpRequest();
var mockAuthService = new Mock <IServiceBase>();
mockAuthService.SetupGet(s => s.Request).Returns(request);
System.Func <IDirectoryRepository> resolveRepo = () => repository;
mockAuthService.Setup(x => x.TryResolve <IDirectoryRepository>()).Returns(resolveRepo);
return(mockAuthService);
}
public FilesController(IWebHostEnvironment webHostEnvironment,
IDirectoryRepository directoryRepository,
IUploadedFileRepository uploadedFileRepository,
ITranslator translator)
{
_webHostEnvironment = webHostEnvironment;
_directoryRepository = directoryRepository;
_uploadedFileRepository = uploadedFileRepository;
_translator = translator;
}
public FilesManager(ICache <FilesManager> cache, IEmailSender emailSender,
IFilesRepository fileRepo, IDirectoryRepository direRepo,
IDirectoryObjectRepository direObjRepo)
{
this.emailSender = emailSender;
this.cache = cache;
this.fileRepo = fileRepo;
this.direRepo = direRepo;
this.direObjRepo = direObjRepo;
}
public static async Task <Directory> GetOrFailAsync(this IDirectoryRepository repository, int id)
{
var directory = await repository.GetAsync(id);
if (directory == null)
{
throw new Exception($"Directory with id '{id}' does not exist.");
}
return(directory);
}
public LoadService(IDataService dataService,
IWebService webService,
IBrandRepository brandRepository,
IDirectoryRepository directoryRepository)
{
this.dataService = dataService;
this.webService = webService;
this.brandRepository = brandRepository;
this.directoryRepository = directoryRepository;
catalogRepository = new CatalogRepository(dataService);
photoRepository = new PhotoRepository(dataService);
}
public MatrixCalculatorService(ILogger <MatrixCalculatorService> logger,
IOptions <CalculatorServiceOptions> options,
IDirectoryRepository directoryRepository,
IMatrixRepository matrixRepository,
ICalculator calculator)
{
_logger = logger;
_directoryRepository = directoryRepository;
_matrixRepository = matrixRepository;
_options = options.Value;
_calculator = calculator;
}
public IEnumerable <CounterRecord> RequestRecords(DateTime runDate, CounterReport report)
{
if (_arguments.JsonRepository != null)
{
using (IDirectoryRepository localJsonRepo = RepositoryFactory.CreateDirectoryRepository(_arguments.JsonRepository))
{
List <DirectoryObjectMetadata> jsonFiles = localJsonRepo.ListFiles().ToList();
if (jsonFiles.Any(x => x.Name == $"{Name} {runDate:yyyy-MM-dd} {report}.json"))
{
DirectoryObjectMetadata directoryObjectMetadata = jsonFiles.First(x => x.Name == $"{Name} {runDate:yyyy-MM-dd} {report}.json");
using (Stream openFile = localJsonRepo.OpenFile(directoryObjectMetadata.Name))
{
string jsonString = new StreamReader(openFile).ReadToEnd();
foreach (var record in JsonConvert.DeserializeObject <IEnumerable <CounterRecord> >(jsonString))
{
yield return(record);
}
}
yield break;
}
}
}
LogMessage($"Requesting {report.ToString()}");
var reportItems = GetResponse(runDate, report.ToString());
if (reportItems == null)
{
yield break;
}
foreach (IReportItem reportItem in reportItems)
{
if (reportItem.ItemDataType == Sushi.DataType.Platform && string.IsNullOrWhiteSpace(reportItem.ItemName))
{
reportItem.ItemName = reportItem.ItemPublisher;
}
yield return(new CounterRecord
{
ItemName = reportItem.ItemName,
ItemPlatform = reportItem.ItemPlatform,
ItemType = Convert(reportItem.ItemDataType),
RunDate = runDate,
Identifiers = reportItem.ItemIdentifier.Bind(a => a.Select(i => Convert(i, reportItem.ItemPlatform, reportItem.ItemDataType)).Where(i => i != null).ToArray(), new CounterIdentifier[] { }),
Metrics = reportItem.ItemPerformance.SelectMany(m => m.Instance).Select(i => new CounterMetric {
MetricType = Convert(i.MetricType), MetricValue = Int32.Parse(i.Count)
}).ToArray()
});
}
}
private IEnumerable <EZProxyLog> ParseFile(IDirectoryRepository source, string file)
{
Stream stream = source.OpenFile(file);
StreamReader reader = new StreamReader(stream);
string line;
int index = 0;
while ((line = reader.ReadLine()) != null)
{
index++;
bool overall = overallExpression.IsMatch(line);
if (!overall && !legacyExpression.IsMatch(line))
{
LogMessage($" ERROR on Line {index} Unknown log format found: {line}");
continue;
//throw new InvalidDataException("ERROR Unknown log format found: " + line);
}
Match match = overall ? overallExpression.Match(line) : legacyExpression.Match(line);
GroupCollection groups = match.Groups;
if (overall)
{
yield return(new EZProxyLog
{
IP = groups["IP"].Value,
Username = ParseUsername(groups["Username"].Value),
DateTime = ParseDate(groups["DateTime"].Value),
Request = groups["Request"].Value,
HTTPCode = Int16.Parse(groups["HTTPCode"].Value),
BytesTransferred = Int32.Parse(groups["BytesTransferred"].Value),
Referer = ParseNull(groups["Referer"].Value),
UserAgent = ParseNull(groups["UserAgent"].Value),
});
}
else
{
yield return(new EZProxyLog
{
IP = groups["IP"].Value,
Username = ParseUsername(groups["Username"].Value),
DateTime = ParseDate(groups["DateTime"].Value),
Request = groups["Request"].Value,
HTTPCode = Int16.Parse(groups["HTTPCode"].Value),
BytesTransferred = Int32.Parse(groups["BytesTransferred"].Value),
});
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="FileFsClient"/> class.
/// </summary>
/// <param name="fileRepository">File repository instance.</param>
/// <param name="directoryRepository">Directory repository instance.</param>
/// <param name="entryRepository">Entry repository instance.</param>
/// <param name="externalFileManager">External file manager instance.</param>
/// <param name="optimizer">Optimizer instance.</param>
/// <param name="transactionWrapper">Transaction wrapper instance.</param>
/// <param name="storageOperationLocker">Storage operation locker instance.</param>
public FileFsClient(
IFileRepository fileRepository,
IDirectoryRepository directoryRepository,
IEntryRepository entryRepository,
IExternalFileManager externalFileManager,
IStorageOptimizer optimizer,
ITransactionWrapper transactionWrapper,
IStorageOperationLocker storageOperationLocker)
{
_fileRepository = fileRepository;
_directoryRepository = directoryRepository;
_entryRepository = entryRepository;
_externalFileManager = externalFileManager;
_optimizer = optimizer;
_transactionWrapper = transactionWrapper;
_storageOperationLocker = storageOperationLocker;
}
public GameSettingsProcessWatcher(
Plugins plugins,
string communicationAddress,
ushort communicationPort,
ComponentAssembler assembler,
IDirectoryRepository directoryRepository,
IGameRepository gameRepository,
IInjectionService injectionService,
ILogger logger,
IProcessWatcher processWatcher)
{
_plugins = plugins;
_communicationAddress = communicationAddress;
_communicationPort = communicationPort;
_assembler = assembler;
_directoryRepository = directoryRepository;
_injectionService = injectionService;
_gameRepository = gameRepository;
_logger = logger;
_processWatcher = processWatcher;
}
public GameSettingsWatchableProcess(
AggregateId gameId,
Plugins plugins,
string communicationAddress,
ushort communicationPort,
ComponentAssembler assembler,
IDirectoryRepository directoryRepository,
IGameRepository gamesRepository,
IInjectionService injectionService,
ILogger logger)
{
GameId = gameId;
_plugins = plugins;
_communicationAddress = communicationAddress;
_communicationPort = communicationPort;
_assembler = assembler;
_directoryRepository = directoryRepository;
_gameRepository = gamesRepository;
_injectionService = injectionService;
_logger = logger;
}
public DataManager(IDirectoryRepository directory, IMaterialRepository material)
{
Directory = directory;
Material = material;
}
public DataManager(IDirectoryRepository directoryRepository, IContentRepository contentRepository)
{
DirectoryRepository = directoryRepository;
ContentRepository = contentRepository;
}
public ActivityDirectory(IDirectoryRepository repository)
{
_repository = repository;
}
public AlcoholDirectory(IDirectoryRepository repository)
{
_repository = repository;
}
public override void Execute(DateTime runDate, Action <string> logMessage, System.Threading.CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(_logDirectoryArgs))
{
logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})");
Regex filePattern = new Regex(_arguments.FilePattern);
foreach (DirectoryObjectMetadata file in source.ListFiles().Where(x => x.Path.Contains(".zip")))
{
string tempZipDirectoryPath = file.Path.Replace(".zip", "");
string tempZipDirectoryName = file.Name.Replace(".zip", "");
ZipFile.ExtractToDirectory(file.Path, tempZipDirectoryPath);
source.DeleteFile(file.Path);
foreach (String unzippedfile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => filePattern.IsMatch(x)))
{
string filename = unzippedfile.Split(new[] { "\\" }, StringSplitOptions.None).Last();
List <String> currentFiles = source.ListFiles().Select(x => x.Name).ToList();
if (!currentFiles.Contains(filename))
{
source.MoveFile(unzippedfile, filename);
}
}
foreach (String gzipFile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => x.Contains(".gz")))
{
string fileNameConcat = tempZipDirectoryName + ".log";
List <String> currentFiles = destination.ListFiles().Select(x => x.Name).ToList();
if (!currentFiles.Contains(fileNameConcat))
{
using (GZipStream gzipStream = new GZipStream(source.OpenFile(gzipFile), CompressionMode.Decompress))
{
using (Stream unzippedDestination = destination.CreateFile(fileNameConcat, Repository.Directory.FileCreationMode.ThrowIfFileExists))
{
gzipStream.CopyTo(unzippedDestination);
}
}
}
}
source.DeleteDirectory(tempZipDirectoryPath);
}
}
List <String> modified = new List <String>();
Int32 newCount = 0;
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
{
logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
IEnumerable <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/");
Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.Operation.Name == Name && d.Repository.Name == source.Name).ToDictionary(d => d.FilePath);
Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name);
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
foreach (DirectoryObjectMetadata file in sourceFiles)
{
if (!dictionary.ContainsKey(file.Path))
{
modified.Add(file.Name);
newCount++;
harvester.DataContext.DirectoryRecords.InsertOnSubmit(new DirectoryRecord
{
OperationID = OperationID,
RepositoryID = repository.ID,
FilePath = file.Path,
FileModifiedDate = file.ModifiedDate,
CreationDate = DateTime.Now,
ModifiedDate = DateTime.Now
});
}
else
{
DirectoryRecord element = dictionary[file.Path];
if (file.ModifiedDate > element.FileModifiedDate)
{
modified.Add(file.Name);
element.FileModifiedDate = file.ModifiedDate;
element.ModifiedDate = DateTime.Now;
}
}
}
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
harvester.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
harvester.DataContext.SubmitChanges();
}
logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");
if (modified.Count == 0)
{
return;
}
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_statisticsArgs))
{
logMessage($"Connected to database '{destination.Name}' ({destination.ConnectionString})");
StreamParser <EZProxyAudit> Parser = new StreamParser <EZProxyAudit>();
List <EZProxyAudit> records = modified.Select(file =>
{
logMessage($"Processing '{file}':");
int lineNumber = 0;
return(Parser.ParseStream(source.OpenFile(file)).Select(x => new EZProxyAudit
{
DateTime = x.DateTime,
Event = x.Event,
IP = x.IP,
Other = x.Other,
Session = x.Session,
Username = x.Username,
LineNumber = lineNumber++,
}));
}).SelectMany(x => x).ToList();
logMessage($"Records Found: {records.Count}");
destination.DataContext.BulkImportEZProxyAudit(records.ToDataReader(r => new object[] { r.DateTime, r.Event, r.IP, r.Username, r.Session, r.Other, r.LineNumber }));
}
}
}
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
List <String> modified = new List <String>();
Int32 newCount = 0;
List <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/").Where(y => y.Name.ToLower().StartsWith("daily_transaction_export_for_vamp") && (y.Name.Contains(".txt") || y.Name.Contains(".csv"))).ToList();
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
{
logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.OperationID == OperationID && d.RepositoryID == harvester.DataContext.Repositories.First(y => y.Name == source.Name).ID).ToDictionary(d => d.FilePath);
foreach (DirectoryObjectMetadata file in sourceFiles)
{
if (!dictionary.ContainsKey(file.Path))
{
modified.Add(file.Name);
newCount++;
}
else
{
DirectoryRecord element = dictionary[file.Path];
if (file.ModifiedDate > element.FileModifiedDate)
{
modified.Add(file.Name);
}
}
}
}
logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");
if (modified.Count == 0 && newCount == 0)
{
logMessage("No Records to be processed.");
return;
}
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_databaseArgs))
{
logMessage($"Connected to destination database '{destination.Name}' ({destination.ConnectionString})");
var records = modified.Select(file =>
{
logMessage($"Processing '{file}':");
Stream inputStream = source.OpenFile(file);
CountingEnumerable <WmsTransactionRecord> counter = new CountingEnumerable <WmsTransactionRecord>(ReadLines(inputStream, GetFileDate(file)));
return(Tuple.Create(counter, (Action)(() =>
{
inputStream.Dispose();
logMessage($"\t{counter.Count} records processed.");
})));
}).SelectMany(i => i).Select(wms => new
{
wms.EventType,
wms.InstitutionName,
ItemBarcode = wms.ItemBarcode == "N/A" ? null : wms.ItemBarcode,
UserBarcode = wms.UserBarcode == "N/A" ? null : wms.UserBarcode,
wms.LoanCheckedOutDate,
wms.LoanDueDate,
wms.RecordDate
}).ToList();
logMessage($"Processed {records.Count} records");
try
{
destination.DataContext.BulkImportTransactions(records.ToDataReader(x => new object[] { x.ItemBarcode, x.UserBarcode, x.LoanDueDate, x.LoanCheckedOutDate, runDate, x.EventType, x.InstitutionName, x.RecordDate }));
UpdateHarvesterRecord(logMessage, sourceFiles, source.Name, _harvesterArgs);
}
catch (Exception ex)
{
Console.WriteLine(ex);
throw;
}
finally
{
destination.DataContext.Connection.Close();
destination.DataContext.Dispose();
}
}
}
}
public override void Execute(DateTime runDate, Action <string> logMessage, System.Threading.CancellationToken cancellationToken)
{
LogMessage = logMessage;
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
List <String> modified = new List <String>();
Int32 newCount = 0;
IEnumerable <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/").ToList();
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
{
logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.Operation.Name == Name && d.Repository.Name == source.Name).ToDictionary(d => d.FilePath);
foreach (DirectoryObjectMetadata file in sourceFiles)
{
if (!dictionary.ContainsKey(file.Path))
{
modified.Add(file.Name);
newCount++;
}
else
{
DirectoryRecord element = dictionary[file.Path];
if (file.ModifiedDate > element.FileModifiedDate)
{
modified.Add(file.Name);
}
}
}
}
if (modified.Count == 0)
{
logMessage("No new files to process");
return;
}
logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_statisticsArgs))
{
logMessage($"Connected to database '{destination.Name}' ({destination.ConnectionString})");
var successfulFiles = new List <DirectoryObjectMetadata>();
var records = modified.Select(file =>
{
IEnumerable <EZProxyLog> logs = ParseFile(source, file);
successfulFiles.Add(sourceFiles.First(x => x.Name == file));
return(Tuple.Create(logs, (Action)(() =>
{
logMessage($"Parsed file {file}");
})));
}).SelectMany(x => x);
try
{
destination.DataContext.BulkImportEZProxyLog(records.ToDataReader(r => new object[] { r.IP, r.Username, r.DateTime, r.Request, r.HTTPCode, r.BytesTransferred, r.Referer, r.UserAgent }));
}
catch (Exception ex)
{
Console.WriteLine(ex);
UpdateHarvesterRecord(LogMessage, successfulFiles, source.Name, _harvesterArgs);
throw;
}
finally
{
destination.DataContext.Connection.Close();
destination.DataContext.Dispose();
}
UpdateHarvesterRecord(LogMessage, successfulFiles, source.Name, _harvesterArgs);
}
}
}
public SmokeDirectory(IDirectoryRepository repository)
{
_repository = repository;
}
/// <inheritdoc />
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(statisticsArguments))
{
destination.DataContext.LogMessage = logMessage;
logMessage($"Connected to destination database '{destination.Name}' ({destination.ConnectionString})");
using (ICounterRepository source = RepositoryFactory.CreateCounterRepository(counterRepoArguments))
{
source.LogMessage = logMessage;
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
Dictionary <CounterReport, List <CounterRecord> > preRecordsReports = source.AvailableReports.ToDictionary(report => report, report => source.RequestRecords(runDate, report).ToList());
if (!preRecordsReports.Any(x => x.Value.Count > 0))
{
logMessage("No Records for this date");
return;
}
if (localJsonArguments != null)
{
foreach (var reportRecord in preRecordsReports)
{
using (IDirectoryRepository directoryRepo = RepositoryFactory.CreateDirectoryRepository(localJsonArguments))
{
using (Stream stream = directoryRepo.CreateFile($"{source.Name} {runDate:yyyy-MM-dd} {reportRecord.Key}.json", FileCreationMode.Overwrite))
{
JsonSerializer serializer = JsonSerializer.Create();
using (JsonTextWriter jsonTextWriter = new JsonTextWriter(new StreamWriter(stream)))
{
serializer.Serialize(jsonTextWriter, reportRecord.Value);
}
}
}
}
}
var preRecords = preRecordsReports.SelectMany(x => x.Value).ToList();
//Add Metrics of vendorRecords to single vendor record
foreach (var recordsByRunDate in preRecords.GroupBy(x => x.RunDate))
{
if (recordsByRunDate.Any(x => x.ItemType == ItemType.Vendor))
{
foreach (var vendorRecord in recordsByRunDate.Where(x => x.ItemType == ItemType.Vendor))
{
vendorRecord.Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Proprietary, IdentifierValue = source.Name
} };
}
}
else
{
preRecords.Add(new CounterRecord {
ItemName = source.Name, ItemType = ItemType.Vendor, RunDate = recordsByRunDate.Select(y => y.RunDate).First(), Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Proprietary, IdentifierValue = source.Name
} }, Metrics = new CounterMetric[] { }
});
}
}
if (preRecords.Any(x => x.ItemType == ItemType.Database))
{
foreach (CounterRecord record in preRecords)
{
if (record.ItemType != ItemType.Database)
{
continue;
}
record.ItemPlatform = source.Name;
record.Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Database, IdentifierValue = record.ItemName
} };
}
}
else
{
preRecords.Add(new CounterRecord {
ItemName = source.Name, ItemType = ItemType.Database, ItemPlatform = source.Name, RunDate = runDate, Identifiers = new[] { new CounterIdentifier {
IdentifierType = IdentifierType.Database, IdentifierValue = source.Name
} }, Metrics = new CounterMetric[] { }
});
}
var where = preRecords.Where(x => x.ItemType == ItemType.Vendor);
Console.WriteLine(where.Count());
var records = preRecords
.Where(x => !string.IsNullOrEmpty(x.ItemName))
.Select(SanitizeIdentifiers)
.GroupBy(x => x, new CounterRecordComparer())
.Select(AggregateDuplicates)
.Select((r, i) => new { Index = i, Record = r });
var counterRecords = records.GroupBy(x => x.Record.RunDate).ToList();
logMessage($"{preRecords.Count + 1} Total Records");
logMessage($"{recordswithNoIdentifiers} Records with no Identifiers");
logMessage($"{counterRecords.Sum(x => x.Count())} Unique Records");
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
destination.DataContext.Connection.Close();
destination.DataContext.Dispose();
destination.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
foreach (var counterGroup in counterRecords)
{
destination.DataContext.BulkImportCounterTransactions(
counterGroup
.ToDataReader(r => new object[] { null, null, null, r.Index, r.Record.ItemName, r.Record.ItemPlatform, r.Record.ItemType, r.Record.RunDate }),
counterGroup
.SelectMany(r => r.Record.Identifiers.Bind(a => a.Select(i => new { r.Index, Identifier = i, r.Record.ItemType })))
.ToDataReader(i => new object[] { i.Index, i.Identifier.IdentifierType, i.Identifier.IdentifierValue, i.ItemType }),
counterGroup
.SelectMany(r => r.Record.Metrics.Bind(a => a.Select(m => new { r.Index, Metric = m })))
.ToDataReader(m => new object[] { m.Index, m.Metric.MetricType, m.Metric.MetricValue }));
//Add Record of report being run
using (var harvester = RepositoryFactory.CreateHarvesterRepository(harvesterArguments))
{
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name);
foreach (CounterReport report in preRecordsReports.Keys)
{
if (!harvester.DataContext.CounterOperationRecords.Any(x => x.OperationID == OperationID && x.RunDate == runDate && x.Report == report.ToString()))
{
harvester.DataContext.CounterOperationRecords.InsertOnSubmit(new CounterOperationRecord
{
OperationID = OperationID,
RepositoryID = repository.ID,
RunDate = runDate,
Report = report.ToString(),
ExecutedDate = DateTime.Now
});
}
else
{
harvester.DataContext.CounterOperationRecords.First(x => x.OperationID == OperationID && x.RunDate == runDate && x.Report == report.ToString()).ExecutedDate = DateTime.Now;
}
}
harvester.DataContext.SubmitChanges();
}
}
}
}
}
public DirectoryService(IDirectoryRepository directoryRepository, IDocumentRepository documentRepository)
{
_directoryRepository = directoryRepository;
_documentRepository = documentRepository;
}
public DataManager(IDirectoryRepository directoryRepository, IMaterialRepository materialRepository)
{
_directoryRepository = directoryRepository;
_materialRepository = materialRepository;
}
public DirectoryService(IDirectoryRepository directoryRepository, IMapper mapper)
{
_directoryRepository = directoryRepository;
_mapper = mapper;
}
public SexDirectory(IDirectoryRepository repository)
{
_repository = repository;
}
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
Regex filePattern = new Regex(@"^((ITU[.]Item_inventories[.]\d{8})|(ITU[.]Circulation_Item_Inventories[.]\d{8}))[.]txt([.]zip)?$", RegexOptions.IgnoreCase);
var sourceFilesBeforeZip = source.ListFiles().Where(y => filePattern.IsMatch(y.Name)).ToArray();
foreach (DirectoryObjectMetadata zipFile in sourceFilesBeforeZip.Where(x => x.Path.Contains(".txt.zip")))
{
if (sourceFilesBeforeZip.All(x => x.Name != zipFile.Name.Replace(".zip", "")))
{
ZipFile.ExtractToDirectory(zipFile.Path, Path.GetDirectoryName(zipFile.Path.Replace(".txt.zip", "")));
}
//source.DeleteFile(zipFile.Path);
}
List <String> modified = new List <String>();
Int32 newCount = 0;
List <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/").Where(y => filePattern.IsMatch(y.Name) && !y.Name.EndsWith(".zip")).ToList();
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
{
logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
Entities.Repository repository = harvester.DataContext.Repositories.First(y => y.Name == source.Name);
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.OperationID == OperationID && d.RepositoryID == repository.ID).ToDictionary(d => d.FilePath);
foreach (DirectoryObjectMetadata file in sourceFiles)
{
if (!dictionary.ContainsKey(file.Path))
{
modified.Add(file.Name);
newCount++;
}
else
{
DirectoryRecord element = dictionary[file.Path];
if (file.ModifiedDate > element.FileModifiedDate)
{
modified.Add(file.Name);
}
}
}
}
if (modified.Count == 0 && newCount == 0)
{
logMessage("No Records to be processed.");
return;
}
logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_databaseArgs))
{
logMessage($"Connected to destination database '{destination.Name}' ({destination.ConnectionString})");
bool exceptionHandled = false;
List <DirectoryObjectMetadata> successfulFiles = new List <DirectoryObjectMetadata>();
StreamParser <WmsInventoryRecord> Parser = new StreamParser <WmsInventoryRecord>();
StreamParser <WmsInventoryRecordEdge> ParserEdge = new StreamParser <WmsInventoryRecordEdge>();
StreamParser <WmsInventoryRecordDiff> ParserDiff = new StreamParser <WmsInventoryRecordDiff>();
StreamParser <WmsInventoryRecord2018> Parser2018 = new StreamParser <WmsInventoryRecord2018>();
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
var completedFiles = modified.Select <string, (string file, DateTime RunDate, IEnumerable <IWmsInventoryRecord> records)>(file =>
{
logMessage($"Processing '{file}':");
Stream inputStream = source.OpenFile(file);
DateTime fileRunDate = GetFileDate(file);
string headers = ExtractHeader(inputStream);
//logMessage(headers);
if (IsWmsInventoryRecord(headers))
{
return(file, fileRunDate, Parser.ParseStream(inputStream));
}
if (IsWmsInventoryRecord2018(headers))
{
return(file, fileRunDate, Parser2018.ParseStream(inputStream));
}
if (IsWmsInventoryRecordDiff(headers))
{
return(file, fileRunDate, ParserDiff.ParseStream(inputStream));
}
if (IsWmsInventoryRecordEdge(headers))
{
return(file, fileRunDate, ParserEdge.ParseStream(inputStream));
}
throw new InvalidDataException($"Header format not recognized: '{headers}'");
}).Select(wmsFileRecord =>
{
var parsed = wmsFileRecord.records.Select(wms => new InventoryRecord
{
OclcNumber = wms.OclcNumber,
Title = ParseTitle(wms.Title),
Author = wms.Author,
MaterialFormat = wms.MaterialFormat.ToString(),
Barcode = wms.Barcode,
Cost = wms.Cost,
LastInventoriedDate = wms.LastInventoriedDate,
DeletedDate = wms.ItemDeletedDate,
ItemType = wms.ItemType.ToString(),
CallNumber = wms.CallNumber,
ShelvingLocation = wms.ShelvingLocation?.ToString(),
CurrentStatus = wms.CurrentStatus?.ToString(),
Description = wms.Description,
RunDate = wmsFileRecord.RunDate,
Anomalous = AnomalousBarcode(wms.Barcode),
}).Where(y => y.Title != null || y.Barcode != null).GroupBy(x => new { x.OclcNumber, x.Barcode, x.RunDate }).Select(x => x.First()).ToArray();
logMessage($"Records Found: {parsed.Length}");
if (cancellationToken.IsCancellationRequested)
{
logMessage("Operation was cancelled");
exceptionHandled = true;
return(null);
}
if (parsed.Length <= 0)
{
logMessage("Failed to parse properly and return any meaningful records. This might mean that non of the parsed records had a Title or Barcode.");
exceptionHandled = true;
return(null);
}
try
{
destination.DataContext.BulkImportInventory(
parsed.ToDataReader(r => new object[] { r.OclcNumber, r.Title, r.MaterialFormat, r.Author, r.Barcode, r.Cost, r.LastInventoriedDate, r.DeletedDate, r.ItemType, r.CallNumber, r.ShelvingLocation, r.CurrentStatus, r.Description, r.RunDate, r.Anomalous }));
}
catch (SqlException ex)
{
logMessage(ex.Message);
if (ex.InnerException != null)
{
logMessage(ex.InnerException.ToString());
}
logMessage("Sql Server was most likely put into an unusable state after this exception and thus the whole operation was canceled.");
exceptionHandled = true;
}
return(sourceFiles.First(x => x.Name == wmsFileRecord.file));
}).Where(x => x != null);
foreach (var success in completedFiles)
{
successfulFiles.Add(success);
}
UpdateHarvesterRecord(logMessage, successfulFiles, source.Name, _harvesterArgs);
if (exceptionHandled)
{
destination.DataContext.Connection.Close();
destination.DataContext.Dispose();
throw new Exception("An Exception was encountered. At least one file failed");
}
}
}
}
public DirectoryService(IDirectoryRepository directoryRepository)
{
_directoryRepository = directoryRepository;
}
public InteresDirectory(IDirectoryRepository repository)
{
_repository = repository;
}
