public async Task ImportsFactsAndAggregatesWhenNotProcessingAggregatesOnlyAndNotAlreadyImported()
{
// arrange
var statisticsBlobContainerUtilityMock = new Mock <IStatisticsBlobContainerUtility>(MockBehavior.Strict);
statisticsBlobContainerUtilityMock
.Setup(m => m.OpenCompressedBlobAsync(It.IsAny <ILeasedLogFile>()))
.Returns(OpenLeasedLogFileStream(_logFileName));
statisticsBlobContainerUtilityMock
.Setup(m => m.ArchiveBlobAsync(_leasedLogFile))
.Returns(Task.FromResult(0));
statisticsBlobContainerUtilityMock
.Setup(m => m.DeleteSourceBlobAsync(_leasedLogFile))
.Returns(Task.FromResult(0));
var warehouseMock = new Mock <IStatisticsWarehouse>(MockBehavior.Strict);
warehouseMock
.Setup(m => m.HasImportedPackageStatisticsAsync(_leasedLogFile.BlobName))
.Returns(Task.FromResult(false));
warehouseMock
.Setup(m => m.CreateAsync(It.IsAny <IReadOnlyCollection <PackageStatistics> >(), _leasedLogFile.BlobName))
.Returns(Task.FromResult(new DataTable()));
warehouseMock
.Setup(m => m.InsertDownloadFactsAsync(It.IsAny <DataTable>(), _leasedLogFile.BlobName))
.Returns(() => Task.FromResult(0));
warehouseMock
.Setup(m => m.StoreLogFileAggregatesAsync(It.IsAny <LogFileAggregates>()))
.Returns(() => Task.FromResult(0));
var logFileProcessor = new LogFileProcessor(
statisticsBlobContainerUtilityMock.Object,
_loggerFactory,
warehouseMock.Object,
_applicationInsightsHelper);
// act
await logFileProcessor.ProcessLogFileAsync(_leasedLogFile, _packageStatisticsParser, aggregatesOnly : false);
// assert
warehouseMock
.Verify(m => m.HasImportedPackageStatisticsAsync(_leasedLogFile.BlobName), Times.Once);
warehouseMock
.Verify(m => m.CreateAsync(It.IsAny <IReadOnlyCollection <PackageStatistics> >(), _leasedLogFile.BlobName), Times.Once);
warehouseMock
.Verify(m => m.InsertDownloadFactsAsync(It.IsAny <DataTable>(), _leasedLogFile.BlobName), Times.Once);
warehouseMock
.Verify(m => m.StoreLogFileAggregatesAsync(It.IsAny <LogFileAggregates>()), Times.Once);
statisticsBlobContainerUtilityMock
.Verify(m => m.ArchiveBlobAsync(_leasedLogFile), Times.Once);
statisticsBlobContainerUtilityMock
.Verify(m => m.DeleteSourceBlobAsync(_leasedLogFile), Times.Once);
}
public async Task DoesNotImportFactsWhenProcessingAggregatesOnly(bool hasImportedToolStatistics)
{
// arrange
var statisticsBlobContainerUtilityMock = new Mock <IStatisticsBlobContainerUtility>(MockBehavior.Strict);
statisticsBlobContainerUtilityMock
.Setup(m => m.OpenCompressedBlobAsync(It.IsAny <ILeasedLogFile>()))
.Returns(OpenLeasedLogFileStream(_logFileName));
var warehouseMock = new Mock <IStatisticsWarehouse>(MockBehavior.Strict);
warehouseMock
.Setup(m => m.HasImportedToolStatisticsAsync(_leasedLogFile.BlobName))
.Returns(Task.FromResult(hasImportedToolStatistics));
warehouseMock
.Setup(m => m.CreateAsync(It.IsAny <IReadOnlyCollection <ToolStatistics> >(), _leasedLogFile.BlobName))
.Returns(Task.FromResult(new DataTable()));
warehouseMock
.Setup(m => m.StoreLogFileAggregatesAsync(It.IsAny <LogFileAggregates>()))
.Returns(() => Task.FromResult(0));
var logFileProcessor = new LogFileProcessor(
statisticsBlobContainerUtilityMock.Object,
_loggerFactory,
warehouseMock.Object);
// act
await logFileProcessor.ProcessLogFileAsync(_leasedLogFile, _packageStatisticsParser, aggregatesOnly : true);
// assert
warehouseMock
.Verify(m => m.HasImportedToolStatisticsAsync(_leasedLogFile.BlobName), Times.Once);
warehouseMock
.Verify(m => m.CreateAsync(It.IsAny <IReadOnlyCollection <PackageStatistics> >(), _leasedLogFile.BlobName), Times.Never);
warehouseMock
.Verify(m => m.InsertDownloadFactsAsync(It.IsAny <DataTable>(), _leasedLogFile.BlobName), Times.Never);
warehouseMock
.Verify(m => m.StoreLogFileAggregatesAsync(It.IsAny <LogFileAggregates>()), Times.Never);
}
static void Main(string[] args)
{
var fileName = $"{AppSettings["logFilesBaseFolder"]}/{AppSettings["currentFileName"]}";
var resultList = new List <LogItem>();
var tasks = new List <Task>();
try
{
if (string.IsNullOrEmpty(fileName))
{
ConsoleLogger.LogError();
return;
}
var clientAddresses = LogFileProcessor.GetGroupedIpAddresses(fileName);
if (clientAddresses.Count == 0)
{
ConsoleLogger.LogError();
return;
}
var groupedAddresses = clientAddresses.GroupBy(x => x);
foreach (var address in groupedAddresses)
{
tasks.Add(Task.Run(() =>
{
try
{
var result = Dns.GetHostEntryAsync(address.Key);
resultList.Add(new LogItem(result.Result.HostName, address.Key, address.Count()));
}
catch (Exception)
{
resultList.Add(new LogItem(null, address.Key, address.Count()));
}
}));
}
Task task = Task.WhenAll(tasks);
ConsoleLogger.LogMessage(AppSettings["pleaseWait"]);
task.Wait();
if (task.Status == TaskStatus.RanToCompletion)
{
resultList = resultList.OrderByDescending(x => x.NumberOfHits).ToList();
foreach (var resultListItem in resultList)
{
ConsoleLogger.LogMessage(resultListItem.ToString());
}
ConsoleLogger.LogMessage(AppSettings["dnsSuccess"]);
}
else
{
ConsoleLogger.LogError();
}
}
catch (Exception)
{
ConsoleLogger.LogError();
}
Console.ReadLine();
}
static void Main(string[] args)
{
var logConfig = new LogConfiguration <DbInstallLogEntry>
{
FileLocateConfig = new FileLocateConfig
{
Directories = new List <string>()
{
@"C:\Dev\log-examples\dbinstall"
},
FileMasks = new List <string>()
{
@"DBInstall.log(.\d*)?"
}
},
StartCheckers = new List <IEntryStartChecker>()
{
new DbInstallEntryStartChecker()
},
EntryParsers = new List <IEntryParser <DbInstallLogEntry> >()
{
new DbInstallEntryParser()
},
MessageParsers = new List <IEntryMessageParser <DbInstallLogEntry> > {
new DacPacDeployStatusParser()
},
Locators = new List <IEntryCollector> {
new DacpacSuccessLocator(), new DbInstallStartLocator()
}
};
var locator = new FileLocator();
var files = locator.Locate(logConfig.FileLocateConfig).ToArray();
files.OrderByDescending(fi => fi.LastWriteTime).ToList().ForEach(f => Console.WriteLine(@"Name: {0}. Created: {1}", f.Name, f.LastWriteTime));
var logFileProcesser = new LogFileProcessor();
foreach (var fileInfo in files)
{
var entries = logFileProcesser.Process(fileInfo.FullName, logConfig.StartCheckers.ToArray(), logConfig.EntryParsers.ToArray()).Cast <DbInstallLogEntry>().ToArray();
var iterator = new EntryIterator <LogEntry>();
iterator.SetList(entries);
foreach (var item in entries)
{
iterator.SetCurrent(item);
logConfig.MessageParsers.ToList().ForEach(p => p.Parse(item));
logConfig.Locators.ToList().ForEach(c => c.Analize(item, iterator));
}
Console.WriteLine(@"File {0} has {1} entries. Errors {2}. Dacpack: {3}", fileInfo.Name, entries.Count(), entries.Count(e => e.IsParseError), entries.Count(e => e.IsDacpack));
entries.Where(e => e.DacpackStatus == "SUCCESS").ToList().ForEach(e => Console.WriteLine(e.Message));
}
var fileWatcher = new FileSystemWatcher();
fileWatcher.Path = Path.GetDirectoryName(files.First().FullName);
fileWatcher.Filter = files.First().Name;
fileWatcher.NotifyFilter = NotifyFilters.LastWrite;
fileWatcher.EnableRaisingEvents = true;
fileWatcher.Changed += (source, e) =>
{
//Console.WriteLine("{0} lines added to the file.", newLines.Count());
};
Console.ReadKey();
}
static void Main(string[] args)
{
// read in arguments, parse them for easy consumption
ParseCommandlineArguments(args);
// setup log processor
_logFileProcessor = LogFileProcessor.Create(_type);
_logFileProcessor.LogFiles = _logs.GetFiles(_logFilter);
// set up logger
Logger.DiskLog = _executionLog;
// Print welcome message
Logger.Info();
Logger.Info(" ##################################################");
Logger.Info(" # Apatche Log Files to Access Database Converter #");
Logger.Info(" # #");
Logger.Info(" # Version 0.3 #");
Logger.Info(" # By: Egil Hansen (http://egilhansen.com) #");
Logger.Info(" ##################################################");
Logger.Info();
Logger.Info("Runtime settings:");
Logger.Info();
Console.ForegroundColor = ConsoleColor.DarkBlue;
Logger.Info(" Database File (db) ................... : {0}", _dbFileName.FullName);
Logger.Info(" Overwrite Existing DB File (overwrite) : {0}", _overwrite.ToString());
Logger.Info(" Logs Directory (logs) ................ : {0}", _logs.FullName);
Logger.Info(" Log filter (filter) .................. : {0}", _logFilter);
Logger.Info(" Log File Type (type) ................. : {0}", _logFileProcessor.Name);
Logger.Info(" Runetime Log (runtimelog) ............ : {0}", _executionLog != null ? _executionLog.FullName : "n/a");
Logger.Info();
Console.ResetColor();
// create target database based on log file type
var csb = new OleDbConnectionStringBuilder {
DataSource = _dbFileName.FullName, Provider = "Microsoft.Jet.OLEDB.4.0"
};
// remove existing database file
if (_dbFileName.Exists && _overwrite)
{
_dbFileName.Delete();
}
// create database
if (_dbFileName.Exists)
{
AddTable(csb);
}
else
{
CreateAccessDatabase(csb.ToString());
}
// parse each log file, add each log entry to database
Logger.Info("Starting log file processing . . .");
Logger.LogOffscreen(string.Empty);
Logger.LogOffscreen("Start time: {0}", DateTime.Now);
Logger.Info();
_logFileProcessor.Process(csb.ToString());
}
