public void Execute_IntegrationTest_SQLite()
{
string filePath = Path.Combine(AppContext.BaseDirectory, Path.GetRandomFileName() + ".dbtest");
List <W3CEvent> logEvents = null;
using (StreamReader logStream = new StreamReader(TestAsset.ReadTextStream(TestAsset.LogFile)))
{
logEvents = W3CEnumerable.FromStream(logStream).ToList();
}
using (SQLiteDbContext dbContext = new SQLiteDbContext(filePath))
{
dbContext.Initialise();
dbContext.BeginTransaction();
// create the project first so we have one
ProjectModel project = DataHelper.CreateProjectModel();
DataHelper.InsertProjectModel(dbContext, project);
// create the log file
LogFileModel logFile = DataHelper.CreateLogFileModel(project.Id);
DataHelper.InsertLogFileModel(dbContext, logFile);
// create the request batch
ICreateRequestBatchCommand createRequestBatchCommand = new CreateRequestBatchCommand(dbContext, new RequestValidator());
createRequestBatchCommand.Execute(logFile.Id, logEvents);
int rowCount = dbContext.ExecuteScalar <int>("SELECT COUNT(*) FROM Requests");
Assert.AreEqual(logEvents.Count, rowCount);
}
}
public void W3C_Parsing()
{
var @event = W3CEnumerable.FromFile("u_ex130609.log").First();
Assert.AreEqual("::1", @event.c_ip, false, CultureInfo.InvariantCulture);
Assert.IsNull(@event.cs_Cookie);
Assert.IsNull(@event.cs_Referer);
Assert.AreEqual(@"Mozilla/5.0+(compatible;+MSIE+10.0;+Windows+NT+6.2;+WOW64;+Trident/6.0)", @event.cs_User_Agent, false, CultureInfo.InvariantCulture);
Assert.IsNull(@event.cs_bytes);
Assert.IsNull(@event.cs_host);
Assert.AreEqual("GET", @event.cs_method, false, CultureInfo.InvariantCulture);
Assert.IsNull(@event.cs_uri_query);
Assert.AreEqual(@"/", @event.cs_uri_stem, false, CultureInfo.InvariantCulture);
Assert.IsNull(@event.cs_username);
Assert.IsNull(@event.cs_version);
Assert.AreEqual(DateTimeKind.Unspecified, @event.dateTime.Kind);
Assert.AreEqual(new DateTime(635063969570000000L, DateTimeKind.Unspecified), @event.dateTime);
Assert.IsNull(@event.s_computername);
Assert.AreEqual("80", @event.s_port, false, CultureInfo.InvariantCulture);
Assert.IsNull(@event.s_sitename);
Assert.IsNull(@event.sc_bytes);
Assert.AreEqual("200", @event.sc_status, false, CultureInfo.InvariantCulture);
Assert.AreEqual("0", @event.sc_substatus, false, CultureInfo.InvariantCulture);
Assert.AreEqual("0", @event.sc_win32_status, false, CultureInfo.InvariantCulture);
Assert.AreEqual("156", @event.time_taken, false, CultureInfo.InvariantCulture);
}
public void W3CbasicRead()
{
var en = W3CEnumerable.FromFile("u_ex130609.log");
int count = en.Count();
Assert.AreEqual(17, count);
}
public EmptyResult Report()
{
var logfilePath = HttpContext.Server.MapPath("~/App_Data/access.log");
var iisLog = W3CEnumerable.FromFile(logfilePath);
List <ReportModel> report = new List <ReportModel>();
foreach (var line in iisLog.Where(x => !string.IsNullOrEmpty(x.c_ip) && !x.c_ip.StartsWith("207.114") && x.s_port == "80" && x.cs_method == "GET")
.GroupBy(x => x.c_ip)
.Select(group => new {
Counter = group.Count(),
IP = group.Key
})
.OrderByDescending(x => x.Counter)
.ThenByDescending(x => Version.Parse(x.IP.ToString())))
{
ReportModel reportItem = new ReportModel();
reportItem.Count = line.Counter;
reportItem.IP = IPAddress.Parse(line.IP);
report.Add(reportItem);
}
CSVExporter.WriteToCSV(report);
return(new EmptyResult());
}
public void GetByUriStemAggregate_Integration_ReturnsData()
{
string filePath = Path.Combine(AppContext.BaseDirectory, Path.GetRandomFileName() + ".dbtest");
List <W3CEvent> logEvents = null;
string uriStemAggregate = Guid.NewGuid().ToString();
int expectedCount = new Random().Next(3, 7);
using (StreamReader logStream = new StreamReader(TestAsset.ReadTextStream(TestAsset.LogFile)))
{
logEvents = W3CEnumerable.FromStream(logStream).ToList();
}
for (int i = 0; i < expectedCount; i++)
{
logEvents[i].cs_uri_stem = uriStemAggregate;
}
using (SQLiteDbContext dbContext = new SQLiteDbContext(filePath))
{
dbContext.Initialise();
dbContext.BeginTransaction();
IRequestRepository requestRepo = new RequestRepository(dbContext);
ICreateRequestBatchCommand createRequestBatchCommand = new CreateRequestBatchCommand(dbContext, new RequestValidator());
// create the project
ProjectModel project = DataHelper.CreateProjectModel();
DataHelper.InsertProjectModel(dbContext, project);
ProjectModel project2 = DataHelper.CreateProjectModel();
DataHelper.InsertProjectModel(dbContext, project2);
// create log file and request records for each
LogFileModel logFile = DataHelper.CreateLogFileModel(project.Id);
DataHelper.InsertLogFileModel(dbContext, logFile);
createRequestBatchCommand.Execute(logFile.Id, logEvents);
LogFileModel logFile2 = DataHelper.CreateLogFileModel(project2.Id);
DataHelper.InsertLogFileModel(dbContext, logFile2);
createRequestBatchCommand.Execute(logFile2.Id, logEvents);
IEnumerable <RequestModel> result = requestRepo.GetByUriStemAggregate(project.Id, uriStemAggregate);
Assert.IsNotNull(result);
Assert.AreEqual(expectedCount, result.Count());
foreach (RequestModel rm in result)
{
Assert.AreEqual(logFile.Id, rm.LogFileId);
}
}
}
public static IEnumerable <W3CEvent> GetLogs()
{
var logPath = ConfigurationManager.AppSettings["LogPath"];
var files = new DirectoryInfo(logPath).GetFiles();
var latest = files.OrderByDescending(f => f.CreationTime).FirstOrDefault();
var copy = latest.FullName + "copy";
File.Copy(latest.FullName, copy);//because IIS has lock on these files
var iisLog = W3CEnumerable.FromFile(copy);
var logs = iisLog.ToList();
File.Delete(copy);
return(logs.OrderByDescending(o => o.dateTime));
}
public void Execute_IntegrationTest_SQLite()
{
string filePath = Path.Combine(AppContext.BaseDirectory, Path.GetRandomFileName() + ".dbtest");
List <W3CEvent> logEvents = null;
using (StreamReader logStream = new StreamReader(TestAsset.ReadTextStream(TestAsset.LogFile)))
{
logEvents = W3CEnumerable.FromStream(logStream).ToList();
}
using (SQLiteDbContext dbContext = new SQLiteDbContext(filePath))
{
dbContext.Initialise();
dbContext.BeginTransaction();
ICreateProjectCommand createProjectCommand = new CreateProjectCommand(dbContext, new ProjectValidator());
ICreateRequestBatchCommand createRequestBatchCommand = new CreateRequestBatchCommand(dbContext, new RequestValidator());
IDeleteLogFileCommand deleteLogFileCommand = new DeleteLogFileCommand(dbContext);
// create the project first so we have one
ProjectModel project = DataHelper.CreateProjectModel();
// create 2 the log files
LogFileModel logFile1 = DataHelper.CreateLogFileModel(project.Id);
DataHelper.InsertLogFileModel(dbContext, logFile1);
LogFileModel logFile2 = DataHelper.CreateLogFileModel(project.Id);
DataHelper.InsertLogFileModel(dbContext, logFile2);
// create the request batch
createRequestBatchCommand.Execute(logFile1.Id, logEvents);
createRequestBatchCommand.Execute(logFile2.Id, logEvents);
int rowCount = dbContext.ExecuteScalar <int>("SELECT COUNT(*) FROM Requests WHERE LogFileId = @LogFileId", new { LogFileId = logFile1.Id });
Assert.AreEqual(logEvents.Count, rowCount);
// run the delete command
deleteLogFileCommand.Execute(logFile1.Id);
// there should be no requests for logFile1, but requests for logFile2 should still exist
rowCount = dbContext.ExecuteScalar <int>("SELECT COUNT(*) FROM Requests WHERE LogFileId = @LogFileId", new { LogFileId = logFile1.Id });
Assert.AreEqual(0, rowCount);
rowCount = dbContext.ExecuteScalar <int>("SELECT COUNT(*) FROM Requests WHERE LogFileId = @LogFileId", new { LogFileId = logFile2.Id });
Assert.AreEqual(logEvents.Count, rowCount);
rowCount = dbContext.ExecuteScalar <int>("SELECT COUNT(*) FROM LogFiles");
Assert.AreEqual(1, rowCount);
}
}
static void Main(string[] args)
{
var txFile = W3CEnumerable.FromFile("access.log"); //assumes file is in the same directory as the exe
var linqOutput = (from txresult in txFile where txresult.s_port == "80" && !txresult.s_ip.StartsWith("207.114") && txresult.cs_method == "GET" group txresult.c_ip by txresult.c_ip into h select new { Count = h.Count(), IP = h.Key });
using (TextWriter writer = new StreamWriter(@"C:\Users\John\output.csv", false, System.Text.Encoding.UTF8)) //replace output path as desired.
using (var csvWriter = new CsvWriter(writer))
{
csvWriter.Configuration.RegisterClassMap <LogClassMap>();
csvWriter.WriteRecords(linqOutput);
writer.Flush();
writer.Close();
}
}
public void GetByLogFile_Integration_ReturnsData()
{
string filePath = Path.Combine(AppContext.BaseDirectory, Path.GetRandomFileName() + ".dbtest");
List <W3CEvent> logEvents = null;
int logFileId = 0;
using (StreamReader logStream = new StreamReader(TestAsset.ReadTextStream(TestAsset.LogFile)))
{
logEvents = W3CEnumerable.FromStream(logStream).ToList().GetRange(0, 10);
}
using (SQLiteDbContext dbContext = new SQLiteDbContext(filePath))
{
dbContext.Initialise();
dbContext.BeginTransaction();
IRequestRepository requestRepo = new RequestRepository(dbContext);
ICreateRequestBatchCommand createRequestBatchCommand = new CreateRequestBatchCommand(dbContext, new RequestValidator());
// create the project
ProjectModel project = DataHelper.CreateProjectModel();
DataHelper.InsertProjectModel(dbContext, project);
// create multiple log file records
for (var i = 0; i < 3; i++)
{
LogFileModel logFile = DataHelper.CreateLogFileModel(project.Id);
DataHelper.InsertLogFileModel(dbContext, logFile);
createRequestBatchCommand.Execute(logFile.Id, logEvents);
if (logFileId == 0)
{
logFileId = logFile.Id;
}
}
IEnumerable <RequestModel> result = requestRepo.GetByLogFile(logFileId);
Assert.IsNotNull(result);
Assert.AreEqual(logEvents.Count, result.Count());
}
}
public static IEnumerable <ISiteLog> GetLogs(string logFolderPath)
{
Stream contentStream = LoadFileContent(logFolderPath);
if (contentStream == null)
{
return(null);
}
StreamReader reader = new StreamReader(contentStream);
IEnumerable <W3CEvent> events = W3CEnumerable.FromStream(reader);
ExcludeMediaItems(ref events); // Exclude js,css,image,font items from logs
IEnumerable <ISiteLog> logs = events.Select(x => new SiteLog
{
LogDate = x.dateTime,
ServerSiteName = x.s_sitename,
ServerComputerName = x.s_computername,
ServerIp = x.s_ip,
ServerPort = x.s_port,
ServerReceivedBytes = x.cs_bytes,
ServerResponseCode = x.sc_status,
ServerWin32Code = x.sc_win32_status,
ServerResponseSubStatus = x.sc_substatus,
ServerResponseTimeMiliseconds = x.time_taken,
ServerSentBytes = x.sc_bytes,
ClientRequestedMethod = x.cs_method,
ClientRequestedUri = x.cs_uri_stem,
ClientRequestedUriQuery = x.cs_uri_query,
ClientUserName = x.cs_username,
ClientIp = x.c_ip,
ClientBrowserVersion = x.cs_version,
ClientUserAgent = x.cs_User_Agent,
ClientRequestedCookie = x.cs_Cookie,
ClientRequestedHost = x.cs_host,
ClientRequestedReferer = x.cs_Referer
});
return(logs);
}
public void Execute_IntegrationTest_SQLite()
{
string filePath = Path.Combine(AppContext.BaseDirectory, Path.GetRandomFileName() + ".dbtest");
List <W3CEvent> logEvents1 = null;
List <W3CEvent> logEvents2 = new List <W3CEvent>();
using (StreamReader logStream = new StreamReader(TestAsset.ReadTextStream(TestAsset.LogFile)))
{
logEvents1 = W3CEnumerable.FromStream(logStream).ToList();
logEvents2.AddRange(logEvents1.GetRange(0, 10));
}
using (SQLiteDbContext dbContext = new SQLiteDbContext(filePath))
{
dbContext.Initialise();
ISetLogFileUnprocessedCommand setLogFileUnprocessedCommand = new SetLogFileUnprocessedCommand(dbContext, _jobRegistrationService);
// create the project first so we have one
ProjectModel project = DataHelper.CreateProjectModel();
DataHelper.InsertProjectModel(dbContext, project);
// create the log files
LogFileModel logFile1 = DataHelper.CreateLogFileModel(project.Id);
logFile1.Status = LogFileStatus.Complete;
DataHelper.InsertLogFileModel(dbContext, logFile1);
LogFileModel logFile2 = DataHelper.CreateLogFileModel(project.Id);
logFile2.Status = LogFileStatus.Complete;
DataHelper.InsertLogFileModel(dbContext, logFile2);
// check that the log file is processed
int processedCount = dbContext.ExecuteScalar <int>("SELECT COUNT(*) FROM LogFiles WHERE ProjectId = @ProjectId AND Status = @Status", new { ProjectId = project.Id, Status = LogFileStatus.Complete });
Assert.AreEqual(2, processedCount);
// execute for a single log file
setLogFileUnprocessedCommand.Execute(logFile1.Id);
processedCount = dbContext.ExecuteScalar <int>("SELECT COUNT(*) FROM LogFiles WHERE ProjectId = @ProjectId AND Status = @Status", new { ProjectId = project.Id, Status = LogFileStatus.Processing });
Assert.AreEqual(1, processedCount);
}
}
public void Execute(int logFileId, string filePath)
{
LogFileModel logFile = _logFileRepo.GetById(logFileId);
try
{
// load the contents of the file
List <W3CEvent> logEvents;
try
{
logEvents = W3CEnumerable.FromFile(filePath).ToList();
}
catch (Exception)
{
throw new FileFormatException("File is not a valid IIS log file");
}
// save the requests
_createRequestBatchCommand.Execute(logFileId, logEvents);
// update the record count of the log file
const string sql = "UPDATE LogFiles SET RecordCount = @RecordCount WHERE Id = @Id";
_dbContext.ExecuteNonQuery(sql, new { Id = logFileId, RecordCount = logEvents.Count });
// delete the file
_fileWrap.Delete(filePath);
// register the job that marks the file as needing aggregate processing
_jobRegistrationService.RegisterResetProcessedLogFileJob(logFile.Id);
}
catch (Exception ex)
{
// try and update the status of the log file record, the file will be marked as in an error state
logFile.Status = BLL.Lookup.LogFileStatus.Error;
logFile.ErrorMsg = ex.Message;
string sql = "UPDATE LogFiles SET Status = @Status, ErrorMsg = @ErrorMsg WHERE Id = @Id";
_dbContext.ExecuteNonQuery(sql, logFile);
}
}
public void Execute_ValidationSucceeds_BatchInserted()
{
int logFileId = new Random().Next(1, 1000);
_requestValidator.Validate(Arg.Any <RequestModel>()).Returns(new ValidationResult());
using (StreamReader logStream = new StreamReader(TestAsset.ReadTextStream(TestAsset.LogFile)))
{
var logEvents = W3CEnumerable.FromStream(logStream).ToList();
int eventCount = logEvents.Count;
Assert.Greater(eventCount, 1);
// execute
_createRequestBatchCommand.Execute(logFileId, logEvents);
// assert
_requestValidator.Received(eventCount).Validate(Arg.Any <RequestModel>());
// should receive eventCount + 1 -> delete also done
_dbContext.Received(eventCount + 1).ExecuteNonQuery(Arg.Any <string>(), Arg.Any <object>());
}
}
