public LogReader()
{
_logQuery = new LogQueryClass();
_input = new COMTSVInputContextClass();
_input.headerRow = false;
_input.iSeparator = "|";
_input.nFields = 3;
}
public int ProcessLogs(string query)
{
LogQueryClass log = new LogQueryClass();
//the input context class type will need changing depending on the type of log
var recordSet = log.Execute(query, new COMW3CInputContextClass());
var enumerableSet = new EnumerableLogRecordSet(recordSet);
var columnCount = recordSet.getColumnCount();
var lastFileName = "File Name";
var lastFileResultCount = 0;
var totalLines = 0;
var toWrite = new List<EventData>();
foreach (var record in enumerableSet)
{
totalLines++;
var result = new JObject();
for (int i = 0; i < columnCount; i++)
{
var value = record.getValue(i);
if (value.GetType() != typeof(DBNull))
{
var colName = recordSet.getColumnName(i);
if (colName.Equals("LogFilename")) {
if (lastFileName.Equals(value))
{
lastFileResultCount++;
}
else
{
_connection.AppendToStreamAsync("logEvents", ExpectedVersion.Any, toWrite).Wait();
//toWrite.Clear();
Console.WriteLine("File {0} - {1} entries", lastFileName, toWrite.Count);
toWrite = new List<EventData>();
lastFileName = value;
lastFileResultCount = 1;
}
}
result[colName] = value;
}
}
toWrite.Add(ToEvent(result));
}
return totalLines;
}
static List <string> ScanRegistry()
{
List <string> keys = new List <string>();
ILogRecordset rs = null;
try
{
LogQueryClass logQ = new LogQueryClass();
COMRegistryInputContextClass registryFormat = new COMRegistryInputContextClass();
string query = @"SELECT Path FROM \ where Value LIKE 'o2Mate%'";
rs = logQ.Execute(query, registryFormat);
for (; !rs.atEnd(); rs.moveNext())
{
keys.Add(rs.getRecord().toNativeString(","));
}
}
finally
{
rs.close();
}
return(keys);
}
private TrafficReport Execute(string query)
{
var report = new TrafficReport();
var parser = new LogQueryClass();
var log = new COMW3CInputContextClass();
var records = parser.Execute(query, log);
while (!records.atEnd())
{
var record = records.getRecord();
var ip = record.getValue("Ip");
var userAgent = record.getValue("UserAgent");
var accessAttempts = record.getValue("AccessAttempts");
var transferedBytes = record.getValue("TransferedBytes");
report.AddEntry(ip, userAgent, accessAttempts, transferedBytes);
records.moveNext();
}
return(report);
}
private DataTable GetDataFromLogs()
{
DataTable dataResults = new DataTable();
LogQueryClass LogParser = new LogQueryClass();
ILogRecordset rs = LogParser.Execute("SELECT * FROM '" + textBoxFrom.Text.Trim() + "'", new COMIISW3CInputContextClass());
for (int i = 0; i < rs.getColumnCount(); i++)
{
if (!dataResults.Columns.Contains(rs.getColumnName(i)))
{
dataResults.Columns.Add(rs.getColumnName(i));
}
}
//dataGridView1.DataSource = dataResults.DefaultView;
ILogRecord rcd;
DataRow row;
while (!rs.atEnd())
{
row = dataResults.NewRow();
rcd = rs.getRecord();
for (int i = 0; i < rs.getColumnCount(); i++)
{
row[rs.getColumnName(i)] = rcd.getValue(rs.getColumnName(i));
}
dataResults.Rows.Add(row);
rs.moveNext();
}
rs.close();
return(dataResults);
}
public override void Import(DateTime @from)
{
var logQuery = new LogQueryClass();
var inputFormat = new COMW3CInputContextClass();
string strQuery = string.Format(@"SELECT to_timestamp(date, time) as date,
s-ip as sourceIP,
cs-method as method,
cs-uri-stem as uri,
cs-uri-query as query,
s-port as port,
c-ip as clientIP,
cs(User-Agent) as userAgent,
cs-host as clientToServerHost,
sc-status as statusCode,
sc-substatus as subStatus,
sc-win32-status as win32Status,
sc-bytes as serverToClientBytes,
cs-bytes as clientToServerBytes,
time-taken as duration FROM {0}", "SOME FILE");
ILogRecordset results = logQuery.Execute(strQuery, inputFormat);
var tweets = new List<Entry>();
while (!results.atEnd())
{
ILogRecord logRecord = results.getRecord();
dynamic date = logRecord.getValue("date");
dynamic sourceIP = logRecord.getValue("sourceIP");
dynamic method = logRecord.getValue("method");
dynamic uri = logRecord.getValue("uri");
dynamic query = logRecord.getValue("query") is DBNull ? string.Empty : logRecord.getValue("query");
dynamic port = logRecord.getValue("port");
dynamic clientIP = logRecord.getValue("clientIP") is DBNull
? string.Empty
: logRecord.getValue("clientIP");
dynamic userAgent = logRecord.getValue("userAgent") is DBNull
? string.Empty
: logRecord.getValue("userAgent");
dynamic clientToServerHost = logRecord.getValue("clientToServerHost") is DBNull
? string.Empty
: logRecord.getValue("clientToServerHost");
dynamic statusCode = logRecord.getValue("statusCode");
dynamic subStatus = logRecord.getValue("subStatus");
dynamic win32Status = logRecord.getValue("win32Status");
dynamic serverToClientBytes = logRecord.getValue("serverToClientBytes");
dynamic clientToServerBytes = logRecord.getValue("clientToServerBytes");
dynamic duration = logRecord.getValue("duration");
tweets.Add(new Entry
{
Date = date,
SourceIP = sourceIP,
Method = method,
Uri = uri,
Query = query,
Port = port,
ClientIP = clientIP,
UserAgent = userAgent,
ClientToServerHost = clientToServerHost,
StatusCode = statusCode,
SubStatus = subStatus,
Win32Status = win32Status,
ServerToClientBytes = serverToClientBytes,
ClientToServerBytes = clientToServerBytes,
Duration = duration
});
results.moveNext();
}
var serializer = new JsonNetSerializer();
string bulkCommand = new BulkCommand(index: "log", type: "iis");
string bulkJson =
new BulkBuilder(serializer)
.BuildCollection(tweets,
(builder, tweet) => builder.Create(tweet)
);
_connection.Post(bulkCommand, bulkJson);
}
public override void Import(DateTime @from)
{
var logQuery = new LogQueryClass();
var inputFormat = new COMW3CInputContextClass();
string strQuery = string.Format(@"SELECT to_timestamp(date, time) as date,
s-ip as sourceIP,
cs-method as method,
cs-uri-stem as uri,
cs-uri-query as query,
s-port as port,
c-ip as clientIP,
cs(User-Agent) as userAgent,
cs-host as clientToServerHost,
sc-status as statusCode,
sc-substatus as subStatus,
sc-win32-status as win32Status,
sc-bytes as serverToClientBytes,
cs-bytes as clientToServerBytes,
time-taken as duration FROM {0}", "SOME FILE");
ILogRecordset results = logQuery.Execute(strQuery, inputFormat);
var tweets = new List <Entry>();
while (!results.atEnd())
{
ILogRecord logRecord = results.getRecord();
dynamic date = logRecord.getValue("date");
dynamic sourceIP = logRecord.getValue("sourceIP");
dynamic method = logRecord.getValue("method");
dynamic uri = logRecord.getValue("uri");
dynamic query = logRecord.getValue("query") is DBNull ? string.Empty : logRecord.getValue("query");
dynamic port = logRecord.getValue("port");
dynamic clientIP = logRecord.getValue("clientIP") is DBNull
? string.Empty
: logRecord.getValue("clientIP");
dynamic userAgent = logRecord.getValue("userAgent") is DBNull
? string.Empty
: logRecord.getValue("userAgent");
dynamic clientToServerHost = logRecord.getValue("clientToServerHost") is DBNull
? string.Empty
: logRecord.getValue("clientToServerHost");
dynamic statusCode = logRecord.getValue("statusCode");
dynamic subStatus = logRecord.getValue("subStatus");
dynamic win32Status = logRecord.getValue("win32Status");
dynamic serverToClientBytes = logRecord.getValue("serverToClientBytes");
dynamic clientToServerBytes = logRecord.getValue("clientToServerBytes");
dynamic duration = logRecord.getValue("duration");
tweets.Add(new Entry
{
Date = date,
SourceIP = sourceIP,
Method = method,
Uri = uri,
Query = query,
Port = port,
ClientIP = clientIP,
UserAgent = userAgent,
ClientToServerHost = clientToServerHost,
StatusCode = statusCode,
SubStatus = subStatus,
Win32Status = win32Status,
ServerToClientBytes = serverToClientBytes,
ClientToServerBytes = clientToServerBytes,
Duration = duration
});
results.moveNext();
}
var serializer = new JsonNetSerializer();
string bulkCommand = new BulkCommand(index: "log", type: "iis");
string bulkJson =
new BulkBuilder(serializer)
.BuildCollection(tweets,
(builder, tweet) => builder.Create(tweet)
);
_connection.Post(bulkCommand, bulkJson);
}
/// <summary>
/// Adds the current log file to the WebStats SQL database.
/// </summary>
/// <returns>Flag to indicate whether the log file was added to the WebStats SQL database successfully.</returns>
private bool AddLogFileToWebStatsSqlDb()
{
bool result;
/* Reasoning for the input context:
*
* 1. Disable checkpoints because incremental parsing of log files does not apply when we are picking up the previous day's log files AFTER updates to them have stopped.
* 2. Disable recursion (set to zero) because all log file paths are read from Whois and there should never be any subdirectories in those paths.
*/
COMIISW3CInputContextClass inputContext = new COMIISW3CInputContextClass();
inputContext.consolidateLogs = true;
//inputContext.iCheckpoint = this._installDirectory + "\\WebStats.IISLogManager.lpc";
inputContext.recurse = 0;
COMSQLOutputContextClass outputContext = new COMSQLOutputContextClass();
outputContext.clearTable = false;
outputContext.createTable = false;
outputContext.fixColNames = true;
outputContext.ignoreIdCols = true;
outputContext.ignoreMinWarns = false;
outputContext.maxStrFieldLen = 8000;
outputContext.transactionRowCount = 0;
outputContext.oConnString = WebStatsODBCSqlDbConnectionString;
LogQueryClass logQuery = new LogQueryClass();
logQuery.ExecuteBatch(GetIISLogSql(), inputContext, outputContext);
result = (logQuery.lastError == 0);
if (result)
{
EventLog.WriteEntry(SourceName, "Logs added to WebStats database.\r\n\r\n" + GetLogFileInfo(), EventLogEntryType.Information, InfoEventID);
}
else
{
EventLog.WriteEntry(SourceName, "Failed to add logs to WebStats database.\r\n\r\n" + GetLogFileInfo(), EventLogEntryType.Information, InfoEventID);
}
return result;
}
//Call references LogParser. Take sql string query and the Table name and execute into a logparser records set. Then, loop through the records
//and save them into a dataTable
private static DataTable parseLog(string query, string TableName)
{
LogQueryClass logParser = new LogQueryClass();
COMTextLineInputContextClass Log = new COMTextLineInputContextClass();
ILogRecordset rsLP = null;
ILogRecord rowLP = null;
rsLP = logParser.Execute(query, Log);
DataTable tab = new DataTable(TableName);
// copy schema
for (int i = 0; i < rsLP.getColumnCount(); i++)
{
DataColumn col = new DataColumn();
col.ColumnName = rsLP.getColumnName(i);
tab.Columns.Add(col);
}
// copy data
while (!rsLP.atEnd())
{
rowLP = rsLP.getRecord();
DataRow row = tab.NewRow();
for (int i = 0; i < rsLP.getColumnCount(); i++)
{
row[i] = Convert.ToString(rowLP.getValue(i));
}
tab.Rows.Add(row);
rsLP.moveNext();
}
return tab;
}
