public void Start(string[] args)
{
m_Parameters = ParameterParse.ParseArguments(args, false /* firstOpFlag */, true /* multipleFiles */);
foreach (KeyValuePair <string, string> kvp in m_Parameters)
{
switch (kvp.Key)
{
case "-p":
case "--period":
m_bucketSeconds = int.Parse(kvp.Value);
break;
case "-o":
case "--output":
m_outFile = kvp.Value;
break;
case "-h":
case "--cmHosts":
m_cmHosts = kvp.Value;
break;
case "--verbose":
m_ParamVerbose++;
break;
case ParameterParse.LAST_PARAM:
m_inFiles = kvp.Value;
break;
case ParameterParse.ERROR_PARAM:
// if we get here, the parser found an error
Logger.Log("Parameter error: " + kvp.Value);
Logger.Log(Invocation());
return;
default:
Logger.Log("ERROR: UNKNOWN PARAMETER: " + kvp.Key);
Logger.Log(Invocation());
return;
}
}
List <HTTPRecord> records = new List <HTTPRecord>();
// Read in the records
if (!String.IsNullOrEmpty(m_inFiles))
{
string[] files = m_inFiles.Split(',');
foreach (string fileName in files)
{
records.AddRange(HTTPRecord.Read(fileName));
}
}
// Find high and low dates and compute the number of buckets
double minDate = double.MaxValue;
double maxDate = double.MinValue;
foreach (HTTPRecord rec in records)
{
minDate = Math.Min(minDate, rec.time);
maxDate = Math.Max(maxDate, rec.time);
}
double bucketBaseTime = minDate;
int numBuckets = ((int)((maxDate - minDate) * LongDate.secondsPerDay)) / m_bucketSeconds;
numBuckets += 1; // add a last bucket for rounding error at the end.
Logger.Log("Number of buckets = {0}", numBuckets);
// Loop through all the records and assign each to a bucket
foreach (HTTPRecord rec in records)
{
rec.bucket = ((int)((rec.time - bucketBaseTime) * LongDate.secondsPerDay)) / m_bucketSeconds;
}
// Specify individual hosts to count accesses with CSV list "host,host,host"
List <string> cmHosts = m_cmHosts.Split(',').ToList <string>();
int numHosts = cmHosts.Count;
if (numHosts == 0)
{
Logger.Log("NUMBER OF Client Manager HOSTS MUST NOT BE ZERO!!");
return;
}
// Initialize each bucket line with the variable sized structures
BucketLine[] bucketLines = new BucketLine[numBuckets];
for (int ii = 0; ii < numBuckets; ii++)
{
bucketLines[ii].bucket = ii;
bucketLines[ii].time = bucketBaseTime + ((double)(ii * m_bucketSeconds) / LongDate.secondsPerDay);
bucketLines[ii].hostCount = new int[numHosts];
bucketLines[ii].logins = 0;
}
// Loop through all the records and fill the bucket info
foreach (HTTPRecord rec in records)
{
int nHost = cmHosts.IndexOf(rec.source);
if (nHost >= 0)
{
bucketLines[rec.bucket].hostCount[nHost]++;
}
if (rec.remain.Contains("/Grid/login/ "))
{
bucketLines[rec.bucket].logins++;
}
}
// Print out all the buckets
bool firstLine = true;
TextWriter outWriter = new StreamWriter(File.Open(m_outFile, FileMode.Create));
if (outWriter != null)
{
using (outWriter)
{
if (firstLine)
{
StringBuilder buff = new StringBuilder();
buff.Append("bucket");
buff.Append(",");
buff.Append("time");
buff.Append(",");
buff.Append("logins");
buff.Append(",");
for (int ii = 0; ii < cmHosts.Count; ii++)
{
buff.Append(cmHosts[ii]);
buff.Append(",");
}
outWriter.WriteLine(buff.ToString());
firstLine = false;
}
foreach (BucketLine buck in bucketLines)
{
StringBuilder buff = new StringBuilder();
buff.Append(buck.bucket.ToString());
buff.Append(",");
buff.Append(buck.time.ToString());
buff.Append(",");
buff.Append(buck.logins.ToString());
buff.Append(",");
for (int ii = 0; ii < buck.hostCount.Length; ii++)
{
buff.Append(buck.hostCount[ii].ToString());
buff.Append(",");
}
outWriter.WriteLine(buff.ToString());
}
}
}
}
static public List <HTTPRecord> Read(string filename)
{
List <HTTPRecord> ret = new List <HTTPRecord>();
TextReader inReader = new StreamReader(File.Open(filename, FileMode.Open));
if (inReader == null)
{
Logger.Log("{0} Read: Failed opening stat file '{1}'", LogHeader, filename);
return(null);
}
string inLine;
Regex pattern = new Regex(@"^(\d+\.\d+\.\d+\.\d+) - - \[(\d+)/(...)/(\d+):(\d+):(\d+):(\d+) .*""(\w+) (.*)$");
using (inReader)
{
while ((inLine = inReader.ReadLine()) != null)
{
try
{
// SAMPLE
// 10.10.10.1 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/ HTTP/1.1" 200 4703 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/ HTTP/1.1" 200 213 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/ HTTP/1.1" 200 211 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/ HTTP/1.1" 200 211 "-" "-"
// 98.229.237.99 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/login/ HTTP/1.1" 200 2586 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:47 -0400] "POST /Grid/ HTTP/1.1" 200 548 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:47 -0400] "POST /Grid/?id=9f54505f-b5aa-4e16-a1ef-d3152a252e08 HTTP/1.1" 200 293 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:47 -0400] "POST /Grid/ HTTP/1.1" 200 290 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:48 -0400] "GET /Grid/?id=cce0f112-878f-4586-a2e2-a8f104bba271 HTTP/1.1" 404 234 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:31:10 -0400] "POST /Grid/ HTTP/1.1" 200 223 "-" "-"
// 98.229.237.99 - - [17/Mar/2013:15:31:10 -0400] "GET /GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429) HTTP/1.1" 200 1432 "-" "Mozilla/5.0 (Windows; U; Windows NT6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 10.10.10.1 - - [17/Mar/2013:15:31:10 -0400] "POST /Grid/ HTTP/1.1" 200 223 "-" "-"
// 98.229.237.99 - - [17/Mar/2013:15:31:10 -0400] "GET /GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429) HTTP/1.1" 200 1432 "-" "Mozilla/5.0 (Windows; U; Windows NT6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 98.229.237.99 - - [17/Mar/2013:15:31:11 -0400] "GET /GridFrontend//static/styles/default/style.css HTTP/1.1" 200 897 "http://107.7.21.234/GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429)" "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 98.229.237.99 - - [17/Mar/2013:15:31:11 -0400] "GET /GridFrontend//static/javascript/jquery.qtip.js HTTP/1.1" 200 10148 "http://107.7.21.234/GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429)" "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 98.229.237.99 - - [17/Mar/2013:15:31:11 -0400] "GET /GridFrontend//static/styles/default/jquery-ui.css HTTP/1.1" 200 5985 "http://107.7.21.234/GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429)" "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 98.229.237.99 - - [17/Mar/2013:15:31:11 -0400] "GET /GridFrontend//static/styles/default/jquery.qtip.css HTTP/1.1" 200 1695 "http://107.7.21.234/GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sstring[] pieces = inLine.Split(',');
Match fields = pattern.Match(inLine);
if (!fields.Success)
{
Logger.Log("PARSING FAILURE: {0}", inLine);
continue;
}
HTTPRecord aRec = new HTTPRecord();
aRec.line = inLine;
aRec.bucket = 0;
int nMonth = "xxJanFebMarAprMayJunJulAugSepOctNovDec".IndexOf(fields.Groups[3].Value) / 3;
aRec.timeDateTime = new DateTime(
int.Parse(fields.Groups[4].Value), // year
nMonth, // month
int.Parse(fields.Groups[2].Value), // month day
int.Parse(fields.Groups[5].Value), // hour
int.Parse(fields.Groups[6].Value), // minute
int.Parse(fields.Groups[7].Value) // second
);
aRec.time = LongDate.DateTimeToExcelDate(aRec.timeDateTime);
aRec.source = fields.Groups[1].Value;
aRec.op = fields.Groups[8].Value;
aRec.remain = fields.Groups[9].Value;
ret.Add(aRec);
// Logger.Log(aRec.ToString());
}
catch (Exception e)
{
Logger.Log("{0} Exception parsing line: '{1}'", LogHeader, inLine);
Logger.Log("{0} Exception parsing line: e: {1} ", LogHeader, e);
continue;
}
}
}
return(ret);
}
