// Loop through all the log records, compute min and max time and number of buckets.
// Also assign each record to a bucket and store the bucket number into the record.
private double[] ComputeBuckets(GlobalRecordCollection gRecords, int resolutionSec)
{
double[] bucketTimes;
double secondsInDay = 24d * 60d * 60d;
double minDate = double.MaxValue;
double maxDate = double.MinValue;
// compute the high and low times
gRecords.ForEveryRecord((r) => {
if (r.time < minDate)
{
minDate = r.time;
}
if (r.time > maxDate)
{
maxDate = r.time;
}
});
// add the bucket number to each of the records
gRecords.ForEveryRecord((r) => {
r.bucket = (long)Math.Round(((r.time - minDate) * secondsInDay / (double)resolutionSec));
});
long numBuckets = (long)((maxDate - minDate) * secondsInDay / (double)resolutionSec);
// Add an extra bucket for rounding at the end
numBuckets += 2;
// Compute a base time for each bucket
bucketTimes = new double[numBuckets];
for (int ii = 0; ii < numBuckets; ii++)
{
bucketTimes[ii] = minDate + (ii * resolutionSec) / secondsInDay;
}
Logger.Log("DEBUG: ComputeBuckets: numBuckets={0}", numBuckets);
return(bucketTimes);
}
// Loop through all the records and build the list of records for each bucket.
// Returns an array of lists. Each element of the array is a bucket and each bucket has
// a list of the records assigned to the bucket.
private List <Records>[] DistributeToBucketArray(GlobalRecordCollection gRecords, long numBuckets)
{
List <Records>[] buckets = new List <Records> [numBuckets];
for (int ii = 0; ii < numBuckets; ii++)
{
buckets[ii] = new List <Records>();
}
gRecords.ForEveryRecord((r) =>
{
try
{
buckets[r.bucket].Add(r);
}
catch (Exception e)
{
Logger.Log("ERROR: adding record to bucket failure. r.bucket={0}, numBuckets={1}, e:{2}", r.bucket, numBuckets, e);
}
});
return(buckets);
}
public void Start(string[] args)
{
m_Parameters = ParameterParse.ParseArguments(args, false /* firstOpFlag */, true /* multipleFiles */);
foreach (KeyValuePair <string, string> kvp in m_Parameters)
{
switch (kvp.Key)
{
case "-o":
case "--output":
m_outFile = kvp.Value;
break;
case "--verbose":
m_ParamVerbose++;
break;
case ParameterParse.LAST_PARAM:
m_inFiles = kvp.Value;
break;
case ParameterParse.ERROR_PARAM:
// if we get here, the parser found an error
Logger.Log("Parameter error: " + kvp.Value);
Logger.Log(Invocation());
return;
default:
Logger.Log("ERROR: UNKNOWN PARAMETER: " + kvp.Key);
Logger.Log(Invocation());
return;
}
}
GlobalRecordCollection globalCollection = new GlobalRecordCollection();
// Read in all log records into approriate structure type for each log line
if (!String.IsNullOrEmpty(m_inFiles))
{
string[] files = m_inFiles.Split(',');
foreach (string fileName in files)
{
if (fileName.Substring(0, connName.Length) == connName)
{
List <Records> syncConn = StatSyncConnector.Read(fileName);
globalCollection.Add(syncConn);
}
else if (fileName.Substring(0, sceneName.Length) == sceneName)
{
List <Records> syncConn = StatScene.Read(fileName);
globalCollection.Add(syncConn);
}
else if (fileName.Substring(0, serverName.Length) == serverName)
{
List <Records> syncConn = StatServer.Read(fileName);
globalCollection.Add(syncConn);
}
else
{
Logger.Log("No handler for file {0}", fileName);
}
}
}
// Compute the number of buckets based on the log records. Compute min and max.
double[] bucketTimes = ComputeBuckets(globalCollection, 10 /* bucket size in seconds */);
long numBuckets = bucketTimes.Length;
if (numBuckets > 0)
{
List <Records>[] buckets = DistributeToBucketArray(globalCollection, numBuckets);
GenerateOutput(globalCollection, buckets, bucketTimes);
}
}
// Output a line for each bucket. Loop through the records in the bucket and figure out the
// logged fields.
// The records are first passed through to get the names of all the logged fields so
// each line of the output file will have the same number of fields.
// Modify GetRecordFileNames() to change the columns that are actually output.
private void GenerateOutput(GlobalRecordCollection gRecords, List <Records>[] buckets, double[] bucketTimes)
{
List <string> fieldNames = new List <string>();
bool firstLine = true;
// Collect all the field names
gRecords.ForEveryRecord((r) => { GetRecordFieldNames(r, ref fieldNames); });
try
{
using (TextWriter outWriter = new StreamWriter(File.Open(m_outFile, FileMode.Create)))
{
for (int ii = 0; ii < buckets.Length; ii++)
{
double bucketTime = 0d;
Dictionary <string, string> outFields = new Dictionary <string, string>();
foreach (Records rec in buckets[ii])
{
bucketTime = bucketTimes[ii];
// Add the output fields (generated in GetRecordFileNames) to this output line.
foreach (string fldname in rec.outputFields.Keys)
{
if (!outFields.ContainsKey(fldname))
{
outFields.Add(fldname, rec.outputFields[fldname]);
}
}
}
// If the first line, output the field names
if (firstLine)
{
StringBuilder tbuff = new StringBuilder();
tbuff.Append("bucket");
tbuff.Append(",");
tbuff.Append("time");
foreach (string fName in fieldNames)
{
tbuff.Append(",");
tbuff.Append(fName);
}
outWriter.WriteLine(tbuff.ToString());
firstLine = false;
}
StringBuilder buff = new StringBuilder();
buff.Append(ii.ToString()); // bucket number
buff.Append(",");
buff.Append(bucketTime.ToString()); // bucket time
foreach (string fName in fieldNames)
{
buff.Append(",");
if (outFields.ContainsKey(fName))
{
buff.Append(outFields[fName]);
}
}
outWriter.WriteLine(buff.ToString());
}
}
}
catch (Exception e)
{
Logger.Log("ERROR WRITING OUTPUT FILE '{0}' {1}", m_outFile, e.Message);
return;
}
}
