private static void WriteLine(TickData data, StreamWriter writer) { writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12}", data.Run, data.Tick, data.IsUltra, data.AverageFulfillmentTime, data.NumberFullfilledRequests, data.AverageKnownFiles, data.AverageKnownNodes, data.NumberTimeouts, data.PercentFulfilled, data.AverageWorkQueueSize, data.WorkQueueVariance, data.NumberDead, data.Count); }
static void Pass1(string inputFilePath, string outputFilePath) { const int numNodes = 100; using (var streamReader = new StreamReader(inputFilePath)) { using (var streamWriter = new StreamWriter(outputFilePath)) { // Strip header row var line = streamReader.ReadLine(); // Write header row in output streamWriter.WriteLine("Run,Tick,IsUltra,AvgFulfillmentTime,NumFulfilled,AvgKnownFiles,AvgKnownNodes," + "NumTimeouts,PercentFulfilled,AvgWorkQueueSize,QueueVariance,NumberDead,Count"); // Loop over Runs, Ticks, and Nodes. Pass 1 aggregates nodes per tick for (var runIterator = 0; runIterator < 30; runIterator++) { for (var tickIterator = 0; tickIterator < 10000; tickIterator++) { var tickData = new TickData { WorkQueueObservations = new List <int>(), IsUltra = false }; var ultraTickData = new TickData { WorkQueueObservations = new List <int>(), IsUltra = true }; var hadUltras = false; for (var nodeIterator = 0; nodeIterator < numNodes; nodeIterator++) { line = streamReader.ReadLine(); if (line == null) { break; } var parts = line.Split(','); var run = int.Parse(parts[0]); var avgFulfillmentTime = double.Parse(parts[1]); var numFulfilled = int.Parse(parts[2]); var numKnownFiles = int.Parse(parts[3]); var numKnownNodes = int.Parse(parts[4]); var numTimeouts = int.Parse(parts[5]); var numUnfulfilled = int.Parse(parts[6]); var percentFulfilled = double.Parse(parts[7]); var workQueueSize = int.Parse(parts[8]); var isDead = bool.Parse(parts[9].Replace("\"", " ")); var isUltra = bool.Parse(parts[10].Replace("\"", " ")); var tickDouble = double.Parse(parts[11]); var tick = (int)tickDouble; if (isUltra) { hadUltras = true; ultraTickData.Run = run; ultraTickData.Tick = tick; ultraTickData.Count++; ultraTickData.AverageFulfillmentTime += avgFulfillmentTime; ultraTickData.NumberFullfilledRequests += numFulfilled; ultraTickData.AverageKnownFiles += numKnownFiles; ultraTickData.AverageKnownNodes += numKnownNodes; ultraTickData.NumberTimeouts += numTimeouts; ultraTickData.AverageWorkQueueSize += workQueueSize; ultraTickData.NumberDead += isDead ? 1 : 0; ultraTickData.WorkQueueObservations.Add(workQueueSize); } else { tickData.Run = run; tickData.Tick = tick; tickData.Count++; tickData.AverageFulfillmentTime += avgFulfillmentTime; tickData.NumberFullfilledRequests += numFulfilled; tickData.AverageKnownFiles += numKnownFiles; tickData.AverageKnownNodes += numKnownNodes; tickData.NumberTimeouts += numTimeouts; tickData.AverageWorkQueueSize += workQueueSize; tickData.NumberDead += isDead ? 1 : 0; tickData.WorkQueueObservations.Add(workQueueSize); } } if (line == null) { break; } // Compute appropriate averages and summaries, and write to file tickData.AverageFulfillmentTime = 1.0 * tickData.AverageFulfillmentTime / tickData.Count; tickData.AverageKnownFiles = 1.0 * tickData.AverageKnownFiles / tickData.Count; tickData.AverageKnownNodes = 1.0 * tickData.AverageKnownNodes / tickData.Count; tickData.AverageWorkQueueSize = 1.0 * tickData.AverageWorkQueueSize / tickData.Count; var denom = (tickData.NumberFullfilledRequests + tickData.NumberTimeouts); if (denom == 0) { denom = 1; } tickData.PercentFulfilled = (1.0 * tickData.NumberFullfilledRequests) / denom; tickData.WorkQueueVariance = Variance(tickData.WorkQueueObservations, tickData.AverageWorkQueueSize); WriteLine(tickData, streamWriter); if (hadUltras) { ultraTickData.AverageFulfillmentTime = 1.0 * ultraTickData.AverageFulfillmentTime / ultraTickData.Count; ultraTickData.AverageKnownFiles = 1.0 * ultraTickData.AverageKnownFiles / ultraTickData.Count; ultraTickData.AverageKnownNodes = 1.0 * ultraTickData.AverageKnownNodes / ultraTickData.Count; ultraTickData.AverageWorkQueueSize = 1.0 * ultraTickData.AverageWorkQueueSize / ultraTickData.Count; var denomU = (ultraTickData.NumberFullfilledRequests + ultraTickData.NumberTimeouts); if (denomU == 0) { denomU = 1; } ultraTickData.PercentFulfilled = (1.0 * ultraTickData.NumberFullfilledRequests) / denomU; ultraTickData.WorkQueueVariance = Variance(ultraTickData.WorkQueueObservations, ultraTickData.AverageWorkQueueSize); WriteLine(ultraTickData, streamWriter); } if (tickIterator % 1000 == 0) { Console.WriteLine("Finished Tick" + tickIterator); } } if (line == null) { break; } Console.WriteLine("Finished Run" + runIterator); } } } }
static void Pass2(string inputFilePath, string outputFilePath) { using (var streamReader = new StreamReader(inputFilePath)) { using (var streamWriter = new StreamWriter(outputFilePath)) { // Strip header row var line = streamReader.ReadLine(); // Write header row in output streamWriter.WriteLine("Run,Tick,IsUltra,AvgFulfillmentTime,NumFulfilled,AvgKnownFiles,AvgKnownNodes," + "NumTimeouts,PercentFulfilled,AvgWorkQueueSize,QueueVariance,NumberDead,Count"); var aggregateTickData = new TickData[10000]; var ultraAggregateTickData = new TickData[10000]; var hadUltras = false; for (var i = 0; i < aggregateTickData.Length; i++) { aggregateTickData[i] = new TickData { IsUltra = false }; ultraAggregateTickData[i] = new TickData { IsUltra = true }; } while (true) { line = streamReader.ReadLine(); if (line == null) { break; } // Parse each line into tickdata var parts = line.Split(','); var run = int.Parse(parts[0]); var tick = int.Parse(parts[1]); var tickIndex = tick - 1; var isUltra = bool.Parse(parts[2]); if (isUltra) { hadUltras = true; ultraAggregateTickData[tickIndex].Tick = tick; ultraAggregateTickData[tickIndex].AverageFulfillmentTime += double.Parse(parts[3]); ultraAggregateTickData[tickIndex].NumberFullfilledRequests += int.Parse(parts[4]); ultraAggregateTickData[tickIndex].AverageKnownFiles += double.Parse(parts[5]); ultraAggregateTickData[tickIndex].AverageKnownNodes += double.Parse(parts[6]); ultraAggregateTickData[tickIndex].NumberTimeouts += int.Parse(parts[7]); ultraAggregateTickData[tickIndex].PercentFulfilled += double.Parse(parts[8]); ultraAggregateTickData[tickIndex].AverageWorkQueueSize += double.Parse(parts[9]); ultraAggregateTickData[tickIndex].WorkQueueVariance += double.Parse(parts[10]); ultraAggregateTickData[tickIndex].NumberDead += int.Parse(parts[11]); ultraAggregateTickData[tickIndex].Count += int.Parse(parts[12]); } else { aggregateTickData[tickIndex].Tick = tick; aggregateTickData[tickIndex].AverageFulfillmentTime += double.Parse(parts[3]); aggregateTickData[tickIndex].NumberFullfilledRequests += int.Parse(parts[4]); aggregateTickData[tickIndex].AverageKnownFiles += double.Parse(parts[5]); aggregateTickData[tickIndex].AverageKnownNodes += double.Parse(parts[6]); aggregateTickData[tickIndex].NumberTimeouts += int.Parse(parts[7]); aggregateTickData[tickIndex].PercentFulfilled += double.Parse(parts[8]); aggregateTickData[tickIndex].AverageWorkQueueSize += double.Parse(parts[9]); aggregateTickData[tickIndex].WorkQueueVariance += double.Parse(parts[10]); aggregateTickData[tickIndex].NumberDead += int.Parse(parts[11]); aggregateTickData[tickIndex].Count += int.Parse(parts[12]); } } // Compute averages for (var i = 0; i < aggregateTickData.Length; i++) { aggregateTickData[i].AverageFulfillmentTime = 1.0 * aggregateTickData[i].AverageFulfillmentTime / 30; aggregateTickData[i].AverageKnownFiles = 1.0 * aggregateTickData[i].AverageKnownFiles / 30; aggregateTickData[i].AverageKnownNodes = 1.0 * aggregateTickData[i].AverageKnownNodes / 30; aggregateTickData[i].PercentFulfilled = 1.0 * aggregateTickData[i].PercentFulfilled / 30; aggregateTickData[i].AverageWorkQueueSize = 1.0 * aggregateTickData[i].AverageWorkQueueSize / 30; aggregateTickData[i].WorkQueueVariance = 1.0 * aggregateTickData[i].WorkQueueVariance / 30; WriteLine(aggregateTickData[i], streamWriter); } if (hadUltras) { for (var i = 0; i < ultraAggregateTickData.Length; i++) { ultraAggregateTickData[i].AverageFulfillmentTime = 1.0 * ultraAggregateTickData[i].AverageFulfillmentTime / 30; ultraAggregateTickData[i].AverageKnownFiles = 1.0 * ultraAggregateTickData[i].AverageKnownFiles / 30; ultraAggregateTickData[i].AverageKnownNodes = 1.0 * ultraAggregateTickData[i].AverageKnownNodes / 30; ultraAggregateTickData[i].PercentFulfilled = 1.0 * ultraAggregateTickData[i].PercentFulfilled / 30; ultraAggregateTickData[i].AverageWorkQueueSize = 1.0 * ultraAggregateTickData[i].AverageWorkQueueSize / 30; ultraAggregateTickData[i].WorkQueueVariance = 1.0 * ultraAggregateTickData[i].WorkQueueVariance / 30; WriteLine(ultraAggregateTickData[i], streamWriter); } } } } }
private static void WriteLine(TickData data, StreamWriter writer) { writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12}", data.Run, data.Tick, data.IsUltra, data.AverageFulfillmentTime, data.NumberFullfilledRequests, data.AverageKnownFiles, data.AverageKnownNodes, data.NumberTimeouts, data.PercentFulfilled, data.AverageWorkQueueSize, data.WorkQueueVariance, data.NumberDead, data.Count); }
static void Pass2(string inputFilePath, string outputFilePath) { using (var streamReader = new StreamReader(inputFilePath)) { using (var streamWriter = new StreamWriter(outputFilePath)) { // Strip header row var line = streamReader.ReadLine(); // Write header row in output streamWriter.WriteLine("Run,Tick,IsUltra,AvgFulfillmentTime,NumFulfilled,AvgKnownFiles,AvgKnownNodes," + "NumTimeouts,PercentFulfilled,AvgWorkQueueSize,QueueVariance,NumberDead,Count"); var aggregateTickData = new TickData[10000]; var ultraAggregateTickData = new TickData[10000]; var hadUltras = false; for (var i = 0; i < aggregateTickData.Length; i++) { aggregateTickData[i] = new TickData { IsUltra = false }; ultraAggregateTickData[i] = new TickData { IsUltra = true }; } while (true) { line = streamReader.ReadLine(); if (line == null) break; // Parse each line into tickdata var parts = line.Split(','); var run = int.Parse(parts[0]); var tick = int.Parse(parts[1]); var tickIndex = tick - 1; var isUltra = bool.Parse(parts[2]); if (isUltra) { hadUltras = true; ultraAggregateTickData[tickIndex].Tick = tick; ultraAggregateTickData[tickIndex].AverageFulfillmentTime += double.Parse(parts[3]); ultraAggregateTickData[tickIndex].NumberFullfilledRequests += int.Parse(parts[4]); ultraAggregateTickData[tickIndex].AverageKnownFiles += double.Parse(parts[5]); ultraAggregateTickData[tickIndex].AverageKnownNodes += double.Parse(parts[6]); ultraAggregateTickData[tickIndex].NumberTimeouts += int.Parse(parts[7]); ultraAggregateTickData[tickIndex].PercentFulfilled += double.Parse(parts[8]); ultraAggregateTickData[tickIndex].AverageWorkQueueSize += double.Parse(parts[9]); ultraAggregateTickData[tickIndex].WorkQueueVariance += double.Parse(parts[10]); ultraAggregateTickData[tickIndex].NumberDead += int.Parse(parts[11]); ultraAggregateTickData[tickIndex].Count += int.Parse(parts[12]); } else { aggregateTickData[tickIndex].Tick = tick; aggregateTickData[tickIndex].AverageFulfillmentTime += double.Parse(parts[3]); aggregateTickData[tickIndex].NumberFullfilledRequests += int.Parse(parts[4]); aggregateTickData[tickIndex].AverageKnownFiles += double.Parse(parts[5]); aggregateTickData[tickIndex].AverageKnownNodes += double.Parse(parts[6]); aggregateTickData[tickIndex].NumberTimeouts += int.Parse(parts[7]); aggregateTickData[tickIndex].PercentFulfilled += double.Parse(parts[8]); aggregateTickData[tickIndex].AverageWorkQueueSize += double.Parse(parts[9]); aggregateTickData[tickIndex].WorkQueueVariance += double.Parse(parts[10]); aggregateTickData[tickIndex].NumberDead += int.Parse(parts[11]); aggregateTickData[tickIndex].Count += int.Parse(parts[12]); } } // Compute averages for (var i = 0; i < aggregateTickData.Length; i++) { aggregateTickData[i].AverageFulfillmentTime = 1.0 * aggregateTickData[i].AverageFulfillmentTime / 30; aggregateTickData[i].AverageKnownFiles = 1.0 * aggregateTickData[i].AverageKnownFiles / 30; aggregateTickData[i].AverageKnownNodes = 1.0 * aggregateTickData[i].AverageKnownNodes / 30; aggregateTickData[i].PercentFulfilled = 1.0 * aggregateTickData[i].PercentFulfilled / 30; aggregateTickData[i].AverageWorkQueueSize = 1.0 * aggregateTickData[i].AverageWorkQueueSize / 30; aggregateTickData[i].WorkQueueVariance = 1.0 * aggregateTickData[i].WorkQueueVariance / 30; WriteLine(aggregateTickData[i], streamWriter); } if (hadUltras) { for (var i = 0; i < ultraAggregateTickData.Length; i++) { ultraAggregateTickData[i].AverageFulfillmentTime = 1.0 * ultraAggregateTickData[i].AverageFulfillmentTime / 30; ultraAggregateTickData[i].AverageKnownFiles = 1.0 * ultraAggregateTickData[i].AverageKnownFiles / 30; ultraAggregateTickData[i].AverageKnownNodes = 1.0 * ultraAggregateTickData[i].AverageKnownNodes / 30; ultraAggregateTickData[i].PercentFulfilled = 1.0 * ultraAggregateTickData[i].PercentFulfilled / 30; ultraAggregateTickData[i].AverageWorkQueueSize = 1.0 * ultraAggregateTickData[i].AverageWorkQueueSize / 30; ultraAggregateTickData[i].WorkQueueVariance = 1.0 * ultraAggregateTickData[i].WorkQueueVariance / 30; WriteLine(ultraAggregateTickData[i], streamWriter); } } } } }
static void Pass1(string inputFilePath, string outputFilePath) { const int numNodes = 100; using (var streamReader = new StreamReader(inputFilePath)) { using (var streamWriter = new StreamWriter(outputFilePath)) { // Strip header row var line = streamReader.ReadLine(); // Write header row in output streamWriter.WriteLine("Run,Tick,IsUltra,AvgFulfillmentTime,NumFulfilled,AvgKnownFiles,AvgKnownNodes," + "NumTimeouts,PercentFulfilled,AvgWorkQueueSize,QueueVariance,NumberDead,Count"); // Loop over Runs, Ticks, and Nodes. Pass 1 aggregates nodes per tick for (var runIterator = 0; runIterator < 30; runIterator++) { for (var tickIterator = 0; tickIterator < 10000; tickIterator++) { var tickData = new TickData { WorkQueueObservations = new List<int>(), IsUltra = false }; var ultraTickData = new TickData { WorkQueueObservations = new List<int>(), IsUltra = true }; var hadUltras = false; for (var nodeIterator = 0; nodeIterator < numNodes; nodeIterator++) { line = streamReader.ReadLine(); if (line == null) break; var parts = line.Split(','); var run = int.Parse(parts[0]); var avgFulfillmentTime = double.Parse(parts[1]); var numFulfilled = int.Parse(parts[2]); var numKnownFiles = int.Parse(parts[3]); var numKnownNodes = int.Parse(parts[4]); var numTimeouts = int.Parse(parts[5]); var numUnfulfilled = int.Parse(parts[6]); var percentFulfilled = double.Parse(parts[7]); var workQueueSize = int.Parse(parts[8]); var isDead = bool.Parse(parts[9].Replace("\"", " ")); var isUltra = bool.Parse(parts[10].Replace("\"", " ")); var tickDouble = double.Parse(parts[11]); var tick = (int)tickDouble; if (isUltra) { hadUltras = true; ultraTickData.Run = run; ultraTickData.Tick = tick; ultraTickData.Count++; ultraTickData.AverageFulfillmentTime += avgFulfillmentTime; ultraTickData.NumberFullfilledRequests += numFulfilled; ultraTickData.AverageKnownFiles += numKnownFiles; ultraTickData.AverageKnownNodes += numKnownNodes; ultraTickData.NumberTimeouts += numTimeouts; ultraTickData.AverageWorkQueueSize += workQueueSize; ultraTickData.NumberDead += isDead ? 1 : 0; ultraTickData.WorkQueueObservations.Add(workQueueSize); } else { tickData.Run = run; tickData.Tick = tick; tickData.Count++; tickData.AverageFulfillmentTime += avgFulfillmentTime; tickData.NumberFullfilledRequests += numFulfilled; tickData.AverageKnownFiles += numKnownFiles; tickData.AverageKnownNodes += numKnownNodes; tickData.NumberTimeouts += numTimeouts; tickData.AverageWorkQueueSize += workQueueSize; tickData.NumberDead += isDead ? 1 : 0; tickData.WorkQueueObservations.Add(workQueueSize); } } if (line == null) break; // Compute appropriate averages and summaries, and write to file tickData.AverageFulfillmentTime = 1.0 * tickData.AverageFulfillmentTime / tickData.Count; tickData.AverageKnownFiles = 1.0 * tickData.AverageKnownFiles / tickData.Count; tickData.AverageKnownNodes = 1.0 * tickData.AverageKnownNodes / tickData.Count; tickData.AverageWorkQueueSize = 1.0 * tickData.AverageWorkQueueSize / tickData.Count; var denom = (tickData.NumberFullfilledRequests + tickData.NumberTimeouts); if (denom == 0) denom = 1; tickData.PercentFulfilled = (1.0 * tickData.NumberFullfilledRequests) / denom; tickData.WorkQueueVariance = Variance(tickData.WorkQueueObservations, tickData.AverageWorkQueueSize); WriteLine(tickData, streamWriter); if (hadUltras) { ultraTickData.AverageFulfillmentTime = 1.0 * ultraTickData.AverageFulfillmentTime / ultraTickData.Count; ultraTickData.AverageKnownFiles = 1.0 * ultraTickData.AverageKnownFiles / ultraTickData.Count; ultraTickData.AverageKnownNodes = 1.0 * ultraTickData.AverageKnownNodes / ultraTickData.Count; ultraTickData.AverageWorkQueueSize = 1.0 * ultraTickData.AverageWorkQueueSize / ultraTickData.Count; var denomU = (ultraTickData.NumberFullfilledRequests + ultraTickData.NumberTimeouts); if (denomU == 0) denomU = 1; ultraTickData.PercentFulfilled = (1.0 * ultraTickData.NumberFullfilledRequests) / denomU; ultraTickData.WorkQueueVariance = Variance(ultraTickData.WorkQueueObservations, ultraTickData.AverageWorkQueueSize); WriteLine(ultraTickData, streamWriter); } if (tickIterator % 1000 == 0) { Console.WriteLine("Finished Tick" + tickIterator); } } if (line == null) break; Console.WriteLine("Finished Run" + runIterator); } } } }