public CsvStats(CsvStats sourceCsvStats, string [] statNamesToFilter = null)
{
Stats = new Dictionary <string, StatSamples>();
Events = new List <CsvEvent>();
if (statNamesToFilter != null)
{
foreach (string statName in statNamesToFilter)
{
List <StatSamples> stats = sourceCsvStats.GetStatsMatchingString(statName);
foreach (StatSamples sourceStat in stats)
{
string key = sourceStat.Name.ToLower();
if (!Stats.ContainsKey(key))
{
Stats.Add(key, new StatSamples(sourceStat));
}
}
}
}
else
{
foreach (StatSamples sourceStat in sourceCsvStats.Stats.Values)
{
AddStat(new StatSamples(sourceStat));
}
}
foreach (CsvEvent ev in sourceCsvStats.Events)
{
Events.Add(new CsvEvent(ev));
}
if (sourceCsvStats.metaData != null)
{
metaData = new CsvMetadata(sourceCsvStats.metaData);
}
}
public void Combine(CsvStats srcStats, bool computeAverage = true)
{
if (metaData != null)
{
metaData.CombineAndValidate(srcStats.metaData);
}
// Collate the stats, removing ones which don't exist in both
int FrameOffset = 0;
Dictionary <string, StatSamples> newStats = new Dictionary <string, StatSamples>();
foreach (StatSamples srcStat in srcStats.Stats.Values.ToArray())
{
if (Stats.ContainsKey(srcStat.Name.ToLower()))
{
StatSamples destStat = GetStat(srcStat.Name);
FrameOffset = Math.Max(FrameOffset, destStat.samples.Count);
foreach (float sample in srcStat.samples)
{
destStat.samples.Add(sample);
}
;
newStats.Add(srcStat.Name.ToLower(), destStat);
}
}
if (computeAverage)
{
// Compute the new average
foreach (StatSamples stat in newStats.Values.ToArray())
{
stat.ComputeAverageAndTotal();
}
}
// Add the events, offsetting the frame numbers
foreach (CsvEvent ev in srcStats.Events)
{
ev.Frame += FrameOffset;
Events.Add(ev);
}
Stats = newStats;
}
public CsvStats StripByEvents(string startString, string endString, bool invert, out int numFramesStripped)
{
CsvStats newCsvStats = new CsvStats();
List <int> startIndices = null;
List <int> endIndices = null;
GetEventFrameIndexDelimiters(startString, endString, out startIndices, out endIndices);
numFramesStripped = 0;
if (startIndices.Count == 0)
{
return(this);
}
numFramesStripped = -1;
int frameCount = 0;
// Strip out samples and recompute averages
foreach (StatSamples stat in Stats.Values)
{
StatSamples newStat = new StatSamples(stat, false);
newStat.samples = new List <float>(SampleCount);
int stripEventIndex = 0;
for (int i = 0; i < stat.samples.Count; i++)
{
int startIndex = (stripEventIndex < startIndices.Count) ? startIndices[stripEventIndex] : stat.samples.Count;
int endIndex = (stripEventIndex < endIndices.Count) ? endIndices[stripEventIndex] : stat.samples.Count;
if (i < startIndex)
{
newStat.samples.Add(stat.samples[i]);
}
else
{
if (i == endIndex)
{
stripEventIndex++;
}
}
}
if (numFramesStripped == -1)
{
numFramesStripped = stat.samples.Count - newStat.samples.Count;
frameCount = stat.samples.Count;
}
newStat.ComputeAverageAndTotal();
newCsvStats.AddStat(newStat);
}
// Strip out the events
int FrameOffset = 0;
{
int stripEventIndex = 0;
for (int i = 0; i < Events.Count; i++)
{
CsvEvent csvEvent = Events[i];
int startIndex = (stripEventIndex < startIndices.Count) ? startIndices[stripEventIndex] : frameCount;
int endIndex = (stripEventIndex < endIndices.Count) ? endIndices[stripEventIndex] : frameCount;
CsvEvent newEvent = new CsvEvent(csvEvent.Name, csvEvent.Frame + FrameOffset);
if (csvEvent.Frame < startIndex)
{
newCsvStats.Events.Add(newEvent);
}
else
{
if (csvEvent.Frame == startIndex)
{
// Check if this is the last event this frame
if (i == Events.Count - 1 || Events[i + 1].Frame != csvEvent.Frame)
{
// Subsequent events will get offset by this amount
FrameOffset -= endIndex - startIndex;
}
}
if (csvEvent.Frame == endIndex + 1)
{
newCsvStats.Events.Add(newEvent);
stripEventIndex++;
}
}
}
}
newCsvStats.metaData = metaData;
return(newCsvStats);
}
// This will average only those stats that exist in all the stat sets
// Stats are averaged per frame, and the the final length is be equal to the longest CSV
// If CSVs are of varying length, this means later frames will be averaged over fewer samples than earlier frames
public static CsvStats AverageByFrame(CsvStats[] statsToAvg, bool bStatsAvarage = false)
{
CsvStats avgStats = new CsvStats();
if (statsToAvg.Length > 0)
{
// We need to have all the frame per each stat in the file
int maxFrames = 0;
// Use the first as stat name basis
string[] statKeys = statsToAvg[0].Stats.Keys.ToArray();
foreach (string statName in statKeys)
{
int maxSamples = 0;
int statCount = 0;
foreach (CsvStats stats in statsToAvg)
{
// Remove from the set of new stats if
// it doesn't exist in one of the set.
if (stats.Stats.ContainsKey(statName))
{
maxSamples = Math.Max(stats.Stats[statName].samples.Count, maxSamples);
statCount++;
}
}
if (statCount == statsToAvg.Length && maxSamples > 0)
{
avgStats.AddStat(new StatSamples(statName));
maxFrames = Math.Max(maxFrames, maxSamples);
}
}
// Copy meta data
avgStats.metaData = statsToAvg[0].metaData;
if (avgStats.metaData != null)
{
foreach (CsvStats stats in statsToAvg)
{
avgStats.metaData.CombineAndValidate(stats.metaData);
}
}
foreach (string statName in avgStats.Stats.Keys)
{
// This should always exist
StatSamples avgSamples = avgStats.GetStat(statName);
if (avgSamples != null)
{
List <int> sampleCounts = new List <int>();
sampleCounts.AddRange(Enumerable.Repeat(0, maxFrames));
// Initialise sample to 0.0
avgSamples.samples.AddRange(Enumerable.Repeat(0.0f, maxFrames));
// Add samples from other stats
foreach (CsvStats stats in statsToAvg)
{
StatSamples statSamples = stats.GetStat(statName);
if ((statSamples != null) && (avgSamples.samples.Count >= statSamples.samples.Count))
{
// This should always be true: avgSamples.samples.Count >= statSamples.samples.Count
for (int i = 0; i < statSamples.samples.Count; i++)
{
avgSamples.samples[i] += statSamples.samples[i];
sampleCounts[i] += 1;
}
}
}
// Average the samples
for (int i = 0; i < avgSamples.samples.Count; i++)
{
avgSamples.samples[i] /= (float)sampleCounts[i];
}
}
if (bStatsAvarage)
{
avgSamples.ComputeAverageAndTotal();
}
}
}
return(avgStats);
}
public void Combine(CsvStats srcStats, bool sumRows = false, bool computeAverage = true)
{
if (metaData != null)
{
metaData.CombineAndValidate(srcStats.metaData);
}
// Collate the stats, removing ones which don't exist in both
int EventFrameOffset = 0;
Dictionary <string, StatSamples> newStats = new Dictionary <string, StatSamples>();
foreach (StatSamples srcStat in srcStats.Stats.Values.ToArray())
{
if (Stats.ContainsKey(srcStat.Name.ToLower()))
{
StatSamples destStat = GetStat(srcStat.Name);
if (sumRows)
{
int n = Math.Min(srcStat.samples.Count, destStat.samples.Count);
// Combine initial stats
for (int i = 0; i < n; i++)
{
destStat.samples[i] += srcStat.samples[i];
}
// Add the remaining stats
for (int i = n; i < srcStat.samples.Count; i++)
{
destStat.samples.Add(srcStat.samples[i]);
}
}
else
{
EventFrameOffset = Math.Max(EventFrameOffset, destStat.samples.Count);
foreach (float sample in srcStat.samples)
{
destStat.samples.Add(sample);
}
;
}
newStats.Add(srcStat.Name.ToLower(), destStat);
}
}
if (computeAverage)
{
// Compute the new average
foreach (StatSamples stat in newStats.Values.ToArray())
{
stat.ComputeAverageAndTotal();
}
}
if (sumRows)
{
// just remove events. Averaging is complicated/doesn't really make sense
Events.Clear();
}
else
{
// Add the events, offsetting the frame numbers
foreach (CsvEvent ev in srcStats.Events)
{
ev.Frame += EventFrameOffset;
Events.Add(ev);
}
}
Stats = newStats;
}
public static CsvStats ReadCSVFromLines(string[] linesArray, string[] statNames, int numRowsToSkip = 0, bool skipReadingData = false)
{
List <string> lines = linesArray.ToList();
// Check if we have any metadata
bool bHasMetaData = LineIsMetadata(lines.Last());
CsvMetadata metaData = null;
// Add the metadata to the stats collection
if (bHasMetaData)
{
string[] lastLine = lines.Last().Split(',');
metaData = new CsvMetadata(lastLine);
// New CSVs from the csv profiler have a header row at the end of the file,
// since the profiler writes out the file incrementally.
if ("1" == metaData.GetValue("hasheaderrowatend", null))
{
// Swap the header row for the one at the end of the file,
// then remove the end one.
lines[0] = lines[lines.Count - 2];
lines.RemoveAt(lines.Count - 2);
}
}
if (numRowsToSkip > 0)
{
lines.RemoveRange(0, numRowsToSkip);
}
string[] headings = lines[0].Split(',');
if (lines.Count > 2 && lines[lines.Count - 1] == "\"")
{
lines[lines.Count - 2] += lines[lines.Count - 1];
lines.RemoveAt(lines.Count - 1);
}
if (skipReadingData)
{
int dataLineCount = bHasMetaData ? lines.Count - 2 : lines.Count - 1;
lines.RemoveRange(1, dataLineCount);
}
// Get the list of lower case stat names, expanding wildcards
string[] statNamesLowercase = null;
if (statNames != null)
{
statNamesLowercase = statNames.Select(s => s.ToLowerInvariant()).ToArray();
{
// Expand the list of stat names based on the wildcards and the headers. We do this here to make sorting simpler
HashSet <string> newStatNamesLowercase = new HashSet <string>();
foreach (string statname in statNamesLowercase)
{
if (statname.EndsWith("*"))
{
int index = statname.LastIndexOf('*');
string prefix = statname.Substring(0, index);
// Expand all the stat names
foreach (string headingStat in headings)
{
if (headingStat.ToLower().StartsWith(prefix))
{
newStatNamesLowercase.Add(headingStat.ToLower());
}
}
}
else
{
newStatNamesLowercase.Add(statname);
}
}
statNamesLowercase = newStatNamesLowercase.ToArray();
}
}
// First line is headings, last line contains build info
int numSamples = lines.Count - (bHasMetaData ? 2 : 1);
// Create the stats
int eventHeadingIndex = -1;
StatSamples[] stats = new StatSamples[headings.Length];
for (int i = 0; i < headings.Length; i++)
{
string heading = headings[i].Trim();
if (heading == "")
{
continue;
}
// find the events column (if there is one)
else if (heading.ToLower() == "events")
{
eventHeadingIndex = i;
}
else if (statNamesLowercase == null || statNamesLowercase.Contains(heading.ToLower()))
{
stats[i] = new StatSamples(heading, numSamples);
}
}
List <CsvEvent> FilteredEvents = new List <CsvEvent>();
if (!skipReadingData)
{
string[] eventStrings = new string[numSamples];
// for (int i = 1; i < numSamples + 1; i++)
Parallel.For(1, numSamples + 1,
i =>
{
int sampleIndex = i - 1;
int statIndex = 0;
string line = lines[i] + "\n";
for (int j = 0; j < line.Length; j++)
{
// Note: we check statIndex<stats.length here in case of truncated CSVs
if (statIndex < stats.Length && stats[statIndex] != null)
{
// Read the stat
float value = 0.0f;
// Skip whitespace
while (line[j] == ' ')
{
j++;
}
bool negative = false;
if (line[j] == '-')
{
negative = true;
j++;
}
// Read the nonfractional part of the number
int num = 0;
while (line[j] >= '0' && line[j] <= '9')
{
num *= 10;
num += line[j] - '0';
j++;
}
value = (float)num;
if (line[j] == '.')
{
// read fractional part
num = 0;
j++;
float multiplier = 0.1f;
while (line[j] >= '0' && line[j] <= '9')
{
value += (float)(line[j] - '0') * multiplier;
j++;
multiplier *= 0.1f;
}
}
if (negative)
{
value = -value;
}
stats[statIndex].samples[sampleIndex] = value;
// Skip everything else until the next newline or comma
while (line[j] != ',' && line[j] != '\n')
{
j++;
}
}
else
{
// Skip parsing
int startJ = j;
while (line[j] != ',' && line[j] != '\n')
{
j++;
}
if (statIndex == eventHeadingIndex)
{
eventStrings[sampleIndex] = line.Substring(startJ, j - startJ);
}
}
statIndex++;
}
}
); // Needed by parallel for
// Read events
for (int i = 0; i < eventStrings.Length; i++)
{
string eventString = eventStrings[i];
if (!string.IsNullOrEmpty(eventString))
{
string[] Events = eventString.Split(';');
foreach (string EventString in Events)
{
if (EventString.Length > 0)
{
CsvEvent ev = new CsvEvent();
ev.Frame = i;
ev.Name = EventString;
FilteredEvents.Add(ev);
}
}
}
}
}
// Make sure the stat ordering matches the order they're passed in
CsvStats csvStats = new CsvStats();
if (statNamesLowercase != null)
{
CsvStats unorderedCsvStats = new CsvStats();
foreach (StatSamples statSamples in stats)
{
if (statSamples != null)
{
// Combine stats if we find a duplicate
if (unorderedCsvStats.Stats.ContainsKey(statSamples.Name.ToLower()))
{
StatSamples existingStat = unorderedCsvStats.GetStat(statSamples.Name);
for (int i = 0; i < statSamples.samples.Count; i++)
{
existingStat.samples[i] += statSamples.samples[i];
}
}
else
{
unorderedCsvStats.AddStat(statSamples);
}
}
}
foreach (string statName in statNamesLowercase)
{
StatSamples stat = unorderedCsvStats.GetStat(statName);
if (stat != null)
{
csvStats.AddStat(stat);
}
}
}
else
{
int c = 0;
foreach (StatSamples statSamples in stats)
{
c++;
if (statSamples != null)
{
if (csvStats.Stats.ContainsKey(statSamples.Name.ToLower()))
{
// Combine stats if we find a duplicate
StatSamples existingStat = csvStats.GetStat(statSamples.Name);
for (int i = 0; i < statSamples.samples.Count; i++)
{
existingStat.samples[i] += statSamples.samples[i];
}
}
else
{
csvStats.AddStat(statSamples);
}
}
}
}
// Compute averages
foreach (StatSamples stat in csvStats.Stats.Values.ToArray())
{
stat.ComputeAverageAndTotal();
}
csvStats.metaData = metaData;
csvStats.Events = FilteredEvents;
return(csvStats);
}