void Run(string[] args)
        {
            string formatString =
                "Format: \n" +
                "  <filename>\n" +
                "OR \n" +
                "  -in <filename>\n" +
                "  -outFormat=<csv|bin>\n" +
                "  [-o outFilename]\n" +
                "  [-force]\n" +
                "  [-verify]\n";


            // Read the command line
            if (args.Length < 1)
            {
                WriteLine(formatString);
                return;
            }


            ReadCommandLine(args);

            string inFilename;
            string outFormat;
            string outFilename = null;

            if (args.Length == 1)
            {
                inFilename = args[0];
                // Determine the output format automatically
                if (isCsvBinFile(inFilename))
                {
                    outFormat = "csv";
                }
                else if (isCsvFile(inFilename))
                {
                    outFormat = "bin";
                }
                else
                {
                    throw new Exception("Unknown input format!");
                }
            }
            else
            {
                // Advanced mode. Output format is required
                inFilename = GetArg("in", true);
                outFormat  = GetArg("outformat", true);
                if (outFormat == "" || inFilename == "")
                {
                    return;
                }
                outFilename = GetArg("o", null);
            }


            bool bBinOut = false;

            if (outFormat == "bin")
            {
                bBinOut = true;
            }
            else if (outFormat == "csv")
            {
                bBinOut = false;
            }
            else
            {
                throw new Exception("Unknown output format!");
            }

            string inFilenameWithoutExtension;

            if (isCsvBinFile(inFilename))
            {
                inFilenameWithoutExtension = inFilename.Substring(0, inFilename.Length - 8);
            }
            else if (isCsvFile(inFilename))
            {
                inFilenameWithoutExtension = inFilename.Substring(0, inFilename.Length - 4);
            }
            else
            {
                throw new Exception("Unexpected input filename extension!");
            }

            if (outFilename == null)
            {
                outFilename = inFilenameWithoutExtension + (bBinOut ? ".csv.bin" : ".csv");
            }

            if (outFilename.ToLowerInvariant() == inFilename.ToLowerInvariant())
            {
                throw new Exception("Input and output filenames can't match!");
            }

            if (System.IO.File.Exists(outFilename) && !GetBoolArg("force"))
            {
                throw new Exception("Output file already exists! Use -force to overwrite anyway.");
            }

            Console.WriteLine("Converting " + inFilename + " to " + outFormat + " format. Output filename: " + outFilename);
            Console.WriteLine("Reading input file...");
            CsvStats csvStats = CsvStats.ReadCSVFile(inFilename, null);

            Console.WriteLine("Writing output file...");
            if (bBinOut)
            {
                csvStats.WriteBinFile(outFilename);
            }
            else
            {
                csvStats.WriteToCSV(outFilename);
            }

            if (GetBoolArg("verify"))
            {
                // TODO: if verify is specified, use a temp intermediate file?
                // TODO: Check metadata, stat counts, stat names
                Console.WriteLine("Verifying output...");
                CsvStats csvStats2 = CsvStats.ReadCSVFile(outFilename, null);
                if (csvStats.SampleCount != csvStats2.SampleCount)
                {
                    throw new Exception("Verify failed!");
                }
            }
        }
        void Run(string[] args)
        {
            // Read the command line
            if (args.Length < 1)
            {
                WriteLine("Invalid args");
                WriteLine(formatString);
                return;
            }

            ReadCommandLine(args);

            string csvOutFilename = GetArg("o", false);

            if (csvOutFilename.Length == 0)
            {
                WriteLine("Missing -o arg");
                WriteLine(formatString);
                return;
            }

            // Set the title
            string title = GetArg("title", false);

            if (title.Length == 0)
            {
                title = MakeShortFilename(csvOutFilename).ToLower().Replace(".csv", "");
            }
            char c = title[0];

            c     = char.ToUpper(c);
            title = c + title.Substring(1);

            string filterOutlierStat      = GetArg("filterOutlierStat", false);
            float  filterOutlierThreshold = GetFloatArg("filterOutlierThreshold", 1000.0f);

            // Whether or not we want stats to be averaged rather than appended
            bool bAverage = GetBoolArg("avg");

            string csvDir          = GetArg("csvDir");
            string csvFilenamesStr = GetArg("csvs", false);

            if (csvDir == null && csvFilenamesStr == null)
            {
                WriteLine("-csvs or -csvdir is required");
                WriteLine(formatString);
                return;
            }

            string searchPattern = GetArg("searchPattern", null);

            if (csvFilenamesStr.Contains("*"))
            {
                // If passed a wildcard to -csvs, this is equivalent to -csvdir . -searchpattern <csvs>
                if (csvDir != null && csvDir != "")
                {
                    throw new Exception("Can't use -csvs with -csvdir");
                }
                csvDir        = ".";
                searchPattern = csvFilenamesStr;
            }

            // Read CSV filenames from a directory or list
            string[] csvFilenames;
            if (csvDir.Length > 0)
            {
                if (searchPattern == null)
                {
                    searchPattern = "*.csv";
                }
                DirectoryInfo di       = new DirectoryInfo(csvDir);
                bool          bRecurse = GetBoolArg("recurse");
                var           files    = di.GetFiles(searchPattern, bRecurse ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
                csvFilenames = new string[files.Length];
                int i = 0;
                foreach (FileInfo csvFile in files)
                {
                    csvFilenames[i] = csvFile.FullName;
                    i++;
                }
            }
            else
            {
                if (csvFilenamesStr.Length == 0)
                {
                    System.Console.Write(formatString);
                    return;
                }
                csvFilenames = csvFilenamesStr.Split(';');
            }

            Console.WriteLine("Collating " + csvFilenames.Length + " csvs:");
            foreach (string csvFilename in csvFilenames)
            {
                Console.WriteLine("  " + csvFilename);
            }
            Console.WriteLine("");

            CsvStats combinedCsvStats = new CsvStats();


            string        metadataFilterString = GetArg("metadataFilter", null);
            List <int>    frameCsvCounts       = new List <int>();
            List <string> allCsvFilenames      = new List <string>();
            int           csvIndex             = 0;

            foreach (string csvFilename in csvFilenames)
            {
                CsvStats srcCsvStats = CsvStats.ReadCSVFile(csvFilename, null);

                // Check for outliers
                bool skip = false;
                if (filterOutlierStat != null)
                {
                    StatSamples outlierStat = srcCsvStats.GetStat(filterOutlierStat);
                    if (outlierStat != null)
                    {
                        foreach (float sample in outlierStat.samples)
                        {
                            if (sample > filterOutlierThreshold)
                            {
                                WriteLine("CSV " + csvFilename + " ignored due to bad " + filterOutlierStat + " value: " + sample);
                                skip = true;
                                break;
                            }
                        }
                    }
                }
                if (metadataFilterString != null)
                {
                    if (srcCsvStats.metaData == null || !CsvStats.DoesMetadataMatchFilter(srcCsvStats.metaData, metadataFilterString))
                    {
                        WriteLine("Skipping CSV " + csvFilename + " due to metadata filter");
                        skip = true;
                    }
                }
                if (skip)
                {
                    continue;
                }

                // Add the CSV filename as the first event if we're not averaging
                if (!bAverage)
                {
                    CsvEvent firstEvent = new CsvEvent();
                    firstEvent.Frame = 0;
                    firstEvent.Name  = "CSV:" + MakeShortFilename(csvFilename).Replace(' ', '_').Replace(',', '_').Replace('\n', '_');
                    srcCsvStats.Events.Insert(0, firstEvent);
                }

                // Combine the stats
                if (csvIndex == 0)
                {
                    combinedCsvStats = srcCsvStats;
                }
                else
                {
                    CsvMetadata metadataA = combinedCsvStats.metaData;
                    CsvMetadata metadataB = srcCsvStats.metaData;

                    // If there is metadata, it should match
                    if (metadataA != null || metadataB != null)
                    {
                        metadataA.CombineAndValidate(metadataB);
                    }
                    combinedCsvStats.Combine(srcCsvStats, bAverage, false);
                }

                // If we're computing the average, update the counts for each frame
                if (bAverage)
                {
                    // Resize frameCsvCounts if necessary
                    for (int i = frameCsvCounts.Count; i < combinedCsvStats.SampleCount; i++)
                    {
                        frameCsvCounts.Add(0);
                    }
                    for (int i = 0; i < srcCsvStats.SampleCount; i++)
                    {
                        frameCsvCounts[i] += 1;
                    }
                }

                allCsvFilenames.Add(Path.GetFileName(csvFilename));
                csvIndex++;
                WriteLine("Csvs Processed: " + csvIndex + " / " + csvFilenames.Length);
            }


            if (bAverage)
            {
                // Divide all samples by the total number of CSVs
                foreach (StatSamples stat in combinedCsvStats.Stats.Values)
                {
                    for (int i = 0; i < stat.samples.Count; i++)
                    {
                        stat.samples[i] /= (float)(frameCsvCounts[i]);
                    }
                }

                // Add a stat for the csv count
                string csvCountStatName = "csvCount";
                if (!combinedCsvStats.Stats.ContainsKey(csvCountStatName))
                {
                    StatSamples csvCountStat = new StatSamples(csvCountStatName);
                    foreach (int count in frameCsvCounts)
                    {
                        csvCountStat.samples.Add((int)count);
                    }
                    combinedCsvStats.Stats.Add(csvCountStatName, csvCountStat);
                }
                if (combinedCsvStats.metaData != null)
                {
                    // Add some metadata
                    combinedCsvStats.metaData.Values.Add("Averaged", allCsvFilenames.Count.ToString());
                    combinedCsvStats.metaData.Values.Add("SourceFiles", string.Join(";", allCsvFilenames));
                }
            }

            combinedCsvStats.ComputeAveragesAndTotal();

            // Write the csv stats to a CSV
            combinedCsvStats.WriteToCSV(csvOutFilename);
        }
        void Run(string[] args)
        {
            // Read the command line
            if (args.Length < 1)
            {
                WriteLine("Invalid args");
                WriteLine(formatString);
                return;
            }

            ReadCommandLine(args);

            string csvOutFilename = GetArg("o", false);

            if (csvOutFilename.Length == 0)
            {
                WriteLine("Missing -o arg");
                WriteLine(formatString);
                return;
            }

            // Set the title
            string title = GetArg("title", false);

            if (title.Length == 0)
            {
                title = MakeShortFilename(csvOutFilename).ToLower().Replace(".csv", "");
            }
            char c = title[0];

            c     = char.ToUpper(c);
            title = c + title.Substring(1);

            // Whether or not we want stats to be averaged rather than appended
            bool bAverage = GetBoolArg("avg");

            // Read CSV filenames from a directory or list
            string[] csvFilenames;
            string   csvDir = GetArg("csvDir");

            if (csvDir.Length > 0)
            {
                DirectoryInfo di    = new DirectoryInfo(csvDir);
                var           files = di.GetFiles("*.csv", SearchOption.AllDirectories);
                csvFilenames = new string[files.Length];
                int i = 0;
                foreach (FileInfo csvFile in files)
                {
                    csvFilenames[i] = csvFile.FullName;
                    i++;
                }
            }
            else
            {
                string csvFilenamesStr = GetArg("csvs", true);
                if (csvFilenamesStr.Length == 0)
                {
                    System.Console.Write(formatString);
                    return;
                }
                csvFilenames = csvFilenamesStr.Split(';');
            }

            CsvStats combinedCsvStats = new CsvStats();

            // List of stats to be averaged
            CsvStats[] statsToAvg = new CsvStats[csvFilenames.Length];

            // Read all the CSVs into one big CSVStats class
            int csvIndex = 0;

            foreach (string csvFilename in csvFilenames)
            {
                CsvStats csvStats = CsvStats.ReadCSVFile(csvFilename, null);

                // Add the CSV filename as the first event (sanitised for CSV)
                CsvEvent firstEvent = null;
                firstEvent       = new CsvEvent();
                firstEvent.Frame = 0;
                firstEvent.Name  = "CSV:" + MakeShortFilename(csvFilename).Replace(' ', '_').Replace(',', '_').Replace('\n', '_');
                csvStats.Events.Insert(0, firstEvent);

                if (bAverage)
                {
                    statsToAvg[csvIndex] = csvStats;
                }
                else
                {
                    // Combine the stats
                    if (csvIndex == 0)
                    {
                        combinedCsvStats = csvStats;
                    }
                    else
                    {
                        CsvMetadata metadataA = combinedCsvStats.metaData;
                        CsvMetadata metadataB = csvStats.metaData;

                        // If there is metadata, it should match
                        if (metadataA != null || metadataB != null)
                        {
                            if (!CsvMetadata.Matches(metadataA, metadataB))
                            {
                                WriteLine("Metadata mismatch!");
                            }
                        }
                        combinedCsvStats.Combine(csvStats, false);
                    }
                }

                csvIndex++;
            }

            if (bAverage)
            {
                // Average stats by frame
                combinedCsvStats = CsvStats.AverageByFrame(statsToAvg);

                if (combinedCsvStats.metaData != null)
                {
                    // Add few more meta information
                    combinedCsvStats.metaData.Values.Add("Averaged", statsToAvg.Length.ToString());
                }

                int      fIt       = 0;
                string[] filenames = new string[csvFilenames.Length];
                foreach (var filename in csvFilenames)
                {
                    filenames[fIt++] = Path.GetFileName(filename);
                }

                if (combinedCsvStats.metaData != null)
                {
                    combinedCsvStats.metaData.Values.Add("Files", string.Join(",", filenames));
                }
            }

            // Write the csv stats to a CSV
            combinedCsvStats.WriteToCSV(csvOutFilename);
        }
Example #4
0
        void Run(string[] args)
        {
            // Read the command line
            if (args.Length < 1)
            {
                WriteLine("Invalid args");
                WriteLine(formatString);
                return;
            }

            ReadCommandLine(args);

            string csvOutFilename = GetArg("o", false);

            if (csvOutFilename.Length == 0)
            {
                WriteLine("Missing -o arg");
                WriteLine(formatString);
                return;
            }

            // Set the title
            string title = GetArg("title", false);

            if (title.Length == 0)
            {
                title = MakeShortFilename(csvOutFilename).ToLower().Replace(".csv", "");
            }
            char c = title[0];

            c     = char.ToUpper(c);
            title = c + title.Substring(1);

            string filterOutlierStat      = GetArg("filterOutlierStat", false);
            float  filterOutlierThreshold = GetFloatArg("filterOutlierThreshold", 1000.0f);

            // Whether or not we want stats to be averaged rather than appended
            bool bAverage = GetBoolArg("avg");

            // Read CSV filenames from a directory or list
            string[] csvFilenames;
            string   csvDir = GetArg("csvDir");

            if (csvDir.Length > 0)
            {
                DirectoryInfo di    = new DirectoryInfo(csvDir);
                var           files = di.GetFiles("*.csv", SearchOption.AllDirectories);
                csvFilenames = new string[files.Length];
                int i = 0;
                foreach (FileInfo csvFile in files)
                {
                    csvFilenames[i] = csvFile.FullName;
                    i++;
                }
            }
            else
            {
                string csvFilenamesStr = GetArg("csvs", true);
                if (csvFilenamesStr.Length == 0)
                {
                    System.Console.Write(formatString);
                    return;
                }
                csvFilenames = csvFilenamesStr.Split(';');
            }

            CsvStats combinedCsvStats = new CsvStats();

            // List of stats to be averaged
            //CsvStats[] statsToAvg = new CsvStats[csvFilenames.Length];

            // Read all the CSVs into one big CSVStats class
            List <int>    frameCsvCounts  = new List <int>();
            List <string> allCsvFilenames = new List <string>();
            int           csvIndex        = 0;

            foreach (string csvFilename in csvFilenames)
            {
                CsvStats srcCsvStats = CsvStats.ReadCSVFile(csvFilename, null);

                // Check for outliers
                bool skip = false;
                if (filterOutlierStat != null)
                {
                    StatSamples outlierStat = srcCsvStats.GetStat(filterOutlierStat);
                    if (outlierStat != null)
                    {
                        foreach (float sample in outlierStat.samples)
                        {
                            if (sample > filterOutlierThreshold)
                            {
                                WriteLine("CSV " + csvFilename + " ignored due to bad " + filterOutlierStat + " value: " + sample);
                                skip = true;
                                break;
                            }
                        }
                    }
                }
                if (skip)
                {
                    continue;
                }

                // Add the CSV filename as the first event if we're not averaging
                if (!bAverage)
                {
                    CsvEvent firstEvent = new CsvEvent();
                    firstEvent.Frame = 0;
                    firstEvent.Name  = "CSV:" + MakeShortFilename(csvFilename).Replace(' ', '_').Replace(',', '_').Replace('\n', '_');
                    srcCsvStats.Events.Insert(0, firstEvent);
                }

                // Combine the stats
                if (csvIndex == 0)
                {
                    combinedCsvStats = srcCsvStats;
                }
                else
                {
                    CsvMetadata metadataA = combinedCsvStats.metaData;
                    CsvMetadata metadataB = srcCsvStats.metaData;

                    // If there is metadata, it should match
                    if (metadataA != null || metadataB != null)
                    {
                        metadataA.CombineAndValidate(metadataB);
                    }
                    combinedCsvStats.Combine(srcCsvStats, bAverage, false);
                    if (bAverage)
                    {
                        // Resize the framecount array to match
                        for (int i = frameCsvCounts.Count; i < combinedCsvStats.SampleCount; i++)
                        {
                            frameCsvCounts.Add(0);
                        }
                        for (int i = 0; i < srcCsvStats.SampleCount; i++)
                        {
                            frameCsvCounts[i] += 1;
                        }
                    }
                }
                allCsvFilenames.Add(Path.GetFileName(csvFilename));
                WriteLine("Csvs Processed: " + csvIndex + " / " + csvFilenames.Length);
                csvIndex++;
            }

            if (bAverage)
            {
                // Divide all samples by the total number of CSVs
                foreach (StatSamples stat in combinedCsvStats.Stats.Values)
                {
                    for (int i = 0; i < stat.samples.Count; i++)
                    {
                        stat.samples[i] /= (float)(frameCsvCounts[i]);
                    }
                }

                // Add a stat for the csv count
                string csvCountStatName = "csvCount";
                if (!combinedCsvStats.Stats.ContainsKey(csvCountStatName))
                {
                    StatSamples csvCountStat = new StatSamples(csvCountStatName);
                    foreach (int count in frameCsvCounts)
                    {
                        csvCountStat.samples.Add((int)count);
                    }
                    combinedCsvStats.Stats.Add(csvCountStatName, csvCountStat);
                }
                if (combinedCsvStats.metaData != null)
                {
                    // Add some metadata
                    combinedCsvStats.metaData.Values.Add("Averaged", allCsvFilenames.Count.ToString());
                    combinedCsvStats.metaData.Values.Add("SourceFiles", string.Join(";", allCsvFilenames));
                }
            }

            combinedCsvStats.ComputeAveragesAndTotal();

            // Write the csv stats to a CSV
            combinedCsvStats.WriteToCSV(csvOutFilename);
        }