public void EventStatisticCacheGetTypeStatisticsIncorrectFilePathsTest()
        {
            TypeCache typecache = new TypeCache();

            EventStatisticCache cache = new EventStatisticCache();
            cache.GetTypeStatistics(typecache, new[] { @"C:\hello\world.etl", testDataFile });
        }
        public void EventStatisticCacheGetTypeStatisticsFilesInvalidTest()
        {
            TypeCache typecache = new TypeCache();

            EventStatisticCache cache = new EventStatisticCache();
            cache.GetTypeStatistics(typecache, new[] { string.Empty, testDataFile });
        }
Esempio n. 3
0
        public void TypeCacheIgnoresTypesContainingImportTest()
        {
            var typeCache = new TypeCache();
            typeCache.Initialize("TypeCacheIgnoresTypesContainingImportTest", @"TestData\TrianglesWithImport.etl");

            Assert.AreEqual(3, typeCache.Cache.Count);
        }
Esempio n. 4
0
        public void TypeCacheIgnoresTypesContainingImportTest()
        {
            var typeCache = new TypeCache();
            typeCache.Init("test5", new[] { @"TestData\TrianglesWithImport.etl" });

            Assert.AreEqual(4, typeCache.Types.Length);
            Assert.AreEqual(3, typeCache.Manifests.Length);
        }
Esempio n. 5
0
        public void TypeCacheInitHappyPathTest()
        {
            var typeCache = new TypeCache();
            typeCache.Initialize("test", new[] { testDataFile });

            var dir = typeCache.CacheDirectory;

            Assert.AreEqual(3, Directory.GetFiles(dir, "*.bond").Length);
            Assert.AreEqual(3, typeCache.Cache.Count);

            Utilities.ForceDeleteDirectory("test");
        }
Esempio n. 6
0
        public void TypeCacheInitHappyPathTest()
        {
            var typeCache = new TypeCache();
            typeCache.Init("test", new[] { testDataFile });

            var dir = typeCache.CacheDirectory;

            Assert.AreEqual(3, Directory.GetFiles(dir, "*.bond").Length);
            Assert.AreEqual(3, typeCache.Manifests.Length);

            Directory.Delete(dir, true);
        }
        public void EventStatisticCacheCreatesCsvTest()
        {
            TypeCache typecache = new TypeCache();
            typecache.Initialize("test2", new[] { testDataFile });

            EventStatisticCache cache = new EventStatisticCache();
            var result = cache.GetTypeStatistics(typecache, new[] { testDataFile });

            Assert.IsTrue(File.Exists(Path.Combine(Environment.CurrentDirectory, "TestData", Path.GetFileNameWithoutExtension(testDataFile) + ".csv")));

            File.Delete(Path.Combine(Environment.CurrentDirectory, Path.GetFileNameWithoutExtension(testDataFile) + ".csv"));
        }
        public void EventStatisticCacheGetTypeStatisticsHappyPathTest()
        {
            TypeCache typecache = new TypeCache();
            typecache.Initialize("test1", new[] { testDataFile });

            EventStatisticCache cache = new EventStatisticCache();
            var result = cache.GetTypeStatistics(typecache, new[] { testDataFile });

            Assert.AreEqual(3, result.Keys.Count);
            Assert.IsTrue(result.All(a => a.Value.EventCount == 10));

            var typeNames = new[] { "IsoscelesTriangle", "EquilateralTriangle", "RightAngledTriangle" };
            Assert.IsTrue(result.All(a => typeNames.Contains(a.Key.Name)));
        }
        public void EventStatisticLoadCsvValuesTest()
        {
            TypeCache typecache = new TypeCache();
            typecache.Initialize("test3", new[] { testDataFile });

            EventStatisticCache cache = new EventStatisticCache();
            var result = cache.GetTypeStatistics(typecache, new[] { testDataFile });

            Assert.IsTrue(File.Exists(Path.Combine(Environment.CurrentDirectory, "TestData", Path.GetFileNameWithoutExtension(testDataFile) + ".csv")));

            File.Delete(Path.Combine(Environment.CurrentDirectory, Path.GetFileNameWithoutExtension(testDataFile) + ".csv"));

            foreach (var item in result)
            {
                switch (item.Key.Name)
                {
                    case "IsoscelesTriangle":
                        Assert.AreEqual(27D, item.Value.AverageByteSize);
                        Assert.AreEqual(270, item.Value.ByteSize);
                        Assert.AreEqual(10, item.Value.EventCount);
                        Assert.AreEqual(0.0370, item.Value.EventsPerSecond);
                        break;

                    case "EquilateralTriangle":
                        Assert.AreEqual(25D, item.Value.AverageByteSize);
                        Assert.AreEqual(250, item.Value.ByteSize);
                        Assert.AreEqual(10, item.Value.EventCount);
                        Assert.AreEqual(0.0370, item.Value.EventsPerSecond);
                        break;

                    case "RightAngledTriangle":
                        Assert.AreEqual(29D, item.Value.AverageByteSize);
                        Assert.AreEqual(290, item.Value.ByteSize);
                        Assert.AreEqual(10, item.Value.EventCount);
                        Assert.AreEqual(0.037, item.Value.EventsPerSecond);
                        break;

                    default:
                        Assert.Fail("Unexpected test result. Test fails.");
                        break;
                }

            }
        }
        public void MultipleTypesStatsCalculationTest()
        {
            TypeCache typecache = new TypeCache();
            typecache.Initialize("MultipleTypesStatsCalculationTest", new[] { testDataFile });

            Assert.AreEqual(4, typecache.Cache.Count);
            Assert.IsTrue(typecache.Cache.All(a => a != null && a.Manifest != null && a.Type != null));

            EventStatisticCache cache = new EventStatisticCache();
            var result = cache.GetTypeStatistics(typecache, new[] { testDataFile });

            Assert.AreEqual(3, result.Count);

            var values = result.Values.ToArray();
            Assert.AreEqual(10, values[0].EventCount);
            Assert.AreEqual(10, values[1].EventCount);
            Assert.AreEqual(10, values[2].EventCount);

            File.Delete(Path.Combine("TestData", Path.GetFileNameWithoutExtension(testDataFile) + ".csv"));
        }
        public void ReadTypesFromSingleManifestTest()
        {
            TypeCache typecache = new TypeCache();
            typecache.Initialize("ReadTypesFromSingleManifestTest", new[] { testDataFile });

            Debug.WriteLine(typecache.CacheDirectory);

            Assert.AreEqual(4, typecache.Cache.Count);

            Debug.WriteLine(typecache.Cache[0].Manifest.ManifestId);
            Debug.WriteLine(typecache.Cache[1].Manifest.ManifestId);
            Debug.WriteLine(typecache.Cache[2].Manifest.ManifestId);
            Debug.WriteLine(typecache.Cache[3].Manifest.ManifestId);

            Debug.WriteLine(typecache.Cache[0].Type.Name);
            Debug.WriteLine(typecache.Cache[1].Type.Name);
            Debug.WriteLine(typecache.Cache[2].Type.Name);
            Debug.WriteLine(typecache.Cache[3].Type.Name);

            Assert.IsTrue(typecache.Cache.All(a => a != null && a.Manifest != null && a.Type != null));
        }
Esempio n. 12
0
 public void TypeCacheInitTargetDirNullTest()
 {
     var typeCache = new TypeCache();
     typeCache.Init(null, new string[] { testDataFile });
 }
Esempio n. 13
0
 public void TypeCacheInitNullFilesTest()
 {
     var typeCache = new TypeCache();
     typeCache.Init("dir", null);
 }
Esempio n. 14
0
 public void TypeCacheInitNoFilesTest()
 {
     var typeCache = new TypeCache();
     typeCache.Init("dir", new string[] { });
     typeCache.Init("dir", null);
 }
Esempio n. 15
0
        private Dictionary<Type, CsvRelatedStats> CalculateTypeStatistics(TypeCache typeCache, string inputFile)
        {
            Console.WriteLine("Getting statistics from file {0}.", inputFile);

            var statsPerType = new Dictionary<Type, CsvRelatedStats>();

            Stopwatch sw = Stopwatch.StartNew();

            var rawCount = from events in BinaryEtwObservable.FromSequentialFiles(inputFile)
                           group events by events.PayloadId
                               into eventTypes
                               from all in
                                   eventTypes.Aggregate(
                                       new { EventCount = (long)0, Bytes = (long)0, minTime = long.MaxValue, maxTime = 0L },
                                       (ac, events) =>
                                       new
                                       {
                                           EventCount = ac.EventCount + 1,
                                           Bytes = ac.Bytes + events.EventPayloadLength,
                                           minTime = Math.Min(ac.minTime, events.ReceiveFileTimeUtc),
                                           maxTime = Math.Max(ac.maxTime, events.ReceiveFileTimeUtc)
                                       })
                               select new { ManifestId = eventTypes.Key, all.EventCount, all.Bytes, all.minTime, all.maxTime };

            var counts = rawCount.ToEnumerable().ToArray();

            sw.Stop();
            Console.WriteLine("Query took {0} milliseconds.", sw.ElapsedMilliseconds);

            foreach (var c in counts)
            {
                var manifest = typeCache.Manifests
                    .FirstOrDefault(m => string.Equals(m.ManifestId, c.ManifestId, StringComparison.OrdinalIgnoreCase));

                if (manifest != null)
                {
                    var line = manifest.Manifest
                        .Split('\n').LastOrDefault(l => l.Trim().StartsWith(@"struct ", StringComparison.OrdinalIgnoreCase));

                    if (line != null)
                    {
                        var className = line.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)[1].Trim();

                        var type = typeCache.Types.FirstOrDefault(t => t.Name == className);

                        if (type != null)
                        {
                            var minDateTime = DateTime.FromFileTimeUtc(c.minTime);
                            var maxDateTime = DateTime.FromFileTimeUtc(c.maxTime);
                            var duration = (maxDateTime - minDateTime).TotalSeconds;
                            if (Math.Abs(duration) < 0.01) duration = 1;

                            var stats = new EventStatistics
                            {
                                AverageByteSize = c.Bytes / c.EventCount,
                                ByteSize = c.Bytes,
                                EventCount = c.EventCount,
                                EventsPerSecond = c.EventCount / duration
                            };

                            statsPerType[type] = new CsvRelatedStats
                                {
                                    ManifestId = c.ManifestId,
                                    Statistics = stats,
                                };
                        }
                    }
                }
            }

            return statsPerType;
        }
        private Dictionary<Type, CsvRelatedStats> CalculateTypeStatistics(TypeCache typeCache, string inputFile)
        {
            Console.WriteLine("Getting statistics from file {0}.", inputFile);

            var statsPerType = new Dictionary<Type, CsvRelatedStats>();

            Stopwatch sw = Stopwatch.StartNew();

            var rawCount = from events in BinaryEtwObservable.FromSequentialFiles(inputFile)
                           group events by events.TypeId
                               into eventTypes
                               from all in
                                   eventTypes.Aggregate(
                                       new { EventCount = (long)0, Bytes = (long)0, minTime = long.MaxValue, maxTime = 0L },
                                       (ac, events) =>
                                       new
                                       {
                                           EventCount = ac.EventCount + 1,
                                           Bytes = ac.Bytes + events.Payload.Length,
                                           minTime = Math.Min(ac.minTime, events.ReceivedTime.ToFileTime()),
                                           maxTime = Math.Max(ac.maxTime, events.ReceivedTime.ToFileTime()),
                                       })
                               select new { ManifestId = eventTypes.Key, all.EventCount, all.Bytes, all.minTime, all.maxTime, };

            var counts = rawCount.ToEnumerable().ToArray();

            sw.Stop();
            Console.WriteLine("Query took {0} milliseconds.", sw.ElapsedMilliseconds);

            foreach (var c in counts)
            {
                var typeCacheItem = typeCache.FindMatchOrDefault(c.ManifestId);

                if (typeCacheItem != null && typeCacheItem.Type != null)
                {
                    var type = typeCacheItem.Type;
                    if (type != null)
                    {
                        var minDateTime = DateTime.FromFileTimeUtc(c.minTime);
                        var maxDateTime = DateTime.FromFileTimeUtc(c.maxTime);
                        var duration = (maxDateTime - minDateTime).TotalSeconds;
                        if (Math.Abs(duration) < 0.01) duration = 1;

                        var stats = new EventStatistics
                        {
                            AverageByteSize = c.Bytes / c.EventCount,
                            ByteSize = c.Bytes,
                            EventCount = c.EventCount,
                            EventsPerSecond = Math.Round(c.EventCount / duration, 3, MidpointRounding.AwayFromZero),
                        };

                        statsPerType[type] = new CsvRelatedStats
                        {
                            ManifestId = c.ManifestId,
                            Statistics = stats,
                        };
                    }
                }
            }

            return statsPerType;
        }
Esempio n. 17
0
        public void TypeCacheParseStructNameIgnoresInvalidTypesTest()
        {
            var testManifest = @"namespace DataModel

            struct EquilateralTriangle{
            1: optional		int64		Side;
            2: optional     string		Source;

            [OccurenceTime("")]
            3: optional     int64		TimeStampUtc;
            };
            struct InvalidTriangle}
            1: optional		int64		Side;
            2: optional     string		Source;

            [OccurenceTime("")]
            3: optional     int64		TimeStampUtc;
            };
            struct ValidTriangle{
            1: optional		int64		Side;
            2: optional     string		Source;

            [OccurenceTime("")]
            3: optional     int64		TimeStampUtc;
            };";
            var typeCache = new TypeCache();
            var tuple = typeCache.ParseClassNames(testManifest);

            Assert.AreEqual("DataModel", tuple.Item1);
            Assert.AreEqual("EquilateralTriangle", tuple.Item2[0]);
            Assert.AreEqual("ValidTriangle", tuple.Item2[1]);
        }
Esempio n. 18
0
        public void TypeCacheParseStructNameHasCurlyBracesHappyPathTest()
        {
            var testManifest = @"namespace DataModel

            struct EquilateralTriangle{
            1: optional		int64		Side;
            2: optional     string		Source;

            [OccurenceTime("")]
            3: optional     int64		TimeStampUtc;
            };
            struct SomeOtherTriangle{
            1: optional		int64		Side;
            2: optional     string		Source;

            [OccurenceTime("")]
            3: optional     int64		TimeStampUtc;
            };";
            var typeCache = new TypeCache();
            var tuple = typeCache.ParseClassNames(testManifest);

            Assert.AreEqual("DataModel", tuple.Item1);
            Assert.AreEqual("EquilateralTriangle", tuple.Item2[0]);
            Assert.AreEqual("SomeOtherTriangle", tuple.Item2[1]);
        }
Esempio n. 19
0
 public void TypeCacheParseClassNamesNullTest()
 {
     var typeCache = new TypeCache();
     typeCache.ParseClassNames(null);
 }
Esempio n. 20
0
 public void TypeCacheParseClassNamesEmptyManifestTest()
 {
     var typeCache = new TypeCache();
     typeCache.ParseClassNames(string.Empty);
 }
        private bool IsCsvLineParsable(string line, TypeCache typeCache, Dictionary<Type, EventStatistics> eventStatsCollection)
        {
            bool isParsable = false;

            var tokens = line.Split(CsvSeparator);

            string eventManifestIdFromCsv = string.Empty;
            string typeNameFromCsv = string.Empty;

            if (tokens != null && tokens.Length == 6 && tokens.All(a => !string.IsNullOrWhiteSpace(a)))
            {
                eventManifestIdFromCsv = tokens[0];
                typeNameFromCsv = tokens[1];

                var typeCacheItem = typeCache.FindMatchOrDefault(eventManifestIdFromCsv);

                if (typeCacheItem != null && typeCacheItem.Type != null && string.Equals(typeCacheItem.Type.Name, typeNameFromCsv, StringComparison.OrdinalIgnoreCase))
                {
                    Debug.WriteLine(typeCacheItem.Type.Name);

                    EventStatistics statsFromCsv = new EventStatistics
                    {
                        AverageByteSize = long.Parse(tokens[5]),
                        ByteSize = long.Parse(tokens[4]),
                        EventCount = long.Parse(tokens[2]),
                        EventsPerSecond = double.Parse(tokens[3]),
                    };

                    EventStatistics existingStats;
                    if (eventStatsCollection.TryGetValue(typeCacheItem.Type, out existingStats))
                    {

                        existingStats = existingStats + statsFromCsv;
                        Console.WriteLine("Stats for type {0} updated.", typeNameFromCsv);
                    }
                    else
                    {
                        eventStatsCollection[typeCacheItem.Type] = statsFromCsv;
                        Console.WriteLine("Stats for type {0} added.", typeNameFromCsv);
                    }

                    isParsable = true;
                }
            }

            return isParsable;
        }
        private bool TryParseStatsFromCsv(string inputFile, TypeCache typeCache, out Dictionary<Type, EventStatistics> eventStatsCollection)
        {
            bool isParsingSuccessful = true;

            bool headersFound = false;
            using (var streamReader = File.OpenText(inputFile))
            {
                eventStatsCollection = new Dictionary<Type, EventStatistics>();

                // At any point parsing is not successful, means current csv is not good enough. Rewrite of csv is required.
                while (!streamReader.EndOfStream && isParsingSuccessful)
                {
                    Console.WriteLine("Starting to parse csv...");

                    string line = string.Empty;

                    // First few lines are not important for stats.
                    while (!headersFound)
                    {
                        line = streamReader.ReadLine();
                        headersFound = !string.IsNullOrWhiteSpace(line) ? line.Contains(this.CsvHeaders) : false;
                    }

                    // headers found
                    if (headersFound)
                    {
                        line = streamReader.ReadLine();

                        if (!string.IsNullOrWhiteSpace(line))
                        {
                            // Every line should pass this test else rewrite of csv is required.
                            isParsingSuccessful = this.IsCsvLineParsable(line, typeCache, eventStatsCollection);
                        }
                    }
                }
            }

            if (eventStatsCollection.Count == 0)
            {
                isParsingSuccessful = false;
            }

            return isParsingSuccessful;
        }
        public IDictionary<Type, EventStatistics> GetTypeStatistics(TypeCache typeCache, params string[] inputFiles)
        {
            Stopwatch sw = Stopwatch.StartNew();

            if (typeCache == null)
            {
                throw new ArgumentNullException("typeCache");
            }

            if (inputFiles == null || inputFiles.Length <= 0 || inputFiles.Any(f => string.IsNullOrWhiteSpace(f)))
            {
                throw new ArgumentNullException("inputFiles");
            }

            if (inputFiles.Any(a => !File.Exists(a)))
            {
                throw new ArgumentException("Some file paths are invalid.");
            }

            var overallStatsPerType = new Dictionary<Type, EventStatistics>();

            Console.WriteLine("Getting Statistics...");

            foreach (var file in inputFiles)
            {
                Dictionary<Type, EventStatistics> statsOfThisFile = null;

                var dir = System.IO.Path.GetDirectoryName(file);
                var fileName = System.IO.Path.GetFileNameWithoutExtension(file) + ".csv";
                var expectedCsv = System.IO.Path.Combine(dir, fileName);

                if (!this.IsStatsCsvAvailable(expectedCsv) ||
                    !this.TryParseStatsFromCsv(expectedCsv, typeCache, out statsOfThisFile))
                {
                    var csvRelatedStatsOfThisFile = this.CalculateTypeStatistics(typeCache, file);

                    if (csvRelatedStatsOfThisFile != null)
                    {
                        statsOfThisFile = new Dictionary<Type, EventStatistics>();
                        foreach (var item in csvRelatedStatsOfThisFile)
                        {
                            statsOfThisFile.Add(item.Key, item.Value.Statistics);
                        }

                        this.CreateCsvStatsFile(expectedCsv, csvRelatedStatsOfThisFile);
                    }
                }

                if (statsOfThisFile != null)
                {
                    foreach (var item in statsOfThisFile)
                    {
                        EventStatistics existingStatsForThisType;
                        if (overallStatsPerType.TryGetValue(item.Key, out existingStatsForThisType))
                        {
                            overallStatsPerType[item.Key] = existingStatsForThisType + item.Value;
                        }
                        else
                        {
                            overallStatsPerType[item.Key] = item.Value;
                        }
                    }
                }
            }

            sw.Stop();
            Console.WriteLine("GetTypeStatistics took {0} milliseconds", sw.ElapsedMilliseconds);

            return overallStatsPerType;
        }