public void EventStatisticCacheGetTypeStatisticsIncorrectFilePathsTest()
{
TypeCache typecache = new TypeCache();
EventStatisticCache cache = new EventStatisticCache();
cache.GetTypeStatistics(typecache, new[] { @"C:\hello\world.etl", testDataFile });
}
public void EventStatisticCacheGetTypeStatisticsFilesInvalidTest()
{
TypeCache typecache = new TypeCache();
EventStatisticCache cache = new EventStatisticCache();
cache.GetTypeStatistics(typecache, new[] { string.Empty, testDataFile });
}
public void TypeCacheIgnoresTypesContainingImportTest()
{
var typeCache = new TypeCache();
typeCache.Initialize("TypeCacheIgnoresTypesContainingImportTest", @"TestData\TrianglesWithImport.etl");
Assert.AreEqual(3, typeCache.Cache.Count);
}
public void TypeCacheIgnoresTypesContainingImportTest()
{
var typeCache = new TypeCache();
typeCache.Init("test5", new[] { @"TestData\TrianglesWithImport.etl" });
Assert.AreEqual(4, typeCache.Types.Length);
Assert.AreEqual(3, typeCache.Manifests.Length);
}
public void TypeCacheInitHappyPathTest()
{
var typeCache = new TypeCache();
typeCache.Initialize("test", new[] { testDataFile });
var dir = typeCache.CacheDirectory;
Assert.AreEqual(3, Directory.GetFiles(dir, "*.bond").Length);
Assert.AreEqual(3, typeCache.Cache.Count);
Utilities.ForceDeleteDirectory("test");
}
public void TypeCacheInitHappyPathTest()
{
var typeCache = new TypeCache();
typeCache.Init("test", new[] { testDataFile });
var dir = typeCache.CacheDirectory;
Assert.AreEqual(3, Directory.GetFiles(dir, "*.bond").Length);
Assert.AreEqual(3, typeCache.Manifests.Length);
Directory.Delete(dir, true);
}
public void EventStatisticCacheCreatesCsvTest()
{
TypeCache typecache = new TypeCache();
typecache.Initialize("test2", new[] { testDataFile });
EventStatisticCache cache = new EventStatisticCache();
var result = cache.GetTypeStatistics(typecache, new[] { testDataFile });
Assert.IsTrue(File.Exists(Path.Combine(Environment.CurrentDirectory, "TestData", Path.GetFileNameWithoutExtension(testDataFile) + ".csv")));
File.Delete(Path.Combine(Environment.CurrentDirectory, Path.GetFileNameWithoutExtension(testDataFile) + ".csv"));
}
public void EventStatisticCacheGetTypeStatisticsHappyPathTest()
{
TypeCache typecache = new TypeCache();
typecache.Initialize("test1", new[] { testDataFile });
EventStatisticCache cache = new EventStatisticCache();
var result = cache.GetTypeStatistics(typecache, new[] { testDataFile });
Assert.AreEqual(3, result.Keys.Count);
Assert.IsTrue(result.All(a => a.Value.EventCount == 10));
var typeNames = new[] { "IsoscelesTriangle", "EquilateralTriangle", "RightAngledTriangle" };
Assert.IsTrue(result.All(a => typeNames.Contains(a.Key.Name)));
}
public void EventStatisticLoadCsvValuesTest()
{
TypeCache typecache = new TypeCache();
typecache.Initialize("test3", new[] { testDataFile });
EventStatisticCache cache = new EventStatisticCache();
var result = cache.GetTypeStatistics(typecache, new[] { testDataFile });
Assert.IsTrue(File.Exists(Path.Combine(Environment.CurrentDirectory, "TestData", Path.GetFileNameWithoutExtension(testDataFile) + ".csv")));
File.Delete(Path.Combine(Environment.CurrentDirectory, Path.GetFileNameWithoutExtension(testDataFile) + ".csv"));
foreach (var item in result)
{
switch (item.Key.Name)
{
case "IsoscelesTriangle":
Assert.AreEqual(27D, item.Value.AverageByteSize);
Assert.AreEqual(270, item.Value.ByteSize);
Assert.AreEqual(10, item.Value.EventCount);
Assert.AreEqual(0.0370, item.Value.EventsPerSecond);
break;
case "EquilateralTriangle":
Assert.AreEqual(25D, item.Value.AverageByteSize);
Assert.AreEqual(250, item.Value.ByteSize);
Assert.AreEqual(10, item.Value.EventCount);
Assert.AreEqual(0.0370, item.Value.EventsPerSecond);
break;
case "RightAngledTriangle":
Assert.AreEqual(29D, item.Value.AverageByteSize);
Assert.AreEqual(290, item.Value.ByteSize);
Assert.AreEqual(10, item.Value.EventCount);
Assert.AreEqual(0.037, item.Value.EventsPerSecond);
break;
default:
Assert.Fail("Unexpected test result. Test fails.");
break;
}
}
}
public void MultipleTypesStatsCalculationTest()
{
TypeCache typecache = new TypeCache();
typecache.Initialize("MultipleTypesStatsCalculationTest", new[] { testDataFile });
Assert.AreEqual(4, typecache.Cache.Count);
Assert.IsTrue(typecache.Cache.All(a => a != null && a.Manifest != null && a.Type != null));
EventStatisticCache cache = new EventStatisticCache();
var result = cache.GetTypeStatistics(typecache, new[] { testDataFile });
Assert.AreEqual(3, result.Count);
var values = result.Values.ToArray();
Assert.AreEqual(10, values[0].EventCount);
Assert.AreEqual(10, values[1].EventCount);
Assert.AreEqual(10, values[2].EventCount);
File.Delete(Path.Combine("TestData", Path.GetFileNameWithoutExtension(testDataFile) + ".csv"));
}
public void ReadTypesFromSingleManifestTest()
{
TypeCache typecache = new TypeCache();
typecache.Initialize("ReadTypesFromSingleManifestTest", new[] { testDataFile });
Debug.WriteLine(typecache.CacheDirectory);
Assert.AreEqual(4, typecache.Cache.Count);
Debug.WriteLine(typecache.Cache[0].Manifest.ManifestId);
Debug.WriteLine(typecache.Cache[1].Manifest.ManifestId);
Debug.WriteLine(typecache.Cache[2].Manifest.ManifestId);
Debug.WriteLine(typecache.Cache[3].Manifest.ManifestId);
Debug.WriteLine(typecache.Cache[0].Type.Name);
Debug.WriteLine(typecache.Cache[1].Type.Name);
Debug.WriteLine(typecache.Cache[2].Type.Name);
Debug.WriteLine(typecache.Cache[3].Type.Name);
Assert.IsTrue(typecache.Cache.All(a => a != null && a.Manifest != null && a.Type != null));
}
public void TypeCacheInitTargetDirNullTest()
{
var typeCache = new TypeCache();
typeCache.Init(null, new string[] { testDataFile });
}
public void TypeCacheInitNullFilesTest()
{
var typeCache = new TypeCache();
typeCache.Init("dir", null);
}
public void TypeCacheInitNoFilesTest()
{
var typeCache = new TypeCache();
typeCache.Init("dir", new string[] { });
typeCache.Init("dir", null);
}
private Dictionary<Type, CsvRelatedStats> CalculateTypeStatistics(TypeCache typeCache, string inputFile)
{
Console.WriteLine("Getting statistics from file {0}.", inputFile);
var statsPerType = new Dictionary<Type, CsvRelatedStats>();
Stopwatch sw = Stopwatch.StartNew();
var rawCount = from events in BinaryEtwObservable.FromSequentialFiles(inputFile)
group events by events.PayloadId
into eventTypes
from all in
eventTypes.Aggregate(
new { EventCount = (long)0, Bytes = (long)0, minTime = long.MaxValue, maxTime = 0L },
(ac, events) =>
new
{
EventCount = ac.EventCount + 1,
Bytes = ac.Bytes + events.EventPayloadLength,
minTime = Math.Min(ac.minTime, events.ReceiveFileTimeUtc),
maxTime = Math.Max(ac.maxTime, events.ReceiveFileTimeUtc)
})
select new { ManifestId = eventTypes.Key, all.EventCount, all.Bytes, all.minTime, all.maxTime };
var counts = rawCount.ToEnumerable().ToArray();
sw.Stop();
Console.WriteLine("Query took {0} milliseconds.", sw.ElapsedMilliseconds);
foreach (var c in counts)
{
var manifest = typeCache.Manifests
.FirstOrDefault(m => string.Equals(m.ManifestId, c.ManifestId, StringComparison.OrdinalIgnoreCase));
if (manifest != null)
{
var line = manifest.Manifest
.Split('\n').LastOrDefault(l => l.Trim().StartsWith(@"struct ", StringComparison.OrdinalIgnoreCase));
if (line != null)
{
var className = line.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)[1].Trim();
var type = typeCache.Types.FirstOrDefault(t => t.Name == className);
if (type != null)
{
var minDateTime = DateTime.FromFileTimeUtc(c.minTime);
var maxDateTime = DateTime.FromFileTimeUtc(c.maxTime);
var duration = (maxDateTime - minDateTime).TotalSeconds;
if (Math.Abs(duration) < 0.01) duration = 1;
var stats = new EventStatistics
{
AverageByteSize = c.Bytes / c.EventCount,
ByteSize = c.Bytes,
EventCount = c.EventCount,
EventsPerSecond = c.EventCount / duration
};
statsPerType[type] = new CsvRelatedStats
{
ManifestId = c.ManifestId,
Statistics = stats,
};
}
}
}
}
return statsPerType;
}
private Dictionary<Type, CsvRelatedStats> CalculateTypeStatistics(TypeCache typeCache, string inputFile)
{
Console.WriteLine("Getting statistics from file {0}.", inputFile);
var statsPerType = new Dictionary<Type, CsvRelatedStats>();
Stopwatch sw = Stopwatch.StartNew();
var rawCount = from events in BinaryEtwObservable.FromSequentialFiles(inputFile)
group events by events.TypeId
into eventTypes
from all in
eventTypes.Aggregate(
new { EventCount = (long)0, Bytes = (long)0, minTime = long.MaxValue, maxTime = 0L },
(ac, events) =>
new
{
EventCount = ac.EventCount + 1,
Bytes = ac.Bytes + events.Payload.Length,
minTime = Math.Min(ac.minTime, events.ReceivedTime.ToFileTime()),
maxTime = Math.Max(ac.maxTime, events.ReceivedTime.ToFileTime()),
})
select new { ManifestId = eventTypes.Key, all.EventCount, all.Bytes, all.minTime, all.maxTime, };
var counts = rawCount.ToEnumerable().ToArray();
sw.Stop();
Console.WriteLine("Query took {0} milliseconds.", sw.ElapsedMilliseconds);
foreach (var c in counts)
{
var typeCacheItem = typeCache.FindMatchOrDefault(c.ManifestId);
if (typeCacheItem != null && typeCacheItem.Type != null)
{
var type = typeCacheItem.Type;
if (type != null)
{
var minDateTime = DateTime.FromFileTimeUtc(c.minTime);
var maxDateTime = DateTime.FromFileTimeUtc(c.maxTime);
var duration = (maxDateTime - minDateTime).TotalSeconds;
if (Math.Abs(duration) < 0.01) duration = 1;
var stats = new EventStatistics
{
AverageByteSize = c.Bytes / c.EventCount,
ByteSize = c.Bytes,
EventCount = c.EventCount,
EventsPerSecond = Math.Round(c.EventCount / duration, 3, MidpointRounding.AwayFromZero),
};
statsPerType[type] = new CsvRelatedStats
{
ManifestId = c.ManifestId,
Statistics = stats,
};
}
}
}
return statsPerType;
}
public void TypeCacheParseStructNameIgnoresInvalidTypesTest()
{
var testManifest = @"namespace DataModel
struct EquilateralTriangle{
1: optional int64 Side;
2: optional string Source;
[OccurenceTime("")]
3: optional int64 TimeStampUtc;
};
struct InvalidTriangle}
1: optional int64 Side;
2: optional string Source;
[OccurenceTime("")]
3: optional int64 TimeStampUtc;
};
struct ValidTriangle{
1: optional int64 Side;
2: optional string Source;
[OccurenceTime("")]
3: optional int64 TimeStampUtc;
};";
var typeCache = new TypeCache();
var tuple = typeCache.ParseClassNames(testManifest);
Assert.AreEqual("DataModel", tuple.Item1);
Assert.AreEqual("EquilateralTriangle", tuple.Item2[0]);
Assert.AreEqual("ValidTriangle", tuple.Item2[1]);
}
public void TypeCacheParseStructNameHasCurlyBracesHappyPathTest()
{
var testManifest = @"namespace DataModel
struct EquilateralTriangle{
1: optional int64 Side;
2: optional string Source;
[OccurenceTime("")]
3: optional int64 TimeStampUtc;
};
struct SomeOtherTriangle{
1: optional int64 Side;
2: optional string Source;
[OccurenceTime("")]
3: optional int64 TimeStampUtc;
};";
var typeCache = new TypeCache();
var tuple = typeCache.ParseClassNames(testManifest);
Assert.AreEqual("DataModel", tuple.Item1);
Assert.AreEqual("EquilateralTriangle", tuple.Item2[0]);
Assert.AreEqual("SomeOtherTriangle", tuple.Item2[1]);
}
public void TypeCacheParseClassNamesNullTest()
{
var typeCache = new TypeCache();
typeCache.ParseClassNames(null);
}
public void TypeCacheParseClassNamesEmptyManifestTest()
{
var typeCache = new TypeCache();
typeCache.ParseClassNames(string.Empty);
}
private bool IsCsvLineParsable(string line, TypeCache typeCache, Dictionary<Type, EventStatistics> eventStatsCollection)
{
bool isParsable = false;
var tokens = line.Split(CsvSeparator);
string eventManifestIdFromCsv = string.Empty;
string typeNameFromCsv = string.Empty;
if (tokens != null && tokens.Length == 6 && tokens.All(a => !string.IsNullOrWhiteSpace(a)))
{
eventManifestIdFromCsv = tokens[0];
typeNameFromCsv = tokens[1];
var typeCacheItem = typeCache.FindMatchOrDefault(eventManifestIdFromCsv);
if (typeCacheItem != null && typeCacheItem.Type != null && string.Equals(typeCacheItem.Type.Name, typeNameFromCsv, StringComparison.OrdinalIgnoreCase))
{
Debug.WriteLine(typeCacheItem.Type.Name);
EventStatistics statsFromCsv = new EventStatistics
{
AverageByteSize = long.Parse(tokens[5]),
ByteSize = long.Parse(tokens[4]),
EventCount = long.Parse(tokens[2]),
EventsPerSecond = double.Parse(tokens[3]),
};
EventStatistics existingStats;
if (eventStatsCollection.TryGetValue(typeCacheItem.Type, out existingStats))
{
existingStats = existingStats + statsFromCsv;
Console.WriteLine("Stats for type {0} updated.", typeNameFromCsv);
}
else
{
eventStatsCollection[typeCacheItem.Type] = statsFromCsv;
Console.WriteLine("Stats for type {0} added.", typeNameFromCsv);
}
isParsable = true;
}
}
return isParsable;
}
private bool TryParseStatsFromCsv(string inputFile, TypeCache typeCache, out Dictionary<Type, EventStatistics> eventStatsCollection)
{
bool isParsingSuccessful = true;
bool headersFound = false;
using (var streamReader = File.OpenText(inputFile))
{
eventStatsCollection = new Dictionary<Type, EventStatistics>();
// At any point parsing is not successful, means current csv is not good enough. Rewrite of csv is required.
while (!streamReader.EndOfStream && isParsingSuccessful)
{
Console.WriteLine("Starting to parse csv...");
string line = string.Empty;
// First few lines are not important for stats.
while (!headersFound)
{
line = streamReader.ReadLine();
headersFound = !string.IsNullOrWhiteSpace(line) ? line.Contains(this.CsvHeaders) : false;
}
// headers found
if (headersFound)
{
line = streamReader.ReadLine();
if (!string.IsNullOrWhiteSpace(line))
{
// Every line should pass this test else rewrite of csv is required.
isParsingSuccessful = this.IsCsvLineParsable(line, typeCache, eventStatsCollection);
}
}
}
}
if (eventStatsCollection.Count == 0)
{
isParsingSuccessful = false;
}
return isParsingSuccessful;
}
public IDictionary<Type, EventStatistics> GetTypeStatistics(TypeCache typeCache, params string[] inputFiles)
{
Stopwatch sw = Stopwatch.StartNew();
if (typeCache == null)
{
throw new ArgumentNullException("typeCache");
}
if (inputFiles == null || inputFiles.Length <= 0 || inputFiles.Any(f => string.IsNullOrWhiteSpace(f)))
{
throw new ArgumentNullException("inputFiles");
}
if (inputFiles.Any(a => !File.Exists(a)))
{
throw new ArgumentException("Some file paths are invalid.");
}
var overallStatsPerType = new Dictionary<Type, EventStatistics>();
Console.WriteLine("Getting Statistics...");
foreach (var file in inputFiles)
{
Dictionary<Type, EventStatistics> statsOfThisFile = null;
var dir = System.IO.Path.GetDirectoryName(file);
var fileName = System.IO.Path.GetFileNameWithoutExtension(file) + ".csv";
var expectedCsv = System.IO.Path.Combine(dir, fileName);
if (!this.IsStatsCsvAvailable(expectedCsv) ||
!this.TryParseStatsFromCsv(expectedCsv, typeCache, out statsOfThisFile))
{
var csvRelatedStatsOfThisFile = this.CalculateTypeStatistics(typeCache, file);
if (csvRelatedStatsOfThisFile != null)
{
statsOfThisFile = new Dictionary<Type, EventStatistics>();
foreach (var item in csvRelatedStatsOfThisFile)
{
statsOfThisFile.Add(item.Key, item.Value.Statistics);
}
this.CreateCsvStatsFile(expectedCsv, csvRelatedStatsOfThisFile);
}
}
if (statsOfThisFile != null)
{
foreach (var item in statsOfThisFile)
{
EventStatistics existingStatsForThisType;
if (overallStatsPerType.TryGetValue(item.Key, out existingStatsForThisType))
{
overallStatsPerType[item.Key] = existingStatsForThisType + item.Value;
}
else
{
overallStatsPerType[item.Key] = item.Value;
}
}
}
}
sw.Stop();
Console.WriteLine("GetTypeStatistics took {0} milliseconds", sw.ElapsedMilliseconds);
return overallStatsPerType;
}
