public static MeasuredTrace GetTraceWithRecordsFromSparseTrace(this MeasurementStore store, MeasuredTrace measuredTrace)
{
return
(store.Traces.Include(t => t.ProcessingRecords)
.Where(t => t.IsSameDataPackage(measuredTrace))
.SingleOrDefault());
}
public static void HydrateTraceMeasurements(this MeasurementStore store, MeasuredTrace measuredTrace)
{
// TODO Go back to dynamically discovering types rather than using hard coded list. For now this works better with EF
// TODO Move filtering to db engine rather than client side. This is super inneficient, but working around EF
foreach (var m in store.Set <CpuSampled>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase)))
{
measuredTrace.AddMeasurement(m);
}
foreach (var m in store.Set <BootPhase>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase)))
{
measuredTrace.AddMeasurement(m);
}
foreach (var m in store.Set <WinlogonSubscriberTask>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase)))
{
measuredTrace.AddMeasurement(m);
}
foreach (var m in store.Set <DiskIo>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase)))
{
measuredTrace.AddMeasurement(m);
}
foreach (var m in store.Set <LogicalDisk>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase)))
{
measuredTrace.AddMeasurement(m);
}
foreach (var m in store.Set <PhysicalDisk>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase)))
{
measuredTrace.AddMeasurement(m);
}
foreach (var m in store.Set <TraceAttribute>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase)))
{
measuredTrace.AddMeasurement(m);
}
}
public static int MoveOneBatch(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig)
{
RichLog.Log.StartMoveFiles();
var movedCount = 0;
var filesToMove = new List <string>(processingConfig.ParallelMovesThrottle);
var fileMoveTasks = new List <Task>();
using (var store = new MeasurementStore(storeConfig))
{
var inScopeTraces = store.Traces.Include(t => t.ProcessingRecords)
.Where(
t =>
t.ProcessingRecords.OrderBy(pr => pr.StateChangeTime).Last().ProcessingState ==
ProcessingState.Discovered)
.Take(processingConfig.ParallelMovesThrottle);
foreach (var t in inScopeTraces)
{
filesToMove.Add(t.ProcessingRecords.Latest().Path);
}
}
foreach (var fileSourcePath in filesToMove.Where(p => !string.IsNullOrEmpty(p)))
{
if (!File.Exists(fileSourcePath))
{
continue;
}
var destinationFullPath = CalculateDestinationPath(fileSourcePath, processingConfig, true);
fileMoveTasks.Add(
Task.Run(() =>
{
if (File.Exists(destinationFullPath))
{
File.Delete(destinationFullPath);
}
File.Move(fileSourcePath, destinationFullPath);
movedCount++;
using (var store = new MeasurementStore(storeConfig))
{
var trace =
store.Traces.Where(
t =>
string.Equals(t.PackageFileNameFull, destinationFullPath,
StringComparison.OrdinalIgnoreCase)).Single();
store.ProcessingRecords.Add(new ProcessingRecord
{
MeasuredTrace = trace,
Path = destinationFullPath,
StateChangeTime = DateTime.UtcNow,
ProcessingState = ProcessingState.Moved
});
store.SaveChanges();
}
}));
}
Task.WaitAll(fileMoveTasks.ToArray());
RichLog.Log.StopMoveFiles(movedCount);
return(movedCount);
}
public static void HydrateTraceMeasurements <TMeasurement>(this MeasurementStore store,
MeasuredTrace measuredTrace) where TMeasurement : class, IMeasurement
{
foreach (var mm in store.Set <TMeasurement>().Include(m => m.Trace).Where(m => m.Trace == measuredTrace))
{
measuredTrace.AddMeasurement(mm);
}
}
public static IEnumerable <MeasuredTrace> GetTraceByFilter(
this MeasurementStore store, Func <MeasuredTrace, bool> filter, bool includeMeasurements = false)
{
foreach (var trace in store.Traces.Include(t => t.ProcessingRecords).Where(t => filter(t)))
{
if (includeMeasurements)
{
store.HydrateTraceMeasurements(trace);
}
yield return(trace);
}
}
public static MeasuredTrace GetTraceByName(
this MeasurementStore store, string packageFileName, bool includeMeasurements = false)
{
var targetTrace = store.Traces.Include(t => t.ProcessingRecords)
.Where(t => string.Equals(t.PackageFileName, packageFileName, StringComparison.OrdinalIgnoreCase))
.SingleOrDefault();
if (includeMeasurements)
{
store.HydrateTraceMeasurements(targetTrace);
}
return(targetTrace);
}
public static int DiscoverOneBatch(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig)
{
var totalDiscoveries = 0;
using (var store = new MeasurementStore(storeConfig))
{
foreach (var incomingDataPath in processingConfig.IncomingDataPaths)
{
var discoveredCount = 0;
RichLog.Log.StartDiscoverFiles(incomingDataPath);
foreach (var filter in processingConfig.IncomingFilePatterns)
{
foreach (
var fileSystemEntryPath in
Directory.EnumerateFileSystemEntries(incomingDataPath, filter,
SearchOption.AllDirectories))
{
var fileInfo = new FileInfo(fileSystemEntryPath);
var sparseTrace = new MeasuredTrace
{
PackageFileName = fileInfo.Name,
PackageFileNameFull =
CalculateDestinationPath(fileSystemEntryPath, processingConfig, true)
};
if (!store.Traces.Any(t => t.IsSameDataPackage(sparseTrace)))
{
store.Traces.Add(sparseTrace);
store.SaveChanges();
}
var dbTrace = store.Traces.Include(t => t.ProcessingRecords).First(t => t.IsSameDataPackage(sparseTrace));
var lastPr = dbTrace.ProcessingRecords.Latest();
if (lastPr == null || lastPr.ProcessingState != ProcessingState.Discovered)
{
store.Set <ProcessingRecord>().Add(new ProcessingRecord
{
MeasuredTrace = dbTrace,
ProcessingState = ProcessingState.Discovered,
StateChangeTime = DateTime.UtcNow,
Path = fileSystemEntryPath
});
store.SaveChanges();
discoveredCount++;
totalDiscoveries++;
}
}
}
RichLog.Log.StopDiscoverFiles(incomingDataPath, discoveredCount);
}
}
return(totalDiscoveries);
}
public static IEnumerable <MeasuredTrace> GetTraceByState(
this MeasurementStore store, ProcessingState processingState)
{
// TODO This is super expensive right now because we are doing most of the filtering on the caller side
// after returing lots of excess data from the db. Unfortunately this was the only ready workaround
// to an EF bug in query compilation
foreach (var trace in store.Traces.Include(t => t.ProcessingRecords))
{
if (trace.ProcessingRecords.Latest().ProcessingState == processingState)
{
yield return(trace);
}
}
}
public static int MeasureOneBatch(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig)
{
RichLog.Log.StartMeasureAndSaveTraces();
var measuredCount = 0;
var measuringTasks = new List <Task>();
var measuringResults = new ConcurrentBag <Trace>();
using (var store = new MeasurementStore(storeConfig))
{
var inScopeTraces = store.Traces.Include(t => t.ProcessingRecords)
.Where(
t =>
t.ProcessingRecords.OrderBy(pr => pr.StateChangeTime).Last().ProcessingState ==
ProcessingState.Moved)
.Take(processingConfig.ParallelMeasuringThrottle);
foreach (var t in inScopeTraces)
{
RichLog.Log.StartMeasureAndSaveItem(t.PackageFileNameFull);
measuringTasks.Add(
Task.Run(() =>
{
Trace traceOut = null;
try
{
using (var tj = new TraceJob(t))
{
tj.StageForProcessing();
tj.RegisterCalipersAllKnown();
traceOut = tj.Measure();
}
}
finally
{
measuredCount++;
if (traceOut == null)
{
traceOut = t;
}
measuringResults.Add(traceOut);
}
}));
}
}
try
{
Task.WaitAll(measuringTasks.ToArray());
}
catch (AggregateException ae)
{
foreach (var e in ae.InnerExceptions)
{
RichLog.Log.TraceAnalyzeFailureDuringProcessEndToEnd(e.Message, e.ToString());
}
}
using (var store = new MeasurementStore(storeConfig))
{
foreach (var t in measuringResults)
{
var addedRows = store.SaveTraceAndMeasurements((MeasuredTrace)t);
store.ProcessingRecords.Add(new ProcessingRecord
{
MeasuredTrace = (MeasuredTrace)t,
StateChangeTime = DateTime.UtcNow,
ProcessingState = ProcessingState.Measured,
Path = t.PackageFileNameFull
});
store.SaveChanges();
RichLog.Log.StopMeasureAndSaveItem(t.PackageFileNameFull, addedRows);
}
}
RichLog.Log.StopMeasureAndSaveTraces(measuredCount);
return(measuredCount);
}
public static DbSet <TEntity> NewDynamicSet <TEntity>(this MeasurementStore store, TEntity entity)
where TEntity : class, IMeasurement
{
return(store.Set <TEntity>());
}