public static MeasuredTrace GetTraceWithRecordsFromSparseTrace(this MeasurementStore store, MeasuredTrace measuredTrace) { return (store.Traces.Include(t => t.ProcessingRecords) .Where(t => t.IsSameDataPackage(measuredTrace)) .SingleOrDefault()); }
public static void HydrateTraceMeasurements(this MeasurementStore store, MeasuredTrace measuredTrace) { // TODO Go back to dynamically discovering types rather than using hard coded list. For now this works better with EF // TODO Move filtering to db engine rather than client side. This is super inneficient, but working around EF foreach (var m in store.Set <CpuSampled>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase))) { measuredTrace.AddMeasurement(m); } foreach (var m in store.Set <BootPhase>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase))) { measuredTrace.AddMeasurement(m); } foreach (var m in store.Set <WinlogonSubscriberTask>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase))) { measuredTrace.AddMeasurement(m); } foreach (var m in store.Set <DiskIo>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase))) { measuredTrace.AddMeasurement(m); } foreach (var m in store.Set <LogicalDisk>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase))) { measuredTrace.AddMeasurement(m); } foreach (var m in store.Set <PhysicalDisk>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase))) { measuredTrace.AddMeasurement(m); } foreach (var m in store.Set <TraceAttribute>().Include(m => m.Trace).Where(m => string.Equals(m.Trace.PackageFileName, measuredTrace.PackageFileName, StringComparison.OrdinalIgnoreCase))) { measuredTrace.AddMeasurement(m); } }
public static int MoveOneBatch(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig) { RichLog.Log.StartMoveFiles(); var movedCount = 0; var filesToMove = new List <string>(processingConfig.ParallelMovesThrottle); var fileMoveTasks = new List <Task>(); using (var store = new MeasurementStore(storeConfig)) { var inScopeTraces = store.Traces.Include(t => t.ProcessingRecords) .Where( t => t.ProcessingRecords.OrderBy(pr => pr.StateChangeTime).Last().ProcessingState == ProcessingState.Discovered) .Take(processingConfig.ParallelMovesThrottle); foreach (var t in inScopeTraces) { filesToMove.Add(t.ProcessingRecords.Latest().Path); } } foreach (var fileSourcePath in filesToMove.Where(p => !string.IsNullOrEmpty(p))) { if (!File.Exists(fileSourcePath)) { continue; } var destinationFullPath = CalculateDestinationPath(fileSourcePath, processingConfig, true); fileMoveTasks.Add( Task.Run(() => { if (File.Exists(destinationFullPath)) { File.Delete(destinationFullPath); } File.Move(fileSourcePath, destinationFullPath); movedCount++; using (var store = new MeasurementStore(storeConfig)) { var trace = store.Traces.Where( t => string.Equals(t.PackageFileNameFull, destinationFullPath, StringComparison.OrdinalIgnoreCase)).Single(); store.ProcessingRecords.Add(new ProcessingRecord { MeasuredTrace = trace, Path = destinationFullPath, StateChangeTime = DateTime.UtcNow, ProcessingState = ProcessingState.Moved }); store.SaveChanges(); } })); } Task.WaitAll(fileMoveTasks.ToArray()); RichLog.Log.StopMoveFiles(movedCount); return(movedCount); }
public static void HydrateTraceMeasurements <TMeasurement>(this MeasurementStore store, MeasuredTrace measuredTrace) where TMeasurement : class, IMeasurement { foreach (var mm in store.Set <TMeasurement>().Include(m => m.Trace).Where(m => m.Trace == measuredTrace)) { measuredTrace.AddMeasurement(mm); } }
public static IEnumerable <MeasuredTrace> GetTraceByFilter( this MeasurementStore store, Func <MeasuredTrace, bool> filter, bool includeMeasurements = false) { foreach (var trace in store.Traces.Include(t => t.ProcessingRecords).Where(t => filter(t))) { if (includeMeasurements) { store.HydrateTraceMeasurements(trace); } yield return(trace); } }
public static MeasuredTrace GetTraceByName( this MeasurementStore store, string packageFileName, bool includeMeasurements = false) { var targetTrace = store.Traces.Include(t => t.ProcessingRecords) .Where(t => string.Equals(t.PackageFileName, packageFileName, StringComparison.OrdinalIgnoreCase)) .SingleOrDefault(); if (includeMeasurements) { store.HydrateTraceMeasurements(targetTrace); } return(targetTrace); }
public static int DiscoverOneBatch(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig) { var totalDiscoveries = 0; using (var store = new MeasurementStore(storeConfig)) { foreach (var incomingDataPath in processingConfig.IncomingDataPaths) { var discoveredCount = 0; RichLog.Log.StartDiscoverFiles(incomingDataPath); foreach (var filter in processingConfig.IncomingFilePatterns) { foreach ( var fileSystemEntryPath in Directory.EnumerateFileSystemEntries(incomingDataPath, filter, SearchOption.AllDirectories)) { var fileInfo = new FileInfo(fileSystemEntryPath); var sparseTrace = new MeasuredTrace { PackageFileName = fileInfo.Name, PackageFileNameFull = CalculateDestinationPath(fileSystemEntryPath, processingConfig, true) }; if (!store.Traces.Any(t => t.IsSameDataPackage(sparseTrace))) { store.Traces.Add(sparseTrace); store.SaveChanges(); } var dbTrace = store.Traces.Include(t => t.ProcessingRecords).First(t => t.IsSameDataPackage(sparseTrace)); var lastPr = dbTrace.ProcessingRecords.Latest(); if (lastPr == null || lastPr.ProcessingState != ProcessingState.Discovered) { store.Set <ProcessingRecord>().Add(new ProcessingRecord { MeasuredTrace = dbTrace, ProcessingState = ProcessingState.Discovered, StateChangeTime = DateTime.UtcNow, Path = fileSystemEntryPath }); store.SaveChanges(); discoveredCount++; totalDiscoveries++; } } } RichLog.Log.StopDiscoverFiles(incomingDataPath, discoveredCount); } } return(totalDiscoveries); }
public static IEnumerable <MeasuredTrace> GetTraceByState( this MeasurementStore store, ProcessingState processingState) { // TODO This is super expensive right now because we are doing most of the filtering on the caller side // after returing lots of excess data from the db. Unfortunately this was the only ready workaround // to an EF bug in query compilation foreach (var trace in store.Traces.Include(t => t.ProcessingRecords)) { if (trace.ProcessingRecords.Latest().ProcessingState == processingState) { yield return(trace); } } }
public static int MeasureOneBatch(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig) { RichLog.Log.StartMeasureAndSaveTraces(); var measuredCount = 0; var measuringTasks = new List <Task>(); var measuringResults = new ConcurrentBag <Trace>(); using (var store = new MeasurementStore(storeConfig)) { var inScopeTraces = store.Traces.Include(t => t.ProcessingRecords) .Where( t => t.ProcessingRecords.OrderBy(pr => pr.StateChangeTime).Last().ProcessingState == ProcessingState.Moved) .Take(processingConfig.ParallelMeasuringThrottle); foreach (var t in inScopeTraces) { RichLog.Log.StartMeasureAndSaveItem(t.PackageFileNameFull); measuringTasks.Add( Task.Run(() => { Trace traceOut = null; try { using (var tj = new TraceJob(t)) { tj.StageForProcessing(); tj.RegisterCalipersAllKnown(); traceOut = tj.Measure(); } } finally { measuredCount++; if (traceOut == null) { traceOut = t; } measuringResults.Add(traceOut); } })); } } try { Task.WaitAll(measuringTasks.ToArray()); } catch (AggregateException ae) { foreach (var e in ae.InnerExceptions) { RichLog.Log.TraceAnalyzeFailureDuringProcessEndToEnd(e.Message, e.ToString()); } } using (var store = new MeasurementStore(storeConfig)) { foreach (var t in measuringResults) { var addedRows = store.SaveTraceAndMeasurements((MeasuredTrace)t); store.ProcessingRecords.Add(new ProcessingRecord { MeasuredTrace = (MeasuredTrace)t, StateChangeTime = DateTime.UtcNow, ProcessingState = ProcessingState.Measured, Path = t.PackageFileNameFull }); store.SaveChanges(); RichLog.Log.StopMeasureAndSaveItem(t.PackageFileNameFull, addedRows); } } RichLog.Log.StopMeasureAndSaveTraces(measuredCount); return(measuredCount); }
public static DbSet <TEntity> NewDynamicSet <TEntity>(this MeasurementStore store, TEntity entity) where TEntity : class, IMeasurement { return(store.Set <TEntity>()); }