public static int MoveOneBatch(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig)
{
RichLog.Log.StartMoveFiles();
var movedCount = 0;
var filesToMove = new List <string>(processingConfig.ParallelMovesThrottle);
var fileMoveTasks = new List <Task>();
using (var store = new MeasurementStore(storeConfig))
{
var inScopeTraces = store.Traces.Include(t => t.ProcessingRecords)
.Where(
t =>
t.ProcessingRecords.OrderBy(pr => pr.StateChangeTime).Last().ProcessingState ==
ProcessingState.Discovered)
.Take(processingConfig.ParallelMovesThrottle);
foreach (var t in inScopeTraces)
{
filesToMove.Add(t.ProcessingRecords.Latest().Path);
}
}
foreach (var fileSourcePath in filesToMove.Where(p => !string.IsNullOrEmpty(p)))
{
if (!File.Exists(fileSourcePath))
{
continue;
}
var destinationFullPath = CalculateDestinationPath(fileSourcePath, processingConfig, true);
fileMoveTasks.Add(
Task.Run(() =>
{
if (File.Exists(destinationFullPath))
{
File.Delete(destinationFullPath);
}
File.Move(fileSourcePath, destinationFullPath);
movedCount++;
using (var store = new MeasurementStore(storeConfig))
{
var trace =
store.Traces.Where(
t =>
string.Equals(t.PackageFileNameFull, destinationFullPath,
StringComparison.OrdinalIgnoreCase)).Single();
store.ProcessingRecords.Add(new ProcessingRecord
{
MeasuredTrace = trace,
Path = destinationFullPath,
StateChangeTime = DateTime.UtcNow,
ProcessingState = ProcessingState.Moved
});
store.SaveChanges();
}
}));
}
Task.WaitAll(fileMoveTasks.ToArray());
RichLog.Log.StopMoveFiles(movedCount);
return(movedCount);
}
public static void InvokeProcessingOnce(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig)
{
RichLog.Log.StartProcessEndToEnd(0);
ForwardMeasureTraceLogging.AddListenerAsNeeded();
AutomationTasks.DiscoverOneBatch(processingConfig, storeConfig);
var moved = AutomationTasks.MoveOneBatch(processingConfig, storeConfig);
var measured = AutomationTasks.MeasureOneBatch(processingConfig, storeConfig);
RichLog.Log.StopProcessEndToEnd(moved, measured);
}
public static int DiscoverOneBatch(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig)
{
var totalDiscoveries = 0;
using (var store = new MeasurementStore(storeConfig))
{
foreach (var incomingDataPath in processingConfig.IncomingDataPaths)
{
var discoveredCount = 0;
RichLog.Log.StartDiscoverFiles(incomingDataPath);
foreach (var filter in processingConfig.IncomingFilePatterns)
{
foreach (
var fileSystemEntryPath in
Directory.EnumerateFileSystemEntries(incomingDataPath, filter,
SearchOption.AllDirectories))
{
var fileInfo = new FileInfo(fileSystemEntryPath);
var sparseTrace = new MeasuredTrace
{
PackageFileName = fileInfo.Name,
PackageFileNameFull =
CalculateDestinationPath(fileSystemEntryPath, processingConfig, true)
};
if (!store.Traces.Any(t => t.IsSameDataPackage(sparseTrace)))
{
store.Traces.Add(sparseTrace);
store.SaveChanges();
}
var dbTrace = store.Traces.Include(t => t.ProcessingRecords).First(t => t.IsSameDataPackage(sparseTrace));
var lastPr = dbTrace.ProcessingRecords.Latest();
if (lastPr == null || lastPr.ProcessingState != ProcessingState.Discovered)
{
store.Set <ProcessingRecord>().Add(new ProcessingRecord
{
MeasuredTrace = dbTrace,
ProcessingState = ProcessingState.Discovered,
StateChangeTime = DateTime.UtcNow,
Path = fileSystemEntryPath
});
store.SaveChanges();
discoveredCount++;
totalDiscoveries++;
}
}
}
RichLog.Log.StopDiscoverFiles(incomingDataPath, discoveredCount);
}
}
return(totalDiscoveries);
}
public static string CalculateDestinationPath(string fileSourcePath, ProcessingConfig processingConfig,
bool prepareSubDirsAsNeeded)
{
var relativeName = new FileInfo(fileSourcePath).Name;
var destinationRootDir = processingConfig.DestinationDataPath;
var tempTrace = new Trace();
var packageType = TraceJobExtension.ResolvePackageType(fileSourcePath);
var adapter = TraceJobExtension.GetPackageAdapter(packageType);
adapter.PopulateTraceAttributesFromFileName(tempTrace, fileSourcePath);
var fileDate = tempTrace.TracePackageTime;
var subDirToken = fileDate.ToString(processingConfig.DestinationSubDirPattern);
var destinationFolderPath = Path.Combine(destinationRootDir, subDirToken);
if (prepareSubDirsAsNeeded)
{
Directory.CreateDirectory(destinationFolderPath);
}
return(Path.Combine(destinationFolderPath, relativeName));
}
public static int MeasureOneBatch(ProcessingConfig processingConfig, MeasurementStoreConfig storeConfig)
{
RichLog.Log.StartMeasureAndSaveTraces();
var measuredCount = 0;
var measuringTasks = new List <Task>();
var measuringResults = new ConcurrentBag <Trace>();
using (var store = new MeasurementStore(storeConfig))
{
var inScopeTraces = store.Traces.Include(t => t.ProcessingRecords)
.Where(
t =>
t.ProcessingRecords.OrderBy(pr => pr.StateChangeTime).Last().ProcessingState ==
ProcessingState.Moved)
.Take(processingConfig.ParallelMeasuringThrottle);
foreach (var t in inScopeTraces)
{
RichLog.Log.StartMeasureAndSaveItem(t.PackageFileNameFull);
measuringTasks.Add(
Task.Run(() =>
{
Trace traceOut = null;
try
{
using (var tj = new TraceJob(t))
{
tj.StageForProcessing();
tj.RegisterCalipersAllKnown();
traceOut = tj.Measure();
}
}
finally
{
measuredCount++;
if (traceOut == null)
{
traceOut = t;
}
measuringResults.Add(traceOut);
}
}));
}
}
try
{
Task.WaitAll(measuringTasks.ToArray());
}
catch (AggregateException ae)
{
foreach (var e in ae.InnerExceptions)
{
RichLog.Log.TraceAnalyzeFailureDuringProcessEndToEnd(e.Message, e.ToString());
}
}
using (var store = new MeasurementStore(storeConfig))
{
foreach (var t in measuringResults)
{
var addedRows = store.SaveTraceAndMeasurements((MeasuredTrace)t);
store.ProcessingRecords.Add(new ProcessingRecord
{
MeasuredTrace = (MeasuredTrace)t,
StateChangeTime = DateTime.UtcNow,
ProcessingState = ProcessingState.Measured,
Path = t.PackageFileNameFull
});
store.SaveChanges();
RichLog.Log.StopMeasureAndSaveItem(t.PackageFileNameFull, addedRows);
}
}
RichLog.Log.StopMeasureAndSaveTraces(measuredCount);
return(measuredCount);
}