public void PrintOptions(ILogger logger, int prefixWidth, int outputWidth)
{
foreach (var option in configuration)
{
var optionText = $" {optionPrefix}{option.Key} <{option.Key.ToUpperInvariant()}>";
logger.WriteResult($"{optionText.PadRight(prefixWidth)}");
var parameters = string.Join(", ", option.Value.GetAllOptions.Value);
var explanation = $"Allowed values: ";
logger.WriteInfo($": {explanation}");
var maxWidth = outputWidth - prefixWidth - explanation.Length - Environment.NewLine.Length - breakText.Length;
var lines = StringAndTextExtensions.Wrap(parameters, maxWidth);
if (lines.Count == 0)
{
logger.WriteLine();
continue;
}
logger.WriteLineInfo($"{lines.First().Trim(trimChars)}");
var padding = new string(' ', prefixWidth);
var explanationPadding = new string(' ', explanation.Length);
foreach (var line in lines.Skip(1))
{
logger.WriteLineInfo($"{padding}{breakText}{explanationPadding}{line.Trim(trimChars)}");
}
}
}
/// <summary>
/// generates project.lock.json that tells compiler where to take dlls and source from
/// and builds executable and copies all required dll's
/// </summary>
public BuildResult Build(GenerateResult generateResult, ILogger logger, Benchmark benchmark, IResolver resolver)
{
var restoreResult = DotNetCliCommandExecutor.ExecuteCommand(
RestoreCommand,
generateResult.ArtifactsPaths.BuildArtifactsDirectoryPath);
logger.WriteLineInfo($"// dotnet restore took {restoreResult.ExecutionTime.TotalSeconds:0.##}s");
if (!restoreResult.IsSuccess)
{
return(BuildResult.Failure(generateResult, new Exception(restoreResult.ProblemDescription)));
}
var buildResult = Build(generateResult);
logger.WriteLineInfo($"// dotnet build took {buildResult.ExecutionTime.TotalSeconds:0.##}s");
if (!buildResult.IsSuccess)
{
// dotnet cli could have succesfully builded the program, but returned 1 as exit code because it had some warnings
// so we need to check whether the exe exists or not, if it does then it is OK
if (File.Exists(generateResult.ArtifactsPaths.ExecutablePath))
{
return(BuildResult.Success(generateResult));
}
return(BuildResult.Failure(generateResult, new Exception(buildResult.ProblemDescription)));
}
return(BuildResult.Success(generateResult));
}
public override void ExportToLog(Summary summary, ILogger logger)
{
if (useCodeBlocks)
{
logger.WriteLine(codeBlockStart);
}
logger = GetRightLogger(logger);
logger.WriteLine();
foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
{
logger.WriteLineInfo(infoLine);
}
logger.WriteLineInfo(summary.JobRuntimes);
logger.WriteLine();
PrintTable(summary.Table, logger);
// TODO: move this logic to an analyser
var benchmarksWithTroubles = summary.Reports.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.WriteLine();
logger.WriteLineError("Benchmarks with issues:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
{
logger.WriteLineError(" " + benchmarkWithTroubles.DisplayInfo);
}
}
}
public override void ExportToLog(Summary summary, ILogger logger)
{
if (useCodeBlocks)
{
logger.WriteLine(codeBlockStart);
}
logger = GetRightLogger(logger);
logger.WriteLine();
foreach (string infoLine in summary.HostEnvironmentInfo.ToFormattedString())
{
logger.WriteLineInfo(infoLine);
}
logger.WriteLineInfo(summary.AllRuntimes);
logger.WriteLine();
PrintTable(summary.Table, logger);
// TODO: move this logic to an analyser
var benchmarksWithTroubles = summary.Reports.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.WriteLine();
logger.WriteLineError("Benchmarks with issues:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
{
logger.WriteLineError(" " + benchmarkWithTroubles.DisplayInfo);
}
}
}
private ExecuteResult Execute(Process process, BenchmarkCase benchmarkCase, SynchronousProcessOutputLoggerWithDiagnoser loggerWithDiagnoser,
ILogger logger, ConsoleExitHandler consoleExitHandler, int launchIndex)
{
logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}");
process.Start();
process.EnsureHighPriority(logger);
if (benchmarkCase.Job.Environment.HasValue(EnvironmentMode.AffinityCharacteristic))
{
process.TrySetAffinity(benchmarkCase.Job.Environment.Affinity, logger);
}
loggerWithDiagnoser.ProcessInput();
if (!process.WaitForExit(milliseconds: (int)ExecuteParameters.ProcessExitTimeout.TotalMilliseconds))
{
logger.WriteLineInfo("// The benchmarking process did not quit on time, it's going to get force killed now.");
consoleExitHandler.KillProcessTree();
}
if (loggerWithDiagnoser.LinesWithResults.Any(line => line.Contains("BadImageFormatException")))
{
logger.WriteLineError("You are probably missing <PlatformTarget>AnyCPU</PlatformTarget> in your .csproj file.");
}
return(new ExecuteResult(true, process.ExitCode, process.Id, loggerWithDiagnoser.LinesWithResults, loggerWithDiagnoser.LinesWithExtraOutput, launchIndex));
}
public override void ExportToLog(Summary summary, ILogger logger)
{
logger.WriteLine("....");
foreach (string infoLine in summary.HostEnvironmentInfo.ToFormattedString())
{
logger.WriteLineInfo(infoLine);
}
logger.WriteLineInfo(summary.AllRuntimes);
logger.WriteLine();
PrintTable(summary.Table, logger);
var benchmarksWithTroubles = summary.Reports
.Where(r => !r.GetResultRuns().Any())
.Select(r => r.BenchmarkCase)
.ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.WriteLine();
logger.WriteLine("[WARNING]");
logger.WriteLineError(".Benchmarks with issues");
logger.WriteLine("====");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
{
logger.WriteLineError("* " + benchmarkWithTroubles.DisplayInfo);
}
logger.WriteLine("====");
}
}
public override void ExportToLog(Summary summary, ILogger logger)
{
logger.WriteLine("....");
foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
{
logger.WriteLineInfo(infoLine);
}
logger.WriteLineInfo(summary.JobRuntimes);
logger.WriteLine();
PrintTable(summary.Table, logger);
var benchmarksWithTroubles = summary.Reports
.Where(r => !r.GetResultRuns().Any())
.Select(r => r.Benchmark)
.ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.WriteLine();
logger.WriteLine("[WARNING]");
logger.WriteLineError(".Benchmarks with issues");
logger.WriteLine("====");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError("* " + benchmarkWithTroubles.DisplayInfo);
logger.WriteLine("====");
}
}
public static void PrintSummary(Project project, ILogger logger)
{
// Summary
logger.WriteLineInfo(" ");
logger.WriteLineInfo("Summary: ");
logger.WriteLineInfo(" ");
foreach (DataStream dataStream in project.DataStreams)
{
String line = string.Format("{0,8}", dataStream.ShortName);
line += string.Format("{0,13}", dataStream.NumOfData);
if (!Double.IsNaN(dataStream.Length))
{
line += string.Format("{0,15}", TimeSpan.FromSeconds(dataStream.Length).ToString(@"hh\:mm\:ss"));
}
line += string.Format("{0,25}", dataStream.StartTime.ToString("yyyy-MM-dd HH:mm:ss"));
line += string.Format("{0,25}", dataStream.EndTime.ToString("yyyy-MM-dd HH:mm:ss"));
if (dataStream.DataLines.Count() > 0)
{
line += string.Format("{0,15}", dataStream.DataLines[0].TimeType.ToString());
}
logger.WriteLineInfo(line);
}
logger.WriteLineInfo(" ");
}
protected void CreateMetadataHeader(ILogger writer)
{
writer.WriteLineInfo(" ");
writer.WriteLineInfo("---------------------------------------------");
writer.WriteLineInfo("Data stream name : " + this.Name);
writer.WriteLineInfo("Data stream short name : " + this.ShortName);
writer.WriteLineInfo("Data location subfolder : " + this.SubFolder);
}
public void DisplayResults(ILogger logger)
{
if (!benchmarkToEtlFile.Any())
{
return;
}
logger.WriteLineInfo($"Exported {benchmarkToEtlFile.Count} trace file(s). Example:");
logger.WriteLineInfo(benchmarkToEtlFile.Values.First());
}
private static List <ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config)
{
var executeResults = new List <ExecuteResult>();
logger.WriteLineInfo("// *** Execute ***");
var launchCount = Math.Max(1, benchmark.Job.LaunchCount.IsAuto ? 2 : benchmark.Job.LaunchCount.Value);
for (int processNumber = 0; processNumber < launchCount; processNumber++)
{
var printedProcessNumber = (benchmark.Job.LaunchCount.IsAuto && processNumber < 2) ? "" : " / " + launchCount.ToString();
logger.WriteLineInfo($"// Launch: {processNumber + 1}{printedProcessNumber}");
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger);
if (!executeResult.FoundExecutable)
{
logger.WriteLineError("Executable not found");
}
executeResults.Add(executeResult);
if (benchmark.Job.LaunchCount.IsAuto && processNumber == 1)
{
var measurements = executeResults.
SelectMany(r => r.Data).
Select(line => Measurement.Parse(logger, line, 0)).
Where(r => r != null).
ToArray();
var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median;
var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median;
var percent = idleApprox / mainApprox * 100;
launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula
}
}
logger.NewLine();
// Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results
if (config.GetDiagnosers().Count() > 0)
{
logger.WriteLineInfo($"// Run, Diagnostic");
config.GetCompositeDiagnoser().Start(benchmark);
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, config.GetCompositeDiagnoser());
var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r != null).ToList();
var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns);
config.GetCompositeDiagnoser().Stop(benchmark, report);
if (!executeResult.FoundExecutable)
{
logger.WriteLineError("Executable not found");
}
logger.NewLine();
}
return(executeResults);
}
public override void WriteMetadata(ILogger writer, CancellationToken cancelletionToken = default(CancellationToken))
{
CreateMetadataHeader(writer);
String subFolder = this.project.Folder + "\\" + this.SubFolder;
writer.WriteLineInfo("Data location: " + subFolder);
writer.WriteLineInfo(" ");
if (Directory.Exists(subFolder) == false)
{
writer.WriteLineInfo("!!! Folder does not exist! Data stream is missing!");
writer.WriteLineWarning(this.Name + ":Folder does not exist! Data stream is missing!");
return;
}
string rootFolder = project.Folder + "\\" + this.SubFolder + "\\";
string[] allFiles = Directory.GetFiles(rootFolder, "*.*", SearchOption.AllDirectories);
string[] files = allFiles.Where(s => s.EndsWith(".JPG")).ToArray();
foreach (String file in files)
{
cancelletionToken.ThrowIfCancellationRequested();
DateTime fileTime = File.GetLastWriteTimeUtc(file);
String subFile = file.Replace(rootFolder, "");
this.DataLines.Add(new ImageDataLine(subFile, fileTime, TimeType.FILE));
}
writer.WriteLineInfo("\t Total number of images : " + DataLines.Count());
writer.WriteLineInfo("\t First file created : " + this.StartTime);
writer.WriteLineInfo("\t Last file created : " + this.EndTime);
writer.WriteLineInfo("\t Frame per second : " + (DataLines.Count() / this.Length).ToString("0.000") + " [fps]");
writer.WriteLineInfo("\t Frequency : " + (this.Length / files.Count()).ToString("0.000") + " [Hz]");
this.OrderDataLines();
if (files.Count() == 0)
{
writer.WriteLineInfo("Empty folder!");
writer.WriteLineWarning(this.Name + ":Empty folder!");
}
int dn = Math.Abs(allFiles.Count() - files.Count());
if (dn != 0)
{
writer.WriteLineInfo("\t No. of unidentified files : " + dn);
writer.WriteLineWarning(this.Name + ":Number of unidentified files in the folder: " + dn);
}
}
private void CopyFile(ILogger logger, string sourcePath, string destinationPath)
{
logger.WriteLineInfo("// Copying {0}", Path.GetFileName(sourcePath));
logger.WriteLineInfo("// from: {0}", Path.GetDirectoryName(sourcePath));
logger.WriteLineInfo("// to: {0}", Path.GetDirectoryName(destinationPath));
try
{
File.Copy(Path.GetFullPath(sourcePath), Path.GetFullPath(destinationPath), overwrite: true);
}
catch (Exception ex)
{
logger.WriteLineError(ex.Message);
throw;
}
}
public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
{
logger.WriteLineInfo($"BuildScript: {generateResult.ArtifactsPaths.BuildScriptFilePath}");
var syntaxTree = CSharpSyntaxTree.ParseText(File.ReadAllText(generateResult.ArtifactsPaths.ProgramCodePath));
var compilationOptions = new CSharpCompilationOptions(
outputKind: OutputKind.ConsoleApplication,
optimizationLevel: OptimizationLevel.Release,
allowUnsafe: true,
platform: GetPlatform(buildPartition.Platform),
deterministic: true);
compilationOptions = compilationOptions.WithIgnoreCorLibraryDuplicatedTypes();
var references = Generator
.GetAllReferences(buildPartition.RepresentativeBenchmarkCase)
.Select(assembly => AssemblyMetadata.CreateFromFile(assembly.Location))
.Concat(FrameworkAssembliesMetadata.Value)
.Distinct()
.Select(uniqueMetadata => uniqueMetadata.GetReference())
.ToList();
var(result, missingReferences) = Build(generateResult, syntaxTree, compilationOptions, references);
if (result.IsBuildSuccess || !missingReferences.Any())
{
return(result);
}
var withMissingReferences = references.Union(missingReferences.Select(assemblyMetadata => assemblyMetadata.GetReference()));
return(Build(generateResult, syntaxTree, compilationOptions, withMissingReferences).result);
}
private ExecuteResult Execute(Process process, Benchmark benchmark, SynchronousProcessOutputLoggerWithDiagnoser loggerWithDiagnoser, ILogger logger)
{
logger.WriteLineInfo("// Execute: " + process.StartInfo.FileName + " " + process.StartInfo.Arguments);
consoleHandler.SetProcess(process);
process.Start();
process.EnsureHighPriority(logger);
if (benchmark.Job.Env.HasValue(EnvMode.AffinityCharacteristic))
{
process.EnsureProcessorAffinity(benchmark.Job.Env.Affinity);
}
loggerWithDiagnoser.ProcessInput();
process.WaitForExit(); // should we add timeout here?
if (process.ExitCode == 0)
{
return(new ExecuteResult(true, process.ExitCode, loggerWithDiagnoser.LinesWithResults, loggerWithDiagnoser.LinesWithExtraOutput));
}
return(new ExecuteResult(true, process.ExitCode, new string[0], new string[0]));
}
/// <summary>
/// update CoreRun folder with newer versions of duplicated dependencies
/// </summary>
private void UpdateDuplicatedDependencies(ArtifactsPaths artifactsPaths, ILogger logger)
{
var publishedDirectory = new DirectoryInfo(artifactsPaths.BinariesDirectoryPath);
var coreRunDirectory = CoreRun.Directory;
foreach (var publishedDependency in publishedDirectory
.EnumerateFileSystemInfos()
.Where(file => file.Extension == ".dll" || file.Extension == ".exe"))
{
var coreRunDependency = new FileInfo(Path.Combine(coreRunDirectory.FullName, publishedDependency.Name));
if (!coreRunDependency.Exists)
{
continue; // the file does not exist in CoreRun directory, we don't need to worry, it will be just loaded from publish directory by CoreRun
}
var publishedVersionInfo = FileVersionInfo.GetVersionInfo(publishedDependency.FullName);
var coreRunVersionInfo = FileVersionInfo.GetVersionInfo(coreRunDependency.FullName);
if (!Version.TryParse(publishedVersionInfo.FileVersion, out var publishedVersion) || !Version.TryParse(coreRunVersionInfo.FileVersion, out var coreRunVersion))
{
continue;
}
if (publishedVersion > coreRunVersion)
{
File.Copy(publishedDependency.FullName, coreRunDependency.FullName, overwrite: true); // we need to ovwerite old things with their newer versions
logger.WriteLineInfo($"Copying {publishedDependency.FullName} to {coreRunDependency.FullName}");
}
}
}
public BuildResult Build(GenerateResult generateResult, ILogger logger, Benchmark benchmark, IResolver resolver)
{
logger.WriteLineInfo($"BuildScript: {generateResult.ArtifactsPaths.BuildScriptFilePath}");
var syntaxTree = CSharpSyntaxTree.ParseText(File.ReadAllText(generateResult.ArtifactsPaths.ProgramCodePath));
var compilationOptions = new CSharpCompilationOptions(
outputKind: OutputKind.ConsoleApplication,
optimizationLevel: OptimizationLevel.Release,
allowUnsafe: true,
platform: GetPlatform(benchmark.Job.ResolveValue(EnvMode.PlatformCharacteristic, resolver)),
deterministic: true);
var references = Generator
.GetAllReferences(benchmark)
.Select(assembly => AssemblyMetadata.CreateFromFile(assembly.Location))
.Concat(FrameworkAssembliesMetadata.Value)
.Distinct()
.Select(uniqueMetadata => uniqueMetadata.GetReference());
(var result, var missingReferences) = Build(generateResult, benchmark, syntaxTree, compilationOptions, references);
if (result.IsBuildSuccess || !missingReferences.Any())
{
return(result);
}
var withMissingReferences = references.Union(missingReferences.Select(assemblyMetadata => assemblyMetadata.GetReference()));
return(Build(generateResult, benchmark, syntaxTree, compilationOptions, withMissingReferences).result);
}
internal static CommandResult ExecuteCommand(
string customDotNetCliPath, string commandWithArguments, string workingDirectory, ILogger logger,
IReadOnlyList <EnvironmentVariable> environmentVariables = null, bool useSharedCompilation = false)
{
commandWithArguments = $"{commandWithArguments} /p:UseSharedCompilation={useSharedCompilation.ToString().ToLowerInvariant()}";
using (var process = new Process {
StartInfo = BuildStartInfo(customDotNetCliPath, workingDirectory, commandWithArguments, environmentVariables)
})
{
var stopwatch = Stopwatch.StartNew();
process.Start();
var standardOutput = process.StandardOutput.ReadToEnd();
var standardError = process.StandardError.ReadToEnd();
process.WaitForExit();
stopwatch.Stop();
logger.WriteLineInfo($"// {commandWithArguments} took {stopwatch.Elapsed.TotalSeconds:0.##}s and exited with {process.ExitCode}");
return(process.ExitCode <= 0
? CommandResult.Success(stopwatch.Elapsed, standardOutput)
: CommandResult.Failure(stopwatch.Elapsed, standardError, standardOutput));
}
}
private ExecuteResult Execute(Process process, BenchmarkCase benchmarkCase, SynchronousProcessOutputLoggerWithDiagnoser loggerWithDiagnoser, ILogger logger)
{
logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}");
process.Start();
process.EnsureHighPriority(logger);
if (benchmarkCase.Job.Environment.HasValue(EnvironmentMode.AffinityCharacteristic))
{
process.TrySetAffinity(benchmarkCase.Job.Environment.Affinity, logger);
}
loggerWithDiagnoser.ProcessInput();
process.WaitForExit(); // should we add timeout here?
if (process.ExitCode == 0)
{
return(new ExecuteResult(true, process.ExitCode, loggerWithDiagnoser.LinesWithResults, loggerWithDiagnoser.LinesWithExtraOutput));
}
if (loggerWithDiagnoser.LinesWithResults.Any(line => line.Contains("BadImageFormatException")))
{
logger.WriteLineError("You are probably missing <PlatformTarget>AnyCPU</PlatformTarget> in your .csproj file.");
}
return(new ExecuteResult(true, process.ExitCode, Array.Empty <string>(), Array.Empty <string>()));
}
private ExecuteResult Execute(Process process, Benchmark benchmark, SynchronousProcessOutputLoggerWithDiagnoser loggerWithDiagnoser, ILogger logger)
{
logger.WriteLineInfo("// Execute: " + process.StartInfo.FileName + " " + process.StartInfo.Arguments);
ConsoleHandler.Instance.SetProcess(process);
process.Start();
process.EnsureHighPriority(logger);
if (benchmark.Job.Env.HasValue(EnvMode.AffinityCharacteristic))
{
process.EnsureProcessorAffinity(benchmark.Job.Env.Affinity);
}
loggerWithDiagnoser.ProcessInput();
process.WaitForExit(); // should we add timeout here?
if (process.ExitCode == 0)
{
return(new ExecuteResult(true, process.ExitCode, loggerWithDiagnoser.LinesWithResults, loggerWithDiagnoser.LinesWithExtraOutput));
}
if (loggerWithDiagnoser.LinesWithResults.Any(line => line.Contains("BadImageFormatException")))
{
logger.WriteLineError("You are probably missing <PlatformTarget>AnyCPU</PlatformTarget> in your .csproj file.");
}
return(new ExecuteResult(true, process.ExitCode, Array.Empty <string>(), Array.Empty <string>()));
}
public override void ExportToLog(Summary summary, ILogger logger)
{
if (useCodeBlocks)
{
logger.WriteLine($"```{codeBlocksSyntax}");
}
logger = GetRightLogger(logger);
logger.WriteLine();
foreach (var infoLine in EnvironmentInfo.GetCurrent().ToFormattedString("Host", true))
{
logger.WriteLineInfo(infoLine);
}
logger.WriteLine();
PrintTable(summary.Table, logger);
// TODO: move this logic to an analyser
var benchmarksWithTroubles = summary.Reports.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.WriteLine();
logger.WriteLineError("Benchmarks with issues:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
{
logger.WriteLineError(" " + benchmarkWithTroubles.ShortInfo);
}
}
}
/// <summary>
/// It fixes the time tags according to the length of the videos relative to the first video.
/// This method assumes that the videos are recorded sequnetially.
/// </summary>
public void FixTimestampsByOrder(ILogger writer = null)
{
writer?.WriteLineInfo("Fix timestamps for " + this.Name);
for (int i = 0; i < DataLines.Count - 1; i++)
{
VideoDataLine currLine = DataLines[i] as VideoDataLine;
VideoDataLine nextLine = DataLines[i + 1] as VideoDataLine;
double dt = currLine.FrameCount * (1 / currLine.FrameRate);
DateTime original = nextLine.TimeStamp;
nextLine.TimeStamp = currLine.TimeStamp.AddSeconds(dt);
writer?.WriteLineInfo("File: " + nextLine.VideoFileName);
writer?.WriteLineInfo(" Original: " + original.ToString("yyyy-MM-dd HH:mm:ss.fff"));
writer?.WriteLineInfo(" New: " + nextLine.TimeStamp.ToString("yyyy-MM-dd HH:mm:ss.fff"));
writer?.WriteLineInfo(" Diff: " + new TimeSpan(nextLine.TimeStamp.Ticks - original.Ticks).ToString(@"hh\:mm\:ss\.fff"));
}
}
/// <summary>Helper method to dump the content of the message into logger.</summary>
/// <param name="logger">The logger the message will be dumped to.</param>
/// <param name="message">The message to log.</param>
internal static void LogMessage([NotNull] this ILogger logger, [NotNull] IMessage message)
{
Code.NotNull(logger, nameof(logger));
Code.NotNull(message, nameof(message));
if (message.MessageSeverity.IsCriticalError())
{
logger.WriteLineError($"{LogImportantInfoPrefix} {message.ToLogString()}");
}
else if (message.MessageSeverity.IsWarningOrHigher())
{
logger.WriteLineInfo($"{LogImportantInfoPrefix} {message.ToLogString()}");
}
else
{
logger.WriteLineInfo($"{LogInfoPrefix} {message.ToLogString()}");
}
}
private void EnsureDependancyInCorrectLocation(ILogger logger, Type type, string outputDir)
{
var fileInfo = new FileInfo(type.Assembly.Location);
if (fileInfo.Name == "mscorlib.dll")
{
return;
}
var expectedLocation = Path.GetFullPath(Path.Combine(outputDir, "..\\" + fileInfo.Name));
if (File.Exists(expectedLocation) == false)
{
logger.WriteLineInfo("// File doesn't exist: {0}", expectedLocation);
logger.WriteLineInfo("// Actually at: {0}", fileInfo.FullName);
CopyFile(logger, fileInfo.FullName, expectedLocation);
}
}
public static void Create(Project project, ILogger logger, CancellationToken cancelletionToken = default(CancellationToken))
{
logger.WriteLineInfo("----------------------------------");
logger.WriteLineInfo(" CAR METADATA file v0.1");
logger.WriteLineInfo(" The Ohio State University, 2017");
logger.WriteLineInfo("----------------------------------");
logger.WriteLineInfo(" ");
logger.WriteLineInfo(" ");
int pi = 0;
foreach (DataStream dataStream in project.DataStreams)
{
pi++;
cancelletionToken.ThrowIfCancellationRequested();
logger.WriteProgress((double)pi / (double)project.DataStreams.Count() * 100.0);
dataStream.WriteMetadata(logger);
}
PrintSummary(project, logger);
logger.WriteLineInfo(" ");
logger.WriteLineInfo("Checking: ");
// Checking Nikon's Novatel MARKTIMEA
ImageDataStream nikonDataStream = (ImageDataStream)project.GetDataStreamByShortName("NIKON");
NovatelDataStream novatelDataStream = (NovatelDataStream)project.GetDataStreamByShortName("NOVATEL");
CheckEvents(nikonDataStream, novatelDataStream, EvenMarkerDataLine.MarkerEventPort.Unknown, logger);
/*if ((nikonDataStream != null) && (novatelDataStream != null))
* {
* if (nikonDataStream.NumOfFiles != novatelDataStream.MarkerEvents.Count())
* {
* logger.WriteLineInfo("WRONG -- Number of #MARKTIME events from the Novatel ASC and the number of Nikon images are different!");
* logger.WriteLineWarning("Number of #MARKTIME events from the Novatel ASC and the number of Nikon images are different!");
* }
* else
* {
* logger.WriteLineInfo("OK -- Number of #MARKTIME events from the Novatel ASC and the number of Nikon images !");
* }
* }*/
// Checking Sony's Septentrio MARKTIME
ImageDataStream son1DataStream = (ImageDataStream)project.GetDataStreamByShortName("SON1");
ImageDataStream son2DataStream = (ImageDataStream)project.GetDataStreamByShortName("SON2");
GPSDataStream septDataStream = (GPSDataStream)project.GetDataStreamByShortName("SEPT");
CheckEvents(son1DataStream, septDataStream, EvenMarkerDataLine.MarkerEventPort.EventA, logger);
CheckEvents(son2DataStream, septDataStream, EvenMarkerDataLine.MarkerEventPort.EventB, logger);
logger.Flush();
}
public static bool IsSupported(Benchmark benchmark, ILogger logger)
{
var result = new List <ValidationError>();
ValidateJob(benchmark.Job, true, result);
if (result.Any())
{
logger.WriteLineInfo($"// Benchmark {benchmark.DisplayInfo}");
logger.WriteLineInfo("// cannot be run in-process. Validation errors:");
foreach (var validationError in result)
{
logger.WriteLineInfo($"// * {validationError.Message}");
}
logger.WriteLine();
return(false);
}
return(true);
}
private static BuildResult Build(ILogger logger, IToolchain toolchain, GenerateResult generateResult, Benchmark benchmark)
{
logger.WriteLineInfo("// *** Build ***");
var buildResult = toolchain.Builder.Build(generateResult, logger, benchmark);
if (buildResult.IsBuildSuccess)
{
logger.WriteLineInfo("// Result = Success");
}
else
{
logger.WriteLineError("// Result = Failure");
if (buildResult.BuildException != null)
{
logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}");
}
}
logger.WriteLine();
return(buildResult);
}
private static ValidationError[] Validate(IList <Benchmark> benchmarks, ILogger logger, IConfig config)
{
logger.WriteLineInfo("// Validating benchmarks:");
var validationErrors = config.GetCompositeValidator().Validate(benchmarks).ToArray();
foreach (var validationError in validationErrors)
{
logger.WriteLineError(validationError.Message);
}
return(validationErrors);
}
private static GenerateResult Generate(ILogger logger, IToolchain toolchain, Benchmark benchmark, string rootArtifactsFolderPath, IConfig config)
{
logger.WriteLineInfo("// *** Generate *** ");
var generateResult = toolchain.Generator.GenerateProject(benchmark, logger, rootArtifactsFolderPath, config);
if (generateResult.IsGenerateSuccess)
{
logger.WriteLineInfo("// Result = Success");
logger.WriteLineInfo($"// {nameof(generateResult.DirectoryPath)} = {generateResult.DirectoryPath}");
}
else
{
logger.WriteLineError("// Result = Failure");
if (generateResult.GenerateException != null)
{
logger.WriteLineError($"// Exception: {generateResult.GenerateException.Message}");
}
}
logger.WriteLine();
return(generateResult);
}
public static bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger)
{
var result = new List<ValidationError>();
result.AddRange(ValidateJob(benchmarkCase.Job, true));
if (benchmarkCase.HasArguments)
result.Add(new ValidationError(true, "Arguments are not supported by the InProcessToolchain yet, see #687 for more details"));
if (result.Any())
{
logger.WriteLineInfo($"// Benchmark {benchmarkCase.DisplayInfo}");
logger.WriteLineInfo("// cannot be run in-process. Validation errors:");
foreach (var validationError in result)
{
logger.WriteLineInfo($"// * {validationError.Message}");
}
logger.WriteLine();
return false;
}
return true;
}
public BuildResult Build(GenerateResult generateResult, ILogger logger, Benchmark benchmark, IResolver resolver)
{
var extraArguments = DotNetCliGenerator.GetCustomArguments(benchmark, resolver);
if (!string.IsNullOrEmpty(RestoreCommand))
{
var restoreResult = DotNetCliCommandExecutor.ExecuteCommand(
CustomDotNetCliPath,
$"{RestoreCommand} {extraArguments}",
generateResult.ArtifactsPaths.BuildArtifactsDirectoryPath);
logger.WriteLineInfo($"// dotnet restore took {restoreResult.ExecutionTime.TotalSeconds:0.##}s");
if (!restoreResult.IsSuccess)
{
return(BuildResult.Failure(generateResult, new Exception(restoreResult.ProblemDescription)));
}
}
var buildResult = Build(
generateResult,
benchmark.Job.ResolveValue(InfrastructureMode.BuildConfigurationCharacteristic, resolver),
extraArguments);
logger.WriteLineInfo($"// dotnet build/publish took {buildResult.ExecutionTime.TotalSeconds:0.##}s");
if (!buildResult.IsSuccess)
{
// dotnet cli could have succesfully builded the program, but returned 1 as exit code because it had some warnings
// so we need to check whether the exe exists or not, if it does then it is OK
if (File.Exists(generateResult.ArtifactsPaths.ExecutablePath))
{
return(BuildResult.Success(generateResult));
}
return(BuildResult.Failure(generateResult, new Exception(buildResult.ProblemDescription)));
}
return(BuildResult.Success(generateResult));
}
internal void PrintOptions(int prefixWidth, int outputWidth)
{
foreach (var option in Configuration)
{
var optionText = $" {OptionPrefix}{option.Key}=<{option.Key.ToUpperInvariant()}>";
logger.WriteResult($"{optionText.PadRight(prefixWidth)}");
var maxWidth = outputWidth - prefixWidth - System.Environment.NewLine.Length - BreakText.Length;
var lines = StringAndTextExtensions.Wrap(option.Value, maxWidth);
if (lines.Count == 0)
{
logger.WriteLine();
continue;
}
logger.WriteLineInfo($"{BreakText}{lines.First().Trim(TrimChars)}");
var padding = new string(' ', prefixWidth);
foreach (var line in lines.Skip(1))
{
logger.WriteLineInfo($"{padding}{BreakText}{line.Trim(TrimChars)}");
}
}
}
public BuildResult Build(GenerateResult generateResult, ILogger logger, Benchmark benchmark)
{
logger.WriteLineInfo($"BuildScript: {generateResult.ArtifactsPaths.BuildScriptFilePath}");
var syntaxTree = CSharpSyntaxTree.ParseText(File.ReadAllText(generateResult.ArtifactsPaths.ProgramCodePath));
var compilationOptions = new CSharpCompilationOptions(
outputKind: OutputKind.ConsoleApplication,
optimizationLevel: OptimizationLevel.Release,
allowUnsafe: true,
platform: GetPlatform(benchmark.Job.Platform),
deterministic: true);
var references = RoslynGenerator
.GetAllReferences(benchmark)
.Select(assembly => AssemblyMetadata.CreateFromFile(assembly.Location))
.Concat(FrameworkAssembliesMetadata.Value)
.Distinct()
.Select(uniqueMetadata => uniqueMetadata.GetReference());
var compilation = CSharpCompilation
.Create(assemblyName: Path.GetFileName(generateResult.ArtifactsPaths.ExecutablePath))
.AddSyntaxTrees(syntaxTree)
.WithOptions(compilationOptions)
.AddReferences(references);
using (var executable = File.Create(generateResult.ArtifactsPaths.ExecutablePath))
{
var emitResult = compilation.Emit(executable);
if(emitResult.Success)
{
return BuildResult.Success(generateResult);
}
foreach (var diagnostic in emitResult.Diagnostics
.Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error))
{
logger.WriteError($"{diagnostic.Id}: {diagnostic.GetMessage()}");
}
return BuildResult.Failure(generateResult);
}
}
private static ValidationError[] Validate(IList<Benchmark> benchmarks, ILogger logger, IConfig config)
{
logger.WriteLineInfo("// Validating benchmarks:");
var validationErrors = config.GetCompositeValidator().Validate(benchmarks).ToArray();
foreach (var validationError in validationErrors)
{
logger.WriteLineError(validationError.Message);
}
return validationErrors;
}
internal void PrintOptions(ILogger logger, int prefixWidth, int outputWidth)
{
foreach (var option in configuration)
{
var optionText = $" {optionPrefix}{option.Key} <{option.Key.ToUpperInvariant()}>";
logger.WriteResult($"{optionText.PadRight(prefixWidth)}");
var maxWidth = outputWidth - prefixWidth - Environment.NewLine.Length - breakText.Length;
var lines = StringAndTextExtensions.Wrap(option.Value, maxWidth);
if (lines.Count == 0)
{
logger.WriteLine();
continue;
}
logger.WriteLineInfo($"{breakText}{lines.First().Trim(trimChars)}");
var padding = new string(' ', prefixWidth);
foreach (var line in lines.Skip(1))
logger.WriteLineInfo($"{padding}{breakText}{line.Trim(trimChars)}");
}
}
private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config, IResolver resolver)
{
var executeResults = new List<ExecuteResult>();
logger.WriteLineInfo("// *** Execute ***");
bool analyzeRunToRunVariance = benchmark.Job.ResolveValue(AccuracyMode.AnalyzeLaunchVarianceCharacteristic, resolver);
bool autoLaunchCount = !benchmark.Job.HasValue(RunMode.LaunchCountCharacteristic);
int defaultValue = analyzeRunToRunVariance ? 2 : 1;
int launchCount = Math.Max(
1,
autoLaunchCount ? defaultValue: benchmark.Job.Run.LaunchCount);
for (int launchIndex = 0; launchIndex < launchCount; launchIndex++)
{
string printedLaunchCount = (analyzeRunToRunVariance &&
autoLaunchCount &&
launchIndex < 2)
? ""
: " / " + launchCount;
logger.WriteLineInfo($"// Launch: {launchIndex + 1}{printedLaunchCount}");
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver);
if (!executeResult.FoundExecutable)
logger.WriteLineError("Executable not found");
if (executeResult.ExitCode != 0)
logger.WriteLineError("ExitCode != 0");
executeResults.Add(executeResult);
var measurements = executeResults
.SelectMany(r => r.Data)
.Select(line => Measurement.Parse(logger, line, 0))
.Where(r => r.IterationMode != IterationMode.Unknown).
ToArray();
if (!measurements.Any())
{
// Something went wrong during the benchmark, don't bother doing more runs
logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!");
break;
}
if (autoLaunchCount && launchIndex == 1 && analyzeRunToRunVariance)
{
// TODO: improve this logic
var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median;
var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median;
var percent = idleApprox / mainApprox * 100;
launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula
}
}
logger.WriteLine();
// Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results
if (config.GetDiagnosers().Any())
{
logger.WriteLineInfo("// Run, Diagnostic");
var compositeDiagnoser = config.GetCompositeDiagnoser();
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver, compositeDiagnoser);
var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList();
var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns);
compositeDiagnoser.ProcessResults(benchmark, report);
if (!executeResult.FoundExecutable)
logger.WriteLineError("Executable not found");
logger.WriteLine();
}
return executeResults;
}
private void CopyFile(ILogger logger, string sourcePath, string destinationPath)
{
logger.WriteLineInfo("// Copying {0}", Path.GetFileName(sourcePath));
logger.WriteLineInfo("// from: {0}", Path.GetDirectoryName(sourcePath));
logger.WriteLineInfo("// to: {0}", Path.GetDirectoryName(destinationPath));
try
{
File.Copy(Path.GetFullPath(sourcePath), Path.GetFullPath(destinationPath), overwrite: true);
}
catch (Exception ex)
{
logger.WriteLineError(ex.Message);
throw;
}
}
private void EnsureDependancyInCorrectLocation(ILogger logger, Type type, string outputDir)
{
var fileInfo = new FileInfo(type.Assembly.Location);
if (fileInfo.Name == "mscorlib.dll")
return;
var expectedLocation = Path.GetFullPath(Path.Combine(outputDir, "..\\" + fileInfo.Name));
if (File.Exists(expectedLocation) == false)
{
logger.WriteLineInfo("// File doesn't exist: {0}", expectedLocation);
logger.WriteLineInfo("// Actually at: {0}", fileInfo.FullName);
CopyFile(logger, fileInfo.FullName, expectedLocation);
}
}
private static Summary Run(Benchmark[] benchmarks, ILogger logger, string title, IConfig config, string rootArtifactsFolderPath, Func<IJob, IToolchain> toolchainProvider)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.ShortInfo}");
logger.WriteLine();
var validationErrors = Validate(benchmarks, logger, config);
if (validationErrors.Any(validationError => validationError.IsCritical))
{
return Summary.CreateFailed(benchmarks, title, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors);
}
var globalChronometer = Chronometer.Start();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
var report = Run(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider);
reports.Add(report);
if (report.GetResultRuns().Any())
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
logger.WriteLine();
}
var clockSpan = globalChronometer.Stop();
var summary = new Summary(title, reports, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors);
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.WriteLine();
logger.WriteLineHeader("// * Export *");
var currentDirectory = Directory.GetCurrentDirectory();
foreach (var file in config.GetCompositeExporter().ExportToFiles(summary))
{
logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}");
}
logger.WriteLine();
logger.WriteLineHeader("// * Detailed results *");
// TODO: make exporter
foreach (var report in reports)
{
logger.WriteLineInfo(report.Benchmark.ShortInfo);
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
logger.WriteLine();
}
LogTotalTime(logger, clockSpan.GetTimeSpan());
logger.WriteLine();
logger.WriteLineHeader("// * Summary *");
MarkdownExporter.Console.ExportToLog(summary, logger);
// TODO: make exporter
var warnings = config.GetCompositeAnalyser().Analyse(summary).ToList();
if (warnings.Count > 0)
{
logger.WriteLine();
logger.WriteLineError("// * Warnings * ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
if (config.GetDiagnosers().Count() > 0)
{
logger.WriteLine();
config.GetCompositeDiagnoser().DisplayResults(logger);
}
logger.WriteLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return summary;
}
private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config)
{
var executeResults = new List<ExecuteResult>();
logger.WriteLineInfo("// *** Execute ***");
var launchCount = Math.Max(1, benchmark.Job.LaunchCount.IsAuto ? 2 : benchmark.Job.LaunchCount.Value);
for (int processNumber = 0; processNumber < launchCount; processNumber++)
{
var printedProcessNumber = (benchmark.Job.LaunchCount.IsAuto && processNumber < 2) ? "" : " / " + launchCount.ToString();
logger.WriteLineInfo($"// Launch: {processNumber + 1}{printedProcessNumber}");
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger);
if (!executeResult.FoundExecutable)
logger.WriteLineError("Executable not found");
executeResults.Add(executeResult);
var measurements = executeResults
.SelectMany(r => r.Data)
.Select(line => Measurement.Parse(logger, line, 0))
.Where(r => r.IterationMode != IterationMode.Unknown).
ToArray();
if (!measurements.Any())
{
// Something went wrong during the benchmark, don't bother doing more runs
logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!");
break;
}
if (benchmark.Job.LaunchCount.IsAuto && processNumber == 1)
{
var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median;
var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median;
var percent = idleApprox / mainApprox * 100;
launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula
}
}
logger.WriteLine();
// Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results
if (config.GetDiagnosers().Count() > 0)
{
logger.WriteLineInfo($"// Run, Diagnostic");
config.GetCompositeDiagnoser().Start(benchmark);
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, config.GetCompositeDiagnoser());
var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList();
var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns);
config.GetCompositeDiagnoser().Stop(benchmark, report);
if (!executeResult.FoundExecutable)
logger.WriteLineError("Executable not found");
logger.WriteLine();
}
return executeResults;
}
private static BuildResult Build(ILogger logger, IToolchain toolchain, GenerateResult generateResult, Benchmark benchmark)
{
logger.WriteLineInfo("// *** Build ***");
var buildResult = toolchain.Builder.Build(generateResult, logger, benchmark);
if (buildResult.IsBuildSuccess)
{
logger.WriteLineInfo("// Result = Success");
}
else
{
logger.WriteLineError("// Result = Failure");
if (buildResult.BuildException != null)
logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}");
}
logger.WriteLine();
return buildResult;
}
private static GenerateResult Generate(ILogger logger, IToolchain toolchain, Benchmark benchmark, string rootArtifactsFolderPath, IConfig config)
{
logger.WriteLineInfo("// *** Generate *** ");
var generateResult = toolchain.Generator.GenerateProject(benchmark, logger, rootArtifactsFolderPath, config);
if (generateResult.IsGenerateSuccess)
{
logger.WriteLineInfo("// Result = Success");
logger.WriteLineInfo($"// {nameof(generateResult.ArtifactsPaths.BinariesDirectoryPath)} = {generateResult.ArtifactsPaths?.BinariesDirectoryPath}");
}
else
{
logger.WriteLineError("// Result = Failure");
if (generateResult.GenerateException != null)
logger.WriteLineError($"// Exception: {generateResult.GenerateException.Message}");
}
logger.WriteLine();
return generateResult;
}
