public void RemoveFile(string path)
{
SourceFile file = SourceFiles.FirstOrDefault(f => string.Compare(f.Path, path) == 0);
if (file != null)
{
SourceFiles.Remove(file);
}
}
internal CompilerContext Compile()
{
var filePath = SourceFiles.FirstOrDefault()?.FilePath;
if (!File.Exists(filePath))
{
return(null);
}
var assName = Prefix + Path.GetFileNameWithoutExtension(filePath) + ".exe";
var cc = new BooScriptCompiler();
var refs = ScriptHelpers.GetReferences();
var imports = ScriptHelpers.GetDefaultImports();
cc.Configure(refs, imports);
//cc.InternalCompiler.Parameters.OutputType = Boo.Lang.Compiler.CompilerOutputType.ConsoleApplication;
cc.InternalCompiler.Parameters.OutputType = Boo.Lang.Compiler.CompilerOutputType.Library;
var context = cc.Compile(assName, new List <string>()
{
filePath
});
if (context.Errors.Count > 0)
{
BooScriptingPlugin.Instance.LogPrintBooErrors(context);
return(context); //dont bother listing warnings...
}
if (context.Warnings.Count > 0)
{
BooScriptingPlugin.Instance.LogPrintBooWarnings(context);
}
if (!Link(context.GeneratedAssembly))
{
BooScriptingPlugin.Instance.LogPrint($"Unable to link boo script '{filePath}'.");
return(null);
}
return(context);
}
bool Link(Assembly ass)
{
//if (!IsBuilt)
// return false;
if (ass == null)
{
return(false);
}
var filePath = SourceFiles.FirstOrDefault()?.FilePath;
if (string.IsNullOrWhiteSpace(filePath))
{
return(false);
}
var linker = new BooModuleLinker(ass, filePath);
OnRun = linker.TryCreateDelegate <Action <object[]> >("OnRun");
GetSchedule = linker.TryCreateDelegate <Func <Scheduler> >("GetSchedule");
return(true);
}
private bool ExecuteActual()
{
if (SourceFiles.Length == 0)
{
Log.LogError($"No SourceFiles to compile");
return(false);
}
ITaskItem?badItem = SourceFiles.FirstOrDefault(sf => string.IsNullOrEmpty(sf.GetMetadata("ObjectFile")));
if (badItem != null)
{
Log.LogError($"Source file {badItem.ItemSpec} is missing ObjectFile metadata.");
return(false);
}
if (!Enum.TryParse(OutputMessageImportance, ignoreCase: true, out MessageImportance messageImportance))
{
Log.LogError($"Invalid value for OutputMessageImportance={OutputMessageImportance}. Valid values: {string.Join(", ", Enum.GetNames(typeof(MessageImportance)))}");
return(false);
}
_totalFiles = SourceFiles.Length;
IDictionary <string, string> envVarsDict = GetEnvironmentVariablesDict();
ConcurrentBag <ITaskItem> outputItems = new();
try
{
List <(string, string)> filesToCompile = new();
foreach (ITaskItem srcItem in SourceFiles)
{
string srcFile = srcItem.ItemSpec;
string objFile = srcItem.GetMetadata("ObjectFile");
string depMetadata = srcItem.GetMetadata("Dependencies");
string[] depFiles = string.IsNullOrEmpty(depMetadata)
? Array.Empty <string>()
: depMetadata.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
if (!ShouldCompile(srcFile, objFile, depFiles, out string reason))
{
Log.LogMessage(MessageImportance.Low, $"Skipping {srcFile} because {reason}.");
outputItems.Add(CreateOutputItemFor(srcFile, objFile));
}
else
{
Log.LogMessage(MessageImportance.Low, $"Compiling {srcFile} because {reason}.");
filesToCompile.Add((srcFile, objFile));
}
}
_numCompiled = SourceFiles.Length - filesToCompile.Count;
if (_numCompiled == _totalFiles)
{
// nothing to do!
OutputFiles = outputItems.ToArray();
return(!Log.HasLoggedErrors);
}
if (_numCompiled > 0)
{
Log.LogMessage(MessageImportance.High, $"[{_numCompiled}/{SourceFiles.Length}] skipped unchanged files");
}
Log.LogMessage(MessageImportance.Low, "Using environment variables:");
foreach (var kvp in envVarsDict)
{
Log.LogMessage(MessageImportance.Low, $"\t{kvp.Key} = {kvp.Value}");
}
string workingDir = Environment.CurrentDirectory;
Log.LogMessage(MessageImportance.Low, $"Using working directory: {workingDir}");
_tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(_tempPath);
int allowedParallelism = DisableParallelCompile ? 1 : Math.Min(SourceFiles.Length, Environment.ProcessorCount);
if (BuildEngine is IBuildEngine9 be9)
{
allowedParallelism = be9.RequestCores(allowedParallelism);
}
/*
* From: https://github.com/dotnet/runtime/issues/46146#issuecomment-754021690
*
* Stephen Toub:
* "As such, by default ForEach works on a scheme whereby each
* thread takes one item each time it goes back to the enumerator,
* and then after a few times of this upgrades to taking two items
* each time it goes back to the enumerator, and then four, and
* then eight, and so on. This amortizes the cost of taking and
* releasing the lock across multiple items, while still enabling
* parallelization for enumerables containing just a few items. It
* does, however, mean that if you've got a case where the body
* takes a really long time and the work for every item is
* heterogeneous, you can end up with an imbalance."
*
* The time taken by individual compile jobs here can vary a
* lot, depending on various factors like file size. This can
* create an imbalance, like mentioned above, and we can end up
* in a situation where one of the partitions has a job that
* takes very long to execute, by which time other partitions
* have completed, so some cores are idle. But the idle
* ones won't get any of the remaining jobs, because they are
* all assigned to that one partition.
*
* Instead, we want to use work-stealing so jobs can be run by any partition.
*/
ParallelLoopResult result = Parallel.ForEach(
Partitioner.Create(filesToCompile, EnumerablePartitionerOptions.NoBuffering),
new ParallelOptions {
MaxDegreeOfParallelism = allowedParallelism
},
(toCompile, state) =>
{
if (!ProcessSourceFile(toCompile.Item1, toCompile.Item2))
{
state.Stop();
}
});
if (!result.IsCompleted && !Log.HasLoggedErrors)
{
Log.LogError("Unknown failure occurred while compiling. Check logs to get more details.");
}
if (!Log.HasLoggedErrors)
{
int numUnchanged = _totalFiles - _numCompiled;
if (numUnchanged > 0)
{
Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalFiles}] unchanged.");
}
}
}
finally
{
if (!string.IsNullOrEmpty(_tempPath))
{
Directory.Delete(_tempPath, true);
}
}
OutputFiles = outputItems.ToArray();
return(!Log.HasLoggedErrors);
bool ProcessSourceFile(string srcFile, string objFile)
{
string tmpObjFile = Path.GetTempFileName();
try
{
string command = $"emcc {Arguments} -c -o \"{tmpObjFile}\" \"{srcFile}\"";
var startTime = DateTime.Now;
// Log the command in a compact format which can be copy pasted
StringBuilder envStr = new StringBuilder(string.Empty);
foreach (var key in envVarsDict.Keys)
{
envStr.Append($"{key}={envVarsDict[key]} ");
}
Log.LogMessage(MessageImportance.Low, $"Exec: {envStr}{command}");
(int exitCode, string output) = Utils.RunShellCommand(
Log,
command,
envVarsDict,
workingDir: Environment.CurrentDirectory,
logStdErrAsMessage: true,
debugMessageImportance: messageImportance,
label: Path.GetFileName(srcFile));
var endTime = DateTime.Now;
var elapsedSecs = (endTime - startTime).TotalSeconds;
if (exitCode != 0)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{output} [took {elapsedSecs:F}s]");
return(false);
}
if (!Utils.CopyIfDifferent(tmpObjFile, objFile, useHash: true))
{
Log.LogMessage(MessageImportance.Low, $"Did not overwrite {objFile} as the contents are unchanged");
}
else
{
Log.LogMessage(MessageImportance.Low, $"Copied {tmpObjFile} to {objFile}");
}
outputItems.Add(CreateOutputItemFor(srcFile, objFile));
int count = Interlocked.Increment(ref _numCompiled);
Log.LogMessage(MessageImportance.High, $"[{count}/{_totalFiles}] {Path.GetFileName(srcFile)} -> {Path.GetFileName(objFile)} [took {elapsedSecs:F}s]");
return(!Log.HasLoggedErrors);
}
catch (Exception ex)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{ex.Message}");
return(false);
}
finally
{
File.Delete(tmpObjFile);
}
}
ITaskItem CreateOutputItemFor(string srcFile, string objFile)
{
ITaskItem newItem = new TaskItem(objFile);
newItem.SetMetadata("SourceFile", srcFile);
return(newItem);
}
}
public override bool Execute()
{
if (SourceFiles.Length == 0)
{
Log.LogError($"No SourceFiles to compile");
return(false);
}
ITaskItem?badItem = SourceFiles.FirstOrDefault(sf => string.IsNullOrEmpty(sf.GetMetadata("ObjectFile")));
if (badItem != null)
{
Log.LogError($"Source file {badItem.ItemSpec} is missing ObjectFile metadata.");
return(false);
}
if (!Enum.TryParse(OutputMessageImportance, ignoreCase: true, out MessageImportance messageImportance))
{
Log.LogError($"Invalid value for OutputMessageImportance={OutputMessageImportance}. Valid values: {string.Join(", ", Enum.GetNames(typeof(MessageImportance)))}");
return(false);
}
_totalFiles = SourceFiles.Length;
IDictionary <string, string> envVarsDict = GetEnvironmentVariablesDict();
ConcurrentBag <ITaskItem> outputItems = new();
try
{
List <(string, string)> filesToCompile = new();
foreach (ITaskItem srcItem in SourceFiles)
{
string srcFile = srcItem.ItemSpec;
string objFile = srcItem.GetMetadata("ObjectFile");
string depMetadata = srcItem.GetMetadata("Dependencies");
string[] depFiles = string.IsNullOrEmpty(depMetadata)
? Array.Empty <string>()
: depMetadata.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
if (!ShouldCompile(srcFile, objFile, depFiles, out string reason))
{
Log.LogMessage(MessageImportance.Low, $"Skipping {srcFile} because {reason}.");
}
else
{
Log.LogMessage(MessageImportance.Low, $"Compiling {srcFile} because {reason}.");
filesToCompile.Add((srcFile, objFile));
}
}
_numCompiled = SourceFiles.Length - filesToCompile.Count;
if (_numCompiled == _totalFiles)
{
// nothing to do!
return(true);
}
if (_numCompiled > 0)
{
Log.LogMessage(MessageImportance.High, $"[{_numCompiled}/{SourceFiles.Length}] skipped unchanged files");
}
Log.LogMessage(MessageImportance.Low, "Using environment variables:");
foreach (var kvp in envVarsDict)
{
Log.LogMessage(MessageImportance.Low, $"\t{kvp.Key} = {kvp.Value}");
}
string workingDir = Environment.CurrentDirectory;
Log.LogMessage(MessageImportance.Low, $"Using working directory: {workingDir}");
_tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(_tempPath);
int allowedParallelism = Math.Min(SourceFiles.Length, Environment.ProcessorCount);
if (BuildEngine is IBuildEngine9 be9)
{
allowedParallelism = be9.RequestCores(allowedParallelism);
}
if (DisableParallelCompile || allowedParallelism == 1)
{
foreach ((string srcFile, string outFile) in filesToCompile)
{
if (!ProcessSourceFile(srcFile, outFile))
{
return(false);
}
}
}
else
{
ParallelLoopResult result = Parallel.ForEach(filesToCompile,
new ParallelOptions {
MaxDegreeOfParallelism = allowedParallelism
},
(toCompile, state) =>
{
if (!ProcessSourceFile(toCompile.Item1, toCompile.Item2))
{
state.Stop();
}
});
if (!result.IsCompleted && !Log.HasLoggedErrors)
{
Log.LogError("Unknown failed occured while compiling");
}
}
if (!Log.HasLoggedErrors)
{
int numUnchanged = _totalFiles - _numCompiled;
if (numUnchanged > 0)
{
Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalFiles}] unchanged.");
}
}
}
finally
{
if (!string.IsNullOrEmpty(_tempPath))
{
Directory.Delete(_tempPath, true);
}
}
OutputFiles = outputItems.ToArray();
return(!Log.HasLoggedErrors);
bool ProcessSourceFile(string srcFile, string objFile)
{
string tmpObjFile = Path.GetTempFileName();
try
{
string command = $"emcc {Arguments} -c -o \"{tmpObjFile}\" \"{srcFile}\"";
var startTime = DateTime.Now;
// Log the command in a compact format which can be copy pasted
StringBuilder envStr = new StringBuilder(string.Empty);
foreach (var key in envVarsDict.Keys)
{
envStr.Append($"{key}={envVarsDict[key]} ");
}
Log.LogMessage(MessageImportance.Low, $"Exec: {envStr}{command}");
(int exitCode, string output) = Utils.RunShellCommand(
Log,
command,
envVarsDict,
workingDir: Environment.CurrentDirectory,
logStdErrAsMessage: true,
debugMessageImportance: messageImportance,
label: Path.GetFileName(srcFile));
var endTime = DateTime.Now;
var elapsedSecs = (endTime - startTime).TotalSeconds;
if (exitCode != 0)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{output} [took {elapsedSecs:F}s]");
return(false);
}
if (!Utils.CopyIfDifferent(tmpObjFile, objFile, useHash: true))
{
Log.LogMessage(MessageImportance.Low, $"Did not overwrite {objFile} as the contents are unchanged");
}
else
{
Log.LogMessage(MessageImportance.Low, $"Copied {tmpObjFile} to {objFile}");
}
ITaskItem newItem = new TaskItem(objFile);
newItem.SetMetadata("SourceFile", srcFile);
outputItems.Add(newItem);
int count = Interlocked.Increment(ref _numCompiled);
Log.LogMessage(MessageImportance.High, $"[{count}/{_totalFiles}] {Path.GetFileName(srcFile)} -> {Path.GetFileName(objFile)} [took {elapsedSecs:F}s]");
return(!Log.HasLoggedErrors);
}
catch (Exception ex)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{ex.Message}");
return(false);
}
finally
{
File.Delete(tmpObjFile);
}
}
}
public override bool Execute()
{
if (SourceFiles.Length == 0)
{
Log.LogError($"No SourceFiles to compile");
return(false);
}
ITaskItem?badItem = SourceFiles.FirstOrDefault(sf => string.IsNullOrEmpty(sf.GetMetadata("ObjectFile")));
if (badItem != null)
{
Log.LogError($"Source file {badItem.ItemSpec} is missing ObjectFile metadata.");
return(false);
}
IDictionary <string, string> envVarsDict = GetEnvironmentVariablesDict();
ConcurrentBag <ITaskItem> outputItems = new();
try
{
Log.LogMessage(MessageImportance.Low, "Using environment variables:");
foreach (var kvp in envVarsDict)
{
Log.LogMessage(MessageImportance.Low, $"\t{kvp.Key} = {kvp.Value}");
}
string workingDir = Environment.CurrentDirectory;
Log.LogMessage(MessageImportance.Low, $"Using working directory: {workingDir}");
_tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(_tempPath);
int allowedParallelism = Math.Min(SourceFiles.Length, Environment.ProcessorCount);
#if false // Enable this when we bump msbuild to 16.1.0
if (BuildEngine is IBuildEngine9 be9)
{
allowedParallelism = be9.RequestCores(allowedParallelism);
}
#endif
if (DisableParallelCompile || allowedParallelism == 1)
{
foreach (ITaskItem srcItem in SourceFiles)
{
if (!ProcessSourceFile(srcItem))
{
return(false);
}
}
}
else
{
ParallelLoopResult result = Parallel.ForEach(SourceFiles,
new ParallelOptions {
MaxDegreeOfParallelism = allowedParallelism
},
(srcItem, state) =>
{
if (!ProcessSourceFile(srcItem))
{
state.Stop();
}
});
if (!result.IsCompleted && !Log.HasLoggedErrors)
{
Log.LogError("Unknown failed occured while compiling");
}
}
}
finally
{
if (!string.IsNullOrEmpty(_tempPath))
{
Directory.Delete(_tempPath, true);
}
}
OutputFiles = outputItems.ToArray();
return(!Log.HasLoggedErrors);
bool ProcessSourceFile(ITaskItem srcItem)
{
string srcFile = srcItem.ItemSpec;
string objFile = srcItem.GetMetadata("ObjectFile");
try
{
string command = $"emcc {Arguments} -c -o {objFile} {srcFile}";
// Log the command in a compact format which can be copy pasted
StringBuilder envStr = new StringBuilder(string.Empty);
foreach (var key in envVarsDict.Keys)
{
envStr.Append($"{key}={envVarsDict[key]} ");
}
Log.LogMessage(MessageImportance.Low, $"Exec: {envStr}{command}");
(int exitCode, string output) = Utils.RunShellCommand(command, envVarsDict, workingDir: Environment.CurrentDirectory);
if (exitCode != 0)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{output}");
return(false);
}
ITaskItem newItem = new TaskItem(objFile);
newItem.SetMetadata("SourceFile", srcFile);
outputItems.Add(newItem);
return(true);
}
catch (Exception ex)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{ex.Message}");
return(false);
}
}
}
