protected GeneratePackageAppxManifest GetNewTask(string manifest,
string?generatedFilename = null,
string?guid = null,
string?displayVersion = null,
string?version = null,
string?displayName = null,
ITaskItem?appIcon = null,
ITaskItem?splashScreen = null) =>
/// <summary>
/// Main entry point.
/// </summary>
public override bool Execute()
{
if (String.IsNullOrEmpty(this.Language))
{
this.Log.LogErrorWithCodeFromResources("General.InvalidValue", nameof(this.Language), "WriteThisAssemblyCodeFile");
return(false);
}
if (this.OutputFile == null && this.OutputDirectory == null)
{
this.Log.LogErrorWithCodeFromResources("WriteCodeFragment.MustSpecifyLocation");
return(false);
}
var code = this.GenerateCode(out var extension);
if (this.Log.HasLoggedErrors)
{
return(false);
}
if (code.Length == 0)
{
this.Log.LogMessageFromResources(MessageImportance.Low, "WriteCodeFragment.NoWorkToDo");
this.OutputFile = null;
return(true);
}
try
{
if (this.OutputFile != null && this.OutputDirectory != null && !Path.IsPathRooted(this.OutputFile.ItemSpec))
{
this.OutputFile = new TaskItem(Path.Combine(this.OutputDirectory.ItemSpec, this.OutputFile.ItemSpec));
}
this.OutputFile ??= new TaskItem(FileUtilities.GetTemporaryFile(this.OutputDirectory !.ItemSpec, extension));
File.WriteAllText(this.OutputFile.ItemSpec, code); // Overwrites file if it already exists (and can be overwritten)
}
catch (Exception ex) when(ExceptionHandling.IsIoRelatedException(ex))
{
this.Log.LogErrorWithCodeFromResources("WriteCodeFragment.CouldNotWriteOutput", (this.OutputFile == null) ? String.Empty : this.OutputFile.ItemSpec, ex.Message);
return(false);
}
this.Log.LogMessageFromResources(MessageImportance.Low, "WriteCodeFragment.GeneratedFile", this.OutputFile.ItemSpec);
return(!this.Log.HasLoggedErrors);
}
private Version GetVersion(ITaskItem?hostItem)
{
var versionAsString = hostItem?.GetMetadata(VersionMetadataName);
if (!NullableString.IsNullOrEmpty(versionAsString))
{
if (Version.TryParse(versionAsString, out var version))
{
return(version);
}
Log.LogError(CommonResources.ItemOfItemGroupMustSpecifyMetadata, hostItem !.ItemSpec, HostsItemGroupName, VersionMetadataName);
}
return(s_versionWithNewUrlFormat);
}
protected override string?BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem?hostItem)
{
var virtualDirectory = hostItem?.GetMetadata(VirtualDirectoryMetadataName) ?? "";
if (!AzureDevOpsUrlParser.TryParseOnPremHttp(relativeUrl, virtualDirectory, out var projectPath, out var repositoryName))
{
Log.LogError(CommonResources.ValueOfWithIdentityIsInvalid, Names.SourceRoot.RepositoryUrlFullName, SourceRoot.ItemSpec, gitUri);
return(null);
}
return
(UriUtilities.Combine(
UriUtilities.Combine(contentUri.ToString(), projectPath), $"_apis/git/repositories/{repositoryName}/items") +
$"?api-version=1.0&versionType=commit&version={revisionId}&path=/*");
}
private bool ExecuteActual()
{
if (SourceFiles.Length == 0)
{
Log.LogError($"No SourceFiles to compile");
return(false);
}
ITaskItem?badItem = SourceFiles.FirstOrDefault(sf => string.IsNullOrEmpty(sf.GetMetadata("ObjectFile")));
if (badItem != null)
{
Log.LogError($"Source file {badItem.ItemSpec} is missing ObjectFile metadata.");
return(false);
}
if (!Enum.TryParse(OutputMessageImportance, ignoreCase: true, out MessageImportance messageImportance))
{
Log.LogError($"Invalid value for OutputMessageImportance={OutputMessageImportance}. Valid values: {string.Join(", ", Enum.GetNames(typeof(MessageImportance)))}");
return(false);
}
_totalFiles = SourceFiles.Length;
IDictionary <string, string> envVarsDict = GetEnvironmentVariablesDict();
ConcurrentBag <ITaskItem> outputItems = new();
try
{
List <(string, string)> filesToCompile = new();
foreach (ITaskItem srcItem in SourceFiles)
{
string srcFile = srcItem.ItemSpec;
string objFile = srcItem.GetMetadata("ObjectFile");
string depMetadata = srcItem.GetMetadata("Dependencies");
string[] depFiles = string.IsNullOrEmpty(depMetadata)
? Array.Empty <string>()
: depMetadata.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
if (!ShouldCompile(srcFile, objFile, depFiles, out string reason))
{
Log.LogMessage(MessageImportance.Low, $"Skipping {srcFile} because {reason}.");
outputItems.Add(CreateOutputItemFor(srcFile, objFile));
}
else
{
Log.LogMessage(MessageImportance.Low, $"Compiling {srcFile} because {reason}.");
filesToCompile.Add((srcFile, objFile));
}
}
_numCompiled = SourceFiles.Length - filesToCompile.Count;
if (_numCompiled == _totalFiles)
{
// nothing to do!
OutputFiles = outputItems.ToArray();
return(!Log.HasLoggedErrors);
}
if (_numCompiled > 0)
{
Log.LogMessage(MessageImportance.High, $"[{_numCompiled}/{SourceFiles.Length}] skipped unchanged files");
}
Log.LogMessage(MessageImportance.Low, "Using environment variables:");
foreach (var kvp in envVarsDict)
{
Log.LogMessage(MessageImportance.Low, $"\t{kvp.Key} = {kvp.Value}");
}
string workingDir = Environment.CurrentDirectory;
Log.LogMessage(MessageImportance.Low, $"Using working directory: {workingDir}");
_tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(_tempPath);
int allowedParallelism = DisableParallelCompile ? 1 : Math.Min(SourceFiles.Length, Environment.ProcessorCount);
if (BuildEngine is IBuildEngine9 be9)
{
allowedParallelism = be9.RequestCores(allowedParallelism);
}
/*
* From: https://github.com/dotnet/runtime/issues/46146#issuecomment-754021690
*
* Stephen Toub:
* "As such, by default ForEach works on a scheme whereby each
* thread takes one item each time it goes back to the enumerator,
* and then after a few times of this upgrades to taking two items
* each time it goes back to the enumerator, and then four, and
* then eight, and so on. This amortizes the cost of taking and
* releasing the lock across multiple items, while still enabling
* parallelization for enumerables containing just a few items. It
* does, however, mean that if you've got a case where the body
* takes a really long time and the work for every item is
* heterogeneous, you can end up with an imbalance."
*
* The time taken by individual compile jobs here can vary a
* lot, depending on various factors like file size. This can
* create an imbalance, like mentioned above, and we can end up
* in a situation where one of the partitions has a job that
* takes very long to execute, by which time other partitions
* have completed, so some cores are idle. But the idle
* ones won't get any of the remaining jobs, because they are
* all assigned to that one partition.
*
* Instead, we want to use work-stealing so jobs can be run by any partition.
*/
ParallelLoopResult result = Parallel.ForEach(
Partitioner.Create(filesToCompile, EnumerablePartitionerOptions.NoBuffering),
new ParallelOptions {
MaxDegreeOfParallelism = allowedParallelism
},
(toCompile, state) =>
{
if (!ProcessSourceFile(toCompile.Item1, toCompile.Item2))
{
state.Stop();
}
});
if (!result.IsCompleted && !Log.HasLoggedErrors)
{
Log.LogError("Unknown failure occurred while compiling. Check logs to get more details.");
}
if (!Log.HasLoggedErrors)
{
int numUnchanged = _totalFiles - _numCompiled;
if (numUnchanged > 0)
{
Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalFiles}] unchanged.");
}
}
}
finally
{
if (!string.IsNullOrEmpty(_tempPath))
{
Directory.Delete(_tempPath, true);
}
}
OutputFiles = outputItems.ToArray();
return(!Log.HasLoggedErrors);
bool ProcessSourceFile(string srcFile, string objFile)
{
string tmpObjFile = Path.GetTempFileName();
try
{
string command = $"emcc {Arguments} -c -o \"{tmpObjFile}\" \"{srcFile}\"";
var startTime = DateTime.Now;
// Log the command in a compact format which can be copy pasted
StringBuilder envStr = new StringBuilder(string.Empty);
foreach (var key in envVarsDict.Keys)
{
envStr.Append($"{key}={envVarsDict[key]} ");
}
Log.LogMessage(MessageImportance.Low, $"Exec: {envStr}{command}");
(int exitCode, string output) = Utils.RunShellCommand(
Log,
command,
envVarsDict,
workingDir: Environment.CurrentDirectory,
logStdErrAsMessage: true,
debugMessageImportance: messageImportance,
label: Path.GetFileName(srcFile));
var endTime = DateTime.Now;
var elapsedSecs = (endTime - startTime).TotalSeconds;
if (exitCode != 0)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{output} [took {elapsedSecs:F}s]");
return(false);
}
if (!Utils.CopyIfDifferent(tmpObjFile, objFile, useHash: true))
{
Log.LogMessage(MessageImportance.Low, $"Did not overwrite {objFile} as the contents are unchanged");
}
else
{
Log.LogMessage(MessageImportance.Low, $"Copied {tmpObjFile} to {objFile}");
}
outputItems.Add(CreateOutputItemFor(srcFile, objFile));
int count = Interlocked.Increment(ref _numCompiled);
Log.LogMessage(MessageImportance.High, $"[{count}/{_totalFiles}] {Path.GetFileName(srcFile)} -> {Path.GetFileName(objFile)} [took {elapsedSecs:F}s]");
return(!Log.HasLoggedErrors);
}
catch (Exception ex)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{ex.Message}");
return(false);
}
finally
{
File.Delete(tmpObjFile);
}
}
ITaskItem CreateOutputItemFor(string srcFile, string objFile)
{
ITaskItem newItem = new TaskItem(objFile);
newItem.SetMetadata("SourceFile", srcFile);
return(newItem);
}
}
public override bool Execute()
{
if (SourceFiles.Length == 0)
{
Log.LogError($"No SourceFiles to compile");
return(false);
}
ITaskItem?badItem = SourceFiles.FirstOrDefault(sf => string.IsNullOrEmpty(sf.GetMetadata("ObjectFile")));
if (badItem != null)
{
Log.LogError($"Source file {badItem.ItemSpec} is missing ObjectFile metadata.");
return(false);
}
if (!Enum.TryParse(OutputMessageImportance, ignoreCase: true, out MessageImportance messageImportance))
{
Log.LogError($"Invalid value for OutputMessageImportance={OutputMessageImportance}. Valid values: {string.Join(", ", Enum.GetNames(typeof(MessageImportance)))}");
return(false);
}
_totalFiles = SourceFiles.Length;
IDictionary <string, string> envVarsDict = GetEnvironmentVariablesDict();
ConcurrentBag <ITaskItem> outputItems = new();
try
{
List <(string, string)> filesToCompile = new();
foreach (ITaskItem srcItem in SourceFiles)
{
string srcFile = srcItem.ItemSpec;
string objFile = srcItem.GetMetadata("ObjectFile");
string depMetadata = srcItem.GetMetadata("Dependencies");
string[] depFiles = string.IsNullOrEmpty(depMetadata)
? Array.Empty <string>()
: depMetadata.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
if (!ShouldCompile(srcFile, objFile, depFiles, out string reason))
{
Log.LogMessage(MessageImportance.Low, $"Skipping {srcFile} because {reason}.");
}
else
{
Log.LogMessage(MessageImportance.Low, $"Compiling {srcFile} because {reason}.");
filesToCompile.Add((srcFile, objFile));
}
}
_numCompiled = SourceFiles.Length - filesToCompile.Count;
if (_numCompiled == _totalFiles)
{
// nothing to do!
return(true);
}
if (_numCompiled > 0)
{
Log.LogMessage(MessageImportance.High, $"[{_numCompiled}/{SourceFiles.Length}] skipped unchanged files");
}
Log.LogMessage(MessageImportance.Low, "Using environment variables:");
foreach (var kvp in envVarsDict)
{
Log.LogMessage(MessageImportance.Low, $"\t{kvp.Key} = {kvp.Value}");
}
string workingDir = Environment.CurrentDirectory;
Log.LogMessage(MessageImportance.Low, $"Using working directory: {workingDir}");
_tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(_tempPath);
int allowedParallelism = Math.Min(SourceFiles.Length, Environment.ProcessorCount);
if (BuildEngine is IBuildEngine9 be9)
{
allowedParallelism = be9.RequestCores(allowedParallelism);
}
if (DisableParallelCompile || allowedParallelism == 1)
{
foreach ((string srcFile, string outFile) in filesToCompile)
{
if (!ProcessSourceFile(srcFile, outFile))
{
return(false);
}
}
}
else
{
ParallelLoopResult result = Parallel.ForEach(filesToCompile,
new ParallelOptions {
MaxDegreeOfParallelism = allowedParallelism
},
(toCompile, state) =>
{
if (!ProcessSourceFile(toCompile.Item1, toCompile.Item2))
{
state.Stop();
}
});
if (!result.IsCompleted && !Log.HasLoggedErrors)
{
Log.LogError("Unknown failed occured while compiling");
}
}
if (!Log.HasLoggedErrors)
{
int numUnchanged = _totalFiles - _numCompiled;
if (numUnchanged > 0)
{
Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalFiles}] unchanged.");
}
}
}
finally
{
if (!string.IsNullOrEmpty(_tempPath))
{
Directory.Delete(_tempPath, true);
}
}
OutputFiles = outputItems.ToArray();
return(!Log.HasLoggedErrors);
bool ProcessSourceFile(string srcFile, string objFile)
{
string tmpObjFile = Path.GetTempFileName();
try
{
string command = $"emcc {Arguments} -c -o \"{tmpObjFile}\" \"{srcFile}\"";
var startTime = DateTime.Now;
// Log the command in a compact format which can be copy pasted
StringBuilder envStr = new StringBuilder(string.Empty);
foreach (var key in envVarsDict.Keys)
{
envStr.Append($"{key}={envVarsDict[key]} ");
}
Log.LogMessage(MessageImportance.Low, $"Exec: {envStr}{command}");
(int exitCode, string output) = Utils.RunShellCommand(
Log,
command,
envVarsDict,
workingDir: Environment.CurrentDirectory,
logStdErrAsMessage: true,
debugMessageImportance: messageImportance,
label: Path.GetFileName(srcFile));
var endTime = DateTime.Now;
var elapsedSecs = (endTime - startTime).TotalSeconds;
if (exitCode != 0)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{output} [took {elapsedSecs:F}s]");
return(false);
}
if (!Utils.CopyIfDifferent(tmpObjFile, objFile, useHash: true))
{
Log.LogMessage(MessageImportance.Low, $"Did not overwrite {objFile} as the contents are unchanged");
}
else
{
Log.LogMessage(MessageImportance.Low, $"Copied {tmpObjFile} to {objFile}");
}
ITaskItem newItem = new TaskItem(objFile);
newItem.SetMetadata("SourceFile", srcFile);
outputItems.Add(newItem);
int count = Interlocked.Increment(ref _numCompiled);
Log.LogMessage(MessageImportance.High, $"[{count}/{_totalFiles}] {Path.GetFileName(srcFile)} -> {Path.GetFileName(objFile)} [took {elapsedSecs:F}s]");
return(!Log.HasLoggedErrors);
}
catch (Exception ex)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{ex.Message}");
return(false);
}
finally
{
File.Delete(tmpObjFile);
}
}
}
protected abstract string?BuildSourceLinkUrl(Uri contentUrl, Uri gitUri, string relativeUrl, string revisionId, ITaskItem?hostItem);
protected override string?BuildSourceLinkUrl(Uri contentUrl, Uri gitUri, string relativeUrl, string revisionId, ITaskItem?hostItem)
=> $"ContentUrl='{contentUrl}' GitUrl='{gitUri}' RelativeUrl='{relativeUrl}' RevisionId='{revisionId}'";
protected override string BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem?hostItem)
{
var trimLeadingSlash = relativeUrl.TrimStart('/');
var trimmedContentUrl = contentUri.ToString().TrimEnd('/', '\\');
// p = project/path
// a = action
// hb = SHA/revision
// f = repo file path
var gitwebRawUrl = UriUtilities.Combine(trimmedContentUrl, $"?p={trimLeadingSlash}.git;a=blob_plain;hb={revisionId};f=*");
return(gitwebRawUrl);
}
public override bool Execute()
{
if (SourceFiles.Length == 0)
{
Log.LogError($"No SourceFiles to compile");
return(false);
}
ITaskItem?badItem = SourceFiles.FirstOrDefault(sf => string.IsNullOrEmpty(sf.GetMetadata("ObjectFile")));
if (badItem != null)
{
Log.LogError($"Source file {badItem.ItemSpec} is missing ObjectFile metadata.");
return(false);
}
IDictionary <string, string> envVarsDict = GetEnvironmentVariablesDict();
ConcurrentBag <ITaskItem> outputItems = new();
try
{
Log.LogMessage(MessageImportance.Low, "Using environment variables:");
foreach (var kvp in envVarsDict)
{
Log.LogMessage(MessageImportance.Low, $"\t{kvp.Key} = {kvp.Value}");
}
string workingDir = Environment.CurrentDirectory;
Log.LogMessage(MessageImportance.Low, $"Using working directory: {workingDir}");
_tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(_tempPath);
int allowedParallelism = Math.Min(SourceFiles.Length, Environment.ProcessorCount);
#if false // Enable this when we bump msbuild to 16.1.0
if (BuildEngine is IBuildEngine9 be9)
{
allowedParallelism = be9.RequestCores(allowedParallelism);
}
#endif
if (DisableParallelCompile || allowedParallelism == 1)
{
foreach (ITaskItem srcItem in SourceFiles)
{
if (!ProcessSourceFile(srcItem))
{
return(false);
}
}
}
else
{
ParallelLoopResult result = Parallel.ForEach(SourceFiles,
new ParallelOptions {
MaxDegreeOfParallelism = allowedParallelism
},
(srcItem, state) =>
{
if (!ProcessSourceFile(srcItem))
{
state.Stop();
}
});
if (!result.IsCompleted && !Log.HasLoggedErrors)
{
Log.LogError("Unknown failed occured while compiling");
}
}
}
finally
{
if (!string.IsNullOrEmpty(_tempPath))
{
Directory.Delete(_tempPath, true);
}
}
OutputFiles = outputItems.ToArray();
return(!Log.HasLoggedErrors);
bool ProcessSourceFile(ITaskItem srcItem)
{
string srcFile = srcItem.ItemSpec;
string objFile = srcItem.GetMetadata("ObjectFile");
try
{
string command = $"emcc {Arguments} -c -o {objFile} {srcFile}";
// Log the command in a compact format which can be copy pasted
StringBuilder envStr = new StringBuilder(string.Empty);
foreach (var key in envVarsDict.Keys)
{
envStr.Append($"{key}={envVarsDict[key]} ");
}
Log.LogMessage(MessageImportance.Low, $"Exec: {envStr}{command}");
(int exitCode, string output) = Utils.RunShellCommand(command, envVarsDict, workingDir: Environment.CurrentDirectory);
if (exitCode != 0)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{output}");
return(false);
}
ITaskItem newItem = new TaskItem(objFile);
newItem.SetMetadata("SourceFile", srcFile);
outputItems.Add(newItem);
return(true);
}
catch (Exception ex)
{
Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{ex.Message}");
return(false);
}
}
}
protected override string?BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem?hostItem) => UriUtilities.Combine(UriUtilities.Combine(contentUri.ToString(), relativeUrl), revisionId + "/*");
protected override string?BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem?hostItem)
{
// The SourceLinkBitbucketGitHost item for bitbucket.org specifies EnterpriseEdition="false".
// Other items that may be specified by the project default to EnterpriseEdition="true" without specifying it.
bool isCloud = bool.TryParse(hostItem?.GetMetadata(IsEnterpriseEditionMetadataName), out var isEnterpriseEdition) && !isEnterpriseEdition;
if (isCloud)
{
return(BuildSourceLinkUrlForCloudEdition(contentUri, relativeUrl, revisionId));
}
if (TryParseEnterpriseUrl(relativeUrl, out var relativeBaseUrl, out var projectName, out var repositoryName))
{
var version = GetBitbucketEnterpriseVersion(hostItem);
return(BuildSourceLinkUrlForEnterpriseEdition(contentUri, relativeBaseUrl, projectName, repositoryName, revisionId, version));
}
Log.LogError(CommonResources.ValueOfWithIdentityIsInvalid, Names.SourceRoot.RepositoryUrlFullName, SourceRoot.ItemSpec, gitUri);
return(null);
}
protected override string?BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem?hostItem)
{
var path = GetVersion(hostItem) >= s_versionWithNewUrlFormat
? "-/raw/" + revisionId + "/*"
: "raw/" + revisionId + "/*";
return(UriUtilities.Combine(UriUtilities.Combine(contentUri.ToString(), relativeUrl), path));
}