/// <summary>
/// Parses template literal fragment like "/literal" fragment in the string <code>"${expression}/literal"</code>.
/// </summary>
protected override ITemplateLiteralFragment ParseTemplateLiteralFragment()
{
var node = CreateNodeAndSetParent <StringIdTemplateLiteralFragment>(m_token);
// TODO: need to change the scanner to use StringId instead of string.
// Or, maybe, switch to ReadOnlySpan<T>.
var text = StringId.Create(m_pathTable.StringTable, m_scanner.TokenValue);
if (!text.IsValid)
{
Contract.Assert(false, I($"Can't create StringId from '{m_scanner.TokenValue}'."));
}
node.TextAsStringId = text;
if (m_scanner.HasExtendedUnicodeEscape)
{
node.HasExtendedUnicodeEscape = true;
}
if (m_scanner.IsUnterminated)
{
node.IsUnterminated = true;
}
NextToken();
FinishNode(node);
return(node);
}
public void CompoundFilterPrecedence()
{
// And has greater precedence than or.
RootFilter expected = new RootFilter(
new BinaryFilter(
new BinaryFilter(
new TagFilter(StringId.Create(StringTable, "Test")),
FilterOperator.And,
new TagFilter(StringId.Create(StringTable, "csc.exe"))),
FilterOperator.Or,
new TagFilter(StringId.Create(StringTable, "Test2"))));
RunPositiveFilterParserTest("tag=\'Test\'andtag=\'csc.exe\'ortag=\'Test2\'", expected);
RootFilter expected2 = new RootFilter(
new BinaryFilter(
new TagFilter(StringId.Create(StringTable, "Test2")),
FilterOperator.Or,
new BinaryFilter(
new TagFilter(StringId.Create(StringTable, "Test")),
FilterOperator.And,
new TagFilter(StringId.Create(StringTable, "csc.exe")))));
RunPositiveFilterParserTest("tag=\'Test2\'ortag=\'csc.exe\'andtag=\'Test\'", expected2);
}
public void NestedDependencyBinaryFilter()
{
RootFilter expected = new RootFilter(
new BinaryFilter(
new DependenciesFilter(
new BinaryFilter(
new TagFilter(StringId.Create(StringTable, "Test1")),
FilterOperator.And,
new TagFilter(StringId.Create(StringTable, "Test2")))),
FilterOperator.And,
new TagFilter(StringId.Create(StringTable, "Test3"))));
RunPositiveFilterParserTest(" ( dpc( tag = \'Test1\' and tag = \'Test2\' ) and tag = \'Test3\' )", expected);
expected = new RootFilter(
new BinaryFilter(
new TagFilter(StringId.Create(StringTable, "Test1")),
FilterOperator.And,
new DependenciesFilter(
new BinaryFilter(
new TagFilter(StringId.Create(StringTable, "Test2")),
FilterOperator.And,
new TagFilter(StringId.Create(StringTable, "Test3"))))));
RunPositiveFilterParserTest(" ( tag = \'Test1\' and dpc( tag = \'Test2\' and tag = \'Test3\' ) )", expected);
}
public void SingleFilterCopyDependents()
{
RootFilter expected = new RootFilter(
new CopyDependentsFilter(new TagFilter(StringId.Create(StringTable, "Test"))));
RunPositiveFilterParserTest("copydpt(tag=\'Test\')", expected);
}
public void TagFilter()
{
RootFilter expected = new RootFilter(
new TagFilter(StringId.Create(StringTable, "Test")));
RunPositiveFilterParserTest("tag=\'Test\'", expected);
}
/// <summary>
/// Creates a package id.
/// </summary>
public static PackageId Create(StringTable stringTable, string name)
{
Contract.Requires(stringTable != null, "stringTable != null");
Contract.Requires(name != null, "name != null");
return(new PackageId(StringId.Create(stringTable, name), PackageVersion.None));
}
public void ComboTest()
{
AbsolutePath.TryCreate(PathTable, X("/d/nonsensePath"), out AbsolutePath path);
var stringId = StringId.Create(PathTable.StringTable, "testString");
var fingerprint = FingerprintUtilities.ZeroFingerprint;
var hash = ContentHashingUtilities.CreateRandom();
var longValue = (long)23;
var stringValue = "asdf";
var intCollection = new int[] { 1, 2, 3 };
// Do some random operations and make sure there's no exceptions
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add(stringValue, longValue);
writer.AddCollection <int, int[]>("ints", intCollection, (w, e) => w.Add(e));
writer.Add(stringValue, hash);
writer.Add(stringValue, stringId);
writer.Add(stringValue, stringId);
writer.Add(stringValue, path, hash);
}
VerifyJsonContent("testString");
VerifyJsonContent(longValue.ToString());
VerifyJsonContent(intCollection.Select(i => i.ToString()).ToArray());
VerifyJsonContent(JsonFingerprinter.ContentHashToString(hash));
}
/// <summary>
/// Creates the default regular expression descriptor (for warnings).
/// </summary>
public static RegexDescriptor CreateDefaultForWarnings(StringTable stringTable)
{
Contract.Requires(stringTable != null, "stringTable can't be null.");
return(new RegexDescriptor(
StringId.Create(stringTable, DefaultWarningPattern),
DefaultOptions));
}
public override int Analyze()
{
var cscExePathAtom = PathAtom.Create(StringTable, "csc.exe");
var cscLookupBySpec = CachedGraph.PipGraph.RetrievePipReferencesOfType(PipType.Process)
.Where(lazyPip => m_executedPipsTracker.WasVisited(lazyPip.PipId.ToNodeId()))
.Select(lazyPip => (Process)lazyPip.HydratePip())
.Where(proc => proc.Executable.Path.GetName(PathTable).CaseInsensitiveEquals(StringTable, cscExePathAtom))
.ToLookup(process => process.Provenance.Token.Path);
var newlineSeparator = StringId.Create(StringTable, Environment.NewLine);
foreach (var specGroup in cscLookupBySpec)
{
var specName = specGroup.Key.GetName(PathTable).ToString(StringTable);
var specOutDirectory = Path.Combine(OutputDirectory, specName);
var specPath = specGroup.Key.ToString(PathTable);
Directory.CreateDirectory(specOutDirectory);
Console.WriteLine(I($"Processing: {specPath}"));
foreach (var cscInvocation in specGroup)
{
var targetPath = Path.Combine(specOutDirectory, cscInvocation.FormattedSemiStableHash + ".csc.args.txt");
Console.WriteLine(I($"Emitting: {targetPath}"));
using (var writer = new StreamWriter(targetPath))
{
writer.WriteLine(I($"Project={specPath}"));
var arguments = GetArgumentsDataFromProcess(cscInvocation)
.With(newlineSeparator, PipDataFragmentEscaping.NoEscaping);
writer.Write(arguments.ToString(PathTable));
}
}
}
return(0);
}
/// <summary>
/// Schedules an IPC pip.
/// </summary>
public IpcPip ScheduleIpcPip(
IIpcMoniker moniker,
PipId?servicePipId,
ProcessBuilder ipcProcessBuilder,
FileArtifact outputFile,
bool isServiceFinalization,
PipConstructionHelper helper = null)
{
var ipcClientInfo = new IpcClientInfo(StringId.Create(Context.StringTable, moniker.Id), new ClientConfig(0, 0));
PipData arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
ReadOnlyArray <FileArtifact> fileDependencies = ipcProcessBuilder.GetInputFilesSoFar();
if (!(helper ?? m_defaultConstructionHelper).TryAddIpc(
ipcClientInfo,
arguments,
outputFile,
servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty,
fileDependencies: fileDependencies,
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .Empty,
isServiceFinalization: isServiceFinalization,
mustRunOnOrchestrator: false,
tags: new string[0],
out IpcPip ipcPip))
{
throw new BuildXLTestException("Failed to add ipc pip");
}
return(ipcPip);
}
private void SetProcessEnvironmentVariables(IReadOnlyDictionary <string, string> environment, ProcessBuilder processBuilder)
{
foreach (KeyValuePair <string, string> kvp in environment)
{
if (kvp.Value != null)
{
var envPipData = new PipDataBuilder(m_context.StringTable);
// Casing for paths is not stable as reported by BuildPrediction. So here we try to guess if the value
// represents a path, and normalize it
string value = kvp.Value;
if (!string.IsNullOrEmpty(value) && AbsolutePath.TryCreate(PathTable, value, out var absolutePath))
{
envPipData.Add(absolutePath);
}
else
{
envPipData.Add(value);
}
processBuilder.SetEnvironmentVariable(
StringId.Create(m_context.StringTable, kvp.Key),
envPipData.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
}
}
public void Test1(bool useStdErr, string text, string errRegex, bool enableMultiLineScanning, string expectedPrintedError)
{
EventListener.NestedLoggerHandler += eventData =>
{
if (eventData.EventId == (int)LogEventId.PipProcessError)
{
var loggedMessage = eventData.Payload.ToArray()[5].ToString();
m_loggedPipFailures.Add(loggedMessage);
}
};
var ops = SplitLines(text)
.Select(l => Operation.Echo(l, useStdErr))
.Concat(new[]
{
Operation.WriteFile(CreateOutputFileArtifact()),
Operation.Fail()
});
var pipBuilder = CreatePipBuilder(ops);
pipBuilder.ErrorRegex = new RegexDescriptor(StringId.Create(Context.StringTable, errRegex), RegexOptions.None);
pipBuilder.EnableMultiLineErrorScanning = enableMultiLineScanning;
SchedulePipBuilder(pipBuilder);
RunScheduler().AssertFailure();
AssertErrorEventLogged(LogEventId.PipProcessError);
XAssert.ArrayEqual(
SplitLines(expectedPrintedError),
m_loggedPipFailures.SelectMany(SplitLines).ToArray());
}
/// <summary>
/// Converts a string to a StringId
/// </summary>
/// <param name="stringTable">StringTable to use to convert the string</param>
/// <param name="str">String to convert to StringId</param>
/// <returns>The result of converting the provided string to a StrindId</returns>
public static StringId StringToId(this StringTable stringTable, string str)
{
// Create will mutate the StringTable if str is not already in the table
// At some point, we should add a TryGet method to StringId because StringTable
// should not mutate as a result of a call to StringToId
return(StringId.Create(stringTable, str));
}
private static EvaluationResult GroupBy(Context context, ArrayLiteral receiver, EvaluationResult arg, EvaluationStackFrame captures)
{
var closure = Converter.ExpectClosure(arg);
using (var disposableFrame = EvaluationStackFrame.Create(closure.Function, captures.Frame))
{
// To avoid warning: access to disposable closure.
var frame = disposableFrame;
var entry = context.TopStack;
var result = receiver.Values.GroupBy(obj =>
{
frame.SetArgument(0, obj);
return(context.InvokeClosure(closure, frame));
});
var arr = result.Select(grp =>
{
var grpArrayLit = ArrayLiteral.CreateWithoutCopy(grp.ToArray(), entry.InvocationLocation, entry.Path);
var bindings = new List <Binding>
{
new Binding(StringId.Create(context.FrontEndContext.StringTable, "key"), grp.Key, location: default(LineInfo)),
new Binding(StringId.Create(context.FrontEndContext.StringTable, "values"), grpArrayLit, location: default(LineInfo)),
};
return(EvaluationResult.Create(ObjectLiteral.Create(bindings, entry.InvocationLocation, entry.Path)));
}).ToArray();
return(EvaluationResult.Create(ArrayLiteral.CreateWithoutCopy(arr, entry.InvocationLocation, entry.Path)));
}
}
public void SingleFilterDoubleNegated()
{
RootFilter expected = new RootFilter(
new TagFilter(StringId.Create(StringTable, "Test")));
RunPositiveFilterParserTest("~(~(tag=\'Test\'))", expected);
}
public PipDataTests(ITestOutputHelper output)
: base(output)
{
m_pathTable = new PathTable();
m_expectedStringId0 = StringId.Create(m_pathTable.StringTable, ExpectedString0);
m_pipDataBuilder = new PipDataBuilder(m_pathTable.StringTable);
m_expectedStringId0 = StringId.Create(m_pathTable.StringTable, ExpectedString0);
m_uniqueEntry0 = AbsolutePath.Create(m_pathTable, A("c", "unique to fragment 0"));
// BEGIN ADDING ARGUMENTS
m_cursorStart = m_pipDataBuilder.CreateCursor();
AddStandardBlock(m_pipDataBuilder);
m_cursor0 = m_pipDataBuilder.CreateCursor();
using (m_pipDataBuilder.StartFragment(Escaping0, m_separator0))
{
m_pipDataBuilder.Add(m_uniqueEntry0);
AddStandardBlock(m_pipDataBuilder);
}
m_cursor1 = m_pipDataBuilder.CreateCursor();
using (m_pipDataBuilder.StartFragment(Escaping1, Separator1))
{
AddStandardBlock(m_pipDataBuilder);
m_pipDataBuilder.Add(UniqueEntry1);
}
m_cursorEnd = m_pipDataBuilder.CreateCursor();
// END ADDING ARGUMENTS
}
public void SingleFilterDependencies()
{
RootFilter expected = new RootFilter(
new DependenciesFilter(new TagFilter(StringId.Create(StringTable, "Test"))));
RunPositiveFilterParserTest("dpc(tag=\'Test\')", expected);
}
public void SpecFilterModuleDottedName()
{
RootFilter expected = new RootFilter(
CreateModuleFilter(StringId.Create(m_context.StringTable, @"My.Test.Module")));
RunPositiveFilterParserTest("module=\'My.Test.Module\'", expected);
}
public void ObservedFileNamesNormalizedTheSameWayInPathsetAndJsonFingerprinter()
{
// This test is guarding codesync between JsonFingerprinter.cs and ObservedPathSet.cs
var fileNames = new string[] { "a", "b", "C" };
var pathTable = new PathTable();
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
observedAccessedFileNames: fileNames,
X("/X/a/b/c"));
var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet);
var sb = new StringBuilder();
using (var writer = new global::BuildXL.Engine.Cache.JsonFingerprinter(sb, pathTable: pathTable))
{
writer.AddCollection <StringId, StringId[]>(
"fileNames",
fileNames.Select(fn => StringId.Create(pathTable.StringTable, fn)).ToArray(),
(w, e) => w.AddFileName(e));
}
var fpOutput = sb.ToString();
XAssert.IsTrue(roundtrip.ObservedAccessedFileNames.All(fileName => fpOutput.Contains($"\"{fileName.ToString(pathTable.StringTable)}\"")));
}
/// <summary>
/// Creates a qualifier space given mappings from keys to list of eligible values.
/// </summary>
public QualifierSpaceId CreateQualifierSpace(IReadOnlyDictionary <string, IReadOnlyList <string> > keyValuesMap)
{
Contract.Requires(keyValuesMap != null);
Contract.Requires(
Contract.ForAll(
keyValuesMap,
kvp => kvp.Key != null && kvp.Value != null && kvp.Value.Count > 0 && Contract.ForAll(kvp.Value, value => value != null)));
Contract.Ensures(IsValidQualifierSpaceId(Contract.Result <QualifierSpaceId>()));
var keyValuesPairIds = new QualifierSpaceEntry[keyValuesMap.Count];
int i = 0;
foreach (var kvp in keyValuesMap)
{
var valuesIds = new StringId[kvp.Value.Count];
for (int j = 0; j < kvp.Value.Count; ++j)
{
valuesIds[j] = StringId.Create(StringTable, kvp.Value[j]);
}
keyValuesPairIds[i] = QualifierSpaceEntry.Create(StringId.Create(StringTable, kvp.Key), valuesIds);
++i;
}
return(CreateQualifierSpace(keyValuesPairIds));
}
public void SingleFilterRequiredInputs()
{
RootFilter expected = new RootFilter(
new DependenciesFilter(new TagFilter(StringId.Create(StringTable, "Test")), ClosureMode.DirectExcludingSelf));
RunPositiveFilterParserTest("requiredfor(tag=\'Test\')", expected);
}
/// <summary>
/// Creates a qualifier space given mappings from keys to list of eligible values.
/// </summary>
public QualifierSpaceId CreateQualifierSpace(params Tuple <string, string[]>[] keyValuesPairs)
{
Contract.Requires(keyValuesPairs != null);
Contract.Requires(
Contract.ForAll(
keyValuesPairs,
pair => pair.Item1 != null && pair.Item2 != null && pair.Item2.Length > 0 && Contract.ForAll(pair.Item2, value => value != null)));
Contract.Ensures(IsValidQualifierSpaceId(Contract.Result <QualifierSpaceId>()));
var keyValuesPairIds = new QualifierSpaceEntry[keyValuesPairs.Length];
for (int i = 0; i < keyValuesPairs.Length; ++i)
{
var kvp = keyValuesPairs[i];
var valuesIds = new StringId[kvp.Item2.Length];
for (int j = 0; j < kvp.Item2.Length; ++j)
{
valuesIds[j] = StringId.Create(StringTable, kvp.Item2[j]);
}
keyValuesPairIds[i] = QualifierSpaceEntry.Create(StringId.Create(StringTable, kvp.Item1), valuesIds);
}
return(CreateQualifierSpace(keyValuesPairIds));
}
public void DifferentOrderInQualifiersResultsInTheSameLogDirectory()
{
var project = CreateProjectWithPredictions("Test.proj");
var key1Key2Qualifier = FrontEndContext.QualifierTable.CreateQualifier(
new Tuple <StringId, StringId>(StringId.Create(StringTable, "key1"), StringId.Create(StringTable, "value1")),
new Tuple <StringId, StringId>(StringId.Create(StringTable, "key2"), StringId.Create(StringTable, "value2")));
var key1Key2Project = Start(currentQualifier: key1Key2Qualifier)
.Add(project)
.ScheduleAll()
.AssertSuccess()
.RetrieveSuccessfulProcess(project);
var key2Key1Qualifier = FrontEndContext.QualifierTable.CreateQualifier(
new Tuple <StringId, StringId>(StringId.Create(StringTable, "key2"), StringId.Create(StringTable, "value2")),
new Tuple <StringId, StringId>(StringId.Create(StringTable, "key1"), StringId.Create(StringTable, "value1")));
var key2Key1Project = Start(currentQualifier: key2Key1Qualifier)
.Add(project)
.ScheduleAll()
.AssertSuccess()
.RetrieveSuccessfulProcess(project);
// File outputs (which includes log files) should be the same, even though the qualifier was built with a different order
Assert.True(key1Key2Project.FileOutputs.SequenceEqual(key2Key1Project.FileOutputs));
}
/// <summary>
/// Factory to create a PipFragment containing a StringId.
/// FOR TESTING PURPOSES ONLY.
/// </summary>
/// <param name="literal">The string literal to insert</param>
/// <param name="stringTable">the associated string table</param>
/// <returns>Fragment with a string literal</returns>
internal static PipFragment FromString(string literal, StringTable stringTable)
{
Contract.Requires(literal != null);
var literalId = StringId.Create(stringTable, literal);
return(new PipFragment(new PipDataEntry[] { literalId }, 0));
}
/// <summary>
/// Configures the process builder to execute the specified commands
/// </summary>
protected virtual void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
JavaScriptProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way
// Otherwise in Windows we force path sets to be all uppercase
processBuilder.Options |= Process.Options.PreservePathSetCasing;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
// Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads,
// any attempt to read into the user profile will fail to compute its corresponding hash
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
// Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen
processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) });
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log"));
processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " "))
{
// Execute the command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand));
// If we need to append arguments to the script command, do it here
if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <JavaScriptArgument> extraArguments))
{
foreach (JavaScriptArgument value in extraArguments)
{
AddJavaScriptArgumentToBuilder(processBuilder.ArgumentsBuilder, value);
}
}
}
}
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
/// <summary>
/// Creates a qualifier by appending or updating the given qualifier (by id) with a key-value pair.
/// </summary>
public QualifierId CreateQualifierWithValue(QualifierId qualifierId, string key, string value)
{
Contract.Requires(IsValidQualifierId(qualifierId));
Contract.RequiresNotNull(key);
Contract.RequiresNotNull(value);
return(CreateQualifierWithValue(qualifierId, StringId.Create(StringTable, key), StringId.Create(StringTable, value)));
}
/// <summary>
/// Creates a package id.
/// </summary>
public static PackageId Create(StringTable stringTable, string name, PackageVersion version)
{
Contract.Requires(stringTable != null);
Contract.Requires(name != null);
Contract.Requires(version != PackageVersion.None);
return(new PackageId(StringId.Create(stringTable, name), version));
}
public void DependentsSelected()
{
RootFilter expected = new RootFilter(
new DependentsFilter(
new TagFilter(StringId.Create(StringTable, "Test"))));
RunPositiveFilterParserTest("+tag=\'Test\'", expected);
}
public void IsNull_Passing_Valid_Value_Returns_False(string value)
{
var expected = false;
var actual = StringId.Create(value);
Assert.AreEqual(expected, actual.IsNull);
}
/// <summary>
/// Creates a package version.
/// </summary>
public static PackageVersion Create(StringTable stringTable, string minVersion, string maxVersion)
{
Contract.Requires(stringTable != null);
Contract.Requires(minVersion != null);
Contract.Requires(maxVersion != null);
return(new PackageVersion(StringId.Create(stringTable, minVersion), StringId.Create(stringTable, maxVersion)));
}