public void NameFilter_filters_substrings_correctly()
{
PackedExecution.Builder builder = ConstructExecution();
NameIndex nameIndex = new NameIndex(builder.PackedExecution.PipTable.PipNameTable);
NameFilter <PipId> nameFilter = new NameFilter <PipId>(
builder.PackedExecution.PipTable,
nameIndex,
pid => builder.PackedExecution.PipTable[pid].Name,
'.',
"rav");
PipId[] results = nameFilter.Filter().OrderBy(pid => pid).ToArray();
XAssert.AreEqual(2, results.Count());
XAssert.AreEqual(new PipId(1), results.First());
XAssert.AreEqual(new PipId(2), results.Last());
NameFilter <PipId> nameFilter2 = new NameFilter <PipId>(
builder.PackedExecution.PipTable,
nameIndex,
pid => builder.PackedExecution.PipTable[pid].Name,
'.',
"RAV");
PipId[] results2 = nameFilter2.Filter().OrderBy(pid => pid).ToArray();
XAssert.AreArraysEqual(results, results2, true);
}
public void RelationTable_can_be_built()
{
PackedExecution packedExecution = new PackedExecution();
PackedExecution.Builder packedExecutionBuilder = new PackedExecution.Builder(packedExecution);
long hash = 1;
string name = "ShellCommon.Shell.ShellCommon.Shell.Merged.Winmetadata";
PipId pipId = packedExecutionBuilder.PipTableBuilder.Add(hash, name, PipType.Process);
PipId pipId2 = packedExecutionBuilder.PipTableBuilder.Add(hash + 1, $"{name}2", PipType.Process);
PipId pipId3 = packedExecutionBuilder.PipTableBuilder.Add(hash + 2, $"{name}3", PipType.Process);
packedExecution.ConstructRelationTables();
RelationTable <PipId, PipId> relationTable = packedExecution.PipDependencies;
RelationTable <PipId, PipId> .Builder builder = new RelationTable <PipId, PipId> .Builder(relationTable);
// add relations in any order
builder.Add(pipId3, pipId2);
builder.Add(pipId3, pipId);
builder.Add(pipId, pipId3);
builder.Add(pipId, pipId2);
// done adding relations; flush to table
builder.Complete();
XAssert.AreArraysEqual(new[] { pipId2, pipId3 }, relationTable[pipId].ToArray(), true);
XAssert.AreArraysEqual(new PipId[0], relationTable[pipId2].ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId, pipId2 }, relationTable[pipId3].ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId2, pipId3 }, relationTable.Enumerate(pipId).ToArray(), true);
XAssert.AreArraysEqual(new PipId[0], relationTable.Enumerate(pipId2).ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId, pipId2 }, relationTable.Enumerate(pipId3).ToArray(), true);
}
/// <summary>
/// Validate Node1 -> ByteArray1 -> Node2 -> ByteArray2
/// </summary>
private T CheckSerializationRoundTrip <T>(T node)
where T : Node
{
byte[] firstPass;
T node2;
using (var writerStream = new MemoryStream())
using (var writer = new BuildXLWriter(false, writerStream, true, false))
{
// Serialize (Node1 -> ByteArray1)
node.Serialize(writer);
// Copy for validation
firstPass = writerStream.ToArray();
}
using (var readerStream = new MemoryStream(firstPass))
using (var reader = new BuildXLReader(true, readerStream, true))
{
// Deserialize (ByteArray1 -> Node2)
DeserializationContext context = new DeserializationContext(null, reader, m_pathTable, GetLineMap());
node2 = Node.Read <T>(context);
}
if (node.Kind == SyntaxKind.None)
{
Assert.Null(node2);
return(null);
}
else
{
Assert.NotNull(node2);
byte[] secondPass;
using (var writerStream2 = new MemoryStream())
using (var writer2 = new BuildXLWriter(false, writerStream2, true, false))
{
// Reserialize (Node2 -> ByteArray2)
node2.Serialize(writer2);
// Copy for validation
secondPass = writerStream2.ToArray();
}
// Compare byte arrays (ByteArray1 and ByteArray2)
XAssert.AreArraysEqual(firstPass, secondPass, true);
// Compare ASTs (Node1 and Node2)
ConstructorTests.ValidateEqual(null, typeof(T), node, node2, string.Empty, null);
return(node2);
}
}
public void TestObjectArrayEqualExhaustive(int bitwidth)
{
int max = 1 << bitwidth;
EnumerateArrayPairs(
max,
i => Convert.ToString(i, 2).ToCharArray().Select(c => c == '0' ? null : new ObjectWithBogusToString(c)),
(i, j, arrI, arrJ) =>
{
XAssert.AreArraysEqual(arrI, arrJ, i == j);
});
}
public void TestCharArrayEqualExhaustive(int bitwidth)
{
int max = 1 << bitwidth;
EnumerateArrayPairs(
max,
i => Convert.ToString(i, 2).ToCharArray(),
(i, j, arrI, arrJ) =>
{
XAssert.AreArraysEqual(arrI, arrJ, i == j);
});
}
public void DedupeReportedFileAccesses()
{
var process = new ReportedProcess(1000, "/usr/bin/touch");
var writeReport = new ReportedFileAccess(ReportedFileOperation.CreateFile,
process,
RequestedAccess.Write,
FileAccessStatus.Allowed,
true,
0,
Usn.Zero,
DesiredAccess.GENERIC_WRITE,
ShareMode.FILE_SHARE_WRITE,
CreationDisposition.CREATE_NEW,
FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
AbsolutePath.Invalid,
"/tmp/out1",
"*");
var readReport = new ReportedFileAccess(ReportedFileOperation.GetFileAttributes,
process,
RequestedAccess.Read,
FileAccessStatus.Allowed,
true,
0,
Usn.Zero,
DesiredAccess.GENERIC_READ,
ShareMode.FILE_SHARE_READ,
CreationDisposition.OPEN_EXISTING,
FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
AbsolutePath.Invalid,
"/tmp/out1",
"*");
HashSet <ReportedFileAccess> allFileAccesses = new HashSet <ReportedFileAccess>()
{
writeReport, readReport
};
HashSet <ReportedFileAccess> explicitFileAccesses = new HashSet <ReportedFileAccess>()
{
readReport, writeReport
};
var runner = CreateSandboxExecRunner();
var dedupedReports = runner.DedupeAccessReports(allFileAccesses, explicitFileAccesses);
XAssert.IsTrue(dedupedReports.Count == 2);
var result = new string[2];
dedupedReports.CopyTo(result);
XAssert.AreArraysEqual(new string[] { " W /tmp/out1", " R /tmp/out1" }, result, true);
}
private void ExpectGraph(GraphReloadResult reloadResult, Pip[] oldPips, string expectedPipIndexesAsString, string expectedEdgesAsString)
{
int[] expectedPipIndexes = expectedPipIndexesAsString.Split(',').Select(i => int.Parse(i)).ToArray();
var graph = SimpleGraph.Parse(oldPips.Length, expectedEdgesAsString);
var newPipGraph = reloadResult.PipGraph;
var newPipTable = newPipGraph.PipTable;
var newGraph = newPipGraph.DirectedGraph;
// check that the new pip table contains expected number of relevant pips
var allPipTypes = new HashSet <PipType>(oldPips.Select(pip => pip.PipType).Distinct());
IEnumerable <Pip> newRelevantPips = HydratePipsByType(newPipTable, relevantTypes: allPipTypes);
Assert.Equal(expectedPipIndexes.Length, newRelevantPips.Count());
// check that for all expected pips there is a node in the new graph
XAssert.All(
expectedPipIndexes,
idx =>
{
Assert.True(newGraph.ContainsNode(oldPips[idx].PipId.ToNodeId()), $"No graph node found for Pip{idx}");
});
// check edges
var newRelevantPipIdValues = new HashSet <uint>(newRelevantPips.Select(pip => pip.PipId.Value));
XAssert.All(
expectedPipIndexes,
idx =>
{
var nodeId = oldPips[idx].PipId.ToNodeId();
var expectedOutgoingEdges = graph
.Edges
.Where(e => e.Src == idx)
.Select(e => oldPips[e.Dest].PipId.ToNodeId().Value)
.OrderBy(v => v)
.ToArray();
var actualOutgoingEdges = newGraph
.GetOutgoingEdges(nodeId)
.Select(e => e.OtherNode.Value)
.Where(v => newRelevantPipIdValues.Contains(v))
.OrderBy(v => v)
.ToArray();
XAssert.AreArraysEqual(expectedOutgoingEdges, actualOutgoingEdges, expectedResult: true);
});
// check stats
Assert.Equal(expectedPipIndexes.Length, reloadResult.Stats.NumPipsReloaded);
}
private void TestGlob(string globFunction, params string[] expectedPaths)
{
var sortedExpectedPaths = expectedPaths.OrderBy(p => p, StringComparer.InvariantCultureIgnoreCase).ToArray();
XAssert.AreArraysEqual(sortedExpectedPaths, expectedPaths, expectedResult: true, "Must pass sorted expected paths");
var result = Build()
.AddSpec("src/project.dsc", "const result = " + globFunction + ";")
.RootSpec("src/project.dsc")
.AddTestFiles()
.EvaluateExpressionWithNoErrors <EvaluatedArrayLiteral>("result");
var actualPaths = new string[result.Count];
for (int i = 0; i < actualPaths.Length; i++)
{
var path = AbsolutePath.Invalid;
switch (result[i].Value)
{
case FileArtifact file:
path = file.Path;
break;
case DirectoryArtifact dir:
path = dir.Path;
break;
default:
XAssert.Fail("Unexpected return value");
break;
}
actualPaths[i] = path.ToString(PathTable);
}
Array.Sort(actualPaths, StringComparer.InvariantCultureIgnoreCase);
XAssert.AreEqual(
expectedPaths.Length,
result.Count,
"Sizes don't line up: encountered: " + string.Join(",", actualPaths.Select(ap => ap.Replace(TestRoot, ""))));
for (int i = 0; i < Math.Min(result.Count, expectedPaths.Length); i++)
{
AssertCanonicalEquality("src\\" + expectedPaths[i], actualPaths[i]);
}
}
public void RelationTable_can_store_multiple_relations()
{
PackedExecution packedExecution = new PackedExecution();
PackedExecution.Builder packedExecutionBuilder = new PackedExecution.Builder(packedExecution);
long hash = 1;
string name = "ShellCommon.Shell.ShellCommon.Shell.Merged.Winmetadata";
PipId pipId = packedExecutionBuilder.PipTableBuilder.Add(hash, name, PipType.Process);
PipId pipId2 = packedExecutionBuilder.PipTableBuilder.Add(hash + 1, $"{name}2", PipType.Process);
PipId pipId3 = packedExecutionBuilder.PipTableBuilder.Add(hash + 2, $"{name}3", PipType.Process);
XAssert.AreNotEqual(pipId, pipId2);
XAssert.AreNotEqual(pipId, pipId3);
XAssert.AreNotEqual(pipId2, pipId3);
packedExecution.ConstructRelationTables();
RelationTable <PipId, PipId> relationTable = packedExecution.PipDependencies;
relationTable.Add(new[] { pipId2, pipId3 }.AsSpan());
XAssert.AreEqual(2, relationTable[pipId].Length);
ReadOnlySpan <PipId> relations = relationTable[pipId];
XAssert.AreEqual(pipId2, relations[0]);
XAssert.AreEqual(pipId3, relations[1]);
relationTable.Add(new[] { pipId }.AsSpan());
XAssert.AreEqual(1, relationTable[pipId2].Length);
relationTable.Add(new[] { pipId, pipId2, pipId3 }.AsSpan());
XAssert.AreEqual(3, relationTable[pipId3].Length);
XAssert.AreArraysEqual(new[] { pipId2, pipId3 }, relationTable[pipId].ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId, pipId2, pipId3 }, relationTable[pipId3].ToArray(), true);
RelationTable <PipId, PipId> inverseRelationTable = relationTable.Invert();
XAssert.AreArraysEqual(new[] { pipId2, pipId3 }, inverseRelationTable[pipId].ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId, pipId3 }, inverseRelationTable[pipId2].ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId, pipId3 }, inverseRelationTable[pipId3].ToArray(), true);
}
private static void CheckRelation(PackedExecution packedExecution, PipId pipId1, PipId pipId2, PipId pipId3, PipId pipId4, RelationTable <PipId, PipId> relationTable)
{
XAssert.AreArraysEqual(new PipId[0], relationTable[pipId1].ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId1 }, relationTable[pipId2].ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId1 }, relationTable[pipId3].ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId2, pipId3 }, relationTable[pipId4].ToArray(), true);
XAssert.AreArraysEqual(new PipId[0], relationTable.Enumerate(pipId1).ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId1 }, relationTable.Enumerate(pipId2).ToArray(), true);
XAssert.AreArraysEqual(new[] { pipId2, pipId3 }, relationTable.Enumerate(pipId4).ToArray(), true);
// try traversing from pipId4; should traverse over pipId3 and reach pipId (as well as pipId2)
IEnumerable <PipId> reachable = relationTable.Traverse(
pipId4,
p => packedExecution.PipTable[p].PipType == PipType.Process
? TraversalFilterResult.AcceptAndHalt
: TraversalFilterResult.RejectAndContinue);
XAssert.AreArraysEqual(new[] { pipId1, pipId2 }, reachable.OrderBy(p => p.Value).ToArray(), true);
}
public void SpannableList_can_be_appended()
{
SpannableList <int> list = new SpannableList <int>(1);
XAssert.AreEqual(0, list.Count);
XAssert.AreEqual(0, list.Count());
XAssert.IsFalse(list.Contains(1));
XAssert.AreEqual(-1, list.IndexOf(1));
XAssert.AreEqual(0, list.AsSpan().Length);
list.Add(1);
XAssert.AreEqual(1, list.Count);
XAssert.AreEqual(1, list.Count());
XAssert.IsTrue(list.Contains(1));
XAssert.AreEqual(0, list.IndexOf(1));
XAssert.AreEqual(1, list.AsSpan().Length);
XAssert.AreEqual(1, list.AsSpan()[0]);
list.Add(2);
XAssert.AreEqual(2, list.Count);
XAssert.AreEqual(2, list.Count());
XAssert.IsTrue(list.Contains(1));
XAssert.IsTrue(list.Contains(2));
XAssert.IsFalse(list.Contains(3));
XAssert.AreEqual(0, list.IndexOf(1));
XAssert.AreEqual(1, list.IndexOf(2));
XAssert.AreEqual(2, list.AsSpan().Length);
XAssert.AreEqual(1, list.AsSpan()[0]);
XAssert.AreEqual(2, list.AsSpan()[1]);
list.AddRange(new[] { 3, 4, 5 }.AsSpan());
XAssert.AreArraysEqual(new[] { 1, 2, 3, 4, 5 }, list.ToArray(), true);
XAssert.AreArraysEqual(new[] { 3, 4 }, list.Enumerate(2, 2).ToArray(), true);
XAssert.AreArraysEqual(new[] { 3, 4 }, list.AsSpan().Slice(2, 2).ToArray(), true);
}
public void SpannableList_can_be_inserted()
{
SpannableList <int> list = new SpannableList <int>(1);
list.Insert(0, 1);
XAssert.AreEqual(1, list.Count);
XAssert.AreEqual(1, list.Count());
XAssert.IsTrue(list.Contains(1));
XAssert.AreEqual(0, list.IndexOf(1));
XAssert.AreEqual(1, list.AsSpan().Length);
XAssert.AreEqual("SpannableList<Int32>[1]{ 1 }", list.ToFullString());
list.Insert(0, 2);
XAssert.AreEqual(2, list.Count);
XAssert.AreEqual(2, list.Count());
XAssert.IsTrue(list.Contains(1));
XAssert.IsTrue(list.Contains(2));
XAssert.IsFalse(list.Contains(3));
XAssert.AreEqual(1, list.IndexOf(1));
XAssert.AreEqual(0, list.IndexOf(2));
XAssert.AreEqual(2, list.AsSpan().Length);
XAssert.AreEqual(2, list.AsSpan()[0]);
XAssert.AreEqual(1, list.AsSpan()[1]);
list.Insert(2, 3);
XAssert.AreEqual(3, list.Count);
XAssert.AreEqual(3, list.Count());
XAssert.IsTrue(list.Contains(1));
XAssert.IsTrue(list.Contains(2));
XAssert.IsTrue(list.Contains(3));
XAssert.AreEqual(1, list.IndexOf(1));
XAssert.AreEqual(0, list.IndexOf(2));
XAssert.AreEqual(2, list.IndexOf(3));
XAssert.AreEqual(3, list.AsSpan().Length);
XAssert.AreArraysEqual(new[] { 2, 1, 3 }, list.ToArray(), true);
}
public void SpannableList_can_be_removed()
{
SpannableList <int> list = new SpannableList <int>(1);
list.Add(1);
list.Add(2);
list.Add(3);
XAssert.AreArraysEqual(new[] { 1, 2, 3 }, list.ToArray(), true);
list.RemoveAt(0);
XAssert.AreEqual(2, list.Count);
XAssert.AreArraysEqual(new[] { 2, 3 }, list.ToArray(), true);
list.RemoveAt(1);
XAssert.AreEqual(1, list.Count);
XAssert.AreArraysEqual(new[] { 2 }, list.ToArray(), true);
XAssert.IsFalse(list.Remove(1));
XAssert.IsTrue(list.Remove(2));
XAssert.AreEqual(0, list.Count);
}
public void NameIndex_contains_all_names()
{
StringTable stringTable = new StringTable();
StringTable.CachingBuilder stringTableBuilder = new StringTable.CachingBuilder(stringTable);
NameTable nameTable = new NameTable('.', stringTable);
NameTable.Builder nameTableBuilder = new NameTable.Builder(nameTable, stringTableBuilder);
NameId id = nameTableBuilder.GetOrAdd("a.b.c");
NameId id2 = nameTableBuilder.GetOrAdd("a.b.d.e");
NameId id3 = nameTableBuilder.GetOrAdd("a.f.g.h");
NameIndex nameIndex = new NameIndex(nameTable);
XAssert.AreEqual(8, nameIndex.Count);
XAssert.AreEqual(3, nameIndex[id].Length);
XAssert.AreEqual(4, nameIndex[id2].Length);
XAssert.AreEqual(4, nameIndex[id3].Length);
// We know these are the string IDs because string IDs get added as the names are constructed,
// and we happened to add names with each successive atom in lexical order.
StringId a = new StringId(1);
StringId b = new StringId(2);
StringId c = new StringId(3);
StringId d = new StringId(4);
StringId e = new StringId(5);
StringId f = new StringId(6);
StringId g = new StringId(7);
StringId h = new StringId(8);
XAssert.AreArraysEqual(new[] { a, b, c }, nameIndex.Enumerate(id).Select(entry => entry.Atom).ToArray(), true);
XAssert.AreArraysEqual(new[] { a, b, d, e }, nameIndex.Enumerate(id2).Select(entry => entry.Atom).ToArray(), true);
XAssert.AreArraysEqual(new[] { a, f, g, h }, nameIndex.Enumerate(id3).Select(entry => entry.Atom).ToArray(), true);
}
public void DirectAffectedDiretoryInputTest()
{
// aInputDir -> (pipA) -> aOutputDir -> (PipB) -> bOutputDir
// |- pip-a-input-file |- pip-a-out-file |- pip-b-out-file
// |- aSubOutputDir
// |- pip-a-out-in-sub-file
// Execepted change affected input for pipB is pip-a-out-file and pip-a-out-in-sub-file
var aInputDir = Path.Combine(ObjectRoot, "input");
var aInputDirPath = AbsolutePath.Create(Context.PathTable, aInputDir);
var aInputDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(aInputDirPath);
var aInputFile = CreateSourceFile(root: aInputDirPath, prefix: "pip-a-input-file");
File.WriteAllText(ArtifactToString(aInputFile), "pipABuild1");
var sealInputDir = SealDirectory(aInputDirPath, SealDirectoryKind.SourceAllDirectories);
var aOutputDir = Path.Combine(ObjectRoot, "aOutputDir");
var aOutputDirPath = AbsolutePath.Create(Context.PathTable, aOutputDir);
var aOutputDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(aOutputDirPath);
var aOutputFileInOutputeDir = CreateOutputFileArtifact(root: aOutputDir, prefix: "pip-a-out-file");
var aOutputSubDir = Path.Combine(aOutputDir, "aSubOutputDir");
var aOutputSubDirPath = AbsolutePath.Create(Context.PathTable, aOutputSubDir);
var aOutputSubDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(aOutputSubDirPath);
var aOutputFileInOutputSubDir = CreateOutputFileArtifact(root: aOutputSubDir, prefix: "pip-a-out-in-sub-file");
var bOutDir = Path.Combine(ObjectRoot, "bOutputDir");
var bOutDirPath = AbsolutePath.Create(Context.PathTable, bOutDir);
var bOutDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(bOutDirPath);
var bOutFileArtifact = CreateOutputFileArtifact(root: bOutDirArtifact, prefix: "pip-b-out-file");
var pipBuilderA = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(aInputFile, doNotInfer: true),
Operation.WriteFile(aOutputFileInOutputeDir, doNotInfer: true),
Operation.CreateDir(aOutputSubDirArtifact, doNotInfer: true),
Operation.WriteFile(aOutputFileInOutputSubDir, doNotInfer: true),
});
pipBuilderA.AddInputDirectory(sealInputDir);
pipBuilderA.AddOutputDirectory(aOutputDirArtifact, SealDirectoryKind.Opaque);
var pipA = SchedulePipBuilder(pipBuilderA);
var changeAffectedWrittenFile = CreateOutputFileArtifact();
var pipBuilderB = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(aOutputFileInOutputSubDir, doNotInfer: true),
Operation.WriteFile(bOutFileArtifact, doNotInfer: true),
});
pipBuilderB.AddInputDirectory(pipA.ProcessOutputs.GetOpaqueDirectory(aOutputDirPath));
pipBuilderB.AddOutputDirectory(bOutDirArtifact, SealDirectoryKind.Opaque);
pipBuilderB.SetChangeAffectedInputListWrittenFilePath(changeAffectedWrittenFile);
var pipB = SchedulePipBuilder(pipBuilderB);
var inputChangesFile = CreateOutputFileArtifact();
File.WriteAllText(ArtifactToString(inputChangesFile), ArtifactToString(aInputFile));
Configuration.Schedule.InputChanges = inputChangesFile.Path;
RunScheduler().AssertSuccess();
string[] actualAffectedSortedInputs = File.ReadAllLines(ArtifactToString(changeAffectedWrittenFile)).OrderBy(p => p, StringComparer.InvariantCultureIgnoreCase).ToArray();
string[] expectedAffectedInputs = { aOutputFileInOutputSubDir.Path.GetName(Context.PathTable).ToString(Context.PathTable.StringTable),
aOutputFileInOutputeDir.Path.GetName(Context.PathTable).ToString(Context.PathTable.StringTable) };
string[] expectedAffectedSortedInputs = expectedAffectedInputs.OrderBy(p => p, StringComparer.InvariantCultureIgnoreCase).ToArray();
XAssert.AreArraysEqual(expectedAffectedSortedInputs, actualAffectedSortedInputs, true);
}