/// <summary> /// Parses template literal fragment like "/literal" fragment in the string <code>"${expression}/literal"</code>. /// </summary> protected override ITemplateLiteralFragment ParseTemplateLiteralFragment() { var node = CreateNodeAndSetParent <StringIdTemplateLiteralFragment>(m_token); // TODO: need to change the scanner to use StringId instead of string. // Or, maybe, switch to ReadOnlySpan<T>. var text = StringId.Create(m_pathTable.StringTable, m_scanner.TokenValue); if (!text.IsValid) { Contract.Assert(false, I($"Can't create StringId from '{m_scanner.TokenValue}'.")); } node.TextAsStringId = text; if (m_scanner.HasExtendedUnicodeEscape) { node.HasExtendedUnicodeEscape = true; } if (m_scanner.IsUnterminated) { node.IsUnterminated = true; } NextToken(); FinishNode(node); return(node); }
public void CompoundFilterPrecedence() { // And has greater precedence than or. RootFilter expected = new RootFilter( new BinaryFilter( new BinaryFilter( new TagFilter(StringId.Create(StringTable, "Test")), FilterOperator.And, new TagFilter(StringId.Create(StringTable, "csc.exe"))), FilterOperator.Or, new TagFilter(StringId.Create(StringTable, "Test2")))); RunPositiveFilterParserTest("tag=\'Test\'andtag=\'csc.exe\'ortag=\'Test2\'", expected); RootFilter expected2 = new RootFilter( new BinaryFilter( new TagFilter(StringId.Create(StringTable, "Test2")), FilterOperator.Or, new BinaryFilter( new TagFilter(StringId.Create(StringTable, "Test")), FilterOperator.And, new TagFilter(StringId.Create(StringTable, "csc.exe"))))); RunPositiveFilterParserTest("tag=\'Test2\'ortag=\'csc.exe\'andtag=\'Test\'", expected2); }
public void NestedDependencyBinaryFilter() { RootFilter expected = new RootFilter( new BinaryFilter( new DependenciesFilter( new BinaryFilter( new TagFilter(StringId.Create(StringTable, "Test1")), FilterOperator.And, new TagFilter(StringId.Create(StringTable, "Test2")))), FilterOperator.And, new TagFilter(StringId.Create(StringTable, "Test3")))); RunPositiveFilterParserTest(" ( dpc( tag = \'Test1\' and tag = \'Test2\' ) and tag = \'Test3\' )", expected); expected = new RootFilter( new BinaryFilter( new TagFilter(StringId.Create(StringTable, "Test1")), FilterOperator.And, new DependenciesFilter( new BinaryFilter( new TagFilter(StringId.Create(StringTable, "Test2")), FilterOperator.And, new TagFilter(StringId.Create(StringTable, "Test3")))))); RunPositiveFilterParserTest(" ( tag = \'Test1\' and dpc( tag = \'Test2\' and tag = \'Test3\' ) )", expected); }
public void SingleFilterCopyDependents() { RootFilter expected = new RootFilter( new CopyDependentsFilter(new TagFilter(StringId.Create(StringTable, "Test")))); RunPositiveFilterParserTest("copydpt(tag=\'Test\')", expected); }
public void TagFilter() { RootFilter expected = new RootFilter( new TagFilter(StringId.Create(StringTable, "Test"))); RunPositiveFilterParserTest("tag=\'Test\'", expected); }
/// <summary> /// Creates a package id. /// </summary> public static PackageId Create(StringTable stringTable, string name) { Contract.Requires(stringTable != null, "stringTable != null"); Contract.Requires(name != null, "name != null"); return(new PackageId(StringId.Create(stringTable, name), PackageVersion.None)); }
public void ComboTest() { AbsolutePath.TryCreate(PathTable, X("/d/nonsensePath"), out AbsolutePath path); var stringId = StringId.Create(PathTable.StringTable, "testString"); var fingerprint = FingerprintUtilities.ZeroFingerprint; var hash = ContentHashingUtilities.CreateRandom(); var longValue = (long)23; var stringValue = "asdf"; var intCollection = new int[] { 1, 2, 3 }; // Do some random operations and make sure there's no exceptions using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add(stringValue, longValue); writer.AddCollection <int, int[]>("ints", intCollection, (w, e) => w.Add(e)); writer.Add(stringValue, hash); writer.Add(stringValue, stringId); writer.Add(stringValue, stringId); writer.Add(stringValue, path, hash); } VerifyJsonContent("testString"); VerifyJsonContent(longValue.ToString()); VerifyJsonContent(intCollection.Select(i => i.ToString()).ToArray()); VerifyJsonContent(JsonFingerprinter.ContentHashToString(hash)); }
/// <summary> /// Creates the default regular expression descriptor (for warnings). /// </summary> public static RegexDescriptor CreateDefaultForWarnings(StringTable stringTable) { Contract.Requires(stringTable != null, "stringTable can't be null."); return(new RegexDescriptor( StringId.Create(stringTable, DefaultWarningPattern), DefaultOptions)); }
public override int Analyze() { var cscExePathAtom = PathAtom.Create(StringTable, "csc.exe"); var cscLookupBySpec = CachedGraph.PipGraph.RetrievePipReferencesOfType(PipType.Process) .Where(lazyPip => m_executedPipsTracker.WasVisited(lazyPip.PipId.ToNodeId())) .Select(lazyPip => (Process)lazyPip.HydratePip()) .Where(proc => proc.Executable.Path.GetName(PathTable).CaseInsensitiveEquals(StringTable, cscExePathAtom)) .ToLookup(process => process.Provenance.Token.Path); var newlineSeparator = StringId.Create(StringTable, Environment.NewLine); foreach (var specGroup in cscLookupBySpec) { var specName = specGroup.Key.GetName(PathTable).ToString(StringTable); var specOutDirectory = Path.Combine(OutputDirectory, specName); var specPath = specGroup.Key.ToString(PathTable); Directory.CreateDirectory(specOutDirectory); Console.WriteLine(I($"Processing: {specPath}")); foreach (var cscInvocation in specGroup) { var targetPath = Path.Combine(specOutDirectory, cscInvocation.FormattedSemiStableHash + ".csc.args.txt"); Console.WriteLine(I($"Emitting: {targetPath}")); using (var writer = new StreamWriter(targetPath)) { writer.WriteLine(I($"Project={specPath}")); var arguments = GetArgumentsDataFromProcess(cscInvocation) .With(newlineSeparator, PipDataFragmentEscaping.NoEscaping); writer.Write(arguments.ToString(PathTable)); } } } return(0); }
/// <summary> /// Schedules an IPC pip. /// </summary> public IpcPip ScheduleIpcPip( IIpcMoniker moniker, PipId?servicePipId, ProcessBuilder ipcProcessBuilder, FileArtifact outputFile, bool isServiceFinalization, PipConstructionHelper helper = null) { var ipcClientInfo = new IpcClientInfo(StringId.Create(Context.StringTable, moniker.Id), new ClientConfig(0, 0)); PipData arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); ReadOnlyArray <FileArtifact> fileDependencies = ipcProcessBuilder.GetInputFilesSoFar(); if (!(helper ?? m_defaultConstructionHelper).TryAddIpc( ipcClientInfo, arguments, outputFile, servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty, fileDependencies: fileDependencies, directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty, skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .Empty, isServiceFinalization: isServiceFinalization, mustRunOnOrchestrator: false, tags: new string[0], out IpcPip ipcPip)) { throw new BuildXLTestException("Failed to add ipc pip"); } return(ipcPip); }
private void SetProcessEnvironmentVariables(IReadOnlyDictionary <string, string> environment, ProcessBuilder processBuilder) { foreach (KeyValuePair <string, string> kvp in environment) { if (kvp.Value != null) { var envPipData = new PipDataBuilder(m_context.StringTable); // Casing for paths is not stable as reported by BuildPrediction. So here we try to guess if the value // represents a path, and normalize it string value = kvp.Value; if (!string.IsNullOrEmpty(value) && AbsolutePath.TryCreate(PathTable, value, out var absolutePath)) { envPipData.Add(absolutePath); } else { envPipData.Add(value); } processBuilder.SetEnvironmentVariable( StringId.Create(m_context.StringTable, kvp.Key), envPipData.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping)); } } }
public void Test1(bool useStdErr, string text, string errRegex, bool enableMultiLineScanning, string expectedPrintedError) { EventListener.NestedLoggerHandler += eventData => { if (eventData.EventId == (int)LogEventId.PipProcessError) { var loggedMessage = eventData.Payload.ToArray()[5].ToString(); m_loggedPipFailures.Add(loggedMessage); } }; var ops = SplitLines(text) .Select(l => Operation.Echo(l, useStdErr)) .Concat(new[] { Operation.WriteFile(CreateOutputFileArtifact()), Operation.Fail() }); var pipBuilder = CreatePipBuilder(ops); pipBuilder.ErrorRegex = new RegexDescriptor(StringId.Create(Context.StringTable, errRegex), RegexOptions.None); pipBuilder.EnableMultiLineErrorScanning = enableMultiLineScanning; SchedulePipBuilder(pipBuilder); RunScheduler().AssertFailure(); AssertErrorEventLogged(LogEventId.PipProcessError); XAssert.ArrayEqual( SplitLines(expectedPrintedError), m_loggedPipFailures.SelectMany(SplitLines).ToArray()); }
/// <summary> /// Converts a string to a StringId /// </summary> /// <param name="stringTable">StringTable to use to convert the string</param> /// <param name="str">String to convert to StringId</param> /// <returns>The result of converting the provided string to a StrindId</returns> public static StringId StringToId(this StringTable stringTable, string str) { // Create will mutate the StringTable if str is not already in the table // At some point, we should add a TryGet method to StringId because StringTable // should not mutate as a result of a call to StringToId return(StringId.Create(stringTable, str)); }
private static EvaluationResult GroupBy(Context context, ArrayLiteral receiver, EvaluationResult arg, EvaluationStackFrame captures) { var closure = Converter.ExpectClosure(arg); using (var disposableFrame = EvaluationStackFrame.Create(closure.Function, captures.Frame)) { // To avoid warning: access to disposable closure. var frame = disposableFrame; var entry = context.TopStack; var result = receiver.Values.GroupBy(obj => { frame.SetArgument(0, obj); return(context.InvokeClosure(closure, frame)); }); var arr = result.Select(grp => { var grpArrayLit = ArrayLiteral.CreateWithoutCopy(grp.ToArray(), entry.InvocationLocation, entry.Path); var bindings = new List <Binding> { new Binding(StringId.Create(context.FrontEndContext.StringTable, "key"), grp.Key, location: default(LineInfo)), new Binding(StringId.Create(context.FrontEndContext.StringTable, "values"), grpArrayLit, location: default(LineInfo)), }; return(EvaluationResult.Create(ObjectLiteral.Create(bindings, entry.InvocationLocation, entry.Path))); }).ToArray(); return(EvaluationResult.Create(ArrayLiteral.CreateWithoutCopy(arr, entry.InvocationLocation, entry.Path))); } }
public void SingleFilterDoubleNegated() { RootFilter expected = new RootFilter( new TagFilter(StringId.Create(StringTable, "Test"))); RunPositiveFilterParserTest("~(~(tag=\'Test\'))", expected); }
public PipDataTests(ITestOutputHelper output) : base(output) { m_pathTable = new PathTable(); m_expectedStringId0 = StringId.Create(m_pathTable.StringTable, ExpectedString0); m_pipDataBuilder = new PipDataBuilder(m_pathTable.StringTable); m_expectedStringId0 = StringId.Create(m_pathTable.StringTable, ExpectedString0); m_uniqueEntry0 = AbsolutePath.Create(m_pathTable, A("c", "unique to fragment 0")); // BEGIN ADDING ARGUMENTS m_cursorStart = m_pipDataBuilder.CreateCursor(); AddStandardBlock(m_pipDataBuilder); m_cursor0 = m_pipDataBuilder.CreateCursor(); using (m_pipDataBuilder.StartFragment(Escaping0, m_separator0)) { m_pipDataBuilder.Add(m_uniqueEntry0); AddStandardBlock(m_pipDataBuilder); } m_cursor1 = m_pipDataBuilder.CreateCursor(); using (m_pipDataBuilder.StartFragment(Escaping1, Separator1)) { AddStandardBlock(m_pipDataBuilder); m_pipDataBuilder.Add(UniqueEntry1); } m_cursorEnd = m_pipDataBuilder.CreateCursor(); // END ADDING ARGUMENTS }
public void SingleFilterDependencies() { RootFilter expected = new RootFilter( new DependenciesFilter(new TagFilter(StringId.Create(StringTable, "Test")))); RunPositiveFilterParserTest("dpc(tag=\'Test\')", expected); }
public void SpecFilterModuleDottedName() { RootFilter expected = new RootFilter( CreateModuleFilter(StringId.Create(m_context.StringTable, @"My.Test.Module"))); RunPositiveFilterParserTest("module=\'My.Test.Module\'", expected); }
public void ObservedFileNamesNormalizedTheSameWayInPathsetAndJsonFingerprinter() { // This test is guarding codesync between JsonFingerprinter.cs and ObservedPathSet.cs var fileNames = new string[] { "a", "b", "C" }; var pathTable = new PathTable(); var pathSet = ObservedPathSetTestUtilities.CreatePathSet( pathTable, observedAccessedFileNames: fileNames, X("/X/a/b/c")); var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet); var sb = new StringBuilder(); using (var writer = new global::BuildXL.Engine.Cache.JsonFingerprinter(sb, pathTable: pathTable)) { writer.AddCollection <StringId, StringId[]>( "fileNames", fileNames.Select(fn => StringId.Create(pathTable.StringTable, fn)).ToArray(), (w, e) => w.AddFileName(e)); } var fpOutput = sb.ToString(); XAssert.IsTrue(roundtrip.ObservedAccessedFileNames.All(fileName => fpOutput.Contains($"\"{fileName.ToString(pathTable.StringTable)}\""))); }
/// <summary> /// Creates a qualifier space given mappings from keys to list of eligible values. /// </summary> public QualifierSpaceId CreateQualifierSpace(IReadOnlyDictionary <string, IReadOnlyList <string> > keyValuesMap) { Contract.Requires(keyValuesMap != null); Contract.Requires( Contract.ForAll( keyValuesMap, kvp => kvp.Key != null && kvp.Value != null && kvp.Value.Count > 0 && Contract.ForAll(kvp.Value, value => value != null))); Contract.Ensures(IsValidQualifierSpaceId(Contract.Result <QualifierSpaceId>())); var keyValuesPairIds = new QualifierSpaceEntry[keyValuesMap.Count]; int i = 0; foreach (var kvp in keyValuesMap) { var valuesIds = new StringId[kvp.Value.Count]; for (int j = 0; j < kvp.Value.Count; ++j) { valuesIds[j] = StringId.Create(StringTable, kvp.Value[j]); } keyValuesPairIds[i] = QualifierSpaceEntry.Create(StringId.Create(StringTable, kvp.Key), valuesIds); ++i; } return(CreateQualifierSpace(keyValuesPairIds)); }
public void SingleFilterRequiredInputs() { RootFilter expected = new RootFilter( new DependenciesFilter(new TagFilter(StringId.Create(StringTable, "Test")), ClosureMode.DirectExcludingSelf)); RunPositiveFilterParserTest("requiredfor(tag=\'Test\')", expected); }
/// <summary> /// Creates a qualifier space given mappings from keys to list of eligible values. /// </summary> public QualifierSpaceId CreateQualifierSpace(params Tuple <string, string[]>[] keyValuesPairs) { Contract.Requires(keyValuesPairs != null); Contract.Requires( Contract.ForAll( keyValuesPairs, pair => pair.Item1 != null && pair.Item2 != null && pair.Item2.Length > 0 && Contract.ForAll(pair.Item2, value => value != null))); Contract.Ensures(IsValidQualifierSpaceId(Contract.Result <QualifierSpaceId>())); var keyValuesPairIds = new QualifierSpaceEntry[keyValuesPairs.Length]; for (int i = 0; i < keyValuesPairs.Length; ++i) { var kvp = keyValuesPairs[i]; var valuesIds = new StringId[kvp.Item2.Length]; for (int j = 0; j < kvp.Item2.Length; ++j) { valuesIds[j] = StringId.Create(StringTable, kvp.Item2[j]); } keyValuesPairIds[i] = QualifierSpaceEntry.Create(StringId.Create(StringTable, kvp.Item1), valuesIds); } return(CreateQualifierSpace(keyValuesPairIds)); }
public void DifferentOrderInQualifiersResultsInTheSameLogDirectory() { var project = CreateProjectWithPredictions("Test.proj"); var key1Key2Qualifier = FrontEndContext.QualifierTable.CreateQualifier( new Tuple <StringId, StringId>(StringId.Create(StringTable, "key1"), StringId.Create(StringTable, "value1")), new Tuple <StringId, StringId>(StringId.Create(StringTable, "key2"), StringId.Create(StringTable, "value2"))); var key1Key2Project = Start(currentQualifier: key1Key2Qualifier) .Add(project) .ScheduleAll() .AssertSuccess() .RetrieveSuccessfulProcess(project); var key2Key1Qualifier = FrontEndContext.QualifierTable.CreateQualifier( new Tuple <StringId, StringId>(StringId.Create(StringTable, "key2"), StringId.Create(StringTable, "value2")), new Tuple <StringId, StringId>(StringId.Create(StringTable, "key1"), StringId.Create(StringTable, "value1"))); var key2Key1Project = Start(currentQualifier: key2Key1Qualifier) .Add(project) .ScheduleAll() .AssertSuccess() .RetrieveSuccessfulProcess(project); // File outputs (which includes log files) should be the same, even though the qualifier was built with a different order Assert.True(key1Key2Project.FileOutputs.SequenceEqual(key2Key1Project.FileOutputs)); }
/// <summary> /// Factory to create a PipFragment containing a StringId. /// FOR TESTING PURPOSES ONLY. /// </summary> /// <param name="literal">The string literal to insert</param> /// <param name="stringTable">the associated string table</param> /// <returns>Fragment with a string literal</returns> internal static PipFragment FromString(string literal, StringTable stringTable) { Contract.Requires(literal != null); var literalId = StringId.Create(stringTable, literal); return(new PipFragment(new PipDataEntry[] { literalId }, 0)); }
/// <summary> /// Configures the process builder to execute the specified commands /// </summary> protected virtual void ConfigureProcessBuilder( ProcessBuilder processBuilder, JavaScriptProject project) { SetCmdTool(processBuilder, project); // Working directory - the directory where the project file lives. processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder); // We allow undeclared inputs to be read processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads; // We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient // to avoid a large number of path sets processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation; // Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way // Otherwise in Windows we force path sets to be all uppercase processBuilder.Options |= Process.Options.PreservePathSetCasing; // By default the double write policy is to allow same content double writes. processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites; // Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads, // any attempt to read into the user profile will fail to compute its corresponding hash processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile))); // Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) }); PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings); var logDirectory = GetLogDirectory(project); processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log")); processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log")); using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " ")) { processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C")); using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " ")) { // Execute the command and redirect the output to a designated log file processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand)); // If we need to append arguments to the script command, do it here if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <JavaScriptArgument> extraArguments)) { foreach (JavaScriptArgument value in extraArguments) { AddJavaScriptArgumentToBuilder(processBuilder.ArgumentsBuilder, value); } } } } FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable); }
/// <summary> /// Creates a qualifier by appending or updating the given qualifier (by id) with a key-value pair. /// </summary> public QualifierId CreateQualifierWithValue(QualifierId qualifierId, string key, string value) { Contract.Requires(IsValidQualifierId(qualifierId)); Contract.RequiresNotNull(key); Contract.RequiresNotNull(value); return(CreateQualifierWithValue(qualifierId, StringId.Create(StringTable, key), StringId.Create(StringTable, value))); }
/// <summary> /// Creates a package id. /// </summary> public static PackageId Create(StringTable stringTable, string name, PackageVersion version) { Contract.Requires(stringTable != null); Contract.Requires(name != null); Contract.Requires(version != PackageVersion.None); return(new PackageId(StringId.Create(stringTable, name), version)); }
public void DependentsSelected() { RootFilter expected = new RootFilter( new DependentsFilter( new TagFilter(StringId.Create(StringTable, "Test")))); RunPositiveFilterParserTest("+tag=\'Test\'", expected); }
public void IsNull_Passing_Valid_Value_Returns_False(string value) { var expected = false; var actual = StringId.Create(value); Assert.AreEqual(expected, actual.IsNull); }
/// <summary> /// Creates a package version. /// </summary> public static PackageVersion Create(StringTable stringTable, string minVersion, string maxVersion) { Contract.Requires(stringTable != null); Contract.Requires(minVersion != null); Contract.Requires(maxVersion != null); return(new PackageVersion(StringId.Create(stringTable, minVersion), StringId.Create(stringTable, maxVersion))); }