Пример #1
0
        /// <nodoc />
        public ApiServer(
            IIpcProvider ipcProvider,
            string ipcMonikerId,
            FileContentManager fileContentManager,
            PipExecutionContext context,
            IServerConfig config,
            PipTwoPhaseCache pipTwoPhaseCache,
            Tracing.IExecutionLogTarget executionLog,
            Tracing.BuildManifestGenerator buildManifestGenerator,
            ServiceManager serviceManger,
            bool verifyFileContentOnBuildManifestHashComputation)
        {
            Contract.Requires(ipcMonikerId != null);
            Contract.Requires(fileContentManager != null);
            Contract.Requires(context != null);
            Contract.Requires(config != null);
            Contract.Requires(pipTwoPhaseCache != null);
            Contract.Requires(executionLog != null);

            m_fileContentManager         = fileContentManager;
            m_server                     = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
            m_context                    = context;
            m_executionLog               = executionLog;
            m_buildManifestGenerator     = buildManifestGenerator;
            m_serviceManger              = serviceManger;
            m_pipTwoPhaseCache           = pipTwoPhaseCache;
            m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, IReadOnlyList <ContentHash> >();
            m_receivedStatistics         = new ConcurrentBigMap <string, long>();
            m_verifyFileContentOnBuildManifestHashComputation = verifyFileContentOnBuildManifestHashComputation;
        }
Пример #2
0
            public Harness(string testOutputDirectory)
            {
                Context = BuildXLContext.CreateInstanceForTesting();
                var config = ConfigurationHelpers.GetDefaultForTesting(Context.PathTable, AbsolutePath.Create(Context.PathTable, System.IO.Path.Combine(testOutputDirectory, "config.dc")));

                m_env = new DummyPipExecutionEnvironment(
                    CreateLoggingContextForTest(),
                    Context,
                    config,
                    subst: FileUtilities.TryGetSubstSourceAndTarget(testOutputDirectory, out var substSource, out var substTarget)
                        ? (substSource, substTarget)
                        : default((string, string)?),
                    sandboxConnection: GetSandboxConnection());
                var sealContentsCache = new ConcurrentBigMap <DirectoryArtifact, int[]>();

                Table = Context.PathTable;

                // Create the paths in the PathTable before creating the CachedFileSystemView
                Path(a);
                Path(b);
                Path(c);
                Path(d);
                Path(e);
                Path(f);
                Path(g);
                Path(h);
                Path(i);
                Path(j);
                Path(k);
                Path(l);

                m_fileSystem = new PipFileSystemView();
                m_fileSystem.Initialize(Table);
            }
Пример #3
0
        private WeakContentFingerprint GenerateSaltedWeakFingerprint(ContentHash hash) => new WeakContentFingerprint(FingerprintUtilities.Hash($"Hash: '{hash.ToHex()}' Salt: '{m_buildManifestHashCacheSalt}'")); // Changes to this string will invalidate all existing cache entries

        /// <nodoc />
        public ApiServer(
            IIpcProvider ipcProvider,
            string ipcMonikerId,
            FileContentManager fileContentManager,
            PipExecutionContext context,
            IServerConfig config,
            EngineCache engineCache,
            Tracing.IExecutionLogTarget executionLog,
            Tracing.BuildManifestGenerator buildManifestGenerator)
        {
            Contract.Requires(ipcMonikerId != null);
            Contract.Requires(fileContentManager != null);
            Contract.Requires(context != null);
            Contract.Requires(config != null);
            Contract.Requires(engineCache != null);
            Contract.Requires(executionLog != null);

            m_fileContentManager         = fileContentManager;
            m_server                     = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
            m_context                    = context;
            m_engineCache                = engineCache;
            m_executionLog               = executionLog;
            m_buildManifestGenerator     = buildManifestGenerator;
            m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, ContentHash>();
            m_buildManifestHashCacheSalt = string.IsNullOrEmpty(Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt)
                ? string.Empty
                : Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt;
        }
Пример #4
0
        /// <summary>
        /// Creates a new binary logger to write to the given stream
        /// </summary>
        /// <param name="logStream">the stream to write events to</param>
        /// <param name="context">the context containing the path table</param>
        /// <param name="logId">the log id to place in the header of execution used to verify with other data structures on load.</param>
        /// <param name="lastStaticAbsolutePathIndex">the last absolute path guaranteed to be in the serialized form of the corresponding path table</param>
        /// <param name="closeStreamOnDispose">specifies whether the stream is closed on disposal of the logger</param>
        /// <param name="onEventWritten">optional callback after each event is written to underlying stream</param>
        public BinaryLogger(Stream logStream, PipExecutionContext context, Guid logId, int lastStaticAbsolutePathIndex = int.MinValue, bool closeStreamOnDispose = true, Action onEventWritten = null)
        {
            m_context = context;
            LogId     = logId;
            m_lastStaticAbsolutePathIndex = lastStaticAbsolutePathIndex;
            m_logStreamWriter             = new BuildXLWriter(debug: false, stream: logStream, leaveOpen: !closeStreamOnDispose, logStats: false);
            m_watch           = Stopwatch.StartNew();
            m_capturedPaths   = new ConcurrentBigMap <AbsolutePath, bool>();
            m_capturedStrings = new ConcurrentBigMap <StringId, bool>();
            m_capturedStrings.Add(StringId.Invalid, true);
            m_writerPool = new ObjectPool <EventWriter>(
                () => new EventWriter(this),
                writer => { writer.Seek(0, SeekOrigin.Begin); return(writer); });

            var logIdBytes = logId.ToByteArray();

            Contract.Assert(logIdBytes.Length == LogIdByteLength);
            logStream.Write(logIdBytes, 0, logIdBytes.Length);
            LogStartTime(DateTime.UtcNow);

            m_pendingEventsDrainingThread = new Thread(
                () =>
            {
                foreach (PooledObjectWrapper <EventWriter> wrapper in m_pendingEvents.GetConsumingEnumerable())
                {
                    var eventWriter = wrapper.Instance;
                    WriteEventData(eventWriter);
                    m_writerPool.PutInstance(wrapper);
                    onEventWritten?.Invoke();
                }
            });
            m_pendingEventsDrainingThread.Start();
        }
Пример #5
0
        /// <summary>
        /// Deserializes an instance of <see cref="PipProducers"/>.
        /// </summary>
        public static PipProducers Deserialize(BuildXLReader reader)
        {
            Contract.Requires(reader != null);

            var producedPaths = new ConcurrentBigMap <PipStableId, HashSet <AbsolutePath> >();
            var pipProducers  = ConcurrentBigMap <AbsolutePath, PipStableId> .Deserialize(
                reader,
                () =>
            {
                var path     = reader.ReadAbsolutePath();
                var producer = reader.ReadPipStableId();

                producedPaths.AddOrUpdate(
                    producer,
                    path,
                    (pip, pathToAdd) => new HashSet <AbsolutePath> {
                    pathToAdd
                },
                    (pip, pathToAdd, existingSet) =>
                {
                    existingSet.Add(pathToAdd);
                    return(existingSet);
                });

                return(new KeyValuePair <AbsolutePath, PipStableId>(path, producer));
            });

            return(new PipProducers(pipProducers, producedPaths));
        }
Пример #6
0
        private static Task <ConcurrentBigMap <AbsolutePath, AbsolutePath> > LoadAsync(Stream stream, string fileName, PathTable pathTable)
        {
            return(ExceptionUtilities.HandleRecoverableIOException(
                       async() =>
            {
                s_fileEnvelope.ReadHeader(stream);
                using (BuildXLReader reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true))
                {
                    var stringTable = await StringTable.DeserializeAsync(reader);
                    var loadedPathTable = await PathTable.DeserializeAsync(reader, Task.FromResult(stringTable));

                    var importedPathIndex = pathTable.Import(loadedPathTable);

                    var pathMapping = new ConcurrentBigMap <AbsolutePath, AbsolutePath>();

                    var count = reader.ReadInt32();
                    for (int i = 0; i < count; i++)
                    {
                        var loadedKey = reader.ReadAbsolutePath();
                        var loadedValue = reader.ReadAbsolutePath();

                        var key = importedPathIndex[loadedKey.Value.Index];
                        var value = importedPathIndex[loadedValue.Value.Index];

                        pathMapping[key] = value;
                    }

                    return pathMapping;
                }
            },
                       ex => { throw new BuildXLException(I($"Failed to read '{fileName}'"), ex); }));
        }
Пример #7
0
        /// <nodoc/>
        public MaterializationDaemon(
            IParser parser,
            DaemonConfig daemonConfig,
            MaterializationDaemonConfig materializationConfig,
            IIpcProvider rpcProvider = null,
            Client bxlClient         = null)
            : base(parser,
                   daemonConfig,
                   !string.IsNullOrWhiteSpace(materializationConfig.LogDir) ? new FileLogger(materializationConfig.LogDir, LogFileName, daemonConfig.Moniker, logVerbose: true, MaterializationDaemonLogPrefix) : daemonConfig.Logger,
                   rpcProvider,
                   bxlClient)
        {
            m_config                = materializationConfig;
            m_actionQueue           = new ActionQueue(m_config.MaxDegreeOfParallelism);
            m_materializationStatus = new ConcurrentBigMap <string, bool>();
            m_counters              = new CounterCollection <MaterializationDaemonCounter>();

            m_macros = new Dictionary <string, string>
            {
                ["$(build.nttree)"] = Environment.GetEnvironmentVariable("_NTTREE")
            };

            m_logger.Info($"MaterializationDaemon config: {JsonConvert.SerializeObject(m_config)}");
            m_logger.Info($"Defined macros (count={m_macros.Count}):{Environment.NewLine}{string.Join(Environment.NewLine, m_macros.Select(kvp => $"{kvp.Key}={kvp.Value}"))}");
        }
Пример #8
0
        /// <summary>
        /// Creates an instance of <see cref="SymlinkDefinitions" />.
        /// </summary>
        public SymlinkDefinitions(PathTable pathTable, ConcurrentBigMap <AbsolutePath, AbsolutePath> symlinkDefinitionMap)
        {
            Contract.Requires(pathTable != null);
            Contract.Requires(symlinkDefinitionMap != null);
            Contract.Requires(pathTable != null);

            m_pathTable                     = pathTable;
            m_symlinkDefinitionMap          = symlinkDefinitionMap;
            m_directorySymlinkContents      = new Dictionary <AbsolutePath, List <AbsolutePath> >();
            m_directoriesContainingSymlinks = new HashSet <HierarchicalNameId>();

            foreach (var symlink in m_symlinkDefinitionMap.Keys)
            {
                var directory = symlink.GetParent(m_pathTable);

                foreach (var pathId in m_pathTable.EnumerateHierarchyBottomUp(directory.Value))
                {
                    if (!m_directoriesContainingSymlinks.Add(pathId))
                    {
                        break;
                    }
                }

                List <AbsolutePath> paths;
                if (!m_directorySymlinkContents.TryGetValue(directory, out paths))
                {
                    paths = new List <AbsolutePath>();
                    m_directorySymlinkContents.Add(directory, paths);
                }

                paths.Add(symlink);
            }
        }
        /// <summary>
        /// Deserializes an instance of <see cref="IncrementalSchedulingPathMapping{T}"/> from a reader.
        /// </summary>
        public static IncrementalSchedulingPathMapping <T> Deserialize(BuildXLReader reader, Func <BinaryReader, T> readValue)
        {
            Contract.Requires(reader != null);

            var valueToPathApproximation = new ConcurrentBigMap <T, List <AbsolutePath> >();
            var pathToValue = ConcurrentBigMap <AbsolutePath, HashSet <T> > .Deserialize(
                reader,
                () =>
            {
                var path       = reader.ReadAbsolutePath();
                var valueCount = reader.ReadInt32Compact();
                var values     = new HashSet <T>();

                for (int i = 0; i < valueCount; i++)
                {
                    T value = readValue(reader);

                    // Reconstruct the approximation map.
                    AddHelper(valueToPathApproximation, value, path);

                    values.Add(value);
                }

                return(new KeyValuePair <AbsolutePath, HashSet <T> >(path, values));
            });

            return(new IncrementalSchedulingPathMapping <T>(pathToValue, valueToPathApproximation));
        }
Пример #10
0
        private static void TestOperationsHelper(bool parallel)
        {
            var map                = new ConcurrentBigMap <int, string>();
            int length             = 100000;
            int expectedAddedCount = length;

            var expectedValues = new string[length];

            // Verify that all bits start off with the default value (false in this case)
            For(length, i =>
            {
                XAssert.IsFalse(map.ContainsKey(i));
            }, parallel);

            XAssert.AreEqual(0, map.Count);

            int addedCount = 0;

            // Verify setting bits
            For(length, i =>
            {
                if ((i % 4) == 3)
                {
                    // Introduce some contention for setting the same key.
                    i = i - 1;
                    Interlocked.Decrement(ref expectedAddedCount);
                }

                if ((i % 7) == 3)
                {
                    // Introduce some concurrent read-only operations
                    // in the parallel case
                    map.ContainsKey(i - 2);
                }

                var stringRepresentation = i.ToString();
                if (map.TryAdd(i, stringRepresentation))
                {
                    expectedValues[i] = stringRepresentation;
                    Interlocked.Increment(ref addedCount);
                }

                XAssert.AreEqual(stringRepresentation, map[i]);
            }, parallel);

            XAssert.AreEqual(expectedAddedCount, addedCount);
            XAssert.AreEqual(expectedAddedCount, map.Count);

            For(length, i =>
            {
                XAssert.AreEqual((i % 4) != 3, map.ContainsKey(i));

                var expectedValue = expectedValues[i];
                if (expectedValue != null)
                {
                    XAssert.AreEqual(expectedValue, map[i]);
                }
            }, parallel);
        }
Пример #11
0
        private PipProducers(ConcurrentBigMap <AbsolutePath, PipStableId> pipProducers, ConcurrentBigMap <PipStableId, HashSet <AbsolutePath> > producedPaths)
        {
            Contract.Requires(pipProducers != null);
            Contract.Requires(producedPaths != null);

            m_pipProducers  = pipProducers;
            m_producedPaths = producedPaths;
        }
Пример #12
0
        /// <summary>
        /// Create an instance of <see cref="PathMapSerializer"/>.
        /// </summary>
        public PathMapSerializer(string filePath)
        {
            Contract.Requires(!string.IsNullOrWhiteSpace(filePath));

            m_filePath    = filePath;
            m_pathTable   = new PathTable();
            m_pathMapping = new ConcurrentBigMap <AbsolutePath, AbsolutePath>();
        }
Пример #13
0
        /// <summary>
        /// Needs to take the flushing lock. Called only from <see cref="FlushAsync(OperationContext)"/>. Refactored
        /// out for clarity.
        /// </summary>
        private void PerformFlush(OperationContext context)
        {
            _database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheFlushes].Increment();

            using (_database.Counters[ContentLocationDatabaseCounters.CacheFlush].Start())
            {
                using (_exchangeLock.AcquireWriteLock())
                {
                    _flushingCache = _cache;
                    _cache         = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
                }

                if (_configuration.FlushSingleTransaction)
                {
                    _database.PersistBatch(context, _flushingCache);
                }
                else
                {
                    var actionBlock = new ActionBlockSlim <KeyValuePair <ShortHash, ContentLocationEntry> >(_configuration.FlushDegreeOfParallelism, kv =>
                    {
                        // Do not lock on GetLock here, as it will cause a deadlock with
                        // SetMachineExistenceAndUpdateDatabase. It is correct not do take any locks as well, because
                        // no Store can happen while flush is running.
                        _database.Persist(context, kv.Key, kv.Value);
                    });

                    foreach (var kv in _flushingCache)
                    {
                        actionBlock.Post(kv);
                    }

                    actionBlock.Complete();
                    actionBlock.CompletionAsync().Wait();
                }

                _database.Counters[ContentLocationDatabaseCounters.NumberOfPersistedEntries].Add(_flushingCache.Count);

                if (_configuration.FlushPreservePercentInMemory > 0)
                {
                    int targetFlushingSize = (int)(_flushingCache.Count * _configuration.FlushPreservePercentInMemory);
                    int removeAmount       = _flushingCache.Count - targetFlushingSize;

                    foreach (var key in _flushingCache.Keys.Take(removeAmount))
                    {
                        _flushingCache.RemoveKey(key);
                    }
                }
                else
                {
                    using (_exchangeLock.AcquireWriteLock())
                    {
                        _flushingCache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
                    }
                }

                _database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCompletedCacheFlushes].Increment();
            }
        }
Пример #14
0
        /// <summary>
        /// Creates a new binary logger to write to the given stream
        /// </summary>
        /// <param name="logStream">the stream to write events to</param>
        /// <param name="context">the context containing the path table</param>
        /// <param name="logId">the log id to place in the header of execution used to verify with other data structures on load.</param>
        /// <param name="lastStaticAbsolutePathIndex">the last absolute path guaranteed to be in the serialized form of the corresponding path table</param>
        /// <param name="closeStreamOnDispose">specifies whether the stream is closed on disposal of the logger</param>
        /// <param name="onEventWritten">optional callback after each event is written to underlying stream</param>
        public BinaryLogger(Stream logStream, PipExecutionContext context, Guid logId, int lastStaticAbsolutePathIndex = int.MinValue, bool closeStreamOnDispose = true, Action onEventWritten = null)
        {
            m_context = context;
            LogId     = logId;
            m_lastStaticAbsolutePathIndex = lastStaticAbsolutePathIndex;
            m_logStreamWriter             = new BuildXLWriter(debug: false, stream: logStream, leaveOpen: !closeStreamOnDispose, logStats: false);
            m_watch           = Stopwatch.StartNew();
            m_capturedPaths   = new ConcurrentBigMap <AbsolutePath, bool>();
            m_capturedStrings = new ConcurrentBigMap <StringId, bool>();
            m_capturedStrings.Add(StringId.Invalid, true);
            m_writerPool = new ObjectPool <EventWriter>(
                () => new EventWriter(this),
                writer => { writer.Seek(0, SeekOrigin.Begin); return(writer); });
            m_onEventWritten = onEventWritten;

            var logIdBytes = logId.ToByteArray();

            Contract.Assert(logIdBytes.Length == LogIdByteLength);
            logStream.Write(logIdBytes, 0, logIdBytes.Length);
            LogStartTime(DateTime.UtcNow);

            m_pendingEventsDrainingThread = new Thread(
                () =>
            {
                // Keeps trying to drain the event queue as long as new events can be added
                // Adding events and completing the queue are properly synchronized already (a write lock
                // is taken before completing), but there is the slim chance we finish draining the queue
                // and before we check for m_completeAdding here again, an Add + CompleteAdding happen, which
                // could leave unprocessed events in the queue. So we also check here whether the queue is not empty
                while (!m_completeAdding || !m_pendingEvents.IsEmpty)
                {
                    // Wait until at least one event is available for consumption
                    m_eventsAvailable.WaitOne();

                    // If new events can still be added, let's give writers the chance to add more events.
                    // This allows for more efficient draining, since therefore the thread synchronization tax is only paid
                    // once for many events. A 100ms lag is acceptable for events to be sitting in the queue. Profiler shows a 10x
                    // aggregated time gain by doing this. Some consideration for why picking this time:
                    // - A greater didn't seem to make a difference (e.g. 500ms was tried with equivalent perf results)
                    // - There are temporal constraints that need events (in particular, the build manifest one) to get serialized
                    // and sent to workers before the pip completion event (which does not go through the binary logger) happens. So
                    // we don't want xlg events to sit in the queue for too long.
                    if (!m_completeAdding)
                    {
                        Thread.Sleep(100);
                    }

                    // Drain all events available on the queue
                    while (m_pendingEvents.TryDequeue(out var action))
                    {
                        action.Run();
                    }
                }
            });

            m_pendingEventsDrainingThread.Start();
        }
Пример #15
0
        /// <summary>
        /// Class constructor
        /// </summary>
        public SealedDirectoryTable(PathTable pathTable)
        {
            IsReadOnly      = false;
            m_patchingState = null;
            m_pathTable     = pathTable;

            m_seals          = new ConcurrentBigMap <DirectoryArtifact, Node>();
            m_pathToSealInfo = new ConcurrentBigMap <HierarchicalNameId, SealInfo>();
        }
        /// <summary>
        /// Writes <see cref="ConcurrentBigMap{TKey, TValue}"/>.
        /// </summary>
        public static void WriteTextMap <K, V>(TextWriter writer, ConcurrentBigMap <K, V> map, Func <K, string> keyToString, Func <V, string> valueToString)
        {
            Contract.Requires(writer != null);
            Contract.Requires(map != null);
            Contract.Requires(keyToString != null);
            Contract.Requires(valueToString != null);

            WriteTextList(writer, map, kvp => I($"{keyToString(kvp.Key)}: {valueToString(kvp.Value)}"));
        }
Пример #17
0
        private PipGraphStaticFingerprints(
            ConcurrentBigMap <ContentFingerprint, PipId> staticFingerprintsToPips,
            ConcurrentBigMap <PipId, ContentFingerprint> pipsToStaticFingerprints)
        {
            Contract.Requires(staticFingerprintsToPips != null);
            Contract.Requires(pipsToStaticFingerprints != null);

            m_staticFingerprintsToPips = staticFingerprintsToPips;
            m_pipsToStaticFingerprints = pipsToStaticFingerprints;
        }
        private IncrementalSchedulingPathMapping(
            ConcurrentBigMap <AbsolutePath, HashSet <T> > pathToValue,
            ConcurrentBigMap <T, List <AbsolutePath> > valueToPathApproximation)
        {
            Contract.Requires(pathToValue != null);
            Contract.Requires(valueToPathApproximation != null);

            m_pathToValue = pathToValue;
            m_valueToPathApproximation = valueToPathApproximation;
        }
Пример #19
0
 /// <summary>
 /// Creates a new file system view
 /// </summary>
 public FileSystemView(
     PathTable pathTable,
     IPipGraphFileSystemView pipGraph,
     ILocalDiskFileSystemView localDiskContentStore,
     bool inferNonExistenceBasedOnParentPathInRealFileSystem = true)
 {
     PathTable           = pathTable;
     PipGraph            = pipGraph;
     LocalDiskFileSystem = localDiskContentStore;
     PathExistenceCache  = new ConcurrentBigMap <AbsolutePath, FileSystemEntry>();
     m_inferNonExistenceBasedOnParentPathInRealFileSystem = inferNonExistenceBasedOnParentPathInRealFileSystem;
 }
Пример #20
0
 /// <summary>
 /// Constructor.
 /// </summary>
 /// <param name="oldPipGraph">Old pip graph.</param>
 /// <param name="oldPipTable">Old pip table.</param>
 /// <param name="graphBuilder">Pip graph builder to which to delegate all "add pip" operations.</param>
 /// <param name="maxDegreeOfParallelism">Max concurrency for graph reloading (<see cref="PartiallyReloadGraph"/>).</param>
 public PatchablePipGraph(
     IReadonlyDirectedGraph oldPipGraph,
     PipTable oldPipTable,
     PipGraph.Builder graphBuilder,
     int maxDegreeOfParallelism)
 {
     m_oldPipGraph             = oldPipGraph;
     m_oldPipTable             = oldPipTable;
     m_builder                 = graphBuilder;
     m_maxDegreeOfParallelism  = maxDegreeOfParallelism;
     m_reloadedSealDirectories = new ConcurrentBigMap <long, DirectoryArtifact>();
     m_reloadedServicePips     = new ConcurrentBigMap <long, PipId>();
     m_pipIdMap                = new ConcurrentBigMap <PipId, PipId>();
 }
Пример #21
0
            public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, string tempDirectory, ISandboxConnection SandboxConnection = null)
            {
                Contract.Requires(context != null);
                Contract.Requires(configuration != null);

                Context              = context;
                LoggingContext       = CreateLoggingContextForTest();
                Configuration        = configuration;
                FileContentTable     = FileContentTable.CreateNew();
                ContentFingerprinter = new PipContentFingerprinter(
                    context.PathTable,
                    artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
                    ExtraFingerprintSalts.Default(),
                    pathExpander: PathExpander);
                PipTable            = pipTable;
                PipFragmentRenderer = this.CreatePipFragmentRenderer();
                IpcProvider         = IpcFactory.GetProvider();
                var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);

                Cache = InMemoryCacheFactory.Create();
                LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker);

                m_sandboxConnectionKext  = SandboxConnection;
                m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>();
                m_wellKnownFiles         = new ConcurrentDictionary <FileArtifact, ContentHash>();
                m_producers      = new ConcurrentDictionary <FileArtifact, Pip>();
                m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable);
                var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore);

                TempCleaner = new TestMoveDeleteCleaner(tempDirectory);

                State = new PipExecutionState(
                    configuration,
                    cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander),
                    unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration,
                    preserveOutputsSalt: ContentHashingUtilities.CreateRandom(),
                    fileAccessWhitelist: FileAccessWhitelist,
                    directoryMembershipFingerprinter: this,
                    pathExpander: PathExpander,
                    executionLog: null,
                    fileSystemView: fileSystemView,
                    fileContentManager: new FileContentManager(this, new NullOperationTracker()),
                    directoryMembershipFinterprinterRuleSet: null);

                m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();

                ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
            }
Пример #22
0
 /// <nodoc />
 public void UnsafeClear()
 {
     // Order is important here, inverse order could cause deadlock.
     using (_flushMutex.AcquireSemaphore())
         using (_exchangeLock.AcquireWriteLock())
         {
             if (_cache.Count != 0)
             {
                 // Nothing guarantees that some number of updates couldn't have happened in between the last flush
                 // and this reset, because acquisition of the write lock happens after the flush lock. The only way
                 // to deal with this situation is to force a flush before this assignment and after locks have been
                 // acquired. However, this isn't required by our code right now; although it is a simple change.
                 _cache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
             }
         }
 }
Пример #23
0
        /// <summary>
        /// Creates a new binary logger to write to the given stream
        /// </summary>
        /// <param name="logStream">the stream to write events to</param>
        /// <param name="context">the context containing the path table</param>
        /// <param name="logId">the log id to place in the header of execution used to verify with other data structures on load.</param>
        /// <param name="lastStaticAbsolutePathIndex">the last absolute path guaranteed to be in the serialized form of the corresponding path table</param>
        /// <param name="closeStreamOnDispose">specifies whether the stream is closed on disposal of the logger</param>
        /// <param name="onEventWritten">optional callback after each event is written to underlying stream</param>
        public BinaryLogger(Stream logStream, PipExecutionContext context, Guid logId, int lastStaticAbsolutePathIndex = int.MinValue, bool closeStreamOnDispose = true, Action onEventWritten = null)
        {
            m_context = context;
            LogId     = logId;
            m_lastStaticAbsolutePathIndex = lastStaticAbsolutePathIndex;
            m_logStreamWriter             = new BuildXLWriter(debug: false, stream: logStream, leaveOpen: !closeStreamOnDispose, logStats: false);
            m_watch           = Stopwatch.StartNew();
            m_capturedPaths   = new ConcurrentBigMap <AbsolutePath, bool>();
            m_capturedStrings = new ConcurrentBigMap <StringId, bool>();
            m_capturedStrings.Add(StringId.Invalid, true);
            m_writerPool = new ObjectPool <EventWriter>(
                () => new EventWriter(this),
                writer => { writer.Seek(0, SeekOrigin.Begin); return(writer); });

            var logIdBytes = logId.ToByteArray();

            Contract.Assert(logIdBytes.Length == LogIdByteLength);
            logStream.Write(logIdBytes, 0, logIdBytes.Length);
            LogStartTime(DateTime.UtcNow);

            m_pendingEventsDrainingThread = new Thread(
                () =>
            {
                try
                {
                    foreach (PooledObjectWrapper <EventWriter> wrapper in m_pendingEvents.GetConsumingEnumerable())
                    {
                        var eventWriter = wrapper.Instance;
                        WriteEventData(eventWriter);
                        m_writerPool.PutInstance(wrapper);
                        onEventWritten?.Invoke();
                    }
                }
                catch (InvalidOperationException)
                {
                    // InvalidOperationException is thrown when calling Take() for a marked-as-completed blocking collection.
                    // However, GetConsumingEnumerable throws an InvalidOperationException here, which is unusual.
                    // In further investigations, we discovered that it might throw one if the collection in BlockingCollection
                    // is passed in the constructor and we externally modify that collection outside of BlockingCollection.
                    // Even we do not do that, we rarely have InvalidOperationException here, which is a NetCore bug.
                    // We reported the bug; but for now, we swallow that exception and we treat it as a signal for completion.
                    return;
                }
            });
            m_pendingEventsDrainingThread.Start();
        }
Пример #24
0
        /// <summary>
        /// Class constructor
        /// </summary>
        public PipExecutionState(
            IConfiguration configuration,
            LoggingContext loggingContext,
            PipTwoPhaseCache cache,
            FileAccessWhitelist fileAccessWhitelist,
            IDirectoryMembershipFingerprinter directoryMembershipFingerprinter,
            SemanticPathExpander pathExpander,
            IExecutionLogTarget executionLog,
            DirectoryMembershipFingerprinterRuleSet directoryMembershipFinterprinterRuleSet,
            FileContentManager fileContentManager,
            IUnsafeSandboxConfiguration unsafeConfiguration,
            PreserveOutputsInfo preserveOutputsSalt,
            FileSystemView fileSystemView,
            bool lazyDeletionOfSharedOpaqueOutputsEnabled,
            ServiceManager serviceManager = null)
        {
            Contract.Requires(fileContentManager != null);
            Contract.Requires(directoryMembershipFingerprinter != null);
            Contract.Requires(pathExpander != null);

            Cache = cache;
            m_fileAccessWhitelist            = fileAccessWhitelist;
            DirectoryMembershipFingerprinter = directoryMembershipFingerprinter;
            ResourceManager           = new ProcessResourceManager(loggingContext);
            m_pathExpander            = new FileContentManagerSemanticPathExpander(fileContentManager, pathExpander);
            ExecutionLog              = executionLog;
            m_rootModuleConfiguration = configuration;
            m_directoryMembershipFingerprinterRuleSet = directoryMembershipFinterprinterRuleSet;
            PathExistenceCache    = new ConcurrentBigMap <AbsolutePath, PathExistence>();
            FileContentManager    = fileContentManager;
            ServiceManager        = serviceManager ?? ServiceManager.Default;
            PipEnvironment        = new PipEnvironment(loggingContext);
            FileSystemView        = fileSystemView;
            m_unsafeConfiguration = unsafeConfiguration;
            m_preserveOutputsSalt = preserveOutputsSalt;
            LazyDeletionOfSharedOpaqueOutputsEnabled = lazyDeletionOfSharedOpaqueOutputsEnabled;

            if (fileSystemView != null)
            {
                fileContentManager.SetLocalDiskFileSystemExistenceView(fileSystemView);
            }
        }
Пример #25
0
        private void WriteFileDependencies(string property, ConcurrentBigMap <AbsolutePath, CompactSet <PipId> > fileToConsumerMap, JsonWriter writer)
        {
            writer.WritePropertyName(property);
            {
                writer.WriteStartArray();

                foreach (var fileEntry in fileToConsumerMap)
                {
                    var file      = fileEntry.Key;
                    var consumers = fileEntry.Value;
                    var path      = ToDisplayFilePath(file);
                    if (path != null)
                    {
                        var fileArtifactContentDecidedEventData = m_fileContentMap[file];
                        // Print : { "Path": "value", "Type": "source/output", "FailPipReferenceCount": "count", "TotalPipsReferenceCount": "count", "Hash": "value", [pip array]}
                        writer.WriteStartObject();
                        WritePropertyAndValue(writer, "Path", path);
                        var typeOfFileArtifact = fileArtifactContentDecidedEventData.FileArtifact.IsSourceFile ? "SourceFile" : "OutputFile";
                        WritePropertyAndValue(writer, "Type", typeOfFileArtifact);
                        WritePropertyAndValue(writer, "FailPipReferenceCount", GetCountOfFailPips(consumers).ToString());
                        WritePropertyAndValue(writer, "TotalPipsReferenceCount", consumers.Count.ToString());
                        WritePropertyAndValue(writer, "Hash", fileArtifactContentDecidedEventData.FileContentInfo.Hash.ToString());
                        writer.WritePropertyName("Pips", true);
                        {
                            writer.WriteStartArray();

                            foreach (var consumer in consumers)
                            {
                                writer.WriteValue(ToDisplayString(consumer));
                            }

                            writer.WriteEndArray();
                        }
                        writer.WriteEndObject();
                        writer.WriteWhitespace(Environment.NewLine);
                    }
                }
                writer.WriteEndArray();
            }
        }
 /// <summary>
 /// Helper to add a key-value to the set of map
 /// </summary>
 private static void AddHelper <TKey, TValue, TCollection>(
     ConcurrentBigMap <TKey, TCollection> map,
     TKey key,
     TValue value)
     where TCollection : ICollection <TValue>, new()
 {
     map.AddOrUpdate(
         key,
         value,
         (k, v) =>
     {
         var s = new TCollection {
             v
         };
         return(s);
     },
         (k, v, s) =>
     {
         s.Add(v);
         return(s);
     });
 }
Пример #27
0
        /// <summary>
        /// Deserialize symlink definitions serialized using <see cref="Serialize(BuildXLWriter, SymlinkDefinitions)"/>
        /// </summary>
        public static Possible <SymlinkDefinitions> Deserialize(LoggingContext loggingContext, PathTable pathTable, BuildXLReader reader)
        {
            try
            {
                bool isNull = reader.ReadBoolean();
                if (isNull)
                {
                    return((SymlinkDefinitions)null);
                }

                var pathMap = ConcurrentBigMap <AbsolutePath, AbsolutePath> .Deserialize(
                    reader,
                    () => new KeyValuePair <AbsolutePath, AbsolutePath>(
                        key : reader.ReadAbsolutePath(),
                        value : reader.ReadAbsolutePath()));

                return(new SymlinkDefinitions(pathTable, pathMap));
            }
            catch (Exception ex)
            {
                Logger.Log.FailedLoadSymlinkFile(loggingContext, ex.GetLogEventMessage());
                return(new Failure <string>("Failed loading symlink definition file"));
            }
        }
Пример #28
0
 /// <summary>
 /// Removes cache entries. This method is not thread safe and for testing only.
 /// </summary>
 public void Forget()
 {
     m_entries = new ConcurrentBigMap <WeakContentFingerprint, Node>();
 }
Пример #29
0
        public void TestConcurrentBigMapOperations()
        {
            var map = new ConcurrentBigMap <int, string>();

            XAssert.IsTrue(map.TryAdd(0, "value"));
            XAssert.IsFalse(map.TryAdd(0, "not added value"));
            XAssert.AreEqual("value", map[0]);
            map[0] = "newValue";
            map[1] = "value1";

            var value0 = "newValue";

            XAssert.AreEqual(value0, map[0]);
            XAssert.IsTrue(map.ContainsKey(0));
            XAssert.IsTrue(map.ContainsKey(1));
            XAssert.IsFalse(map.ContainsKey(12));

            XAssert.AreEqual(2, map.Count);

            // Test TryGetValue
            string value1;

            XAssert.IsTrue(map.TryGetValue(1, out value1));
            XAssert.AreEqual("value1", value1);
            string value31;

            XAssert.IsFalse(map.TryGetValue(31, out value31));

            // Test update
            XAssert.IsFalse(map.TryUpdate(1, "notUpdatedValue1", "notActualValue1"));
            XAssert.AreEqual("value1", map[1]);
            XAssert.IsTrue(map.TryUpdate(1, "updatedValue1", "value1"));
            value1 = map[1];
            XAssert.AreEqual("updatedValue1", value1);

            // Test remove
            int    beforeFailedRemoveCount = map.Count;
            string value23;

            XAssert.IsFalse(map.TryRemove(23, out value23));
            XAssert.AreEqual(beforeFailedRemoveCount, map.Count);
            map.Add(23, "value23");
            XAssert.AreEqual(beforeFailedRemoveCount + 1, map.Count);
            XAssert.IsTrue(map.TryRemove(23, out value23));
            XAssert.AreEqual("value23", value23);
            XAssert.AreEqual(beforeFailedRemoveCount, map.Count);

            Assert.Equal(new int[] { 0, 1 }, map.Keys.ToArray());
            Assert.Equal(new string[] { value0, value1 }, map.Values.ToArray());

            XAssert.AreEqual(2, map.Count);

            string addedData    = "added data";
            string notAddedData = "not added data";
            var    result       = map.GetOrAdd(2, addedData, (key, data0) => data0);

            XAssert.IsFalse(result.IsFound);
            XAssert.AreEqual(addedData, result.Item.Value);
            XAssert.AreEqual(addedData, map[2]);

            // Ensure entry is not updated for get or add
            result = map.GetOrAdd(2, notAddedData, (key, data0) => data0);
            XAssert.IsTrue(result.IsFound);
            XAssert.AreEqual(addedData, result.Item.Value);
            XAssert.AreEqual(addedData, map[2]);

            Func <int, string, string, string> updateFunction =
                (key, data0, currentValue) => "updated " + currentValue;

            var updatedData = updateFunction(2, notAddedData, addedData);

            result = map.AddOrUpdate(2, notAddedData, (key, data0) => data0, updateFunction);
            XAssert.IsTrue(result.IsFound);
            XAssert.AreEqual(addedData, result.OldItem.Value);
            XAssert.AreEqual(updatedData, result.Item.Value);
            XAssert.AreEqual(updatedData, map[2]);

            result = map.AddOrUpdate(3, addedData, (key, data0) => data0, updateFunction);
            XAssert.IsFalse(result.IsFound);
            XAssert.AreEqual(addedData, result.Item.Value);
            XAssert.AreEqual(addedData, map[3]);

            TestOperationsHelper(parallel: false);
        }
Пример #30
0
        /// <summary>
        /// Needs to take the flushing lock. Called only from <see cref="FlushAsync(OperationContext)"/>. Refactored
        /// out for clarity.
        /// </summary>
        private CounterCollection <FlushableCacheCounters> PerformFlush(OperationContext context)
        {
            _database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheFlushes].Increment();
            var counters = new CounterCollection <FlushableCacheCounters>();

            using (_database.Counters[ContentLocationDatabaseCounters.CacheFlush].Start())
            {
                using (_exchangeLock.AcquireWriteLock())
                {
                    _flushingCache = _cache;
                    _cache         = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
                }

                using (counters[FlushableCacheCounters.FlushingTime].Start()) {
                    var threads = _configuration.FlushDegreeOfParallelism;
                    if (threads <= 0)
                    {
                        threads = Environment.ProcessorCount;
                    }

                    if (_configuration.FlushSingleTransaction)
                    {
                        if (_configuration.FlushDegreeOfParallelism == 1 || _flushingCache.Count <= _configuration.FlushTransactionSize)
                        {
                            _database.PersistBatch(context, _flushingCache);
                        }
                        else
                        {
                            var actionBlock = new ActionBlockSlim <IEnumerable <KeyValuePair <ShortHash, ContentLocationEntry> > >(threads, kvs =>
                            {
                                _database.PersistBatch(context, kvs);
                            });

                            foreach (var kvs in _flushingCache.GetPages(_configuration.FlushTransactionSize))
                            {
                                actionBlock.Post(kvs);
                            }

                            actionBlock.Complete();
                            actionBlock.CompletionAsync().Wait();
                        }
                    }
                    else
                    {
                        var actionBlock = new ActionBlockSlim <KeyValuePair <ShortHash, ContentLocationEntry> >(threads, kv =>
                        {
                            // Do not lock on GetLock here, as it will cause a deadlock with
                            // SetMachineExistenceAndUpdateDatabase. It is correct not do take any locks as well, because
                            // no Store can happen while flush is running.
                            _database.Persist(context, kv.Key, kv.Value);
                        });

                        foreach (var kv in _flushingCache)
                        {
                            actionBlock.Post(kv);
                        }

                        actionBlock.Complete();
                        actionBlock.CompletionAsync().Wait();
                    }
                }

                counters[FlushableCacheCounters.Persisted].Add(_flushingCache.Count);

                _database.Counters[ContentLocationDatabaseCounters.NumberOfPersistedEntries].Add(_flushingCache.Count);

                using (counters[FlushableCacheCounters.CleanupTime].Start())
                {
                    if (_configuration.FlushPreservePercentInMemory > 0)
                    {
                        int targetFlushingSize = (int)(_flushingCache.Count * _configuration.FlushPreservePercentInMemory);
                        int removeAmount       = _flushingCache.Count - targetFlushingSize;

                        foreach (var key in _flushingCache.Keys.Take(removeAmount))
                        {
                            _flushingCache.RemoveKey(key);
                        }
                    }
                    else
                    {
                        using (_exchangeLock.AcquireWriteLock())
                        {
                            _flushingCache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
                        }
                    }
                }

                counters[FlushableCacheCounters.Leftover].Add(_flushingCache.Count);
                _database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCompletedCacheFlushes].Increment();
            }

            counters[FlushableCacheCounters.Growth].Add(_cache.Count);
            return(counters);
        }