Ejemplo n.º 1
0
            public void It_should_set_IsOperationRunning_to_true()
            {
                var operationTracker = new OperationTracker();

                operationTracker.BeginOperation();
                Assert.That(operationTracker.IsOperationRunning, Is.True);
            }
Ejemplo n.º 2
0
            public void It_should_not_decrement_OperationsRunning_below_zero()
            {
                var operationTracker = new OperationTracker();

                operationTracker.EndOperation();
                Assert.That(operationTracker.OperationsRunning, Is.EqualTo(0));
            }
Ejemplo n.º 3
0
        public void TestOperationsHelper(bool parallel)
        {
            LoggingContext log = new LoggingContext("op");
            var            operationTracker = new OperationTracker(new LoggingContext("test"));
            int            length           = 100000;

            using (var globalContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, new PipId(15234), PipType.Process, log))
                using (globalContext.StartOperation(PipExecutorCounter.ExecutePipStepDuration))
                {
                    For(length, i =>
                    {
                        using (var context = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, log))
                            using (context.StartOperation(PipExecutorCounter.ExecutePipStepDuration))
                                using (var outerContext = context.StartAsyncOperation(PipExecutorCounter.FileContentManagerTryMaterializeOuterDuration))
                                    using (outerContext.StartOperation(PipExecutorCounter.FileContentManagerTryMaterializeDuration))
                                    {
                                    }

                        using (var outerContext = globalContext.StartAsyncOperation(PipExecutorCounter.FileContentManagerTryMaterializeOuterDuration))
                            using (outerContext.StartOperation(PipExecutorCounter.FileContentManagerTryMaterializeDuration))
                            {
                            }
                    }, parallel);
                }
        }
Ejemplo n.º 4
0
        /// <summary>
        /// Sets logging context and start time of the pip
        /// </summary>
        public void Start(OperationTracker tracker, LoggingContext loggingContext)
        {
            Contract.Assert(Step == PipExecutionStep.Start || IsDistributedWorker);

            OperationContext = tracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, PipId, PipType, loggingContext, OperationCompleted);
            StartTime        = DateTime.UtcNow;
        }
Ejemplo n.º 5
0
            public void It_should_increment_OperationsRunning()
            {
                var operationTracker = new OperationTracker();

                operationTracker.BeginOperation();
                Assert.That(operationTracker.OperationsRunning, Is.EqualTo(1));
            }
Ejemplo n.º 6
0
            public void It_should_invoke_setTargetPropertyAction_with_a_value_of_true()
            {
                var isOperationRunning = false;
                var operationTracker   = new OperationTracker(value => { isOperationRunning = value; });

                operationTracker.BeginOperation();
                Assert.That(isOperationRunning, Is.True);
            }
Ejemplo n.º 7
0
        internal void Start(LoggingContext loggingContext, OperationTracker operationTracker)
        {
            Contract.Requires(!IsStarted);
            Contract.Ensures(IsStarted);

            m_executePhaseLoggingContext = loggingContext;
            m_operationTracker           = operationTracker;
            IsStarted = true;
        }
Ejemplo n.º 8
0
        public void Setup()
        {
            _operationTracker = new OperationTracker();
            _fileSystem       = new InMemoryFilesystem();
            _node             = new LocalNode(_operationTracker, _fileSystem);

            _fileSystem.AddRoot(Path.GetPathRoot(Path.GetTempPath()));
            _fileSystem.CreateDirectory(Path.GetTempPath());
        }
Ejemplo n.º 9
0
        protected virtual IAnchorRunspaceProxy GetRunspace()
        {
            IAnchorRunspaceProxy result;

            using (OperationTracker.Create(this.anchorContext.Logger, "Creating a new runspace.", new object[0]))
            {
                result = AnchorRunspaceProxy.CreateRunspaceForDatacenterAdmin(this.anchorContext, "Mailbox Load Balance");
            }
            return(result);
        }
Ejemplo n.º 10
0
 OperationTracker CreateTrackerForNewOperation(string operationName, CancellationToken operationCancellation)
 {
     if (currentOperationTracker != null && !currentOperationTracker.cancellation.IsCancellationRequested)
     {
         throw new InvalidOperationException(
                   string.Format("Impossible to start new operation '{0}' while previous one '{1}' is not finished or cancelled",
                                 operationName, currentOperationTracker.name));
     }
     currentOperationTracker = new OperationTracker(this, operationName, operationCancellation);
     return(currentOperationTracker);
 }
Ejemplo n.º 11
0
        public void OperationTracker_Basics1()
        {
            var o = new OperationTracker();

            //
            // Enter a few times.
            //

            var g1 = o.Enter();
            var g2 = o.Enter();

            //
            // Double-dispose is fine.
            //

            g1.Dispose();
            g1.Dispose();

            //
            // Initiate dispose.
            //

            var d = o.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
                    .AsTask()
#endif
            ;

            Assert.IsFalse(d.IsCompleted);

            //
            // Cannot enter again.
            //

            Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());

            //
            // Exit the last operation.
            //

            g2.Dispose();

            //
            // Tracker is disposed now.
            //

            d.Wait();

            //
            // Cannot enter again.
            //

            Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());
        }
 protected override void ProcessRequest()
 {
     using (OperationTracker.Create(this.logger, "Rebalancing {0} from {1} to {2}.", new object[]
     {
         this.rebalanceData.RebalanceInformation,
         this.rebalanceData.SourceDatabase,
         this.rebalanceData.TargetDatabase
     }))
     {
         this.logger.Log(MigrationEventType.Information, "MOVE: Moving {0} from {1} to {2}.", new object[]
         {
             this.rebalanceData.RebalanceInformation,
             this.rebalanceData.SourceDatabase,
             this.rebalanceData.TargetDatabase
         });
         Band[] bands = this.rebalanceData.RebalanceInformation.Metrics.OfType <Band>().ToArray <Band>();
         TopologyExtractorFactory entitySelectorFactory = this.serviceContext.TopologyExtractorFactoryContextPool.GetContext(this.clientFactory, bands, this.nonMovableOrgs, this.logger).GetEntitySelectorFactory();
         DirectoryDatabase        database              = this.directoryProvider.GetDatabase(this.rebalanceData.SourceDatabase.Guid);
         LoadContainer            container             = entitySelectorFactory.GetExtractor(database).ExtractTopology();
         IOperationRetryManager   operationRetryManager = LoadBalanceOperationRetryManager.Create(this.logger);
         foreach (LoadMetric loadMetric in this.rebalanceData.RebalanceInformation.Metrics)
         {
             EntitySelector selector = loadMetric.GetSelector(container, this.rebalanceData.ConstraintSetIdentity, this.rebalanceData.RebalanceInformation[loadMetric]);
             if (selector.IsEmpty)
             {
                 this.logger.Log(MigrationEventType.Information, "Could not find any mailbox for metric {0} in database {1}.", new object[]
                 {
                     loadMetric,
                     this.rebalanceData.SourceDatabase
                 });
             }
             else
             {
                 this.logger.Log(MigrationEventType.Information, "Found mailboxes matching the metric {0} in database {1}. Requesting the injections.", new object[]
                 {
                     loadMetric,
                     this.rebalanceData.SourceDatabase
                 });
                 operationRetryManager.TryRun(delegate
                 {
                     DirectoryDatabase database2 = (DirectoryDatabase)this.directoryProvider.GetDirectoryObject(this.rebalanceData.TargetDatabase.DirectoryObjectIdentity);
                     using (IInjectorService injectorClientForDatabase = this.clientFactory.GetInjectorClientForDatabase(database2))
                     {
                         injectorClientForDatabase.InjectMoves(this.rebalanceData.TargetDatabase.Guid, this.rebalanceData.RebalanceBatchName, selector.GetEntities(this.rebalanceData.TargetDatabase));
                     }
                 });
             }
         }
     }
 }
Ejemplo n.º 13
0
 protected override void ProcessRequest()
 {
     using (OperationTracker.Create(this.logger, "Applying processor {0} to mailbox {1}.", new object[]
     {
         this.Processor.GetType().FullName,
         this.Mailbox.Identity
     }))
     {
         using (RunspaceReservation runspaceReservation = this.cmdletPool.AcquireRunspace())
         {
             this.Processor.ProcessMailbox(this.Mailbox, runspaceReservation.Runspace);
         }
     }
 }
 private void RefreshCachedValue()
 {
     using (OperationTracker.Create(this.logger, "Refreshing topology for {0}.", new object[]
     {
         this.directoryObject
     }))
     {
         this.cachedValue = this.topologyExtractor.ExtractTopology();
         ExAssert.RetailAssert(this.cachedValue != null, "ExtractTopology for directoryObject: {0} should never return null.  TopologyExtractor: {1}", new object[]
         {
             this.directoryObject,
             this.topologyExtractor
         });
     }
 }
Ejemplo n.º 15
0
        private static async Task <PipResult> Execute(BuildXLContext context, FileContentTable fileContentTable, IConfiguration config, Pip pip)
        {
            Contract.Requires(context != null);
            Contract.Requires(fileContentTable != null);
            Contract.Requires(config != null);
            Contract.Requires(pip != null);

            var loggingContext   = BuildXLTestBase.CreateLoggingContextForTest();
            var operationTracker = new OperationTracker(loggingContext);

            using (var env = new Test.BuildXL.Scheduler.Utils.DummyPipExecutionEnvironment(loggingContext, context, config, fileContentTable))
                using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, env.LoggingContext))
                {
                    return(await Test.BuildXL.Scheduler.TestPipExecutor.ExecuteAsync(operationContext, env, pip));
                }
        }
        private IPhysicalMailbox LoadMailboxData()
        {
            IPhysicalMailbox result;

            using (OperationTracker.Create(this.logger, "Retrieving single mailbox {0} data from database {1}", new object[]
            {
                this.Guid,
                this.database.Identity
            }))
            {
                using (IPhysicalDatabase physicalDatabaseConnection = this.clientFactory.GetPhysicalDatabaseConnection(this.database))
                {
                    result = (physicalDatabaseConnection.GetMailbox(this.Guid) ?? EmptyPhysicalMailbox.Instance);
                }
            }
            return(result);
        }
        private void BeginProcessingMailbox(DirectoryMailbox mailbox)
        {
            IRequestQueue requestQueue = (mailbox.Parent != null) ? this.context.QueueManager.GetProcessingQueue(mailbox.Parent) : this.context.QueueManager.MainProcessingQueue;

            string[] source = LoadBalanceADSettings.Instance.Value.ExcludedMailboxProcessors.Split(new char[]
            {
                ','
            });
            IList <MailboxProcessor> list = this.processors(this.context.QueueManager) ?? ((IList <MailboxProcessor>)Array <MailboxProcessor> .Empty);

            foreach (MailboxProcessor mailboxProcessor in list)
            {
                if (source.Contains(mailboxProcessor.Name))
                {
                    this.context.Logger.LogVerbose("Skipping processor {0} because it is disabled.", new object[]
                    {
                        mailboxProcessor.Name
                    });
                }
                else if (!mailboxProcessor.ShouldProcess(mailbox))
                {
                    this.context.Logger.LogVerbose("Processor {0} doesn't want to process the mailbox, ignored.", new object[]
                    {
                        mailboxProcessor.Name
                    });
                }
                else if (mailboxProcessor.RequiresRunspace)
                {
                    requestQueue.EnqueueRequest(new ProcessMailboxRequest(mailbox, mailboxProcessor, this.context.Logger, this.context.CmdletPool));
                }
                else
                {
                    using (OperationTracker.Create(this.context.Logger, "Applying processor {0} to mailbox {1}.", new object[]
                    {
                        mailboxProcessor.GetType().FullName,
                        mailbox.Identity
                    }))
                    {
                        mailboxProcessor.ProcessMailbox(mailbox, null);
                    }
                }
            }
        }
Ejemplo n.º 18
0
        internal void Start(LoggingContext loggingContext, OperationTracker operationTracker)
        {
            Contract.Requires(!IsStarted);
            Contract.Ensures(IsStarted);

            m_executePhaseLoggingContext = loggingContext;
            m_operationTracker           = operationTracker;

            foreach (var servicePipId in m_pipGraph.GetServicePipIds())
            {
                var serviceMutable = (ProcessMutablePipState)m_pipGraph.PipTable.GetMutable(servicePipId);
                foreach (var finalizationPipId in serviceMutable.ServiceInfo.FinalizationPipIds)
                {
                    m_finalizationPipToServicePipMap[finalizationPipId] = servicePipId;
                }
            }

            IsStarted = true;
        }
Ejemplo n.º 19
0
        protected override void ProcessRequest()
        {
            this.logger.LogInformation("Starting to drain database {0} with batch name {1}.", new object[]
            {
                this.directoryDatabase.Identity,
                this.batchName
            });
            IOperationRetryManager operationRetryManager = LoadBalanceOperationRetryManager.Create(this.logger);

            using (OperationTracker.Create(this.logger, "Moving mailboxes out of {0}.", new object[]
            {
                this.directoryDatabase.Identity
            }))
            {
                foreach (DirectoryMailbox mailboxToMove2 in this.directoryDatabase.GetMailboxes())
                {
                    DirectoryMailbox mailboxToMove = mailboxToMove2;
                    operationRetryManager.TryRun(delegate
                    {
                        this.moveInjector.InjectMoveForMailbox(mailboxToMove, this.batchName);
                    });
                }
            }
            this.logger.LogInformation("Draining database {0}: Cleaning soft deleted mailboxes.", new object[]
            {
                this.directoryDatabase.Identity
            });
            using (OperationTracker.Create(this.logger, "Starting soft deleted cleanup for {0}.", new object[]
            {
                this.directoryDatabase.Identity
            }))
            {
                this.serviceContext.CleanupSoftDeletedMailboxesOnDatabase(this.directoryDatabase.Identity, ByteQuantifiedSize.Zero);
            }
            this.logger.LogInformation("Finished processing the draining of database {0}.", new object[]
            {
                this.directoryDatabase.Identity
            });
            if (this.OnDrainFinished != null)
            {
                this.OnDrainFinished(this.directoryDatabase);
            }
        }
        private PerProcessPipPerformanceInformation CreateSamplePip(int index)
        {
            Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) =>
            {
                PipResult result;
                var       operationTracker = new OperationTracker(runnablePip.LoggingContext);
                var       pip = runnablePip.Pip;
                using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext))
                {
                    result = await TestPipExecutor.ExecuteAsync(operationContext, m_executionEnvironment, pip);
                }

                return(result);
            };

            var pathTable = m_context.PathTable;

            var executable   = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, X("/x/pkgs/tool.exe")));
            var dependencies = new HashSet <FileArtifact> {
                executable
            };

            var processBuilder = new ProcessBuilder()
                                 .WithExecutable(executable)
                                 .WithWorkingDirectory(AbsolutePath.Create(pathTable, X("/x/obj/working")))
                                 .WithArguments(PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-loadargs"))
                                 .WithStandardDirectory(AbsolutePath.Create(pathTable, X("/x/obj/working.std")))
                                 .WithDependencies(dependencies)
                                 .WithContext(m_context);

            var dataBuilder = new PipDataBuilder(m_context.PathTable.StringTable);
            var pipData     = dataBuilder.ToPipData(" ", PipDataFragmentEscaping.NoEscaping);
            var pip         = processBuilder.WithArguments(pipData).Build();
            var pipId       = m_executionEnvironment.PipTable.Add((uint)(index + 1), pip);

            var runnableProcessPip = (ProcessRunnablePip)(RunnablePip.Create(m_loggingContext, m_executionEnvironment, pipId, PipType.Process, 0, taskFactory, 0));

            m_runnablePips.Add(index, runnableProcessPip);      // For verification

            return(GeneratePipInfoWithRunnablePipAndIndex(ref runnableProcessPip, index));
        }
Ejemplo n.º 21
0
 private void PopulateDirectoryObjectFromIdentity(LoadEntity entity)
 {
     using (OperationTracker.Create(this.logger, "Re-hydrating directory object of type {0} on load entity.", new object[]
     {
         entity.DirectoryObjectIdentity.ObjectType
     }))
     {
         if (entity.DirectoryObjectIdentity != null)
         {
             try
             {
                 entity.DirectoryObject = this.directory.GetDirectoryObject(entity.DirectoryObjectIdentity);
             }
             catch (LocalizedException exception)
             {
                 this.logger.LogError(exception, "Failed to rehydrate object with identity '{0}'.", new object[]
                 {
                     entity.DirectoryObjectIdentity
                 });
             }
         }
     }
 }
Ejemplo n.º 22
0
        public void TestSanitizeForJSON(string oldValue, string expectValue)
        {
            var sanitizedDescription = OperationTracker.SanitizeForJSON(oldValue);

            XAssert.AreEqual(expectValue, sanitizedDescription);
        }
Ejemplo n.º 23
0
        public void OperationTracker_Concurrent()
        {
            //
            // Create enough competing threads so we have concurrency the moment we dispose the tracker.
            //

            int n = Environment.ProcessorCount * 4;

            using var c = new CountdownEvent(n);
            using var e = new ManualResetEvent(initialState: false);

            var tasks = new Thread[n];

            var t = new OperationTracker();

            //
            // Each thread will enter, then signal having arrived within the tracked scope, and then wait for a signal to continue.
            //

            for (int i = 0; i < n; i++)
            {
                var u = new Thread(() =>
                {
                    using var _ = t.Enter();

                    c.Signal();
                    e.WaitOne();

                    Thread.Yield();
                });

                tasks[i] = u;

                u.Start();
            }

            //
            // Wait for all threads to arrive, then let them loose.
            //

            c.Wait();
            e.Set();

            //
            // Initiate and await completion of closing the tracker.
            //

            t.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
            .AsTask()
#endif
            .Wait();

            //
            // Join all threads.
            //

            foreach (var u in tasks)
            {
                u.Join();
            }
        }
Ejemplo n.º 24
0
        /// <summary>
        /// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
        /// </summary>
        public DummyPipExecutionEnvironment(
            LoggingContext loggingContext,
            PipExecutionContext context,
            IConfiguration config,
            FileContentTable fileContentTable = null,
            EngineCache pipCache = null,
            SemanticPathExpander semanticPathExpander           = null,
            PipContentFingerprinter.PipDataLookup pipDataLookup = null,
            FileAccessWhitelist fileAccessWhitelist             = null,
            bool allowUnspecifiedSealedDirectories = false,
            PipTable pipTable        = null,
            IIpcProvider ipcProvider = null,
            IKextConnection sandboxedKextConnection = null)
        {
            Contract.Requires(context != null);
            Contract.Requires(config != null);

            LoggingContext = loggingContext;
            Context        = context;

            // Ensure paths visible when debugging
            PathTable.DebugPathTable = Context.PathTable;
            Configuration            = config;
            PipTable             = pipTable;
            PathExpander         = semanticPathExpander ?? SemanticPathExpander.Default;
            ContentFingerprinter = new PipContentFingerprinter(
                Context.PathTable,
                artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
                new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
                pathExpander: PathExpander,
                pipDataLookup: pipDataLookup);
            PipFragmentRenderer = this.CreatePipFragmentRenderer();
            IpcProvider         = ipcProvider ?? IpcFactory.GetProvider();

            FileContentTable    = fileContentTable ?? FileContentTable.CreateNew();
            Cache               = pipCache;
            FileAccessWhitelist = fileAccessWhitelist;
            m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
            m_sandboxedKextConnection           = sandboxedKextConnection;

            if (Cache == null)
            {
                Cache = InMemoryCacheFactory.Create(context);
            }

            var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);

            LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
            PipGraphView          = new TestPipGraphFilesystemView(Context.PathTable);
            m_operationTracker    = new OperationTracker(loggingContext);

            var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);

            var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;

            if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
            {
                preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
            }

            State = new PipExecutionState(
                config,
                cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
                fileAccessWhitelist: FileAccessWhitelist,
                directoryMembershipFingerprinter: this,
                pathExpander: PathExpander,
                executionLog: ExecutionLogRecorder,
                fileSystemView: fileSystemView,
                fileContentManager: GetFileContentManager(),
                directoryMembershipFinterprinterRuleSet: null,
                unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
                preserveOutputsSalt: preserveOutputsSalt,
                serviceManager: new DummyServiceManager());

            m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();

            ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
        }
Ejemplo n.º 25
0
        public MailboxProvisioningResult GetDatabase(MailboxProvisioningData provisioningData)
        {
            MailboxProvisioningResult result;

            using (OperationTracker.Create(this.logger, "Selecting database for '{0}' using {1}.", new object[]
            {
                provisioningData,
                base.GetType().Name
            }))
            {
                MailboxProvisioningResult mailboxProvisioningResult = new MailboxProvisioningResult
                {
                    Status = MailboxProvisioningResultStatus.ConstraintCouldNotBeSatisfied,
                    MailboxProvisioningConstraints = provisioningData.MailboxProvisioningConstraints
                };
                IMailboxProvisioningConstraints provisioningConstraint      = provisioningData.MailboxProvisioningConstraints ?? new MailboxProvisioningConstraints();
                IEnumerable <LoadContainer>     databasesMatchingConstraint = this.GetDatabasesMatchingConstraint(provisioningConstraint);
                List <LoadContainer>            list = new List <LoadContainer>(300);
                foreach (LoadContainer loadContainer in databasesMatchingConstraint)
                {
                    mailboxProvisioningResult.Status = MailboxProvisioningResultStatus.InsufficientCapacity;
                    LoadMetric loadMetric;
                    long       num;
                    long       num2;
                    if (!loadContainer.CanAcceptRegularLoad)
                    {
                        this.logger.Log(MigrationEventType.Instrumentation, "Database '{0}' cannot accept regular load, skipping.", new object[]
                        {
                            loadContainer.DirectoryObjectIdentity
                        });
                    }
                    else if (!loadContainer.AvailableCapacity.SupportsAdditional(provisioningData.ConsumedLoad, out loadMetric, out num, out num2))
                    {
                        this.logger.Log(MigrationEventType.Instrumentation, "Database '{0}' does not have sufficient capacity for the provisioning request. The {1} requested units of {2} would exceed the {3} available. Skipped.", new object[]
                        {
                            loadContainer.DirectoryObjectIdentity,
                            num,
                            loadMetric,
                            num2
                        });
                    }
                    else
                    {
                        list.Add(loadContainer);
                    }
                }
                if (list.Any <LoadContainer>())
                {
                    LoadContainer loadContainer2 = list[new Random().Next(0, list.Count)];
                    mailboxProvisioningResult.Status   = MailboxProvisioningResultStatus.Valid;
                    mailboxProvisioningResult.Database = loadContainer2.DirectoryObjectIdentity;
                    loadContainer2.CommittedLoad      += provisioningData.ConsumedLoad;
                    this.logger.Log(MigrationEventType.Instrumentation, "Selected database {0} with {1} max, {2} consumed and {3} available.", new object[]
                    {
                        loadContainer2.DirectoryObjectIdentity,
                        loadContainer2.MaximumLoad,
                        loadContainer2.ConsumedLoad,
                        loadContainer2.AvailableCapacity
                    });
                }
                result = mailboxProvisioningResult;
            }
            return(result);
        }
Ejemplo n.º 26
0
        public void OperationTracker_Basics2()
        {
            var o = new OperationTracker();

            //
            // Enter a few times.
            //

            var g1 = o.Enter();
            var g2 = o.Enter();

            //
            // Initiate dispose a first time.
            //

            var d1 = o.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
                     .AsTask()
#endif
            ;

            Assert.IsFalse(d1.IsCompleted);

            //
            // Cannot enter again.
            //

            Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());

            //
            // Initiate dispose a second time.
            //

            var d2 = o.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
                     .AsTask()
#endif
            ;

            Assert.IsFalse(d2.IsCompleted);

            //
            // Exit one of the operations.
            //

            g2.Dispose();

            Assert.IsFalse(d1.IsCompleted);
            Assert.IsFalse(d2.IsCompleted);

            //
            // Cannot enter again.
            //

            Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());

            //
            // Exit the other operation.
            //

            g1.Dispose();

            //
            // Tracker is disposed now.
            //

            d1.Wait();
            d2.Wait();

            //
            // Cannot enter again.
            //

            Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());

            //
            // Dispose is a no-op now.
            //

            o.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
            .AsTask()
#endif
            .Wait();

            //
            // Cannot enter again.
            //

            Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());
        }
Ejemplo n.º 27
0
        public async Task Stress()
        {
            const int N              = 5;
            const int M              = N * N;
            var       context        = BuildXLContext.CreateInstanceForTesting();
            var       loggingContext = CreateLoggingContextForTest();
            var       pathTable      = context.PathTable;

            using (var tempFiles = new TempFileStorage(canGetFileNames: true))
            {
                var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles);

                using (var pipTable = new PipTable(
                           context.PathTable,
                           context.SymbolTable,
                           initialBufferSize: 1024,
                           maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3,
                           debug: false))
                {
                    var executionEnvironment = new PipQueueTestExecutionEnvironment(
                        context,
                        config,
                        pipTable,
                        Path.Combine(TestOutputDirectory, "temp"),
                        TryGetSubstSourceAndTarget(out string substSource, out string substTarget) ? (substSource, substTarget) : default((string, string)?),
                        GetSandboxConnection());

                    Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) =>
                    {
                        PipResult result;
                        var       operationTracker = new OperationTracker(runnablePip.LoggingContext);
                        var       pip = runnablePip.Pip;
                        using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext))
                        {
                            result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip);
                        }

                        executionEnvironment.MarkExecuted(pip);
                        return(result);
                    };

                    string       executable         = CmdHelper.OsShellExe;
                    FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));

                    // This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed).
                    executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile);

                    using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
                    {
                        // phase 1: create some files
                        var baseFileArtifacts = new List <FileArtifact>();
                        for (int i = 0; i < N; i++)
                        {
                            string       destination             = tempFiles.GetUniqueFileName();
                            AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
                            FileArtifact destinationArtifact     = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
                            baseFileArtifacts.Add(destinationArtifact);

                            PipData contents = PipDataBuilder.CreatePipData(
                                context.StringTable,
                                " ",
                                PipDataFragmentEscaping.CRuntimeArgumentRules,
                                i.ToString(CultureInfo.InvariantCulture));

                            var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
                            var pipId     = pipTable.Add((uint)(i + 1), writeFile);

                            var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable));
                            executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash);

                            var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0);
                            runnable.Start(new OperationTracker(loggingContext), loggingContext);
                            runnable.SetDispatcherKind(DispatcherKind.IO);
                            phase1PipQueue.Enqueue(runnable);
                        }

                        phase1PipQueue.SetAsFinalized();
                        phase1PipQueue.DrainQueues();
                        await Task.WhenAll(
                            Enumerable.Range(0, 2).Select(
                                async range =>
                        {
                            using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
                            {
                                // phase 2: do some more with those files
                                var pips         = new ConcurrentDictionary <PipId, Tuple <string, int> >();
                                var checkerTasks = new ConcurrentQueue <Task>();
                                Action <PipId, Task <PipResult> > callback =
                                    (id, task) =>
                                {
                                    XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion);
                                    XAssert.IsFalse(task.Result.Status.IndicatesFailure());
                                    Tuple <string, int> t;
                                    if (!pips.TryRemove(id, out t))
                                    {
                                        XAssert.Fail();
                                    }

                                    checkerTasks.Enqueue(
                                        Task.Run(
                                            () =>
                                    {
                                        string actual = File.ReadAllText(t.Item1).Trim();

                                        // TODO: Make this async
                                        XAssert.AreEqual(actual, t.Item2.ToString());
                                    }));
                                };
                                var r = new Random(0);
                                for (int i = 0; i < M; i++)
                                {
                                    int sourceIndex             = r.Next(baseFileArtifacts.Count);
                                    FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex];

                                    string destination = tempFiles.GetUniqueFileName();
                                    AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
                                    FileArtifact destinationArtifact     = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
                                    Pip pip;

                                    DispatcherKind queueKind;
                                    switch (r.Next(2))
                                    {
                                    case 0:
                                        pip       = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
                                        queueKind = DispatcherKind.IO;
                                        executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
                                        break;

                                    case 1:
                                        string workingDirectory =
                                            OperatingSystemHelper.IsUnixOS ? "/tmp" :
                                            Environment.GetFolderPath(Environment.SpecialFolder.Windows);

                                        AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);

                                        var pipData = OperatingSystemHelper.IsUnixOS ?
                                                      PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") :
                                                      PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact);

                                        queueKind = DispatcherKind.CPU;
                                        pip       = new Process(
                                            executableArtifact,
                                            workingDirectoryAbsolutePath,
                                            pipData,
                                            FileArtifact.Invalid,
                                            PipData.Invalid,
                                            ReadOnlyArray <EnvironmentVariable> .Empty,
                                            FileArtifact.Invalid,
                                            FileArtifact.Invalid,
                                            FileArtifact.Invalid,
                                            tempFiles.GetUniqueDirectory(pathTable),
                                            null,
                                            null,
                                            ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact),
                                            ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()),
                                            ReadOnlyArray <DirectoryArtifact> .Empty,
                                            ReadOnlyArray <DirectoryArtifact> .Empty,
                                            ReadOnlyArray <PipId> .Empty,
                                            ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)),
                                            ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)),
                                            ReadOnlyArray <StringId> .Empty,
                                            ReadOnlyArray <int> .Empty,
                                            ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
                                            provenance: PipProvenance.CreateDummy(context),
                                            toolDescription: StringId.Invalid,
                                            additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
                                        executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
                                        break;

                                    default:
                                        Contract.Assert(false);
                                        continue;
                                    }

                                    var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip);

                                    Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) =>
                                    {
                                        var task      = taskFactory(runnablePip);
                                        var pipResult = await task;
                                        callback(pipId, task);
                                    };

                                    var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0);
                                    runnable.Start(new OperationTracker(loggingContext), loggingContext);
                                    runnable.SetDispatcherKind(queueKind);
                                    phase2PipQueue.Enqueue(runnable);

                                    if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex)))
                                    {
                                        Contract.Assert(false);
                                    }
                                }

                                phase2PipQueue.SetAsFinalized();
                                phase2PipQueue.DrainQueues();
                                XAssert.AreEqual(0, pips.Count);
                                await Task.WhenAll(checkerTasks);
                            }
                        }));
                    }
                }
            }
        }
Ejemplo n.º 28
0
 internal Tracker(OperationTracker parent)
 {
     _parent = parent;
 }