/// <inheritdoc/>
        public override Task <RpcResponse> AttachCompleted(AttachCompletionInfo message, ServerCallContext context)
        {
            var bondMessage = message.ToOpenBond();

            m_masterService.AttachCompleted(bondMessage);

            return(Task.FromResult(new RpcResponse()));
        }
Esempio n. 2
0
        public static OpenBond.AttachCompletionInfo ToOpenBond(this AttachCompletionInfo message)
        {
            return(new OpenBond.AttachCompletionInfo()
            {
                WorkerId = message.WorkerId,

                AvailableRamMb = message.AvailableRamMb,
                MaxConcurrency = message.MaxConcurrency,
                WorkerCacheValidationContentHash = message.WorkerCacheValidationContentHash.ToBondContentHash(),
            });
        }
Esempio n. 3
0
        private async Task <bool> SendAttachCompletedAfterProcessBuildRequestStartedAsync()
        {
            if (!m_isGrpcEnabled)
            {
#if !DISABLE_FEATURE_BOND_RPC
                m_bondMasterClient.Start(m_services, OnConnectionTimeOutAsync);
#endif
            }

            var cacheValidationContent     = Guid.NewGuid().ToByteArray();
            var cacheValidationContentHash = ContentHashingUtilities.HashBytes(cacheValidationContent);

            var possiblyStored = await m_environment.Cache.ArtifactContentCache.TryStoreAsync(
                new MemoryStream(cacheValidationContent),
                cacheValidationContentHash);

            if (!possiblyStored.Succeeded)
            {
                Logger.Log.DistributionFailedToStoreValidationContentToWorkerCacheWithException(
                    m_appLoggingContext,
                    cacheValidationContentHash.ToHex(),
                    possiblyStored.Failure.DescribeIncludingInnerFailures());

                Exit(timedOut: true, "Failed to validate retrieve content from master via cache");
                return(false);
            }

            var attachCompletionInfo = new AttachCompletionInfo
            {
                WorkerId       = WorkerId,
                MaxConcurrency = m_maxProcesses,
                AvailableRamMb = m_scheduler.LocalWorker.TotalMemoryMb,
                WorkerCacheValidationContentHash = cacheValidationContentHash.ToBondContentHash(),
            };

            Contract.Assert(attachCompletionInfo.WorkerCacheValidationContentHash != null, "worker cache validation content hash is null");

            var attachCompletionResult = await m_masterClient.AttachCompletedAsync(attachCompletionInfo);

            if (!attachCompletionResult.Succeeded)
            {
                Logger.Log.DistributionInactiveMaster(m_appLoggingContext, (int)attachCompletionResult.Duration.TotalMinutes);
                Exit(timedOut: true, "Failed to attach to master");
                return(true);
            }
            else
            {
                m_notifyMasterExecutionLogTarget = new NotifyMasterExecutionLogTarget(WorkerId, m_masterClient, m_environment.Context, m_scheduler.PipGraph.GraphId, m_scheduler.PipGraph.MaxAbsolutePathIndex, m_services);
                m_scheduler.AddExecutionLogTarget(m_notifyMasterExecutionLogTarget);
                m_sendThread.Start();
            }

            return(true);
        }
        public static OpenBond.AttachCompletionInfo ToOpenBond(this AttachCompletionInfo message)
        {
            return(new OpenBond.AttachCompletionInfo()
            {
                WorkerId = message.WorkerId,

                AvailableRamMb = message.AvailableRamMb,
                AvailableCommitMb = message.AvailableCommitMb,
                MaxProcesses = message.MaxProcesses,
                MaxMaterialize = message.MaxMaterialize,
                WorkerCacheValidationContentHash = message.WorkerCacheValidationContentHash.ToBondContentHash(),
            });
        }
Esempio n. 5
0
        private async Task <bool> SendAttachCompletedAfterProcessBuildRequestStartedAsync()
        {
            var cacheValidationContent     = Guid.NewGuid().ToByteArray();
            var cacheValidationContentHash = ContentHashingUtilities.HashBytes(cacheValidationContent);

            var possiblyStored = await m_environment.Cache.ArtifactContentCache.TryStoreAsync(
                new MemoryStream(cacheValidationContent),
                cacheValidationContentHash);

            if (!possiblyStored.Succeeded)
            {
                Logger.Log.DistributionFailedToStoreValidationContentToWorkerCacheWithException(
                    m_appLoggingContext,
                    cacheValidationContentHash.ToHex(),
                    possiblyStored.Failure.DescribeIncludingInnerFailures());

                Exit("Failed to validate retrieve content from master via cache", isUnexpected: true);
                return(false);
            }

            var attachCompletionInfo = new AttachCompletionInfo
            {
                WorkerId          = WorkerId,
                MaxProcesses      = m_config.Schedule.MaxProcesses,
                MaxMaterialize    = m_config.Schedule.MaxMaterialize,
                AvailableRamMb    = m_scheduler.LocalWorker.TotalRamMb,
                AvailableCommitMb = m_scheduler.LocalWorker.TotalCommitMb,
                WorkerCacheValidationContentHash = cacheValidationContentHash.ToBondContentHash(),
            };

            Contract.Assert(attachCompletionInfo.WorkerCacheValidationContentHash != null, "worker cache validation content hash is null");

            var attachCompletionResult = await m_masterClient.AttachCompletedAsync(attachCompletionInfo);

            if (!attachCompletionResult.Succeeded)
            {
                Exit($"Failed to attach to master. Duration: {(int)attachCompletionResult.Duration.TotalMinutes}", isUnexpected: true);
                return(true);
            }
            else
            {
                m_notifyMasterExecutionLogTarget = new NotifyMasterExecutionLogTarget(WorkerId, m_masterClient, m_environment.Context, m_scheduler.PipGraph.GraphId, m_scheduler.PipGraph.MaxAbsolutePathIndex, m_services);
                m_scheduler.AddExecutionLogTarget(m_notifyMasterExecutionLogTarget);
                m_sendThread.Start();
            }

            return(true);
        }
Esempio n. 6
0
        public async void AttachCompletedAsync(AttachCompletionInfo attachCompletionInfo)
        {
            Contract.Requires(attachCompletionInfo != null);

            // There is a nearly impossible race condition where the node may still be
            // in the Starting state (i.e. waiting for ACK of Attach call) so we try to transition
            // from Starting AND Started
            var isStatusUpdated = ChangeStatus(WorkerNodeStatus.Starting, WorkerNodeStatus.Attached);

            isStatusUpdated |= ChangeStatus(WorkerNodeStatus.Started, WorkerNodeStatus.Attached);

            if (!isStatusUpdated || m_workerClient == null)
            {
                // If the status is not changed to Attached due to the current status,
                // then no need to validate cache connection.
                // The worker might have already validated the cache and it is running.
                // Or the worker might have been stopped due to the master termination.
                return;
            }

            m_cacheValidationContentHash = attachCompletionInfo.WorkerCacheValidationContentHash;
            TotalProcessSlots            = attachCompletionInfo.MaxConcurrency;
            TotalMemoryMb = attachCompletionInfo.AvailableRamMb;

            var validateCacheSuccess = await ValidateCacheConnection();

            if (validateCacheSuccess)
            {
                ChangeStatus(WorkerNodeStatus.Attached, WorkerNodeStatus.Running);
                Volatile.Write(ref m_everAvailable, true);
                m_sendThread.Start();
            }
            else
            {
                await FinishAsync("ValidateCacheConnection failed");
            }
        }
Esempio n. 7
0
        /// <summary>
        /// Completes the attachment of a worker.
        /// </summary>
        public void AttachCompleted(AttachCompletionInfo attachCompletionInfo)
        {
            var worker = GetWorkerById(attachCompletionInfo.WorkerId);

            worker.AttachCompletedAsync(attachCompletionInfo);
        }
Esempio n. 8
0
        /// <summary>
        /// Completes the attachment of a worker.
        /// </summary>
        void IOrchestratorService.AttachCompleted(AttachCompletionInfo attachCompletionInfo)
        {
            var worker = GetWorkerById(attachCompletionInfo.WorkerId);

            worker.AttachCompletedAsync(attachCompletionInfo);
        }