public IDistributionStrategy AddShard(string shard, IDistributionConfiguration configuration, Boolean needTransfer) { if (string.IsNullOrEmpty(shard)) { throw new Exception("Shard Name cannot be null or empty. Failed to configure distribution"); } _syncLock.AcquireWriterLock(Timeout.Infinite); try { shard = shard.ToLower(); if (_shards.Count < 1) { _shards.Add(shard); _nonShardedDistribution = new NonShardedDistribution(shard); return(this); } if (_shards.Contains(shard)) { throw new Exception("Shard '" + shard + "' already Exists"); } _shards.Add(shard); return(this); } finally { _syncLock.ReleaseWriterLock(); } }
/// <summary> /// Class constructor /// </summary> /// <param name="appLoggingContext">Application-level logging context</param> /// <param name="maxProcesses">the maximum number of concurrent pips on the worker</param> /// <param name="config">Distribution config</param>\ /// <param name="buildId">the build id</param> public WorkerService(LoggingContext appLoggingContext, int maxProcesses, IDistributionConfiguration config, string buildId) { m_isGrpcEnabled = config.IsGrpcEnabled; m_appLoggingContext = appLoggingContext; m_maxProcesses = maxProcesses; m_port = config.BuildServicePort; m_services = new DistributionServices(buildId); if (m_isGrpcEnabled) { m_workerServer = new Grpc.GrpcWorkerServer(this, appLoggingContext, buildId); } else { #if !DISABLE_FEATURE_BOND_RPC m_bondWorkerService = new InternalBond.BondWorkerServer(appLoggingContext, this, m_port, m_services); m_workerServer = m_bondWorkerService; #endif } m_attachCompletionSource = TaskSourceSlim.Create <bool>(); m_exitCompletionSource = TaskSourceSlim.Create <bool>(); m_workerRunnablePipObserver = new WorkerRunnablePipObserver(this); m_sendThread = new Thread(SendBuildResults); }
public MasterService(IDistributionConfiguration config, LoggingContext loggingContext, string buildId) { Contract.Requires(config != null && config.BuildRole == DistributedBuildRoles.Master); Contract.Ensures(m_remoteWorkers != null); m_isGrpcEnabled = config.IsGrpcEnabled; // Create all remote workers m_buildServicePort = config.BuildServicePort; m_remoteWorkers = new RemoteWorker[config.BuildWorkers.Count]; m_loggingContext = loggingContext; DistributionServices = new DistributionServices(buildId); for (int i = 0; i < m_remoteWorkers.Length; i++) { var configWorker = config.BuildWorkers[i]; var workerId = i + 1; // 0 represents the local worker. var serviceLocation = new ServiceLocation { IpAddress = configWorker.IpAddress, Port = configWorker.BuildServicePort }; m_remoteWorkers[i] = new RemoteWorker(m_isGrpcEnabled, loggingContext, (uint)workerId, this, serviceLocation); } if (m_isGrpcEnabled) { m_masterServer = new Grpc.GrpcMasterServer(loggingContext, this, buildId); } else { #if !DISABLE_FEATURE_BOND_RPC m_masterServer = new InternalBond.BondMasterServer(loggingContext, this); #endif } }
public IDistributionStrategy RemoveShard(string shard, IDistributionConfiguration configuration, bool gracefull) { if (string.IsNullOrEmpty(shard)) { throw new ArgumentNullException(shard); } _syncLock.AcquireWriterLock(Timeout.Infinite); shard = shard.ToLower(); try { if (_shards.Contains(shard)) { if (_shards.Count < 2) { if (shard.Equals(_shards.First())) { throw new Exception("Shard '" + shard + "' is the only shard in the cluster. Cannot remove it"); } else { return(this); } } } else { //In case shard has already been removed. For e.g. a gracefull -removal is followed by a forcefull removal. if (_nonShardedDistribution.Bucket.CurrentShard.Equals(shard)) { _nonShardedDistribution.Bucket.CurrentShard = _nonShardedDistribution.Bucket.FinalShard; _nonShardedDistribution.Bucket.Status = BucketStatus.Functional; } return(this); } _shards.Remove(shard); if (_nonShardedDistribution.Bucket.FinalShard.Equals(shard)) { _nonShardedDistribution.Bucket.FinalShard = _shards.First(); if (gracefull) { _nonShardedDistribution.Bucket.Status = BucketStatus.NeedTransfer; } else { _nonShardedDistribution.Bucket.CurrentShard = _nonShardedDistribution.Bucket.FinalShard; _nonShardedDistribution.Bucket.Status = BucketStatus.Functional; } } return(this); } finally { _syncLock.ReleaseWriterLock(); } }
/// <summary> /// Class constructor /// </summary> /// <param name="appLoggingContext">Application-level logging context</param> /// <param name="maxProcesses">the maximum number of concurrent pips on the worker</param> /// <param name="config">Distribution config</param>\ /// <param name="buildId">the build id</param> public WorkerService(LoggingContext appLoggingContext, int maxProcesses, IDistributionConfiguration config, string buildId) { m_appLoggingContext = appLoggingContext; m_maxProcesses = maxProcesses; m_port = config.BuildServicePort; m_services = new DistributionServices(buildId); m_workerServer = new Grpc.GrpcWorkerServer(this, appLoggingContext, buildId); m_attachCompletionSource = TaskSourceSlim.Create <bool>(); m_exitCompletionSource = TaskSourceSlim.Create <bool>(); m_workerRunnablePipObserver = new WorkerRunnablePipObserver(this); m_sendThread = new Thread(SendBuildResults); }
/// <nodoc /> public DistributionConfiguration(IDistributionConfiguration template) { Contract.Assume(template != null); BuildRole = template.BuildRole; BuildServicePort = template.BuildServicePort; ValidateDistribution = template.ValidateDistribution; EnableSourceFileMaterialization = template.EnableSourceFileMaterialization; ReplicateOutputsToWorkers = template.ReplicateOutputsToWorkers; BuildWorkers = new List <IDistributionServiceLocation>(template.BuildWorkers.Select(location => new DistributionServiceLocation(location))); DistributeCacheLookups = template.DistributeCacheLookups; MinimumWorkers = template.MinimumWorkers; IsGrpcEnabled = template.IsGrpcEnabled; }
/// <nodoc /> public DistributionConfiguration(IDistributionConfiguration template) { Contract.Assume(template != null); BuildRole = template.BuildRole; BuildServicePort = template.BuildServicePort; ValidateDistribution = template.ValidateDistribution; EnableSourceFileMaterialization = template.EnableSourceFileMaterialization; ReplicateOutputsToWorkers = template.ReplicateOutputsToWorkers; BuildWorkers = new List <IDistributionServiceLocation>(template.BuildWorkers.Select(location => new DistributionServiceLocation(location))); DistributeCacheLookups = template.DistributeCacheLookups; MinimumWorkers = template.MinimumWorkers; EarlyWorkerRelease = template.EarlyWorkerRelease; EarlyWorkerReleaseMultiplier = template.EarlyWorkerReleaseMultiplier; FireForgetMaterializeOutput = template.FireForgetMaterializeOutput; }
/// <nodoc /> public DistributionConfiguration(IDistributionConfiguration template) { Contract.Assume(template != null); BuildRole = template.BuildRole; BuildServicePort = template.BuildServicePort; ValidateDistribution = template.ValidateDistribution; ReplicateOutputsToWorkers = template.ReplicateOutputsToWorkers; BuildWorkers = new List <IDistributionServiceLocation>(template.BuildWorkers.Select(location => new DistributionServiceLocation(location))); DistributeCacheLookups = template.DistributeCacheLookups; MinimumWorkers = template.MinimumWorkers; LowWorkersWarningThreshold = template.LowWorkersWarningThreshold; EarlyWorkerRelease = template.EarlyWorkerRelease; EarlyWorkerReleaseMultiplier = template.EarlyWorkerReleaseMultiplier; FireForgetMaterializeOutput = template.FireForgetMaterializeOutput; NumRetryFailedPipsOnAnotherWorker = template.NumRetryFailedPipsOnAnotherWorker; }
public OrchestratorService(IDistributionConfiguration config, LoggingContext loggingContext, DistributedInvocationId invocationId, PipExecutionContext context) : base(invocationId) { Contract.Requires(config != null && config.BuildRole.IsOrchestrator()); // Create all remote workers m_buildServicePort = config.BuildServicePort; m_remoteWorkers = new RemoteWorker[config.BuildWorkers.Count]; m_loggingContext = loggingContext; for (int i = 0; i < m_remoteWorkers.Length; i++) { var configWorker = config.BuildWorkers[i]; var workerId = i + 1; // 0 represents the local worker. var serviceLocation = new ServiceLocation { IpAddress = configWorker.IpAddress, Port = configWorker.BuildServicePort }; m_remoteWorkers[i] = new RemoteWorker(loggingContext, (uint)workerId, this, serviceLocation, context); } m_orchestratorServer = new Grpc.GrpcOrchestratorServer(loggingContext, this, invocationId); }
public MasterService(IDistributionConfiguration config, LoggingContext loggingContext, string buildId) { Contract.Requires(config != null && config.BuildRole == DistributedBuildRoles.Master); Contract.Ensures(m_remoteWorkers != null); // Create all remote workers m_buildServicePort = config.BuildServicePort; m_remoteWorkers = new RemoteWorker[config.BuildWorkers.Count]; m_loggingContext = loggingContext; DistributionServices = new DistributionServices(buildId); for (int i = 0; i < m_remoteWorkers.Length; i++) { var configWorker = config.BuildWorkers[i]; var workerId = i + 1; // 0 represents the local worker. var serviceLocation = new ServiceLocation { IpAddress = configWorker.IpAddress, Port = configWorker.BuildServicePort }; m_remoteWorkers[i] = new RemoteWorker(loggingContext, (uint)workerId, this, serviceLocation); } m_masterServer = new Grpc.GrpcMasterServer(loggingContext, this, buildId); }
/// <summary> /// Whether this build is running in CloudBuild /// </summary> public static bool ReplicateOutputsToWorkers(this IDistributionConfiguration configuration) { return(configuration.ReplicateOutputsToWorkers ?? false); }
/// <summary> /// Whether workers should send results of materializeoutput step to the orchestrator. /// </summary> public static bool FireForgetMaterializeOutput(this IDistributionConfiguration configuration) { return(configuration.FireForgetMaterializeOutput ?? false); }
public bool RemoveShardFromCluster(string cluster, string shard, IDistributionConfiguration configuration, Boolean isGraceFull) { return(_session.RemoveShardFromCluster(cluster, shard, configuration, isGraceFull)); }
public bool AddShardToCluster(string cluster, string shard, ShardConfiguration shardConfiguration, IDistributionConfiguration distributionConfiguration) { return(_session.AddShardToCluster(cluster, shard, shardConfiguration, distributionConfiguration)); }