public async Task BasicTests(int maxWorkers, IWorkerInfo manager, IEnumerable <IWorkerInfo> workers, IEnumerable <IWorkerInfo> toRemoves)
{
var activityId = Guid.NewGuid().ToString();
var settings = new ScaleSettings {
MaxWorkers = maxWorkers
};
var mockManager = new Mock <MockScaleManager>(MockBehavior.Default, MockBehavior.Strict, settings)
{
CallBase = true
};
// Test
using (var scaleManager = mockManager.Object)
{
// Setup
if (toRemoves.Any())
{
scaleManager.MockScaleTracer.Setup(t => t.TraceInformation(activityId, manager, It.Is <string>(s => s.Contains("exceeds maximum number"))));
}
foreach (var toRemove in toRemoves)
{
mockManager.Setup(m => m.MockRequestRemoveWorker(activityId, manager, toRemove))
.Returns(Task.CompletedTask);
}
// test
var actual = await scaleManager.MockTryRemoveIfMaxWorkers(activityId, workers, manager);
// assert
mockManager.VerifyAll();
scaleManager.VerifyAll();
Assert.Equal(toRemoves.Any(), actual);
}
}
/// <summary>
/// nominate itself to be a manager
/// </summary>
protected virtual async Task <IWorkerInfo> SetManager(string activityId, IWorkerInfo worker, IWorkerInfo current)
{
var tableLock = await _table.AcquireLock();
_tracer.TraceInformation(activityId, worker, string.Format("Acquire table lock id: {0}", tableLock.Id));
try
{
var manager = await _table.GetManager();
// other worker already takes up manager position
if (!ScaleUtils.WorkerEquals(manager, current))
{
return(manager);
}
await _table.SetManager(worker);
_tracer.TraceInformation(activityId, worker, "This worker is set to be a manager.");
return(worker);
}
finally
{
await tableLock.Release();
_tracer.TraceInformation(activityId, worker, string.Format("Release table lock id: {0}", tableLock.Id));
}
}
public async Task BasicTests(IWorkerInfo worker, IWorkerInfo current)
{
var activityId = Guid.NewGuid().ToString();
var mockManager = new Mock <MockScaleManager> {
CallBase = true
};
using (var manager = mockManager.Object)
{
// Setup
mockManager.Setup(m => m.MockPingWorker(activityId, worker))
.Returns(Task.CompletedTask);
mockManager.Setup(m => m.MockEnsureManager(activityId, worker))
.Returns(Task.FromResult <IWorkerInfo>(current));
manager.MockWorkerInfoProvider.Setup(p => p.GetWorkerInfo(activityId))
.Returns(Task.FromResult <IWorkerInfo>(worker));
if (current == worker)
{
mockManager.Setup(m => m.MockMakeScaleDecision(activityId, worker))
.Returns(Task.CompletedTask);
mockManager.Setup(m => m.MockCheckStaleWorker(activityId, worker))
.Returns(Task.CompletedTask);
}
// test
await manager.MockProcessWorkItem(activityId);
// assert
mockManager.VerifyAll();
manager.VerifyAll();
}
}
/// <summary>
/// add worker to table and ping to keep alive
/// </summary>
protected virtual async Task PingWorker(string activityId, IWorkerInfo worker)
{
// if ping was unsuccessful, keep pinging. this is to address
// the issue where site continue to run on an unassigned worker.
if (!_pingResult || _pingWorkerUtc < DateTime.UtcNow)
{
// if PingWorker throws, we will not update the worker status
// this worker will be stale and eventually removed.
_pingResult = await _eventHandler.PingWorker(activityId, worker);
_pingWorkerUtc = DateTime.UtcNow.Add(_settings.WorkerPingInterval);
}
// check if worker is valid for the site
if (_pingResult)
{
await _table.AddOrUpdate(worker);
_tracer.TraceUpdateWorker(activityId, worker, string.Format("Worker loadfactor {0} updated", worker.LoadFactor));
}
else
{
_tracer.TraceWarning(activityId, worker, string.Format("Worker does not belong to the site."));
await _table.Delete(worker);
_tracer.TraceRemoveWorker(activityId, worker, "Worker removed");
throw new InvalidOperationException("The worker does not belong to the site.");
}
}
public async Task BasicTests(IWorkerInfo manager, IEnumerable <IWorkerInfo> workers, IWorkerInfo loadFactorMaxWorker)
{
var activityId = Guid.NewGuid().ToString();
var mockManager = new Mock <MockScaleManager>(MockBehavior.Default)
{
CallBase = true
};
// Test
using (var scaleManager = mockManager.Object)
{
// Setup
if (loadFactorMaxWorker != null)
{
scaleManager.MockScaleTracer.Setup(t => t.TraceInformation(activityId, loadFactorMaxWorker, It.Is <string>(s => s.Contains("have int.MaxValue loadfactor"))));
mockManager.Setup(m => m.MockRequestAddWorker(activityId, workers, manager, false))
.Returns(Task.FromResult(true));
}
// test
var actual = await scaleManager.MockTryAddIfLoadFactorMaxWorker(activityId, workers, manager);
// assert
mockManager.VerifyAll();
scaleManager.VerifyAll();
Assert.Equal(loadFactorMaxWorker != null, actual);
}
}
public async Task BasicTests(double maxBusyWorkerRatio, IWorkerInfo manager, IEnumerable <IWorkerInfo> workers, bool expected)
{
var activityId = Guid.NewGuid().ToString();
var settings = new ScaleSettings
{
MaxBusyWorkerRatio = maxBusyWorkerRatio,
BusyWorkerLoadFactor = 80
};
var mockManager = new Mock <MockScaleManager>(MockBehavior.Default, MockBehavior.Strict, settings)
{
CallBase = true
};
// Test
using (var scaleManager = mockManager.Object)
{
// Setup
if (expected)
{
scaleManager.MockScaleTracer.Setup(t => t.TraceInformation(activityId, manager, It.Is <string>(s => s.Contains("exceeds maximum busy worker ratio"))));
mockManager.Setup(m => m.MockRequestAddWorker(activityId, workers, manager, false))
.Returns(Task.FromResult(true));
}
// test
var actual = await scaleManager.MockTryAddIfMaxBusyWorkerRatio(activityId, workers, manager);
// assert
mockManager.VerifyAll();
scaleManager.VerifyAll();
Assert.Equal(expected, actual);
}
}
public async Task SuccessfulSetTests(IWorkerInfo worker, IWorkerInfo current)
{
var activityId = Guid.NewGuid().ToString();
var tableLock = new MockWorkerTableLock();
// Test
using (var scaleManager = new MockScaleManager(MockBehavior.Strict))
{
// Setup
scaleManager.MockWorkerTable.Setup(t => t.AcquireLock())
.Returns(() => tableLock.AcquireLock());
scaleManager.MockWorkerTable.Setup(t => t.GetManager())
.Returns(Task.FromResult(current));
scaleManager.MockWorkerTable.Setup(t => t.SetManager(worker))
.Returns(Task.CompletedTask);
scaleManager.MockScaleTracer.Setup(t => t.TraceInformation(activityId, worker, It.Is <string>(c => c.Contains("Acquire table lock"))));
scaleManager.MockScaleTracer.Setup(t => t.TraceInformation(activityId, worker, It.Is <string>(c => c.Contains("Release table lock"))));
scaleManager.MockScaleTracer.Setup(t => t.TraceInformation(activityId, worker, It.Is <string>(c => c.Contains("is set to be a manager"))));
// test
var newManager = await scaleManager.MockSetManager(activityId, worker, current);
// assert
scaleManager.VerifyAll();
Assert.True(ScaleUtils.Equals(newManager, worker));
Assert.False(ScaleUtils.Equals(newManager, current));
}
}
public void Consume(IConsumeContext <WorkerAvailable <TMessage> > context)
{
IWorkerInfo <TMessage> worker = _workerCache.GetWorker <TMessage>(context.Message.ControlUri, x =>
{
if (_log.IsInfoEnabled)
{
_log.InfoFormat("Discovered New Worker: {0}", context.Message.ControlUri);
}
WorkerInfo workerInfo = new WorkerInfo(context.Message.ControlUri, context.Message.DataUri);
return(new WorkerInfo <TMessage>(workerInfo));
});
worker.Update(context.Message.InProgress,
context.Message.InProgressLimit,
context.Message.Pending,
context.Message.PendingLimit,
context.Message.Updated);
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Worker {0}: {1} in progress, {2} pending", worker.DataUri, worker.InProgress,
worker.Pending);
}
}
public async Task Delete(IWorkerInfo worker)
{
var table = await GetWorkerCloudTable();
var entity = (AppServiceWorkerInfo)worker;
entity.ETag = "*";
try
{
var operation = TableOperation.Delete(entity);
await table.ExecuteAsync(operation);
}
catch (StorageException ex)
{
var webException = ex.InnerException as WebException;
if (webException != null)
{
var response = webException.Response as HttpWebResponse;
if (response != null && response.StatusCode == HttpStatusCode.NotFound)
{
return;
}
}
throw;
}
}
protected virtual async Task RequestRemoveWorker(string activityId, IWorkerInfo manager, IWorkerInfo toRemove)
{
await _eventHandler.RemoveWorker(activityId, toRemove);
await _table.Delete(toRemove);
_tracer.TraceRemoveWorker(activityId, toRemove, string.Format("Worker removed by manager ({0})", manager.ToDisplayString()));
}
/// <summary>
/// this routine checks stale worker performed by manager
/// if ping request failed, we will request remove that worker
/// if ping request Worker Not Found, we will remove from the table
/// </summary>
protected virtual async Task CheckStaleWorker(string activityId, IWorkerInfo manager)
{
const int CheckStaleBatch = 5;
if (DateTime.UtcNow < _staleWorkerCheckUtc)
{
return;
}
try
{
var stales = await _table.ListStale();
_tracer.TraceInformation(activityId, manager, stales.GetSummary("Stale"));
await Task.WhenAll(stales.Take(CheckStaleBatch).Select(async stale =>
{
Exception exception = null;
bool validWorker = false;
try
{
validWorker = await _eventHandler.PingWorker(activityId, stale);
}
catch (Exception ex)
{
exception = ex;
}
// worker failed to response
if (exception != null)
{
_tracer.TraceWarning(activityId, stale, string.Format("Stale worker (LastModifiedTimeUtc={0}) failed ping request {1}", stale.LastModifiedTimeUtc, exception));
await RequestRemoveWorker(activityId, manager, stale);
}
else if (!validWorker)
{
_tracer.TraceWarning(activityId, stale, string.Format("Stale worker (LastModifiedTimeUtc={0}) does not belong to the site.", stale.LastModifiedTimeUtc));
await _table.Delete(stale);
_tracer.TraceRemoveWorker(activityId, stale, string.Format("Stale worker (LastModifiedTimeUtc={0}) removed by manager ({1}).", stale.LastModifiedTimeUtc, manager.ToDisplayString()));
}
else
{
_tracer.TraceInformation(activityId, stale, string.Format("Stale worker (LastModifiedTimeUtc={0}) ping successfully by manager ({1}).", stale.LastModifiedTimeUtc, manager.ToDisplayString()));
}
}));
}
catch (Exception ex)
{
_tracer.TraceError(activityId, manager, string.Format("CheckStaleWorker failed with {0}", ex));
}
finally
{
_staleWorkerCheckUtc = DateTime.UtcNow.Add(_settings.StaleWorkerCheckInterval);
}
}
public async Task AddOrUpdate(IWorkerInfo worker)
{
var table = await GetWorkerCloudTable();
var entity = (AppServiceWorkerInfo)worker;
entity.ETag = "*";
var operation = TableOperation.InsertOrReplace(entity);
await table.ExecuteAsync(operation);
}
/// <summary>
/// this routine makes scaling decision performed by manager
/// - remove if exceeds MaxWorkers
/// - add any int.MaxValue
/// - remove any int.MinValue
/// - add if busy workers exceeds MaxBusyWorkerRatio
/// - remove if free workers exceeds MaxFreeWorkerRatio
/// - shrink back to homestamp if no busy worker
/// </summary>
protected virtual async Task MakeScaleDecision(string activityId, IWorkerInfo manager)
{
if (DateTime.UtcNow < _scaleCheckUtc)
{
return;
}
try
{
var workers = await _table.ListNonStale();
_tracer.TraceInformation(activityId, manager, workers.GetSummary("NonStale"));
if (await TryRemoveIfMaxWorkers(activityId, workers, manager))
{
return;
}
if (await TryAddIfLoadFactorMaxWorker(activityId, workers, manager))
{
return;
}
if (await TrySwapIfLoadFactorMinWorker(activityId, workers, manager))
{
return;
}
if (await TryAddIfMaxBusyWorkerRatio(activityId, workers, manager))
{
return;
}
if (await TryRemoveIfMaxFreeWorkerRatio(activityId, workers, manager))
{
return;
}
if (await TryRemoveSlaveWorker(activityId, workers, manager))
{
return;
}
}
catch (Exception ex)
{
_tracer.TraceError(activityId, manager, string.Format("MakeScaleDecision failed with {0}", ex));
}
finally
{
_scaleCheckUtc = DateTime.UtcNow.Add(_settings.ScaleCheckInterval);
}
}
public static bool WorkerEquals(IWorkerInfo src, IWorkerInfo dst)
{
if (src == null && dst == null)
{
return(true);
}
else if (src == null || dst == null)
{
return(false);
}
return(string.Equals(src.StampName, dst.StampName, StringComparison.OrdinalIgnoreCase) &&
string.Equals(src.WorkerName, dst.WorkerName, StringComparison.OrdinalIgnoreCase));
}
/// <summary>
/// This requests FE to remove this specific worker.
/// - FE should return success regardless if worker belongs.
/// - Any unexpected error will throw.
/// </summary>
public async Task RemoveWorker(string activityId, IWorkerInfo worker)
{
var stampHostName = GetStampHostName(worker.StampName);
var details = string.Format("Remove worker request from {0}:{1}", AppServiceSettings.CurrentStampName, AppServiceSettings.WorkerName);
var pathAndQuery = string.Format("https://{0}/operations/removeworker/{1}/{2}?token={3}",
stampHostName,
AppServiceSettings.SiteName,
worker.WorkerName,
GetToken());
using (var response = await SendAsync(activityId, HttpMethod.Delete, pathAndQuery, worker, details))
{
response.EnsureSuccessStatusCode();
}
}
public async Task SetManager(IWorkerInfo worker)
{
// update manager row
var entity = new AppServiceWorkerInfo
{
PartitionKey = AppServiceSettings.ManagerPartitionKey,
RowKey = AppServiceSettings.ManagerRowKey,
StampName = worker.StampName,
WorkerName = worker.WorkerName,
ETag = "*"
};
var operation = TableOperation.InsertOrReplace(entity);
var table = await GetWorkerCloudTable();
await table.ExecuteAsync(operation);
}
/// <summary>
/// this routine ensure a manager
/// - check if existing active manager
/// - if yes, check (prefer homestamp) manager
/// - if no, become (prefer homestamp) manager
/// </summary>
protected virtual async Task <IWorkerInfo> EnsureManager(string activityId, IWorkerInfo worker)
{
var manager = await _table.GetManager();
if (DateTime.UtcNow < _managerCheckUtc)
{
return(manager);
}
try
{
// no manager or current one stale
if (manager == null || manager.IsStale)
{
// if this worker is homestamp or no homestamp worker exists
if (worker.IsHomeStamp)
{
return(await SetManager(activityId, worker, manager));
}
else
{
var workers = await _table.ListNonStale();
if (!workers.Any(w => w.IsHomeStamp))
{
return(await SetManager(activityId, worker, manager));
}
}
}
else if (!manager.IsHomeStamp)
{
// prefer home stamp
if (worker.IsHomeStamp)
{
return(await SetManager(activityId, worker, manager));
}
}
return(manager);
}
finally
{
_managerCheckUtc = DateTime.UtcNow.Add(_settings.ManagerCheckInterval);
}
}
public async Task BasicTests(IWorkerInfo manager, IWorkerInfo toRemove)
{
var activityId = Guid.NewGuid().ToString();
// Test
using (var scaleManager = new MockScaleManager(MockBehavior.Strict))
{
scaleManager.MockScaleHandler.Setup(s => s.RemoveWorker(activityId, toRemove))
.Returns(Task.CompletedTask);
scaleManager.MockWorkerTable.Setup(t => t.Delete(toRemove))
.Returns(Task.CompletedTask);
scaleManager.MockScaleTracer.Setup(t => t.TraceRemoveWorker(activityId, toRemove, It.Is <string>(s => s.Contains(manager.ToDisplayString()))));
// test
await scaleManager.MockRequestRemoveWorker(activityId, manager, toRemove);
// assert
scaleManager.VerifyAll();
}
}
public async Task BasicTests(IWorkerInfo manager, IEnumerable <IWorkerInfo> workers, bool added, IWorkerInfo toRemove)
{
var activityId = Guid.NewGuid().ToString();
var settings = new ScaleSettings
{
BusyWorkerLoadFactor = 80,
FreeWorkerLoadFactor = 20
};
var mockManager = new Mock <MockScaleManager>(MockBehavior.Default, MockBehavior.Strict, settings)
{
CallBase = true
};
// Test
using (var scaleManager = mockManager.Object)
{
// Setup
if (toRemove != null)
{
scaleManager.MockScaleTracer.Setup(t => t.TraceInformation(activityId, toRemove, It.Is <string>(s => s.Contains("remove slave worker"))));
mockManager.Setup(m => m.MockRequestAddWorker(activityId, Enumerable.Empty <IWorkerInfo>(), manager, true))
.Returns(Task.FromResult(added));
if (added)
{
mockManager.Setup(m => m.MockRequestRemoveWorker(activityId, manager, toRemove))
.Returns(Task.FromResult(true));
}
}
// test
var actual = await scaleManager.MockTryRemoveSlaveWorker(activityId, workers, manager);
// assert
mockManager.VerifyAll();
scaleManager.VerifyAll();
Assert.Equal(toRemove != null, actual);
}
}
IEnumerable <Action <IConsumeContext <TMessage> > > Handle(IWorkerInfo <TMessage> worker)
{
yield return(context =>
{
context.BaseContext.NotifyConsume(context,
typeof(DistributorMessageSink <TMessage>).ToShortTypeName(), null);
IEndpoint endpoint = context.Bus.GetEndpoint(worker.DataUri);
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Sending {0}[{1}] to {2}", typeof(TMessage).ToShortTypeName(),
context.MessageId, worker.DataUri);
}
var distributed = new Distributed <TMessage>(context.Message, context.ResponseAddress);
endpoint.Send(distributed, x =>
{
x.SetRequestId(context.RequestId);
x.SetConversationId(context.ConversationId);
x.SetCorrelationId(context.CorrelationId);
x.SetSourceAddress(context.SourceAddress);
x.SetDestinationAddress(context.DestinationAddress);
x.SetResponseAddress(context.ResponseAddress);
x.SetFaultAddress(context.FaultAddress);
x.SetNetwork(context.Network);
if (context.ExpirationTime.HasValue)
{
x.SetExpirationTime(context.ExpirationTime.Value);
}
context.Headers.Each(header => x.SetHeader(header.Key, header.Value));
x.SetHeader("mt.worker.uri", worker.DataUri.ToString());
});
});
}
/// <summary>
/// this routine does ..
/// - ping and update worker status
/// - ensure manager
/// - if manager, make scale decision
/// - if manager, stale worker management
/// </summary>
protected virtual async Task ProcessWorkItem(string activityId)
{
// get worker status
var worker = await _provider.GetWorkerInfo(activityId);
_worker = worker;
// update worker status and keep alive
await PingWorker(activityId, worker);
// select manager
var manager = await EnsureManager(activityId, worker);
// if this is manager, perform scale decision and stale worker management
if (ScaleUtils.WorkerEquals(worker, manager))
{
// perform scale decision
await MakeScaleDecision(activityId, worker);
// stale worker management
await CheckStaleWorker(activityId, worker);
}
}
/// <summary>
/// This pings a specific worker. The outcome could be ..
/// - FE may return 404 Worker Not Found. This routine will return false and
/// worker will be removed from the table (done by manager or worker itself).
/// - Worker returns success (2xx). Worker itself will update its status on the table.
/// - Other than that throws.
/// </summary>
public async Task <bool> PingWorker(string activityId, IWorkerInfo worker)
{
var stampHostName = GetStampHostName(worker.StampName);
var details = string.Format("Ping worker request from {0}:{1}", AppServiceSettings.CurrentStampName, AppServiceSettings.WorkerName);
var pathAndQuery = string.Format("https://{0}/operations/keepalive/{1}/{2}?token={3}",
stampHostName,
AppServiceSettings.SiteName,
worker.WorkerName,
GetToken());
using (var response = await SendAsync(activityId, HttpMethod.Get, pathAndQuery, worker, details))
{
try
{
response.EnsureSuccessStatusCode();
return(true);
}
catch (HttpRequestException)
{
// Worker is not valid for the stamp
if (response.StatusCode == HttpStatusCode.NotFound &&
string.Equals(response.ReasonPhrase, WorkerNotFound, StringComparison.OrdinalIgnoreCase))
{
return(false);
}
// Worker is not valid for the stamp
if (response.StatusCode == HttpStatusCode.ServiceUnavailable &&
string.Equals(response.ReasonPhrase, SiteUnavailableFromMiniArr, StringComparison.OrdinalIgnoreCase))
{
return(false);
}
throw;
}
}
}
public async Task BasicTests(IWorkerInfo worker, IWorkerInfo current, IEnumerable <IWorkerInfo> workers, IWorkerInfo expected)
{
var activityId = Guid.NewGuid().ToString();
var mockManager = new Mock <MockScaleManager>(MockBehavior.Default)
{
CallBase = true
};
// Test
using (var scaleManager = mockManager.Object)
{
// Setup
IWorkerInfo newManager = null;
if (expected != current)
{
mockManager.Setup(m => m.MockSetManager(activityId, worker, current))
.Callback((string acticityId1, IWorkerInfo info1, IWorkerInfo current1) => newManager = info1)
.Returns(() => Task.FromResult(newManager));
}
scaleManager.MockWorkerTable.Setup(t => t.GetManager())
.Returns(Task.FromResult <IWorkerInfo>(current));
if (workers != null)
{
scaleManager.MockWorkerTable.Setup(t => t.List())
.Returns(Task.FromResult(workers));
}
// test
var actual = await scaleManager.MockEnsureManager(activityId, worker);
// assert
mockManager.VerifyAll();
scaleManager.VerifyAll();
Assert.True(ScaleUtils.Equals(expected, actual));
}
}
public CachedWorker(IWorkerInfo worker)
{
Worker = worker;
MessageWorkers = new GenericTypeCache <IWorkerInfo>(typeof(IWorkerInfo <>),
type => (IWorkerInfo)FastActivator.Create(typeof(WorkerInfo <>), new Type[] { type }));
}
public static string ToDisplayString(this IWorkerInfo worker)
{
return(string.Join(":", worker.StampName, worker.WorkerName));
}
void IScaleTracer.TraceRemoveWorker(string activityId, IWorkerInfo workerInfo, string details)
{
SetActivityId(activityId);
RemoveWorker(workerInfo.SiteName, workerInfo.StampName, workerInfo.WorkerName, details);
}
void IScaleTracer.TraceHttp(string activityId, IWorkerInfo workerInfo, string verb, string address, int statusCode, string startTime, string endTime, int latencyInMilliseconds, string requestContent, string details)
{
SetActivityId(activityId);
Http(workerInfo.SiteName, workerInfo.StampName, workerInfo.WorkerName, verb, address, statusCode, startTime, endTime, latencyInMilliseconds, requestContent, details);
}
void IScaleTracer.TraceInformation(string activityId, IWorkerInfo workerInfo, string details)
{
SetActivityId(activityId);
Information(workerInfo.SiteName, workerInfo.StampName, workerInfo.WorkerName, details);
}
void IScaleTracer.TraceUpdateWorker(string activityId, IWorkerInfo workerInfo, string details)
{
SetActivityId(activityId);
UpdateWorker(workerInfo.SiteName, workerInfo.StampName, workerInfo.WorkerName, workerInfo.LoadFactor, details);
}
protected virtual async Task <bool> RequestAddWorker(string activityId, IEnumerable <IWorkerInfo> workers, IWorkerInfo manager, bool force)
{
string addedStampName = null;
if (!force && workers.Count() >= _settings.MaxWorkers)
{
_tracer.TraceWarning(activityId, manager, string.Format("Unable to add new worker due to maximum number of workers ({0}) allowed.", _settings.MaxWorkers));
}
else
{
// try on each stamps
var stampNames = workers.GroupBy(w => w.StampName).Select(g => g.Key);
addedStampName = await _eventHandler.AddWorker(activityId, stampNames, 1);
if (!string.IsNullOrEmpty(addedStampName))
{
_tracer.TraceAddWorker(activityId, manager, string.Format("New worker added to {0} stamp", addedStampName));
}
else
{
_tracer.TraceWarning(activityId, manager, string.Format("Unable to add worker to existing {0} stamps.", stampNames.ToDisplayString()));
}
}
return(!string.IsNullOrEmpty(addedStampName));
}
