public int RemoveStaleEntries(uint expirationTime) { int removedCount = 0; lock (this.innerList) { LinkedListNode <InstanceEndpoint> current = this.innerList.First; while (current != null) { LinkedListNode <InstanceEndpoint> next = current.Next; InstanceEndpoint instance = current.Value; if (instance.LastRefreshTimestamp < expirationTime) { this.innerList.Remove(current); removedCount++; Debug.WriteLine("UpdateWorkerList: Removed worker {0} from the list.", (object)instance.IPAddress); AntaresEventProvider.EventWriteLBHttpDispatchEndpointInfoMessage(site.Name, instance.IPAddress, "UpdateWorkerList", "Removing worker from routing list"); } current = next; } } return(removedCount); }
private int UpdateWorkerList(SiteMetadata site, string[] currentWorkerSet) { uint now = GetCurrentTickCount(); // Union the cached list of known workers with the list provided by the FE. The list from the FE // may be stale, so we have to accept new workers generously and carefully age out stale workers. foreach (string ipAddress in currentWorkerSet) { InstanceEndpoint instance = site.Endpoints.GetOrAdd(ipAddress); instance.LastRefreshTimestamp = now; // Clear the busy status for workers whose busy status is ready to expire. if (instance.IsBusy && instance.IsBusyUntil < now) { site.Endpoints.ClearBusyStatus(instance); } // Periodically trace the health statistics of the workers if (instance.NextMetricsTraceTime == 0) { instance.NextMetricsTraceTime = now + WorkerMetricsTraceInterval; } else if (now > instance.NextMetricsTraceTime && Monitor.TryEnter(instance)) { try { Debug.WriteLine("UpdateWorkerList: Worker metrics for site {0}: {1}", site.Name, instance.ToString()); AntaresEventProvider.EventWriteLBHttpDispatchEndpointMetrics(site.Name, instance.IPAddress, instance.PendingRequestCount, instance.IsBusy, instance.Weight); instance.NextMetricsTraceTime = now + WorkerMetricsTraceInterval; } finally { Monitor.Exit(instance); } } } // This is a performance-sensitive code path so we throttle the age-out logic to avoid using excess CPU. if (now >= site.NextWorkerExpirationTick && Monitor.TryEnter(site)) { try { site.Endpoints.RemoveStaleEntries(site.NextWorkerExpirationTick); // Wait M seconds before doing another worker expiration check. site.NextWorkerExpirationTick = now + WorkerExpirationCheckInterval; } finally { Monitor.Exit(site); } } site.IsBurstMode = site.Endpoints.Count < this.burstLimit; return(site.Endpoints.Count); }
internal void SetIsBusy(InstanceEndpoint instance) { lock (this.innerList) { uint duration = SiteRequestDispatcher.BusyStatusDuration; instance.IsBusyUntil = HttpScaleEnvironment.TickCount + duration; instance.IsBusy = true; Debug.WriteLine("Set instance {0} as busy for the next {1}ms. New weight: {2}", instance.IPAddress, duration, instance.Weight); AntaresEventProvider.EventWriteLBHttpDispatchEndpointInfoMessage(site.Name, instance.IPAddress, "SetIsBusy", string.Format("Set instance busy for {0}ms. New weight: {1}", duration, instance.Weight)); // Busy adds a large amount of weight to an instance, moving it towards the back. this.UpdatePositionFromBack(instance.Node); } }
internal void ClearBusyStatus(InstanceEndpoint instance) { lock (this.innerList) { instance.IsBusy = false; // Clearing the busy flag is going to give it a big priority boost, // though it's not clear where it will end up. Since most instances will // have similar status, it should be safe to assume clearing a busy flag // will move it toward the front of the priority list. this.UpdatePositionFromFront(instance.Node); Debug.WriteLine("Removed busy status from {0}. New weight: {1}", instance.IPAddress, instance.Weight); AntaresEventProvider.EventWriteLBHttpDispatchEndpointInfoMessage(site.Name, instance.IPAddress, "ClearBusyStatus", "Removing busy status. New weight: " + instance.Weight); } }
// Caller must be holding an async lock private async Task <InstanceEndpoint> ScaleOutAsync( SiteMetadata site, int targetInstanceCount, InstanceEndpoint previousEndpoint) { Debug.WriteLine("Attempting to scale out to " + targetInstanceCount); AntaresEventProvider.EventWriteLBHttpDispatchSiteInfoMessage(site.Name, "ScaleOut", string.Format("Attempting to scale out to {0} instances.", targetInstanceCount)); string[] ipAddresses = await this.OnScaleOutAsync(site.Name, targetInstanceCount); if (ipAddresses != null) { this.UpdateWorkerList(site, ipAddresses); } // It is expected in most cases that this will return the newly added worker (if any). return(site.Endpoints.ReserveBestInstance(previousEndpoint)); }
public InstanceEndpoint GetOrAdd(string ipAddress) { InstanceEndpoint newInstance; lock (this.innerList) { InstanceEndpoint existingInstance; if (this.TryGetValueInternal(ipAddress, out existingInstance)) { return(existingInstance); } newInstance = new InstanceEndpoint(ipAddress); newInstance.Node = this.innerList.AddFirst(newInstance); } Debug.WriteLine("UpdateWorkerList: Added worker {0}.", (object)ipAddress); AntaresEventProvider.EventWriteLBHttpDispatchEndpointInfoMessage(this.site.Name, ipAddress, "UpdateWorkerList", "Added worker"); return(newInstance); }
public void OnRequestCompleted( string requestId, string siteName, int statusCode, string statusPhrase, string ipAddress) { SiteMetadata site; if (!this.knownSites.TryGetValue(siteName, out site)) { throw new ArgumentException(string.Format("Site '{0}' does not exist in the list of known sites.", siteName)); } InstanceEndpoint instance; if (!site.Endpoints.TryGetValue(ipAddress, out instance)) { Debug.WriteLine("OnRequestCompleted: Worker '{0}' was not found.", (object)ipAddress); AntaresEventProvider.EventWriteLBHttpDispatchEndpointInfoMessage(site.Name, ipAddress, "OnRequestCompleted", "Worker not found"); return; } site.Endpoints.OnRequestCompleted(instance); // If this is the request we're using as a health sample, clear it so another request can be selected. if (instance.HealthTrackingRequestId == requestId) { instance.HealthTrackingRequestId = null; } // If the request is rejected because the instance is too busy, mark the endpoint as busy for the next N seconds. // This flag will be cleared the next time a request arrives after the busy status expires. if (statusCode == 429 && statusPhrase == "Instance Too Busy") { site.Endpoints.SetIsBusy(instance); } }
public virtual async Task <string> DispatchRequestAsync( string requestId, string siteName, string defaultHostName, string[] knownWorkers) { SiteMetadata site = this.knownSites.GetOrAdd(siteName, name => new SiteMetadata(siteName)); this.UpdateWorkerList(site, knownWorkers); InstanceEndpoint bestInstance = site.Endpoints.ReserveBestInstance(null); if (site.IsBurstMode) { if (bestInstance.PendingRequestCount <= 1) { // Endpoint is idle - choose it. return(bestInstance.IPAddress); } else { // All endpoints are occupied; try to scale out. using (var scaleLock = await site.ScaleLock.LockAsync(MaxLockTime)) { if (scaleLock.TimedOut) { // The caller should return 429 to avoid overloading the current role Debug.WriteLine("Timed-out waiting to start a scale operation in burst mode."); AntaresEventProvider.EventWriteLBHttpDispatchSiteWarningMessage(site.Name, "DispatchRequest", string.Format("Timed-out ({0}ms) waiting to start a scale operation in burst mode. Reverting to {1}.", MaxLockTime, bestInstance.IPAddress)); return(bestInstance.IPAddress); } if (site.IsBurstMode) { // No instances are idle, scale-out and send the request to the new instance. // Note that there will be a cold-start penalty for this request, but it's // been decided that this is better than sending a request to an existing // but potentially CPU-pegged instance. int targetInstanceCount = Math.Min(this.burstLimit, site.Endpoints.Count + 1); InstanceEndpoint newInstance = await this.ScaleOutAsync( site, targetInstanceCount, bestInstance); return(newInstance.IPAddress); } } } } // Pick one request at a time to use for latency tracking uint now = GetCurrentTickCount(); if (Interlocked.CompareExchange(ref bestInstance.HealthTrackingRequestId, requestId, null) == null) { bestInstance.HealthTrackingRequestStartTime = now; } if (bestInstance.PendingRequestCount <= 1) { // This is an idle worker (the current request is the pending one). return(bestInstance.IPAddress); } if (bestInstance.IsBusy) { using (var result = await site.ScaleLock.LockAsync(MaxLockTime)) { if (result.TimedOut) { // Scale operations are unhealthy Debug.WriteLine("Timed-out waiting to start a scale operation on busy instance."); AntaresEventProvider.EventWriteLBHttpDispatchSiteWarningMessage(site.Name, "DispatchRequest", string.Format("Timed-out ({0}ms) waiting to start a scale operation on a busy instance {1}.", MaxLockTime, bestInstance.IPAddress)); return(bestInstance.IPAddress); } if (!bestInstance.IsBusy) { // The instance became healthy while we were waiting. return(bestInstance.IPAddress); } // Serialize scale-out requests to avoid overloading our infrastructure. Each serialized scale // request will request one more instance from the previous request which can result in rapid-scale out. bestInstance = await this.ScaleOutAsync(site, site.Endpoints.Count + 1, bestInstance); if (bestInstance.IsBusy) { // Scale-out failed Debug.WriteLine("Best instance is still busy after a scale operation."); AntaresEventProvider.EventWriteLBHttpDispatchSiteWarningMessage(site.Name, "DispatchRequest", string.Format("Best instance {0} is still busy after a scale operation.", bestInstance.IPAddress)); } return(bestInstance.IPAddress); } } bool isQuestionable = bestInstance.PendingRequestCount > QuestionablePendingRequestCount || bestInstance.HealthTrackingRequestStartTime < now - QuestionableRequestLatency; if (isQuestionable && bestInstance.NextAllowedPingTime <= now && Interlocked.CompareExchange(ref bestInstance.PingLock, 1, 0) == 0) { bool isHealthy; try { isHealthy = await this.PingAsync(siteName, bestInstance, defaultHostName); } finally { bestInstance.NextAllowedPingTime = now + PingInterval; bestInstance.PingLock = 0; } if (!isHealthy) { site.Endpoints.SetIsBusy(bestInstance); // Serialize scale-out requests to avoid overloading our infrastructure. Each serialized scale // request will request one more instance from the previous request which can result in rapid-scale out. using (var result = await site.ScaleLock.LockAsync(MaxLockTime)) { if (result.TimedOut) { // Scale operations are unhealthy Debug.WriteLine("Timed-out waiting to start a scale operation after unhealthy ping."); AntaresEventProvider.EventWriteLBHttpDispatchSiteWarningMessage(site.Name, "DispatchRequest", string.Format("Timed-out ({0}ms) waiting to start a scale operation after unhealthy ping to {1}.", MaxLockTime, bestInstance.IPAddress)); return(bestInstance.IPAddress); } bestInstance = await this.ScaleOutAsync(site, site.Endpoints.Count + 1, bestInstance); if (bestInstance.IsBusy) { // Scale-out failed Debug.WriteLine("Best worker is still busy after a ping-initiated scale."); AntaresEventProvider.EventWriteLBHttpDispatchSiteWarningMessage(site.Name, "DispatchRequest", string.Format("Best worker {0} is still busy after a ping-initiated scale.", bestInstance.IPAddress)); } return(bestInstance.IPAddress); } } } return(bestInstance.IPAddress); }
private static void WebDeployPublishTrace(object sender, DeploymentTraceEventArgs e) { AntaresEventProvider.EventWritePublishFailOverServiceDebugEvent(e.Message); }
private void Publish(object unusedState) { Operation operation = null; try { lock (_pendingPublishOperations) { operation = _pendingPublishOperations.Dequeue(); } if (operation == null) { PublishHelper.LogVerboseInformation("Publish: Thread {0} did not find any operations to process", Thread.CurrentThread.ManagedThreadId); return; } // This was not required earlier but now that every site will be synced twice for every incoming publish (Look at the postsync override in publishextender), // there are chances that same operation might get scheduled in concurrent threads which is not a supported web deploy scenario. bool canContinue = true; lock (_operationsInProgress) { if (_operationsInProgress.Contains(operation.SiteName)) { PublishHelper.LogVerboseInformation("Publish: An operation for {0} is already in progress. Thread {1} will mark it as not started to be scheduled later.", operation.SiteName, Thread.CurrentThread.ManagedThreadId); operation.Status = PublishOperationStatus.NotStarted; canContinue = false; } else { _operationsInProgress.Add(operation.SiteName); } } if (!canContinue) { lock (_completedOperations) { _completedOperations.Enqueue(operation); } return; } operation.ThreadID = Thread.CurrentThread.ManagedThreadId; operation.Status = PublishOperationStatus.Error; DeploymentBaseOptions srcBaseOptions = new DeploymentBaseOptions(); AntaresEventProvider.EventWritePublishFailOverServiceProgressInformation(operation.ThreadID, operation.SiteName); using (DeploymentObject depObj = DeploymentManager.CreateObject(DeploymentWellKnownProvider.Manifest, PublishHelper.GetPublishManifest(operation.PhysicalPath), srcBaseOptions)) { try { DeploymentBaseOptions destBaseOptions = new DeploymentBaseOptions(); destBaseOptions.ComputerName = operation.PublishUrl; string modifiedPhysicalPath = operation.PhysicalPath; string ipDefaultValue = GetFileServerIP(ref modifiedPhysicalPath); var fileServerIPParameter = new DeploymentSyncParameter( FileServerIPParameterName, FileServerIPParameterDescription, ipDefaultValue, string.Empty); var fileServerIPEntry = new DeploymentSyncParameterEntry( DeploymentSyncParameterEntryKind.ProviderPath, DeploymentWellKnownProvider.ContentPath.ToString(), string.Empty, string.Empty); fileServerIPParameter.Add(fileServerIPEntry); var contentPathParameter = new DeploymentSyncParameter( ContentPathParameterName, ContentPathParameterDescription, modifiedPhysicalPath, DeploymentWellKnownTag.PhysicalPath.ToString()); var contentParamEntry = new DeploymentSyncParameterEntry( DeploymentSyncParameterEntryKind.ProviderPath, DeploymentWellKnownProvider.ContentPath.ToString(), string.Empty, string.Empty); contentPathParameter.Add(contentParamEntry); var setAclParameter = new DeploymentSyncParameter( SetAclParameterName, SetAclParameterDescription, modifiedPhysicalPath, DeploymentWellKnownTag.SetAcl.ToString()); var setAclParamEntry = new DeploymentSyncParameterEntry( DeploymentSyncParameterEntryKind.ProviderPath, DeploymentWellKnownProvider.SetAcl.ToString(), string.Empty, string.Empty); setAclParameter.Add(setAclParamEntry); depObj.SyncParameters.Add(fileServerIPParameter); depObj.SyncParameters.Add(contentPathParameter); depObj.SyncParameters.Add(setAclParameter); destBaseOptions.UserName = operation.AdminCredential.UserName; destBaseOptions.Password = operation.AdminCredential.Password; destBaseOptions.AuthenticationType = "basic"; destBaseOptions.Trace += new EventHandler <DeploymentTraceEventArgs>(WebDeployPublishTrace); destBaseOptions.TraceLevel = System.Diagnostics.TraceLevel.Verbose; depObj.SyncTo(destBaseOptions, new DeploymentSyncOptions()); operation.Status = PublishOperationStatus.Completed; AntaresEventProvider.EventWritePublishFailOverServicePublishComplete(operation.SiteName); } catch (Exception e) { AntaresEventProvider.EventWritePublishFailOverServiceFailedToPublishSite(operation.SiteName, e.ToString()); } } } catch (Exception e) { if ((e is DeploymentDetailedException && ((DeploymentDetailedException)e).ErrorCode == DeploymentErrorCode.FileOrFolderNotFound) || e is WebHostingObjectNotFoundException) { operation.Status = PublishOperationStatus.SourceOrDestinationInvalid; } AntaresEventProvider.EventWritePublishFailOverServiceFailedToGetSourceSite(operation.SiteName, e.ToString()); } finally { lock (_completedOperations) { PublishHelper.LogVerboseInformation("Publish: Thread {0} qeuing completed operation for site: {1}", operation.ThreadID, operation.SiteName); _completedOperations.Enqueue(operation); } lock (_operationsInProgress) { _operationsInProgress.Remove(operation.SiteName); } _continueEvent.Set(); PublishHelper.LogVerboseInformation("Publish: Thread {0} exiting", Thread.CurrentThread.ManagedThreadId); } }
/// <summary> /// Web Deploy does not parameterize or run any rules (parameterization is one of the rules) when deletion is used. /// So to work around that deletecontent does a simple publish of empty content to the server and gets the IP that is hosting the volume. /// After finding the IP the deletion is called on that content path. This will effectively delete "\\someipaddress\volumename\guidfolder" /// </summary> /// <param name="unusedState">This is not used. This is the signature for the waitcallback expected by threadpool.</param> private void DeleteContent(object unusedState) { Operation operation = null; try { lock (_pendingDeleteOperations) { operation = _pendingDeleteOperations.Dequeue(); } if (operation == null) { PublishHelper.LogVerboseInformation("DeleteContent: Thread {0} did not find any operations to process", Thread.CurrentThread.ManagedThreadId); return; } operation.ThreadID = Thread.CurrentThread.ManagedThreadId; operation.Status = PublishOperationStatus.Error; DeploymentBaseOptions srcBaseOptions = new DeploymentBaseOptions(); AntaresEventProvider.EventWritePublishFailOverServiceProgressInformation(operation.ThreadID, operation.SiteName); string remoteIP = string.Empty; bool errorOcurred = false; using (DeploymentObject depObj = DeploymentManager.CreateObject(DeploymentWellKnownProvider.ContentPath, Path.GetTempPath(), srcBaseOptions)) { try { DeploymentBaseOptions destBaseOptions = new DeploymentBaseOptions(); destBaseOptions.ComputerName = operation.PublishUrl; string modifiedPhysicalPath = operation.PhysicalPath; string ipDefaultValue = GetFileServerIP(ref modifiedPhysicalPath); var queryServerIPParameter = new DeploymentSyncParameter( QueryServerIPParameterName, QueryServerIPParameterDescription, string.Empty, DeploymentWellKnownTag.PhysicalPath.ToString()); var queryServerIPParameterEntry = new DeploymentSyncParameterEntry( DeploymentSyncParameterEntryKind.ProviderPath, DeploymentWellKnownProvider.ContentPath.ToString(), string.Empty, string.Empty); queryServerIPParameter.Add(queryServerIPParameterEntry); var contentPathParameter = new DeploymentSyncParameter( ContentPathParameterName, ContentPathParameterDescription, modifiedPhysicalPath, DeploymentWellKnownTag.PhysicalPath.ToString()); var contentParamEntry = new DeploymentSyncParameterEntry( DeploymentSyncParameterEntryKind.ProviderPath, DeploymentWellKnownProvider.ContentPath.ToString(), string.Empty, string.Empty); contentPathParameter.Add(contentParamEntry); depObj.SyncParameters.Add(contentPathParameter); depObj.SyncParameters.Add(queryServerIPParameter); destBaseOptions.UserName = operation.AdminCredential.UserName; destBaseOptions.Password = operation.AdminCredential.Password; destBaseOptions.AuthenticationType = "basic"; depObj.SyncTo(destBaseOptions, new DeploymentSyncOptions()); } catch (Exception e) { bool unhandledException = false; // In cases where the site was deleted on the source before it was ever synced to the destination // mark as the destination invalid and set the status so that its never attempted again. if (e is DeploymentDetailedException && ((DeploymentDetailedException)e).ErrorCode == DeploymentErrorCode.ERROR_INVALID_PATH) { string[] messageArray = e.Message.Split(new string[] { "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries); if (messageArray.Length > 0) { remoteIP = messageArray[0]; } else { errorOcurred = true; } } else if (e is DeploymentException) { errorOcurred = true; } else { // its not a deployment exception. This is an exception not handled by web deploy such as duplicate key exception. // This needs to be retried. unhandledException = true; errorOcurred = true; } if (errorOcurred) { AntaresEventProvider.EventWritePublishFailOverServiceFailedToPublishSite(operation.SiteName, e.ToString()); operation.Status = unhandledException ? PublishOperationStatus.Error : PublishOperationStatus.SourceOrDestinationInvalid; } } } if (!errorOcurred) { string[] pathParts = operation.PhysicalPath.Split(new char[] { '\\' }, StringSplitOptions.RemoveEmptyEntries); string remotePath = string.Concat(@"\\", remoteIP, @"\", pathParts[1], @"\", pathParts[2]); using (DeploymentObject depObj = DeploymentManager.CreateObject(DeploymentWellKnownProvider.Auto, string.Empty, srcBaseOptions)) { try { DeploymentBaseOptions destBaseOptions = new DeploymentBaseOptions(); destBaseOptions.ComputerName = operation.PublishUrl; destBaseOptions.UserName = operation.AdminCredential.UserName; destBaseOptions.Password = operation.AdminCredential.Password; destBaseOptions.AuthenticationType = "basic"; destBaseOptions.Trace += new EventHandler <DeploymentTraceEventArgs>(WebDeployPublishTrace); destBaseOptions.TraceLevel = System.Diagnostics.TraceLevel.Verbose; var syncOptions = new DeploymentSyncOptions(); syncOptions.DeleteDestination = true; depObj.SyncTo(DeploymentWellKnownProvider.ContentPath, remotePath, destBaseOptions, syncOptions); operation.Status = PublishOperationStatus.Completed; AntaresEventProvider.EventWritePublishFailOverServicePublishComplete(operation.SiteName); } catch (Exception e) { var ex = e as DeploymentDetailedException; if (ex != null && ex.ErrorCode == DeploymentErrorCode.FileOrFolderNotFound) { operation.Status = PublishOperationStatus.SourceOrDestinationInvalid; } else { AntaresEventProvider.EventWritePublishFailOverServiceFailedToPublishSite(operation.SiteName, e.ToString()); } } } } } finally { lock (_completedOperations) { PublishHelper.LogVerboseInformation("DeleteContent: Thread {0} queuing completed operation for site: {1}", operation.ThreadID, operation.SiteName); _completedOperations.Enqueue(operation); } _continueEvent.Set(); PublishHelper.LogVerboseInformation("DeleteContent: Thread {0} exiting.", Thread.CurrentThread.ManagedThreadId); } }
private void Controller() { while (true) { PublishHelper.LogVerboseInformation("ControllerThread: Waiting for operations"); int eventId = WaitHandle.WaitAny(_waitHandleArray, Interval); if (eventId == 0) { //If exitevent is signalled, the process is exiting. break; } if (!PublishHelper.ShouldBeginProcessingPublishOperation) { PublishHelper.LogVerboseInformation("ControllerThread: BeginProcessing is currently off"); _continueEvent.Reset(); continue; } int maxConcurrentOperations = PublishHelper.MaxConcurrentSyncOperations; ThreadPool.SetMaxThreads(maxConcurrentOperations, maxConcurrentOperations); if (!_operationInitialized) { _operationInitialized = true; PublishHelper.LogVerboseInformation("ControllerThread: Initializing publish operation"); } #region Schedule Operations _continueEvent.Reset(); PublishOperation publishOperation; try { while (PublishHelper.GetNextPublishOperation(out publishOperation)) { if (_exitEvent.WaitOne(0)) { //If this is signalled, the process is exiting return; } string physicalPath = string.Empty; bool siteInvalid = false; try { // if the site was added or updated and then deleted, the add and update // operations get always scheduled before the delete. // so if the site is deleted, this will throw object not found exception. // set the object as invalid so that this is never attempted again. physicalPath = publishOperation.PhysicalPath; } catch (WebHostingObjectNotFoundException) { siteInvalid = true; } WaitCallback callback = null; Operation operation = new Operation( publishOperation.OperationId, publishOperation.SiteName, physicalPath, PublishHelper.PublishUrl, PublishHelper.AdminCredential); int currentOperationCount = 0; if (siteInvalid) { operation.Status = PublishOperationStatus.SourceOrDestinationInvalid; lock (_completedOperations) { PublishHelper.LogVerboseInformation("ControllerThread: Queuing completed operation for site: {0} as it does not exist", operation.SiteName); _completedOperations.Enqueue(operation); } continue; } PublishHelper.LogVerboseInformation("ControllerThread: Adding Operation {0} for {1} to the queue", operation.SiteName, publishOperation.SiteState); if (publishOperation.SiteState == SiteState.Deleted) { lock (_pendingDeleteOperations) { _pendingDeleteOperations.Enqueue(operation); currentOperationCount += _pendingDeleteOperations.Count; } callback = new WaitCallback(DeleteContent); } else { lock (_pendingPublishOperations) { _pendingPublishOperations.Enqueue(operation); currentOperationCount += _pendingPublishOperations.Count; } callback = new WaitCallback(Publish); } AntaresEventProvider.EventWritePublishFailOverServiceOperationQueued(operation.SiteName); ThreadPool.QueueUserWorkItem(callback); AntaresEventProvider.EventWritePublishFailOverServiceDebugEvent(String.Format("ControllerThread: Current in queue: {0}, maxCount: {1}", currentOperationCount, maxConcurrentOperations)); if (currentOperationCount >= maxConcurrentOperations) { /// Dont queue more operations than the max count to let other publishers pick these operations up. /// This does not mean that we will wait for the entire interval. As soon as a thread finishes, the continue event will /// get signalled and new operations will get picked up till maxcount operations are queued again. break; } } } catch (Exception ex) { AntaresEventProvider.EventWritePublishFailOverServiceUnableToGetNextOperation(ex.ToString()); } #endregion } }