/// <summary>
/// Create storage blob container.
/// </summary>
/// <param name="container">blob container</param>
public static void CreateContainerWithRetry(CloudBlobContainer container)
{
RetryHelper <object> .InvokeOperation(
() =>
{
if (container.CreateIfNotExists())
{
BrokerTracing.TraceInfo("[AzureQueueManager].CreateContainerWithRetry: Create the container {0}", container.Name);
}
return(null);
},
(e, count) =>
{
BrokerTracing.TraceError("Failed to create the container {0}: {1}. Retry Count = {2}", container.Name, e, count);
StorageException se = e as StorageException;
if (se != null)
{
string errorCode = BurstUtility.GetStorageErrorCode(se);
// According to test, the error code is ResourceAlreadyExists.
// There is no doc about this, add ContainerAlreadyExists here.
// TODO: Azure storage SDK 2.0
if ( // errorCode == StorageErrorCodeStrings.ResourceAlreadyExists ||
errorCode == StorageErrorCodeStrings.ContainerAlreadyExists)
{
Thread.Sleep(TimeSpan.FromMinutes(1));
}
}
});
}
/// <summary>
/// Create storage queue
/// </summary>
/// <param name="queue">storage queue</param>
/// <remarks>
/// CreateIfNotExist method throws StorageClientException when queue is
/// being deleted, so sleep for a while before retry.
/// </remarks>
public static void CreateQueueWithRetry(CloudQueue queue)
{
RetryHelper <object> .InvokeOperation(
() =>
{
if (queue.CreateIfNotExists())
{
BrokerTracing.TraceInfo("[AzureQueueManager].CreateQueueWithRetry: Create the queue {0}", queue.Name);
}
return(null);
},
(e, count) =>
{
BrokerTracing.TraceError("Failed to create the queue {0}: {1}. Retry Count = {2}", queue.Name, e, count);
StorageException se = e as StorageException;
if (se != null)
{
if (BurstUtility.GetStorageErrorCode(se) == QueueErrorCodeStrings.QueueAlreadyExists)
{
Thread.Sleep(TimeSpan.FromMinutes(1));
}
}
});
}
/// <summary>
/// Open the broker launcher service
/// </summary>
public void OpenService()
{
Trace.TraceInformation("Open service.");
//TODO: SF: remove the singleton implementation
//SingletonRegistry.Initialize(SingletonRegistry.RegistryMode.WindowsNonHA);
// for debug attach
//Thread.Sleep(60 * 1000);
bool isOnAzure = SoaHelper.IsOnAzure();
if (isOnAzure)
{
this.StartNodeMappingCacheService();
}
try
{
// richci: if this is a console application we are running in MSCS in production. Make
// sure only one instance of the console app is running at a time.
if (!isOnAzure && IsConsoleApplication && !AcquireSingleProcessLock())
{
// If another instance already created the mutex, release this handle
ReleaseSingleProcessLock();
throw new InvalidOperationException("Only one instance of the process can be run a time");
}
if (false) //!isOnAzure && !IsConsoleApplication && Win32API.IsFailoverBrokerNode())
{
//
// If this is a brokerlauncher service running as service on a failover BN, dont
// open WCF endpoints. In this configuration, the broker launcher windows service is
// for mgmt operations only. All application traffic will go through brokerlaunchers
// running as console apps in MSCS resource groups
//
// Otherwise this a HpcBroker windows service on FO BN handling mgmt operations only
//
this.launcherInstance = new BrokerLauncher(true, this.context);
}
else
{
Trace.TraceInformation("Open broker launcher service host.");
this.launcherInstance = new BrokerLauncher(false, this.context);
this.launcherHost = new ServiceHost(this.launcherInstance, new Uri(SoaHelper.GetBrokerLauncherAddress(HostName)));
BindingHelper.ApplyDefaultThrottlingBehavior(this.launcherHost);
this.launcherHost.AddServiceEndpoint(typeof(IBrokerLauncher), BindingHelper.HardCodedUnSecureNetTcpBinding, string.Empty);
this.launcherHost.AddServiceEndpoint(typeof(IBrokerLauncher), BindingHelper.HardCodedUnSecureNetTcpBinding, "Internal");
this.launcherHost.AddServiceEndpoint(typeof(IBrokerLauncher), BindingHelper.HardCodedUnSecureNetTcpBinding, "AAD");
// this.launcherHost.Credentials.UseInternalAuthenticationAsync(true).GetAwaiter().GetResult();
string addFormat = SoaHelper.BrokerLauncherAadAddressFormat;
ServiceAuthorizationBehavior myServiceBehavior = this.launcherHost.Description.Behaviors.Find <ServiceAuthorizationBehavior>();
myServiceBehavior.PrincipalPermissionMode = PrincipalPermissionMode.None;
this.launcherHost.Open();
if (BrokerLauncherSettings.Default.EnableAzureStorageQueueEndpoint)
{
if (string.IsNullOrEmpty(BrokerLauncherSettings.Default.AzureStorageConnectionString))
{
Trace.TraceError("AzureStorageConnectionString is null or empty while EnableAzureStorageQueueEndpoint is set to true");
}
else
{
this.watcher = new BrokerLauncherCloudQueueWatcher(this.launcherInstance, BrokerLauncherSettings.Default.AzureStorageConnectionString);
}
}
Trace.TraceInformation("Open broker launcher service succeeded.");
TraceHelper.TraceEvent(TraceEventType.Information, "Open broker launcher service succeeded.");
if (SoaHelper.IsSchedulerOnAzure())
{
// Broker service is enabled on scheduler node for on-premise and scheduler on Azure cluster.
// SoaDiagSvc is not expected to run on the Azure cluster.
return;
}
ISchedulerHelper helper = SchedulerHelperFactory.GetSchedulerHelper(this.context);
#if HPCPACK
ThreadPool.QueueUserWorkItem(
(object state) =>
{
try
{
RetryHelper <object> .InvokeOperation(
() =>
{
this.soaDiagAuthenticator = new SoaDiagAuthenticator();
SoaDiagService diagServiceInstance = new SoaDiagService(helper.GetClusterInfoAsync, this.soaDiagAuthenticator);
this.diagServiceHost = new ServiceHost(
diagServiceInstance,
#if DEBUG
new Uri("http://localhost/SoaDiagService"),
#endif
new Uri(SoaHelper.GetDiagServiceAddress(HostName)));
BindingHelper.ApplyDefaultThrottlingBehavior(this.diagServiceHost, SoaDiagSvcMaxConcurrentCalls);
var endpoint = this.diagServiceHost.AddServiceEndpoint(typeof(ISoaDiagService), BindingHelper.HardCodedDiagServiceNetTcpBinding, string.Empty);
endpoint.Behaviors.Add(new SoaDiagServiceErrorHandler());
#if DEBUG
var httpEndpoint = this.diagServiceHost.AddServiceEndpoint(typeof(ISoaDiagService), new BasicHttpBinding(), string.Empty);
httpEndpoint.Behaviors.Add(new SoaDiagServiceErrorHandler());
#endif
this.diagServiceHost.Open();
TraceHelper.TraceEvent(TraceEventType.Information, "Open soa diag service succeeded.");
this.cleanupService = new DiagCleanupService(helper.GetClusterInfoAsync);
this.cleanupService.Start();
TraceHelper.TraceEvent(TraceEventType.Information, "Open soa diag cleanup service succeeded.");
return(null);
},
(ex, count) =>
{
TraceHelper.TraceEvent(TraceEventType.Error, "Failed to open soa diag service: {0}. Retry Count = {1}", ex, count);
this.CloseSoaDiagService();
Thread.Sleep(RetryPeriod);
});
}
catch (Exception e)
{
TraceHelper.TraceEvent(TraceEventType.Error, "Failed to open soa diag service after all retry: {0}", e);
}
});
#endif
#if HPCPACK
ThreadPool.QueueUserWorkItem((object state) =>
{
try
{
RetryHelper <object> .InvokeOperation(
() =>
{
this.azureStorageCleaner = new AzureStorageCleaner(helper);
this.azureStorageCleaner.Start();
TraceHelper.TraceEvent(TraceEventType.Information,
"Open Azure storage cleanup service succeeded.");
return(null);
},
(ex, count) =>
{
TraceHelper.TraceEvent(
TraceEventType.Error,
"Failed to open Azure storage cleanup service: {0}. Retry Count = {1}",
ex,
count);
if (this.azureStorageCleaner != null)
{
this.azureStorageCleaner.Close();
}
Thread.Sleep(RetryPeriod);
});
}
catch (Exception e)
{
TraceHelper.TraceEvent(TraceEventType.Error,
"Failed to open Azure storage cleanup service after all retry: {0}", e);
}
});
#endif
}
}
catch (Exception e)
{
TraceHelper.TraceEvent(TraceEventType.Critical, "Failed to open service: {0}", e);
throw;
}
}