/// <summary>
/// Marks a batch as closed so that no further jobs are accepted, anymore
/// </summary>
private void MarkBatchAsClosed(string batchId)
{
using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString))
{
var batch = repo.GetBatches().Where(b => b.Id.Equals(batchId)).FirstOrDefault();
if (batch != null)
{
batch.Status = BatchStatus.Closed;
}
repo.UpdateBatch(batch);
}
}
/// <summary>
/// Performs initialization of storage access etc.
/// </summary>
public virtual void Initialize()
{
// Get the storage connection string
_storageConnectionString = CloudConfigurationManager.GetSetting(GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME);
if (string.IsNullOrEmpty(_storageConnectionString.Trim()))
{
throw new Exception("Missing Storage Connection String " + GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME);
}
_serviceBusConnectionString = CloudConfigurationManager.GetSetting(GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME);
if (string.IsNullOrEmpty(_serviceBusConnectionString.Trim()))
{
throw new Exception("Missing ServiceBus Connection String " + GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME);
}
_cancellationServiceBusTopicName = string.Format("{0}_{1}", GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_TOPICPREFIX, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_CANCELJOBS);
// Create the queue client for processing the queues of the different batches and create the default-batch queue
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_storageConnectionString);
_queueClient = storageAccount.CreateCloudQueueClient();
_queueClient.RetryPolicy = new Microsoft.WindowsAzure.Storage.RetryPolicies.LinearRetry(
TimeSpan.FromSeconds(GlobalConstants.STORAGE_RETRY_MILLISECONDS_BETWEEN_RETRY),
GlobalConstants.STORAGE_RETRY_MAX_ATTEMPTS);
_sharedQueue = _queueClient.GetQueueReference(GlobalConstants.DEFAULT_BATCH_ID);
if (!_sharedQueue.Exists())
{
_sharedQueue.Create();
}
// add the shared queue info to the table storage if it doesn't exist
using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString))
{
if (repo.GetBatches().Where(b => b.Id.Equals(GlobalConstants.DEFAULT_BATCH_ID)).FirstOrDefault() == null)
{
var batch = new Batch();
batch.Id = GlobalConstants.DEFAULT_BATCH_ID;
batch.BatchName = GlobalConstants.DEFAULT_BATCH_NAME;
batch.Status = BatchStatus.Open;
batch.Priority = 999;
repo.CreateBatch(new Repositories.Entities.BatchEntity(batch));
}
}
}
public IQueryable <Batch> GetBatches()
{
try
{
string getAllIdentifierForLogs = "--get all--";
// Log the query request
GeresEventSource.Log.WebApiMonitoringQueryBatchReceived(getAllIdentifierForLogs);
// Retrieve all batches from the table
using (var repo = RepositoryFactory.CreateBatchRepository(CloudConfigurationManager.GetSetting(GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME)))
{
var allBatches = repo.GetBatches();
// Log the query completion
GeresEventSource.Log.WebApiMonitoringQueryBatchSuccessful(getAllIdentifierForLogs, allBatches.Count());
// Return the batches to the client
var allBatchDtos = (from b in allBatches
select new Batch()
{
Id = b.Id,
BatchName = b.Name,
Priority = b.Priority,
RequiresDedicatedWorker = b.RequiresDedicatedWorker,
Status = b.Status
});
return(allBatchDtos.AsQueryable());
}
}
catch (HttpResponseException ex)
{
throw ex;
}
catch (Exception ex)
{
Trace.TraceError("WebAPI - JobMonitorController -- Unknown exception occured: {0}{1}{2}", ex.Message, Environment.NewLine, ex.StackTrace);
GeresEventSource.Log.WebApiUnknownExceptionOccured(ex.Message, ex.StackTrace);
throw new HttpResponseException(HttpStatusCode.InternalServerError);
}
}
/// <summary>
/// A batch of jobs is ready for processing - create a dedicated queue for the batch
/// </summary>
public Batch CreateBatch(Batch newBatch)
{
// create an identifier for the batch
newBatch.Id = Guid.NewGuid().ToString();
newBatch.Status = BatchStatus.Open;
// create a new queue for this batch
var batchQueue = _queueClient.GetQueueReference(newBatch.Id);
batchQueue.CreateIfNotExists();
// add the queue info to the table storage
using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString))
{
repo.CreateBatch(new Repositories.Entities.BatchEntity(newBatch));
}
return(newBatch);
}
/// <summary>
/// Determines if the batch queue is available (not closed) or not.
/// </summary>
private bool IsBatchQueueOpen(string batchId)
{
var result = false;
using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString))
{
var batch = repo.GetBatches().Where(b => b.Id.Equals(batchId)).FirstOrDefault();
if (batch != null)
{
if (batch.Status == BatchStatus.Open)
{
result = true;
}
}
}
return(result);
}
public static NonFactoredTabPresenter Create(INonFactoredTabView view)
{
return(new NonFactoredTabPresenter(view, RepositoryFactory.CreateBatchRepository()));
}
public void Initialize(ITenantManager tenantManager)
{
// Set the Tenant Manager
_tenantManager = tenantManager;
// Create the factory for built-in jobs
_builtInJobsFactory = new Geres.Engine.JobFactories.JobWorkerFactory();
// Read configuration settings
_storageConnectionString = CloudConfigurationManager.GetSetting(GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME);
if (string.IsNullOrEmpty(_storageConnectionString.Trim()))
{
throw new Exception("Missing configuration setting " + GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME);
}
_serviceBusConnectionString = CloudConfigurationManager.GetSetting(GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME);
if (string.IsNullOrEmpty(_serviceBusConnectionString.Trim()))
{
throw new Exception("Missing configuration setting " + GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME);
}
// Parse further configuration settings
_messageLockForProcessingInMinutes = int.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.MAX_MESSAGE_LOCK_TIME_IN_MIN));
_messageMaxRetryAttempts = int.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.MAX_MESSAGE_RETRY_ATTEMPTS));
// Read the setting for the single-job-cancellations
_singleJobCancellationEnabled = bool.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.GERES_CONFIG_JOB_SINGLECANCELLATION_ENABLED));
_singleJobCancellationTimeWindow = TimeSpan.FromSeconds(int.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.GERES_CONFIG_JOB_SINGLECANCELLATION_TIMEWINDOW_IN_SECONDS)));
_singleJobCancellationMessageTimeToLive = TimeSpan.FromSeconds(int.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.GERES_CONFIG_JOB_SINGLECANCELLATION_MESSAGETIMETOLIVE)));
// Retrieve storage account from connection string.
_storageAccount = CloudStorageAccount.Parse(_storageConnectionString);
_queueClient = _storageAccount.CreateCloudQueueClient();
// Create the default batch queue if it does not exist
var defaultBatchQueue = _queueClient.GetQueueReference(GlobalConstants.DEFAULT_BATCH_ID);
defaultBatchQueue.CreateIfNotExists();
// create a retry policy
_requestOptions = new QueueRequestOptions
{
// create a retry policy, retry every 30 seconds for a maximum of 10 times
RetryPolicy = new LinearRetry(
TimeSpan.FromMilliseconds(GlobalConstants.STORAGE_RETRY_MILLISECONDS_BETWEEN_RETRY),
GlobalConstants.STORAGE_RETRY_MAX_ATTEMPTS),
};
// Create the service bus connection
// Initialize the connection to Service Bus Queue
_notificationServiceBusClient = new JobNotificationServiceBus(
_serviceBusConnectionString,
string.Format("{0}_{1}", GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_TOPICPREFIX, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_JOBSTATUS));
// Create the service bus connection
// Initialize the connection to Service Bus Queue
_cancellationServiceBusClient = new JobCancellationServiceBus(
_serviceBusConnectionString,
string.Format("{0}_{1}", GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_TOPICPREFIX, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_CANCELJOBS),
_singleJobCancellationTimeWindow,
_singleJobCancellationMessageTimeToLive);
// Create all required repositories
_jobsRepository = RepositoryFactory.CreateJobsRepository(_storageConnectionString);
_batchRepository = RepositoryFactory.CreateBatchRepository(_storageConnectionString);
}
public static ScheduleTabPresenter Create(IScheduleTabView view)
{
return(new ScheduleTabPresenter(view, RepositoryFactory.CreateBatchRepository()));
}
public static RepurchasesTabPresenter Create(IRepurchasesTabView view)
{
return(new RepurchasesTabPresenter(view, RepositoryFactory.CreateBatchRepository()));
}
public static BatchChargesTabPresenter Create(IBatchChargesTabView view)
{
return(new BatchChargesTabPresenter(view, RepositoryFactory.CreateBatchRepository()));
}
public static InvoiceBatchesPresenter Create(IInvoiceBatchesView view)
{
return(new InvoiceBatchesPresenter(view, RepositoryFactory.CreateBatchRepository()));
}
public static CreditsTabPresenter Create(ICreditsTabView view)
{
return(new CreditsTabPresenter(view, RepositoryFactory.CreateBatchRepository()));
}
/// <summary>
/// Initializes all required resources by the AutoSclaer
/// </summary>
public void Initialize(IDictionary <string, string> resourceDetails, IAutoScalerPolicy policy)
{
if (resourceDetails == null)
{
throw new ArgumentException("Argument 'resourceDetails' cannot be null, missing properties for initialization in AutoScaler!");
}
//
// Read configuration settings
//
_defaultBatchId = resourceDetails[Constants.QUEUE_HANDLER_INIT_PROP_DEFAULT_BATCH_ID];
_storageConn = resourceDetails[Constants.QUEUE_HANDLER_INIT_PROP_STORAGECONNECTION];
_serviceBusConn = resourceDetails[GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME];
_checkForJobProcessorCommandsIntervalInSeconds = int.Parse(resourceDetails[GlobalConstants.GERES_CONFIG_AUTOSCALER_CHECKFORJOBHOSTUPDATES_INTERVAL]);
if (string.IsNullOrEmpty(_defaultBatchId) || string.IsNullOrEmpty(_storageConn))
{
throw new ArgumentException("Missing properties in resourceDetails parameter!");
}
//
// Check the loaded policy
//
if (policy == null)
{
throw new ArgumentException("Missing property for autoscaler policy!");
}
_policy = policy;
//
// Create all required repositories
//
_jobHostRepository = RepositoryFactory.CreateJobHostRepository(_storageConn, RoleEnvironment.DeploymentId);
_batchRepository = RepositoryFactory.CreateBatchRepository(_storageConn);
_roleOpsRepository = RepositoryFactory.CreateRoleOperationStatusRepository(_storageConn);
//
// Connect to storage for the jobs-queue
//
var cloudAccount = CloudStorageAccount.Parse(_storageConn);
_cloudQueueClient = cloudAccount.CreateCloudQueueClient();
_defaultQueue = _cloudQueueClient.GetQueueReference(_defaultBatchId);
_defaultQueue.CreateIfNotExists();
//
// Connect to the service bus for autoscaler commands
//
_jobHostServiceBus = new JobHostServiceBus(_serviceBusConn, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_COMMANDSFORAUTOSCALER,
GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_COMMANDSFORJOBHOST);
var subscribeToJobHostCommandsTask = Task.Run(() =>
{
SubscriptionClient client = null;
while (true)
{
try
{
if (client == null || client.IsClosed)
{
client = _jobHostServiceBus.CreateSubscription(GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_COMMANDSFORAUTOSCALER,
GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBHOSTUPDATES);
}
var msg = client.Receive(TimeSpan.FromSeconds(_checkForJobProcessorCommandsIntervalInSeconds));
if (msg != null)
{
if (ProcessCommandFromJobHost(msg))
{
msg.Complete();
}
else
{
msg.Abandon();
}
}
}
catch (Exception ex)
{
client = null;
Geres.Diagnostics.GeresEventSource.Log.AutoScalerWorkerAutoScalerListenForJobHostUpdatesFailed
(
RoleEnvironment.CurrentRoleInstance.Id, RoleEnvironment.DeploymentId,
string.Format("Topic: {0}, Subscription: {1}", GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBHOSTUPDATES,
GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_COMMANDSFORAUTOSCALER),
ex.Message,
ex.StackTrace
);
}
}
});
}
/// <summary>
/// Internal mechanism to handle jobs being put on the queue
/// </summary>
private string SubmitJobToQueue(Job job, string batchId)
{
// Log that we're trying to submit a job
GeresEventSource.Log.EngineJobControllerTrySubmittingJob(job.JobName, job.JobType, batchId);
// Initialize the properties of the job
job.JobId = Guid.NewGuid().ToString();
job.SubmittedAt = DateTime.UtcNow;
job.ScheduledBy = GetCurrentIdentityName();
job.Status = JobStatus.Submitted;
job.JobProcessorPackageName = job.JobProcessorPackageName;
job.TenantName = job.TenantName;
// Log the submission of the Message
GeresEventSource.Log.EngineJobControllerSubmittingJobToQueue
(
job.JobId, job.JobName, job.JobType, job.SubmittedAt, job.ScheduledBy, batchId, job.JobProcessorPackageName, false
);
// the message is simply the id of the
var message = new CloudQueueMessage(job.JobId);
CloudQueue queue = null;
string batchName = "";
// Determine which queue to use
if (!string.IsNullOrEmpty(batchId))
{
// use a custom queue if the batch does exist
using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString))
{
var batchExisting = repo.GetBatch(batchId);
if (batchExisting == null)
{
throw new InvalidOperationException(GlobalConstants.BATCH_DOES_NOT_EXIST_EXCEPTION);
}
else
{
batchName = batchExisting.Name;
}
}
// Create the queue for the batch if it does not exist
queue = _queueClient.GetQueueReference(batchId);
queue.CreateIfNotExists();
}
else
{
// use Q0
queue = _sharedQueue;
batchId = GlobalConstants.DEFAULT_BATCH_ID;
batchName = GlobalConstants.DEFAULT_BATCH_NAME;
}
// Update the repository with the job
try
{
using (var repo = RepositoryFactory.CreateJobsRepository(_storageConnectionString))
{
var jobEntity = new Repositories.Entities.JobEntity(job);
jobEntity.BatchId = batchId;
jobEntity.BatchName = batchName;
repo.CreateJob(jobEntity);
}
}
catch (Exception ex)
{
GeresEventSource.Log.EngineJobControllerFailedUpdatingJob(job.JobId, job.JobName, job.JobType, ex.Message, ex.StackTrace);
throw new ApplicationException(GlobalConstants.SUBMIT_JOB_FAILED, ex);
}
// Try submitting the Job to the Queue
try
{
queue.AddMessage(message);
GeresEventSource.Log.EngineJobControllerSubmittedToBatch(job.JobId, job.JobName, job.JobType, batchId, batchName);
}
catch (Exception ex)
{
GeresEventSource.Log.EngineJobControllerSubmittingToBatchQueueFailed(job.JobId, job.JobName, job.JobType, batchId, batchName, ex.Message, ex.StackTrace);
try
{
using (var repo = RepositoryFactory.CreateJobsRepository(_storageConnectionString))
{
job.Status = JobStatus.Failed;
repo.UpdateJob(new Repositories.Entities.JobEntity(job));
}
}
catch (Exception exi)
{
GeresEventSource.Log.EngineJobControllerFailedUpdatingJob(job.JobId, job.JobName, job.JobType, exi.Message, exi.StackTrace);
}
throw new ApplicationException(GlobalConstants.SUBMIT_JOB_FAILED, ex);
}
return(job.JobId);
}
