/// <summary> /// Marks a batch as closed so that no further jobs are accepted, anymore /// </summary> private void MarkBatchAsClosed(string batchId) { using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString)) { var batch = repo.GetBatches().Where(b => b.Id.Equals(batchId)).FirstOrDefault(); if (batch != null) { batch.Status = BatchStatus.Closed; } repo.UpdateBatch(batch); } }
/// <summary> /// Performs initialization of storage access etc. /// </summary> public virtual void Initialize() { // Get the storage connection string _storageConnectionString = CloudConfigurationManager.GetSetting(GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME); if (string.IsNullOrEmpty(_storageConnectionString.Trim())) { throw new Exception("Missing Storage Connection String " + GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME); } _serviceBusConnectionString = CloudConfigurationManager.GetSetting(GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME); if (string.IsNullOrEmpty(_serviceBusConnectionString.Trim())) { throw new Exception("Missing ServiceBus Connection String " + GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME); } _cancellationServiceBusTopicName = string.Format("{0}_{1}", GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_TOPICPREFIX, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_CANCELJOBS); // Create the queue client for processing the queues of the different batches and create the default-batch queue CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_storageConnectionString); _queueClient = storageAccount.CreateCloudQueueClient(); _queueClient.RetryPolicy = new Microsoft.WindowsAzure.Storage.RetryPolicies.LinearRetry( TimeSpan.FromSeconds(GlobalConstants.STORAGE_RETRY_MILLISECONDS_BETWEEN_RETRY), GlobalConstants.STORAGE_RETRY_MAX_ATTEMPTS); _sharedQueue = _queueClient.GetQueueReference(GlobalConstants.DEFAULT_BATCH_ID); if (!_sharedQueue.Exists()) { _sharedQueue.Create(); } // add the shared queue info to the table storage if it doesn't exist using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString)) { if (repo.GetBatches().Where(b => b.Id.Equals(GlobalConstants.DEFAULT_BATCH_ID)).FirstOrDefault() == null) { var batch = new Batch(); batch.Id = GlobalConstants.DEFAULT_BATCH_ID; batch.BatchName = GlobalConstants.DEFAULT_BATCH_NAME; batch.Status = BatchStatus.Open; batch.Priority = 999; repo.CreateBatch(new Repositories.Entities.BatchEntity(batch)); } } }
public IQueryable <Batch> GetBatches() { try { string getAllIdentifierForLogs = "--get all--"; // Log the query request GeresEventSource.Log.WebApiMonitoringQueryBatchReceived(getAllIdentifierForLogs); // Retrieve all batches from the table using (var repo = RepositoryFactory.CreateBatchRepository(CloudConfigurationManager.GetSetting(GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME))) { var allBatches = repo.GetBatches(); // Log the query completion GeresEventSource.Log.WebApiMonitoringQueryBatchSuccessful(getAllIdentifierForLogs, allBatches.Count()); // Return the batches to the client var allBatchDtos = (from b in allBatches select new Batch() { Id = b.Id, BatchName = b.Name, Priority = b.Priority, RequiresDedicatedWorker = b.RequiresDedicatedWorker, Status = b.Status }); return(allBatchDtos.AsQueryable()); } } catch (HttpResponseException ex) { throw ex; } catch (Exception ex) { Trace.TraceError("WebAPI - JobMonitorController -- Unknown exception occured: {0}{1}{2}", ex.Message, Environment.NewLine, ex.StackTrace); GeresEventSource.Log.WebApiUnknownExceptionOccured(ex.Message, ex.StackTrace); throw new HttpResponseException(HttpStatusCode.InternalServerError); } }
/// <summary> /// A batch of jobs is ready for processing - create a dedicated queue for the batch /// </summary> public Batch CreateBatch(Batch newBatch) { // create an identifier for the batch newBatch.Id = Guid.NewGuid().ToString(); newBatch.Status = BatchStatus.Open; // create a new queue for this batch var batchQueue = _queueClient.GetQueueReference(newBatch.Id); batchQueue.CreateIfNotExists(); // add the queue info to the table storage using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString)) { repo.CreateBatch(new Repositories.Entities.BatchEntity(newBatch)); } return(newBatch); }
/// <summary> /// Determines if the batch queue is available (not closed) or not. /// </summary> private bool IsBatchQueueOpen(string batchId) { var result = false; using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString)) { var batch = repo.GetBatches().Where(b => b.Id.Equals(batchId)).FirstOrDefault(); if (batch != null) { if (batch.Status == BatchStatus.Open) { result = true; } } } return(result); }
public static NonFactoredTabPresenter Create(INonFactoredTabView view) { return(new NonFactoredTabPresenter(view, RepositoryFactory.CreateBatchRepository())); }
public void Initialize(ITenantManager tenantManager) { // Set the Tenant Manager _tenantManager = tenantManager; // Create the factory for built-in jobs _builtInJobsFactory = new Geres.Engine.JobFactories.JobWorkerFactory(); // Read configuration settings _storageConnectionString = CloudConfigurationManager.GetSetting(GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME); if (string.IsNullOrEmpty(_storageConnectionString.Trim())) { throw new Exception("Missing configuration setting " + GlobalConstants.STORAGE_CONNECTIONSTRING_CONFIGNAME); } _serviceBusConnectionString = CloudConfigurationManager.GetSetting(GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME); if (string.IsNullOrEmpty(_serviceBusConnectionString.Trim())) { throw new Exception("Missing configuration setting " + GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME); } // Parse further configuration settings _messageLockForProcessingInMinutes = int.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.MAX_MESSAGE_LOCK_TIME_IN_MIN)); _messageMaxRetryAttempts = int.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.MAX_MESSAGE_RETRY_ATTEMPTS)); // Read the setting for the single-job-cancellations _singleJobCancellationEnabled = bool.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.GERES_CONFIG_JOB_SINGLECANCELLATION_ENABLED)); _singleJobCancellationTimeWindow = TimeSpan.FromSeconds(int.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.GERES_CONFIG_JOB_SINGLECANCELLATION_TIMEWINDOW_IN_SECONDS))); _singleJobCancellationMessageTimeToLive = TimeSpan.FromSeconds(int.Parse(CloudConfigurationManager.GetSetting(GlobalConstants.GERES_CONFIG_JOB_SINGLECANCELLATION_MESSAGETIMETOLIVE))); // Retrieve storage account from connection string. _storageAccount = CloudStorageAccount.Parse(_storageConnectionString); _queueClient = _storageAccount.CreateCloudQueueClient(); // Create the default batch queue if it does not exist var defaultBatchQueue = _queueClient.GetQueueReference(GlobalConstants.DEFAULT_BATCH_ID); defaultBatchQueue.CreateIfNotExists(); // create a retry policy _requestOptions = new QueueRequestOptions { // create a retry policy, retry every 30 seconds for a maximum of 10 times RetryPolicy = new LinearRetry( TimeSpan.FromMilliseconds(GlobalConstants.STORAGE_RETRY_MILLISECONDS_BETWEEN_RETRY), GlobalConstants.STORAGE_RETRY_MAX_ATTEMPTS), }; // Create the service bus connection // Initialize the connection to Service Bus Queue _notificationServiceBusClient = new JobNotificationServiceBus( _serviceBusConnectionString, string.Format("{0}_{1}", GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_TOPICPREFIX, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_JOBSTATUS)); // Create the service bus connection // Initialize the connection to Service Bus Queue _cancellationServiceBusClient = new JobCancellationServiceBus( _serviceBusConnectionString, string.Format("{0}_{1}", GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_TOPICPREFIX, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_CANCELJOBS), _singleJobCancellationTimeWindow, _singleJobCancellationMessageTimeToLive); // Create all required repositories _jobsRepository = RepositoryFactory.CreateJobsRepository(_storageConnectionString); _batchRepository = RepositoryFactory.CreateBatchRepository(_storageConnectionString); }
public static ScheduleTabPresenter Create(IScheduleTabView view) { return(new ScheduleTabPresenter(view, RepositoryFactory.CreateBatchRepository())); }
public static RepurchasesTabPresenter Create(IRepurchasesTabView view) { return(new RepurchasesTabPresenter(view, RepositoryFactory.CreateBatchRepository())); }
public static BatchChargesTabPresenter Create(IBatchChargesTabView view) { return(new BatchChargesTabPresenter(view, RepositoryFactory.CreateBatchRepository())); }
public static InvoiceBatchesPresenter Create(IInvoiceBatchesView view) { return(new InvoiceBatchesPresenter(view, RepositoryFactory.CreateBatchRepository())); }
public static CreditsTabPresenter Create(ICreditsTabView view) { return(new CreditsTabPresenter(view, RepositoryFactory.CreateBatchRepository())); }
/// <summary> /// Initializes all required resources by the AutoSclaer /// </summary> public void Initialize(IDictionary <string, string> resourceDetails, IAutoScalerPolicy policy) { if (resourceDetails == null) { throw new ArgumentException("Argument 'resourceDetails' cannot be null, missing properties for initialization in AutoScaler!"); } // // Read configuration settings // _defaultBatchId = resourceDetails[Constants.QUEUE_HANDLER_INIT_PROP_DEFAULT_BATCH_ID]; _storageConn = resourceDetails[Constants.QUEUE_HANDLER_INIT_PROP_STORAGECONNECTION]; _serviceBusConn = resourceDetails[GlobalConstants.SERVICEBUS_INTERNAL_CONNECTIONSTRING_CONFIGNAME]; _checkForJobProcessorCommandsIntervalInSeconds = int.Parse(resourceDetails[GlobalConstants.GERES_CONFIG_AUTOSCALER_CHECKFORJOBHOSTUPDATES_INTERVAL]); if (string.IsNullOrEmpty(_defaultBatchId) || string.IsNullOrEmpty(_storageConn)) { throw new ArgumentException("Missing properties in resourceDetails parameter!"); } // // Check the loaded policy // if (policy == null) { throw new ArgumentException("Missing property for autoscaler policy!"); } _policy = policy; // // Create all required repositories // _jobHostRepository = RepositoryFactory.CreateJobHostRepository(_storageConn, RoleEnvironment.DeploymentId); _batchRepository = RepositoryFactory.CreateBatchRepository(_storageConn); _roleOpsRepository = RepositoryFactory.CreateRoleOperationStatusRepository(_storageConn); // // Connect to storage for the jobs-queue // var cloudAccount = CloudStorageAccount.Parse(_storageConn); _cloudQueueClient = cloudAccount.CreateCloudQueueClient(); _defaultQueue = _cloudQueueClient.GetQueueReference(_defaultBatchId); _defaultQueue.CreateIfNotExists(); // // Connect to the service bus for autoscaler commands // _jobHostServiceBus = new JobHostServiceBus(_serviceBusConn, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_COMMANDSFORAUTOSCALER, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_COMMANDSFORJOBHOST); var subscribeToJobHostCommandsTask = Task.Run(() => { SubscriptionClient client = null; while (true) { try { if (client == null || client.IsClosed) { client = _jobHostServiceBus.CreateSubscription(GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_COMMANDSFORAUTOSCALER, GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBHOSTUPDATES); } var msg = client.Receive(TimeSpan.FromSeconds(_checkForJobProcessorCommandsIntervalInSeconds)); if (msg != null) { if (ProcessCommandFromJobHost(msg)) { msg.Complete(); } else { msg.Abandon(); } } } catch (Exception ex) { client = null; Geres.Diagnostics.GeresEventSource.Log.AutoScalerWorkerAutoScalerListenForJobHostUpdatesFailed ( RoleEnvironment.CurrentRoleInstance.Id, RoleEnvironment.DeploymentId, string.Format("Topic: {0}, Subscription: {1}", GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBHOSTUPDATES, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_COMMANDSFORAUTOSCALER), ex.Message, ex.StackTrace ); } } }); }
/// <summary> /// Internal mechanism to handle jobs being put on the queue /// </summary> private string SubmitJobToQueue(Job job, string batchId) { // Log that we're trying to submit a job GeresEventSource.Log.EngineJobControllerTrySubmittingJob(job.JobName, job.JobType, batchId); // Initialize the properties of the job job.JobId = Guid.NewGuid().ToString(); job.SubmittedAt = DateTime.UtcNow; job.ScheduledBy = GetCurrentIdentityName(); job.Status = JobStatus.Submitted; job.JobProcessorPackageName = job.JobProcessorPackageName; job.TenantName = job.TenantName; // Log the submission of the Message GeresEventSource.Log.EngineJobControllerSubmittingJobToQueue ( job.JobId, job.JobName, job.JobType, job.SubmittedAt, job.ScheduledBy, batchId, job.JobProcessorPackageName, false ); // the message is simply the id of the var message = new CloudQueueMessage(job.JobId); CloudQueue queue = null; string batchName = ""; // Determine which queue to use if (!string.IsNullOrEmpty(batchId)) { // use a custom queue if the batch does exist using (var repo = RepositoryFactory.CreateBatchRepository(_storageConnectionString)) { var batchExisting = repo.GetBatch(batchId); if (batchExisting == null) { throw new InvalidOperationException(GlobalConstants.BATCH_DOES_NOT_EXIST_EXCEPTION); } else { batchName = batchExisting.Name; } } // Create the queue for the batch if it does not exist queue = _queueClient.GetQueueReference(batchId); queue.CreateIfNotExists(); } else { // use Q0 queue = _sharedQueue; batchId = GlobalConstants.DEFAULT_BATCH_ID; batchName = GlobalConstants.DEFAULT_BATCH_NAME; } // Update the repository with the job try { using (var repo = RepositoryFactory.CreateJobsRepository(_storageConnectionString)) { var jobEntity = new Repositories.Entities.JobEntity(job); jobEntity.BatchId = batchId; jobEntity.BatchName = batchName; repo.CreateJob(jobEntity); } } catch (Exception ex) { GeresEventSource.Log.EngineJobControllerFailedUpdatingJob(job.JobId, job.JobName, job.JobType, ex.Message, ex.StackTrace); throw new ApplicationException(GlobalConstants.SUBMIT_JOB_FAILED, ex); } // Try submitting the Job to the Queue try { queue.AddMessage(message); GeresEventSource.Log.EngineJobControllerSubmittedToBatch(job.JobId, job.JobName, job.JobType, batchId, batchName); } catch (Exception ex) { GeresEventSource.Log.EngineJobControllerSubmittingToBatchQueueFailed(job.JobId, job.JobName, job.JobType, batchId, batchName, ex.Message, ex.StackTrace); try { using (var repo = RepositoryFactory.CreateJobsRepository(_storageConnectionString)) { job.Status = JobStatus.Failed; repo.UpdateJob(new Repositories.Entities.JobEntity(job)); } } catch (Exception exi) { GeresEventSource.Log.EngineJobControllerFailedUpdatingJob(job.JobId, job.JobName, job.JobType, exi.Message, exi.StackTrace); } throw new ApplicationException(GlobalConstants.SUBMIT_JOB_FAILED, ex); } return(job.JobId); }