public IEnumerable<JobChunk> SplitJob(JobInfo info) { Logger.Log.Instance.Info(string.Format("DefaultJobChunksProvider. Split job. JobId '{0}'", info.JobId)); var blobDirectory = AzureClient.Instance.BlobClient.GetBlobDirectoryReference(info.DataSource.ToString()); if (blobDirectory == null) throw new JobProcessorException("Data source is not available"); var blobs = blobDirectory.ListBlobs(); foreach (var blob in blobs) { var chunk = new JobChunk() { Data = blob.Uri.ToString(), IsBlob = true, BlobContainer = blob.Container.Name, BlobName = (blob as CloudBlockBlob).Name, Handler = info.Handler, Mode = ProcessingMode.Map, ResponseQueueName = JobProcessor.RoleSettings.ChunkResponseQueue }; chunk.ChunkUid.ChunkId = Guid.NewGuid().ToString(); chunk.ChunkUid.JobId = info.JobId; chunk.ChunkUid.JobName = info.JobName; Logger.Log.Instance.Info(string.Format("DefaultJobChunksProvider. Create new map chunk for JobId '{0}'. ChunkId: '{1}'", chunk.ChunkUid.JobId, chunk.ChunkUid.ChunkId)); yield return chunk; } }
public bool ProcessJob(JobInfo jobInfo, Action<JobInfo, JobProcessStatus> callback) { try { Logger.Log.Instance.Info(string.Format("JobProcessManager. Process job. Message Id: {0}, PopReceipt: {1}", jobInfo.JobMessageId, jobInfo.PopReceipt)); _jobHistoryUpdater.UpdateJobStatus(jobInfo, JobProcessStatus.New); if(_jobProcessDataCollection.ContainsKey(jobInfo.JobId)) _jobProcessDataCollection.Remove(jobInfo.JobId); _jobProcessDataCollection.Add(jobInfo.JobId, new JobProcessData(_factory, jobInfo)); _jobProcessDataCollection[jobInfo.JobId].JobCompleteCallback = callback; foreach (var chunk in _jobChunksProvider.SplitJob(jobInfo)) { _jobJobChunkRegistrator.RegisterNewMapChunk(chunk.ChunkUid); _jobHistoryUpdater.AddChunckInfo(chunk); _chunkDispatcher.Dispatch(chunk); } _jobHistoryUpdater.UpdateJobStatus(jobInfo, JobProcessStatus.MapRunning); return true; } catch (Exception ex) { Logger.Log.Instance.Exception(ex, "JobProcessManager. Job processing failed."); return false; } }
public bool ReturnJob(JobInfo jobInfo) { try { if (jobInfo == null) return false; Logger.Log.Instance.Info(string.Format("DefaultJobSupplier. Return message to the queue. Id: {0}, PopReceipt: {1}", jobInfo.JobMessageId, jobInfo.PopReceipt)); // Implement retrun Job //var message = new CloudQueueMessage(string.Empty) //{ // //Id = jobInfo.JobMessageId, // //PopReceipt = jobInfo.PopReceipt //}; //_queue.UpdateMessage(message, new TimeSpan(0), MessageUpdateFields.Visibility); return true; } catch (Exception ex) { Logger.Log.Instance.Exception(ex, string.Format("DefaultJobSupplier. Failed to return the job (JobId: {1}, MessageId: {2}, popReceipt: {3}) from the queue {0}.", RoleSettings.JobRequestsQueueName, jobInfo.JobId, jobInfo.JobMessageId, jobInfo.PopReceipt)); return false; } }
private void OnResumeJobProcessing(JobInfo jobInfo, JobProcessStatus status) { Logger.Log.Instance.Info(string.Format("JobMonitor. OnResumeJobProcessing. MessageId: {0}, PopReceipt: {1}. Status: {2}", jobInfo.JobMessageId, jobInfo.PopReceipt, status)); try { switch (status) { case JobProcessStatus.Completed: _jobSupplier.RemoveJob(jobInfo); break; case JobProcessStatus.Failed: _jobSupplier.ReturnJob(jobInfo); break; default: Logger.Log.Instance.Warning("JobMonitor. OnResumeJobProcessing - unsupported status"); break; } } catch (Exception ex) { Logger.Log.Instance.Exception(ex, string.Format("JobMonitor. OnResumeJobProcessing error. MessageId: {0}, Status: {1}", jobInfo.JobMessageId, status)); } }
public bool RemoveJob(JobInfo jobInfo) { try { if (jobInfo == null) return false; Logger.Log.Instance.Info(string.Format("DefaultJobSupplier. Remove message. Id: {0}, PopReceipt: {1}", jobInfo.JobMessageId, jobInfo.PopReceipt)); _queue.DeleteMessage(jobInfo.JobMessageId, jobInfo.PopReceipt); return true; } catch (Exception ex) { Logger.Log.Instance.Exception(ex, string.Format("DefaultJobSupplier. Failed to remove the job (JobId: {1}, MessageId: {2}, popReceipt: {3}) from the queue {0}.", RoleSettings.JobRequestsQueueName, jobInfo.JobId, jobInfo.JobMessageId, jobInfo.PopReceipt)); return false; } }
public ResultsCollector(JobInfo jobInfo) { _jobInfo = jobInfo; _results = new SortedList<string, List<object>>(); }
public MapResultsCollector(JobInfo jobInfo) : base(jobInfo) { }
private JobInfo parseQueueMessage(CloudQueueMessage message) { if (message == null) { Logger.Log.Instance.Info(string.Format("DefaultJobSupplier. Cannot parse queue null message")); } Logger.Log.Instance.Info(string.Format("DefaultJobSupplier. Parse queue message into JobInfo. Id: {0}, PopReceipt: {1}", message.Id, message.PopReceipt)); var jsonMessage = Newtonsoft.Json.Linq.JObject.Parse(message.AsString); var jobInfo = new JobInfo(); jobInfo.JobId = jsonMessage.Property("JobId").Value.ToString() + "_" + Guid.NewGuid().ToString("N"); jobInfo.JobName = jsonMessage.Property("Name").Value.ToString(); jobInfo.DataSource = jsonMessage.Property("DataSource").Value.ToString(); jobInfo.Handler = jsonMessage.Property("Handler").Value.ToString(); jobInfo.JobMessageId = message.Id; jobInfo.PopReceipt = message.PopReceipt; return jobInfo; }
public ReduceResultsCollector(JobInfo jobInfo) : base(jobInfo) { }
public JobProcessData(IFactory factory, JobInfo jobInfo) { this.JobInfo = jobInfo; MapResultsCollector = factory.CreateMapResultsCollector(jobInfo); ReduceResultsCollector = factory.CreateReduceResultsCollector(jobInfo); }