public IEnumerable<JobChunk> SplitJob(JobInfo info)
{
Logger.Log.Instance.Info(string.Format("DefaultJobChunksProvider. Split job. JobId '{0}'",
info.JobId));
var blobDirectory = AzureClient.Instance.BlobClient.GetBlobDirectoryReference(info.DataSource.ToString());
if (blobDirectory == null)
throw new JobProcessorException("Data source is not available");
var blobs = blobDirectory.ListBlobs();
foreach (var blob in blobs)
{
var chunk = new JobChunk()
{
Data = blob.Uri.ToString(),
IsBlob = true,
BlobContainer = blob.Container.Name,
BlobName = (blob as CloudBlockBlob).Name,
Handler = info.Handler,
Mode = ProcessingMode.Map,
ResponseQueueName = JobProcessor.RoleSettings.ChunkResponseQueue
};
chunk.ChunkUid.ChunkId = Guid.NewGuid().ToString();
chunk.ChunkUid.JobId = info.JobId;
chunk.ChunkUid.JobName = info.JobName;
Logger.Log.Instance.Info(string.Format("DefaultJobChunksProvider. Create new map chunk for JobId '{0}'. ChunkId: '{1}'",
chunk.ChunkUid.JobId,
chunk.ChunkUid.ChunkId));
yield return chunk;
}
}
public bool ProcessJob(JobInfo jobInfo, Action<JobInfo, JobProcessStatus> callback)
{
try
{
Logger.Log.Instance.Info(string.Format("JobProcessManager. Process job. Message Id: {0}, PopReceipt: {1}",
jobInfo.JobMessageId,
jobInfo.PopReceipt));
_jobHistoryUpdater.UpdateJobStatus(jobInfo, JobProcessStatus.New);
if(_jobProcessDataCollection.ContainsKey(jobInfo.JobId))
_jobProcessDataCollection.Remove(jobInfo.JobId);
_jobProcessDataCollection.Add(jobInfo.JobId, new JobProcessData(_factory, jobInfo));
_jobProcessDataCollection[jobInfo.JobId].JobCompleteCallback = callback;
foreach (var chunk in _jobChunksProvider.SplitJob(jobInfo))
{
_jobJobChunkRegistrator.RegisterNewMapChunk(chunk.ChunkUid);
_jobHistoryUpdater.AddChunckInfo(chunk);
_chunkDispatcher.Dispatch(chunk);
}
_jobHistoryUpdater.UpdateJobStatus(jobInfo, JobProcessStatus.MapRunning);
return true;
}
catch (Exception ex)
{
Logger.Log.Instance.Exception(ex, "JobProcessManager. Job processing failed.");
return false;
}
}
public bool ReturnJob(JobInfo jobInfo)
{
try
{
if (jobInfo == null)
return false;
Logger.Log.Instance.Info(string.Format("DefaultJobSupplier. Return message to the queue. Id: {0}, PopReceipt: {1}",
jobInfo.JobMessageId,
jobInfo.PopReceipt));
// Implement retrun Job
//var message = new CloudQueueMessage(string.Empty)
//{
// //Id = jobInfo.JobMessageId,
// //PopReceipt = jobInfo.PopReceipt
//};
//_queue.UpdateMessage(message, new TimeSpan(0), MessageUpdateFields.Visibility);
return true;
}
catch (Exception ex)
{
Logger.Log.Instance.Exception(ex, string.Format("DefaultJobSupplier. Failed to return the job (JobId: {1}, MessageId: {2}, popReceipt: {3}) from the queue {0}.", RoleSettings.JobRequestsQueueName, jobInfo.JobId, jobInfo.JobMessageId, jobInfo.PopReceipt));
return false;
}
}
private void OnResumeJobProcessing(JobInfo jobInfo, JobProcessStatus status)
{
Logger.Log.Instance.Info(string.Format("JobMonitor. OnResumeJobProcessing. MessageId: {0}, PopReceipt: {1}. Status: {2}",
jobInfo.JobMessageId,
jobInfo.PopReceipt,
status));
try
{
switch (status)
{
case JobProcessStatus.Completed:
_jobSupplier.RemoveJob(jobInfo);
break;
case JobProcessStatus.Failed:
_jobSupplier.ReturnJob(jobInfo);
break;
default:
Logger.Log.Instance.Warning("JobMonitor. OnResumeJobProcessing - unsupported status");
break;
}
}
catch (Exception ex)
{
Logger.Log.Instance.Exception(ex, string.Format("JobMonitor. OnResumeJobProcessing error. MessageId: {0}, Status: {1}",
jobInfo.JobMessageId,
status));
}
}
public bool RemoveJob(JobInfo jobInfo)
{
try
{
if (jobInfo == null)
return false;
Logger.Log.Instance.Info(string.Format("DefaultJobSupplier. Remove message. Id: {0}, PopReceipt: {1}",
jobInfo.JobMessageId,
jobInfo.PopReceipt));
_queue.DeleteMessage(jobInfo.JobMessageId, jobInfo.PopReceipt);
return true;
}
catch (Exception ex)
{
Logger.Log.Instance.Exception(ex, string.Format("DefaultJobSupplier. Failed to remove the job (JobId: {1}, MessageId: {2}, popReceipt: {3}) from the queue {0}.", RoleSettings.JobRequestsQueueName, jobInfo.JobId, jobInfo.JobMessageId, jobInfo.PopReceipt));
return false;
}
}
public ResultsCollector(JobInfo jobInfo)
{
_jobInfo = jobInfo;
_results = new SortedList<string, List<object>>();
}
public MapResultsCollector(JobInfo jobInfo)
: base(jobInfo)
{
}
private JobInfo parseQueueMessage(CloudQueueMessage message)
{
if (message == null)
{
Logger.Log.Instance.Info(string.Format("DefaultJobSupplier. Cannot parse queue null message"));
}
Logger.Log.Instance.Info(string.Format("DefaultJobSupplier. Parse queue message into JobInfo. Id: {0}, PopReceipt: {1}",
message.Id,
message.PopReceipt));
var jsonMessage = Newtonsoft.Json.Linq.JObject.Parse(message.AsString);
var jobInfo = new JobInfo();
jobInfo.JobId = jsonMessage.Property("JobId").Value.ToString() + "_" + Guid.NewGuid().ToString("N");
jobInfo.JobName = jsonMessage.Property("Name").Value.ToString();
jobInfo.DataSource = jsonMessage.Property("DataSource").Value.ToString();
jobInfo.Handler = jsonMessage.Property("Handler").Value.ToString();
jobInfo.JobMessageId = message.Id;
jobInfo.PopReceipt = message.PopReceipt;
return jobInfo;
}
public ReduceResultsCollector(JobInfo jobInfo)
: base(jobInfo)
{
}
public JobProcessData(IFactory factory, JobInfo jobInfo)
{
this.JobInfo = jobInfo;
MapResultsCollector = factory.CreateMapResultsCollector(jobInfo);
ReduceResultsCollector = factory.CreateReduceResultsCollector(jobInfo);
}