private ObjectBlockRequest <T> ConvertToObjectBlockRequest <T>(IObjectBlockSettings <T> settings)
{
var request = new ObjectBlockRequest <T>(settings.Object,
_taskConfiguration.MaxLengthForNonCompressedData);
request.ApplicationName = _taskExecutionInstance.ApplicationName;
request.TaskName = _taskExecutionInstance.TaskName;
request.TaskExecutionId = _taskExecutionInstance.TaskExecutionId;
request.TaskDeathMode = _taskExecutionOptions.TaskDeathMode;
if (_taskExecutionOptions.TaskDeathMode == TaskDeathMode.KeepAlive)
{
request.KeepAliveDeathThreshold = _taskExecutionOptions.KeepAliveDeathThreshold.Value;
}
else
{
request.OverrideDeathThreshold = _taskExecutionOptions.OverrideThreshold.Value;
}
request.ReprocessReferenceValue = settings.ReferenceValueToReprocess;
request.ReprocessOption = settings.ReprocessOption;
SetConfigurationOverridableSettings(request, settings);
return(request);
}
public IList <IObjectBlockContext <T> > GenerateObjectBlocks <T>(ObjectBlockRequest <T> blockRequest)
{
var blocks = new List <IObjectBlockContext <T> >();
if (!string.IsNullOrEmpty(blockRequest.ReprocessReferenceValue))
{
blocks = LoadObjectBlocksOfTask(blockRequest);
}
else
{
var forceBlocks = GetForcedObjectBlocks(blockRequest);
blocks.AddRange(forceBlocks);
if (GetBlocksRemaining(blockRequest, blocks) > 0)
{
LoadFailedAndDeadObjectBlocks(blockRequest, blocks);
}
if (GetBlocksRemaining(blockRequest, blocks) > 0 && blockRequest.Object != null)
{
var newBlock = GenerateNewObjectBlock(blockRequest);
blocks.Add(newBlock);
}
}
if (!blocks.Any())
{
LogEmptyBlockEvent(blockRequest.TaskExecutionId, blockRequest.ApplicationName, blockRequest.TaskName);
}
return(blocks.OrderBy(x => long.Parse(x.Block.ObjectBlockId)).ToList());
}
private List <IObjectBlockContext <T> > GetDeadObjectBlocks <T>(ObjectBlockRequest <T> blockRequest, int blockCountLimit)
{
var deadBlockRequest = CreateDeadBlocksRequest(blockRequest, blockCountLimit);
var deadBlocks = _blockRepository.FindDeadObjectBlocks <T>(deadBlockRequest);
return(CreateObjectBlockContexts(blockRequest, deadBlocks));
}
private async Task <ObjectBlock <T> > GenerateObjectBlockAsync <T>(ObjectBlockRequest <T> blockRequest)
{
var request = new ObjectBlockCreateRequest <T>(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName),
blockRequest.TaskExecutionId,
blockRequest.Object,
blockRequest.CompressionThreshold);
var objectBlock = (await _blockRepository.AddObjectBlockAsync(request).ConfigureAwait(false)).Block;
await Task.Delay(10).ConfigureAwait(false); // guarantee that each block has a unique created date
return(objectBlock);
}
private List <IObjectBlockContext <T> > CreateObjectBlockContexts <T>(ObjectBlockRequest <T> blockRequest, IList <ObjectBlock <T> > objectBlocks)
{
var blocks = new List <IObjectBlockContext <T> >();
foreach (var objectBlock in objectBlocks)
{
var blockContext = CreateObjectBlockContext(blockRequest, objectBlock);
blocks.Add(blockContext);
}
return(blocks);
}
private ObjectBlock <T> GenerateObjectBlock <T>(ObjectBlockRequest <T> blockRequest)
{
var request = new ObjectBlockCreateRequest <T>(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName),
blockRequest.TaskExecutionId,
blockRequest.Object,
blockRequest.CompressionThreshold);
var objectBlock = _blockRepository.AddObjectBlock(request).Block;
Thread.Sleep(5); // guarantee that each block has a unique created date
return(objectBlock);
}
private List <IObjectBlockContext <T> > GetFailedObjectBlocks <T>(ObjectBlockRequest <T> blockRequest, int blockCountLimit)
{
var failedBlockRequest = new FindFailedBlocksRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName),
blockRequest.TaskExecutionId,
blockRequest.BlockType,
DateTime.UtcNow - blockRequest.FailedTaskDetectionRange,
DateTime.UtcNow,
blockCountLimit,
blockRequest.FailedTaskRetryLimit
);
var failedBlocks = _blockRepository.FindFailedObjectBlocks <T>(failedBlockRequest);
return(CreateObjectBlockContexts <T>(blockRequest, failedBlocks));
}
private async Task LoadFailedAndDeadObjectBlocksAsync <T>(ObjectBlockRequest <T> blockRequest, List <IObjectBlockContext <T> > blocks)
{
int blocksRemaining = GetBlocksRemaining(blockRequest, blocks);
if (blockRequest.ReprocessDeadTasks)
{
blocks.AddRange(await GetDeadObjectBlocksAsync(blockRequest, blocksRemaining).ConfigureAwait(false));
blocksRemaining = blockRequest.MaxBlocks - blocks.Count;
}
if (GetBlocksRemaining(blockRequest, blocks) > 0 && blockRequest.ReprocessFailedTasks)
{
blocks.AddRange(await GetFailedObjectBlocksAsync(blockRequest, blocksRemaining).ConfigureAwait(false));
}
}
private void LoadFailedAndDeadObjectBlocks <T>(ObjectBlockRequest <T> blockRequest, List <IObjectBlockContext <T> > blocks)
{
int blocksRemaining = GetBlocksRemaining(blockRequest, blocks);
if (blockRequest.ReprocessDeadTasks)
{
blocks.AddRange(GetDeadObjectBlocks(blockRequest, blocksRemaining));
blocksRemaining = blockRequest.MaxBlocks - blocks.Count;
}
if (GetBlocksRemaining(blockRequest, blocks) > 0 && blockRequest.ReprocessFailedTasks)
{
blocks.AddRange(GetFailedObjectBlocks(blockRequest, blocksRemaining));
}
}
private List <IObjectBlockContext <T> > LoadObjectBlocksOfTask <T>(ObjectBlockRequest <T> blockRequest)
{
var failedBlockRequest = new FindBlocksOfTaskRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName),
blockRequest.TaskExecutionId,
blockRequest.BlockType,
blockRequest.ReprocessReferenceValue,
blockRequest.ReprocessOption);
var blocksOfTask = _blockRepository.FindObjectBlocksOfTask <T>(failedBlockRequest);
if (!blocksOfTask.Any())
{
return(new List <IObjectBlockContext <T> >());
}
return(CreateObjectBlockContexts(blockRequest, blocksOfTask));
}
private IObjectBlockContext <T> CreateObjectBlockContext <T>(ObjectBlockRequest <T> blockRequest, ObjectBlock <T> objectBlock, int forcedBlockQueueId = 0)
{
var attempt = objectBlock.Attempt + 1;
var createRequest = new BlockExecutionCreateRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName),
blockRequest.TaskExecutionId,
blockRequest.BlockType,
objectBlock.ObjectBlockId,
attempt);
var blockExecutionId = _blockRepository.AddObjectBlockExecution(createRequest);
var blockContext = new ObjectBlockContext <T>(_objectBlockRepository,
_taskExecutionRepository,
blockRequest.ApplicationName,
blockRequest.TaskName,
blockRequest.TaskExecutionId,
objectBlock,
blockExecutionId,
forcedBlockQueueId.ToString());
return(blockContext);
}
private List <IObjectBlockContext <T> > GetForcedObjectBlocks <T>(ObjectBlockRequest <T> blockRequest)
{
var forcedBlockRequest = new QueuedForcedBlocksRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName),
blockRequest.TaskExecutionId,
blockRequest.BlockType);
var queuedForcedBlocks = _blockRepository.GetQueuedForcedObjectBlocks <T>(forcedBlockRequest);
var forcedBlocks = new List <IObjectBlockContext <T> >();
foreach (var queuedForcedBlock in queuedForcedBlocks)
{
forcedBlocks.Add(CreateObjectBlockContext(blockRequest, queuedForcedBlock.ObjectBlock, queuedForcedBlock.ForcedBlockQueueId));
}
if (forcedBlocks.Any())
{
DequeueForcedBlocks(blockRequest, forcedBlocks.Select(x => x.ForcedBlockQueueId).ToList());
}
return(forcedBlocks);
}
private async Task <List <IObjectBlockContext <T> > > LoadObjectBlocksOfTaskAsync <T>(ObjectBlockRequest <T> blockRequest)
{
var failedBlockRequest = new FindBlocksOfTaskRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName),
blockRequest.TaskExecutionId,
blockRequest.BlockType,
blockRequest.ReprocessReferenceValue,
blockRequest.ReprocessOption);
var blocksOfTask = await _blockRepository.FindObjectBlocksOfTaskAsync <T>(failedBlockRequest).ConfigureAwait(false);
if (!blocksOfTask.Any())
{
return(new List <IObjectBlockContext <T> >());
}
return(await CreateObjectBlockContextsAsync(blockRequest, blocksOfTask).ConfigureAwait(false));
}
private async Task <List <IObjectBlockContext <T> > > GetDeadObjectBlocksAsync <T>(ObjectBlockRequest <T> blockRequest, int blockCountLimit)
{
var deadBlockRequest = CreateDeadBlocksRequest(blockRequest, blockCountLimit);
var deadBlocks = await _blockRepository.FindDeadObjectBlocksAsync <T>(deadBlockRequest).ConfigureAwait(false);
return(await CreateObjectBlockContextsAsync(blockRequest, deadBlocks).ConfigureAwait(false));
}
private IObjectBlockContext <T> GenerateNewObjectBlock <T>(ObjectBlockRequest <T> blockRequest)
{
var newObjectBlock = GenerateObjectBlock(blockRequest);
return(CreateObjectBlockContext(blockRequest, newObjectBlock));
}
private async Task <List <IObjectBlockContext <T> > > GetFailedObjectBlocksAsync <T>(ObjectBlockRequest <T> blockRequest, int blockCountLimit)
{
var failedBlockRequest = new FindFailedBlocksRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName),
blockRequest.TaskExecutionId,
blockRequest.BlockType,
DateTime.UtcNow - blockRequest.FailedTaskDetectionRange,
DateTime.UtcNow,
blockCountLimit,
blockRequest.FailedTaskRetryLimit
);
var failedBlocks = await _blockRepository.FindFailedObjectBlocksAsync <T>(failedBlockRequest).ConfigureAwait(false);
return(await CreateObjectBlockContextsAsync <T>(blockRequest, failedBlocks).ConfigureAwait(false));
}
private async Task <List <IObjectBlockContext <T> > > CreateObjectBlockContextsAsync <T>(ObjectBlockRequest <T> blockRequest, IList <ObjectBlock <T> > objectBlocks)
{
var blocks = new List <IObjectBlockContext <T> >();
foreach (var objectBlock in objectBlocks)
{
var blockContext = await CreateObjectBlockContextAsync(blockRequest, objectBlock).ConfigureAwait(false);
blocks.Add(blockContext);
}
return(blocks);
}
private async Task <IObjectBlockContext <T> > GenerateNewObjectBlockAsync <T>(ObjectBlockRequest <T> blockRequest)
{
var newObjectBlock = await GenerateObjectBlockAsync(blockRequest).ConfigureAwait(false);
return(await CreateObjectBlockContextAsync(blockRequest, newObjectBlock).ConfigureAwait(false));
}
private async Task <List <IObjectBlockContext <T> > > GetForcedObjectBlocksAsync <T>(ObjectBlockRequest <T> blockRequest)
{
var forcedBlockRequest = new QueuedForcedBlocksRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName),
blockRequest.TaskExecutionId,
blockRequest.BlockType);
var queuedForcedBlocks = await _blockRepository.GetQueuedForcedObjectBlocksAsync <T>(forcedBlockRequest).ConfigureAwait(false);
var forcedBlocks = new List <IObjectBlockContext <T> >();
foreach (var queuedForcedBlock in queuedForcedBlocks)
{
forcedBlocks.Add(await CreateObjectBlockContextAsync(blockRequest, queuedForcedBlock.ObjectBlock, queuedForcedBlock.ForcedBlockQueueId).ConfigureAwait(false));
}
if (forcedBlocks.Any())
{
await DequeueForcedBlocksAsync(blockRequest, forcedBlocks.Select(x => x.ForcedBlockQueueId).ToList()).ConfigureAwait(false);
}
return(forcedBlocks);
}
