public async Task TestBatchFlowSubJobsAsync()
{
var root = await _client.CreateNewJobAsync(new AddJobDto("root"));
var addSubDto = new BatchAddJobDto()
{
Children = Enumerable.Range(0, 1000).Select(s => new AddJobDto(s.ToString())).ToList(),
ParentJobId = root.JobId
};
var errorRes = await _client.BatchAddChildrenAsync(addSubDto);
Assert.AreEqual(false, errorRes.Any());
var updateJobBlock = new ActionBlock <long>(async jobId =>
{
await _client.UpdateJobStatesAsync(jobId, new UpdateJobStateDto(JobState.Running));
await _client.UpdateJobStatesAsync(jobId, new UpdateJobStateDto(JobState.RanToCompletion));
},
Helper.GetOutOfGrainExecutionOptions());
foreach (var child in addSubDto.Children)
{
await updateJobBlock.PostToBlockUntilSuccessAsync(child.JobId.Value);
}
updateJobBlock.Complete();
await updateJobBlock.Completion;
await _client.UpdateJobStatesAsync(root.JobId, new UpdateJobStateDto(JobState.Running));
await _client.UpdateJobStatesAsync(root.JobId, new UpdateJobStateDto(JobState.RanToCompletion));
}
private async Task ManipulateBufferAsync(IEnumerable <AddToBufferDto> bufferedGrains, bool syncState = true)
{
var flushAction = new ActionBlock <AddToBufferDto>(async item =>
{
switch (item.GrainType)
{
case BufferedGrainInterfaceType.JobGrain:
await _client.GetGrain <IJobGrainInMem>(item.GrainIntId).DeactivateAsync(syncState);
break;
case BufferedGrainInterfaceType.DescendantsRefGrain:
await _client.GetGrain <IDescendantsRefGrainInMem>(item.GrainIntId).DeactivateAsync(syncState);
break;
case BufferedGrainInterfaceType.JobTreeStatisticsGrain:
await _client.GetGrain <IJobTreeStatisticsGrainInMem>(item.GrainIntId)
.DeactivateAsync(syncState);
break;
default:
throw new ArgumentOutOfRangeException();
}
}, Helper.GetOutOfGrainExecutionOptions());
foreach (var item in bufferedGrains)
{
await flushAction.PostToBlockUntilSuccessAsync(item);
}
flushAction.Complete();
await flushAction.Completion;
}
private async Task MergeShardingIndexAsync()
{
try
{
var current = DateTimeOffset.Now;
var deleteAction = new ActionBlock <ITableEntity>(
async entity => await Client.GetTableReference(_tableName)
.ExecuteAsync(TableOperation.Delete(entity)),
Helper.GetOutOfGrainExecutionOptions());
var timeIndexSeq =
Helper.GetTimeIndexRange(current.AddHours(-_indexConfig.TrackTimeIndexCount),
current);
foreach (var index in timeIndexSeq)
{
var token = new TableContinuationToken();
var shardGrain = GrainFactory.GetGrain <IShardJobIndexGrain>(index);
var aggregator = GrainFactory.GetGrain <IAggregateJobIndexGrain>(index);
var indexResults = new List <JobIndexInternal>();
while (token != null && indexResults.Count < _indexConfig.MaxRoundSize)
{
var result = await shardGrain.FetchWithTokenAsync(token);
token = result.ContinuationToken;
if (result.Results?.Count > 0)
{
indexResults.AddRange(result.Results);
}
}
if (indexResults.Count > 0)
{
await aggregator.MergeIntoIndicesAsync(indexResults);
}
foreach (var entity in indexResults)
{
await deleteAction.PostToBlockUntilSuccessAsync(
new TableEntity(entity.PartitionKey, entity.RowKey)
{
ETag = "*"
});
}
_logger.Info($"Merged index to {index}, count: {indexResults.Count}");
}
deleteAction.Complete();
await deleteAction.Completion;
}
catch (Exception e)
{
_logger.LogError(e, $"Error in {nameof(MergeShardingIndexAsync)}");
}
SharedData.LastMergeTimePoint = DateTimeOffset.Now;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
var current = DateTimeOffset.Now;
var deleteAction = new ActionBlock <TableEntity>(
entity => Client.GetTableReference(_tableName).ExecuteAsync(TableOperation.Delete(entity)),
Helper.GetOutOfGrainExecutionOptions());
var timeIndexSeq =
Helper.GetTimeIndexRange(current.AddHours(-_indexConfig.TrackTimeIndexCount),
current);
foreach (var index in timeIndexSeq)
{
var token = new TableContinuationToken();
var shardGrain = _client.GetGrain <IShardJobIndexGrain>(index);
var aggregator = _client.GetGrain <IAggregateJobIndexGrain>(index);
var indexResults = new List <JobIndexInternal>();
while (token != null && indexResults.Count < _indexConfig.MaxRoundSize)
{
var result = await shardGrain.FetchWithTokenAsync(token);
token = result.ContinuationToken;
if (result.Results?.Count > 0)
{
indexResults.AddRange(result.Results);
}
}
if (indexResults.Count > 0)
{
await aggregator.MergeIntoIndicesAsync(indexResults);
}
foreach (var entity in indexResults)
{
await deleteAction.PostToBlockUntilSuccessAsync(
new TableEntity(entity.PartitionKey, entity.RowKey)
{
ETag = "*"
});
}
}
deleteAction.Complete();
await deleteAction.Completion;
}
catch (Exception e)
{
_logger.LogError(e, $"Error in {nameof(MergeJobIndexWorker)}");
}
await Task.Delay(_indexConfig.IndexMergeInterval, stoppingToken);
}
}
public async Task <ReturnQueryIndexDto> QueryJobsAsync(QueryJobIndexDto dto)
{
if (dto.Start > dto.End)
{
throw new InvalidOperationException("the start date cannot be greater than the end date");
}
if (dto.PageNumber <= 0)
{
dto.PageNumber = 1;
}
if (dto.PageSize <= 0)
{
dto.PageSize = 10;
}
var timeIndices = Helper.GetTimeIndexRange(dto.Start, dto.End);
var result = new ConcurrentDictionary <long, JobIndexInternal>();
var queryIndicesBlock = new ActionBlock <string>(
async index =>
{
var readOnlyGrainIndices =
await _client.GetGrain <IAggregateJobIndexGrain>(index).QueryAsync(dto.Predicate);
foreach (var innerIndex in readOnlyGrainIndices)
{
result[innerIndex.JobId] = innerIndex;
}
}, Helper.GetOutOfGrainExecutionOptions());
foreach (var index in timeIndices)
{
await queryIndicesBlock.PostToBlockUntilSuccessAsync(index);
}
queryIndicesBlock.Complete();
await queryIndicesBlock.Completion;
return(new ReturnQueryIndexDto
{
Indices = _mapper.Map <List <JobIndex> >(result.Values
.OrderByDescending(s => s.JobId)
.Skip((dto.PageNumber - 1) * dto.PageNumber)
.Take(dto.PageSize)
.ToList()),
IndexGrainHit = timeIndices.Count,
TotalCount = result.Count
});
}
public async Task <List <JobEntity> > BatchGetJobEntitiesAsync(IEnumerable <long> jobIds)
{
var jobs = new ConcurrentBag <JobEntityState>();
var getJobInfoProcessor = new ActionBlock <long>(async jobId =>
{
var jobGrain = _client.GetGrain <IJobGrain>(jobId);
jobs.Add(await jobGrain.GetJobAsync());
}, Helper.GetOutOfGrainExecutionOptions());
foreach (var childJobId in jobIds)
{
await getJobInfoProcessor.PostToBlockUntilSuccessAsync(childJobId);
}
getJobInfoProcessor.Complete();
await getJobInfoProcessor.Completion;
return(_mapper.Map <List <JobEntity> >(jobs.OrderByDescending(s => s.JobId).ToList()));
}
public async Task <Dictionary <long, JobTreeStatistics> > GetJobStatisticsListByIdsAsync(IEnumerable <long> ids)
{
var statisticsContainer = new ConcurrentBag <JobTreeStatistics>();
var getStatisticsProcessor = new ActionBlock <long>(async jobId =>
{
var statistics = await GetJobStatisticsByIdAsync(jobId);
statisticsContainer.Add(statistics);
}, Helper.GetOutOfGrainExecutionOptions());
foreach (var id in ids)
{
await getStatisticsProcessor.PostToBlockUntilSuccessAsync(id);
}
getStatisticsProcessor.Complete();
await getStatisticsProcessor.Completion;
return(statisticsContainer.ToDictionary(s => s.JobId, s => s));
}
public async Task <List <JobIndexInternal> > QueryAsync(string queryStr)
{
var queryResult = new ConcurrentBag <JobIndexInternal>();
var queryActionBlock = new ActionBlock <int>(async index =>
{
var grain = GrainFactory.GetGrain <IRollingJobIndexGrain>(Helper.GetRollingIndexId(this.GetPrimaryKeyString(), index));
foreach (var item in await grain.QueryAsync(queryStr))
{
queryResult.Add(item);
}
}, Helper.GetGrainInternalExecutionOptions());
for (var i = 0; i <= State.RollingIndexCount; i++)
{
await queryActionBlock.PostToBlockUntilSuccessAsync(i);
}
queryActionBlock.Complete();
await queryActionBlock.Completion;
return(queryResult.ToList());
}
public async Task <List <JobEntity> > GetDescendantEntitiesAsync(long id)
{
await _client.GetGrain <IJobGrain>(id).GetJobEntityAsync();
var childrenIds = await _client.GetGrain <IDescendantsRefGrain>(id).GetChildrenAsync();
var jobs = new ConcurrentBag <JobEntityState>();
var getJobInfoProcessor = new ActionBlock <long>(async jobId =>
{
var jobGrain = _client.GetGrain <IJobGrain>(jobId);
jobs.Add(await jobGrain.GetJobEntityAsync());
}, Helper.GetOutOfGrainExecutionOptions());
foreach (var childJobId in childrenIds)
{
await getJobInfoProcessor.PostToBlockUntilSuccessAsync(childJobId);
}
getJobInfoProcessor.Complete();
await getJobInfoProcessor.Completion;
return(_mapper.Map <List <JobEntity> >(jobs.OrderBy(s => s.JobId).ToList()));
}