private void WithGetRecordsSuccess(params string[] payload)
{
_kinesisClient.GetRecordsAsync(Arg.Any <GetRecordsRequest>())
.Returns(async info => new GetRecordsResponse
{
Records = payload
.Select(x => new Record {
Data = new MemoryStream(Encoding.UTF8.GetBytes(x))
})
.ToList(),
NextShardIterator = _nextShardIterator < 0 ? null : "next-" + (Interlocked.Increment(ref _nextShardIterator).ToString())
});
}
private async Task <string> TryProcessRecordsAsync(Checkpoint checkPoint, string iterator, CancellationToken token)
{
GetRecordsRequest request = new GetRecordsRequest
{
Limit = 250,
ShardIterator = iterator,
};
GetRecordsResponse response = await _client.GetRecordsAsync(request, token);
UpdateHeartbeatDelay(response.MillisBehindLatest);
if (response.Records.Any())
{
ProcessRecordsAsync(response, checkPoint).Wait();
CheckpointRepository.Save(checkPoint);
}
if (response.Records.Count < 10)
{
Trace.WriteLine("Sleeping as no new data.");
Task.Delay(1000, token).Wait(token);
}
else
{
// _logger.LogInformation("Got {0} records from Kinesis", response.Records.Count);
}
iterator = response.NextShardIterator;
return(iterator);
}
private async Task <RecordResponse> GetRecordResponse(KShard shard, CancellationToken ctx)
{
var request = new GetRecordsRequest {
ShardIterator = shard.ShardIterator
};
while (!ctx.IsCancellationRequested)
{
var record = await _client.GetRecordsAsync(request, ctx);
if (record.Records.Count > 0)
{
return(RecordResponse.Create(record, ctx));
}
request.ShardIterator = record.NextShardIterator;
await Task.Delay(1000, ctx);
}
return(RecordResponse.Empty);
}
private Amazon.Kinesis.Model.GetRecordsResponse CallAWSServiceOperation(IAmazonKinesis client, Amazon.Kinesis.Model.GetRecordsRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Kinesis", "GetRecords");
try
{
#if DESKTOP
return(client.GetRecords(request));
#elif CORECLR
return(client.GetRecordsAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
