/// <summary>
/// Returns a shard iterator. A shard iterator provides information about how to retrieve
/// the stream records from within a shard. Use the shard iterator in a subsequent <code>GetRecords</code>
/// request to read the stream records from the shard.
///
/// <note>
/// <para>
/// A shard iterator expires 15 minutes after it is returned to the requester.
/// </para>
/// </note>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the GetShardIterator service method.</param>
///
/// <returns>The response from the GetShardIterator service method, as returned by DynamoDBStreams.</returns>
/// <exception cref="Amazon.DynamoDBv2.Model.InternalServerErrorException">
/// An error occurred on the server side.
/// </exception>
/// <exception cref="Amazon.DynamoDBv2.Model.ResourceNotFoundException">
/// The operation tried to access a nonexistent stream.
/// </exception>
/// <exception cref="Amazon.DynamoDBv2.Model.TrimmedDataAccessException">
/// The operation attempted to read past the oldest stream record in a shard.
///
///
/// <para>
/// In DynamoDB Streams, there is a 24 hour limit on data retention. Stream records whose
/// age exceeds this limit are subject to removal (trimming) from the stream. You might
/// receive a TrimmedDataAccessException if:
/// </para>
/// <ul> <li>You request a shard iterator with a sequence number older than the trim
/// point (24 hours).</li> <li>You obtain a shard iterator, but before you use the iterator
/// in a <i>GetRecords</i> request, a stream record in the shard exceeds the 24 hour period
/// and is trimmed. This causes the iterator to access a record that no longer exists.</li>
/// </ul>
/// </exception>
public GetShardIteratorResponse GetShardIterator(GetShardIteratorRequest request)
{
var marshaller = new GetShardIteratorRequestMarshaller();
var unmarshaller = GetShardIteratorResponseUnmarshaller.Instance;
return(Invoke <GetShardIteratorRequest, GetShardIteratorResponse>(request, marshaller, unmarshaller));
}
internal virtual GetShardIteratorResponse GetShardIterator(GetShardIteratorRequest request)
{
var marshaller = GetShardIteratorRequestMarshaller.Instance;
var unmarshaller = GetShardIteratorResponseUnmarshaller.Instance;
return(Invoke <GetShardIteratorRequest, GetShardIteratorResponse>(request, marshaller, unmarshaller));
}
/// <summary>
/// Returns a shard iterator. A shard iterator provides information about how to retrieve
/// the stream records from within a shard. Use the shard iterator in a subsequent <code>GetRecords</code>
/// request to read the stream records from the shard.
///
/// <note>
/// <para>
/// A shard iterator expires 15 minutes after it is returned to the requester.
/// </para>
/// </note>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the GetShardIterator service method.</param>
///
/// <returns>The response from the GetShardIterator service method, as returned by DynamoDBStreams.</returns>
/// <exception cref="Amazon.DynamoDBv2.Model.InternalServerErrorException">
/// An error occurred on the server side.
/// </exception>
/// <exception cref="Amazon.DynamoDBv2.Model.ResourceNotFoundException">
/// The operation tried to access a nonexistent stream.
/// </exception>
/// <exception cref="Amazon.DynamoDBv2.Model.TrimmedDataAccessException">
/// The operation attempted to read past the oldest stream record in a shard.
///
///
/// <para>
/// In DynamoDB Streams, there is a 24 hour limit on data retention. Stream records whose
/// age exceeds this limit are subject to removal (trimming) from the stream. You might
/// receive a TrimmedDataAccessException if:
/// </para>
/// <ul> <li>
/// <para>
/// You request a shard iterator with a sequence number older than the trim point (24
/// hours).
/// </para>
/// </li> <li>
/// <para>
/// You obtain a shard iterator, but before you use the iterator in a <code>GetRecords</code>
/// request, a stream record in the shard exceeds the 24 hour period and is trimmed. This
/// causes the iterator to access a record that no longer exists.
/// </para>
/// </li> </ul>
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/streams-dynamodb-2012-08-10/GetShardIterator">REST API Reference for GetShardIterator Operation</seealso>
public virtual GetShardIteratorResponse GetShardIterator(GetShardIteratorRequest request)
{
var options = new InvokeOptions();
options.RequestMarshaller = GetShardIteratorRequestMarshaller.Instance;
options.ResponseUnmarshaller = GetShardIteratorResponseUnmarshaller.Instance;
return Invoke<GetShardIteratorResponse>(request, options);
}
/// <summary>
/// Initiates the asynchronous execution of the GetShardIterator operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the GetShardIterator operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/streams-dynamodb-2012-08-10/GetShardIterator">REST API Reference for GetShardIterator Operation</seealso>
public virtual Task <GetShardIteratorResponse> GetShardIteratorAsync(GetShardIteratorRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new GetShardIteratorRequestMarshaller();
var unmarshaller = GetShardIteratorResponseUnmarshaller.Instance;
return(InvokeAsync <GetShardIteratorRequest, GetShardIteratorResponse>(request, marshaller,
unmarshaller, cancellationToken));
}
/// <summary>
/// Initiates the asynchronous execution of the GetShardIterator operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the GetShardIterator operation on AmazonDynamoDBStreamsClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndGetShardIterator
/// operation.</returns>
public IAsyncResult BeginGetShardIterator(GetShardIteratorRequest request, AsyncCallback callback, object state)
{
var marshaller = new GetShardIteratorRequestMarshaller();
var unmarshaller = GetShardIteratorResponseUnmarshaller.Instance;
return(BeginInvoke <GetShardIteratorRequest>(request, marshaller, unmarshaller,
callback, state));
}
/// <summary>
/// Returns a shard iterator. A shard iterator provides information about how to retrieve
/// the stream records from within a shard. Use the shard iterator in a subsequent <code>GetRecords</code>
/// request to read the stream records from the shard.
///
/// <note>
/// <para>
/// A shard iterator expires 15 minutes after it is returned to the requester.
/// </para>
/// </note>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the GetShardIterator service method.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
///
/// <returns>The response from the GetShardIterator service method, as returned by DynamoDBStreams.</returns>
/// <exception cref="Amazon.DynamoDBv2.Model.InternalServerErrorException">
/// An error occurred on the server side.
/// </exception>
/// <exception cref="Amazon.DynamoDBv2.Model.ResourceNotFoundException">
/// The operation tried to access a nonexistent stream.
/// </exception>
/// <exception cref="Amazon.DynamoDBv2.Model.TrimmedDataAccessException">
/// The operation attempted to read past the oldest stream record in a shard.
///
///
/// <para>
/// In DynamoDB Streams, there is a 24 hour limit on data retention. Stream records whose
/// age exceeds this limit are subject to removal (trimming) from the stream. You might
/// receive a TrimmedDataAccessException if:
/// </para>
/// <ul> <li>
/// <para>
/// You request a shard iterator with a sequence number older than the trim point (24
/// hours).
/// </para>
/// </li> <li>
/// <para>
/// You obtain a shard iterator, but before you use the iterator in a <code>GetRecords</code>
/// request, a stream record in the shard exceeds the 24 hour period and is trimmed. This
/// causes the iterator to access a record that no longer exists.
/// </para>
/// </li> </ul>
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/streams-dynamodb-2012-08-10/GetShardIterator">REST API Reference for GetShardIterator Operation</seealso>
public virtual Task<GetShardIteratorResponse> GetShardIteratorAsync(GetShardIteratorRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var options = new InvokeOptions();
options.RequestMarshaller = GetShardIteratorRequestMarshaller.Instance;
options.ResponseUnmarshaller = GetShardIteratorResponseUnmarshaller.Instance;
return InvokeAsync<GetShardIteratorResponse>(request, options, cancellationToken);
}
/// <summary>
/// Initiates the asynchronous execution of the GetShardIterator operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the GetShardIterator operation on AmazonDynamoDBStreamsClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndGetShardIterator
/// operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/streams-dynamodb-2012-08-10/GetShardIterator">REST API Reference for GetShardIterator Operation</seealso>
public virtual IAsyncResult BeginGetShardIterator(GetShardIteratorRequest request, AsyncCallback callback, object state)
{
var options = new InvokeOptions();
options.RequestMarshaller = GetShardIteratorRequestMarshaller.Instance;
options.ResponseUnmarshaller = GetShardIteratorResponseUnmarshaller.Instance;
return(BeginInvoke(request, options, callback, state));
}
static async Task Main(string[] args)
{
Environment.SetEnvironmentVariable("AWS_PROFILE", "testuser");
var streamsClient = new AmazonDynamoDBStreamsClient();
var dynamoDbClient = new AmazonDynamoDBClient();
var tableDecriptor = await dynamoDbClient.DescribeTableAsync("GameScores");
var streamArn = tableDecriptor.Table.LatestStreamArn;
string exclusiveStartShardId = null;
while (true)
{
var result = await streamsClient.DescribeStreamAsync(new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = exclusiveStartShardId
});
var shards = result.StreamDescription.Shards;
foreach (var shard in shards)
{
var shardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var shardIteratorResponse = await streamsClient.GetShardIteratorAsync(shardIteratorRequest);
var currentShardIterator = shardIteratorResponse.ShardIterator;
while (currentShardIterator != null)
{
var recordsResponse = await streamsClient.GetRecordsAsync(new GetRecordsRequest()
{
ShardIterator = currentShardIterator
});
foreach (var record in recordsResponse.Records)
{
var data = record.Dynamodb;
Console.WriteLine("Table changed");
}
currentShardIterator = recordsResponse.NextShardIterator;
}
}
}
}
private static void ReadFromStream()
{
/*Config example*/
//AmazonKinesisConfig config = new AmazonKinesisConfig();
//config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1;
//AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
/*instance example*/
var kinesisClient = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1);
String kinesisStreamName = "your-kinesis-name";
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = kinesisStreamName;
DescribeStreamResponse describeResponse = kinesisClient.DescribeStream(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = kinesisStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = kinesisClient.GetShardIterator(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = kinesisClient.GetRecords(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string json = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("Json string: " + json);
}
}
iteratorId = nextIterator;
}
}
}
private async Task <string> GetShardIterator(string shardId)
{
var request = new GetShardIteratorRequest
{
StreamName = _streamName,
ShardIteratorType = ShardIteratorType.AFTER_SEQUENCE_NUMBER,
StartingSequenceNumber = _lastSequenceNumber,
ShardId = shardId
};
var response = await _client.GetShardIteratorAsync(request);
return(response.ShardIterator);
}
private async Task <GetShardIteratorResponse> GetShardIteratorAsync(Checkpoint checkPoint)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest
{
StreamName = checkPoint.StreamName,
ShardId = checkPoint.ShardId,
ShardIteratorType = ShardIteratorType.AT_SEQUENCE_NUMBER,
StartingSequenceNumber = checkPoint.LastSequenceNumber,
};
GetShardIteratorResponse iteratorResponse = await _client.GetShardIteratorAsync(iteratorRequest);
return(iteratorResponse);
}
private async Task ReadFromStream()
{
var describeRequest = new DescribeStreamRequest
{
StreamName = _streamName,
};
var describeStreamResponse = await _kinesisClient.DescribeStreamAsync(describeRequest);
var shards = describeStreamResponse.StreamDescription.Shards;
foreach (var shard in shards)
{
var getShardIteratorRequest = new GetShardIteratorRequest
{
StreamName = _streamName,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON,
};
var getShardIteratorResponse = await _kinesisClient.GetShardIteratorAsync(getShardIteratorRequest);
var shardIterator = getShardIteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(shardIterator))
{
var getRecordsRequest = new GetRecordsRequest
{
Limit = 100,
ShardIterator = shardIterator,
};
var getRecordsResponse = await _kinesisClient.GetRecordsAsync(getRecordsRequest);
var nextIterator = getRecordsResponse.NextShardIterator;
var records = getRecordsResponse.Records;
if (records.Count > 0)
{
Console.WriteLine($"Received {records.Count} records.");
foreach (var record in records)
{
var dataMessage = await JsonSerializer.DeserializeAsync <DataMessage>(record.Data);
Console.WriteLine($"DataMessage Id={dataMessage.Id}, CreatedOn={dataMessage.CreatedOn.ToString("yyyy-MM-dd HH:mm")}");
}
}
shardIterator = nextIterator;
}
}
}
private async Task GenerateShardIteratorRequest(KShard shard)
{
var shardIteratorRequest = new GetShardIteratorRequest()
{
ShardId = shard.ShardId,
ShardIteratorType = shard.ShardIteratorType,
StartingSequenceNumber = shard.SequenceNumber,
StreamName = _utilities.StreamName
};
var response = await _client.GetShardIteratorAsync(shardIteratorRequest, _cancellationTokenSource.Token);
shard.SetNextShardIterator(response.ShardIterator);
}
private async Task ReadFromStream()
{
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = myStreamName;
DescribeStreamResponse describeResponse =
await kinesisClient.DescribeStreamAsync(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = myStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = await kinesisClient.GetShardIteratorAsync(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = await kinesisClient.GetRecordsAsync(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string theMessage = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("message string: " + theMessage);
}
}
iteratorId = nextIterator;
}
}
}
public AWSDynamoDBStream(BasicAWSCredentials basicAWSCredentials, RegionEndpoint regionEndpoint, string tableName, AWSDynamoDBIteratorType type)
{
this.TableName = tableName;
this.AmazonDynamoDBStreamsClient = new AmazonDynamoDBStreamsClient(basicAWSCredentials, regionEndpoint);
this.AWSDynamoDBIteratorType = type;
var listStreams = AmazonDynamoDBStreamsClient.ListStreams(new ListStreamsRequest()
{
TableName = this.TableName
});
this.StreamArn = listStreams.Streams.First().StreamArn;
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = this.StreamArn
};
var describeStreamResponse = this.AmazonDynamoDBStreamsClient.DescribeStream(describeStreamRequest);
var shards = describeStreamResponse.StreamDescription.Shards;
GetShardIteratorRequest getShardIteratorRequest = null;
if (this.AWSDynamoDBIteratorType == AWSDynamoDBIteratorType.TRIM_HORIZON)
{
getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = this.StreamArn,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON,
ShardId = shards.First().ShardId,
};
}
if (this.AWSDynamoDBIteratorType == AWSDynamoDBIteratorType.LATEST)
{
getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = this.StreamArn,
ShardIteratorType = ShardIteratorType.LATEST,
ShardId = shards.Last().ShardId,
//SequenceNumber = shards.First().SequenceNumberRange.StartingSequenceNumber
};
}
var shardIteratorResponse = this.AmazonDynamoDBStreamsClient.GetShardIterator(getShardIteratorRequest);
this.LatestShardID = shardIteratorResponse.ShardIterator;
}
private void GetShardIterator()
{
var request = new GetShardIteratorRequest
{
ShardId = _settings.ShardId,
StreamName = _settings.StreamName,
ShardIteratorType = _settings.ShardIteratorType
};
if (_settings.AtTimestamp.HasValue)
{
request.Timestamp = _settings.AtTimestamp.Value;
}
_kinesisClient.GetShardIteratorAsync(request)
.PipeTo(_self,
success: result => new GetShardIteratorSuccess(result),
failure: ex => new GetShardIteratorFailure(ex));
}
private GetShardIteratorResult InnerGetShardIterator(GetShardIteratorRequest getShardIteratorParam)
{
ObsWebServiceRequest obsWebServiceRequest = new DISWebServiceRequest();
OBS.Runtime.Internal.IRequest requestobs = new DISDefaultRequest(obsWebServiceRequest, Constants.SERVICENAME)
{
HttpMethod = HttpMethodName.GET.ToString()
};
string resourcePath = ResourcePathBuilder.Standard()
.WithProjectId(_disConfig.GetProjectId())
.WithResource(new CursorResource(null))
.Build();
requestobs.ResourcePath = resourcePath;
var results = Request <GetShardIteratorResult>(getShardIteratorParam, requestobs);
return(results);
}
private async Task ProcessShard(CancellationToken stoppingToken, Shard shard)
{
// Get an iterator for the current shard
var iteratorRequest = new GetShardIteratorRequest
{
StreamArn = _latestStreamArn,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var iteratorResult = await _dynamoDbStreams.GetShardIteratorAsync(iteratorRequest, stoppingToken);
var iterator = iteratorResult.ShardIterator;
// Shard iterator is not null until the Shard is sealed (marked as READ_ONLY).
while (iterator != null)
{
// Use the shard iterator to read the stream records
var recordsResult = await _dynamoDbStreams.GetRecordsAsync(
new GetRecordsRequest
{
ShardIterator = iterator,
Limit = BatchSize
});
var records = recordsResult.Records;
foreach (var record in records)
{
_recordProcessor.ProcessRecord(record.Dynamodb);
}
iterator = recordsResult.NextShardIterator;
//we need to pause between iterations as get records has a limit
await Task.Delay(1000);
}
throw new ShardHasBeenClosedException(
$"The shard has been closed and we must re-get shards. Shard {shard.ShardId}"
);
}
/// <summary>
/// 获取数据游标
/// </summary>
/// <param name="streamName">通道名称</param>
/// <param name="shardId">分区ID</param>
public static GetShardIteratorResult GetCursorDemo(string streamName, string shardId)
{
var dic = new DISIngestionClient();
//该参数与游标类型AT_TIMESTAMP搭配使用
long timestamp = 1543397197333;
//该参数与游标类型AT_SEQUENCE_NUMBER、AFTER_SEQUENCE_NUMBER搭配使用
string startingSequenceNumber = "0";
//AT_SEQUENCE_NUMBER：从特定序列号所在的记录开始读取。此类型为默认游标类型。
//AFTER_SEQUENCE_NUMBER：从特定序列号后的记录开始读取。
//TRIM_HORIZON：从分区中时间最长的记录开始读取。
//LATEST：在分区中最新的记录之后开始读取，以确保始终读取分区中的最新数据。
//AT_TIMESTAMP：从特定时间戳开始读取。
string shardIteratorType = "AT_SEQUENCE_NUMBER";
var request = new GetShardIteratorRequest
{
//通道名称
StreamName = streamName,
//分区值
ShardId = shardId,
//游标类型，可选参数
ShardIteratorType = shardIteratorType,
//序列号，可选参数
StartingSequenceNumber = startingSequenceNumber,
//时间戳，可选参数
Timestamp = timestamp
};
var response = dic.GetShardIterator(request);
Console.Out.WriteLine(response);
return(response);
}
/// <summary>
/// This operation returns a shard iterator in ShardIterator.
/// More info: http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public GetShardIteratorResponse GetShardIterator(GetShardIteratorRequest request)
{
return(Invoke <GetShardIteratorResponse>(request, "GetShardIterator"));
}
private async Task ReadRefPersonStream()
{
Dictionary <string, Models.RefPerson> refPersons = new Dictionary <string, Models.RefPerson>();
refPersons.Clear();
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
string streamArn = Models.MyAWSConfigs.DynamodbRefPersonTableStreamArn;
//int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
//&& processedRecordCount < maxItemCount
while (currentShardIter != null)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
string changedRefPersonCamera = record.Dynamodb.NewImage["camera"].S;
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1") ? true : false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = changedRefPersonCamera;
refPerson.lastLocation = cameras[changedRefPersonCamera].location;
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id, Models.MyAWSConfigs.RefImagesBucketName);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (!refPersons.ContainsKey(refPerson.id))
{
refPersons.Add(refPerson.id, refPerson);
}
}
else
{
if (refPersons.ContainsKey(refPerson.id))
{
refPersons.Remove(refPerson.id);
}
}
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
public GetShardIteratorResult GetShardIterator(GetShardIteratorRequest getShardIteratorParam)
{
return(InnerGetShardIterator(getShardIteratorParam));
}
private async Task ReadStream()
{
List <Models.RefPerson> refPersons = new List <Models.RefPerson>();
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
string streamArn = "arn:aws:dynamodb:ap-southeast-2:358403828169:table/ref_persons/stream/2019-11-18T05:31:40.045";
int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.dynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
while (currentShardIter != null && processedRecordCount < maxItemCount)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
//Console.WriteLine($"{changedRefPersonId}:{changedRefPersonStatus}:{changedRefPersonName}:{changedRefPersonDescription}");
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1")?true:false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = "mobile_stream_1";
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (refPersons.FindAll(p => p.id == refPerson.id).Count == 0)
{
refPersons.Add(refPerson);
}
}
else
{
refPersons.RemoveAll(p => p.id == refPerson.id);
}
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
private async Task ReadHistoryStream()
{
string streamArn = Models.MyAWSConfigs.DynamodbHistoryTableStreamArn;
DateTime notifyTime = DateTime.Now;
notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);
//int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
//&& processedRecordCount < maxItemCount
while (currentShardIter != null)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
if (DateTime.Now >= notifyTime)
{
string id = record.Dynamodb.NewImage["id"].S;
string name = record.Dynamodb.NewImage["name"].S;
string camId = record.Dynamodb.NewImage["cameraId"].S;
string camLocation = cameras[camId].location;
string notification = $"{name} has been detected by camera {camId} at {camLocation}";
string trayMessage = $"{name} has been detected";
AddNewNotification(notification, id).Wait();
notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);
notifyIcon.Visible = true;
notifyIcon.ShowBalloonTip(1000, "New Person Detected", trayMessage, System.Windows.Forms.ToolTipIcon.Info);
}
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
public async Task Test1()
{
// Transaction put with an outbox message.
var transactWriteItems = new List <TransactWriteItem>();
for (int i = 0; i < 5; i++)
{
var orgDoc = new Document
{
["Id"] = 123,
["Name"] = $"org-123-{i}",
["Foo"] = Document.FromJson("{\"plot\" : \"Nothing happens at all.\",\"rating\" : 0}")
};
transactWriteItems.Add(new TransactWriteItem
{
Put = new Put
{
TableName = OrgsTableName,
Item = orgDoc.ToAttributeMap()
}
});
var outboxDoc = new Document
{
["Id"] = $"123-{i}",
["Created"] = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
["Body"] = Document.FromJson("{\"plot\" : \"Nothing happens at all.\",\"rating\" : 0}")
};
transactWriteItems.Add(new TransactWriteItem
{
Put = new Put
{
TableName = OutboxTableName,
Item = outboxDoc.ToAttributeMap()
}
});
}
var transactWriteItemsRequest = new TransactWriteItemsRequest
{
TransactItems = transactWriteItems,
};
await _client.TransactWriteItemsAsync(transactWriteItemsRequest);
// Read the changes to outbox change stream
string lastEvaluatedShardId = null;
do
{
var describeStreamRequest = new DescribeStreamRequest
{
StreamArn = _tableLatestStreamArn,
ExclusiveStartShardId = lastEvaluatedShardId
};
var describeStreamResponse = await _streamsClient.DescribeStreamAsync(describeStreamRequest);
foreach (var shard in describeStreamResponse.StreamDescription.Shards)
{
var getShardIteratorRequest = new GetShardIteratorRequest
{
StreamArn = _tableLatestStreamArn,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var getShardIteratorResponse = await _streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
var currentShardIterator = getShardIteratorResponse.ShardIterator;
var iterations = 0; // loop will continue for some time until the stream shard is closed, this just short circuits things for the test.
while (currentShardIterator != null && iterations < 10)
{
var getRecordsRequest = new GetRecordsRequest
{
ShardIterator = currentShardIterator,
};
var getRecordsResponse = await _streamsClient.GetRecordsAsync(getRecordsRequest);
foreach (var record in getRecordsResponse.Records)
{
_testOutputHelper.WriteLine($"{record.EventID} {record.EventName} {record.EventSource} {record.Dynamodb.NewImage.StreamViewType}");
}
currentShardIterator = getRecordsResponse.NextShardIterator;
iterations++;
}
}
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
public static async Task ReadFromKinesis()
{
dicPrefixo = new Dictionary <string, string>();
var file = "last_iterator_id.txt";
_client = new AmazonKinesisClient(workConfig["AccessKeyAWS"], workConfig["SecretKeyAWS"], RegionEndpoint.USEast1);
try
{
var describer = await _client.DescribeStreamAsync(new DescribeStreamRequest { StreamName = workConfig["QueueNamedAWS"] });
var shards = describer.StreamDescription.Shards;
foreach (var shard in shards)
{
var iterator_request = new GetShardIteratorRequest
{
StreamName = workConfig["QueueNamedAWS"],
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.LATEST,
Timestamp = DateTime.MinValue
};
var iterator = await _client.GetShardIteratorAsync(iterator_request);
string curr_iterator_id = iterator.ShardIterator;
string last_iterator_id = File.Exists(file) ? File.ReadAllText(file) : string.Empty;
if (!string.IsNullOrEmpty(last_iterator_id))
{
curr_iterator_id = last_iterator_id;
}
while (!string.IsNullOrEmpty(curr_iterator_id))
{
var response = await _client.GetRecordsAsync(new GetRecordsRequest { ShardIterator = curr_iterator_id, Limit = 1 });
var next_iterator_id = response.NextShardIterator;
var records = response.Records;
//Console.Write("\r\n" + "Sequencial: " + curr_iterator_id + "\r\n");
if (records.Count > 0)
{
foreach (var record in records)
{
var strData = Encoding.UTF8.GetString(record.Data.ToArray());
var words = strData.Split(new string[] { "\\n" }, StringSplitOptions.None);
foreach (var item in words)
{
Console.Write(item);
var lines = item.Split('\n');
for (int i = 0; i < lines.Count() - 1; i++)
{
//GravaLog.GravaLogInformacao("Split linha registro - ServiceReader Linha: 194");
var msg_partes = lines[i].Split(',');
//GravaLog.GravaLogInformacao("msg_partes - ServiceReader Linha: 197");
var addressAndTag = msg_partes[0].Replace("ns=2;s=", "");
//GravaLog.GravaLogInformacao("addressAndTag - ServiceReader Linha: 200");
string prefixoTagAtual;
//como o codigo para ler o numero fila do tipo 1 foi separado do tipo 2 então agora o separador é constante.
char separator = '.'; //Configuracoes.NumeroFila == "1" ? '\\' : '.';
if (addressAndTag.IndexOf(separator) == -1)
{
continue;
}
else
{
var tagPartes = addressAndTag.Split(separator);
//GravaLog.GravaLogInformacao("tagPartes - ServiceReader Linha: 214");
var tagPartesPrefixo = tagPartes[0].Split(separator);
//GravaLog.GravaLogInformacao("tagPartesPrefixo - ServiceReader Linha: 217");
prefixoTagAtual = tagPartes[3];
//GravaLog.GravaLogInformacao("prefixoTagAtual - ServiceReader Linha: 220");
nomeTagAtual = tagPartes[4].Split('\\').LastOrDefault();
//GravaLog.GravaLogInformacao("nomeTagAtual - ServiceReader Linha: 223");
}
string strTagCombinada = prefixoTagAtual + "." + nomeTagAtual;
CultureInfo culture = new CultureInfo("en-US");
DateTime dtRegistro;
//GravaLog.GravaLogInformacao("dtRegistro - ServiceReader Linha: 231");
dtRegistro = Convert.ToDateTime(msg_partes[2].TrimEnd(), new CultureInfo("en-US"));
//Grava Registro
//GravaLog.GravaLogInformacao("Data.SetFila_Tabela - Grava fila completa - ServiceReader Linha: 236");
Data.SetFila_Tabela(Conexao.GetConnectionHist(workConfig), lines[i], dtRegistro, workConfig["QueueNamedAWS"],
nomeTagAtual, msg_partes[1].TrimEnd(), prefixoTagAtual, curr_iterator_id);
Int32 id_t_bitola = 0;
GravaLog.GravaLogInformacao("Verifica se há nome da bitola no dicionario - ServiceReader Linha: 242");
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SP_BITOLA", out string valueTagBitola))
{
GravaLog.GravaLogInformacao("Retorna nome da bitola - ServiceReader Linha: 245");
id_t_bitola = Data.GetIdBitola(Conexao.GetConnectionProd(workConfig), valueTagBitola);
}
//Grava na Tabela de Mudança
if (dicPrefixo.TryGetValue(strTagCombinada, out string valueTag))
{
GravaLog.GravaLogInformacao("Verifica mudança de valores - ServiceReader Linha: 253");
if (valueTag != null && valueTag != String.Empty)
{
GravaLog.GravaLogInformacao("Troca o valor da Tag no dicionario - ServiceReader Linha: 256");
dicPrefixo.Remove(strTagCombinada);
dicPrefixo.Add(strTagCombinada, msg_partes[1].TrimEnd());
if (msg_partes[1].TrimEnd() != valueTag)
{
GravaLog.GravaLogInformacao("Data.SetFila_Tabela_Mudanca - Grava mudanca na tabela - Tag: " + nomeTagAtual +
" Valor Anterior: " + valueTag + " Valor Atual: " + msg_partes[1].TrimEnd() + " - ServiceReader Linha: 263");
Data.SetFila_Tabela_Mudanca(Conexao.GetConnectionHist(workConfig), workConfig["QueueNamedAWS"], nomeTagAtual,
msg_partes[1].TrimEnd(), prefixoTagAtual, valueTag);
GravaLog.GravaLogInformacao("Identificada Tag que retorna posicao do canal - ServiceReader Linha: 268");
if (nomeTagAtual == "GRVATUAL" || nomeTagAtual == "CC_CURPGVNB")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SS_BD1OUT", out string valueTagBD1OUT))
{
GravaLog.GravaLogInformacao("BD1 desligada - ServiceReader Linha: 276");
if (valueTagBD1OUT == "1")
{
continue;
}
}
GravaLog.GravaLogInformacao("Retorna ID da gaiola - ServiceReader Linha: 281");
Gaiola gaiola = null;
if (prefixoTagAtual == "HH113")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".TB_Z13", out string valueBD2))
{
if (Convert.ToInt32(valueBD2) == 0)
{
gaiola = RetornaGaiola("HF113");
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
}
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
if (gaiola != null && gaiola.Num_Bloco > 0)
{
id_t_bitola = Data.GetIdBitolaNumBloco(Conexao.GetConnectionProd(workConfig), gaiola.Num_Bloco);
if (id_t_bitola > 0)
{
GravaLog.GravaLogInformacao("Data.SetLaminacaoCambio_Canal - Grava dados do canal - " + "ID Bitola: - " + id_t_bitola + " ID Gaiola: " + gaiola.Id +
" Posicao: " + Convert.ToInt32(msg_partes[1].TrimEnd()) + " - ServiceReader Linha: 288");
Data.SetLaminacaoCambio_Canal(Conexao.GetConnectionProd(workConfig), id_t_bitola, Convert.ToInt32(msg_partes[1].TrimEnd()), gaiola.Id, gaiola.Num_Bloco);
}
}
}
}
}
}
else
{
dicPrefixo.Add(strTagCombinada, msg_partes[1].TrimEnd());
GravaLog.GravaLogInformacao("Identificada Tag que retorna posicao do canal - dicionario limpo - ServiceReader Linha: 320");
if (nomeTagAtual == "GRVATUAL" || nomeTagAtual == "CC_CURPGVNB")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SS_BD1OUT", out string valueTagBD1OUT))
{
if (valueTagBD1OUT == "1")
{
continue;
}
}
Gaiola gaiola = null;
if (prefixoTagAtual == "HH113")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".TB_Z13", out string valueBD2))
{
if (Convert.ToInt32(valueBD2) == 0)
{
gaiola = RetornaGaiola("HF113");
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
}
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
if (gaiola != null && gaiola.Num_Bloco > 0)
{
id_t_bitola = Data.GetIdBitolaNumBloco(Conexao.GetConnectionProd(workConfig), gaiola.Num_Bloco);
if (id_t_bitola > 0)
{
GravaLog.GravaLogInformacao("Data.SetLaminacaoCambio_Canal - Grava dados do canal - " + "ID Bitola: - " + id_t_bitola + " ID Gaiola: " + gaiola.Id +
" Posicao: " + Convert.ToInt32(msg_partes[1].TrimEnd()) + " - ServiceReader Linha: 319");
Data.SetLaminacaoCambio_Canal(Conexao.GetConnectionProd(workConfig), id_t_bitola, Convert.ToInt32(msg_partes[1].TrimEnd()), gaiola.Id, gaiola.Num_Bloco);
}
}
}
}
}
}
}
}
//GravaLog.GravaLogInformacao("Novo registro da fila");
File.WriteAllText(file, next_iterator_id);
curr_iterator_id = next_iterator_id;
}
}
}
catch (Exception ex)
{
logger.LogCritical($"KinesisWorker.ServiceReader exceção em {DateTimeOffset.Now:dd/MM/yyy HH:mm:ss} ... {ex.Message}");
GravaLog.GravaLogErro(ex.Message);
if (!stoppingToken.IsCancellationRequested)
{
await ReadFromKinesis();
}
}
finally
{
if (!stoppingToken.IsCancellationRequested)
{
await ReadFromKinesis();
}
}
}
/// <summary>
/// 下载数据
/// </summary>
/// <param name="streamName">通道名称</param>
/// <param name="shardId">分区ID</param>
public static void RunConsumerDemo(string streamName, string shardId)
{
var dic = new DISIngestionClient();
//该参数与游标类型AT_TIMESTAMP搭配使用
long timestamp = 1543397197333;
//该参数与游标类型AT_SEQUENCE_NUMBER、AFTER_SEQUENCE_NUMBER搭配使用
const string startingSequenceNumber = "0";
//AT_SEQUENCE_NUMBER：从特定序列号所在的记录开始读取。此类型为默认游标类型。
//AFTER_SEQUENCE_NUMBER：从特定序列号后的记录开始读取。
//TRIM_HORIZON：从分区中时间最长的记录开始读取。
//LATEST：在分区中最新的记录之后开始读取，以确保始终读取分区中的最新数据。
//AT_TIMESTAMP：从特定时间戳开始读取。
const string shardIteratorType = "AT_SEQUENCE_NUMBER";
var request = new GetShardIteratorRequest
{
//通道名称
StreamName = streamName,
//分区值
ShardId = shardId,
//序列号，可选参数
StartingSequenceNumber = startingSequenceNumber,
//游标类型，可选参数
ShardIteratorType = shardIteratorType,
//时间戳，可选参数
Timestamp = timestamp
};
var recordsRequest = new GetRecordsRequest();
var response = dic.GetShardIterator(request);
Console.Out.WriteLine(response);
var iterator = response.ShardIterator;
//下载数据，这里的进程不会退出，只要检测到有数据就下载数据
while (true)
{
//数据游标。
recordsRequest.ShardIterator = iterator;
GetRecordsResult recordResponse = dic.GetRecords(recordsRequest);
// 下一批数据游标
iterator = recordResponse.NextShardIterator;
if (recordResponse.Records.Count > 0)
{
foreach (var record in recordResponse.Records)
{
Console.WriteLine("Record[{0}] = {1}", record.SequenceNumber, DecodeData(record.Data));
}
Thread.Sleep(1000);
}
else
{
Thread.Sleep(1000);
}
}
}
public void GetShardIteratorAsync(GetShardIteratorRequest request, AmazonServiceCallback <GetShardIteratorRequest, GetShardIteratorResponse> callback, AsyncOptions options = null)
{
throw new System.NotImplementedException();
}
public static async Task TestStreamAsync()
{
var config = new DynamoDBContextConfig
{
Conversion = DynamoDBEntryConversion.V2,
ConsistentRead = true
};
var Context = new DynamoDBContext(new AmazonDynamoDBClient());
#region test_stream_read
// Save list to table
var list = new TODOList
{
User = "******",
ListId = Guid.NewGuid().ToString(),
Complete = false,
Name = "StreamTest",
CreateDate = DateTime.UtcNow,
UpdateDate = DateTime.UtcNow,
Items = new List <TODOListItem>
{
new TODOListItem {
Description = "Task1", Complete = true
},
new TODOListItem {
Description = "Task2", Complete = false
}
}
};
await Context.SaveAsync(list);
using (var streamClient = new AmazonDynamoDBStreamsClient())
{
// Function for reading records continuously from a shard.
Func <string, CancellationToken, Task> shardReader = async(iterator, token) =>
{
while (!token.IsCancellationRequested)
{
var response = (await streamClient.GetRecordsAsync(new GetRecordsRequest {
ShardIterator = iterator
}));
// Update position in shard iterator
iterator = response.NextShardIterator;
// This is what you would write in a Lambda function processing DynamoDB Streams.
foreach (var record in response.Records)
{
var newVersion = record.Dynamodb.NewImage;
Console.WriteLine($"Item read: {newVersion["User"].S}/{newVersion["ListId"].S}");
}
}
};
// Find the arn for the DynamoDB Stream
var streamArn = (await streamClient.ListStreamsAsync(new ListStreamsRequest {
TableName = "TODOList"
}))
.Streams.FirstOrDefault()?.StreamArn;
Console.WriteLine($"The stream arn is {streamArn} shards");
// A stream is made of shards.
var shards = (await streamClient.DescribeStreamAsync(new DescribeStreamRequest {
StreamArn = streamArn
}))
.StreamDescription.Shards;
Console.WriteLine($"The stream currently has {shards.Count} shards");
// Execute a separate reader for each shard.
var cancelSource = new CancellationTokenSource();
var readerTasks = new Task[shards.Count];
for (int i = 0; i < readerTasks.Length; i++)
{
var shardIteratorRequest = new GetShardIteratorRequest
{
StreamArn = streamArn,
ShardId = shards[i].ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var shardIterator = (await streamClient.GetShardIteratorAsync(shardIteratorRequest))
.ShardIterator;
Console.WriteLine($"Shard Iterator {i}: {shardIterator}");
readerTasks[i] = shardReader(shardIterator, cancelSource.Token);
}
Task.WaitAll(readerTasks, TimeSpan.FromSeconds(5));
cancelSource.Cancel();
}
#endregion
}
private void UpdateData()
{
using (IAmazonKinesis klient = new AmazonKinesisClient(_credentials, Amazon.RegionEndpoint.USWest2))
{
ListStreamsResponse resp = klient.ListStreams();
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = StreamName;
DescribeStreamResponse describeResponse = klient.DescribeStream(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
SequenceNumberRange range = shard.SequenceNumberRange;
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = StreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType;
GetShardIteratorResponse iteratorResponse = klient.GetShardIterator(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (IsRunning && !string.IsNullOrEmpty(iteratorId))
{
Thread.Sleep(1);
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = klient.GetRecords(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Amazon.Kinesis.Model.Record> records = getResponse.Records;
if (records.Count > 0)
{
if (IsDebug)
{
string message = string.Format("Received {0} records. ", records.Count);
Console.WriteLine(message);
foreach (IDebugObserver observer in _debugObservers)
{
observer.WriteDebug(message);
}
}
foreach (Amazon.Kinesis.Model.Record record in records)
{
if (!IsRunning)
{
break;
}
Thread.Sleep(1);
string json = Encoding.UTF8.GetString(record.Data.ToArray());
if (IsDebug)
{
string message = "Json string: " + json;
Console.WriteLine(message);
foreach (IDebugObserver observer in _debugObservers)
{
observer.WriteDebug(message);
}
}
T obj = JsonConvert.DeserializeObject <T>(json);
if (obj != null && _dataHandler != null)
{
_dataHandler(obj);
}
}
}
iteratorId = nextIterator;
}
if (!IsRunning)
{
break;
}
}
}
}
