internal virtual DescribeStreamResponse DescribeStream(DescribeStreamRequest request)
{
var options = new InvokeOptions();
options.RequestMarshaller = DescribeStreamRequestMarshaller.Instance;
options.ResponseUnmarshaller = DescribeStreamResponseUnmarshaller.Instance;
return(Invoke <DescribeStreamResponse>(request, options));
}
/// <summary>
/// Initiates the asynchronous execution of the DescribeStream operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the DescribeStream operation on AmazonDynamoDBStreamsClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndDescribeStream
/// operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/streams-dynamodb-2012-08-10/DescribeStream">REST API Reference for DescribeStream Operation</seealso>
public virtual IAsyncResult BeginDescribeStream(DescribeStreamRequest request, AsyncCallback callback, object state)
{
var options = new InvokeOptions();
options.RequestMarshaller = DescribeStreamRequestMarshaller.Instance;
options.ResponseUnmarshaller = DescribeStreamResponseUnmarshaller.Instance;
return(BeginInvoke(request, options, callback, state));
}
/// <summary>
/// Initiates the asynchronous execution of the DescribeStream operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the DescribeStream operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/streams-dynamodb-2012-08-10/DescribeStream">REST API Reference for DescribeStream Operation</seealso>
public virtual Task <DescribeStreamResponse> DescribeStreamAsync(DescribeStreamRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var options = new InvokeOptions();
options.RequestMarshaller = DescribeStreamRequestMarshaller.Instance;
options.ResponseUnmarshaller = DescribeStreamResponseUnmarshaller.Instance;
return(InvokeAsync <DescribeStreamResponse>(request, options, cancellationToken));
}
/// <summary>
/// 删除通道
/// </summary>
/// <param name="streamName">通道名称</param>
/// <returns></returns>
public static ResponseResult DeleteStream(string streamName)
{
var dic = new DISIngestionClient();
var request = new DescribeStreamRequest {
StreamName = streamName
};
ResponseResult response = dic.DeleteStream(request);
Console.WriteLine(response);
return(response);
}
public KinesisTask(Executor executor, PutRecordsRequest request, DescribeStreamRequest dsRequest, KinesisRquestType reqType, object context, ResponseCallback response_cb, KinesisClientReturn kinesisClientReturn, DateTime deadline, DateTime expiration, int timeout)
: base(deadline, expiration)
{
executor_ = executor;
context_ = context;
response_cb_ = response_cb;
dsRequest_ = dsRequest;
timeout_ = timeout;
finished_ = false;
reqType_ = reqType;
prRequest_ = request;
}
private static void ReadFromStream()
{
/*Config example*/
//AmazonKinesisConfig config = new AmazonKinesisConfig();
//config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1;
//AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
/*instance example*/
var kinesisClient = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1);
String kinesisStreamName = "your-kinesis-name";
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = kinesisStreamName;
DescribeStreamResponse describeResponse = kinesisClient.DescribeStream(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = kinesisStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = kinesisClient.GetShardIterator(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = kinesisClient.GetRecords(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string json = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("Json string: " + json);
}
}
iteratorId = nextIterator;
}
}
}
private async Task ReadFromStream()
{
var describeRequest = new DescribeStreamRequest
{
StreamName = _streamName,
};
var describeStreamResponse = await _kinesisClient.DescribeStreamAsync(describeRequest);
var shards = describeStreamResponse.StreamDescription.Shards;
foreach (var shard in shards)
{
var getShardIteratorRequest = new GetShardIteratorRequest
{
StreamName = _streamName,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON,
};
var getShardIteratorResponse = await _kinesisClient.GetShardIteratorAsync(getShardIteratorRequest);
var shardIterator = getShardIteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(shardIterator))
{
var getRecordsRequest = new GetRecordsRequest
{
Limit = 100,
ShardIterator = shardIterator,
};
var getRecordsResponse = await _kinesisClient.GetRecordsAsync(getRecordsRequest);
var nextIterator = getRecordsResponse.NextShardIterator;
var records = getRecordsResponse.Records;
if (records.Count > 0)
{
Console.WriteLine($"Received {records.Count} records.");
foreach (var record in records)
{
var dataMessage = await JsonSerializer.DeserializeAsync <DataMessage>(record.Data);
Console.WriteLine($"DataMessage Id={dataMessage.Id}, CreatedOn={dataMessage.CreatedOn.ToString("yyyy-MM-dd HH:mm")}");
}
}
shardIterator = nextIterator;
}
}
}
public DescribeStreamResponse GetStreamResponse(string streamName = null)
{
if (string.IsNullOrEmpty(streamName) && string.IsNullOrEmpty(_streamName))
{
throw new Exception("Please specify a stream name to get the stream response.");
}
var request = new DescribeStreamRequest()
{
StreamName = streamName ?? _streamName
};
return(AsyncHelper.RunSync(() => _client.DescribeStreamAsync(request)));
}
public async Task <DescribeStreamResponse> GetStreamResponseAsync(string streamName = null)
{
if (string.IsNullOrEmpty(streamName) && string.IsNullOrEmpty(_streamName))
{
throw new Exception("Please specify a stream name to get the stream response.");
}
var request = new DescribeStreamRequest()
{
StreamName = streamName ?? _streamName
};
return(await _client.DescribeStreamAsync(request));
}
static DescribeStreamResponse DescribeStream(IAmazonKinesis kinesisClient, string streamName)
{
var request = new DescribeStreamRequest {
StreamName = streamName
};
try
{
var response = kinesisClient.DescribeStream(request);
return(response);
}
catch (ResourceNotFoundException)
{
return(null);
}
}
static DescribeStreamResponse DescribeStream(IAmazonKinesis kinesisClient, string streamName)
{
var request = new DescribeStreamRequest {
StreamName = streamName
};
try
{
var describeStreamTask = kinesisClient.DescribeStreamAsync(request);
return(describeStreamTask.GetAwaiter().GetResult());
}
catch (ResourceNotFoundException)
{
return(null);
}
}
private bool DoesKinesisStreamExists(AmazonKinesisClient kinesisClient, string streamName)
{
var describeStreamReq = new DescribeStreamRequest();
describeStreamReq.StreamName = streamName;
var describeResult = kinesisClient.DescribeStreamAsync(describeStreamReq).Result;
string streamStatus = describeResult.StreamDescription.StreamStatus;
if (streamStatus == StreamStatus.ACTIVE)
{
return(true);
}
else
{
return(false);
}
}
private async Task ReadFromStream()
{
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = myStreamName;
DescribeStreamResponse describeResponse =
await kinesisClient.DescribeStreamAsync(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = myStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = await kinesisClient.GetShardIteratorAsync(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = await kinesisClient.GetRecordsAsync(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string theMessage = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("message string: " + theMessage);
}
}
iteratorId = nextIterator;
}
}
}
public AWSDynamoDBStream(BasicAWSCredentials basicAWSCredentials, RegionEndpoint regionEndpoint, string tableName, AWSDynamoDBIteratorType type)
{
this.TableName = tableName;
this.AmazonDynamoDBStreamsClient = new AmazonDynamoDBStreamsClient(basicAWSCredentials, regionEndpoint);
this.AWSDynamoDBIteratorType = type;
var listStreams = AmazonDynamoDBStreamsClient.ListStreams(new ListStreamsRequest()
{
TableName = this.TableName
});
this.StreamArn = listStreams.Streams.First().StreamArn;
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = this.StreamArn
};
var describeStreamResponse = this.AmazonDynamoDBStreamsClient.DescribeStream(describeStreamRequest);
var shards = describeStreamResponse.StreamDescription.Shards;
GetShardIteratorRequest getShardIteratorRequest = null;
if (this.AWSDynamoDBIteratorType == AWSDynamoDBIteratorType.TRIM_HORIZON)
{
getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = this.StreamArn,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON,
ShardId = shards.First().ShardId,
};
}
if (this.AWSDynamoDBIteratorType == AWSDynamoDBIteratorType.LATEST)
{
getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = this.StreamArn,
ShardIteratorType = ShardIteratorType.LATEST,
ShardId = shards.Last().ShardId,
//SequenceNumber = shards.First().SequenceNumberRange.StartingSequenceNumber
};
}
var shardIteratorResponse = this.AmazonDynamoDBStreamsClient.GetShardIterator(getShardIteratorRequest);
this.LatestShardID = shardIteratorResponse.ShardIterator;
}
/// <summary>
/// 删除通道
/// </summary>
/// <param name="describeStreamRequest"></param>
/// <returns></returns>
public ResponseResult DeleteStream(DescribeStreamRequest describeStreamRequest)
{
ObsWebServiceRequest obsWebServiceRequest = new DISWebServiceRequest();
IRequest requestobs = new DISDefaultRequest(obsWebServiceRequest, Constants.SERVICENAME)
{
HttpMethod = HttpMethodName.DELETE.ToString()
};
var resourcePath = ResourcePathBuilder.Standard()
.WithProjectId(_disConfig.GetProjectId())
.WithResource(new StreamResource(null, describeStreamRequest.StreamName))
.Build();
requestobs.ResourcePath = resourcePath;
ResponseResult results = this.Request <ResponseResult>(describeStreamRequest, requestobs);
return(results);
}
/// <summary>
/// Called when the Kinesis Stream Awaiter is created in the CF script, it will wait on the specified stream to enter
/// ACTIVE status
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task <CustomResourceResponse> CreateAsync(CustomResourceRequest request, ILambdaContext context)
{
if (request.ResourceProperties.ContainsKey("StreamName"))
{
context.LogInfo($"Beginning await for Kinesis stream {request.ResourceProperties["StreamName"]}.");
DescribeStreamRequest Request = new DescribeStreamRequest()
{
StreamName = request.ResourceProperties["StreamName"].ToString()
};
while (true)
{
if (context.RemainingTime.TotalMilliseconds < 1500)
{
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.FAILED, "Timeout waiting for stream to become active.", request));
}
DescribeStreamResponse Response = await this._KinesisClient.DescribeStreamAsync(Request);
if ((int)Response.HttpStatusCode < 300)
{
if (Response.StreamDescription.StreamStatus == StreamStatus.ACTIVE)
{
break;
}
}
else
{
context.LogWarning($"Received an unsuccessful response to the describe stream request: {(int)Response.HttpStatusCode}.");
}
Thread.Sleep(_WaitTimeInMillis);
}
context.LogInfo($"Successfully created Kinesis stream {Request.StreamName}.");
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.SUCCESS, "Created", Request.StreamName, request.StackId, request.RequestId, request.LogicalResourceId));
}
else
{
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.FAILED, "The StreamName property was not provided.", "stream", request.StackId, request.RequestId, request.LogicalResourceId));
}
}
/// <summary>
/// This method waits a maximum of 10 minutes for the specified stream to become active.
/// <param name="myStreamName">Name of the stream whose active status is waited upon.</param>
/// </summary>
private static bool WaitForStreamToBecomeAvailable(string myStreamName)
{
var deadline = DateTime.UtcNow + TimeSpan.FromMinutes(10);
while (DateTime.UtcNow < deadline)
{
DescribeStreamRequest describeStreamReq = new DescribeStreamRequest();
describeStreamReq.StreamName = myStreamName;
DescribeStreamResponse describeResult = oClient.DescribeStream(describeStreamReq);
string streamStatus = describeResult.HttpStatusCode.ToString();
Console.Error.WriteLine(" - current state: " + streamStatus);
if (streamStatus == StreamStatus.ACTIVE)
{
return(true);
}
Thread.Sleep(TimeSpan.FromSeconds(20));
}
throw new Exception("Stream " + myStreamName + " never went active.");
}
/// <summary>
/// This method waits a maximum of 10 minutes for the specified stream to become active.
/// <param name="myStreamName">Name of the stream whose active status is waited upon.</param>
/// </summary>
private static void WaitForStreamToBecomeAvailable(string myStreamName)
{
var deadline = DateTime.UtcNow + TimeSpan.FromMinutes(10);
while (DateTime.UtcNow < deadline)
{
DescribeStreamRequest describeStreamReq = new DescribeStreamRequest();
describeStreamReq.StreamName = myStreamName;
var describeResult = kinesisClient.DescribeStreamAsync(describeStreamReq).Result;
string streamStatus = describeResult.StreamDescription.StreamStatus;
Console.Error.WriteLine(" - current state: " + streamStatus);
if (streamStatus == StreamStatus.ACTIVE)
{
return;
}
Thread.Sleep(TimeSpan.FromSeconds(20));
}
throw new Exception("Stream " + myStreamName + " never went active.");
}
public void Start()
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest
{
Limit = 10,
StreamName = _streamName
};
var streamDescription = _client.DescribeStreamAsync(describeStreamRequest).Result;
var shards = streamDescription.StreamDescription.Shards;
_logger.LogInformation("Found {0} shards for stream {1}", shards.Count, _streamName);
_shardReaders.Clear();
foreach (Shard shard in shards)
{
StartReadingShart(shard);
}
}
protected virtual void KinesisClientUpdate(string start_shard_id)
{
try
{
var dsr = new DescribeStreamRequest()
{
Stream = stream, Limit = 100
};
if (!string.IsNullOrEmpty(start_shard_id))
{
dsr.ExclusiveStartShardId = start_shard_id;
}
kinesis_client.DescribeStreamRequest(dsr, (r) => { update_callback(r); }, null, DateTime.Now, DateTime.Now.AddDays(1));
}
catch (Exception e)
{
update_callback(new AwsKinesisResult(e.ToString(), null, DateTime.Now, DateTime.Now));
}
}
//todo: handle multiple shards, aws sdk help says one thread per shard. Is _client thread safe?
//todo: figure out algorithm to merge sequence in correct order
private async Task <string[]> GetShardIds()
{
string lastShardId = null;
var shardIds = new List <string>();
while (true)
{
var request = new DescribeStreamRequest
{
StreamName = _streamName,
ExclusiveStartShardId = lastShardId
};
var stream = (await _client.DescribeStreamAsync(request)).StreamDescription;
shardIds.AddRange(stream.Shards.Select(shard => shard.ShardId));
if (!stream.HasMoreShards)
{
break;
}
lastShardId = shardIds.Last();
}
return(shardIds.ToArray());
}
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonKinesisClient client = new AmazonKinesisClient(creds, config);
DescribeStreamResponse resp = new DescribeStreamResponse();
DescribeStreamRequest req = new DescribeStreamRequest
{
Limit = maxItems
};
resp = client.DescribeStream(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.StreamDescription.Shards)
{
AddObject(obj);
}
}
/// <summary>
/// 查询通道列表
/// </summary>
/// <param name="startStreamName">起始通道名称</param>
/// <param name="limit">每次查询时返回的通道数量</param>
/// <returns></returns>
public static DescribeStreamListResult DescribeStreamList(string startStreamName, int?limit)
{
DescribeStreamListResult response = null;
var dic = new DISIngestionClient();
var request = new DescribeStreamRequest();
if (!string.IsNullOrWhiteSpace(startStreamName))
{
//从该通道开始返回通道列表，返回的通道列表不包括此通道名称。
request.StartStreamName = startStreamName;
}
if (limit != null)
{
//单次请求返回通道列表的最大数量
request.Limit = limit.Value;
}
response = dic.DescribeStreamList(request);
var reqJson = JsonConvert.SerializeObject(response);
Console.WriteLine(reqJson);
return(response);
}
/// <summary>
/// This operation returns the following information about the stream: the current status of the
/// stream, the stream Amazon Resource Name (ARN), and an array of shard objects that comprise the stream.
///
/// More info: http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStream.html
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public DescribeStreamResponse DescribeStream(DescribeStreamRequest request)
{
return(Invoke <DescribeStreamResponse>(request, "DescribeStream"));
}
private async Task ReadStream()
{
List <Models.RefPerson> refPersons = new List <Models.RefPerson>();
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
string streamArn = "arn:aws:dynamodb:ap-southeast-2:358403828169:table/ref_persons/stream/2019-11-18T05:31:40.045";
int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.dynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
while (currentShardIter != null && processedRecordCount < maxItemCount)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
//Console.WriteLine($"{changedRefPersonId}:{changedRefPersonStatus}:{changedRefPersonName}:{changedRefPersonDescription}");
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1")?true:false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = "mobile_stream_1";
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (refPersons.FindAll(p => p.id == refPerson.id).Count == 0)
{
refPersons.Add(refPerson);
}
}
else
{
refPersons.RemoveAll(p => p.id == refPerson.id);
}
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
private void UpdateData()
{
using (IAmazonKinesis klient = new AmazonKinesisClient(_credentials, Amazon.RegionEndpoint.USWest2))
{
ListStreamsResponse resp = klient.ListStreams();
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = StreamName;
DescribeStreamResponse describeResponse = klient.DescribeStream(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
SequenceNumberRange range = shard.SequenceNumberRange;
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = StreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType;
GetShardIteratorResponse iteratorResponse = klient.GetShardIterator(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (IsRunning && !string.IsNullOrEmpty(iteratorId))
{
Thread.Sleep(1);
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = klient.GetRecords(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Amazon.Kinesis.Model.Record> records = getResponse.Records;
if (records.Count > 0)
{
if (IsDebug)
{
string message = string.Format("Received {0} records. ", records.Count);
Console.WriteLine(message);
foreach (IDebugObserver observer in _debugObservers)
{
observer.WriteDebug(message);
}
}
foreach (Amazon.Kinesis.Model.Record record in records)
{
if (!IsRunning)
{
break;
}
Thread.Sleep(1);
string json = Encoding.UTF8.GetString(record.Data.ToArray());
if (IsDebug)
{
string message = "Json string: " + json;
Console.WriteLine(message);
foreach (IDebugObserver observer in _debugObservers)
{
observer.WriteDebug(message);
}
}
T obj = JsonConvert.DeserializeObject <T>(json);
if (obj != null && _dataHandler != null)
{
_dataHandler(obj);
}
}
}
iteratorId = nextIterator;
}
if (!IsRunning)
{
break;
}
}
}
}
public async Task Test1()
{
// Transaction put with an outbox message.
var transactWriteItems = new List <TransactWriteItem>();
for (int i = 0; i < 5; i++)
{
var orgDoc = new Document
{
["Id"] = 123,
["Name"] = $"org-123-{i}",
["Foo"] = Document.FromJson("{\"plot\" : \"Nothing happens at all.\",\"rating\" : 0}")
};
transactWriteItems.Add(new TransactWriteItem
{
Put = new Put
{
TableName = OrgsTableName,
Item = orgDoc.ToAttributeMap()
}
});
var outboxDoc = new Document
{
["Id"] = $"123-{i}",
["Created"] = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
["Body"] = Document.FromJson("{\"plot\" : \"Nothing happens at all.\",\"rating\" : 0}")
};
transactWriteItems.Add(new TransactWriteItem
{
Put = new Put
{
TableName = OutboxTableName,
Item = outboxDoc.ToAttributeMap()
}
});
}
var transactWriteItemsRequest = new TransactWriteItemsRequest
{
TransactItems = transactWriteItems,
};
await _client.TransactWriteItemsAsync(transactWriteItemsRequest);
// Read the changes to outbox change stream
string lastEvaluatedShardId = null;
do
{
var describeStreamRequest = new DescribeStreamRequest
{
StreamArn = _tableLatestStreamArn,
ExclusiveStartShardId = lastEvaluatedShardId
};
var describeStreamResponse = await _streamsClient.DescribeStreamAsync(describeStreamRequest);
foreach (var shard in describeStreamResponse.StreamDescription.Shards)
{
var getShardIteratorRequest = new GetShardIteratorRequest
{
StreamArn = _tableLatestStreamArn,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var getShardIteratorResponse = await _streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
var currentShardIterator = getShardIteratorResponse.ShardIterator;
var iterations = 0; // loop will continue for some time until the stream shard is closed, this just short circuits things for the test.
while (currentShardIterator != null && iterations < 10)
{
var getRecordsRequest = new GetRecordsRequest
{
ShardIterator = currentShardIterator,
};
var getRecordsResponse = await _streamsClient.GetRecordsAsync(getRecordsRequest);
foreach (var record in getRecordsResponse.Records)
{
_testOutputHelper.WriteLine($"{record.EventID} {record.EventName} {record.EventSource} {record.Dynamodb.NewImage.StreamViewType}");
}
currentShardIterator = getRecordsResponse.NextShardIterator;
iterations++;
}
}
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
/// <summary>
/// Returns information about a stream, including the current status of the stream, its
/// Amazon Resource Name (ARN), the composition of its shards, and its corresponding DynamoDB
/// table.
///
/// <note>
/// <para>
/// You can call <code>DescribeStream</code> at a maximum rate of 10 times per second.
/// </para>
/// </note>
/// <para>
/// Each shard in the stream has a <code>SequenceNumberRange</code> associated with it.
/// If the <code>SequenceNumberRange</code> has a <code>StartingSequenceNumber</code>
/// but no <code>EndingSequenceNumber</code>, then the shard is still open (able to receive
/// more stream records). If both <code>StartingSequenceNumber</code> and <code>EndingSequenceNumber</code>
/// are present, then that shard is closed and can no longer receive more data.
/// </para>
/// </summary>
/// <param name="streamArn">The Amazon Resource Name (ARN) for the stream.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
///
/// <returns>The response from the DescribeStream service method, as returned by DynamoDBStreams.</returns>
/// <exception cref="Amazon.DynamoDBv2.Model.InternalServerErrorException">
/// An error occurred on the server side.
/// </exception>
/// <exception cref="Amazon.DynamoDBv2.Model.ResourceNotFoundException">
/// The operation tried to access a nonexistent stream.
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/streams-dynamodb-2012-08-10/DescribeStream">REST API Reference for DescribeStream Operation</seealso>
public virtual Task<DescribeStreamResponse> DescribeStreamAsync(string streamArn, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var request = new DescribeStreamRequest();
request.StreamArn = streamArn;
return DescribeStreamAsync(request, cancellationToken);
}
/// <summary>
/// Returns information about a stream, including the current status of the stream, its
/// Amazon Resource Name (ARN), the composition of its shards, and its corresponding DynamoDB
/// table.
///
/// <note>
/// <para>
/// You can call <code>DescribeStream</code> at a maximum rate of 10 times per second.
/// </para>
/// </note>
/// <para>
/// Each shard in the stream has a <code>SequenceNumberRange</code> associated with it.
/// If the <code>SequenceNumberRange</code> has a <code>StartingSequenceNumber</code>
/// but no <code>EndingSequenceNumber</code>, then the shard is still open (able to receive
/// more stream records). If both <code>StartingSequenceNumber</code> and <code>EndingSequenceNumber</code>
/// are present, then that shard is closed and can no longer receive more data.
/// </para>
/// </summary>
/// <param name="streamArn">The Amazon Resource Name (ARN) for the stream.</param>
///
/// <returns>The response from the DescribeStream service method, as returned by DynamoDBStreams.</returns>
/// <exception cref="Amazon.DynamoDBv2.Model.InternalServerErrorException">
/// An error occurred on the server side.
/// </exception>
/// <exception cref="Amazon.DynamoDBv2.Model.ResourceNotFoundException">
/// The operation tried to access a nonexistent stream.
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/streams-dynamodb-2012-08-10/DescribeStream">REST API Reference for DescribeStream Operation</seealso>
public virtual DescribeStreamResponse DescribeStream(string streamArn)
{
var request = new DescribeStreamRequest();
request.StreamArn = streamArn;
return DescribeStream(request);
}
private async Task ReadRefPersonStream()
{
Dictionary <string, Models.RefPerson> refPersons = new Dictionary <string, Models.RefPerson>();
refPersons.Clear();
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
string streamArn = Models.MyAWSConfigs.DynamodbRefPersonTableStreamArn;
//int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
//&& processedRecordCount < maxItemCount
while (currentShardIter != null)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
string changedRefPersonCamera = record.Dynamodb.NewImage["camera"].S;
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1") ? true : false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = changedRefPersonCamera;
refPerson.lastLocation = cameras[changedRefPersonCamera].location;
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id, Models.MyAWSConfigs.RefImagesBucketName);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (!refPersons.ContainsKey(refPerson.id))
{
refPersons.Add(refPerson.id, refPerson);
}
}
else
{
if (refPersons.ContainsKey(refPerson.id))
{
refPersons.Remove(refPerson.id);
}
}
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
