public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonKinesisClient client = new AmazonKinesisClient(creds, config);
DescribeStreamResponse resp = new DescribeStreamResponse();
do
{
DescribeStreamRequest req = new DescribeStreamRequest
{
ExclusiveStartShardId = resp.StreamDescription.Shards[0].ShardId
,
Limit = maxItems
};
resp = client.DescribeStream(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.StreamDescription.Shards)
{
AddObject(obj);
}
}while (!string.IsNullOrEmpty(resp.StreamDescription.Shards[0].ShardId));
}
private static void ReadFromStream()
{
/*Config example*/
//AmazonKinesisConfig config = new AmazonKinesisConfig();
//config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1;
//AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
/*instance example*/
var kinesisClient = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1);
String kinesisStreamName = "your-kinesis-name";
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = kinesisStreamName;
DescribeStreamResponse describeResponse = kinesisClient.DescribeStream(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = kinesisStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = kinesisClient.GetShardIterator(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = kinesisClient.GetRecords(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string json = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("Json string: " + json);
}
}
iteratorId = nextIterator;
}
}
}
public DescribeStreamResponse GetStreamResponse(string streamName = null)
{
if (string.IsNullOrEmpty(streamName) && string.IsNullOrEmpty(_streamName))
{
throw new Exception("Please specify a stream name to get the stream response.");
}
var request = new DescribeStreamRequest()
{
StreamName = streamName ?? _streamName
};
return(_client.DescribeStream(request));
}
/// <summary>
/// This method waits a maximum of 10 minutes for the specified stream to become active.
/// <param name="myStreamName">Name of the stream whose active status is waited upon.</param>
/// </summary>
private static void WaitForStreamToBecomeAvailable(string myStreamName)
{
var deadline = DateTime.UtcNow + TimeSpan.FromMinutes(10);
while (DateTime.UtcNow < deadline)
{
DescribeStreamRequest describeStreamReq = new DescribeStreamRequest();
describeStreamReq.StreamName = myStreamName;
DescribeStreamResult describeResult = kinesisClient.DescribeStream(describeStreamReq);
string streamStatus = describeResult.StreamDescription.StreamStatus;
Console.Error.WriteLine(" - current state: " + streamStatus);
if (streamStatus == StreamStatus.ACTIVE)
{
return;
}
Thread.Sleep(TimeSpan.FromSeconds(20));
}
throw new Exception("Stream " + myStreamName + " never went active.");
}
private void UpdateData()
{
using (IAmazonKinesis klient = new AmazonKinesisClient(_credentials, Amazon.RegionEndpoint.USWest2))
{
ListStreamsResponse resp = klient.ListStreams();
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = StreamName;
DescribeStreamResponse describeResponse = klient.DescribeStream(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
SequenceNumberRange range = shard.SequenceNumberRange;
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = StreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType;
GetShardIteratorResponse iteratorResponse = klient.GetShardIterator(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (IsRunning && !string.IsNullOrEmpty(iteratorId))
{
Thread.Sleep(1);
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = klient.GetRecords(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Amazon.Kinesis.Model.Record> records = getResponse.Records;
if (records.Count > 0)
{
if (IsDebug)
{
string message = string.Format("Received {0} records. ", records.Count);
Console.WriteLine(message);
foreach (IDebugObserver observer in _debugObservers)
{
observer.WriteDebug(message);
}
}
foreach (Amazon.Kinesis.Model.Record record in records)
{
if (!IsRunning)
{
break;
}
Thread.Sleep(1);
string json = Encoding.UTF8.GetString(record.Data.ToArray());
if (IsDebug)
{
string message = "Json string: " + json;
Console.WriteLine(message);
foreach (IDebugObserver observer in _debugObservers)
{
observer.WriteDebug(message);
}
}
T obj = JsonConvert.DeserializeObject <T>(json);
if (obj != null && _dataHandler != null)
{
_dataHandler(obj);
}
}
}
iteratorId = nextIterator;
}
if (!IsRunning)
{
break;
}
}
}
}
public List <JObject> GetRecords(Filter filter, string additionalFilterStr)
{
List <JObject> jsonObjects = new List <JObject>();
try
{
var response = _amazonKinesisClient.DescribeStream(new DescribeStreamRequest()
{
StreamName = _streamName
});
foreach (var shard in response.StreamDescription.Shards)
{
var getRecordsRequest = new GetRecordsRequest();
getRecordsRequest.ShardIterator =
_amazonKinesisClient.GetShardIterator(new GetShardIteratorRequest()
{
StreamName = _streamName,
ShardIteratorType = ShardIteratorType.AT_TIMESTAMP,
ShardId = shard.ShardId,
Timestamp = _timePeriod
})
.ShardIterator;
getRecordsRequest.Limit = ConfigurationManager.AppSettings["FetchRecordsLimit"] != null?int.Parse(ConfigurationManager.AppSettings["FetchRecordsLimit"]) : 100;
GetRecordsResponse recordResponse;
dynamic additionalFilter = JObject.Parse(additionalFilterStr);
do
{
recordResponse = _amazonKinesisClient.GetRecords(getRecordsRequest);
getRecordsRequest.ShardIterator = recordResponse.NextShardIterator;
foreach (var record in recordResponse.Records)
{
var sr = new StreamReader(record.Data);
var myStr = sr.ReadToEnd();
dynamic jsonStr = JObject.Parse(myStr);
bool isObjectQualified = FilterJsonObjectArray(filter, additionalFilter, jsonStr);
if (isObjectQualified)
{
jsonObjects.Add(jsonStr);
}
}
} while (recordResponse.MillisBehindLatest > 0);
Console.Out.WriteLine("response = {0}", recordResponse);
}
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
return(jsonObjects);
}
