private async Task <List <Quote> > getQuotesByIds(IEnumerable <string> ids)
{
// ETL
List <Dictionary <string, AttributeValue> > listOfIdsToBeFetched
= new List <Dictionary <string, AttributeValue> >();
foreach (string id in ids)
{
listOfIdsToBeFetched.Add(new Dictionary <string, AttributeValue>
{
{ DataDefinitions.QUOTES_TABLE_HASH_KEY, new AttributeValue {
N = id
} }
});
}
BatchGetItemRequest request = new BatchGetItemRequest(new Dictionary <string, KeysAndAttributes>
{
{ DataDefinitions.QUOTES_TABLE, new KeysAndAttributes {
Keys = listOfIdsToBeFetched
} }
});
BatchGetItemResponse response = await _client.BatchGetItemAsync(request);
// Extract data from the complex structure of response
// Will return records
List <Quote> quotes = new List <Quote>();
foreach (Dictionary <string, AttributeValue> pair in response.Responses[DataDefinitions.QUOTES_TABLE])
{
quotes.Add(Quote.ToQuoteFromTable(pair));
}
return(quotes);
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
BatchGetItemResponse response = new BatchGetItemResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("ConsumedCapacity", targetDepth))
{
var unmarshaller = new ListUnmarshaller <ConsumedCapacity, ConsumedCapacityUnmarshaller>(ConsumedCapacityUnmarshaller.Instance);
response.ConsumedCapacity = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("Responses", targetDepth))
{
var unmarshaller = new DictionaryUnmarshaller <string, List <Dictionary <string, AttributeValue> >, StringUnmarshaller, ListUnmarshaller <Dictionary <string, AttributeValue>, DictionaryUnmarshaller <string, AttributeValue, StringUnmarshaller, AttributeValueUnmarshaller> > >(StringUnmarshaller.Instance, new ListUnmarshaller <Dictionary <string, AttributeValue>, DictionaryUnmarshaller <string, AttributeValue, StringUnmarshaller, AttributeValueUnmarshaller> >(new DictionaryUnmarshaller <string, AttributeValue, StringUnmarshaller, AttributeValueUnmarshaller>(StringUnmarshaller.Instance, AttributeValueUnmarshaller.Instance)));
response.Responses = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("UnprocessedKeys", targetDepth))
{
var unmarshaller = new DictionaryUnmarshaller <string, KeysAndAttributes, StringUnmarshaller, KeysAndAttributesUnmarshaller>(StringUnmarshaller.Instance, KeysAndAttributesUnmarshaller.Instance);
response.UnprocessedKeys = unmarshaller.Unmarshall(context);
continue;
}
}
return(response);
}
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonDynamoDBConfig config = new AmazonDynamoDBConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonDynamoDBClient client = new AmazonDynamoDBClient(creds, config);
BatchGetItemResponse resp = new BatchGetItemResponse();
do
{
BatchGetItemRequest req = new BatchGetItemRequest
{
RequestItems = resp.UnprocessedKeys
};
resp = client.BatchGetItem(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.Responses)
{
AddObject(obj);
}
foreach (var obj in resp.UnprocessedKeys)
{
AddObject(obj);
}
}while (resp.UnprocessedKeys.Count > 0);
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
BatchGetItemResponse response = new BatchGetItemResponse();
context.Read();
UnmarshallResult(context, response);
return(response);
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
BatchGetItemResponse response = new BatchGetItemResponse();
context.Read();
response.BatchGetItemResult = BatchGetItemResultUnmarshaller.GetInstance().Unmarshall(context);
return(response);
}
public async Task <string[]> GetSeenItems(string[] items, CancellationToken token)
{
BatchGetItemResponse response = await dynamo.BatchGetItemAsync(new BatchGetItemRequest
{
RequestItems = new Dictionary <string, KeysAndAttributes>
{
{
StateTableName,
new KeysAndAttributes
{
Keys = items.Select(x => new Dictionary <string, AttributeValue> {
{ ItemIdKey, new AttributeValue(x.ToString()) }
}).ToList()
}
}
}
}, token);
return(response.Responses[StateTableName].Select(x => x[ItemIdKey].S).ToArray());
}
private static void UnmarshallResult(JsonUnmarshallerContext context, BatchGetItemResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = originalDepth + 1;
while (context.Read())
{
if (context.TestExpression("Responses", targetDepth))
{
context.Read();
response.Responses = new Dictionary <String, List <Dictionary <string, AttributeValue> > >();
if (context.CurrentTokenType == JsonToken.Null)
{
continue;
}
KeyValueUnmarshaller <string, List <Dictionary <string, AttributeValue> >, StringUnmarshaller, ListUnmarshaller <Dictionary <string, AttributeValue>, DictionaryUnmarshaller <string, AttributeValue, StringUnmarshaller, AttributeValueUnmarshaller> > > unmarshaller = new KeyValueUnmarshaller <string, List <Dictionary <string, AttributeValue> >, StringUnmarshaller, ListUnmarshaller <Dictionary <string, AttributeValue>, DictionaryUnmarshaller <string, AttributeValue, StringUnmarshaller, AttributeValueUnmarshaller> > >(StringUnmarshaller.GetInstance(), new ListUnmarshaller <Dictionary <string, AttributeValue>, DictionaryUnmarshaller <string, AttributeValue, StringUnmarshaller, AttributeValueUnmarshaller> >(new DictionaryUnmarshaller <string, AttributeValue, StringUnmarshaller, AttributeValueUnmarshaller>(StringUnmarshaller.GetInstance(), AttributeValueUnmarshaller.GetInstance())));
while (context.Read())
{
JsonToken token = context.CurrentTokenType;
if (token == JsonToken.ArrayStart || token == JsonToken.ObjectStart)
{
continue;
}
if (token == JsonToken.ArrayEnd || token == JsonToken.ObjectEnd)
{
break;
}
KeyValuePair <string, List <Dictionary <string, AttributeValue> > > kvp = unmarshaller.Unmarshall(context);
response.Responses.Add(kvp.Key, kvp.Value);
}
continue;
}
if (context.TestExpression("UnprocessedKeys", targetDepth))
{
context.Read();
response.UnprocessedKeys = new Dictionary <String, KeysAndAttributes>();
if (context.CurrentTokenType == JsonToken.Null)
{
continue;
}
KeyValueUnmarshaller <string, KeysAndAttributes, StringUnmarshaller, KeysAndAttributesUnmarshaller> unmarshaller = new KeyValueUnmarshaller <string, KeysAndAttributes, StringUnmarshaller, KeysAndAttributesUnmarshaller>(StringUnmarshaller.GetInstance(), KeysAndAttributesUnmarshaller.GetInstance());
while (context.Read())
{
JsonToken token = context.CurrentTokenType;
if (token == JsonToken.ArrayStart || token == JsonToken.ObjectStart)
{
continue;
}
if (token == JsonToken.ArrayEnd || token == JsonToken.ObjectEnd)
{
break;
}
KeyValuePair <string, KeysAndAttributes> kvp = unmarshaller.Unmarshall(context);
response.UnprocessedKeys.Add(kvp.Key, kvp.Value);
}
continue;
}
if (context.TestExpression("ConsumedCapacity", targetDepth))
{
context.Read();
response.ConsumedCapacity = new List <ConsumedCapacity>();
if (context.CurrentTokenType == JsonToken.Null)
{
continue;
}
ConsumedCapacityUnmarshaller unmarshaller = ConsumedCapacityUnmarshaller.GetInstance();
while (context.Read())
{
JsonToken token = context.CurrentTokenType;
if (token == JsonToken.ArrayStart)
{
continue;
}
if (token == JsonToken.ArrayEnd)
{
break;
}
response.ConsumedCapacity.Add(unmarshaller.Unmarshall(context));
}
continue;
}
if (context.CurrentDepth <= originalDepth)
{
return;
}
}
return;
}
private byte[] GetRestOfTheData(string id, int itemsCount)
{
if (itemsCount == 2)
{
GetItemResponse response;
using (var client = CreateClient())
{
response = client.GetItem(_tableName, new Dictionary <string, AttributeValue>()
{
{ DynamoDBMeta.IdColumn, new AttributeValue(id + "1") }
});
}
if (response.HttpStatusCode != HttpStatusCode.OK)
{
throw CreateInvalidResponseException(
response.ResponseMetadata == null ? null : response.ResponseMetadata.Metadata,
response.HttpStatusCode);
}
return(response.Item[DynamoDBMeta.DataColumn].B.ToArray());
}
var map = new Dictionary <int, byte[]>();
BatchGetItemResponse batchGetItemResponse = null;
using (var client = CreateClient())
{
do
{
var req = batchGetItemResponse != null ? new BatchGetItemRequest(batchGetItemResponse.UnprocessedKeys) : new BatchGetItemRequest(new Dictionary <string, KeysAndAttributes>()
{
{
_tableName, new KeysAndAttributes()
{
AttributesToGet = new List <string>()
{
DynamoDBMeta.DataColumn, DynamoDBMeta.IdColumn
},
Keys =
Enumerable.Range(1, itemsCount).Select(
x => new Dictionary <string, AttributeValue>()
{
{
DynamoDBMeta.IdColumn,
new AttributeValue(id +
x.ToString(CultureInfo.InvariantCulture))
}
}).ToList()
}
}
});
batchGetItemResponse = client.BatchGetItem(req);
foreach (var response in batchGetItemResponse.Responses[_tableName])
{
map[Int32.Parse(response[DynamoDBMeta.IdColumn].S.Last().ToString(CultureInfo.InvariantCulture))] =
response[DynamoDBMeta.DataColumn].B.ToArray();
}
} while (batchGetItemResponse.UnprocessedKeys != null && batchGetItemResponse.UnprocessedKeys.Count > 0);
}
var retVal = new byte[map.Sum(x => x.Value.Length)];
var startIndex = 0;
for (var i = 1; i < itemsCount; i++)
{
var data = map[i];
for (var j = 0; j < data.Length; j++)
{
retVal[j + startIndex] = data[j];
}
startIndex += data.Length;
}
return(retVal);
}
public async Task <IEnumerable <TObject> > GetBatchItemsAsync <TObject>(string tableName,
IEnumerable <object> keyValues)
{
var keyValuesJsonData = keyValues.Select(JsonConvert.SerializeObject);
var keyValuesDocument = keyValuesJsonData.Select(Document.FromJson);
Table table;
try
{
table = Table.LoadTable(dynamoDBClient, tableName);
}
catch (ResourceNotFoundException e)
{
throw new TableNameFailException($"Not found the source '{tableName}'.", e);
}
foreach (var k in table.Keys)
{
foreach (var document in keyValuesDocument)
{
if (!document.ContainsKey(k.Key))
{
throw new PrimaryKeyNameFailException($"Not found the primary key '{k.Key}' in query.", null);
}
}
}
var request = new BatchGetItemRequest();
request.RequestItems = new Dictionary <string, KeysAndAttributes>
{
{
tableName,
new KeysAndAttributes
{
Keys = keyValuesDocument.Select(d => d.ToAttributeMap()).ToList()
}
}
};
BatchGetItemResponse response = await dynamoDBClient.BatchGetItemAsync(request);
var items = response.Responses[tableName];
var resultItems = new List <TObject>();
foreach (var item in items)
{
var document = Document.FromAttributeMap(item);
var jsonData = document.ToJson();
TObject data;
try
{
data = JsonConvert.DeserializeObject <TObject>(jsonData);
}
catch (JsonReaderException e)
{
throw new IncorrectDataFormatException(
$"'{typeof(TObject).Name}' and object '{jsonData}' have a different data format",
e);
}
resultItems.Add(data);
}
return(resultItems);
}
