public async Task BatchAddItemsAsync(IEnumerable <DynamoDBItem> items)
{
var requests = new List <WriteRequest>();
foreach (var item in items)
{
var putRq = new PutRequest(item.ToDictionary());
requests.Add(new WriteRequest(putRq));
}
var batchRq = new Dictionary <string, List <WriteRequest> > {
{ TableName, requests }
};
await _dynamoDbClient.BatchWriteItemAsync(batchRq);
}
static async Task RemoveData25Async(
IAmazonDynamoDB client, string tableName, string partitionKey, string score,
string[] hashList)
{
var request = new Dictionary <string, List <WriteRequest> >()
{
[tableName] = hashList.Select(h =>
{
var dataId = DynamoDbScoreDataConstant.PrefixAnnotation + score + h;
return(new WriteRequest()
{
DeleteRequest = new DeleteRequest()
{
Key = new Dictionary <string, AttributeValue>()
{
[DynamoDbScoreDataPropertyNames.OwnerId] = new AttributeValue(partitionKey),
[DynamoDbScoreDataPropertyNames.DataId] = new AttributeValue(dataId),
},
},
});
}).ToList(),
};
try
{
await client.BatchWriteItemAsync(request);
}
catch (Exception ex)
{
// TODO 削除時に失敗したデータを取得しリトライ処理を入れる
Console.WriteLine(ex.Message);
throw;
}
}
static async Task SendToDynamoAsync(IEnumerable <object> entities, IAmazonDynamoDB client,
ICollection <Dictionary <string, List <WriteRequest> > > entitiesToRetry)
{
var item = new BatchWriteItemRequest
{
RequestItems = new Dictionary <string, List <WriteRequest> >
{
{
"mp-dyndb-entities-qa", entities.Select(t => new WriteRequest
{
PutRequest = new PutRequest(Document.FromJson(JsonConvert.SerializeObject(t))
.ToAttributeMap())
}).ToList()
}
}
};
var result = await client.BatchWriteItemAsync(item);
entitiesToRetry.Add(result.UnprocessedItems);
if (result.HttpStatusCode != HttpStatusCode.OK)
{
throw new Exception();
}
}
async Task PostAsync(IEnumerable <JObject> values, ILambdaContext?context)
{
context?.Logger.LogLine("Sending batch to dynamo");
var requests = values.Select(e => new WriteRequest
{
PutRequest = new PutRequest(Document.FromJson(e.ToString(Formatting.None)).ToAttributeMap())
});
var batch = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> >
{
[tableName] = requests.ToList()
});
var result = await dynamoClient.BatchWriteItemAsync(batch);
if (result.HttpStatusCode != HttpStatusCode.OK)
{
var metadata = JsonConvert.SerializeObject(result.ResponseMetadata.Metadata, new JsonSerializerSettings {
MaxDepth = 3
});
context?.Logger.LogLine($"Dynamo Call failed with {result.HttpStatusCode} {metadata}");
context?.Logger.LogLine($"Data: {string.Join(Environment.NewLine, values.Select(s => s.ToString()))}");
throw new Exception(metadata);
}
}
static async Task DeleteItemRelations25Async(IAmazonDynamoDB client, string tableName, string partitionKey, string score, string[] ids)
{
var request = new Dictionary <string, List <WriteRequest> >()
{
[tableName] = ids.Select(id => new WriteRequest()
{
DeleteRequest = new DeleteRequest()
{
Key = new Dictionary <string, AttributeValue>()
{
[DynamoDbScoreItemRelationPropertyNames.OwnerId] = new AttributeValue(partitionKey),
[DynamoDbScoreItemRelationPropertyNames.ItemRelation] = new AttributeValue(id + score),
}
}
}).ToList(),
};
try
{
await client.BatchWriteItemAsync(request);
}
catch (Exception ex)
{
// TODO 削除時に失敗したデータを取得しリトライ処理を入れる
Console.WriteLine(ex.Message);
throw;
}
}
public async Task <bool> CreateAsync <T>(IReadOnlyList <T> request, CancellationToken token = default)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
var batches = Batch(request);
foreach (var batch in batches)
{
var response = await client.BatchWriteItemAsync(batch, token);
// TODO TO: Make this transactional across batches i.e. if one fails we need to handle it.
// TODO TO: Add logging/codes which describe failures
if (response.HttpStatusCode != HttpStatusCode.OK)
{
return(false);
}
}
new PutItemRequest()
{
TableName = tableName,
Item = Document.FromJson("").ToAttributeMap(),
};
return(true);
}
private Task BatchDeleteAsync(IEnumerable <string> messageIds)
{
var requests = messageIds.Select(x =>
new WriteRequest(
new DeleteRequest(
new Dictionary <string, AttributeValue> {
{ MESSAGE_ID, new AttributeValue(x) }
}
)
)
).ToList();
var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> > {
{ _tableName, requests }
});
return(_dynamoClient.BatchWriteItemAsync(request));
}
public async Task <BatchWriteItemResponse> BatchWriteItemAsync(IEnumerable <WriteRequest> writeRequests)
{
return(await _dynamoClient.BatchWriteItemAsync(new BatchWriteItemRequest {
RequestItems = new Dictionary <string, List <WriteRequest> > {
{ _songTableName, writeRequests.ToList() }
}
}));
}
public async Task DynamoDbUpdateCacheVideoDataAsync(List <WriteRequest> writeRequests)
{
var batchPutRequest = new BatchWriteItemRequest {
RequestItems = new Dictionary <string, List <WriteRequest> > {
{ _dynamoDbVideoTableName, writeRequests }
}
};
await _dynamoDbClient.BatchWriteItemAsync(batchPutRequest);
}
private async Task CallUntilCompletionAsync(BatchWriteItemRequest request, Dictionary <string, Dictionary <Key, Document> > documentMap, IAmazonDynamoDB client, CancellationToken cancellationToken)
#endif
{
do
{
var result = await client.BatchWriteItemAsync(request, cancellationToken).ConfigureAwait(false);
request.RequestItems = result.UnprocessedItems;
Dictionary <Key, Document> unprocessedDocuments = new Dictionary <Key, Document>(keyComparer);
foreach (var unprocessedItems in result.UnprocessedItems)
{
string tableName = unprocessedItems.Key;
Table table = tableMap[tableName];
Dictionary <Key, Document> tableDocumentMap = documentMap[tableName];
foreach (var writeRequest in unprocessedItems.Value)
{
if (writeRequest.PutRequest != null)
{
var doc = table.FromAttributeMap(writeRequest.PutRequest.Item);
var key = table.MakeKey(doc);
Document document = null;
if (tableDocumentMap.TryGetValue(key, out document))
{
// Remove unprocessed requests from the document map
// and copy them to unprocessed documents.
unprocessedDocuments.Add(key, document);
tableDocumentMap.Remove(key);
}
}
}
// Commit the remaining documents in the document map
foreach (var document in tableDocumentMap.Values)
{
document.CommitChanges();
}
// Replace existing documents with just the unprocessed documents
documentMap[tableName] = unprocessedDocuments;
}
} while (request.RequestItems.Count > 0);
// Commit any remaining documents in document map.
// This would only happen if we are not able to match the items sent in the request
// with the items returned back as unprocessed items.
foreach (var tableDocumentMap in documentMap.Values)
{
foreach (var document in tableDocumentMap.Values)
{
document.CommitChanges();
}
}
}
/// <summary>
/// Removes items from a DynamoDB table.
/// </summary>
/// <param name="client">An initialized DynamoDB client object.</param>
/// <param name="tableName">The name of the table from which the item
/// will be deleted.</param>
/// <param name="idstring">The id of the item to be deleted.</param>
/// <param name="area">The area of the item to delete.</param>
/// <returns>The response from the call to BatchWriteItemAsync.</returns>
public static async Task <BatchWriteItemResponse> RemoveItemsAsync(
IAmazonDynamoDB client,
string tableName,
string idstring,
string area)
{
var ids = idstring.Split(" ");
var writeRequests = new List <WriteRequest>();
var items = new Dictionary <string, List <WriteRequest> >();
for (int i = 0; i < ids.Length; i++)
{
var writeRequest = new WriteRequest
{
// For the operation to delete an item, if you provide a primary key value
// that does not exist in the table, there is no error.
DeleteRequest = new DeleteRequest
{
Key = new Dictionary <string, AttributeValue>()
{
{
"ID",
new AttributeValue
{
S = ids[i],
}
},
{
"Area",
new AttributeValue
{
S = area,
}
},
},
},
};
writeRequests.Add(writeRequest);
}
items.Add(tableName, writeRequests);
var request = new BatchWriteItemRequest(items);
var response = await client.BatchWriteItemAsync(request);
return(response);
}
public async Task InsertResults(ElectionStatistics electionStatistics)
{
try
{
_logger.LogInformation($"Inserting results");
Dictionary <string, AttributeValue> item = new Dictionary <string, AttributeValue>();
item["id"] = new AttributeValue {
S = electionStatistics.Id
};
item["fileType"] = new AttributeValue {
S = electionStatistics.Type
};
item["fileSource"] = new AttributeValue {
S = electionStatistics.Source
};
item["electionId"] = new AttributeValue {
S = electionStatistics.ElectionId
};
item["statisticsJson"] = new AttributeValue {
S = electionStatistics.StatisticsJson
};
item["csvTimestamp"] = new AttributeValue {
N = electionStatistics.Timestamp.ToString()
};
List <WriteRequest> items = new List <WriteRequest>
{
new WriteRequest
{
PutRequest = new PutRequest
{
Item = item
}
}
};
Dictionary <string, List <WriteRequest> > requestItems = new Dictionary <string, List <WriteRequest> >();
requestItems[_config.TableName] = items;
await _dynamoDb.BatchWriteItemAsync(requestItems);
}
catch (Exception e)
{
_logger.LogError(e, "Couldn't insert results");
}
}
public async Task InsertResults(ElectionStatistics electionStatistics)
{
try
{
_logger.LogInformation($"Inserting results");
var tableExists = await TableIsReady();
if (!tableExists)
{
await CreateTable();
}
Dictionary <string, AttributeValue> item = new Dictionary <string, AttributeValue>();
item["id"] = new AttributeValue {
S = electionStatistics.Id
};
item["csvType"] = new AttributeValue {
S = electionStatistics.Type
};
item["csvLocation"] = new AttributeValue {
S = electionStatistics.Location
};
item["statisticsJson"] = new AttributeValue {
S = electionStatistics.StatisticsJson
};
item["csvTimestamp"] = new AttributeValue {
N = electionStatistics.FileTimestamp.ToString()
};
List <WriteRequest> items = new List <WriteRequest>();
items.Add(new WriteRequest
{
PutRequest = new PutRequest {
Item = item
}
});
Dictionary <string, List <WriteRequest> > requestItems = new Dictionary <string, List <WriteRequest> >();
requestItems[_config.TableName] = items;
await _dynamoDb.BatchWriteItemAsync(requestItems);
}
catch (Exception e)
{
_logger.LogError(e, "Couldn't insert results");
}
}
async Task Load(ConcurrentBag <DynamoDbMetrics> batchMetrics, List <BatchWriteItemRequest> batches)
{
foreach (var batch in batches)
{
var batchMetric = new DynamoDbMetrics("Load", OperationType.Load);
var response = await batchMetric.HandleAsync(client.BatchWriteItemAsync(batch));
if (response.UnprocessedItems.Any() || response.HttpStatusCode != HttpStatusCode.OK ||
response.ItemCollectionMetrics.Any() || response.ResponseMetadata.Metadata.Any())
{
batchMetric.Error += JsonConvert.SerializeObject(response) + Environment.NewLine;
}
batchMetric.WriteCapacityUnits = response.ConsumedCapacity.Capacity;
batchMetrics.Add(batchMetric);
}
}
private async Task BatchWriteItem(List <WriteRequest> list)
{
var request = new BatchWriteItemRequest
{
ReturnConsumedCapacity = ReturnConsumedCapacity.TOTAL,
};
request.RequestItems["Music"] = list;
BatchWriteItemResponse response;
do
{
response = await DDBClient.BatchWriteItemAsync(request);
request.RequestItems = response.UnprocessedItems;
} while (response.UnprocessedItems.Count > 0);
}
async Task <BatchWriteItemResponse> IAmazonDynamoDB.BatchWriteItemAsync(BatchWriteItemRequest request, CancellationToken cancellationToken)
{
try {
var response = await _dynamoDBClient.BatchWriteItemAsync(request, cancellationToken);
_callback?.Invoke(new {
Action = nameof(IAmazonDynamoDB.BatchWriteItemAsync),
Request = request,
Response = response
});
return(response);
} catch (Exception e) {
_callback?.Invoke(new {
Action = nameof(IAmazonDynamoDB.BatchWriteItemAsync),
Request = request,
Exception = new {
Type = e.GetType().FullName,
Message = e.Message
}
});
throw;
}
}
public static BatchWriteItemResponse BatchWriteItem(this IAmazonDynamoDB client, BatchWriteItemRequest request)
{
return(client.BatchWriteItemAsync(request).GetResult());
}
private async Task DeleteRecords(string table, List <int> ids)
{
var writeRequests = new List <WriteRequest>();
var writeRequestsCollection = new List <List <WriteRequest> >();
int count = 0;
try
{
foreach (var id in ids)
{
count++;
var key = new Dictionary <string, AttributeValue> {
{ "Id", new AttributeValue {
N = id.ToString()
} }
};
writeRequests.Add(new WriteRequest()
{
DeleteRequest = new DeleteRequest {
Key = key
}
});
if (count == 25)
{
writeRequestsCollection.Add(writeRequests);
writeRequests = new List <WriteRequest>();
count = 0;
}
}
if (count > 0)
{
writeRequestsCollection.Add(writeRequests);
}
foreach (var requests in writeRequestsCollection)
{
var requestItems = new Dictionary <string, List <WriteRequest> > {
{ table, requests }
};
var batchRequest = new BatchWriteItemRequest {
RequestItems = requestItems
};
await _dynamoDBClient.BatchWriteItemAsync(batchRequest);
}
}
catch (Exception e)
{
_logger.LogError(@$ "Error deleting records from: {table} table.{_newLine} Error: {e}.{_newLine}");
throw e;
}
}
async Task DeleteUserEntries(FilePath path, string[] users)
{
var deletes = from user in users select new WriteRequest(new DeleteRequest(
new Dictionary <string, AttributeValue> {
{ "user", new AttributeValue(user) },
{ "path", new AttributeValue(path.NormalizedPath) }
}
));
List <Task <BatchResult> > operations = new List <Task <BatchResult> >();
// Batch sizes are limited to 25 operations, as per https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
foreach (var batch in Batch(deletes, 25))
{
operations.Add(
client.BatchWriteItemAsync(
new Dictionary <string, List <WriteRequest> > {
{ this.tableName, batch }
}).ContinueWith <BatchResult>(task => {
if (task.IsFaulted)
{
return(new BatchError(task.Exception));
}
var response = task.Result;
if (response.UnprocessedItems.Count != 0)
{
return(new BatchUnprocessed(response.UnprocessedItems[this.tableName]));
}
return(new BatchSuccess());
})
);
}
var failed = false;
var results = await Task.WhenAll(operations);
foreach (BatchResult result in results)
{
if (result is BatchSuccess)
{
continue;
}
else if (result is BatchUnprocessed u)
{
failed = true;
LambdaLogger.Log($"Failed to delete user file entries: {u}");
}
else if (result is BatchError e)
{
failed = true;
LambdaLogger.Log($"Failed to delete user file entries with error: {e}");
}
else
{
failed = true;
LambdaLogger.Log($"PROGRAM ERROR, unknown type of batch result: {result}");
}
}
if (failed)
{
throw new ServerException("Failed to delete user file entries");
}
}
public async Task ProcessUrl(UrlInfo urlInfo)
{
// stop processing because we have reached the maximum depth
if (urlInfo.Depth <= 0)
{
LogInfo($"Ignoring URL '{urlInfo.Url}' because we have reached the maximum depth");
return;
}
// get page contents of wikipedia article
var httpClient = new HttpClient();
var url = urlInfo.Url;
var response = await httpClient.GetAsync(url);
var strContents = await response.Content.ReadAsStringAsync();
// iterate through links found in page contents
var foundLinks = new List <Uri>();
var matchedLink = false;
string backLink = null;
foreach (var link in HelperFunctions.FindLinks(strContents))
{
// format internal links and attempt to parse for validity
var llink = HelperFunctions.FixInternalLink(link, urlInfo.Url);
Uri parsedLink = null;
try {
parsedLink = new Uri(llink);
} catch {
continue;
}
// ignore external urls
if (!parsedLink.Host.Equals(urlInfo.Url.Host))
{
continue;
}
// check for Target article
if (parsedLink.AbsolutePath == urlInfo.Target.AbsolutePath)
{
matchedLink = true;
backLink = parsedLink.ToString();
break;
}
// filter out unecessary links
if (HelperFunctions.FilterLink(parsedLink))
{
continue;
}
foundLinks.Add(parsedLink);
}
var item = new Dictionary <string, AttributeValue>();
// create solution item
if (matchedLink)
{
item["WikiId"] = new AttributeValue {
S = $"{urlInfo.Origin}::{backLink}"
};
item["CrawlUrl"] = new AttributeValue {
S = $"{backLink}"
};
item["CrawlDepth"] = new AttributeValue {
N = "0"
};
item["CrawlOrigin"] = new AttributeValue {
S = $"{urlInfo.Origin}"
};
item["CrawlTarget"] = new AttributeValue {
S = $"{urlInfo.Target}"
};
item["CrawlBackLink"] = new AttributeValue {
S = $"{urlInfo.Url}"
};
}
// only enqueue child links if depth is greater than 1
if (!matchedLink && urlInfo.Depth > 1)
{
// take the first link and add it to the table
var linkToQueue = foundLinks.Take(LINKS_PER_PAGE).First();
item["WikiId"] = new AttributeValue {
S = $"{urlInfo.Origin}::{linkToQueue}"
};
item["CrawlUrl"] = new AttributeValue {
S = $"{linkToQueue}"
};
item["CrawlDepth"] = new AttributeValue {
N = $"{urlInfo.Depth - 1}"
};
item["CrawlOrigin"] = new AttributeValue {
S = $"{urlInfo.Origin}"
};
item["CrawlTarget"] = new AttributeValue {
S = $"{urlInfo.Target}"
};
item["CrawlBackLink"] = new AttributeValue {
S = $"{urlInfo.Url}"
};
}
// write the item to the table
if (matchedLink || urlInfo.Depth > 1)
{
var writeRequest = new BatchWriteItemRequest {
RequestItems = new Dictionary <string, List <WriteRequest> > {
[TABLE_NAME] = new List <WriteRequest> {
new WriteRequest {
PutRequest = new PutRequest {
Item = item
}
},
}
}
};
var writeResponse = await _client.BatchWriteItemAsync(writeRequest);
}
}
public static async Task <BatchWriteItemResponse> AddItemsAsync(bool debug, IAmazonDynamoDB client, string table, string[] inputs, int index)
{
var writeRequests = new List <WriteRequest>();
string[] headers = inputs[0].Split(",");
int numcolumns = headers.Length;
string line;
// Read the rest of the file, line by line
for (int input = 1; input < inputs.Length; input++)
{
line = inputs[input];
// Split line into columns
string[] values = line.Split(',');
// If we don't have the right number of parts, something's wrong
if (values.Length != numcolumns)
{
Console.WriteLine("Did not have " + numcolumns.ToString() + " columns in: ");
Console.WriteLine(line);
return(null);
}
var item = new Dictionary <string, AttributeValue>
{
{ "ID", new AttributeValue {
S = index.ToString()
} }
};
DebugPrint(debug, "Set ID string attribute to " + index.ToString());
for (int i = 0; i < numcolumns; i++)
{
if ((headers[i] == "Customer_ID") || (headers[i] == "Order_ID") || (headers[i] == "Order_Customer") || (headers[i] == "Order_Product") || (headers[i] == "Product_ID") || (headers[i] == "Product_Quantity") || (headers[i] == "Product_Cost"))
{
item.Add(headers[i], new AttributeValue {
N = values[i]
});
DebugPrint(debug, "Set " + headers[i] + " int attribute to " + values[i]);
}
else if (headers[i] == "Order_Date")
{
// The datetime format is:
// YYYY-MM-DD HH:MM:SS
DateTime myDateTime = DateTime.ParseExact(values[i], "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
TimeSpan timeSpan = myDateTime - new DateTime(1970, 1, 1, 0, 0, 0);
item.Add(headers[i], new AttributeValue {
N = ((long)timeSpan.TotalSeconds).ToString()
});
DebugPrint(debug, "Set " + headers[i] + " int (long) attribute to " + ((long)timeSpan.TotalSeconds).ToString());
}
else
{
item.Add(headers[i], new AttributeValue {
S = values[i]
});
DebugPrint(debug, "Set " + headers[i] + " string attribute to " + values[i]);
}
}
DebugPrint(debug, "");
index++;
WriteRequest putRequest = new WriteRequest(new PutRequest(item));
writeRequests.Add(putRequest);
}
var requestItems = new Dictionary <string, List <WriteRequest> >
{
{ table, writeRequests }
};
var request = new BatchWriteItemRequest
{
ReturnConsumedCapacity = "TOTAL",
RequestItems = requestItems
};
var response = await client.BatchWriteItemAsync(request);
return(response);
}