public Task BatchUpsertAsync(IEnumerable <TEntity> entities)
{
var batch = Context.CreateBatchWrite <TEntity>();
batch.AddPutItems(entities);
return(batch.ExecuteAsync());
}
public async Task BatchStoreAsync <T>(IEnumerable <T> items) where T : class
{
var itemBatch = DbContext.CreateBatchWrite <T>();
itemBatch.AddPutItems(items);
await itemBatch.ExecuteAsync();
}
/// <summary>
/// Save for individual item
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="t"></param>
public static void Save <T>(T t)
{
var batch = _context.CreateBatchWrite <T>();
batch.AddPutItem(t);
_context.ExecuteBatchWriteAsync(new BatchWrite[] { batch });
//batch.ExecuteAsync();
}
public void SaveAreas(List <Area> areas)
{
Parallel.ForEach(areas.ChunkBy <Area>(200), (item, state, index) =>
{
Console.WriteLine("Initiating a new chunk. Index: " + index);
var bulkInsert = context.CreateBatchWrite <Area>();
bulkInsert.AddPutItems(item);
bulkInsert.ExecuteAsync().Wait();
Console.WriteLine("Chunk inserted successfully. Index" + index);
});
}
static async Task Main()
{
var region = Amazon.RegionEndpoint.APNortheast1; // 使用するリージョン
var client = new AmazonDynamoDBClient(region);
// 使用するDynamoDBテーブルを作成する
await CreateTableIfNotExistsAsync(client, TABLE_NAME);
using (var context = new DynamoDBContext(client))
{
//
// データの追加 (PutItem処理)
//
var item = new SampleTableData
{
PartitionKey = 0,
SortKey = "SK0",
Property1 = DateTime.Now
};
await context.SaveAsync(item);
//
// 複数データの追加 (BatchWriteItem処理)
//
var batchWrite = context.CreateBatchWrite <SampleTableData>();
var items = Enumerable.Range(1, 9).Select(i => new SampleTableData
{
PartitionKey = 0,
SortKey = "SK" + i,
Property1 = DateTime.Now
});
batchWrite.AddPutItems(items);
await batchWrite.ExecuteAsync();
//
// データの取得 (Query処理)
//
const int scanPartitionKey = 0;
var queriedItems = await context.QueryAsync <SampleTableData>(scanPartitionKey).GetRemainingAsync();
//
// データの削除 (DeleteItem処理)
//
await context.DeleteAsync(queriedItems.First());
//
// 複数データの削除 (DeleteItem処理)
//
var batchDelete = context.CreateBatchWrite <SampleTableData>();
batchDelete.AddDeleteItems(queriedItems.Skip(1).Take(4));
await batchDelete.ExecuteAsync();
}
}
private static void SingleTableBatchWrite(DynamoDBContext context)
{
Book book1 = new Book
{
Id = 902,
InPublication = true,
ISBN = "902-11-11-1111",
PageCount = "100",
Price = 10,
ProductCategory = "Book",
Title = "My book3 in batch write"
};
Book book2 = new Book
{
Id = 903,
InPublication = true,
ISBN = "903-11-11-1111",
PageCount = "200",
Price = 10,
ProductCategory = "Book",
Title = "My book4 in batch write"
};
var bookBatch = context.CreateBatchWrite <Book>();
bookBatch.AddPutItems(new List <Book> {
book1, book2
});
Console.WriteLine("Performing batch write in SingleTableBatchWrite().");
bookBatch.Execute();
}
public void AddBatch(List <Type> types)
{
var batchTypes = _context.CreateBatchWrite <Type>();
batchTypes.AddPutItems(types);
batchTypes.Execute();
}
public void AddBatch(List <Widget> widgets)
{
var batchTypes = _context.CreateBatchWrite <Widget>();
batchTypes.AddPutItems(widgets);
batchTypes.Execute();
}
private async Task <List <double> > ExecuteInsertWorkload(List <string> ids, ILambdaContext context)
{
List <double> insertTimeList = new List <double>();
//Prepare Data Model for Insert
var twitterModels = await QueryTwitterStreamData(ids, context).ConfigureAwait(false);
AmazonDynamoDBConfig ddbConfig = new AmazonDynamoDBConfig();
ddbConfig.ServiceURL = "http://34.246.18.10:8000";
AmazonDynamoDBClient amazonDynamoDbClient =
new AmazonDynamoDBClient(ddbConfig);
string tableName = "twitter-stream-data-insert";
foreach (var twitterStreamModel in twitterModels)
{
var sw = Stopwatch.StartNew();
IDynamoDBContext dynamoDbContext = new DynamoDBContext(amazonDynamoDbClient);
var dynamoDbBatch = dynamoDbContext.CreateBatchWrite <TwitterStreamModel>(new DynamoDBOperationConfig
{
OverrideTableName = tableName
});
dynamoDbBatch.AddPutItem(twitterStreamModel);
await dynamoDbBatch.ExecuteAsync().ConfigureAwait(false);
sw.Stop();
insertTimeList.Add(sw.Elapsed.TotalMilliseconds);
}
return(insertTimeList);
}
public override void LoadUnitTestData(DataContainer dataContainer)
{
Console.WriteLine($"Using connection string {dataContainer.ConnectionString}");
using (var client = new DynamoDBClientBuilder(dataContainer.ConnectionString).GetClient()) {
client.CreateTableAsync("CONTACTS_AUDIT",
new List <KeySchemaElement>()
{
new KeySchemaElement("ID", KeyType.HASH)
},
new List <AttributeDefinition>()
{
new AttributeDefinition("ID", ScalarAttributeType.S)
}, new ProvisionedThroughput(10, 10)
).Wait();
client.CreateTableAsync("EMAILS",
new List <KeySchemaElement>()
{
new KeySchemaElement("ID", KeyType.HASH)
},
new List <AttributeDefinition>()
{
new AttributeDefinition("ID", ScalarAttributeType.S)
}, new ProvisionedThroughput(10, 10)
).Wait();
client.CreateTableAsync("LEADS",
new List <KeySchemaElement>()
{
new KeySchemaElement("ID", KeyType.HASH)
},
new List <AttributeDefinition>()
{
new AttributeDefinition("ID", ScalarAttributeType.S)
}, new ProvisionedThroughput(10, 10)
).Wait();
client.CreateTableAsync("PEOPLE",
new List <KeySchemaElement>()
{
new KeySchemaElement("Id", KeyType.HASH)
},
new List <AttributeDefinition>()
{
new AttributeDefinition("Id", ScalarAttributeType.S)
}, new ProvisionedThroughput(10, 10)
).Wait();
var contextConfig = new DynamoDBContextConfig()
{
TableNamePrefix = ""
};
var context = new DynamoDBContext(client, contextConfig);
var people = GetPeople(200);
var contactsBatch = context.CreateBatchWrite <Person>();
contactsBatch.AddPutItems(people);
contactsBatch.ExecuteAsync().GetAwaiter().GetResult();
}
}
async Task FlushBatch(DynamoDBContext dynamoDbContext, IEnumerable <ZipCode> batch)
{
var batchWrite = dynamoDbContext.CreateBatchWrite <ZipCode>();
batchWrite.AddPutItems(batch);
await batchWrite.ExecuteAsync();
}
public async Task Write(List <Item> items)
{
if (items.Count < 1)
{
return;
}
var dynamoItems = ConvertToDynamo(items);
// Calculate the write rate
decimal sizeKb = _sizeCapacity * 1000;
var rowSize = (int)Math.Round(sizeKb / dynamoItems[0].ToString().Length) * 10;
var chunkSize = rowSize * _writeCapacity;
var index = 0;
var retryMax = 10;
var retryCount = 0;
var retryWait = 1000;
var chunk = dynamoItems.GetRange(index, chunkSize);
while (chunk.Count > 0)
{
try
{
var batchWrite = _context.CreateBatchWrite <DynamoItem>();
batchWrite.AddPutItems(chunk);
await batchWrite.ExecuteAsync();
System.Threading.Thread.Sleep(1000); // write per second
// Reset for next write
index += chunkSize;
chunk = dynamoItems.GetRange(index, chunkSize);
retryCount = 0;
retryWait = 1000;
}
catch (ProvisionedThroughputExceededException)
{
if (retryCount < retryMax)
{
retryCount++;
retryWait = retryWait * 2;
Console.WriteLine($" {DateTime.Now}: ERROR! Exceeded write capcity. At chunk: {index} Retry count at: {retryCount} Retrying in: {retryWait} ms");
System.Threading.Thread.Sleep(retryWait);
}
else
{
Console.WriteLine($" {DateTime.Now}: ERROR! Exceeded maximum retry. Aborting write process!");
return;
}
}
catch (ArgumentException)
{
// Load the last remaining chunk
chunkSize = dynamoItems.Count - index;
chunk = dynamoItems.GetRange(index, chunkSize);
}
}
}
public DBThrottler(DynamoDBContext dynamoContext, int maxBatchNo, int sleepTimeMs)
{
context = dynamoContext;
maxBatchWrite = maxBatchNo;
this.sleepTimeMs = sleepTimeMs;
items = new List <BatchWrite <T> >();
items.Add(context.CreateBatchWrite <T>());
}
public async Task BatchInserts(List <Profile> profiles)
{
BatchWrite <Profile> batchWrite = _context.CreateBatchWrite <Profile>();
batchWrite.AddPutItems(profiles);
await batchWrite.ExecuteAsync();
Console.WriteLine("Items inserted successfully");
}
public virtual void Delete(List <TEntity> listEntities)
{
using (var client = new AmazonDynamoDBClient(_fcaSecrets.AwsCredentials, _fcaSecrets.AwsRegion))
using (var context = new DynamoDBContext(client))
{
var entryBatch = context.CreateBatchWrite <TEntity>(_fcaSecrets.DefaultDbOperationConfig);
entryBatch.AddDeleteItems(listEntities);
entryBatch.ExecuteAsync();
}
}
public void SaveList <T>(IEnumerable <T> items)
{
var itemBatch = _dynamoDBContext.CreateBatchWrite <T>();
foreach (var item in items)
{
itemBatch.AddPutItem(item);
}
itemBatch.ExecuteAsync();
}
public async Task AddBulk(IEnumerable <CarDTO> cars)
{
// Adding the uid manually.. this sucks
cars.Select(c => { c.Id = Guid.NewGuid().ToString(); return(c); }).ToList();
// Start the batch
BatchWrite <Car> bulkBatch = dbContext.CreateBatchWrite <Car>();
bulkBatch.AddPutItems(mapper.Map <IEnumerable <Car> >(cars));
await bulkBatch.ExecuteAsync();
}
public bool Remove(List <TEntity> lstObj)
{
using DynamoDBContext context = GetContext();
var batch = context.CreateBatchWrite <TEntity>();
batch.AddDeleteItems(lstObj);
var task = batch.ExecuteAsync();
task.Wait();
return(task.IsCompletedSuccessfully);
}
/// <summary>
/// The BatchStore Method allows you to store a list of items of type T to dynamoDb
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="items"></param>
public void BatchStore <T>(IEnumerable <T> items) where T : class
{
var itemBatch = DbContext.CreateBatchWrite <T>();
foreach (var item in items)
{
itemBatch.AddPutItem(item);
}
itemBatch.Execute();
}
/// <summary>
/// The BatchStoreAsync Method will accept an IEnumerable of an Object that inherits from IPersistentItem to store multiple POCO objects in DynamoDB
/// </summary>
/// <param name="items"></param>
public virtual async Task BatchStoreAsync(IEnumerable <TItem> items)
{
var itemBatch = _dbContext.CreateBatchWrite <TItem>();
foreach (var item in items)
{
itemBatch.AddPutItem(item);
}
await itemBatch.ExecuteAsync().ConfigureAwait(false);
}
public void AddItem(T item)
{
items[currentItemSlot].AddPutItem(item);
itemsInBatch++;
if (itemsInBatch >= maxBatchWrite)
{
items.Add(context.CreateBatchWrite <T>());
itemsInBatch = 0;
currentItemSlot++;
}
}
public async Task AddItems <T>(IEnumerable <T> items)
{
IDynamoDBContext context = new DynamoDBContext(client, new DynamoDBContextConfig()
{
ConsistentRead = true
});
BatchWrite <T> batch = context.CreateBatchWrite <T>();
batch.AddPutItems(items);
await batch.ExecuteAsync();
}
public async Task Create(List <T> entityList, CancellationToken cancellationToken = default)
{
try
{
var batch = DynamoDBContext.CreateBatchWrite <T>();
batch.AddPutItems(entityList);
await batch.ExecuteAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
LogDynamoException(ref ex);
throw;
}
}
private static void MultiTableBatchWrite(DynamoDBContext context)
{
// 1. New Forum item.
Forum newForum = new Forum
{
Name = "Test BatchWrite Forum",
Threads = 0
};
var forumBatch = context.CreateBatchWrite <Forum>();
forumBatch.AddPutItem(newForum);
// 2. New Thread item.
Thread newThread = new Thread
{
ForumName = "S3 forum",
Subject = "My sample question",
KeywordTags = new List <string> {
"S3", "Bucket"
},
Message = "Message text"
};
DynamoDBOperationConfig config = new DynamoDBOperationConfig();
config.SkipVersionCheck = true;
var threadBatch = context.CreateBatchWrite <Thread>(config);
threadBatch.AddPutItem(newThread);
threadBatch.AddDeleteKey("some partition key value", "some sort key value");
var superBatch = new MultiTableBatchWrite(forumBatch, threadBatch);
Console.WriteLine("Performing batch write in MultiTableBatchWrite().");
superBatch.Execute();
}
public void LogMatches(IEnumerable<Match> matches)
{
using (var context = new DynamoDBContext(client))
{
var dynamoDbMatches = new List<DynamoDbMatch>();
foreach(var match in matches)
{
dynamoDbMatches.Add(ToDynamoDbMatch(match));
}
var batchWriter = context.CreateBatchWrite<DynamoDbMatch>();
batchWriter.AddPutItems(dynamoDbMatches);
batchWriter.Execute();
}
}
protected override async Task EmitBatchAsync(IEnumerable <LogEvent> events)
{
var records = events.Select(x =>
{
var document = new LogDocument(x, x.RenderMessage(_formatProvider));
var sw = new StringWriter();
_formatter.Format(x, sw);
document.Properties = sw.ToString();
return(document);
});
AmazonDynamoDBClient client = null;
try
{
if (_credentials != null)
{
if (_regionEndpoint != null)
{
client = new AmazonDynamoDBClient(_credentials, _regionEndpoint);
}
else
{
client = new AmazonDynamoDBClient(_credentials, AmazonDynamoDbConfig);
}
}
else
{
client = new AmazonDynamoDBClient(AmazonDynamoDbConfig);
}
using (var context = new DynamoDBContext(client))
{
var batchWrite = context.CreateBatchWrite <LogDocument>(OperationConfig);
batchWrite.AddPutItems(records);
await batchWrite.ExecuteAsync();
}
}
catch (Exception exception)
{
SelfLog.WriteLine("Unable to write events to DynamoDB Sink for {0}: {1}", _tableName, exception);
}
finally
{
if (client != null)
{
client.Dispose();
}
}
}
public BatchWrite GetBatchWrite(DynamoDBContext dbContext)
{
var entityBatch = dbContext.CreateBatchWrite <TEntity>();
if (_data_to_Delete.Any())
{
entityBatch.AddPutItems(_data_to_Delete);
}
if (_data_to_Put.Any())
{
entityBatch.AddPutItems(_data_to_Put);
}
return(entityBatch);
}
public BatchWrite <TEntity> GetBatchWrite(List <TEntity> entities, DynamoDbBatchOperator batchOperator)
{
using DynamoDBContext context = GetContext();
BatchWrite <TEntity> batch = context.CreateBatchWrite <TEntity>();
switch (batchOperator)
{
case DynamoDbBatchOperator.Delete:
batch.AddDeleteItems(entities);
break;
case DynamoDbBatchOperator.Put:
batch.AddPutItems(entities);
break;
}
return(batch);
}
/// <summary>
/// The BatchStore Method allows you to store a list of items of type T to dynamoDb
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="items"></param>
public void BatchStore <T>(IEnumerable <T> items) where T : class
{
try
{
var itemBatch = DbContext.CreateBatchWrite <T>();
foreach (var item in items)
{
itemBatch.AddPutItem(item);
}
itemBatch.Execute();
}
catch
{
throw new Exception("Require collection not exists. Please try after a while");
}
}
public async Task InsertNewItems(List <Profile> profiles)
{
if (profiles != null && profiles.Count > 0)
{
try
{
BatchWrite <Profile> batchWrite = _context.CreateBatchWrite <Profile>();
batchWrite.AddPutItems(profiles);
Console.WriteLine("Inserting list of profiles...");
await batchWrite.ExecuteAsync();
Console.WriteLine("Done");
}
catch (Exception ex)
{
Console.WriteLine("[!] Insert data failed");
Console.WriteLine(ex);
}
}
}
private async Task SaveObjectToDynamoDb(string jsonPayload)
{
var twitterObject = JsonConvert.DeserializeObject <TwitterStreamModel>(jsonPayload);
AmazonDynamoDBConfig ddbConfig = new AmazonDynamoDBConfig();
ddbConfig.ServiceURL = "http://34.246.18.10:8000";
AmazonDynamoDBClient amazonDynamoDbClient =
new AmazonDynamoDBClient(ddbConfig);
var sw = Stopwatch.StartNew();
IDynamoDBContext dynamoDbContext = new DynamoDBContext(amazonDynamoDbClient);
var dynamoDbBatch = dynamoDbContext.CreateBatchWrite <TwitterStreamModel>();
dynamoDbBatch.AddPutItem(twitterObject);
await dynamoDbBatch.ExecuteAsync().ConfigureAwait(false);
sw.Stop();
Console.WriteLine($"Insert Id: {twitterObject.id} take {sw.ElapsedMilliseconds} ms");
}
protected override async Task EmitBatchAsync(IEnumerable<LogEvent> events)
{
var records = events.Select(x => new LogDocument(x, x.RenderMessage(_formatProvider)));
try
{
using (var client = new AmazonDynamoDBClient(AmazonDynamoDbConfig))
{
using (var context = new DynamoDBContext(client))
{
var batchWrite = context.CreateBatchWrite<LogDocument>(OperationConfig);
batchWrite.AddPutItems(records);
await batchWrite.ExecuteAsync();
}
}
}
catch (Exception exception)
{
SelfLog.WriteLine("Unable to write events to DynamoDB Sink for {0}: {1}", _tableName, exception);
}
}
private static void TestCRUDOperations(DynamoDBContext context)
{
List<MonthlyTotal> Import = new List<MonthlyTotal>();
string oneTimeFile = @"C: \Users\wildbillcat\Downloads\unknown_20160206082017.csv";
int i = 0;
using (CsvReader csv = new CsvReader(System.IO.File.OpenText(oneTimeFile)))
{
CsvReader Header = new CsvReader(System.IO.File.OpenText(oneTimeFile));
Header.Read();
string parsedate = Header.FieldHeaders[2].Substring(21, 10);
Header.Dispose();
DateTime ImportDate = Convert.ToDateTime(parsedate);
while (csv.Read())
{
//MonthlyTotal SKUDocument = context.Load<MonthlyTotal>(SKUNumber, ImportDate, new DynamoDBOperationConfig { ConsistentRead = true });
//if(SKUDocument == null)
//{
// //SKU isnt in Datebase, will have to create it.
// SKUDocument = new MonthlyTotal()
// {
// SKUId = SKUNumber,
// Month = ImportDate,
// Total = csv.GetField<double>(2)
// };
//}
MonthlyTotal SKUDocument = new MonthlyTotal()
{
SKUId = csv.GetField<int>(0),
Month = ImportDate,
Total = csv.GetField<double>(2)
};
//context.Save(SKUDocument);
Import.Add(SKUDocument);
i++;
if(i > 1000)
{
i = 0;
Console.Write("SKU :");
Console.WriteLine(SKUDocument.SKUId);
}
}
}
var importBatch = context.CreateBatchWrite<MonthlyTotal>();
importBatch.AddPutItems(Import);
importBatch.Execute();
Console.WriteLine("Wrote to NoSQL DB");
Console.WriteLine();
}
