public Task BatchUpsertAsync(IEnumerable <TEntity> entities) { var batch = Context.CreateBatchWrite <TEntity>(); batch.AddPutItems(entities); return(batch.ExecuteAsync()); }
public async Task BatchStoreAsync <T>(IEnumerable <T> items) where T : class { var itemBatch = DbContext.CreateBatchWrite <T>(); itemBatch.AddPutItems(items); await itemBatch.ExecuteAsync(); }
/// <summary> /// Save for individual item /// </summary> /// <typeparam name="T"></typeparam> /// <param name="t"></param> public static void Save <T>(T t) { var batch = _context.CreateBatchWrite <T>(); batch.AddPutItem(t); _context.ExecuteBatchWriteAsync(new BatchWrite[] { batch }); //batch.ExecuteAsync(); }
public void SaveAreas(List <Area> areas) { Parallel.ForEach(areas.ChunkBy <Area>(200), (item, state, index) => { Console.WriteLine("Initiating a new chunk. Index: " + index); var bulkInsert = context.CreateBatchWrite <Area>(); bulkInsert.AddPutItems(item); bulkInsert.ExecuteAsync().Wait(); Console.WriteLine("Chunk inserted successfully. Index" + index); }); }
static async Task Main() { var region = Amazon.RegionEndpoint.APNortheast1; // 使用するリージョン var client = new AmazonDynamoDBClient(region); // 使用するDynamoDBテーブルを作成する await CreateTableIfNotExistsAsync(client, TABLE_NAME); using (var context = new DynamoDBContext(client)) { // // データの追加 (PutItem処理) // var item = new SampleTableData { PartitionKey = 0, SortKey = "SK0", Property1 = DateTime.Now }; await context.SaveAsync(item); // // 複数データの追加 (BatchWriteItem処理) // var batchWrite = context.CreateBatchWrite <SampleTableData>(); var items = Enumerable.Range(1, 9).Select(i => new SampleTableData { PartitionKey = 0, SortKey = "SK" + i, Property1 = DateTime.Now }); batchWrite.AddPutItems(items); await batchWrite.ExecuteAsync(); // // データの取得 (Query処理) // const int scanPartitionKey = 0; var queriedItems = await context.QueryAsync <SampleTableData>(scanPartitionKey).GetRemainingAsync(); // // データの削除 (DeleteItem処理) // await context.DeleteAsync(queriedItems.First()); // // 複数データの削除 (DeleteItem処理) // var batchDelete = context.CreateBatchWrite <SampleTableData>(); batchDelete.AddDeleteItems(queriedItems.Skip(1).Take(4)); await batchDelete.ExecuteAsync(); } }
private static void SingleTableBatchWrite(DynamoDBContext context) { Book book1 = new Book { Id = 902, InPublication = true, ISBN = "902-11-11-1111", PageCount = "100", Price = 10, ProductCategory = "Book", Title = "My book3 in batch write" }; Book book2 = new Book { Id = 903, InPublication = true, ISBN = "903-11-11-1111", PageCount = "200", Price = 10, ProductCategory = "Book", Title = "My book4 in batch write" }; var bookBatch = context.CreateBatchWrite <Book>(); bookBatch.AddPutItems(new List <Book> { book1, book2 }); Console.WriteLine("Performing batch write in SingleTableBatchWrite()."); bookBatch.Execute(); }
public void AddBatch(List <Type> types) { var batchTypes = _context.CreateBatchWrite <Type>(); batchTypes.AddPutItems(types); batchTypes.Execute(); }
public void AddBatch(List <Widget> widgets) { var batchTypes = _context.CreateBatchWrite <Widget>(); batchTypes.AddPutItems(widgets); batchTypes.Execute(); }
private async Task <List <double> > ExecuteInsertWorkload(List <string> ids, ILambdaContext context) { List <double> insertTimeList = new List <double>(); //Prepare Data Model for Insert var twitterModels = await QueryTwitterStreamData(ids, context).ConfigureAwait(false); AmazonDynamoDBConfig ddbConfig = new AmazonDynamoDBConfig(); ddbConfig.ServiceURL = "http://34.246.18.10:8000"; AmazonDynamoDBClient amazonDynamoDbClient = new AmazonDynamoDBClient(ddbConfig); string tableName = "twitter-stream-data-insert"; foreach (var twitterStreamModel in twitterModels) { var sw = Stopwatch.StartNew(); IDynamoDBContext dynamoDbContext = new DynamoDBContext(amazonDynamoDbClient); var dynamoDbBatch = dynamoDbContext.CreateBatchWrite <TwitterStreamModel>(new DynamoDBOperationConfig { OverrideTableName = tableName }); dynamoDbBatch.AddPutItem(twitterStreamModel); await dynamoDbBatch.ExecuteAsync().ConfigureAwait(false); sw.Stop(); insertTimeList.Add(sw.Elapsed.TotalMilliseconds); } return(insertTimeList); }
public override void LoadUnitTestData(DataContainer dataContainer) { Console.WriteLine($"Using connection string {dataContainer.ConnectionString}"); using (var client = new DynamoDBClientBuilder(dataContainer.ConnectionString).GetClient()) { client.CreateTableAsync("CONTACTS_AUDIT", new List <KeySchemaElement>() { new KeySchemaElement("ID", KeyType.HASH) }, new List <AttributeDefinition>() { new AttributeDefinition("ID", ScalarAttributeType.S) }, new ProvisionedThroughput(10, 10) ).Wait(); client.CreateTableAsync("EMAILS", new List <KeySchemaElement>() { new KeySchemaElement("ID", KeyType.HASH) }, new List <AttributeDefinition>() { new AttributeDefinition("ID", ScalarAttributeType.S) }, new ProvisionedThroughput(10, 10) ).Wait(); client.CreateTableAsync("LEADS", new List <KeySchemaElement>() { new KeySchemaElement("ID", KeyType.HASH) }, new List <AttributeDefinition>() { new AttributeDefinition("ID", ScalarAttributeType.S) }, new ProvisionedThroughput(10, 10) ).Wait(); client.CreateTableAsync("PEOPLE", new List <KeySchemaElement>() { new KeySchemaElement("Id", KeyType.HASH) }, new List <AttributeDefinition>() { new AttributeDefinition("Id", ScalarAttributeType.S) }, new ProvisionedThroughput(10, 10) ).Wait(); var contextConfig = new DynamoDBContextConfig() { TableNamePrefix = "" }; var context = new DynamoDBContext(client, contextConfig); var people = GetPeople(200); var contactsBatch = context.CreateBatchWrite <Person>(); contactsBatch.AddPutItems(people); contactsBatch.ExecuteAsync().GetAwaiter().GetResult(); } }
async Task FlushBatch(DynamoDBContext dynamoDbContext, IEnumerable <ZipCode> batch) { var batchWrite = dynamoDbContext.CreateBatchWrite <ZipCode>(); batchWrite.AddPutItems(batch); await batchWrite.ExecuteAsync(); }
public async Task Write(List <Item> items) { if (items.Count < 1) { return; } var dynamoItems = ConvertToDynamo(items); // Calculate the write rate decimal sizeKb = _sizeCapacity * 1000; var rowSize = (int)Math.Round(sizeKb / dynamoItems[0].ToString().Length) * 10; var chunkSize = rowSize * _writeCapacity; var index = 0; var retryMax = 10; var retryCount = 0; var retryWait = 1000; var chunk = dynamoItems.GetRange(index, chunkSize); while (chunk.Count > 0) { try { var batchWrite = _context.CreateBatchWrite <DynamoItem>(); batchWrite.AddPutItems(chunk); await batchWrite.ExecuteAsync(); System.Threading.Thread.Sleep(1000); // write per second // Reset for next write index += chunkSize; chunk = dynamoItems.GetRange(index, chunkSize); retryCount = 0; retryWait = 1000; } catch (ProvisionedThroughputExceededException) { if (retryCount < retryMax) { retryCount++; retryWait = retryWait * 2; Console.WriteLine($" {DateTime.Now}: ERROR! Exceeded write capcity. At chunk: {index} Retry count at: {retryCount} Retrying in: {retryWait} ms"); System.Threading.Thread.Sleep(retryWait); } else { Console.WriteLine($" {DateTime.Now}: ERROR! Exceeded maximum retry. Aborting write process!"); return; } } catch (ArgumentException) { // Load the last remaining chunk chunkSize = dynamoItems.Count - index; chunk = dynamoItems.GetRange(index, chunkSize); } } }
public DBThrottler(DynamoDBContext dynamoContext, int maxBatchNo, int sleepTimeMs) { context = dynamoContext; maxBatchWrite = maxBatchNo; this.sleepTimeMs = sleepTimeMs; items = new List <BatchWrite <T> >(); items.Add(context.CreateBatchWrite <T>()); }
public async Task BatchInserts(List <Profile> profiles) { BatchWrite <Profile> batchWrite = _context.CreateBatchWrite <Profile>(); batchWrite.AddPutItems(profiles); await batchWrite.ExecuteAsync(); Console.WriteLine("Items inserted successfully"); }
public virtual void Delete(List <TEntity> listEntities) { using (var client = new AmazonDynamoDBClient(_fcaSecrets.AwsCredentials, _fcaSecrets.AwsRegion)) using (var context = new DynamoDBContext(client)) { var entryBatch = context.CreateBatchWrite <TEntity>(_fcaSecrets.DefaultDbOperationConfig); entryBatch.AddDeleteItems(listEntities); entryBatch.ExecuteAsync(); } }
public void SaveList <T>(IEnumerable <T> items) { var itemBatch = _dynamoDBContext.CreateBatchWrite <T>(); foreach (var item in items) { itemBatch.AddPutItem(item); } itemBatch.ExecuteAsync(); }
public async Task AddBulk(IEnumerable <CarDTO> cars) { // Adding the uid manually.. this sucks cars.Select(c => { c.Id = Guid.NewGuid().ToString(); return(c); }).ToList(); // Start the batch BatchWrite <Car> bulkBatch = dbContext.CreateBatchWrite <Car>(); bulkBatch.AddPutItems(mapper.Map <IEnumerable <Car> >(cars)); await bulkBatch.ExecuteAsync(); }
public bool Remove(List <TEntity> lstObj) { using DynamoDBContext context = GetContext(); var batch = context.CreateBatchWrite <TEntity>(); batch.AddDeleteItems(lstObj); var task = batch.ExecuteAsync(); task.Wait(); return(task.IsCompletedSuccessfully); }
/// <summary> /// The BatchStore Method allows you to store a list of items of type T to dynamoDb /// </summary> /// <typeparam name="T"></typeparam> /// <param name="items"></param> public void BatchStore <T>(IEnumerable <T> items) where T : class { var itemBatch = DbContext.CreateBatchWrite <T>(); foreach (var item in items) { itemBatch.AddPutItem(item); } itemBatch.Execute(); }
/// <summary> /// The BatchStoreAsync Method will accept an IEnumerable of an Object that inherits from IPersistentItem to store multiple POCO objects in DynamoDB /// </summary> /// <param name="items"></param> public virtual async Task BatchStoreAsync(IEnumerable <TItem> items) { var itemBatch = _dbContext.CreateBatchWrite <TItem>(); foreach (var item in items) { itemBatch.AddPutItem(item); } await itemBatch.ExecuteAsync().ConfigureAwait(false); }
public void AddItem(T item) { items[currentItemSlot].AddPutItem(item); itemsInBatch++; if (itemsInBatch >= maxBatchWrite) { items.Add(context.CreateBatchWrite <T>()); itemsInBatch = 0; currentItemSlot++; } }
public async Task AddItems <T>(IEnumerable <T> items) { IDynamoDBContext context = new DynamoDBContext(client, new DynamoDBContextConfig() { ConsistentRead = true }); BatchWrite <T> batch = context.CreateBatchWrite <T>(); batch.AddPutItems(items); await batch.ExecuteAsync(); }
public async Task Create(List <T> entityList, CancellationToken cancellationToken = default) { try { var batch = DynamoDBContext.CreateBatchWrite <T>(); batch.AddPutItems(entityList); await batch.ExecuteAsync(cancellationToken).ConfigureAwait(false); } catch (Exception ex) { LogDynamoException(ref ex); throw; } }
private static void MultiTableBatchWrite(DynamoDBContext context) { // 1. New Forum item. Forum newForum = new Forum { Name = "Test BatchWrite Forum", Threads = 0 }; var forumBatch = context.CreateBatchWrite <Forum>(); forumBatch.AddPutItem(newForum); // 2. New Thread item. Thread newThread = new Thread { ForumName = "S3 forum", Subject = "My sample question", KeywordTags = new List <string> { "S3", "Bucket" }, Message = "Message text" }; DynamoDBOperationConfig config = new DynamoDBOperationConfig(); config.SkipVersionCheck = true; var threadBatch = context.CreateBatchWrite <Thread>(config); threadBatch.AddPutItem(newThread); threadBatch.AddDeleteKey("some partition key value", "some sort key value"); var superBatch = new MultiTableBatchWrite(forumBatch, threadBatch); Console.WriteLine("Performing batch write in MultiTableBatchWrite()."); superBatch.Execute(); }
public void LogMatches(IEnumerable<Match> matches) { using (var context = new DynamoDBContext(client)) { var dynamoDbMatches = new List<DynamoDbMatch>(); foreach(var match in matches) { dynamoDbMatches.Add(ToDynamoDbMatch(match)); } var batchWriter = context.CreateBatchWrite<DynamoDbMatch>(); batchWriter.AddPutItems(dynamoDbMatches); batchWriter.Execute(); } }
protected override async Task EmitBatchAsync(IEnumerable <LogEvent> events) { var records = events.Select(x => { var document = new LogDocument(x, x.RenderMessage(_formatProvider)); var sw = new StringWriter(); _formatter.Format(x, sw); document.Properties = sw.ToString(); return(document); }); AmazonDynamoDBClient client = null; try { if (_credentials != null) { if (_regionEndpoint != null) { client = new AmazonDynamoDBClient(_credentials, _regionEndpoint); } else { client = new AmazonDynamoDBClient(_credentials, AmazonDynamoDbConfig); } } else { client = new AmazonDynamoDBClient(AmazonDynamoDbConfig); } using (var context = new DynamoDBContext(client)) { var batchWrite = context.CreateBatchWrite <LogDocument>(OperationConfig); batchWrite.AddPutItems(records); await batchWrite.ExecuteAsync(); } } catch (Exception exception) { SelfLog.WriteLine("Unable to write events to DynamoDB Sink for {0}: {1}", _tableName, exception); } finally { if (client != null) { client.Dispose(); } } }
public BatchWrite GetBatchWrite(DynamoDBContext dbContext) { var entityBatch = dbContext.CreateBatchWrite <TEntity>(); if (_data_to_Delete.Any()) { entityBatch.AddPutItems(_data_to_Delete); } if (_data_to_Put.Any()) { entityBatch.AddPutItems(_data_to_Put); } return(entityBatch); }
public BatchWrite <TEntity> GetBatchWrite(List <TEntity> entities, DynamoDbBatchOperator batchOperator) { using DynamoDBContext context = GetContext(); BatchWrite <TEntity> batch = context.CreateBatchWrite <TEntity>(); switch (batchOperator) { case DynamoDbBatchOperator.Delete: batch.AddDeleteItems(entities); break; case DynamoDbBatchOperator.Put: batch.AddPutItems(entities); break; } return(batch); }
/// <summary> /// The BatchStore Method allows you to store a list of items of type T to dynamoDb /// </summary> /// <typeparam name="T"></typeparam> /// <param name="items"></param> public void BatchStore <T>(IEnumerable <T> items) where T : class { try { var itemBatch = DbContext.CreateBatchWrite <T>(); foreach (var item in items) { itemBatch.AddPutItem(item); } itemBatch.Execute(); } catch { throw new Exception("Require collection not exists. Please try after a while"); } }
public async Task InsertNewItems(List <Profile> profiles) { if (profiles != null && profiles.Count > 0) { try { BatchWrite <Profile> batchWrite = _context.CreateBatchWrite <Profile>(); batchWrite.AddPutItems(profiles); Console.WriteLine("Inserting list of profiles..."); await batchWrite.ExecuteAsync(); Console.WriteLine("Done"); } catch (Exception ex) { Console.WriteLine("[!] Insert data failed"); Console.WriteLine(ex); } } }
private async Task SaveObjectToDynamoDb(string jsonPayload) { var twitterObject = JsonConvert.DeserializeObject <TwitterStreamModel>(jsonPayload); AmazonDynamoDBConfig ddbConfig = new AmazonDynamoDBConfig(); ddbConfig.ServiceURL = "http://34.246.18.10:8000"; AmazonDynamoDBClient amazonDynamoDbClient = new AmazonDynamoDBClient(ddbConfig); var sw = Stopwatch.StartNew(); IDynamoDBContext dynamoDbContext = new DynamoDBContext(amazonDynamoDbClient); var dynamoDbBatch = dynamoDbContext.CreateBatchWrite <TwitterStreamModel>(); dynamoDbBatch.AddPutItem(twitterObject); await dynamoDbBatch.ExecuteAsync().ConfigureAwait(false); sw.Stop(); Console.WriteLine($"Insert Id: {twitterObject.id} take {sw.ElapsedMilliseconds} ms"); }
protected override async Task EmitBatchAsync(IEnumerable<LogEvent> events) { var records = events.Select(x => new LogDocument(x, x.RenderMessage(_formatProvider))); try { using (var client = new AmazonDynamoDBClient(AmazonDynamoDbConfig)) { using (var context = new DynamoDBContext(client)) { var batchWrite = context.CreateBatchWrite<LogDocument>(OperationConfig); batchWrite.AddPutItems(records); await batchWrite.ExecuteAsync(); } } } catch (Exception exception) { SelfLog.WriteLine("Unable to write events to DynamoDB Sink for {0}: {1}", _tableName, exception); } }
private static void TestCRUDOperations(DynamoDBContext context) { List<MonthlyTotal> Import = new List<MonthlyTotal>(); string oneTimeFile = @"C: \Users\wildbillcat\Downloads\unknown_20160206082017.csv"; int i = 0; using (CsvReader csv = new CsvReader(System.IO.File.OpenText(oneTimeFile))) { CsvReader Header = new CsvReader(System.IO.File.OpenText(oneTimeFile)); Header.Read(); string parsedate = Header.FieldHeaders[2].Substring(21, 10); Header.Dispose(); DateTime ImportDate = Convert.ToDateTime(parsedate); while (csv.Read()) { //MonthlyTotal SKUDocument = context.Load<MonthlyTotal>(SKUNumber, ImportDate, new DynamoDBOperationConfig { ConsistentRead = true }); //if(SKUDocument == null) //{ // //SKU isnt in Datebase, will have to create it. // SKUDocument = new MonthlyTotal() // { // SKUId = SKUNumber, // Month = ImportDate, // Total = csv.GetField<double>(2) // }; //} MonthlyTotal SKUDocument = new MonthlyTotal() { SKUId = csv.GetField<int>(0), Month = ImportDate, Total = csv.GetField<double>(2) }; //context.Save(SKUDocument); Import.Add(SKUDocument); i++; if(i > 1000) { i = 0; Console.Write("SKU :"); Console.WriteLine(SKUDocument.SKUId); } } } var importBatch = context.CreateBatchWrite<MonthlyTotal>(); importBatch.AddPutItems(Import); importBatch.Execute(); Console.WriteLine("Wrote to NoSQL DB"); Console.WriteLine(); }