コード例 #1
0
ファイル: Repository.cs プロジェクト: afgonzal/futbol
        public Task BatchUpsertAsync(IEnumerable <TEntity> entities)
        {
            var batch = Context.CreateBatchWrite <TEntity>();

            batch.AddPutItems(entities);
            return(batch.ExecuteAsync());
        }
コード例 #2
0
        public async Task BatchStoreAsync <T>(IEnumerable <T> items) where T : class
        {
            var itemBatch = DbContext.CreateBatchWrite <T>();

            itemBatch.AddPutItems(items);

            await itemBatch.ExecuteAsync();
        }
コード例 #3
0
        /// <summary>
        /// Save for individual item
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="t"></param>
        public static void Save <T>(T t)
        {
            var batch = _context.CreateBatchWrite <T>();

            batch.AddPutItem(t);

            _context.ExecuteBatchWriteAsync(new BatchWrite[] { batch });
            //batch.ExecuteAsync();
        }
コード例 #4
0
ファイル: RegionService.cs プロジェクト: 10pearls/tilebase
 public void SaveAreas(List <Area> areas)
 {
     Parallel.ForEach(areas.ChunkBy <Area>(200), (item, state, index) =>
     {
         Console.WriteLine("Initiating a new chunk. Index: " + index);
         var bulkInsert = context.CreateBatchWrite <Area>();
         bulkInsert.AddPutItems(item);
         bulkInsert.ExecuteAsync().Wait();
         Console.WriteLine("Chunk inserted successfully. Index" + index);
     });
 }
コード例 #5
0
ファイル: AmazonDynamoDB.cs プロジェクト: mahiya/sample-codes
        static async Task Main()
        {
            var region = Amazon.RegionEndpoint.APNortheast1; // 使用するリージョン
            var client = new AmazonDynamoDBClient(region);

            // 使用するDynamoDBテーブルを作成する
            await CreateTableIfNotExistsAsync(client, TABLE_NAME);

            using (var context = new DynamoDBContext(client))
            {
                //
                // データの追加 (PutItem処理)
                //
                var item = new SampleTableData
                {
                    PartitionKey = 0,
                    SortKey      = "SK0",
                    Property1    = DateTime.Now
                };
                await context.SaveAsync(item);

                //
                // 複数データの追加 (BatchWriteItem処理)
                //
                var batchWrite = context.CreateBatchWrite <SampleTableData>();
                var items      = Enumerable.Range(1, 9).Select(i => new SampleTableData
                {
                    PartitionKey = 0,
                    SortKey      = "SK" + i,
                    Property1    = DateTime.Now
                });
                batchWrite.AddPutItems(items);
                await batchWrite.ExecuteAsync();

                //
                // データの取得 (Query処理)
                //
                const int scanPartitionKey = 0;
                var       queriedItems     = await context.QueryAsync <SampleTableData>(scanPartitionKey).GetRemainingAsync();

                //
                // データの削除 (DeleteItem処理)
                //
                await context.DeleteAsync(queriedItems.First());

                //
                // 複数データの削除 (DeleteItem処理)
                //
                var batchDelete = context.CreateBatchWrite <SampleTableData>();
                batchDelete.AddDeleteItems(queriedItems.Skip(1).Take(4));
                await batchDelete.ExecuteAsync();
            }
        }
コード例 #6
0
        private static void SingleTableBatchWrite(DynamoDBContext context)
        {
            Book book1 = new Book
            {
                Id              = 902,
                InPublication   = true,
                ISBN            = "902-11-11-1111",
                PageCount       = "100",
                Price           = 10,
                ProductCategory = "Book",
                Title           = "My book3 in batch write"
            };
            Book book2 = new Book
            {
                Id              = 903,
                InPublication   = true,
                ISBN            = "903-11-11-1111",
                PageCount       = "200",
                Price           = 10,
                ProductCategory = "Book",
                Title           = "My book4 in batch write"
            };

            var bookBatch = context.CreateBatchWrite <Book>();

            bookBatch.AddPutItems(new List <Book> {
                book1, book2
            });

            Console.WriteLine("Performing batch write in SingleTableBatchWrite().");
            bookBatch.Execute();
        }
コード例 #7
0
        public void AddBatch(List <Type> types)
        {
            var batchTypes = _context.CreateBatchWrite <Type>();

            batchTypes.AddPutItems(types);
            batchTypes.Execute();
        }
コード例 #8
0
        public void AddBatch(List <Widget> widgets)
        {
            var batchTypes = _context.CreateBatchWrite <Widget>();

            batchTypes.AddPutItems(widgets);
            batchTypes.Execute();
        }
コード例 #9
0
        private async Task <List <double> > ExecuteInsertWorkload(List <string> ids, ILambdaContext context)
        {
            List <double> insertTimeList = new List <double>();

            //Prepare Data Model for Insert
            var twitterModels = await QueryTwitterStreamData(ids, context).ConfigureAwait(false);

            AmazonDynamoDBConfig ddbConfig = new AmazonDynamoDBConfig();

            ddbConfig.ServiceURL = "http://34.246.18.10:8000";

            AmazonDynamoDBClient amazonDynamoDbClient =
                new AmazonDynamoDBClient(ddbConfig);


            string tableName = "twitter-stream-data-insert";

            foreach (var twitterStreamModel in twitterModels)
            {
                var sw = Stopwatch.StartNew();
                IDynamoDBContext dynamoDbContext = new DynamoDBContext(amazonDynamoDbClient);
                var dynamoDbBatch = dynamoDbContext.CreateBatchWrite <TwitterStreamModel>(new DynamoDBOperationConfig
                {
                    OverrideTableName = tableName
                });
                dynamoDbBatch.AddPutItem(twitterStreamModel);

                await dynamoDbBatch.ExecuteAsync().ConfigureAwait(false);

                sw.Stop();
                insertTimeList.Add(sw.Elapsed.TotalMilliseconds);
            }

            return(insertTimeList);
        }
コード例 #10
0
        public override void LoadUnitTestData(DataContainer dataContainer)
        {
            Console.WriteLine($"Using connection string {dataContainer.ConnectionString}");
            using (var client = new DynamoDBClientBuilder(dataContainer.ConnectionString).GetClient()) {
                client.CreateTableAsync("CONTACTS_AUDIT",
                                        new List <KeySchemaElement>()
                {
                    new KeySchemaElement("ID", KeyType.HASH)
                },
                                        new List <AttributeDefinition>()
                {
                    new AttributeDefinition("ID", ScalarAttributeType.S)
                }, new ProvisionedThroughput(10, 10)
                                        ).Wait();
                client.CreateTableAsync("EMAILS",
                                        new List <KeySchemaElement>()
                {
                    new KeySchemaElement("ID", KeyType.HASH)
                },
                                        new List <AttributeDefinition>()
                {
                    new AttributeDefinition("ID", ScalarAttributeType.S)
                }, new ProvisionedThroughput(10, 10)
                                        ).Wait();
                client.CreateTableAsync("LEADS",
                                        new List <KeySchemaElement>()
                {
                    new KeySchemaElement("ID", KeyType.HASH)
                },
                                        new List <AttributeDefinition>()
                {
                    new AttributeDefinition("ID", ScalarAttributeType.S)
                }, new ProvisionedThroughput(10, 10)
                                        ).Wait();
                client.CreateTableAsync("PEOPLE",
                                        new List <KeySchemaElement>()
                {
                    new KeySchemaElement("Id", KeyType.HASH)
                },
                                        new List <AttributeDefinition>()
                {
                    new AttributeDefinition("Id", ScalarAttributeType.S)
                }, new ProvisionedThroughput(10, 10)
                                        ).Wait();


                var contextConfig = new DynamoDBContextConfig()
                {
                    TableNamePrefix = ""
                };
                var context = new DynamoDBContext(client, contextConfig);

                var people = GetPeople(200);

                var contactsBatch = context.CreateBatchWrite <Person>();
                contactsBatch.AddPutItems(people);
                contactsBatch.ExecuteAsync().GetAwaiter().GetResult();
            }
        }
コード例 #11
0
        async Task FlushBatch(DynamoDBContext dynamoDbContext, IEnumerable <ZipCode> batch)
        {
            var batchWrite = dynamoDbContext.CreateBatchWrite <ZipCode>();

            batchWrite.AddPutItems(batch);

            await batchWrite.ExecuteAsync();
        }
コード例 #12
0
ファイル: DynamoDb.cs プロジェクト: johnlee/AwsDbBenchmark
        public async Task Write(List <Item> items)
        {
            if (items.Count < 1)
            {
                return;
            }
            var dynamoItems = ConvertToDynamo(items);

            // Calculate the write rate
            decimal sizeKb    = _sizeCapacity * 1000;
            var     rowSize   = (int)Math.Round(sizeKb / dynamoItems[0].ToString().Length) * 10;
            var     chunkSize = rowSize * _writeCapacity;

            var index      = 0;
            var retryMax   = 10;
            var retryCount = 0;
            var retryWait  = 1000;
            var chunk      = dynamoItems.GetRange(index, chunkSize);

            while (chunk.Count > 0)
            {
                try
                {
                    var batchWrite = _context.CreateBatchWrite <DynamoItem>();
                    batchWrite.AddPutItems(chunk);
                    await batchWrite.ExecuteAsync();

                    System.Threading.Thread.Sleep(1000); // write per second

                    // Reset for next write
                    index     += chunkSize;
                    chunk      = dynamoItems.GetRange(index, chunkSize);
                    retryCount = 0;
                    retryWait  = 1000;
                }
                catch (ProvisionedThroughputExceededException)
                {
                    if (retryCount < retryMax)
                    {
                        retryCount++;
                        retryWait = retryWait * 2;
                        Console.WriteLine($"  {DateTime.Now}: ERROR! Exceeded write capcity. At chunk: {index} Retry count at: {retryCount} Retrying in: {retryWait} ms");
                        System.Threading.Thread.Sleep(retryWait);
                    }
                    else
                    {
                        Console.WriteLine($"  {DateTime.Now}: ERROR! Exceeded maximum retry. Aborting write process!");
                        return;
                    }
                }
                catch (ArgumentException)
                {
                    // Load the last remaining chunk
                    chunkSize = dynamoItems.Count - index;
                    chunk     = dynamoItems.GetRange(index, chunkSize);
                }
            }
        }
コード例 #13
0
        public DBThrottler(DynamoDBContext dynamoContext, int maxBatchNo, int sleepTimeMs)
        {
            context          = dynamoContext;
            maxBatchWrite    = maxBatchNo;
            this.sleepTimeMs = sleepTimeMs;

            items = new List <BatchWrite <T> >();
            items.Add(context.CreateBatchWrite <T>());
        }
コード例 #14
0
        public async Task BatchInserts(List <Profile> profiles)
        {
            BatchWrite <Profile> batchWrite = _context.CreateBatchWrite <Profile>();

            batchWrite.AddPutItems(profiles);
            await batchWrite.ExecuteAsync();

            Console.WriteLine("Items inserted successfully");
        }
コード例 #15
0
 public virtual void Delete(List <TEntity> listEntities)
 {
     using (var client = new AmazonDynamoDBClient(_fcaSecrets.AwsCredentials, _fcaSecrets.AwsRegion))
         using (var context = new DynamoDBContext(client))
         {
             var entryBatch = context.CreateBatchWrite <TEntity>(_fcaSecrets.DefaultDbOperationConfig);
             entryBatch.AddDeleteItems(listEntities);
             entryBatch.ExecuteAsync();
         }
 }
コード例 #16
0
        public void SaveList <T>(IEnumerable <T> items)
        {
            var itemBatch = _dynamoDBContext.CreateBatchWrite <T>();

            foreach (var item in items)
            {
                itemBatch.AddPutItem(item);
            }
            itemBatch.ExecuteAsync();
        }
コード例 #17
0
        public async Task AddBulk(IEnumerable <CarDTO> cars)
        {
            // Adding the uid manually.. this sucks
            cars.Select(c => { c.Id = Guid.NewGuid().ToString(); return(c); }).ToList();

            // Start the batch
            BatchWrite <Car> bulkBatch = dbContext.CreateBatchWrite <Car>();

            bulkBatch.AddPutItems(mapper.Map <IEnumerable <Car> >(cars));
            await bulkBatch.ExecuteAsync();
        }
コード例 #18
0
        public bool Remove(List <TEntity> lstObj)
        {
            using DynamoDBContext context = GetContext();
            var batch = context.CreateBatchWrite <TEntity>();

            batch.AddDeleteItems(lstObj);
            var task = batch.ExecuteAsync();

            task.Wait();
            return(task.IsCompletedSuccessfully);
        }
コード例 #19
0
        /// <summary>
        /// The BatchStore Method allows you to store a list of items of type T to dynamoDb
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="items"></param>
        public void BatchStore <T>(IEnumerable <T> items) where T : class
        {
            var itemBatch = DbContext.CreateBatchWrite <T>();

            foreach (var item in items)
            {
                itemBatch.AddPutItem(item);
            }

            itemBatch.Execute();
        }
コード例 #20
0
        /// <summary>
        /// The BatchStoreAsync Method will accept an IEnumerable of an Object that inherits from IPersistentItem to store multiple POCO objects in DynamoDB
        /// </summary>
        /// <param name="items"></param>
        public virtual async Task BatchStoreAsync(IEnumerable <TItem> items)
        {
            var itemBatch = _dbContext.CreateBatchWrite <TItem>();

            foreach (var item in items)
            {
                itemBatch.AddPutItem(item);
            }

            await itemBatch.ExecuteAsync().ConfigureAwait(false);
        }
コード例 #21
0
        public void AddItem(T item)
        {
            items[currentItemSlot].AddPutItem(item);
            itemsInBatch++;

            if (itemsInBatch >= maxBatchWrite)
            {
                items.Add(context.CreateBatchWrite <T>());
                itemsInBatch = 0;
                currentItemSlot++;
            }
        }
コード例 #22
0
        public async Task AddItems <T>(IEnumerable <T> items)
        {
            IDynamoDBContext context = new DynamoDBContext(client, new DynamoDBContextConfig()
            {
                ConsistentRead = true
            });
            BatchWrite <T> batch = context.CreateBatchWrite <T>();

            batch.AddPutItems(items);

            await batch.ExecuteAsync();
        }
コード例 #23
0
 public async Task Create(List <T> entityList, CancellationToken cancellationToken = default)
 {
     try
     {
         var batch = DynamoDBContext.CreateBatchWrite <T>();
         batch.AddPutItems(entityList);
         await batch.ExecuteAsync(cancellationToken).ConfigureAwait(false);
     }
     catch (Exception ex)
     {
         LogDynamoException(ref ex);
         throw;
     }
 }
コード例 #24
0
        private static void MultiTableBatchWrite(DynamoDBContext context)
        {
            // 1. New Forum item.
            Forum newForum = new Forum
            {
                Name    = "Test BatchWrite Forum",
                Threads = 0
            };
            var forumBatch = context.CreateBatchWrite <Forum>();

            forumBatch.AddPutItem(newForum);

            // 2. New Thread item.
            Thread newThread = new Thread
            {
                ForumName   = "S3 forum",
                Subject     = "My sample question",
                KeywordTags = new List <string> {
                    "S3", "Bucket"
                },
                Message = "Message text"
            };

            DynamoDBOperationConfig config = new DynamoDBOperationConfig();

            config.SkipVersionCheck = true;
            var threadBatch = context.CreateBatchWrite <Thread>(config);

            threadBatch.AddPutItem(newThread);
            threadBatch.AddDeleteKey("some partition key value", "some sort key value");

            var superBatch = new MultiTableBatchWrite(forumBatch, threadBatch);

            Console.WriteLine("Performing batch write in MultiTableBatchWrite().");
            superBatch.Execute();
        }
コード例 #25
0
        public void LogMatches(IEnumerable<Match> matches)
        {
            using (var context = new DynamoDBContext(client))
            {
                var dynamoDbMatches = new List<DynamoDbMatch>();
                foreach(var match in matches)
                {
                    dynamoDbMatches.Add(ToDynamoDbMatch(match));
                }

                var batchWriter = context.CreateBatchWrite<DynamoDbMatch>();
                batchWriter.AddPutItems(dynamoDbMatches);
                batchWriter.Execute();
            }
        }
コード例 #26
0
        protected override async Task EmitBatchAsync(IEnumerable <LogEvent> events)
        {
            var records = events.Select(x =>
            {
                var document = new LogDocument(x, x.RenderMessage(_formatProvider));
                var sw       = new StringWriter();
                _formatter.Format(x, sw);
                document.Properties = sw.ToString();
                return(document);
            });
            AmazonDynamoDBClient client = null;

            try
            {
                if (_credentials != null)
                {
                    if (_regionEndpoint != null)
                    {
                        client = new AmazonDynamoDBClient(_credentials, _regionEndpoint);
                    }
                    else
                    {
                        client = new AmazonDynamoDBClient(_credentials, AmazonDynamoDbConfig);
                    }
                }
                else
                {
                    client = new AmazonDynamoDBClient(AmazonDynamoDbConfig);
                }
                using (var context = new DynamoDBContext(client))
                {
                    var batchWrite = context.CreateBatchWrite <LogDocument>(OperationConfig);
                    batchWrite.AddPutItems(records);
                    await batchWrite.ExecuteAsync();
                }
            }
            catch (Exception exception)
            {
                SelfLog.WriteLine("Unable to write events to DynamoDB Sink for {0}: {1}", _tableName, exception);
            }
            finally
            {
                if (client != null)
                {
                    client.Dispose();
                }
            }
        }
コード例 #27
0
        public BatchWrite GetBatchWrite(DynamoDBContext dbContext)
        {
            var entityBatch = dbContext.CreateBatchWrite <TEntity>();

            if (_data_to_Delete.Any())
            {
                entityBatch.AddPutItems(_data_to_Delete);
            }

            if (_data_to_Put.Any())
            {
                entityBatch.AddPutItems(_data_to_Put);
            }

            return(entityBatch);
        }
コード例 #28
0
        public BatchWrite <TEntity> GetBatchWrite(List <TEntity> entities, DynamoDbBatchOperator batchOperator)
        {
            using DynamoDBContext context = GetContext();
            BatchWrite <TEntity> batch = context.CreateBatchWrite <TEntity>();

            switch (batchOperator)
            {
            case DynamoDbBatchOperator.Delete:
                batch.AddDeleteItems(entities);
                break;

            case DynamoDbBatchOperator.Put:
                batch.AddPutItems(entities);
                break;
            }
            return(batch);
        }
コード例 #29
0
        /// <summary>
        ///     The BatchStore Method allows you to store a list of items of type T to dynamoDb
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="items"></param>
        public void BatchStore <T>(IEnumerable <T> items) where T : class
        {
            try
            {
                var itemBatch = DbContext.CreateBatchWrite <T>();

                foreach (var item in items)
                {
                    itemBatch.AddPutItem(item);
                }

                itemBatch.Execute();
            }
            catch
            {
                throw new Exception("Require collection not exists. Please try after a while");
            }
        }
コード例 #30
0
        public async Task InsertNewItems(List <Profile> profiles)
        {
            if (profiles != null && profiles.Count > 0)
            {
                try
                {
                    BatchWrite <Profile> batchWrite = _context.CreateBatchWrite <Profile>();
                    batchWrite.AddPutItems(profiles);
                    Console.WriteLine("Inserting list of profiles...");
                    await batchWrite.ExecuteAsync();

                    Console.WriteLine("Done");
                }
                catch (Exception ex)
                {
                    Console.WriteLine("[!] Insert data failed");
                    Console.WriteLine(ex);
                }
            }
        }
コード例 #31
0
        private async Task SaveObjectToDynamoDb(string jsonPayload)
        {
            var twitterObject = JsonConvert.DeserializeObject <TwitterStreamModel>(jsonPayload);

            AmazonDynamoDBConfig ddbConfig = new AmazonDynamoDBConfig();

            ddbConfig.ServiceURL = "http://34.246.18.10:8000";
            AmazonDynamoDBClient amazonDynamoDbClient =
                new AmazonDynamoDBClient(ddbConfig);

            var sw = Stopwatch.StartNew();
            IDynamoDBContext dynamoDbContext = new DynamoDBContext(amazonDynamoDbClient);
            var dynamoDbBatch = dynamoDbContext.CreateBatchWrite <TwitterStreamModel>();

            dynamoDbBatch.AddPutItem(twitterObject);
            await dynamoDbBatch.ExecuteAsync().ConfigureAwait(false);

            sw.Stop();
            Console.WriteLine($"Insert Id: {twitterObject.id} take {sw.ElapsedMilliseconds} ms");
        }
コード例 #32
0
    protected override async Task EmitBatchAsync(IEnumerable<LogEvent> events)
    {
      var records = events.Select(x => new LogDocument(x, x.RenderMessage(_formatProvider)));

      try
      {
        using (var client = new AmazonDynamoDBClient(AmazonDynamoDbConfig))
        {
          using (var context = new DynamoDBContext(client))
          {
            var batchWrite = context.CreateBatchWrite<LogDocument>(OperationConfig);
            batchWrite.AddPutItems(records);
            await batchWrite.ExecuteAsync();
          }
        }
      }
      catch (Exception exception)
      {
        SelfLog.WriteLine("Unable to write events to DynamoDB Sink for {0}: {1}", _tableName, exception);
      }
    }
コード例 #33
0
ファイル: Program.cs プロジェクト: wildbillcat/Dynamo
        private static void TestCRUDOperations(DynamoDBContext context)
        {
            List<MonthlyTotal> Import = new List<MonthlyTotal>();
            string oneTimeFile = @"C: \Users\wildbillcat\Downloads\unknown_20160206082017.csv";
            int i = 0;
            using (CsvReader csv = new CsvReader(System.IO.File.OpenText(oneTimeFile)))
            {
                CsvReader Header = new CsvReader(System.IO.File.OpenText(oneTimeFile));
                Header.Read();
                string parsedate = Header.FieldHeaders[2].Substring(21, 10);
                Header.Dispose();
                DateTime ImportDate = Convert.ToDateTime(parsedate);
                while (csv.Read())
                {
                    //MonthlyTotal SKUDocument = context.Load<MonthlyTotal>(SKUNumber, ImportDate, new DynamoDBOperationConfig { ConsistentRead = true });
                    //if(SKUDocument == null)
                    //{
                    //    //SKU isnt in Datebase, will have to create it.
                    //    SKUDocument = new MonthlyTotal()
                    //    {
                    //        SKUId = SKUNumber,
                    //        Month = ImportDate,
                    //        Total = csv.GetField<double>(2)
                    //    };
                    //}
                    MonthlyTotal SKUDocument = new MonthlyTotal()
                    {
                        SKUId = csv.GetField<int>(0),
                        Month = ImportDate,
                        Total = csv.GetField<double>(2)
                    };
                    //context.Save(SKUDocument);
                    Import.Add(SKUDocument);
                    i++;
                    if(i > 1000)
                    {
                        i = 0;
                        Console.Write("SKU :");
                        Console.WriteLine(SKUDocument.SKUId);
                    }
                }
            }
            var importBatch = context.CreateBatchWrite<MonthlyTotal>();
            importBatch.AddPutItems(Import);

            importBatch.Execute();
            Console.WriteLine("Wrote to NoSQL DB");
            Console.WriteLine();
        }