Пример #1
0
        public async Task BatchInserts(List <Profile> profiles)
        {
            BatchWrite <Profile> batchWrite = _context.CreateBatchWrite <Profile>();

            batchWrite.AddPutItems(profiles);
            await batchWrite.ExecuteAsync();

            Console.WriteLine("Items inserted successfully");
        }
        private async Task <bool> AddManyItems(List <T> items)
        {
            _ = UpdateTableTracker();
            BatchWrite <T> batch = _context.CreateBatchWrite <T>();

            batch.AddPutItems(items);
            await batch.ExecuteAsync();

            return(true);
        }
Пример #3
0
        public async Task AddBulk(IEnumerable <CarDTO> cars)
        {
            // Adding the uid manually.. this sucks
            cars.Select(c => { c.Id = Guid.NewGuid().ToString(); return(c); }).ToList();

            // Start the batch
            BatchWrite <Car> bulkBatch = dbContext.CreateBatchWrite <Car>();

            bulkBatch.AddPutItems(mapper.Map <IEnumerable <Car> >(cars));
            await bulkBatch.ExecuteAsync();
        }
        public async Task AddItems <T>(IEnumerable <T> items)
        {
            IDynamoDBContext context = new DynamoDBContext(client, new DynamoDBContextConfig()
            {
                ConsistentRead = true
            });
            BatchWrite <T> batch = context.CreateBatchWrite <T>();

            batch.AddPutItems(items);

            await batch.ExecuteAsync();
        }
Пример #5
0
        public BatchWrite <TEntity> GetBatchWrite(List <TEntity> entities, DynamoDbBatchOperator batchOperator)
        {
            using DynamoDBContext context = GetContext();
            BatchWrite <TEntity> batch = context.CreateBatchWrite <TEntity>();

            switch (batchOperator)
            {
            case DynamoDbBatchOperator.Delete:
                batch.AddDeleteItems(entities);
                break;

            case DynamoDbBatchOperator.Put:
                batch.AddPutItems(entities);
                break;
            }
            return(batch);
        }
Пример #6
0
        public async Task InsertNewItems(List <Profile> profiles)
        {
            if (profiles != null && profiles.Count > 0)
            {
                try
                {
                    BatchWrite <Profile> batchWrite = _context.CreateBatchWrite <Profile>();
                    batchWrite.AddPutItems(profiles);
                    Console.WriteLine("Inserting list of profiles...");
                    await batchWrite.ExecuteAsync();

                    Console.WriteLine("Done");
                }
                catch (Exception ex)
                {
                    Console.WriteLine("[!] Insert data failed");
                    Console.WriteLine(ex);
                }
            }
        }
Пример #7
0
        private void RunBloomMigration(CancellationToken token)
        {
            BlockHeader GetMissingBlockHeader(long i)
            {
                if (_logger.IsWarn)
                {
                    _logger.Warn(GetLogMessage("warning", $"Header for block {i} not found. Logs will not be searchable for this block."));
                }
                return(EmptyHeader);
            }

            if (_api.BloomStorage == null)
            {
                throw new StepDependencyException(nameof(_api.BloomStorage));
            }
            if (_api.BlockTree == null)
            {
                throw new StepDependencyException(nameof(_api.BlockTree));
            }
            if (_api.ChainLevelInfoRepository == null)
            {
                throw new StepDependencyException(nameof(_api.ChainLevelInfoRepository));
            }

            IBlockTree    blockTree = _api.BlockTree;
            IBloomStorage storage   = _api.BloomStorage;
            long          to        = MinBlockNumber;
            long          synced    = storage.MigratedBlockNumber + 1;
            long          from      = synced;

            _migrateCount = to + 1;
            _averages     = _api.BloomStorage.Averages.ToArray();
            IChainLevelInfoRepository?chainLevelInfoRepository = _api.ChainLevelInfoRepository;

            _progress.Update(synced);

            if (_logger.IsInfo)
            {
                _logger.Info(GetLogMessage("started"));
            }

            using (Timer timer = new Timer(1000)
            {
                Enabled = true
            })
            {
                timer.Elapsed += (_, _) =>
                {
                    if (_logger.IsInfo)
                    {
                        _logger.Info(GetLogMessage("in progress"));
                    }
                };

                try
                {
                    storage.Migrate(GetHeadersForMigration());
                }
                finally
                {
                    _progress.MarkEnd();
                    _stopwatch?.Stop();
                }

                IEnumerable <BlockHeader> GetHeadersForMigration()
                {
                    bool TryGetMainChainBlockHashFromLevel(long number, out Keccak?blockHash)
                    {
                        using BatchWrite batch = chainLevelInfoRepository.StartBatch();
                        ChainLevelInfo?level = chainLevelInfoRepository.LoadLevel(number);

                        if (level != null)
                        {
                            if (!level.HasBlockOnMainChain)
                            {
                                if (level.BlockInfos.Length > 0)
                                {
                                    level.HasBlockOnMainChain = true;
                                    chainLevelInfoRepository.PersistLevel(number, level, batch);
                                }
                            }

                            blockHash = level.MainChainBlock?.BlockHash;
                            return(blockHash != null);
                        }
                        else
                        {
                            blockHash = null;
                            return(false);
                        }
                    }

                    for (long i = from; i <= to; i++)
                    {
                        if (token.IsCancellationRequested)
                        {
                            timer.Stop();
                            if (_logger.IsInfo)
                            {
                                _logger.Info(GetLogMessage("cancelled"));
                            }
                            yield break;
                        }

                        if (TryGetMainChainBlockHashFromLevel(i, out Keccak? blockHash))
                        {
                            BlockHeader?header = blockTree.FindHeader(blockHash !, BlockTreeLookupOptions.None);
                            yield return(header ?? GetMissingBlockHeader(i));
                        }
                        else
                        {
                            yield return(GetMissingBlockHeader(i));
                        }

                        _progress.Update(++synced);
                    }
                }
            }

            if (!token.IsCancellationRequested)
            {
                if (_logger.IsInfo)
                {
                    _logger.Info(GetLogMessage("finished"));
                }
            }
        }
Пример #8
0
        private async Task LoadData(IEnumerable <DashboardEventRaw> data, string source)
        {
            if (data == null)
            {
                this._context.LogError("Null was provided to LoadData, no data to load.");
                return;
            }

            BatchWrite <DashboardEventParsed> batch = ddbContext.CreateBatchWrite <DashboardEventParsed>();

            int misses = 0;

            foreach (DashboardEventRaw item in data)
            {
                DashboardEventParsed parsed;

                try
                {
                    parsed = DashboardEventParsed.FromRawEvent(item);
                    batch.AddPutItem(parsed);

                    if (parsed.Timeline.StartTimeWasFoundInDescription == false)
                    {
                        this._context.LogError($"Did not find start/end in description: {item.Description}");
                        misses += 1;
                    }
                }
                catch (Exception e)
                {
                    this._context.LogError(e);
                }
            }

            try
            {
                await batch.ExecuteAsync();
            }
            catch (Exception e)
            {
                this._context.LogError(e);
            }

            await cwClient.PutMetricDataAsync(new PutMetricDataRequest()
            {
                MetricData = new List <MetricDatum>()
                {
                    new MetricDatum()
                    {
                        Value        = misses,
                        MetricName   = Environment.GetEnvironmentVariable("ExtractionFailureMetricName"),
                        TimestampUtc = DateTime.UtcNow,
                        Unit         = StandardUnit.Count,
                        Dimensions   = new List <Dimension>()
                        {
                            new Dimension()
                            {
                                Name  = "source",
                                Value = source
                            }
                        }
                    }
                },
                Namespace = "SHD"
            });
        }
Пример #9
0
        private void RunMigration(CancellationToken token)
        {
            Block GetMissingBlock(long i, Keccak?blockHash)
            {
                if (_logger.IsWarn)
                {
                    _logger.Warn(GetLogMessage("warning", $"Block {i} not found. Logs will not be searchable for this block."));
                }
                EmptyBlock.Header.Number = i;
                EmptyBlock.Header.Hash   = blockHash;
                return(EmptyBlock);
            }

            long synced     = 1;
            IDb  receiptsDb = _dbProvider.ReceiptsDb;

            _progress.Reset(synced);

            if (_logger.IsInfo)
            {
                _logger.Info(GetLogMessage("started"));
            }

            using (Timer timer = new Timer(1000)
            {
                Enabled = true
            })
            {
                timer.Elapsed += (ElapsedEventHandler)((o, e) =>
                {
                    if (_logger.IsInfo)
                    {
                        _logger.Info(GetLogMessage("in progress"));
                    }
                });

                try
                {
                    foreach (Block block in GetBlockBodiesForMigration())
                    {
                        TxReceipt?[] receipts        = _receiptStorage.Get(block);
                        TxReceipt[]  notNullReceipts = receipts.Length == 0 ? receipts : receipts.Where(r => r != null).ToArray();

                        if (receipts.Length == 0 || notNullReceipts.Length != 0) // if notNullReceipts.Length is 0 and receipts are not 0 - we are missing all receipts, they are not processed yet.
                        {
                            _receiptStorage.Insert(block, notNullReceipts);
                            _receiptStorage.MigratedBlockNumber = block.Number;

                            for (int i = 0; i < notNullReceipts.Length; i++)
                            {
                                receiptsDb.Delete(notNullReceipts[i].TxHash !);
                            }

                            if (notNullReceipts.Length != receipts.Length)
                            {
                                if (_logger.IsWarn)
                                {
                                    _logger.Warn(GetLogMessage("warning", $"Block {block.ToString(Block.Format.FullHashAndNumber)} is missing {receipts.Length - notNullReceipts.Length} of {receipts.Length} receipts!"));
                                }
                            }
                        }
                        else if (block.Number <= _blockTree.Head?.Number)
                        {
                            if (_logger.IsWarn)
                            {
                                _logger.Warn(GetLogMessage("warning", $"Block {block.ToString(Block.Format.FullHashAndNumber)} is missing {receipts.Length - notNullReceipts.Length} of {receipts.Length} receipts!"));
                            }
                        }
                    }
                }
                finally
                {
                    _progress.MarkEnd();
                    _stopwatch?.Stop();
                }

                IEnumerable <Block> GetBlockBodiesForMigration()
                {
                    bool TryGetMainChainBlockHashFromLevel(long number, out Keccak?blockHash)
                    {
                        using BatchWrite batch = _chainLevelInfoRepository.StartBatch();
                        ChainLevelInfo level = _chainLevelInfoRepository.LoadLevel(number);

                        if (level != null)
                        {
                            if (!level.HasBlockOnMainChain)
                            {
                                if (level.BlockInfos.Length > 0)
                                {
                                    level.HasBlockOnMainChain = true;
                                    _chainLevelInfoRepository.PersistLevel(number, level, batch);
                                }
                            }

                            blockHash = level.MainChainBlock?.BlockHash;
                            return(blockHash != null);
                        }
                        else
                        {
                            blockHash = null;
                            return(false);
                        }
                    }

                    for (long i = _toBlock - 1; i > 0; i--)
                    {
                        if (token.IsCancellationRequested)
                        {
                            timer.Stop();
                            if (_logger.IsInfo)
                            {
                                _logger.Info(GetLogMessage("cancelled"));
                            }
                            yield break;
                        }

                        if (TryGetMainChainBlockHashFromLevel(i, out Keccak? blockHash))
                        {
                            Block header = _blockTree.FindBlock(blockHash, BlockTreeLookupOptions.None);
                            yield return(header ?? GetMissingBlock(i, blockHash));
                        }

                        _progress.Update(++synced);
                    }
                }
            }

            if (!token.IsCancellationRequested)
            {
                if (_logger.IsInfo)
                {
                    _logger.Info(GetLogMessage("finished"));
                }
            }
        }