public void ProcessETLMode(IRecordProcessor rp) { inputDatabase.Connect(); MapFields(rp); rp.setDatasetName(inputDatabase.NormTableName(tableName)); string[] sqls = rp.GetSelectStatements(); if (Progress != null) { Progress.ReportProgress(0); } foreach (string sql in sqls) { try { DbDataReader reader = inputDatabase.Execute(sql); int count = 1; rp.Reset(); //Extract while (reader.Read()) { //Transform rp.ProcessRecord(reader); if (Progress != null) { Progress.ReportProgress(0, "Read records: " + count.ToString()); } count++; } reader.Close(); reader = null; //Load if (outputDatabase.FileName != null) { //if output is file - Write one WriteData(); } } catch (Exception ex) { throw ex; } // Call Close when done reading. } inputDatabase.Close(); }
private async Task ProcessShard(CancellationToken stoppingToken, Shard shard) { // Get an iterator for the current shard var iteratorRequest = new GetShardIteratorRequest { StreamArn = _latestStreamArn, ShardId = shard.ShardId, ShardIteratorType = ShardIteratorType.TRIM_HORIZON }; var iteratorResult = await _dynamoDbStreams.GetShardIteratorAsync(iteratorRequest, stoppingToken); var iterator = iteratorResult.ShardIterator; // Shard iterator is not null until the Shard is sealed (marked as READ_ONLY). while (iterator != null) { // Use the shard iterator to read the stream records var recordsResult = await _dynamoDbStreams.GetRecordsAsync( new GetRecordsRequest { ShardIterator = iterator, Limit = BatchSize }); var records = recordsResult.Records; foreach (var record in records) { _recordProcessor.ProcessRecord(record.Dynamodb); } iterator = recordsResult.NextShardIterator; //we need to pause between iterations as get records has a limit await Task.Delay(1000); } throw new ShardHasBeenClosedException( $"The shard has been closed and we must re-get shards. Shard {shard.ShardId}" ); }