public void ProcessETLMode(IRecordProcessor rp)
{
inputDatabase.Connect();
MapFields(rp);
rp.setDatasetName(inputDatabase.NormTableName(tableName));
string[] sqls = rp.GetSelectStatements();
if (Progress != null)
{
Progress.ReportProgress(0);
}
foreach (string sql in sqls)
{
try
{
DbDataReader reader = inputDatabase.Execute(sql);
int count = 1;
rp.Reset();
//Extract
while (reader.Read())
{
//Transform
rp.ProcessRecord(reader);
if (Progress != null)
{
Progress.ReportProgress(0, "Read records: " + count.ToString());
}
count++;
}
reader.Close();
reader = null;
//Load
if (outputDatabase.FileName != null)
{
//if output is file - Write one
WriteData();
}
}
catch (Exception ex)
{
throw ex;
}
// Call Close when done reading.
}
inputDatabase.Close();
}
private async Task ProcessShard(CancellationToken stoppingToken, Shard shard)
{
// Get an iterator for the current shard
var iteratorRequest = new GetShardIteratorRequest
{
StreamArn = _latestStreamArn,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var iteratorResult = await _dynamoDbStreams.GetShardIteratorAsync(iteratorRequest, stoppingToken);
var iterator = iteratorResult.ShardIterator;
// Shard iterator is not null until the Shard is sealed (marked as READ_ONLY).
while (iterator != null)
{
// Use the shard iterator to read the stream records
var recordsResult = await _dynamoDbStreams.GetRecordsAsync(
new GetRecordsRequest
{
ShardIterator = iterator,
Limit = BatchSize
});
var records = recordsResult.Records;
foreach (var record in records)
{
_recordProcessor.ProcessRecord(record.Dynamodb);
}
iterator = recordsResult.NextShardIterator;
//we need to pause between iterations as get records has a limit
await Task.Delay(1000);
}
throw new ShardHasBeenClosedException(
$"The shard has been closed and we must re-get shards. Shard {shard.ShardId}"
);
}
