Task SendMessage(UnicastTransportOperation transportOperation)
{
Logger.Debug("Send to " + transportOperation.Destination);
var toBeReceived = transportOperation.GetTimeToBeReceived();
var timeToBeReceived = toBeReceived.HasValue && toBeReceived.Value < TimeSpan.MaxValue
? toBeReceived
: null;
if (timeToBeReceived != null && timeToBeReceived.Value == TimeSpan.Zero)
{
var messageType = transportOperation.Message.Headers[Headers.EnclosedMessageTypes].Split(',').First();
Logger.WarnFormat("TimeToBeReceived is set to zero for message of type '{0}'. Cannot send operation.",
messageType);
return(Task.FromResult(0));
}
// TimeToBeReceived was not specified on message - go for maximum set by SDK
if (timeToBeReceived == TimeSpan.MaxValue)
{
timeToBeReceived = null;
}
var messageWrapper = BuildMessageWrapper(transportOperation, toBeReceived, transportOperation.Destination);
var topic = producerFactory.GetProducer().Topic(transportOperation.Destination);
var messageStream = new MemoryStream();
KafkaTransportInfrastructure.GetSerializer().Serialize(messageWrapper, messageStream);
return(topic.Produce(messageStream.ToArray()));
}
private async Task StreamTableAsync(CancellationToken token,
string executionId,
TableSchema tableSchema,
SerializationMode serializationMode,
bool sendWithKey,
int batchSize,
int printPercentProgressMod)
{
string topicName = _kafkaTopicPrefix + tableSchema.TableName.ToLower();
var rowCount = await _cdcReaderClient.GetRowCountAsync(tableSchema);
Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} has {rowCount} rows to export");
int progress = 0;
using (var producer = ProducerFactory.GetProducer(topicName, tableSchema, serializationMode, sendWithKey, _kafkaBootstrapServers, _schemaRegistryUrl))
{
long ctr = 0;
PrimaryKeyValue lastRetrievedKey = null;
var existingOffset = await _cdcReaderClient.GetLastFullLoadOffsetAsync(executionId, tableSchema.TableName);
if (existingOffset.Result == CdcReader.State.Result.NoStoredState)
{
Console.WriteLine($"Table {tableSchema.TableName} - No previous stored offset. Starting from first row");
var firstBatch = await _cdcReaderClient.GetFirstBatchAsync(tableSchema, batchSize);
ctr = await PublishAsync(producer, token, firstBatch, ctr);
lastRetrievedKey = firstBatch.LastRowKey;
await _cdcReaderClient.StoreFullLoadOffsetAsync(executionId, tableSchema.TableName, firstBatch.LastRowKey);
}
else
{
Console.WriteLine($"Table {tableSchema.TableName} - No data to export");
lastRetrievedKey = existingOffset.State;
}
bool finished = false;
while (!token.IsCancellationRequested && !finished)
{
var changes = new List <RowChange>();
var batch = await _cdcReaderClient.GetBatchAsync(tableSchema, lastRetrievedKey, batchSize);
ctr = await PublishAsync(producer, token, batch, ctr);
int latestProgress = (int)(((double)ctr / (double)rowCount) * 100);
if (progress != latestProgress && latestProgress % printPercentProgressMod == 0)
{
Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} - Progress at {latestProgress}% ({ctr} records)");
}
progress = latestProgress;
lastRetrievedKey = batch.LastRowKey;
await _cdcReaderClient.StoreFullLoadOffsetAsync(executionId, tableSchema.TableName, lastRetrievedKey);
if (!batch.Records.Any() || batch.Records.Count < batchSize)
{
finished = true;
}
}
if (token.IsCancellationRequested)
{
Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} - cancelled at progress at {progress}% ({ctr} records)");
}
else
{
Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} - complete ({ctr} records)");
}
}
}
private async Task StartPublishingChanges(CancellationToken token,
string executionId,
string tableName,
TimeSpan maxInterval,
int batchSize,
bool sendWithKey,
SerializationMode serializationMode)
{
var tableTopic = _tableTopicPrefix + tableName.ToLower();
var tableSchema = await _cdcReaderClient.GetTableSchemaAsync(tableName);
using (var producer = ProducerFactory.GetProducer(tableTopic, tableSchema, serializationMode, sendWithKey, _kafkaBootstrapServers, _schemaRegistryUrl))
{
var cdcState = await SetInitialStateAsync(token, producer, executionId, tableSchema, maxInterval);
var sw = new Stopwatch();
while (!token.IsCancellationRequested)
{
cdcState.ToLsn = await _cdcReaderClient.GetMaxLsnAsync();
sw.Start();
Console.WriteLine($"Table {tableName} - Starting to export LSN range {GetBigInteger(cdcState.FromLsn)} to {GetBigInteger(cdcState.ToLsn)}");
bool more = true;
int blockCounter = 0;
while (!token.IsCancellationRequested && more)
{
if (GetBigInteger(cdcState.FromLsn) <= GetBigInteger(cdcState.ToLsn))
{
blockCounter++;
ChangeBatch batch = null;
if (cdcState.UnfinishedLsn)
{
batch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, cdcState.FromLsn, cdcState.FromSeqVal, cdcState.ToLsn, batchSize);
}
else
{
batch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, cdcState.FromLsn, cdcState.ToLsn, batchSize);
}
if (batch.Changes.Any())
{
Console.WriteLine($"Table {tableName} - Retrieved block #{blockCounter} with {batch.Changes.Count} changes");
foreach (var change in batch.Changes)
{
await producer.SendAsync(token, change);
cdcState.FromLsn = change.Lsn;
cdcState.FromSeqVal = change.SeqVal;
}
more = batch.MoreChanges;
cdcState.UnfinishedLsn = batch.MoreOfLastTransaction;
if (cdcState.UnfinishedLsn)
{
cdcState.FromSeqVal = Increment(cdcState.FromSeqVal);
}
else
{
cdcState.FromLsn = Increment(cdcState.FromLsn);
}
var offset = GetOffset(cdcState);
await BlockingStoreCdcOffsetAsync(token, executionId, tableName, offset);
}
else
{
more = false;
cdcState.UnfinishedLsn = false;
Console.WriteLine($"Table {tableName} - No changes");
}
}
else
{
more = false;
cdcState.UnfinishedLsn = false;
Console.WriteLine($"Table {tableName} - No changes");
}
}
var remainingMs = maxInterval.TotalMilliseconds - sw.Elapsed.TotalMilliseconds;
if (remainingMs > 0)
{
await Task.Delay((int)remainingMs);
}
sw.Reset();
}
}
}
