public async Task SyncFrom(ulong since, string[] excludeOriginReplicas)
{
var lastPos = since;
_log.Info($"Received replication request to sync from {lastPos}");
var until = _lmdb.Read(txn => _wlTable.GetLastClock(txn))?.Item1 ?? 0;
while (!_cancellationToken.IsCancellationRequested && lastPos <= until)
{
var page = _lmdb.Read(txn =>
{
var logPage = _wlTable.GetLogPage(txn, lastPos, _replicationConfig.PageSize)
.Where(logItem => logItem.Item1 <= until && !excludeOriginReplicas.Contains(logItem.Item2.OriginatorReplicaId))
. // TODO: Add check for IncludeAcked
ToArray();
foreach (var logItem in logPage)
{
if (logItem.Item2.LoggedEventCase == WriteLogEvent.LoggedEventOneofCase.Updated && (logItem.Item2.Updated.Value == null || logItem.Item2.Updated.Value.IsEmpty))
{
var latestValue = GetValue(txn, logItem.Item2.Updated.Key);
if (latestValue != null)
{
logItem.Item2.Updated.Value = latestValue; // TODO: Handle "Value not found"
}
else
{
_log.Info($"No value for the key {logItem.Item2.Updated.Key}");
}
}
}
return(logPage);
});
if (page.Length == 0)
{
break;
}
{
if (_replicationConfig.UseBatching)
{
lastPos = await ProcessInBatches(page);
}
else
{
foreach (var pageItem in page)
{
var(pos, item) = pageItem;
if (!_cancellationToken.IsCancellationRequested)
{
await _responseStreamWriteAsync(new SyncPacket
{
ReplicaId = _ownReplicaId,
Item = new Item {
LogEvent = item
}
});
lastPos = pos;
}
}
}
lastPos++;
}
}
await _responseStreamWriteAsync(new SyncPacket
{
ReplicaId = _ownReplicaId,
SkipPos = new SkipPos {
LastPos = lastPos
}
});
_log.Info($"Page replicated. Last pos: {lastPos}");
async Task <ulong> ProcessInBatches((ulong, WriteLogEvent)[] page)
public static TableKey[] KeysByPrefix(this LightningPersistence lmdb, Table table, TableKey prefix, uint page, uint pageSize) => lmdb.Read(txn => txn.KeysByPrefix(table, prefix, page, pageSize).ToArray());
public ulong?GetLastPos(string replicaId) => _lmdb.Read(txn => GetLastPos(txn, replicaId));
public static bool ContainsKey(this LightningPersistence lmdb, Table table, TableKey key) => lmdb.Read(txn => txn.ContainsKey(table, key));
