/// <summary>
/// Performs synchronization in an infinite loop. Periodically checks if source version has increased to trigger replication.
/// </summary>
/// <param name="token">The cancellation token.</param>
public void SyncLoop(CancellationToken token)
{
var currentVersions = Enumerable.Repeat(0L, Config.ReplicationSets.Count).ToList();
if (!Initialized)
{
Init();
}
while (true)
{
if (token.IsCancellationRequested)
{
Log.Info("Stopping replication.");
return;
}
var start = DateTime.UtcNow;
Error = false;
for (int i = 0; i < Config.ReplicationSets.Count; i++)
{
var replicationSet = Config.ReplicationSets[i];
var currentVersion = currentVersions[i];
long version = 0;
try
{
using (var db = GetDatabase(replicationSet.Source.ConnectionString, DatabaseType.SqlServer2008))
version = db.ExecuteScalar <long>("select CHANGE_TRACKING_CURRENT_VERSION()");
Log.Debug($"Current version of source in replication set {replicationSet.Name} is {version}.");
if (version > currentVersion)
{
Log.Info($"Current version of source in replication set {replicationSet.Name} has increased from {currentVersion} to {version}: Starting replication.");
var tables = Tables[i];
var success = Sync(replicationSet, tables, version);
if (success)
{
currentVersions[i] = version;
}
Synced?.Invoke(this, new SyncEventArgs {
ReplicationSet = replicationSet, Version = version
});
}
}
catch (Exception ex)
{
Log.Error(ex, $"Error occurred during replication of set {replicationSet.Name}.");
Error = true;
}
if (token.IsCancellationRequested)
{
Log.Info("Stopping replication.");
return;
}
}
Log.Info($"Finished replication {(Error ? "with" : "without")} errors");
var delay = (int)Math.Round(Math.Max(0, (TimeSpan.FromSeconds(Interval) - (DateTime.UtcNow - start)).TotalSeconds) * 1000, MidpointRounding.AwayFromZero);
Thread.Sleep(delay);
}
}
internal virtual void OnSynced() => Synced?.Invoke(this, EventArgs.Empty);
private static void OnSynced() => Synced?.Invoke(null, EventArgs.Empty);
private async Task PeerSyncAsync(CancellationToken token, PeerInfo peerInfo)
{
bool wasCancelled = false;
ISynchronizationPeer peer = peerInfo.Peer;
BigInteger bestNumber = BlockTree.BestSuggested.Number;
const int maxLookup = 64;
int ancestorLookupLevel = 0;
bool isCommonAncestorKnown = false;
while (peerInfo.NumberAvailable > bestNumber && peerInfo.NumberReceived <= peerInfo.NumberAvailable)
{
if (ancestorLookupLevel > maxLookup)
{
throw new InvalidOperationException("Cannot find ancestor"); // TODO: remodel this after full sync test is added
}
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
if (!isCommonAncestorKnown)
{
// TODO: cases when many peers used for sync and one peer finished sync and then we need resync - we should start from common point and not NumberReceived that may be far in the past
_logger.Info($"Finding common ancestor for {peerInfo.Peer.NodeId}");
isCommonAncestorKnown = true;
}
BigInteger blocksLeft = peerInfo.NumberAvailable - peerInfo.NumberReceived;
// TODO: fault handling on tasks
int blocksToRequest = (int)BigInteger.Min(blocksLeft + 1, BatchSize);
if (_logger.IsDebugEnabled)
{
_logger.Debug($"Sync request to peer with {peerInfo.NumberAvailable} blocks. Got {peerInfo.NumberReceived} and asking for {blocksToRequest} more.");
}
Task <BlockHeader[]> headersTask = peer.GetBlockHeaders(peerInfo.NumberReceived, blocksToRequest, 0, token);
_currentSyncTask = headersTask;
BlockHeader[] headers = await headersTask;
if (_currentSyncTask.IsCanceled)
{
wasCancelled = true;
break;
}
if (_currentSyncTask.IsFaulted)
{
_logger.Error("Failed to retrieve headers when synchronizing", _currentSyncTask.Exception);
throw _currentSyncTask.Exception;
}
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
List <Keccak> hashes = new List <Keccak>();
Dictionary <Keccak, BlockHeader> headersByHash = new Dictionary <Keccak, BlockHeader>();
for (int i = 1; i < headers.Length; i++)
{
hashes.Add(headers[i].Hash);
headersByHash[headers[i].Hash] = headers[i];
}
Task <Block[]> bodiesTask = peer.GetBlocks(hashes.ToArray(), token);
_currentSyncTask = bodiesTask;
Block[] blocks = await bodiesTask;
if (_currentSyncTask.IsCanceled)
{
wasCancelled = true;
break;
}
if (_currentSyncTask.IsFaulted)
{
_logger.Error("Failed to retrieve bodies when synchronizing", _currentSyncTask.Exception);
throw _currentSyncTask.Exception;
}
ancestorLookupLevel = 0;
for (int i = 0; i < blocks.Length; i++)
{
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
blocks[i].Header = headersByHash[hashes[i]];
}
if (blocks.Length > 0)
{
Block parent = BlockTree.FindParent(blocks[0]);
if (parent == null)
{
ancestorLookupLevel += BatchSize;
peerInfo.NumberReceived -= BatchSize;
continue;
}
}
// Parity 1.11 non canonical blocks when testing on 27/06
for (int i = 0; i < blocks.Length; i++)
{
if (i != 0 && blocks[i].ParentHash != blocks[i - 1].Hash)
{
throw new EthSynchronizationException("Peer send an inconsistent block list");
}
}
for (int i = 0; i < blocks.Length; i++)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Received {blocks[i]} from {peer.NodeId}");
}
if (_blockValidator.ValidateSuggestedBlock(blocks[i]))
{
AddBlockResult addResult = BlockTree.SuggestBlock(blocks[i]);
if (addResult == AddBlockResult.UnknownParent)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Block {blocks[i].Number} ignored (unknown parent)");
}
if (i == 0)
{
if (_logger.IsWarnEnabled)
{
_logger.Warn("Resyncing split");
}
peerInfo.NumberReceived -= 1;
var syncTask = Task.Run(() => PeerSyncAsync(_syncCancellationTokenSource.Token, peerInfo), _syncCancellationTokenSource.Token);
await syncTask;
}
else
{
const string message = "Peer sent an inconsistent batch of block headers";
_logger.Error(message);
throw new EthSynchronizationException(message);
}
}
if (_logger.IsDebugEnabled)
{
_logger.Debug($"Block {blocks[i].Number} suggested for processing");
}
}
else
{
if (_logger.IsWarnEnabled)
{
_logger.Warn($"Block {blocks[i].Number} skipped (validation failed)");
}
}
}
peerInfo.NumberReceived = blocks[blocks.Length - 1].Number;
bestNumber = BlockTree.BestSuggested.Number;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Stopping sync processes with Node: {peerInfo.Peer.NodeId}, wasCancelled: {wasCancelled}");
}
if (!wasCancelled)
{
peerInfo.IsSynced = true;
Synced?.Invoke(this, new SyncEventArgs(peerInfo.Peer));
}
}