private bool ReplicateChangesToDestination(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs)
{
var scriptResult = ApplyConversionScript(cfg, docs);
if (scriptResult.Data.Count == 0)
{
return(true);
}
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
var countOfItems = scriptResult.Data.Sum(x => x.Value.Count);
try
{
using (var writer = new ElasticsearchDestinationWriter(Database, cfg, replicationStats))
{
if (writer.Execute(scriptResult))
{
log.Debug("Replicated changes of {0} for replication {1}", string.Join(", ", docs.Select(d => d.Key)), cfg.Name);
replicationStats.CompleteSuccess(countOfItems);
}
else
{
log.Debug("Replicated changes (with some errors) of {0} for replication {1}", string.Join(", ", docs.Select(d => d.Key)), cfg.Name);
replicationStats.Success(countOfItems);
}
}
return(true);
}
catch (Exception e)
{
log.WarnException("Failure to replicate changes to Elasticsearch for: " + cfg.Name, e);
SqlReplicationStatistics replicationStatistics;
DateTime newTime;
if (statistics.TryGetValue(cfg.Name, out replicationStatistics) == false)
{
newTime = SystemTime.UtcNow.AddSeconds(5);
}
else
{
var totalSeconds = (SystemTime.UtcNow - replicationStatistics.LastErrorTime).TotalSeconds;
newTime = SystemTime.UtcNow.AddSeconds(Math.Max(60 * 15, Math.Min(5, totalSeconds + 5)));
}
replicationStats.RecordWriteError(e, Database, countOfItems, newTime);
return(false);
}
}
private bool ReplicateDeletionsToDestination(SqlReplicationConfig cfg, IEnumerable <ListItem> deletedDocs)
{
var identifiers = deletedDocs.Select(x => x.Key).ToList();
if (identifiers.Count == 0)
{
return(true);
}
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
using (var writer = new ElasticsearchDestinationWriter(Database, cfg, replicationStats))
{
foreach (var sqlReplicationTable in cfg.SqlReplicationTables)
{
writer.DeleteItems(sqlReplicationTable.TableName, sqlReplicationTable.DocumentKeyColumn, identifiers);
}
writer.Commit();
log.Debug("Replicated deletes of {0} for config {1}", string.Join(", ", identifiers), cfg.Name);
}
return(true);
}
private bool ReplicateChangesToDestination(SqlReplicationConfig cfg, IEnumerable<JsonDocument> docs)
{
var scriptResult = ApplyConversionScript(cfg, docs);
if (scriptResult.Data.Count == 0)
return true;
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
var countOfItems = scriptResult.Data.Sum(x => x.Value.Count);
try
{
using (var writer = new ElasticsearchDestinationWriter(Database, cfg, replicationStats))
{
if (writer.Execute(scriptResult))
{
log.Debug("Replicated changes of {0} for replication {1}", string.Join(", ", docs.Select(d => d.Key)), cfg.Name);
replicationStats.CompleteSuccess(countOfItems);
}
else
{
log.Debug("Replicated changes (with some errors) of {0} for replication {1}", string.Join(", ", docs.Select(d => d.Key)), cfg.Name);
replicationStats.Success(countOfItems);
}
}
return true;
}
catch (Exception e)
{
log.WarnException("Failure to replicate changes to Elasticsearch for: " + cfg.Name, e);
SqlReplicationStatistics replicationStatistics;
DateTime newTime;
if (statistics.TryGetValue(cfg.Name, out replicationStatistics) == false)
{
newTime = SystemTime.UtcNow.AddSeconds(5);
}
else
{
var totalSeconds = (SystemTime.UtcNow - replicationStatistics.LastErrorTime).TotalSeconds;
newTime = SystemTime.UtcNow.AddSeconds(Math.Max(60 * 15, Math.Min(5, totalSeconds + 5)));
}
replicationStats.RecordWriteError(e, Database, countOfItems, newTime);
return false;
}
}
private bool ReplicateDeletionsToDestination(SqlReplicationConfig cfg, IEnumerable<ListItem> deletedDocs)
{
var identifiers = deletedDocs.Select(x => x.Key).ToList();
if (identifiers.Count == 0)
return true;
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
using (var writer = new ElasticsearchDestinationWriter(Database, cfg, replicationStats))
{
foreach (var sqlReplicationTable in cfg.SqlReplicationTables)
{
writer.DeleteItems(sqlReplicationTable.TableName, sqlReplicationTable.DocumentKeyColumn, identifiers);
}
writer.Commit();
log.Debug("Replicated deletes of {0} for config {1}", string.Join(", ", identifiers), cfg.Name);
}
return true;
}