public SqlReplicationConfig ToSqlReplicationConfig()
{
var result = new SqlReplicationConfig
{
Id = Id,
Name = Name,
FactoryName = FactoryName,
RavenEntityName = RavenEntityName,
Script = Script,
Disabled = disabled,
SqlReplicationTables = new List <SqlReplicationTable>(sqlReplicationTables)
};
if (string.IsNullOrWhiteSpace(ConnectionString) == false)
{
result.ConnectionString = ConnectionString;
}
else if (string.IsNullOrWhiteSpace(ConnectionStringName) == false)
{
result.ConnectionStringName = connectionStringName;
}
else
{
result.ConnectionStringSettingName = connectionStringSettingName;
}
return(result);
}
public ElasticsearchReplicationConfig(SqlReplicationConfig other)
{
this.ConnectionString = other.ConnectionString;
this.FactoryName = other.FactoryName;
this.Script = other.Script;
this.Disabled = other.Disabled;
this.Id = other.Id;
this.Name = other.Name;
this.RavenEntityName = other.RavenEntityName;
this.SqlReplicationTables = other.SqlReplicationTables;
}
private Etag GetLastEtagFor(SqlReplicationStatus replicationStatus, SqlReplicationConfig sqlReplicationConfig)
{
var lastEtag = Etag.Empty;
var lastEtagHolder = replicationStatus.LastReplicatedEtags.FirstOrDefault(
x => string.Equals(sqlReplicationConfig.Name, x.Name, StringComparison.InvariantCultureIgnoreCase));
if (lastEtagHolder != null)
{
lastEtag = lastEtagHolder.LastDocEtag;
}
return(lastEtag);
}
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs)
{
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
var result = new ConversionScriptResult();
foreach (var jsonDocument in docs)
{
Database.WorkContext.CancellationToken.ThrowIfCancellationRequested();
if (string.IsNullOrEmpty(cfg.RavenEntityName) == false)
{
var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName);
if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false)
{
continue;
}
}
var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key);
try
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
var document = jsonDocument.ToJson();
document[Constants.DocumentIdFieldName] = jsonDocument.Key;
patcher.Apply(document, new ScriptedPatchRequest
{
Script = cfg.Script
}, jsonDocument.SerializedSizeOnDisk);
if (log.IsDebugEnabled && patcher.Debug.Count > 0)
{
log.Debug("Debug output for doc: {0} for script {1}:\r\n.{2}", jsonDocument.Key, cfg.Name, string.Join("\r\n", patcher.Debug));
patcher.Debug.Clear();
}
replicationStats.ScriptSuccess();
}
catch (ParseException e)
{
replicationStats.MarkScriptAsInvalid(Database, cfg.Script);
log.WarnException("Could not parse Elasticsearch Replication script for " + cfg.Name, e);
return(result);
}
catch (Exception e)
{
replicationStats.RecordScriptError(Database);
log.WarnException("Could not process Elasticsearch Replication script for " + cfg.Name + ", skipping document: " + jsonDocument.Key, e);
}
}
return(result);
}
public static SqlReplicationConfigModel FromSqlReplicationConfig(SqlReplicationConfig config)
{
return(new SqlReplicationConfigModel
{
Id = config.Id,
Name = config.Name,
FactoryName = config.FactoryName,
RavenEntityName = config.RavenEntityName,
Script = config.Script,
ConnectionString = config.ConnectionString,
ConnectionStringName = config.ConnectionStringName,
ConnectionStringSettingName = config.ConnectionStringSettingName,
Disabled = config.Disabled,
SqlReplicationTables = new ObservableCollection <SqlReplicationTable>(config.SqlReplicationTables)
});
}
private bool ReplicateChangesToDestination(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs)
{
var scriptResult = ApplyConversionScript(cfg, docs);
if (scriptResult.Data.Count == 0)
{
return(true);
}
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
var countOfItems = scriptResult.Data.Sum(x => x.Value.Count);
try
{
using (var writer = new ElasticsearchDestinationWriter(Database, cfg, replicationStats))
{
if (writer.Execute(scriptResult))
{
log.Debug("Replicated changes of {0} for replication {1}", string.Join(", ", docs.Select(d => d.Key)), cfg.Name);
replicationStats.CompleteSuccess(countOfItems);
}
else
{
log.Debug("Replicated changes (with some errors) of {0} for replication {1}", string.Join(", ", docs.Select(d => d.Key)), cfg.Name);
replicationStats.Success(countOfItems);
}
}
return(true);
}
catch (Exception e)
{
log.WarnException("Failure to replicate changes to Elasticsearch for: " + cfg.Name, e);
SqlReplicationStatistics replicationStatistics;
DateTime newTime;
if (statistics.TryGetValue(cfg.Name, out replicationStatistics) == false)
{
newTime = SystemTime.UtcNow.AddSeconds(5);
}
else
{
var totalSeconds = (SystemTime.UtcNow - replicationStatistics.LastErrorTime).TotalSeconds;
newTime = SystemTime.UtcNow.AddSeconds(Math.Max(60 * 15, Math.Min(5, totalSeconds + 5)));
}
replicationStats.RecordWriteError(e, Database, countOfItems, newTime);
return(false);
}
}
public ElasticsearchDestinationWriter(DocumentDatabase database, SqlReplicationConfig _cfg, SqlReplicationStatistics replicationStatistics)
{
var cfg = new ElasticsearchReplicationConfig(_cfg);
this.database = database;
this.cfg = cfg;
this.targetIndexName = cfg.FactoryName.ToLowerInvariant(); // Elasticsearch requires all index names to be lowercased
this.replicationStatistics = replicationStatistics;
try
{
elasticsearchClient = cfg.GetElasticClient();
}
catch (UriFormatException e)
{
if (database != null)
{
database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Invalid Elasticsearch URL provided",
Message = "Elasticsearch Replication could not parse one of the provided node URLs",
UniqueKey = "Elasticsearch Replication Connection Error: " + cfg.ConnectionString
});
}
}
catch (Exception e)
{
if (database != null)
{
database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Elasticsearch Replication could not open connection",
Message = "Elasticsearch Replication could not open connection to " + cfg.ConnectionString,
UniqueKey = "Elasticsearch Replication Connection Error: " + cfg.ConnectionString
});
}
throw;
}
}
private bool HasChanges(SqlReplicationConfigModel local, SqlReplicationConfig remote)
{
if (remote == null)
{
return(false);
}
if (local.RavenEntityName != remote.RavenEntityName)
{
return(true);
}
if (local.Script != remote.Script)
{
return(true);
}
if (local.Disabled != remote.Disabled)
{
return(true);
}
if (local.ConnectionString != remote.ConnectionString)
{
return(true);
}
if (local.ConnectionStringName != remote.ConnectionStringName)
{
return(true);
}
if (local.ConnectionStringSettingName != remote.ConnectionStringSettingName)
{
return(true);
}
if (local.FactoryName != remote.FactoryName)
{
return(true);
}
return(false);
}
private SqlReplicationConfig UpdateConfig(SqlReplicationConfig config, SqlReplicationConfigModel sqlReplicationConfig)
{
if (config == null)
{
return(sqlReplicationConfig.ToSqlReplicationConfig());
}
config.ConnectionString = sqlReplicationConfig.ConnectionString;
config.ConnectionStringName = sqlReplicationConfig.ConnectionStringName;
config.ConnectionStringSettingName = sqlReplicationConfig.ConnectionStringSettingName;
config.Disabled = sqlReplicationConfig.Disabled;
config.FactoryName = sqlReplicationConfig.FactoryName;
config.Id = sqlReplicationConfig.Id;
config.Name = sqlReplicationConfig.Name;
config.RavenEntityName = sqlReplicationConfig.RavenEntityName;
config.Script = sqlReplicationConfig.Script;
config.SqlReplicationTables = new List <SqlReplicationTable>(sqlReplicationConfig.SqlReplicationTables);
return(config);
}
private bool ReplicateDeletionsToDestination(SqlReplicationConfig cfg, IEnumerable <ListItem> deletedDocs)
{
var identifiers = deletedDocs.Select(x => x.Key).ToList();
if (identifiers.Count == 0)
{
return(true);
}
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
using (var writer = new ElasticsearchDestinationWriter(Database, cfg, replicationStats))
{
foreach (var sqlReplicationTable in cfg.SqlReplicationTables)
{
writer.DeleteItems(sqlReplicationTable.TableName, sqlReplicationTable.DocumentKeyColumn, identifiers);
}
writer.Commit();
log.Debug("Replicated deletes of {0} for config {1}", string.Join(", ", identifiers), cfg.Name);
}
return(true);
}
private static string GetSqlReplicationDeletionName(SqlReplicationConfig replicationConfig)
{
return(ElasticsearchReplicationConstants.ElasticsearchReplicationDeletions + replicationConfig.Name);
}
