public SqlReplicationConfig ToSqlReplicationConfig() { var result = new SqlReplicationConfig { Id = Id, Name = Name, FactoryName = FactoryName, RavenEntityName = RavenEntityName, Script = Script, Disabled = disabled, SqlReplicationTables = new List <SqlReplicationTable>(sqlReplicationTables) }; if (string.IsNullOrWhiteSpace(ConnectionString) == false) { result.ConnectionString = ConnectionString; } else if (string.IsNullOrWhiteSpace(ConnectionStringName) == false) { result.ConnectionStringName = connectionStringName; } else { result.ConnectionStringSettingName = connectionStringSettingName; } return(result); }
public ElasticsearchReplicationConfig(SqlReplicationConfig other) { this.ConnectionString = other.ConnectionString; this.FactoryName = other.FactoryName; this.Script = other.Script; this.Disabled = other.Disabled; this.Id = other.Id; this.Name = other.Name; this.RavenEntityName = other.RavenEntityName; this.SqlReplicationTables = other.SqlReplicationTables; }
private Etag GetLastEtagFor(SqlReplicationStatus replicationStatus, SqlReplicationConfig sqlReplicationConfig) { var lastEtag = Etag.Empty; var lastEtagHolder = replicationStatus.LastReplicatedEtags.FirstOrDefault( x => string.Equals(sqlReplicationConfig.Name, x.Name, StringComparison.InvariantCultureIgnoreCase)); if (lastEtagHolder != null) { lastEtag = lastEtagHolder.LastDocEtag; } return(lastEtag); }
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs) { var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name)); var result = new ConversionScriptResult(); foreach (var jsonDocument in docs) { Database.WorkContext.CancellationToken.ThrowIfCancellationRequested(); if (string.IsNullOrEmpty(cfg.RavenEntityName) == false) { var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName); if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false) { continue; } } var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key); try { DocumentRetriever.EnsureIdInMetadata(jsonDocument); var document = jsonDocument.ToJson(); document[Constants.DocumentIdFieldName] = jsonDocument.Key; patcher.Apply(document, new ScriptedPatchRequest { Script = cfg.Script }, jsonDocument.SerializedSizeOnDisk); if (log.IsDebugEnabled && patcher.Debug.Count > 0) { log.Debug("Debug output for doc: {0} for script {1}:\r\n.{2}", jsonDocument.Key, cfg.Name, string.Join("\r\n", patcher.Debug)); patcher.Debug.Clear(); } replicationStats.ScriptSuccess(); } catch (ParseException e) { replicationStats.MarkScriptAsInvalid(Database, cfg.Script); log.WarnException("Could not parse Elasticsearch Replication script for " + cfg.Name, e); return(result); } catch (Exception e) { replicationStats.RecordScriptError(Database); log.WarnException("Could not process Elasticsearch Replication script for " + cfg.Name + ", skipping document: " + jsonDocument.Key, e); } } return(result); }
public static SqlReplicationConfigModel FromSqlReplicationConfig(SqlReplicationConfig config) { return(new SqlReplicationConfigModel { Id = config.Id, Name = config.Name, FactoryName = config.FactoryName, RavenEntityName = config.RavenEntityName, Script = config.Script, ConnectionString = config.ConnectionString, ConnectionStringName = config.ConnectionStringName, ConnectionStringSettingName = config.ConnectionStringSettingName, Disabled = config.Disabled, SqlReplicationTables = new ObservableCollection <SqlReplicationTable>(config.SqlReplicationTables) }); }
private bool ReplicateChangesToDestination(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs) { var scriptResult = ApplyConversionScript(cfg, docs); if (scriptResult.Data.Count == 0) { return(true); } var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name)); var countOfItems = scriptResult.Data.Sum(x => x.Value.Count); try { using (var writer = new ElasticsearchDestinationWriter(Database, cfg, replicationStats)) { if (writer.Execute(scriptResult)) { log.Debug("Replicated changes of {0} for replication {1}", string.Join(", ", docs.Select(d => d.Key)), cfg.Name); replicationStats.CompleteSuccess(countOfItems); } else { log.Debug("Replicated changes (with some errors) of {0} for replication {1}", string.Join(", ", docs.Select(d => d.Key)), cfg.Name); replicationStats.Success(countOfItems); } } return(true); } catch (Exception e) { log.WarnException("Failure to replicate changes to Elasticsearch for: " + cfg.Name, e); SqlReplicationStatistics replicationStatistics; DateTime newTime; if (statistics.TryGetValue(cfg.Name, out replicationStatistics) == false) { newTime = SystemTime.UtcNow.AddSeconds(5); } else { var totalSeconds = (SystemTime.UtcNow - replicationStatistics.LastErrorTime).TotalSeconds; newTime = SystemTime.UtcNow.AddSeconds(Math.Max(60 * 15, Math.Min(5, totalSeconds + 5))); } replicationStats.RecordWriteError(e, Database, countOfItems, newTime); return(false); } }
public ElasticsearchDestinationWriter(DocumentDatabase database, SqlReplicationConfig _cfg, SqlReplicationStatistics replicationStatistics) { var cfg = new ElasticsearchReplicationConfig(_cfg); this.database = database; this.cfg = cfg; this.targetIndexName = cfg.FactoryName.ToLowerInvariant(); // Elasticsearch requires all index names to be lowercased this.replicationStatistics = replicationStatistics; try { elasticsearchClient = cfg.GetElasticClient(); } catch (UriFormatException e) { if (database != null) { database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Invalid Elasticsearch URL provided", Message = "Elasticsearch Replication could not parse one of the provided node URLs", UniqueKey = "Elasticsearch Replication Connection Error: " + cfg.ConnectionString }); } } catch (Exception e) { if (database != null) { database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Elasticsearch Replication could not open connection", Message = "Elasticsearch Replication could not open connection to " + cfg.ConnectionString, UniqueKey = "Elasticsearch Replication Connection Error: " + cfg.ConnectionString }); } throw; } }
private bool HasChanges(SqlReplicationConfigModel local, SqlReplicationConfig remote) { if (remote == null) { return(false); } if (local.RavenEntityName != remote.RavenEntityName) { return(true); } if (local.Script != remote.Script) { return(true); } if (local.Disabled != remote.Disabled) { return(true); } if (local.ConnectionString != remote.ConnectionString) { return(true); } if (local.ConnectionStringName != remote.ConnectionStringName) { return(true); } if (local.ConnectionStringSettingName != remote.ConnectionStringSettingName) { return(true); } if (local.FactoryName != remote.FactoryName) { return(true); } return(false); }
private SqlReplicationConfig UpdateConfig(SqlReplicationConfig config, SqlReplicationConfigModel sqlReplicationConfig) { if (config == null) { return(sqlReplicationConfig.ToSqlReplicationConfig()); } config.ConnectionString = sqlReplicationConfig.ConnectionString; config.ConnectionStringName = sqlReplicationConfig.ConnectionStringName; config.ConnectionStringSettingName = sqlReplicationConfig.ConnectionStringSettingName; config.Disabled = sqlReplicationConfig.Disabled; config.FactoryName = sqlReplicationConfig.FactoryName; config.Id = sqlReplicationConfig.Id; config.Name = sqlReplicationConfig.Name; config.RavenEntityName = sqlReplicationConfig.RavenEntityName; config.Script = sqlReplicationConfig.Script; config.SqlReplicationTables = new List <SqlReplicationTable>(sqlReplicationConfig.SqlReplicationTables); return(config); }
private bool ReplicateDeletionsToDestination(SqlReplicationConfig cfg, IEnumerable <ListItem> deletedDocs) { var identifiers = deletedDocs.Select(x => x.Key).ToList(); if (identifiers.Count == 0) { return(true); } var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name)); using (var writer = new ElasticsearchDestinationWriter(Database, cfg, replicationStats)) { foreach (var sqlReplicationTable in cfg.SqlReplicationTables) { writer.DeleteItems(sqlReplicationTable.TableName, sqlReplicationTable.DocumentKeyColumn, identifiers); } writer.Commit(); log.Debug("Replicated deletes of {0} for config {1}", string.Join(", ", identifiers), cfg.Name); } return(true); }
private static string GetSqlReplicationDeletionName(SqlReplicationConfig replicationConfig) { return(ElasticsearchReplicationConstants.ElasticsearchReplicationDeletions + replicationConfig.Name); }