private static ReplicationStrategy GetReplicationStrategyFromConfiguration(string keyspaceName)
{
if (Configuration.ConfigurationFileExists())
{
var keyspaces = JSON.Parse(Configuration.GetConfiguration()).Linq
.SingleOrDefault(x => x.Key.Equals("keyspaces", StringComparison.OrdinalIgnoreCase)).Value?.Linq
.ToDictionary(x => x.Key, x => x.Value, StringComparer.OrdinalIgnoreCase);
#pragma warning disable CS8620
if (keyspaces?.GetValue(keyspaceName) != null)
{
return(ReplicationStrategy.FromSortedDictionary(new SortedDictionary <string, string>((keyspaces[keyspaceName].Linq.ToDictionary(x => x.Key, x => x.Value.Value)))));
}
else if (keyspaces?.GetValue(Configuration.DefaultKeyspaceKey) != null)
#pragma warning restore CS8620
{
return(ReplicationStrategy.FromSortedDictionary(new SortedDictionary <string, string>((keyspaces[Configuration.DefaultKeyspaceKey].Linq.ToDictionary(x => x.Key, x => x.Value.Value)))));
}
else
{
return(CreateSimpleStrategy(1)); //Default if the the keyspace name is not present and there is no default
}
}
else
{
return(CreateSimpleStrategy(1)); //Default if the file is not present
}
}
private static ReplicationStrategy GetReplicationStrategyFromConfiguration(string keyspaceName)
{
if (Configuration.ConfigurationFileExists())
{
var configuration = Configuration.GetConfiguration()
.FromJson <Dictionary <string, Dictionary <string, object> > >()
.GetValue("keyspaces", new Dictionary <string, object>());
if (configuration.GetValue(keyspaceName) != null)
{
return(ReplicationStrategy.FromSortedDictionary(new SortedDictionary <string, string>((configuration[keyspaceName] as Dictionary <string, object>).ToDictionary(kv => kv.Key, kv => kv.Value as string))));
}
else if (configuration.GetValue(Configuration.DefaultKeyspaceKey) != null)
{
return(ReplicationStrategy.FromSortedDictionary(new SortedDictionary <string, string>((configuration[Configuration.DefaultKeyspaceKey] as Dictionary <string, object>).ToDictionary(kv => kv.Key, kv => kv.Value as string))));
}
else
{
return(CreateSimpleStrategy(1)); //Default if the the keyspace name is not present and there is no default
}
}
else
{
return(CreateSimpleStrategy(1)); //Default if the file is not present
}
}
/// <summary>
/// store or update the file content with stream
/// </summary>
/// <param name="stream">stream</param>
/// <param name="fileName">file name</param>
/// <param name="fid">fid</param>
/// <param name="replicationStrategy">replication strategy</param>
/// <returns>Write Result</returns>
public WriteResult Upload(Stream stream, string fileName = null, string fid = null,
ReplicationStrategy replicationStrategy = ReplicationStrategy.None)
{
var assignResult = _weedCSharpClient.Assign(new AssignParams(replicationStrategy));
return(_weedCSharpClient.Write(assignResult.WeedFSFile, assignResult.Location, stream, fileName));
}
private void ReplicateSingleSideBySideIndex(ReplicationStrategy destination, IndexDefinition indexDefinition, IndexDefinition sideBySideIndexDefinition)
{
var url = $"{destination.ConnectionStringOptions.Url}/replication/side-by-side?{GetDebugInformation()}";
IndexReplaceDocument indexReplaceDocument;
try
{
indexReplaceDocument = Database.Documents.Get(Constants.IndexReplacePrefix + sideBySideIndexDefinition.Name, null).DataAsJson.JsonDeserialization <IndexReplaceDocument>();
}
catch (Exception e)
{
Log.Warn("Cannot get side-by-side index replacement document. Aborting operation. (this exception should not happen and the cause should be investigated)", e);
return;
}
var sideBySideReplicationInfo = new SideBySideReplicationInfo
{
Index = indexDefinition,
SideBySideIndex = sideBySideIndexDefinition,
OriginDatabaseId = destination.CurrentDatabaseId,
IndexReplaceDocument = indexReplaceDocument
};
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethod.Post, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(sideBySideReplicationInfo));
replicationRequest.ExecuteRequest();
}
public FilterReplicatedDocs(DocumentActions docActions, ReplicationStrategy strategy, PrefetchingBehavior prefetchingBehavior, string destinationId, Etag lastEtag)
{
this.docActions = docActions;
this.strategy = strategy;
this.prefetchingBehavior = prefetchingBehavior;
this.destinationId = destinationId;
this.lastEtag = lastEtag;
}
private void ReplicateIndexesMultiPut(ReplicationStrategy destination, List <IndexToAdd> indexesToAdd)
{
var serializedIndexDefinitions = RavenJToken.FromObject(indexesToAdd.ToArray());
var url = $"{destination.ConnectionStringOptions.Url}/indexes?{GetDebugInformation()}";
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Put, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(serializedIndexDefinitions);
replicationRequest.ExecuteRequest();
}
public MeterMetric GetReplicationBatchSizeMetric(ReplicationStrategy destination)
{
return(ReplicationBatchSizeMeter.GetOrAdd(destination.ConnectionStringOptions.Url,
s =>
{
var newMetric = dbMetrics.Meter("metrics", "docs/min for " + s, "Replication docs/min Counter", TimeUnit.Minutes);
MetricsTicker.Instance.AddMeterMetric(newMetric);
return newMetric;
}));
}
public static CassandraKeyspace Retreive(string keyspaceName, WrappedConnection wrappedConnection)
{
var replication = wrappedConnection.Query <SortedDictionary <string, string> >($"select replication from system_schema.keyspaces where keyspace_name = '{keyspaceName}'");
if (replication == null || !replication.Any())
{
return(new CassandraKeyspace(keyspaceName, GetReplicationStrategyFromConfiguration(keyspaceName), wrappedConnection));
}
return(new CassandraKeyspace(keyspaceName, ReplicationStrategy.FromSortedDictionary(replication), wrappedConnection));
}
private DocumentCount GetDocumentsLeftCountForEtl(ReplicationStrategy replicationStrategy, string destinationUrl, string databaseName)
{
var sourcesDocument = replicationTask.GetLastReplicatedEtagFrom(replicationStrategy);
if (sourcesDocument == null)
{
throw new InvalidOperationException($"Couldn't get last replicated etag for destination url: {destinationUrl} and database: {databaseName}");
}
long storageCount = 0;
var earlyExit = new Reference <bool>();
database.TransactionalStorage.Batch(actions =>
{
//get document count since last replicated etag
long skipped = 0;
var destinationId = sourcesDocument.ServerInstanceId.ToString();
storageCount = actions.Documents.GetDocumentIdsAfterEtag(
sourcesDocument.LastDocumentEtag, MaxDocumentsToCheck,
WillDocumentBeReplicated(replicationStrategy, destinationId),
earlyExit, database.WorkContext.CancellationToken).Count();
});
long diffFromIndex = 0;
if (earlyExit.Value)
{
//we couldn't get an accurate document left count from the storage,
//we'll try to get an approximation from the index
var entityNames = replicationStrategy.SpecifiedCollections.Keys.ToHashSet();
var query = QueryBuilder.GetQueryForAllMatchingDocumentsForIndex(database, entityNames);
//get count of documents with specified collections on this server
var localDocumentCount = GetDocumentCountForEntityNames(query);
//get count of documents with specified collections on destination server
var url = $"{replicationStrategy.ConnectionStringOptions.Url}/admin/replication/replicated-docs-by-entity-names";
var request = requestFactory.Create(url, HttpMethods.Post, replicationStrategy.ConnectionStringOptions);
request.Write(query);
var remoteDocumentCount = request.ExecuteRequest <long>();
diffFromIndex = localDocumentCount - remoteDocumentCount;
}
return(new DocumentCount
{
Count = earlyExit.Value == false ? storageCount : Math.Max(storageCount, diffFromIndex),
Type = earlyExit.Value == false ? CountType.Accurate : CountType.Approximate,
IsEtl = true
});
}
private DocumentCount GetDocumentsLeftCount(ReplicationStrategy replicationStrategy,
string destinationUrl, string databaseName)
{
//first check the stats
var localDocumentCount = database.Statistics.CountOfDocuments;
var url = $"{replicationStrategy.ConnectionStringOptions.Url}/stats";
var request = requestFactory.Create(url, HttpMethods.Get, replicationStrategy.ConnectionStringOptions);
var remoteDocumentCount = request.ExecuteRequest <DatabaseStatistics>().CountOfDocuments;
var difference = localDocumentCount - remoteDocumentCount;
if (difference > MaxDocumentsToCheck)
{
return(new DocumentCount
{
Count = difference,
Type = CountType.Approximate,
IsEtl = false
});
}
var sourcesDocument = replicationTask.GetLastReplicatedEtagFrom(replicationStrategy);
if (sourcesDocument == null)
{
throw new InvalidOperationException($"Couldn't get last replicated etag for destination url: {destinationUrl} and database: {databaseName}");
}
long count = 0;
var earlyExit = new Reference <bool>();
database.TransactionalStorage.Batch(actions =>
{
//get document count since last replicated etag
var destinationId = sourcesDocument.ServerInstanceId.ToString();
count = actions.Documents.GetDocumentIdsAfterEtag(
sourcesDocument.LastDocumentEtag, MaxDocumentsToCheck,
WillDocumentBeReplicated(replicationStrategy, destinationId), earlyExit,
database.WorkContext.CancellationToken).Count();
});
return(new DocumentCount
{
Count = earlyExit.Value == false ? count : Math.Max(count, difference),
Type = earlyExit.Value == false ? CountType.Accurate : CountType.Approximate,
IsEtl = false
});
}
private void ReplicateIndexDeletionIfNeeded(
List <JsonDocument> indexTombstones,
ReplicationStrategy destination,
Dictionary <string, int> replicatedIndexTombstones)
{
if (indexTombstones.Count == 0)
{
return;
}
foreach (var tombstone in indexTombstones)
{
try
{
int value;
//In case the index was recreated under the same name we will increase the destination count for this tombstone
//As if we sent the delete request but without actually sending the request, ending with a NOOP and deleting the index tombstone.
if (Database.IndexStorage.HasIndex(tombstone.Key))
{
replicatedIndexTombstones.TryGetValue(tombstone.Key, out value);
replicatedIndexTombstones[tombstone.Key] = value + 1;
continue;
}
var url = string.Format("{0}/indexes/{1}?{2}&{3}",
destination.ConnectionStringOptions.Url,
Uri.EscapeUriString(tombstone.Key),
GetTombstoneVersion(tombstone, IndexDefinitionStorage.IndexVersionKey, Constants.IndexVersion),
GetDebugInformation());
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Delete, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(EmptyRequestBody));
replicationRequest.ExecuteRequest();
Log.Info("Replicated index deletion (index name = {0})", tombstone.Key);
replicatedIndexTombstones.TryGetValue(tombstone.Key, out value);
replicatedIndexTombstones[tombstone.Key] = value + 1;
}
catch (Exception e)
{
Replication.HandleRequestBufferingErrors(e, destination);
Log.ErrorException(string.Format("Failed to replicate index deletion (index name = {0})", tombstone.Key), e);
}
}
}
private Func <string, RavenJObject, Func <JsonDocument>, bool> WillDocumentBeReplicated(
ReplicationStrategy strategy, string destinationId, Action skippedAction = null)
{
return((key, metadata, getDocument) =>
{
string _;
if (strategy.FilterDocuments(destinationId, key, metadata, out _) == false)
{
skippedAction?.Invoke();
return false;
}
if (strategy.IsETL == false)
{
//not ETL replication
return true;
}
var collection = metadata.Value <string>(Constants.RavenEntityName);
string script;
if (string.IsNullOrEmpty(collection) ||
strategy.SpecifiedCollections.TryGetValue(collection, out script) == false)
{
//document has no collection or this collection will not be replicated
skippedAction?.Invoke();
return false;
}
if (string.IsNullOrEmpty(script) || metadata.ContainsKey(Constants.RavenDeleteMarker))
{
//no filter script or this document is deleted
return true;
}
if (WillReplicateByScript(script, getDocument()) == false)
{
skippedAction?.Invoke();
return false;
}
return true;
});
}
private void ReplicateSingleTransformer(ReplicationStrategy destination, TransformerDefinition definition)
{
try
{
var clonedTransformer = definition.Clone();
clonedTransformer.TransfomerId = 0;
var url = destination.ConnectionStringOptions.Url + "/transformers/" + Uri.EscapeUriString(definition.Name) + "?" + GetDebugInformation();
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethod.Put, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(clonedTransformer));
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
Replication.HandleRequestBufferingErrors(e, destination);
Log.WarnException("Could not replicate transformer " + definition.Name + " to " + destination.ConnectionStringOptions.Url, e);
}
}
private static ReplicationStrategy GetReplicationStrategyFromConfiguration(string keyspaceName)
{
if (Configuration.ConfigurationFileExists())
{
var configuration = JObject.Parse(Configuration.GetConfiguration())["keyspaces"];
if (configuration[keyspaceName] != null)
{
return(ReplicationStrategy.FromSortedDictionary(configuration[keyspaceName].ToObject <SortedDictionary <string, string> >()));
}
else if (configuration[Configuration.DefaultKeyspaceKey] != null)
{
return(ReplicationStrategy.FromSortedDictionary(configuration[Configuration.DefaultKeyspaceKey].ToObject <SortedDictionary <string, string> >()));
}
else
{
return(CreateSimpleStrategy(1)); //Default if the the keyspace name is not present and there is no default
}
}
else
{
return(CreateSimpleStrategy(1)); //Default if the file is not present
}
}
public HistogramMetric GetReplicationBatchSizeHistogram(ReplicationStrategy destination)
{
return(ReplicationBatchSizeHistogram.GetOrAdd(destination.ConnectionStringOptions.Url,
s => dbMetrics.Histogram("metrics", "Replication docs/min Histogram")));
}
public FilterAndTransformSpecifiedCollections(DocumentDatabase database, ReplicationStrategy strategy, string destinationId)
{
this.database = database;
this.strategy = strategy;
this.destinationId = destinationId;
}
public WriteResult Upload(Stream stream, string fileName = null, string fid = null,
ReplicationStrategy replicationStrategy = ReplicationStrategy.None)
{
return(WeedCSharpClientSubject.Instance.Upload(stream, fileName, fid, replicationStrategy));
}
public abstract TData GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination);
public CassandraKeyspace(string keyspaceName, ReplicationStrategy replicationStrategy, WrappedConnection wrappedConnection)
: base(keyspaceName, wrappedConnection)
{
_replicationStrategy = replicationStrategy ?? throw new ArgumentNullException(nameof(replicationStrategy));
}
private ReplicatonNodeState CheckDestinationConnectionState(ReplicationStrategy destination)
{
return(CheckConnectionState(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions));
}
public override TimeTicks GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination)
{
var sourceReplicationInformationWithBatchInformation = task.GetLastReplicatedEtagFrom(destination);
if (sourceReplicationInformationWithBatchInformation == null)
{
return(null);
}
if (sourceReplicationInformationWithBatchInformation.LastModified.HasValue == false)
{
return(null);
}
return(new TimeTicks(SystemTime.UtcNow - sourceReplicationInformationWithBatchInformation.LastModified.Value));
}
public MeterMetric GetReplicationDurationMetric(ReplicationStrategy destination)
{
return(ReplicationDurationMeter.GetOrAdd(destination.ConnectionStringOptions.Url,
s => dbMetrics.Meter("metrics", "duration", "Replication duration Counter", TimeUnit.Minutes)));
}
public HistogramMetric GetReplicationDurationHistogram(ReplicationStrategy destination)
{
return(ReplicationDurationHistogram.GetOrAdd(destination.ConnectionStringOptions.Url,
s => dbMetrics.Histogram("metrics", "Replication duration Histogram for " + s)));
}
public override OctetString GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination)
{
throw new NotSupportedException();
}
private void ReplicateTransformerDeletionIfNeeded(List <JsonDocument> transformerTombstones, ReplicationStrategy destination, Dictionary <string, int> replicatedTransformerTombstones)
{
if (transformerTombstones.Count == 0)
{
return;
}
foreach (var tombstone in transformerTombstones)
{
try
{
int value;
if (Database.Transformers.GetTransformerDefinition(tombstone.Key) != null) //if in the meantime the transformer was recreated under the same name
{
replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value);
replicatedTransformerTombstones[tombstone.Key] = value + 1;
continue;
}
var url = string.Format("{0}/transformers/{1}?{2}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(tombstone.Key), GetDebugInformation());
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Delete, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(EmptyRequestBody));
replicationRequest.ExecuteRequest();
Log.Info("Replicated transformer deletion (transformer name = {0})", tombstone.Key);
replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value);
replicatedTransformerTombstones[tombstone.Key] = value + 1;
}
catch (Exception e)
{
Replication.HandleRequestBufferingErrors(e, destination);
Log.ErrorException(string.Format("Failed to replicate transformer deletion (transformer name = {0})", tombstone.Key), e);
}
}
}
public MeterMetric GetReplicationBatchSizeMetric(ReplicationStrategy destination)
{
return(ReplicationBatchSizeMeter.GetOrAdd(destination.ConnectionStringOptions.Url,
s => dbMetrics.Meter("metrics", "docs/min for " + s, "Replication docs/min Counter", TimeUnit.Minutes)));
}
public object Clone()
{
Cluster cluster = new Cluster();
cluster.OpTimeout = OpTimeout;
cluster.StatsRepInterval = StatsRepInterval;
cluster.ReplicationStrategy = ReplicationStrategy != null ? (Alachisoft.NCache.Config.Dom.ReplicationStrategy)ReplicationStrategy.Clone() : null;
cluster.Channel = Channel != null ? (Channel)Channel.Clone() : null;
return(cluster);
}
public ConcurrentQueue <ReplicationPerformanceStats> GetReplicationPerformanceStats(ReplicationStrategy destination)
{
return(ReplicationPerformanceStats.GetOrAdd(destination.ConnectionStringOptions.Url,
s => new ConcurrentQueue <ReplicationPerformanceStats>()));
}
public override OctetString GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination)
{
return(new OctetString(destination.ConnectionStringOptions.Url.ForDatabase(destination.ConnectionStringOptions.DefaultDatabase)));
}
