private static ReplicationStrategy GetReplicationStrategyFromConfiguration(string keyspaceName) { if (Configuration.ConfigurationFileExists()) { var keyspaces = JSON.Parse(Configuration.GetConfiguration()).Linq .SingleOrDefault(x => x.Key.Equals("keyspaces", StringComparison.OrdinalIgnoreCase)).Value?.Linq .ToDictionary(x => x.Key, x => x.Value, StringComparer.OrdinalIgnoreCase); #pragma warning disable CS8620 if (keyspaces?.GetValue(keyspaceName) != null) { return(ReplicationStrategy.FromSortedDictionary(new SortedDictionary <string, string>((keyspaces[keyspaceName].Linq.ToDictionary(x => x.Key, x => x.Value.Value))))); } else if (keyspaces?.GetValue(Configuration.DefaultKeyspaceKey) != null) #pragma warning restore CS8620 { return(ReplicationStrategy.FromSortedDictionary(new SortedDictionary <string, string>((keyspaces[Configuration.DefaultKeyspaceKey].Linq.ToDictionary(x => x.Key, x => x.Value.Value))))); } else { return(CreateSimpleStrategy(1)); //Default if the the keyspace name is not present and there is no default } } else { return(CreateSimpleStrategy(1)); //Default if the file is not present } }
private static ReplicationStrategy GetReplicationStrategyFromConfiguration(string keyspaceName) { if (Configuration.ConfigurationFileExists()) { var configuration = Configuration.GetConfiguration() .FromJson <Dictionary <string, Dictionary <string, object> > >() .GetValue("keyspaces", new Dictionary <string, object>()); if (configuration.GetValue(keyspaceName) != null) { return(ReplicationStrategy.FromSortedDictionary(new SortedDictionary <string, string>((configuration[keyspaceName] as Dictionary <string, object>).ToDictionary(kv => kv.Key, kv => kv.Value as string)))); } else if (configuration.GetValue(Configuration.DefaultKeyspaceKey) != null) { return(ReplicationStrategy.FromSortedDictionary(new SortedDictionary <string, string>((configuration[Configuration.DefaultKeyspaceKey] as Dictionary <string, object>).ToDictionary(kv => kv.Key, kv => kv.Value as string)))); } else { return(CreateSimpleStrategy(1)); //Default if the the keyspace name is not present and there is no default } } else { return(CreateSimpleStrategy(1)); //Default if the file is not present } }
/// <summary> /// store or update the file content with stream /// </summary> /// <param name="stream">stream</param> /// <param name="fileName">file name</param> /// <param name="fid">fid</param> /// <param name="replicationStrategy">replication strategy</param> /// <returns>Write Result</returns> public WriteResult Upload(Stream stream, string fileName = null, string fid = null, ReplicationStrategy replicationStrategy = ReplicationStrategy.None) { var assignResult = _weedCSharpClient.Assign(new AssignParams(replicationStrategy)); return(_weedCSharpClient.Write(assignResult.WeedFSFile, assignResult.Location, stream, fileName)); }
private void ReplicateSingleSideBySideIndex(ReplicationStrategy destination, IndexDefinition indexDefinition, IndexDefinition sideBySideIndexDefinition) { var url = $"{destination.ConnectionStringOptions.Url}/replication/side-by-side?{GetDebugInformation()}"; IndexReplaceDocument indexReplaceDocument; try { indexReplaceDocument = Database.Documents.Get(Constants.IndexReplacePrefix + sideBySideIndexDefinition.Name, null).DataAsJson.JsonDeserialization <IndexReplaceDocument>(); } catch (Exception e) { Log.Warn("Cannot get side-by-side index replacement document. Aborting operation. (this exception should not happen and the cause should be investigated)", e); return; } var sideBySideReplicationInfo = new SideBySideReplicationInfo { Index = indexDefinition, SideBySideIndex = sideBySideIndexDefinition, OriginDatabaseId = destination.CurrentDatabaseId, IndexReplaceDocument = indexReplaceDocument }; var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethod.Post, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(sideBySideReplicationInfo)); replicationRequest.ExecuteRequest(); }
public FilterReplicatedDocs(DocumentActions docActions, ReplicationStrategy strategy, PrefetchingBehavior prefetchingBehavior, string destinationId, Etag lastEtag) { this.docActions = docActions; this.strategy = strategy; this.prefetchingBehavior = prefetchingBehavior; this.destinationId = destinationId; this.lastEtag = lastEtag; }
private void ReplicateIndexesMultiPut(ReplicationStrategy destination, List <IndexToAdd> indexesToAdd) { var serializedIndexDefinitions = RavenJToken.FromObject(indexesToAdd.ToArray()); var url = $"{destination.ConnectionStringOptions.Url}/indexes?{GetDebugInformation()}"; var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Put, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination)); replicationRequest.Write(serializedIndexDefinitions); replicationRequest.ExecuteRequest(); }
public MeterMetric GetReplicationBatchSizeMetric(ReplicationStrategy destination) { return(ReplicationBatchSizeMeter.GetOrAdd(destination.ConnectionStringOptions.Url, s => { var newMetric = dbMetrics.Meter("metrics", "docs/min for " + s, "Replication docs/min Counter", TimeUnit.Minutes); MetricsTicker.Instance.AddMeterMetric(newMetric); return newMetric; })); }
public static CassandraKeyspace Retreive(string keyspaceName, WrappedConnection wrappedConnection) { var replication = wrappedConnection.Query <SortedDictionary <string, string> >($"select replication from system_schema.keyspaces where keyspace_name = '{keyspaceName}'"); if (replication == null || !replication.Any()) { return(new CassandraKeyspace(keyspaceName, GetReplicationStrategyFromConfiguration(keyspaceName), wrappedConnection)); } return(new CassandraKeyspace(keyspaceName, ReplicationStrategy.FromSortedDictionary(replication), wrappedConnection)); }
private DocumentCount GetDocumentsLeftCountForEtl(ReplicationStrategy replicationStrategy, string destinationUrl, string databaseName) { var sourcesDocument = replicationTask.GetLastReplicatedEtagFrom(replicationStrategy); if (sourcesDocument == null) { throw new InvalidOperationException($"Couldn't get last replicated etag for destination url: {destinationUrl} and database: {databaseName}"); } long storageCount = 0; var earlyExit = new Reference <bool>(); database.TransactionalStorage.Batch(actions => { //get document count since last replicated etag long skipped = 0; var destinationId = sourcesDocument.ServerInstanceId.ToString(); storageCount = actions.Documents.GetDocumentIdsAfterEtag( sourcesDocument.LastDocumentEtag, MaxDocumentsToCheck, WillDocumentBeReplicated(replicationStrategy, destinationId), earlyExit, database.WorkContext.CancellationToken).Count(); }); long diffFromIndex = 0; if (earlyExit.Value) { //we couldn't get an accurate document left count from the storage, //we'll try to get an approximation from the index var entityNames = replicationStrategy.SpecifiedCollections.Keys.ToHashSet(); var query = QueryBuilder.GetQueryForAllMatchingDocumentsForIndex(database, entityNames); //get count of documents with specified collections on this server var localDocumentCount = GetDocumentCountForEntityNames(query); //get count of documents with specified collections on destination server var url = $"{replicationStrategy.ConnectionStringOptions.Url}/admin/replication/replicated-docs-by-entity-names"; var request = requestFactory.Create(url, HttpMethods.Post, replicationStrategy.ConnectionStringOptions); request.Write(query); var remoteDocumentCount = request.ExecuteRequest <long>(); diffFromIndex = localDocumentCount - remoteDocumentCount; } return(new DocumentCount { Count = earlyExit.Value == false ? storageCount : Math.Max(storageCount, diffFromIndex), Type = earlyExit.Value == false ? CountType.Accurate : CountType.Approximate, IsEtl = true }); }
private DocumentCount GetDocumentsLeftCount(ReplicationStrategy replicationStrategy, string destinationUrl, string databaseName) { //first check the stats var localDocumentCount = database.Statistics.CountOfDocuments; var url = $"{replicationStrategy.ConnectionStringOptions.Url}/stats"; var request = requestFactory.Create(url, HttpMethods.Get, replicationStrategy.ConnectionStringOptions); var remoteDocumentCount = request.ExecuteRequest <DatabaseStatistics>().CountOfDocuments; var difference = localDocumentCount - remoteDocumentCount; if (difference > MaxDocumentsToCheck) { return(new DocumentCount { Count = difference, Type = CountType.Approximate, IsEtl = false }); } var sourcesDocument = replicationTask.GetLastReplicatedEtagFrom(replicationStrategy); if (sourcesDocument == null) { throw new InvalidOperationException($"Couldn't get last replicated etag for destination url: {destinationUrl} and database: {databaseName}"); } long count = 0; var earlyExit = new Reference <bool>(); database.TransactionalStorage.Batch(actions => { //get document count since last replicated etag var destinationId = sourcesDocument.ServerInstanceId.ToString(); count = actions.Documents.GetDocumentIdsAfterEtag( sourcesDocument.LastDocumentEtag, MaxDocumentsToCheck, WillDocumentBeReplicated(replicationStrategy, destinationId), earlyExit, database.WorkContext.CancellationToken).Count(); }); return(new DocumentCount { Count = earlyExit.Value == false ? count : Math.Max(count, difference), Type = earlyExit.Value == false ? CountType.Accurate : CountType.Approximate, IsEtl = false }); }
private void ReplicateIndexDeletionIfNeeded( List <JsonDocument> indexTombstones, ReplicationStrategy destination, Dictionary <string, int> replicatedIndexTombstones) { if (indexTombstones.Count == 0) { return; } foreach (var tombstone in indexTombstones) { try { int value; //In case the index was recreated under the same name we will increase the destination count for this tombstone //As if we sent the delete request but without actually sending the request, ending with a NOOP and deleting the index tombstone. if (Database.IndexStorage.HasIndex(tombstone.Key)) { replicatedIndexTombstones.TryGetValue(tombstone.Key, out value); replicatedIndexTombstones[tombstone.Key] = value + 1; continue; } var url = string.Format("{0}/indexes/{1}?{2}&{3}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(tombstone.Key), GetTombstoneVersion(tombstone, IndexDefinitionStorage.IndexVersionKey, Constants.IndexVersion), GetDebugInformation()); var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Delete, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(EmptyRequestBody)); replicationRequest.ExecuteRequest(); Log.Info("Replicated index deletion (index name = {0})", tombstone.Key); replicatedIndexTombstones.TryGetValue(tombstone.Key, out value); replicatedIndexTombstones[tombstone.Key] = value + 1; } catch (Exception e) { Replication.HandleRequestBufferingErrors(e, destination); Log.ErrorException(string.Format("Failed to replicate index deletion (index name = {0})", tombstone.Key), e); } } }
private Func <string, RavenJObject, Func <JsonDocument>, bool> WillDocumentBeReplicated( ReplicationStrategy strategy, string destinationId, Action skippedAction = null) { return((key, metadata, getDocument) => { string _; if (strategy.FilterDocuments(destinationId, key, metadata, out _) == false) { skippedAction?.Invoke(); return false; } if (strategy.IsETL == false) { //not ETL replication return true; } var collection = metadata.Value <string>(Constants.RavenEntityName); string script; if (string.IsNullOrEmpty(collection) || strategy.SpecifiedCollections.TryGetValue(collection, out script) == false) { //document has no collection or this collection will not be replicated skippedAction?.Invoke(); return false; } if (string.IsNullOrEmpty(script) || metadata.ContainsKey(Constants.RavenDeleteMarker)) { //no filter script or this document is deleted return true; } if (WillReplicateByScript(script, getDocument()) == false) { skippedAction?.Invoke(); return false; } return true; }); }
private void ReplicateSingleTransformer(ReplicationStrategy destination, TransformerDefinition definition) { try { var clonedTransformer = definition.Clone(); clonedTransformer.TransfomerId = 0; var url = destination.ConnectionStringOptions.Url + "/transformers/" + Uri.EscapeUriString(definition.Name) + "?" + GetDebugInformation(); var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethod.Put, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(clonedTransformer)); replicationRequest.ExecuteRequest(); } catch (Exception e) { Replication.HandleRequestBufferingErrors(e, destination); Log.WarnException("Could not replicate transformer " + definition.Name + " to " + destination.ConnectionStringOptions.Url, e); } }
private static ReplicationStrategy GetReplicationStrategyFromConfiguration(string keyspaceName) { if (Configuration.ConfigurationFileExists()) { var configuration = JObject.Parse(Configuration.GetConfiguration())["keyspaces"]; if (configuration[keyspaceName] != null) { return(ReplicationStrategy.FromSortedDictionary(configuration[keyspaceName].ToObject <SortedDictionary <string, string> >())); } else if (configuration[Configuration.DefaultKeyspaceKey] != null) { return(ReplicationStrategy.FromSortedDictionary(configuration[Configuration.DefaultKeyspaceKey].ToObject <SortedDictionary <string, string> >())); } else { return(CreateSimpleStrategy(1)); //Default if the the keyspace name is not present and there is no default } } else { return(CreateSimpleStrategy(1)); //Default if the file is not present } }
public HistogramMetric GetReplicationBatchSizeHistogram(ReplicationStrategy destination) { return(ReplicationBatchSizeHistogram.GetOrAdd(destination.ConnectionStringOptions.Url, s => dbMetrics.Histogram("metrics", "Replication docs/min Histogram"))); }
public FilterAndTransformSpecifiedCollections(DocumentDatabase database, ReplicationStrategy strategy, string destinationId) { this.database = database; this.strategy = strategy; this.destinationId = destinationId; }
public WriteResult Upload(Stream stream, string fileName = null, string fid = null, ReplicationStrategy replicationStrategy = ReplicationStrategy.None) { return(WeedCSharpClientSubject.Instance.Upload(stream, fileName, fid, replicationStrategy)); }
public abstract TData GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination);
public CassandraKeyspace(string keyspaceName, ReplicationStrategy replicationStrategy, WrappedConnection wrappedConnection) : base(keyspaceName, wrappedConnection) { _replicationStrategy = replicationStrategy ?? throw new ArgumentNullException(nameof(replicationStrategy)); }
private ReplicatonNodeState CheckDestinationConnectionState(ReplicationStrategy destination) { return(CheckConnectionState(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions)); }
public override TimeTicks GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination) { var sourceReplicationInformationWithBatchInformation = task.GetLastReplicatedEtagFrom(destination); if (sourceReplicationInformationWithBatchInformation == null) { return(null); } if (sourceReplicationInformationWithBatchInformation.LastModified.HasValue == false) { return(null); } return(new TimeTicks(SystemTime.UtcNow - sourceReplicationInformationWithBatchInformation.LastModified.Value)); }
public MeterMetric GetReplicationDurationMetric(ReplicationStrategy destination) { return(ReplicationDurationMeter.GetOrAdd(destination.ConnectionStringOptions.Url, s => dbMetrics.Meter("metrics", "duration", "Replication duration Counter", TimeUnit.Minutes))); }
public HistogramMetric GetReplicationDurationHistogram(ReplicationStrategy destination) { return(ReplicationDurationHistogram.GetOrAdd(destination.ConnectionStringOptions.Url, s => dbMetrics.Histogram("metrics", "Replication duration Histogram for " + s))); }
public override OctetString GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination) { throw new NotSupportedException(); }
private void ReplicateTransformerDeletionIfNeeded(List <JsonDocument> transformerTombstones, ReplicationStrategy destination, Dictionary <string, int> replicatedTransformerTombstones) { if (transformerTombstones.Count == 0) { return; } foreach (var tombstone in transformerTombstones) { try { int value; if (Database.Transformers.GetTransformerDefinition(tombstone.Key) != null) //if in the meantime the transformer was recreated under the same name { replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value); replicatedTransformerTombstones[tombstone.Key] = value + 1; continue; } var url = string.Format("{0}/transformers/{1}?{2}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(tombstone.Key), GetDebugInformation()); var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Delete, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(EmptyRequestBody)); replicationRequest.ExecuteRequest(); Log.Info("Replicated transformer deletion (transformer name = {0})", tombstone.Key); replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value); replicatedTransformerTombstones[tombstone.Key] = value + 1; } catch (Exception e) { Replication.HandleRequestBufferingErrors(e, destination); Log.ErrorException(string.Format("Failed to replicate transformer deletion (transformer name = {0})", tombstone.Key), e); } } }
public MeterMetric GetReplicationBatchSizeMetric(ReplicationStrategy destination) { return(ReplicationBatchSizeMeter.GetOrAdd(destination.ConnectionStringOptions.Url, s => dbMetrics.Meter("metrics", "docs/min for " + s, "Replication docs/min Counter", TimeUnit.Minutes))); }
public object Clone() { Cluster cluster = new Cluster(); cluster.OpTimeout = OpTimeout; cluster.StatsRepInterval = StatsRepInterval; cluster.ReplicationStrategy = ReplicationStrategy != null ? (Alachisoft.NCache.Config.Dom.ReplicationStrategy)ReplicationStrategy.Clone() : null; cluster.Channel = Channel != null ? (Channel)Channel.Clone() : null; return(cluster); }
public ConcurrentQueue <ReplicationPerformanceStats> GetReplicationPerformanceStats(ReplicationStrategy destination) { return(ReplicationPerformanceStats.GetOrAdd(destination.ConnectionStringOptions.Url, s => new ConcurrentQueue <ReplicationPerformanceStats>())); }
public override OctetString GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination) { return(new OctetString(destination.ConnectionStringOptions.Url.ForDatabase(destination.ConnectionStringOptions.DefaultDatabase))); }