public JArray GetDocuments(int start, int pageSize, Guid?etag) { var list = new JArray(); TransactionalStorage.Batch(actions => { IEnumerable <JsonDocument> documents; if (etag == null) { documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start); } else { documents = actions.Documents.GetDocumentsAfter(etag.Value); } var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents.Take(pageSize)) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) { continue; } list.Add(document.ToJson()); } }); return(list); }
private List <JsonDocument> GetJsonDocsFromDisk(Guid etag) { List <JsonDocument> jsonDocs = null; var untilEtag = GetNextEtagInMemory(); context.TransactionalStorage.Batch(actions => { jsonDocs = actions.Documents .GetDocumentsAfter( etag, autoTuner.NumberOfItemsToIndexInSingleBatch, autoTuner.MaximumSizeAllowedToFetchFromStorage, untilEtag: untilEtag) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .ToList(); }); if (untilEtag == null) { MaybeAddFutureBatch(jsonDocs); } return(jsonDocs); }
private List <JsonDocument> GetJsonDocsFromDisk(Etag etag, Etag untilEtag) { List <JsonDocument> jsonDocs = null; context.TransactionalStorage.Batch(actions => { jsonDocs = actions.Documents .GetDocumentsAfter( etag, autoTuner.NumberOfItemsToIndexInSingleBatch, context.CancellationToken, maxSize: autoTuner.MaximumSizeAllowedToFetchFromStorage, untilEtag: untilEtag, timeout: autoTuner.FetchingDocumentsFromDiskTimeout) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .ToList(); }); if (untilEtag == null) { MaybeAddFutureBatch(jsonDocs); } return(jsonDocs); }
public void AfterCommit(JsonDocument[] docs) { if (context.Configuration.DisableDocumentPreFetchingForIndexing) { return; } if (futureIndexBatches.Count > 512 || // this is optimization, and we need to make sure we don't overuse memory docs.Length == 0) { return; } foreach (var doc in docs) { DocumentRetriever.EnsureIdInMetadata(doc); } futureIndexBatches.Add(new FutureIndexBatch { StartingEtag = GetLowestEtag(docs), Task = new CompletedTask <JsonResults>(new JsonResults { Results = docs, LoadedFromDisk = false }), Age = Interlocked.Increment(ref currentIndexingAge) }); }
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource) { RavenJArray jsonDocuments = null; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag) .Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs .Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source .Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Take(100) .Select(x => x.ToJson())); }); } catch (Exception e) { log.Warn("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(jsonDocuments); }
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray jsonDocuments = null; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag, 100) .Where(destination.FilterDocuments) .Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(jsonDocuments); }
public JsonDocument Get(string key, TransactionInformation transactionInformation) { if (key == null) { throw new ArgumentNullException("key"); } key = key.Trim(); JsonDocument document = null; if (transactionInformation == null || Database.InFlightTransactionalState.TryGet(key, transactionInformation, out document) == false) { // first we check the dtc state, then the storage, to avoid race conditions var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(transactionInformation, key); TransactionalStorage.Batch(actions => { document = actions.Documents.DocumentByKey(key); }); if (nonAuthoritativeInformationBehavior != null) { document = nonAuthoritativeInformationBehavior(document); } } DocumentRetriever.EnsureIdInMetadata(document); return(new DocumentRetriever(null, Database.ReadTriggers, Database.InFlightTransactionalState) .ExecuteReadTriggers(document, transactionInformation, ReadOperation.Load)); }
private Dictionary <string, List <ItemToReplicate> > ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs) { var dictionary = new Dictionary <string, List <ItemToReplicate> >(); foreach (var jsonDocument in docs) { if (string.IsNullOrEmpty(cfg.RavenEntityName) == false) { var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName); if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false) { continue; } } var patcher = new SqlReplicationScriptedJsonPatcher(Database, dictionary, jsonDocument.Key); try { DocumentRetriever.EnsureIdInMetadata(jsonDocument); jsonDocument.Metadata[Constants.DocumentIdFieldName] = jsonDocument.Key; var document = jsonDocument.ToJson(); patcher.Apply(document, new ScriptedPatchRequest { Script = cfg.Script }); } catch (Exception e) { log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping this document", e); } } return(dictionary); }
private JArray GetJsonDocuments(Guid etag) { JArray jsonDocuments = null; try { var instanceId = docDb.TransactionalStorage.Id.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonDocuments = new JArray(actions.Documents.GetDocumentsAfter(etag) .Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs .Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) == instanceId) // only replicate documents created on this instance .Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Take(100) .Select(x => x.ToJson())); }); } catch (Exception e) { log.Warn("Could not get documents to replicate after: " + etag, e); } return(jsonDocuments); }
public void AfterCommit(JsonDocument[] docs) { if (futureIndexBatches.Count > 512 || // this is optimization, and we need to make sure we don't overuse memory docs.Length == 0) { return; } foreach (var doc in docs) { DocumentRetriever.EnsureIdInMetadata(doc); } futureIndexBatches.Add(new FutureIndexBatch { StartingEtag = DecrementEtag(GetLowestEtag(docs)), Task = new CompletedTask <JsonResults>(new JsonResults { Results = docs, LoadedFromDisk = false }), Age = currentIndexingAge }); }
private void StartGetDocumentOnBackground() { while (run) { DocumentDatabase.TransactionalStorage.Batch(accessor => { var documents = accessor.Documents.GetDocumentsAfter(lastEtagSeen, 128) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .ToArray(); if (documents.Length == 0) { return; } lastEtagSeen = documents.Last().Etag; log.Debug("Docs: {0}", string.Join(", ", documents.Select(x => x.Key))); getDocumentsState.Enqueue(new GetDocumentState(lastEtagSeen, documents.Length)); }); } }
public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation) { if (key == null) { throw new ArgumentNullException("key"); } key = key.Trim(); JsonDocumentMetadata document = null; if (transactionInformation == null || Database.InFlightTransactionalState.TryGet(key, transactionInformation, out document) == false) { var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocumentMetadata>(transactionInformation, key); TransactionalStorage.Batch(actions => { document = actions.Documents.DocumentMetadataByKey(key); }); if (nonAuthoritativeInformationBehavior != null) { document = nonAuthoritativeInformationBehavior(document); } } DocumentRetriever.EnsureIdInMetadata(document); return(new DocumentRetriever(null, Database.ReadTriggers, Database.InFlightTransactionalState) .ProcessReadVetoes(document, transactionInformation, ReadOperation.Load)); }
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs) { var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name)); var result = new ConversionScriptResult(); foreach (var jsonDocument in docs) { Database.WorkContext.CancellationToken.ThrowIfCancellationRequested(); if (string.IsNullOrEmpty(cfg.RavenEntityName) == false) { var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName); if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false) { continue; } } var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key); using (var scope = new SqlReplicationScriptedJsonPatcherOperationScope(Database)) { try { DocumentRetriever.EnsureIdInMetadata(jsonDocument); var document = jsonDocument.ToJson(); document[Constants.DocumentIdFieldName] = jsonDocument.Key; patcher.Apply(scope, document, new ScriptedPatchRequest { Script = cfg.Script }, jsonDocument.SerializedSizeOnDisk); if (log.IsDebugEnabled && patcher.Debug.Count > 0) { log.Debug("Debug output for doc: {0} for script {1}:\r\n.{2}", jsonDocument.Key, cfg.Name, string.Join("\r\n", patcher.Debug)); patcher.Debug.Clear(); } replicationStats.ScriptSuccess(); } catch (ParseException e) { replicationStats.MarkScriptAsInvalid(Database, cfg.Script); log.WarnException("Could not parse SQL Replication script for " + cfg.Name, e); return(result); } catch (Exception e) { replicationStats.RecordScriptError(Database); log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping document: " + jsonDocument.Key, e); } } } return(result); }
private void ExecuteIndexingInternal(IEnumerable <IndexToWorkOn> indexesToWorkOn, Action <JsonDocument[]> indexingOp) { var lastIndexedGuidForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToGuid(); JsonDocument[] jsonDocs = null; try { transactionalStorage.Batch(actions => { jsonDocs = actions.Documents.GetDocumentsAfter(lastIndexedGuidForAllIndexes) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM .ToArray(); }); if (jsonDocs.Length > 0) { indexingOp(jsonDocs); } } finally { if (jsonDocs != null && jsonDocs.Length > 0) { var last = jsonDocs.Last(); Debug.Assert(last.Etag != null); Debug.Assert(last.LastModified != null); var lastEtag = last.Etag.Value; var lastModified = last.LastModified.Value; var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray()); // whatever we succeeded in indexing or not, we have to update this // because otherwise we keep trying to re-index failed documents transactionalStorage.Batch(actions => { foreach (var indexToWorkOn in indexesToWorkOn) { if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) > 0) { continue; } actions.Indexing.UpdateLastIndexed(indexToWorkOn.IndexName, lastEtag, lastModified); } }); } } }
public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation) { JsonDocumentMetadata document = null; TransactionalStorage.Batch(actions => { document = actions.Documents.DocumentMetadataByKey(key, transactionInformation); }); DocumentRetriever.EnsureIdInMetadata(document); return(new DocumentRetriever(null, ReadTriggers) .ProcessReadVetoes(document, transactionInformation, ReadOperation.Load)); }
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs) { var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name)); var result = new ConversionScriptResult(); foreach (var jsonDocument in docs) { if (string.IsNullOrEmpty(cfg.RavenEntityName) == false) { var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName); if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false) { continue; } } var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key); try { DocumentRetriever.EnsureIdInMetadata(jsonDocument); jsonDocument.Metadata[Constants.DocumentIdFieldName] = jsonDocument.Key; var document = jsonDocument.ToJson(); patcher.Apply(document, new ScriptedPatchRequest { Script = cfg.Script }); replicationStats.ScriptSuccess(); } catch (ParseException e) { replicationStats.MarkScriptAsInvalid(Database, cfg.Script); log.WarnException("Could parse SQL Replication script for " + cfg.Name, e); return(result); } catch (Exception e) { replicationStats.RecordScriptError(Database); log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping this document", e); } } return(result); }
public static int UpdateDocumentsAfter(this DocumentDatabase db, Guid eTag, Func <JsonDocument, bool> updateDoc, CancellationToken cancellationToken, TransactionInformation transactionInformation) { var initialTime = DateTime.UtcNow; log.Trace("UpdateDocumentsAfter started whith Etag {0}", eTag); int updatedDocCount = 0; db.TransactionalStorage.Batch(action => { var documentRetriever = new DocumentRetriever(action, db.ReadTriggers); foreach (var doc in action.Documents.GetDocumentsAfter(eTag, int.MaxValue)) { DocumentRetriever.EnsureIdInMetadata(doc); if (cancellationToken.IsCancellationRequested) { log.Trace("UpdateDocumentsAfter has been cancelled"); } cancellationToken.ThrowIfCancellationRequested(); var document = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document != null && updateDoc(document)) { try { db.Put(document.Key, document.Etag, document.DataAsJson, document.Metadata, transactionInformation); updatedDocCount++; log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount); } catch (ConcurrencyException) { log.Trace("ConcurrencyException caught in UpdateDocumentsAfter"); if (db.UpdateDocumentWithRetries(document.Key, updateDoc, transactionInformation)) { updatedDocCount++; log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount); } } } } }); log.Trace("UpdateDocumentsAfter ETag {0} completed successfully. {1} documents updated in {2}", eTag, updatedDocCount, DateTime.UtcNow.Subtract(initialTime)); return(updatedDocCount); }
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs) { if (context.Configuration.DisableDocumentPreFetchingForIndexing || docs.Length == 0) { return; } if (inMemoryDocs.Count > // don't use too much, this is an optimization and we need to be careful about using too much mem context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { return; } foreach (var jsonDocument in docs) { DocumentRetriever.EnsureIdInMetadata(jsonDocument); inMemoryDocs.Enqueue(jsonDocument); } }
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs) { if (context.Configuration.DisableDocumentPreFetchingForIndexing || docs.Length == 0) { return; } if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem context.Configuration.MaxNumberOfItemsToPreFetchForIndexing || prefetchingQueue.Aggregate(0, (x, c) => x + SelectSerializedSizeOnDiskIfNotNull(c)) > context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit) { return; } foreach (var jsonDocument in docs) { DocumentRetriever.EnsureIdInMetadata(jsonDocument); prefetchingQueue.Add(jsonDocument); } }
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action <RavenJObject> addDocument) { TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize, WorkContext.CancellationToken); var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState); int docCount = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); if (etag != null) { etag = doc.Etag; } DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever .ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } addDocument(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) { break; } start += docCount; } }); }
private List <JsonDocument> GetJsonDocsFromDisk(Etag etag, Etag untilEtag) { List <JsonDocument> jsonDocs = null; context.TransactionalStorage.Batch(actions => { //limit how much data we load from disk --> better adhere to memory limits var totalSizeAllowedToLoadInBytes = (context.Configuration.MemoryLimitForIndexingInMb * 1024 * 1024) - (prefetchingQueue.Aggregate(0, (acc, doc) => acc + doc.SerializedSizeOnDisk) + autoTuner.CurrentlyUsedBatchSizesInBytes.Values.Sum()); // at any rate, we will load a min of 512Kb docs var maxSize = Math.Max( Math.Min(totalSizeAllowedToLoadInBytes, autoTuner.MaximumSizeAllowedToFetchFromStorageInBytes), 1024 * 512); jsonDocs = actions.Documents .GetDocumentsAfter( etag, autoTuner.NumberOfItemsToIndexInSingleBatch, context.CancellationToken, maxSize, untilEtag, autoTuner.FetchingDocumentsFromDiskTimeout ) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .ToList(); }); if (untilEtag == null) { MaybeAddFutureBatch(jsonDocs); } return(jsonDocs); }
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs) { if (context.Configuration.DisableDocumentPreFetching || docs.Length == 0 || DisableCollectingDocumentsAfterCommit) { return; } if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem context.Configuration.MaxNumberOfItemsToPreFetch || prefetchingQueue.Aggregate(0, (x, c) => x + SelectSerializedSizeOnDiskIfNotNull(c)) > context.Configuration.AvailableMemoryForRaisingBatchSizeLimit) { return; } Etag lowestEtag = null; foreach (var jsonDocument in docs) { DocumentRetriever.EnsureIdInMetadata(jsonDocument); prefetchingQueue.Add(jsonDocument); if (ShouldHandleUnusedDocumentsAddedAfterCommit && (lowestEtag == null || jsonDocument.Etag.CompareTo(lowestEtag) < 0)) { lowestEtag = jsonDocument.Etag; } } if (ShouldHandleUnusedDocumentsAddedAfterCommit && lowestEtag != null) { if (lowestInMemoryDocumentAddedAfterCommit == null || lowestEtag.CompareTo(lowestInMemoryDocumentAddedAfterCommit.Etag) < 0) { lowestInMemoryDocumentAddedAfterCommit = new DocAddedAfterCommit { Etag = lowestEtag, AddedAt = SystemTime.UtcNow }; } } }
public void AfterCommit(JsonDocument[] docs) { if (docs.Length == 0) { return; } foreach (var doc in docs) { DocumentRetriever.EnsureIdInMetadata(doc); } futureIndexBatches.Add(new FutureIndexBatch { StartingEtag = DecrementEtag(GetLowestEtag(docs)), Task = new CompletedTask <JsonResults>(new JsonResults { Results = docs, LoadedFromDisk = false }), Age = currentIndexingAge }); }
private JsonResults GetJsonDocsFromDisk(Guid lastIndexed) { JsonDocument[] jsonDocs = null; transactionalStorage.Batch(actions => { jsonDocs = actions.Documents .GetDocumentsAfter( lastIndexed, autoTuner.NumberOfItemsToIndexInSingleBatch, autoTuner.MaximumSizeAllowedToFetchFromStorage) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .ToArray(); }); return(new JsonResults { Results = jsonDocs, LoadedFromDisk = true }); }
public JArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize) { var list = new JArray(); TransactionalStorage.Batch(actions => { var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start) .Take(pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) { continue; } list.Add(document.ToJson()); } }); return(list); }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, ref int nextStart, Action <RavenJObject> addDoc, string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, string skipAfter = null) { if (idPrefix == null) { throw new ArgumentNullException("idPrefix"); } idPrefix = idPrefix.Trim(); var canPerformRapidPagination = nextStart > 0 && start == nextStart; var actualStart = canPerformRapidPagination ? start : 0; var addedDocs = 0; var matchedDocs = 0; TransactionalStorage.Batch( actions => { var docsToSkip = canPerformRapidPagination ? 0 : start; int docCount; AbstractTransformer storedTransformer = null; if (transformer != null) { storedTransformer = IndexDefinitionStorage.GetTransformer(transformer); if (storedTransformer == null) { throw new InvalidOperationException("No transformer with the name: " + transformer); } } do { docCount = 0; var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter); var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters); foreach (var doc in docs) { token.ThrowIfCancellationRequested(); docCount++; var keyTest = doc.Key.Substring(idPrefix.Length); if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest)) { continue; } DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc; document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } matchedDocs++; if (matchedDocs <= docsToSkip) { continue; } token.ThrowIfCancellationRequested(); if (storedTransformer != null) { using (new CurrentTransformationScope(Database, documentRetriever)) { var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) }) .Select(x => JsonExtensions.ToJObject(x)) .ToArray(); if (transformed.Length == 0) { throw new InvalidOperationException("The transform results function failed on a document: " + document.Key); } var transformedJsonDocument = new JsonDocument { Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag), NonAuthoritativeInformation = document.NonAuthoritativeInformation, LastModified = document.LastModified, DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } }, }; addDoc(transformedJsonDocument.ToJson()); } } else { addDoc(document.ToJson()); } addedDocs++; if (addedDocs >= pageSize) { break; } } actualStart += pageSize; }while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue); }); if (addedDocs != pageSize) { nextStart = start; // will mark as last page } else if (canPerformRapidPagination) { nextStart = start + matchedDocs; } else { nextStart = actualStart; } }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); var relevantConfigs = config.Where(x => { if (x.Disabled) { return(false); } var sqlReplicationStatistics = statistics.GetOrDefault(x.Name); if (sqlReplicationStatistics == null) { return(true); } return(SystemTime.UtcNow >= sqlReplicationStatistics.LastErrorTime); }) // have error or the timeout expired .ToList(); if (relevantConfigs.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus); if (leastReplicatedEtag == null) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } List <JsonDocument> documents; using (prefetchingBehavior.DocumentBatchFrom(leastReplicatedEtag, out documents)) { Etag latestEtag = null, lastBatchEtag = null; if (documents.Count != 0) { lastBatchEtag = documents[documents.Count - 1].Etag; } var replicationDuration = Stopwatch.StartNew(); documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here if (documents.Count != 0) { latestEtag = documents[documents.Count - 1].Etag; } documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false); var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >(); foreach (var relevantConfig in relevantConfigs) { var cfg = relevantConfig; Database.TransactionalStorage.Batch(accessor => { deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg), GetLastEtagFor(localReplicationStatus, cfg), latestEtag, 1024) .ToList(); }); } // No documents AND there aren't any deletes to replicate if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0) { if (latestEtag != null) { // so we filtered some documents, let us update the etag about that. foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags) { if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0) { lastReplicatedEtag.LastDocEtag = latestEtag; } } latestEtag = Etag.Max(latestEtag, lastBatchEtag); SaveNewReplicationStatus(localReplicationStatus, latestEtag); } else // no point in waiting if we just saved a new doc { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); } continue; } var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >(); try { var itemsToReplicate = documents.Select(x => { DocumentRetriever.EnsureIdInMetadata(x); var doc = x.ToJson(); doc[Constants.DocumentIdFieldName] = x.Key; return(new ReplicatedDoc { Document = doc, Etag = x.Etag, Key = x.Key, SerializedSizeOnDisk = x.SerializedSizeOnDisk }); }).ToList(); BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig => { try { var startTime = SystemTime.UtcNow; Stopwatch spRepTime = new Stopwatch(); spRepTime.Start(); var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig); var deletedDocs = deletedDocsByConfig[replicationConfig]; var docsToReplicate = itemsToReplicate .Where(x => lastReplicatedEtag.CompareTo(x.Etag) <= 0) // haven't replicate the etag yet .Where(document => { var info = Database.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0) { log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, replicationConfig.Name, info.TouchedEtag); return(false); } } return(true); }) .ToList(); var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicate); if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicate.Count == 0) { currentLatestEtag = lastBatchEtag; } int countOfReplicatedItems = 0; if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) && ReplicateChangesToDestination(replicationConfig, docsToReplicate, out countOfReplicatedItems)) { if (deletedDocs.Count > 0) { Database.TransactionalStorage.Batch(accessor => accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag)); } successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag)); } spRepTime.Stop(); var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick); var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig); sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds); UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicate.Count); } catch (Exception e) { log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Sql Replication failure to replication", Message = "Sql Replication could not replicate to " + replicationConfig.Name, UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name }); } }); if (successes.Count == 0) { continue; } foreach (var t in successes) { var cfg = t.Item1; var currentLatestEtag = t.Item2; var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = currentLatestEtag ?? Etag.Empty }); } else { destEtag.LastDocEtag = currentLatestEtag = currentLatestEtag ?? destEtag.LastDocEtag; } latestEtag = Etag.Max(latestEtag, currentLatestEtag); } latestEtag = Etag.Max(latestEtag, lastBatchEtag); SaveNewReplicationStatus(localReplicationStatus, latestEtag); } finally { AfterReplicationCompleted(successes.Count); var min = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray())); if (min != null) { var lastMinReplicatedEtag = min.ToEtag(); prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag); prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed); } } } } }
public RelationalDatabaseWriter.TableQuerySummary[] SimulateSqlReplicationSQLQueries(string strDocumentId, SqlReplicationConfig sqlReplication, bool performRolledbackTransaction, out Alert alert) { alert = null; RelationalDatabaseWriter.TableQuerySummary[] resutls = null; try { var stats = new SqlReplicationStatistics(sqlReplication.Name); var jsonDocument = Database.Documents.Get(strDocumentId, null); DocumentRetriever.EnsureIdInMetadata(jsonDocument); var doc = jsonDocument.ToJson(); doc[Constants.DocumentIdFieldName] = jsonDocument.Key; var docs = new List <ReplicatedDoc> { new ReplicatedDoc { Document = doc, Etag = jsonDocument.Etag, Key = jsonDocument.Key, SerializedSizeOnDisk = jsonDocument.SerializedSizeOnDisk } }; var scriptResult = ApplyConversionScript(sqlReplication, docs); var connectionsDoc = Database.Documents.Get(ConnectionsDocumentName, null); var sqlReplicationConnections = connectionsDoc.DataAsJson.JsonDeserialization <SqlReplicationConnections>(); if (PrepareSqlReplicationConfig(sqlReplication, sqlReplication.Name, stats, sqlReplicationConnections, false, false)) { if (performRolledbackTransaction) { using (var writer = new RelationalDatabaseWriter(Database, sqlReplication, stats)) { resutls = writer.RolledBackExecute(scriptResult).ToArray(); } } else { var simulatedwriter = new RelationalDatabaseWriterSimulator(Database, sqlReplication, stats); resutls = new List <RelationalDatabaseWriter.TableQuerySummary>() { new RelationalDatabaseWriter.TableQuerySummary() { Commands = simulatedwriter.SimulateExecuteCommandText(scriptResult) .Select(x => new RelationalDatabaseWriter.TableQuerySummary.CommandData() { CommandText = x }).ToArray() } }.ToArray(); } } alert = stats.LastAlert; } catch (Exception e) { alert = new Alert() { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = "Last SQL replication operation for " + sqlReplication.Name + " was failed", Title = "SQL replication error", Exception = e.ToString(), UniqueKey = "Sql Replication Error: " + sqlReplication.Name }; } return(resutls); }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int docsSinceLastReplEtag = 0; List <JsonDocument> docsToReplicate; List <JsonDocument> filteredDocsToReplicate; result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; while (true) { docsToReplicate = GetDocsToReplicate(actions, result); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.GetRecentTouchesFor(document.Key); if (info != null) { if (Etag.IsGreaterThan(info.PreTouchEtag, result.LastEtag) == false) { return(false); } } return(destination.FilterDocuments(destinationId, document.Key, document.Metadata)); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag.Value; if (lastDoc.LastModified.HasValue) { result.LastLastModified = lastDoc.LastModified.Value; } } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); } log.Debug(() => { if (docsSinceLastReplEtag == 0) { return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag)); } if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) { return(string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination)); } var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag)); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(result); }
public RemoteQueryResults Query(LinearQuery query) { var viewGenerator = queryCache.GetOrAdd(query.Query, s => new DynamicViewCompiler("query", new IndexDefinition { Map = query.Query, }, new AbstractDynamicCompilationExtension[0]) { RequiresSelectNewAnonymousType = false }.GenerateInstance()); var results = new List <string>(); var errors = new List <string>(); int lastResult = 0; int finalResult = 0; remoteStorage.Batch(actions => { var firstAndLastDocumentIds = actions.Documents.FirstAndLastDocumentIds(); finalResult = firstAndLastDocumentIds.Item2; var start = Math.Max(firstAndLastDocumentIds.Item1, query.Start); var matchingDocs = actions.Documents.DocumentsById(start, firstAndLastDocumentIds.Item2); if (string.IsNullOrEmpty(viewGenerator.ForEntityName) == false) //optimization { matchingDocs = matchingDocs.Where(x => x.Item1.Metadata.Value <string>("Raven-Entity-Name") == viewGenerator.ForEntityName); } var docs = matchingDocs .Select(x => { DocumentRetriever.EnsureIdInMetadata(x.Item1); return(x); }) .Select(x => { lastResult = x.Item2; return(new DynamicJsonObject(x.Item1.ToJson())); }); var robustEnumerator = new RobustEnumerator { OnError = (exception, o) => errors.Add(String.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results.AddRange( robustEnumerator.RobustEnumeration(docs, viewGenerator.MapDefinition) .Take(query.PageSize) .Select(result => JsonExtensions.ToJObject(result).ToString()) ); }); return(new RemoteQueryResults { LastScannedResult = lastResult, TotalResults = finalResult, Errors = errors.ToArray(), QueryCacheSize = queryCache.Count, Results = results.ToArray() }); }