public CurrentTransformationScope(DocumentDatabase database, DocumentRetriever documentRetriever) { this.database = database; retriever = documentRetriever; old = current; current = this; }
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray jsonDocuments = null; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag, 100) .Where(destination.FilterDocuments) .Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(jsonDocuments); }
private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs) { var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) { return; // index was deleted, probably } var dateTime = jsonDocs.Min(x => x.LastModified); var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); try { log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index); context.IndexStorage.Index(index, viewGenerator, jsonDocs .Select(doc => documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Index)) .Where(doc => doc != null) .Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime); } catch (Exception e) { if (actions.IsWriteConflict(e)) { return; } log.WarnFormat(e, "Failed to index documents for index: {0}", index); } }
private Dictionary <string, List <ItemToReplicate> > ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs) { var dictionary = new Dictionary <string, List <ItemToReplicate> >(); foreach (var jsonDocument in docs) { if (string.IsNullOrEmpty(cfg.RavenEntityName) == false) { var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName); if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false) { continue; } } var patcher = new SqlReplicationScriptedJsonPatcher(Database, dictionary, jsonDocument.Key); try { DocumentRetriever.EnsureIdInMetadata(jsonDocument); jsonDocument.Metadata[Constants.DocumentIdFieldName] = jsonDocument.Key; var document = jsonDocument.ToJson(); patcher.Apply(document, new ScriptedPatchRequest { Script = cfg.Script }); } catch (Exception e) { log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping this document", e); } } return(dictionary); }
private JArray GetJsonDocuments(Guid etag) { JArray jsonDocuments = null; try { var instanceId = docDb.TransactionalStorage.Id.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonDocuments = new JArray(actions.Documents.GetDocumentsAfter(etag) .Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs .Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) == instanceId) // only replicate documents created on this instance .Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Take(100) .Select(x => x.ToJson())); }); } catch (Exception e) { log.Warn("Could not get documents to replicate after: " + etag, e); } return(jsonDocuments); }
public void AfterCommit(JsonDocument[] docs) { if (context.Configuration.DisableDocumentPreFetchingForIndexing) { return; } if (futureIndexBatches.Count > 512 || // this is optimization, and we need to make sure we don't overuse memory docs.Length == 0) { return; } foreach (var doc in docs) { DocumentRetriever.EnsureIdInMetadata(doc); } futureIndexBatches.Add(new FutureIndexBatch { StartingEtag = GetLowestEtag(docs), Task = new CompletedTask <JsonResults>(new JsonResults { Results = docs, LoadedFromDisk = false }), Age = Interlocked.Increment(ref currentIndexingAge) }); }
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument) { Etag lastDocumentReadEtag = null; TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize, token); var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, Database.InFlightTransactionalState); int docCount = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); if (etag != null) { etag = doc.Etag; } JsonDocument.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } returnedDocs = true; Database.WorkContext.UpdateFoundWork(); bool canContinue = addDocument(document); if (!canContinue) { break; } lastDocumentReadEtag = etag; } if (returnedDocs || docCount == 0) { break; } start += docCount; } }); return(lastDocumentReadEtag); }
private List <JsonDocument> GetJsonDocsFromDisk(Etag etag, Etag untilEtag) { List <JsonDocument> jsonDocs = null; context.TransactionalStorage.Batch(actions => { jsonDocs = actions.Documents .GetDocumentsAfter( etag, autoTuner.NumberOfItemsToIndexInSingleBatch, context.CancellationToken, maxSize: autoTuner.MaximumSizeAllowedToFetchFromStorage, untilEtag: untilEtag, timeout: autoTuner.FetchingDocumentsFromDiskTimeout) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .ToList(); }); if (untilEtag == null) { MaybeAddFutureBatch(jsonDocs); } return(jsonDocs); }
public JArray GetDocuments(int start, int pageSize, Guid?etag) { var list = new JArray(); TransactionalStorage.Batch(actions => { IEnumerable <JsonDocument> documents; if (etag == null) { documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start); } else { documents = actions.Documents.GetDocumentsAfter(etag.Value); } var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents.Take(pageSize)) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) { continue; } list.Add(document.ToJson()); } }); return(list); }
private List <JsonDocument> GetJsonDocsFromDisk(Guid etag) { List <JsonDocument> jsonDocs = null; var untilEtag = GetNextEtagInMemory(); context.TransactionalStorage.Batch(actions => { jsonDocs = actions.Documents .GetDocumentsAfter( etag, autoTuner.NumberOfItemsToIndexInSingleBatch, autoTuner.MaximumSizeAllowedToFetchFromStorage, untilEtag: untilEtag) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .ToList(); }); if (untilEtag == null) { MaybeAddFutureBatch(jsonDocs); } return(jsonDocs); }
private void StartGetDocumentOnBackground() { while (run) { DocumentDatabase.TransactionalStorage.Batch(accessor => { var documents = accessor.Documents.GetDocumentsAfter(lastEtagSeen, 128) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .ToArray(); if (documents.Length == 0) { return; } lastEtagSeen = documents.Last().Etag; log.Debug("Docs: {0}", string.Join(", ", documents.Select(x => x.Key))); getDocumentsState.Enqueue(new GetDocumentState(lastEtagSeen, documents.Length)); }); } }
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource) { RavenJArray jsonDocuments = null; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag) .Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs .Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source .Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Take(100) .Select(x => x.ToJson())); }); } catch (Exception e) { log.Warn("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(jsonDocuments); }
public JsonDocument Get(string key, TransactionInformation transactionInformation) { if (key == null) { throw new ArgumentNullException("key"); } key = key.Trim(); JsonDocument document = null; if (transactionInformation == null || Database.InFlightTransactionalState.TryGet(key, transactionInformation, out document) == false) { // first we check the dtc state, then the storage, to avoid race conditions var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(transactionInformation, key); TransactionalStorage.Batch(actions => { document = actions.Documents.DocumentByKey(key); }); if (nonAuthoritativeInformationBehavior != null) { document = nonAuthoritativeInformationBehavior(document); } } DocumentRetriever.EnsureIdInMetadata(document); return(new DocumentRetriever(null, Database.ReadTriggers, Database.InFlightTransactionalState) .ExecuteReadTriggers(document, transactionInformation, ReadOperation.Load)); }
public void AfterCommit(JsonDocument[] docs) { if (futureIndexBatches.Count > 512 || // this is optimization, and we need to make sure we don't overuse memory docs.Length == 0) { return; } foreach (var doc in docs) { DocumentRetriever.EnsureIdInMetadata(doc); } futureIndexBatches.Add(new FutureIndexBatch { StartingEtag = DecrementEtag(GetLowestEtag(docs)), Task = new CompletedTask <JsonResults>(new JsonResults { Results = docs, LoadedFromDisk = false }), Age = currentIndexingAge }); }
public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation) { if (key == null) { throw new ArgumentNullException("key"); } key = key.Trim(); JsonDocumentMetadata document = null; if (transactionInformation == null || Database.InFlightTransactionalState.TryGet(key, transactionInformation, out document) == false) { var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocumentMetadata>(transactionInformation, key); TransactionalStorage.Batch(actions => { document = actions.Documents.DocumentMetadataByKey(key); }); if (nonAuthoritativeInformationBehavior != null) { document = nonAuthoritativeInformationBehavior(document); } } DocumentRetriever.EnsureIdInMetadata(document); return(new DocumentRetriever(null, Database.ReadTriggers, Database.InFlightTransactionalState) .ProcessReadVetoes(document, transactionInformation, ReadOperation.Load)); }
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs) { var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name)); var result = new ConversionScriptResult(); foreach (var jsonDocument in docs) { Database.WorkContext.CancellationToken.ThrowIfCancellationRequested(); if (string.IsNullOrEmpty(cfg.RavenEntityName) == false) { var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName); if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false) { continue; } } var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key); using (var scope = new SqlReplicationScriptedJsonPatcherOperationScope(Database)) { try { DocumentRetriever.EnsureIdInMetadata(jsonDocument); var document = jsonDocument.ToJson(); document[Constants.DocumentIdFieldName] = jsonDocument.Key; patcher.Apply(scope, document, new ScriptedPatchRequest { Script = cfg.Script }, jsonDocument.SerializedSizeOnDisk); if (log.IsDebugEnabled && patcher.Debug.Count > 0) { log.Debug("Debug output for doc: {0} for script {1}:\r\n.{2}", jsonDocument.Key, cfg.Name, string.Join("\r\n", patcher.Debug)); patcher.Debug.Clear(); } replicationStats.ScriptSuccess(); } catch (ParseException e) { replicationStats.MarkScriptAsInvalid(Database, cfg.Script); log.WarnException("Could not parse SQL Replication script for " + cfg.Name, e); return(result); } catch (Exception e) { replicationStats.RecordScriptError(Database); log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping document: " + jsonDocument.Key, e); } } } return(result); }
private void ExecuteIndexingInternal(IEnumerable <IndexToWorkOn> indexesToWorkOn, Action <JsonDocument[]> indexingOp) { var lastIndexedGuidForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToGuid(); JsonDocument[] jsonDocs = null; try { transactionalStorage.Batch(actions => { jsonDocs = actions.Documents.GetDocumentsAfter(lastIndexedGuidForAllIndexes) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM .ToArray(); }); if (jsonDocs.Length > 0) { indexingOp(jsonDocs); } } finally { if (jsonDocs != null && jsonDocs.Length > 0) { var last = jsonDocs.Last(); Debug.Assert(last.Etag != null); Debug.Assert(last.LastModified != null); var lastEtag = last.Etag.Value; var lastModified = last.LastModified.Value; var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray()); // whatever we succeeded in indexing or not, we have to update this // because otherwise we keep trying to re-index failed documents transactionalStorage.Batch(actions => { foreach (var indexToWorkOn in indexesToWorkOn) { if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) > 0) { continue; } actions.Indexing.UpdateLastIndexed(indexToWorkOn.IndexName, lastEtag, lastModified); } }); } } }
private IEnumerable <RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable <JsonDocument> results, List <string> transformerErrors, CancellationToken token) { if (query.PageSize <= 0) // maybe they just want the stats? { return(Enumerable.Empty <RavenJObject>()); } IndexingFunc transformFunc = null; // Check an explicitly declared one first if (string.IsNullOrEmpty(query.ResultsTransformer) == false) { var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer); if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null) { transformFunc = transformGenerator.TransformResultsDefinition; } else { throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found"); } } else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null) { transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source); } if (transformFunc == null) { return(results.Select(x => x.ToJson())); } var dynamicJsonObjects = results.Select(x => new DynamicLuceneOrParentDocumntObject(docRetriever, x.ToJson())); var robustEnumerator = new RobustEnumerator(token, 100) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; return(robustEnumerator.RobustEnumeration( dynamicJsonObjects.Cast <object>().GetEnumerator(), transformFunc) .Select(JsonExtensions.ToJObject)); }
public JsonDocument GetWithTransformer(string key, string transformer, TransactionInformation transactionInformation, Dictionary <string, RavenJToken> transformerParameters, out HashSet <string> itemsToInclude) { JsonDocument result = null; DocumentRetriever docRetriever = null; TransactionalStorage.Batch( actions => { docRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters); using (new CurrentTransformationScope(Database, docRetriever)) { var document = Get(key, transactionInformation); if (document == null) { return; } if (document.Metadata.ContainsKey("Raven-Read-Veto")) { result = document; return; } var storedTransformer = IndexDefinitionStorage.GetTransformer(transformer); if (storedTransformer == null) { throw new InvalidOperationException("No transformer with the name: " + transformer); } var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) }) .Select(x => JsonExtensions.ToJObject(x)) .ToArray(); if (transformed.Length == 0) { return; } result = new JsonDocument { Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(docRetriever.Etag), NonAuthoritativeInformation = document.NonAuthoritativeInformation, LastModified = document.LastModified, DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } }, }; } }); itemsToInclude = docRetriever.ItemsToInclude; return(result); }
protected override bool TryGetOrCreateResourceStore(string tenantId, out IResourceStore database) { if (ResourcesStoresCache.TryGetValue(tenantId, out database)) { return(true); } JsonDocument jsonDocument; using (DocumentRetriever.DisableReadTriggers()) jsonDocument = DefaultDatabase.Get("Raven/Databases/" + tenantId, null); if (jsonDocument == null) { return(false); } var document = jsonDocument.DataAsJson.JsonDeserialization <DatabaseDocument>(); database = ResourcesStoresCache.GetOrAdd(tenantId, s => { var config = new InMemoryRavenConfiguration { Settings = DefaultConfiguration.Settings, }; foreach (var setting in document.Settings) { config.Settings[setting.Key] = setting.Value; } var dataDir = config.Settings["Raven/DataDir"]; if (dataDir == null) { throw new InvalidOperationException("Could not find Raven/DataDir"); } if (dataDir.StartsWith("~/") || dataDir.StartsWith(@"~\")) { var baseDataPath = Path.GetDirectoryName(DefaultDatabase.Configuration.DataDirectory); if (baseDataPath == null) { throw new InvalidOperationException("Could not find root data path"); } config.Settings["Raven/DataDir"] = Path.Combine(baseDataPath, dataDir.Substring(2)); } config.Settings["Raven/VirtualDir"] = config.Settings["Raven/VirtualDir"] + "/" + tenantId; config.Initialize(); var documentDatabase = new DocumentDatabase(config); documentDatabase.SpinBackgroundWorkers(); return(documentDatabase); }); return(true); }
public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation) { JsonDocumentMetadata document = null; TransactionalStorage.Batch(actions => { document = actions.Documents.DocumentMetadataByKey(key, transactionInformation); }); DocumentRetriever.EnsureIdInMetadata(document); return(new DocumentRetriever(null, ReadTriggers) .ProcessReadVetoes(document, transactionInformation, ReadOperation.Load)); }
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs) { var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name)); var result = new ConversionScriptResult(); foreach (var jsonDocument in docs) { if (string.IsNullOrEmpty(cfg.RavenEntityName) == false) { var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName); if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false) { continue; } } var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key); try { DocumentRetriever.EnsureIdInMetadata(jsonDocument); jsonDocument.Metadata[Constants.DocumentIdFieldName] = jsonDocument.Key; var document = jsonDocument.ToJson(); patcher.Apply(document, new ScriptedPatchRequest { Script = cfg.Script }); replicationStats.ScriptSuccess(); } catch (ParseException e) { replicationStats.MarkScriptAsInvalid(Database, cfg.Script); log.WarnException("Could parse SQL Replication script for " + cfg.Name, e); return(result); } catch (Exception e) { replicationStats.RecordScriptError(Database); log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping this document", e); } } return(result); }
public Etag GetDocumentsWithIdStartingWith(string idPrefix, int pageSize, Etag etag, CancellationToken token, Action <JsonDocument> addDocument) { TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = actions.Documents.GetDocumentsAfterWithIdStartingWith(etag, idPrefix, pageSize, token, timeout: TimeSpan.FromSeconds(2)); var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, Database.InFlightTransactionalState); int docCount = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); etag = doc.Etag; JsonDocument.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } addDocument(document); returnedDocs = true; Database.WorkContext.UpdateFoundWork(); } if (returnedDocs || docCount == 0) { break; } } }); return(etag); }
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs) { if (context.Configuration.DisableDocumentPreFetchingForIndexing || docs.Length == 0) { return; } if (inMemoryDocs.Count > // don't use too much, this is an optimization and we need to be careful about using too much mem context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { return; } foreach (var jsonDocument in docs) { DocumentRetriever.EnsureIdInMetadata(jsonDocument); inMemoryDocs.Enqueue(jsonDocument); } }
public static int UpdateDocumentsAfter(this DocumentDatabase db, Guid eTag, Func <JsonDocument, bool> updateDoc, CancellationToken cancellationToken, TransactionInformation transactionInformation) { var initialTime = DateTime.UtcNow; log.Trace("UpdateDocumentsAfter started whith Etag {0}", eTag); int updatedDocCount = 0; db.TransactionalStorage.Batch(action => { var documentRetriever = new DocumentRetriever(action, db.ReadTriggers); foreach (var doc in action.Documents.GetDocumentsAfter(eTag, int.MaxValue)) { DocumentRetriever.EnsureIdInMetadata(doc); if (cancellationToken.IsCancellationRequested) { log.Trace("UpdateDocumentsAfter has been cancelled"); } cancellationToken.ThrowIfCancellationRequested(); var document = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document != null && updateDoc(document)) { try { db.Put(document.Key, document.Etag, document.DataAsJson, document.Metadata, transactionInformation); updatedDocCount++; log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount); } catch (ConcurrencyException) { log.Trace("ConcurrencyException caught in UpdateDocumentsAfter"); if (db.UpdateDocumentWithRetries(document.Key, updateDoc, transactionInformation)) { updatedDocCount++; log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount); } } } } }); log.Trace("UpdateDocumentsAfter ETag {0} completed successfully. {1} documents updated in {2}", eTag, updatedDocCount, DateTime.UtcNow.Subtract(initialTime)); return(updatedDocCount); }
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs) { if (context.Configuration.DisableDocumentPreFetchingForIndexing || docs.Length == 0) { return; } if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem context.Configuration.MaxNumberOfItemsToPreFetchForIndexing || prefetchingQueue.Aggregate(0, (x, c) => x + SelectSerializedSizeOnDiskIfNotNull(c)) > context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit) { return; } foreach (var jsonDocument in docs) { DocumentRetriever.EnsureIdInMetadata(jsonDocument); prefetchingQueue.Add(jsonDocument); } }
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action <RavenJObject> addDocument) { TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize, WorkContext.CancellationToken); var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState); int docCount = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); if (etag != null) { etag = doc.Etag; } DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever .ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } addDocument(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) { break; } start += docCount; } }); }
private List <JsonDocument> GetJsonDocsFromDisk(Etag etag, Etag untilEtag) { List <JsonDocument> jsonDocs = null; context.TransactionalStorage.Batch(actions => { //limit how much data we load from disk --> better adhere to memory limits var totalSizeAllowedToLoadInBytes = (context.Configuration.MemoryLimitForIndexingInMb * 1024 * 1024) - (prefetchingQueue.Aggregate(0, (acc, doc) => acc + doc.SerializedSizeOnDisk) + autoTuner.CurrentlyUsedBatchSizesInBytes.Values.Sum()); // at any rate, we will load a min of 512Kb docs var maxSize = Math.Max( Math.Min(totalSizeAllowedToLoadInBytes, autoTuner.MaximumSizeAllowedToFetchFromStorageInBytes), 1024 * 512); jsonDocs = actions.Documents .GetDocumentsAfter( etag, autoTuner.NumberOfItemsToIndexInSingleBatch, context.CancellationToken, maxSize, untilEtag, autoTuner.FetchingDocumentsFromDiskTimeout ) .Where(x => x != null) .Select(doc => { DocumentRetriever.EnsureIdInMetadata(doc); return(doc); }) .ToList(); }); if (untilEtag == null) { MaybeAddFutureBatch(jsonDocs); } return(jsonDocs); }
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs) { if (context.Configuration.DisableDocumentPreFetching || docs.Length == 0 || DisableCollectingDocumentsAfterCommit) { return; } if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem context.Configuration.MaxNumberOfItemsToPreFetch || prefetchingQueue.Aggregate(0, (x, c) => x + SelectSerializedSizeOnDiskIfNotNull(c)) > context.Configuration.AvailableMemoryForRaisingBatchSizeLimit) { return; } Etag lowestEtag = null; foreach (var jsonDocument in docs) { DocumentRetriever.EnsureIdInMetadata(jsonDocument); prefetchingQueue.Add(jsonDocument); if (ShouldHandleUnusedDocumentsAddedAfterCommit && (lowestEtag == null || jsonDocument.Etag.CompareTo(lowestEtag) < 0)) { lowestEtag = jsonDocument.Etag; } } if (ShouldHandleUnusedDocumentsAddedAfterCommit && lowestEtag != null) { if (lowestInMemoryDocumentAddedAfterCommit == null || lowestEtag.CompareTo(lowestInMemoryDocumentAddedAfterCommit.Etag) < 0) { lowestInMemoryDocumentAddedAfterCommit = new DocAddedAfterCommit { Etag = lowestEtag, AddedAt = SystemTime.UtcNow }; } } }
public void AfterCommit(JsonDocument[] docs) { if (docs.Length == 0) { return; } foreach (var doc in docs) { DocumentRetriever.EnsureIdInMetadata(doc); } futureIndexBatches.Add(new FutureIndexBatch { StartingEtag = DecrementEtag(GetLowestEtag(docs)), Task = new CompletedTask <JsonResults>(new JsonResults { Results = docs, LoadedFromDisk = false }), Age = currentIndexingAge }); }
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable<JsonDocument> results, List<string> transformerErrors, CancellationToken token) { if (query.PageSize <= 0) // maybe they just want the stats? { return Enumerable.Empty<RavenJObject>(); } IndexingFunc transformFunc = null; // Check an explicitly declared one first if (string.IsNullOrEmpty(query.ResultsTransformer) == false) { var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer); if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null) transformFunc = transformGenerator.TransformResultsDefinition; else throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found"); } else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null) { transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source); } if (transformFunc == null) return results.Select(x => x.ToJson()); var dynamicJsonObjects = results.Select(x => new DynamicLuceneOrParentDocumntObject(docRetriever, x.ToJson())); var robustEnumerator = new RobustEnumerator(token, 100) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; return robustEnumerator.RobustEnumeration( dynamicJsonObjects.Cast<object>().GetEnumerator(), transformFunc) .Select(JsonExtensions.ToJObject); }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, Action<RavenJObject> addDoc) { if (idPrefix == null) throw new ArgumentNullException("idPrefix"); idPrefix = idPrefix.Trim(); TransactionalStorage.Batch( actions => { var docsToSkip = start; var addedDocs = 0; var matchedDocs = 0; int docCount; start = 0; do { docCount = 0; var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState); foreach (var doc in docs) { token.ThrowIfCancellationRequested(); docCount++; var keyTest = doc.Key.Substring(idPrefix.Length); if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest)) continue; DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc; document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) continue; matchedDocs++; if (matchedDocs <= docsToSkip) continue; token.ThrowIfCancellationRequested(); addDoc(document.ToJson()); addedDocs++; if (addedDocs >= pageSize) break; } start += pageSize; } while (docCount > 0 && addedDocs < pageSize && start >= 0 && start < int.MaxValue); }); }
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action<RavenJObject> addDocument) { TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState); int docCount = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); if (etag != null) etag = doc.Etag; DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever .ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) continue; addDocument(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) break; start += docCount; } }); }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, Action<RavenJObject> addDoc) { if (idPrefix == null) throw new ArgumentNullException("idPrefix"); idPrefix = idPrefix.Trim(); TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { int docCount = 0; var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState); foreach (var doc in documents) { docCount++; string keyTest = doc.Key.Substring(idPrefix.Length); if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest)) continue; DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key); JsonDocument document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc; document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; addDoc(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) break; start += docCount; } }); }
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag) { var list = new RavenJArray(); TransactionalStorage.Batch(actions => { while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag.Value, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); int docCount = 0; foreach (var doc in documents) { docCount++; if(etag != null) etag = doc.Etag; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } if (list.Length != 0 || docCount == 0) break; start += docCount; } }); return list; }
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable<JsonDocument> results, List<string> transformerErrors) { if (query.PageSize <= 0) // maybe they just want the stats? { return Enumerable.Empty<RavenJObject>(); } IndexingFunc transformFunc = null; // Check an explicitly declared one first if (query.ResultsTransformer != null) { var transformGenerator = IndexDefinitionStorage.GetTransfomer(query.ResultsTransformer); if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null) transformFunc = transformGenerator.TransformResultsDefinition; } else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null) { transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source); } if (transformFunc == null) return results.Select(x => x.ToJson()); var dynamicJsonObjects = results.Select(x => new DynamicJsonObject(x.ToJson())).ToArray(); var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; return robustEnumerator.RobustEnumeration( dynamicJsonObjects.Cast<object>().GetEnumerator(), transformFunc) .Select(JsonExtensions.ToJObject); }
public QueryResult Query(string index, IndexQuery query) { index = IndexDefinitionStorage.FixupIndexName(index); var list = new List<JObject>(); var stale = false; Tuple<DateTime, Guid> indexTimestamp = null; TransactionalStorage.Batch( actions => { string entityName = null; var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator != null) entityName = viewGenerator.ForEntityName; stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers); var indexDefinition = GetIndexDefinition(index); var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, query.FieldsToFetch,query.AggregationOperation)) select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, query.FieldsToFetch, query.AggregationOperation) into doc where doc != null select doc; var transformerErrors = new List<string>(); IEnumerable<JObject> results; if (viewGenerator != null && query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null) { var robustEnumerator = new RobustEnumerator { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray(); results = robustEnumerator.RobustEnumeration( dynamicJsonObjects, source => viewGenerator.TransformResultsDefinition(docRetriever, source)) .Select(JsonExtensions.ToJObject); } else { results = collection.Select(x => x.ToJson()); } list.AddRange(results); if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } }); return new QueryResult { IndexName = index, Results = list, IsStale = stale, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2 }; }
public QueryResultWithIncludes Query(string index, IndexQuery query) { index = IndexDefinitionStorage.FixupIndexName(index); var list = new List<RavenJObject>(); var highlightings = new Dictionary<string, Dictionary<string, string[]>>(); var stale = false; Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty); Guid resultEtag = Guid.Empty; var nonAuthoritativeInformation = false; var idsToLoad = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase); TransactionalStorage.Batch( actions => { var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) throw new IndexDoesNotExistsException("Could not find index named: " + index); resultEtag = GetIndexEtag(index, null); stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers, idsToLoad); var indexDefinition = GetIndexDefinition(index); var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation, viewGenerator.ReduceDefinition == null ? Constants.DocumentIdFieldName : Constants.ReduceKeyFieldName); Func<IndexQueryResult, bool> shouldIncludeInResults = result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch); var collection = from queryResult in IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers) select new { Document = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch), Fragments = queryResult.Highligtings } into docWithFragments where docWithFragments.Document != null let _ = nonAuthoritativeInformation |= (docWithFragments.Document.NonAuthoritativeInformation ?? false) select docWithFragments; var transformerErrors = new List<string>(); IEnumerable<RavenJObject> results; if (query.SkipTransformResults == false && query.PageSize > 0 && // maybe they just want the stats? viewGenerator.TransformResultsDefinition != null) { var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.Document.ToJson())).ToArray(); var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results = robustEnumerator.RobustEnumeration( dynamicJsonObjects.Cast<object>().GetEnumerator(), source => viewGenerator.TransformResultsDefinition(docRetriever, source)) .Select(JsonExtensions.ToJObject); } else { var resultList = new List<RavenJObject>(); foreach (var docWithFragments in collection) { resultList.Add(docWithFragments.Document.ToJson()); if (docWithFragments.Fragments != null && docWithFragments.Document.Key != null) highlightings.Add(docWithFragments.Document.Key, docWithFragments.Fragments); } results = resultList; } list.AddRange(results); if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } }); return new QueryResultWithIncludes { IndexName = index, Results = list, IsStale = stale, NonAuthoritativeInformation = nonAuthoritativeInformation, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2, ResultEtag = resultEtag, IdsToInclude = idsToLoad, LastQueryTime = SystemTime.UtcNow, Highlightings = highlightings }; }
public QueryResultWithIncludes Query(string index, IndexQuery query, Action<QueryHeaderInformation> headerInfo, Action<RavenJObject> onResult) { index = IndexDefinitionStorage.FixupIndexName(index); var highlightings = new Dictionary<string, Dictionary<string, string[]>>(); Func<IndexQueryResult, object> tryRecordHighlighting = queryResult => { if (queryResult.Highligtings != null && queryResult.Key != null) highlightings.Add(queryResult.Key, queryResult.Highligtings); return null; }; var stale = false; Tuple<DateTime, Etag> indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty); Etag resultEtag = Etag.Empty; var nonAuthoritativeInformation = false; if (string.IsNullOrEmpty(query.ResultsTransformer) == false) { query.FieldsToFetch = new[] { Constants.AllFields }; } var idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase); TransactionalStorage.Batch( actions => { var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) throw new IndexDoesNotExistsException("Could not find index named: " + index); resultEtag = GetIndexEtag(index, null); stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers, query.QueryInputs, idsToLoad); var indexDefinition = GetIndexDefinition(index); var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation, viewGenerator.ReduceDefinition == null ? Constants.DocumentIdFieldName : Constants.ReduceKeyFieldName); Func<IndexQueryResult, bool> shouldIncludeInResults = result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch); var indexQueryResults = IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers); indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults); var transformerErrors = new List<string>(); var results = GetQueryResults(query, viewGenerator, docRetriever, from queryResult in indexQueryResults let doc = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch) where doc != null let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false) let __ = tryRecordHighlighting(queryResult) select doc, transformerErrors); if (headerInfo != null) { headerInfo(new QueryHeaderInformation { Index = index, IsStable = stale, ResultEtag = resultEtag, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2, TotalResults = query.TotalSize.Value }); } using (new CurrentTransformationScope(docRetriever)) { foreach (var result in results) { onResult(result); } if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } } }); return new QueryResultWithIncludes { IndexName = index, IsStale = stale, NonAuthoritativeInformation = nonAuthoritativeInformation, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2, ResultEtag = resultEtag, IdsToInclude = idsToLoad, LastQueryTime = SystemTime.UtcNow, Highlightings = highlightings }; }
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag) { var list = new RavenJArray(); TransactionalStorage.Batch(actions => { IEnumerable<JsonDocument> documents; if (etag == null) documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start); else documents = actions.Documents.GetDocumentsAfter(etag.Value); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents.Take(pageSize)) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } }); return list; }
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize) { var list = new RavenJArray(); TransactionalStorage.Batch(actions => { var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start) .Take(pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } }); return list; }
public QueryResult Query(string index, IndexQuery query) { index = IndexDefinitionStorage.FixupIndexName(index); var list = new List<RavenJObject>(); var stale = false; Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty); TransactionalStorage.Batch( actions => { var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) throw new InvalidOperationException("Could not find index named: " + index); stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers); var indexDefinition = GetIndexDefinition(index); var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation, viewGenerator.ReduceDefinition == null ? Constants.DocumentIdFieldName : Constants.ReduceKeyFieldName); var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch, IndexQueryTriggers) select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch) into doc where doc != null select doc; var transformerErrors = new List<string>(); IEnumerable<RavenJObject> results; if (query.SkipTransformResults == false && query.PageSize > 0 && // maybe they just want the stats? viewGenerator.TransformResultsDefinition != null) { var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray(); var robustEnumerator = new RobustEnumerator(dynamicJsonObjects.Length) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results = robustEnumerator.RobustEnumeration( dynamicJsonObjects, source => viewGenerator.TransformResultsDefinition(docRetriever, source)) .Select(JsonExtensions.ToJObject); } else { results = collection.Select(x => x.ToJson()); } if (query.PageSize > 0) // maybe they just want the query stats? list.AddRange(results); if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } }); return new QueryResult { IndexName = index, Results = list, IsStale = stale, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2 }; }
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize) { if (idPrefix == null) throw new ArgumentNullException("idPrefix"); idPrefix = idPrefix.Trim(); var list = new RavenJArray(); TransactionalStorage.Batch(actions => { var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false) continue; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } }); return list; }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize, Action<RavenJObject> addDoc) { if (idPrefix == null) throw new ArgumentNullException("idPrefix"); idPrefix = idPrefix.Trim(); TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { int docCount = 0; var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { docCount++; if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false) continue; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; addDoc(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) break; start += docCount; } }); }
private JArray GetJsonDocuments(Guid etag) { JArray jsonDocuments = null; try { var instanceId = docDb.TransactionalStorage.Id.ToString(); docDb.TransactionalStorage.Batch(actions => { var docRetr = new DocumentRetriever(actions, Enumerable.Empty<AbstractReadTrigger>()); jsonDocuments = new JArray(actions.Documents.GetDocumentsAfter(etag) .Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs .Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) == instanceId) // only replicate documents created on this instance .Where(x=> x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict .Select(x=> { docRetr.EnsureIdInMetadata(x); return x; }) .Take(100) .Select(x => x.ToJson())); }); } catch (Exception e) { log.Warn("Could not get documents to replicate after: " + etag, e); } return jsonDocuments; }
public void GetDocuments(int start, int pageSize, Etag etag, Action<RavenJObject> addDocument) { TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); int docCount = 0; foreach (var doc in documents) { docCount++; if (etag != null) etag = doc.Etag; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; addDocument(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) break; start += docCount; } }); }
public JArray GetDocuments(int start, int pageSize, Guid? etag) { var list = new JArray(); TransactionalStorage.Batch(actions => { IEnumerable<JsonDocument> documents; if (etag == null) documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start); else documents = actions.Documents.GetDocumentsAfter(etag.Value); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents.Take(pageSize)) { var document = documentRetriever.ExecuteReadTriggers(doc, null, // here we want to have the Load semantic, not Query, because we need this to be // as close as possible to the full database contents ReadOperation.Load); if (document == null) continue; if (document.Metadata.Property("@id") == null) document.Metadata.Add("@id", new JValue(doc.Key)); list.Add(document.ToJson()); } }); return list; }
public JsonDocument GetWithTransformer(string key, string transformer, TransactionInformation transactionInformation, Dictionary<string, RavenJToken> queryInputs) { JsonDocument result = null; TransactionalStorage.Batch( actions => { var docRetriever = new DocumentRetriever(actions, ReadTriggers, queryInputs); using (new CurrentTransformationScope(docRetriever)) { var document = Get(key, transactionInformation); if (document == null) return; var storedTransformer = IndexDefinitionStorage.GetTransfomer(transformer); if (storedTransformer == null) throw new InvalidOperationException("No transfomer with the name: " + transformer); var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) }) .Select(x => JsonExtensions.ToJObject(x)) .ToArray(); if (transformed.Length == 0) return; result = new JsonDocument { Etag = document.Etag, NonAuthoritativeInformation = document.NonAuthoritativeInformation, LastModified = document.LastModified, }; result.DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } }; } }); return result; }
public bool IndexDocuments(IStorageActionsAccessor actions, string index, Guid etagToIndexFrom) { log.DebugFormat("Indexing documents for {0}, etag to index from: {1}", index, etagToIndexFrom); var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) return false; // index was deleted, probably var jsonDocs = actions.Documents.GetDocumentsAfter(etagToIndexFrom) .Where(x => x != null) .Take(10000) // ensure that we won't go overboard with reading and blow up with OOM .ToArray(); if(jsonDocs.Length == 0) return false; var dateTime = jsonDocs.Select(x=>x.LastModified).Min(); var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); try { log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index); context.IndexStorage.Index(index, viewGenerator, jsonDocs .Select(doc => documentRetriever.ProcessReadVetoes(doc, null, ReadOperation.Index)) .Where(doc => doc != null) .Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime); return true; } catch (Exception e) { log.WarnFormat(e, "Failed to index documents for index: {0}", index); return false; } finally { // whatever we succeeded in indexing or not, we have to update this // because otherwise we keep trying to re-index failed documents var last = jsonDocs.Last(); actions.Indexing.UpdateLastIndexed(index, last.Etag, last.LastModified); } }
public JsonDocument GetWithTransformer(string key, string transformer, TransactionInformation transactionInformation, Dictionary<string, RavenJToken> queryInputs) { JsonDocument result = null; TransactionalStorage.Batch( actions => { var docRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState, queryInputs); using (new CurrentTransformationScope(this, docRetriever)) { var document = Get(key, transactionInformation); if (document == null) return; if (document.Metadata.ContainsKey("Raven-Read-Veto") || document.Metadata.ContainsKey(Constants.RavenReplicationConflict)) { result = document; return; } var storedTransformer = IndexDefinitionStorage.GetTransformer(transformer); if (storedTransformer == null) throw new InvalidOperationException("No transformer with the name: " + transformer); var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) }) .Select(x => JsonExtensions.ToJObject(x)) .ToArray(); if (transformed.Length == 0) return; result = new JsonDocument { Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(docRetriever.Etag), NonAuthoritativeInformation = document.NonAuthoritativeInformation, LastModified = document.LastModified, DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } }, }; } }); return result; }