protected Task <IOperationResult> ExecuteDelete(IndexQueryServerSide query, Index index, QueryOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, OperationCancelToken token) { return(ExecuteOperation(query, index, options, context, onProgress, (key, retrieveDetails) => { var command = new DeleteDocumentCommand(key, null, Database); return new BulkOperationCommand <DeleteDocumentCommand>(command, retrieveDetails, x => new BulkOperationResult.DeleteDetails { Id = key, Etag = x.DeleteResult?.Etag }); }, token)); }
private void GetDocuments(DocumentsOperationContext context, Transformer transformer, bool metadataOnly) { // everything here operates on all docs var actualEtag = ComputeAllDocumentsEtag(context); if (GetLongFromHeaders("If-None-Match") == actualEtag) { HttpContext.Response.StatusCode = 304; return; } HttpContext.Response.Headers["ETag"] = "\"" + actualEtag + "\""; var etag = GetLongQueryString("etag", false); IEnumerable <Document> documents; bool isLoadStartingWith = false; if (etag != null) { documents = Database.DocumentsStorage.GetDocumentsFrom(context, etag.Value, GetStart(), GetPageSize()); } else if (HttpContext.Request.Query.ContainsKey("startsWith")) { isLoadStartingWith = true; documents = Database.DocumentsStorage.GetDocumentsStartingWith(context, HttpContext.Request.Query["startsWith"], HttpContext.Request.Query["matches"], HttpContext.Request.Query["excludes"], GetStart(), GetPageSize() ); } else // recent docs { documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, GetStart(), GetPageSize()); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { if (isLoadStartingWith) { writer.WriteStartObject(); writer.WritePropertyName("Results"); } if (transformer != null) { var transformerParameters = GetTransformerParameters(context); using (var scope = transformer.OpenTransformationScope(transformerParameters, null, Database.DocumentsStorage, Database.TransformerStore, context)) { writer.WriteDocuments(context, scope.Transform(documents).ToList(), metadataOnly); } } else { writer.WriteDocuments(context, documents, metadataOnly); } if (isLoadStartingWith) { writer.WriteEndObject(); } } }
protected override long ExecuteCmd(DocumentsOperationContext context) { throw new NotSupportedException("Should only call Execute() here"); }
public RetrieveDocumentIdsVisitor(TransactionOperationContext serverContext, DocumentsOperationContext context, QueryMetadata metadata, ByteStringContext allocator) : base(metadata.Query.QueryText) { _query = metadata.Query; _serverContext = serverContext; _context = context; _metadata = metadata; _allocator = allocator; }
public StreamDestination(Stream stream, DocumentsOperationContext context, DatabaseSource source) { _stream = stream; _context = context; _source = source; }
public TermsQueryResultServerSide ExecuteGetTermsQuery(string indexName, string field, string fromValue, long?existingResultEtag, int pageSize, DocumentsOperationContext context, OperationCancelToken token, out Index index) { Exception lastException = null; for (var i = 0; i < NumberOfRetries; i++) { try { index = GetIndex(indexName); var etag = index.GetIndexEtag(null); if (etag == existingResultEtag) { return(TermsQueryResultServerSide.NotModifiedResult); } return(index.GetTerms(field, fromValue, pageSize, context, token)); } catch (ObjectDisposedException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } lastException = e; WaitForIndexBeingLikelyReplacedDuringQuery().GetAwaiter().GetResult(); } catch (OperationCanceledException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } if (token.Token.IsCancellationRequested) { throw; } lastException = e; WaitForIndexBeingLikelyReplacedDuringQuery().GetAwaiter().GetResult(); } } throw CreateRetriesFailedException(lastException); }
public override async Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { Exception lastException = null; for (var i = 0; i < NumberOfRetries; i++) { try { return(await GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, context, onProgress, token)); } catch (ObjectDisposedException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } catch (OperationCanceledException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } if (token.Token.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } } throw CreateRetriesFailedException(lastException); }
public override async Task <DocumentQueryResult> ExecuteQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token) { var sw = Stopwatch.StartNew(); var result = await GetRunner(query).ExecuteQuery(query, documentsContext, existingResultEtag, token); result.DurationInMs = (long)sw.Elapsed.TotalMilliseconds; return(result); }
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamDocumentQueryResultWriter writer, OperationCancelToken token) { return(GetRunner(query).ExecuteStreamQuery(query, documentsContext, response, writer, token)); }
public override Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { return(GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, context, onProgress, token)); }
public List <DynamicQueryToIndexMatcher.Explanation> ExplainDynamicIndexSelection(IndexQueryServerSide query, DocumentsOperationContext context) { if (query.Metadata.IsDynamic == false) { throw new InvalidOperationException("Explain can only work on dynamic indexes"); } if (_dynamic is DynamicQueryRunner d) { return(d.ExplainIndexSelection(query, context)); } throw new NotSupportedException(InvalidQueryRunner.ErrorMessage); }
public override Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { return(GetRunner(query).ExecuteDeleteQuery(query, options, context, onProgress, token)); }
public override Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { return(GetRunner(query).ExecuteIndexEntriesQuery(query, context, existingResultEtag, token)); }
protected Task <IOperationResult> ExecutePatch(IndexQueryServerSide query, Index index, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, OperationCancelToken token) { return(ExecuteOperation(query, index, options, context, onProgress, (key, retrieveDetails) => { var command = new PatchDocumentCommand(context, key, expectedChangeVector: null, skipPatchIfChangeVectorMismatch: false, patch: (patch, patchArgs), patchIfMissing: (null, null), database: Database, debugMode: false, isTest: false); return new BulkOperationCommand <PatchDocumentCommand>(command, retrieveDetails, x => new BulkOperationResult.PatchDetails { Id = key, ChangeVector = x.PatchResult.ChangeVector, Status = x.PatchResult.Status }); }, token));
public override async Task ExecuteStreamIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamQueryResultWriter <BlittableJsonReaderObject> writer, OperationCancelToken token) { Exception lastException = null; for (var i = 0; i < NumberOfRetries; i++) { try { documentsContext.CloseTransaction(); await GetRunner(query).ExecuteStreamIndexEntriesQuery(query, documentsContext, response, writer, token); return; } catch (ObjectDisposedException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } catch (OperationCanceledException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } if (token.Token.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } } throw CreateRetriesFailedException(lastException); }
public async Task <FacetedQueryResult> ExecuteFacetedQuery(IndexQueryServerSide query, long?existingResultEtag, DocumentsOperationContext documentsContext, OperationCancelToken token) { if (query.Metadata.IsDynamic) { throw new InvalidQueryException("Facet query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters); } var sw = Stopwatch.StartNew(); var result = await _static.ExecuteFacetedQuery(query, existingResultEtag, documentsContext, token); result.DurationInMs = (long)sw.Elapsed.TotalMilliseconds; return(result); }
public async Task <FacetedQueryResult> ExecuteFacetedQuery(IndexQueryServerSide query, long?existingResultEtag, DocumentsOperationContext documentsContext, OperationCancelToken token) { if (query.Metadata.IsDynamic) { throw new InvalidQueryException("Facet query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters); } Exception lastException = null; for (var i = 0; i < NumberOfRetries; i++) { try { var sw = Stopwatch.StartNew(); var result = await _static.ExecuteFacetedQuery(query, existingResultEtag, documentsContext, token); result.DurationInMs = (long)sw.Elapsed.TotalMilliseconds; return(result); } catch (ObjectDisposedException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } catch (OperationCanceledException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } if (token.Token.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } } throw CreateRetriesFailedException(lastException); }
public TermsQueryResultServerSide ExecuteGetTermsQuery(Index index, string field, string fromValue, long?existingResultEtag, int pageSize, DocumentsOperationContext context, OperationCancelToken token) { var etag = index.GetIndexEtag(); if (etag == existingResultEtag) { return(TermsQueryResultServerSide.NotModifiedResult); } return(index.GetTerms(field, fromValue, pageSize, context, token)); }
public override async Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { Exception lastException = null; for (var i = 0; i < NumberOfRetries; i++) { try { return(await GetRunner(query).ExecuteIndexEntriesQuery(query, context, existingResultEtag, token)); } catch (ObjectDisposedException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } catch (OperationCanceledException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } if (token.Token.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } } throw CreateRetriesFailedException(lastException); }
public async Task <SuggestionQueryResult> ExecuteSuggestionQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { if (query.Metadata.IsDynamic) { throw new InvalidQueryException("Suggestion query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters); } var sw = Stopwatch.StartNew(); if (query.Metadata.SelectFields.Length != 1 || query.Metadata.SelectFields[0].IsSuggest == false) { throw new InvalidQueryException("Suggestion query must have one suggest token in SELECT.", query.Metadata.QueryText, query.QueryParameters); } var selectField = (SuggestionField)query.Metadata.SelectFields[0]; var index = GetIndex(query.Metadata.IndexName); var indexDefinition = index.GetIndexDefinition(); if (indexDefinition.Fields.TryGetValue(selectField.Name, out IndexFieldOptions field) == false) { throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' does not have a field '{selectField.Name}'."); } if (field.Suggestions == null) { throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' does not have suggestions configured for field '{selectField.Name}'."); } if (field.Suggestions.Value == false) { throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' have suggestions explicitly disabled for field '{selectField.Name}'."); } if (existingResultEtag.HasValue) { var etag = index.GetIndexEtag(); if (etag == existingResultEtag.Value) { return(SuggestionQueryResult.NotModifiedResult); } } var result = await index.SuggestionQuery(query, context, token); result.DurationInMs = (int)sw.Elapsed.TotalMilliseconds; return(result); }
public override async Task <DocumentQueryResult> ExecuteQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token) { Exception lastException = null; for (var i = 0; i < NumberOfRetries; i++) { try { Stopwatch sw = null; QueryTimingsScope scope; DocumentQueryResult result; using (scope = query.Timings?.Start()) { if (scope == null) { sw = Stopwatch.StartNew(); } result = await GetRunner(query).ExecuteQuery(query, documentsContext, existingResultEtag, token); } result.DurationInMs = sw != null ? (long)sw.Elapsed.TotalMilliseconds : (long)scope.Duration.TotalMilliseconds; return(result); } catch (ObjectDisposedException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } catch (OperationCanceledException e) { if (Database.DatabaseShutdown.IsCancellationRequested) { throw; } if (token.Token.IsCancellationRequested) { throw; } lastException = e; await WaitForIndexBeingLikelyReplacedDuringQuery(); } } throw CreateRetriesFailedException(lastException); }
public (BlittableJsonReaderObject Document, string Id) Test(MigrationTestSettings settings, DatabaseSchema dbSchema, DocumentsOperationContext context) { using (var enumerationConnection = OpenConnection()) using (var referencesConnection = OpenConnection()) { var collectionToImport = settings.Collection; var tableSchema = dbSchema.GetTable(collectionToImport.SourceTableSchema, collectionToImport.SourceTableName); var specialColumns = dbSchema.FindSpecialColumns(collectionToImport.SourceTableSchema, collectionToImport.SourceTableName); string CollectionNameProvider(string schema, string name) => settings.CollectionsMapping.Single(x => x.TableSchema == schema && x.TableName == name).CollectionName; using (var patcher = new JsPatcher(collectionToImport, context)) { var references = ResolveReferences(collectionToImport, dbSchema, CollectionNameProvider); InitializeDataProviders(references, referencesConnection); try { Dictionary <string, object> queryParameters = null; var queryToUse = settings.Mode == MigrationTestMode.First ? GetQueryForCollection(collectionToImport) // we limit rows count internally by passing rowsLimit: 1 : GetQueryByPrimaryKey(collectionToImport, tableSchema, settings.PrimaryKeyValues, out queryParameters); foreach (var doc in EnumerateTable(queryToUse, collectionToImport.ColumnsMapping, specialColumns, collectionToImport.AttachmentNameMapping, enumerationConnection, rowsLimit: 1, queryParameters: queryParameters)) { doc.SetCollection(collectionToImport.Name); var id = GenerateDocumentId(doc.Collection, GetColumns(doc.SpecialColumnsValues, tableSchema.PrimaryKeyColumns)); FillDocumentFields(doc.Object, doc.SpecialColumnsValues, references, "", doc.Attachments); return(patcher.Patch(doc.ToBlittable(context)), id); } } catch (JavaScriptException e) { if (e.InnerException is Jint.Runtime.JavaScriptException innerException && string.Equals(innerException.Message, "skip", StringComparison.OrdinalIgnoreCase)) { throw new InvalidOperationException("Document was skipped", e); }
private (List <Slice>, string) ExtractIdsFromQuery(IndexQueryServerSide query, DocumentsOperationContext context) { if (string.IsNullOrWhiteSpace(query.Query)) { return(null, null); } if (query.Metadata.Query.Where == null) { return(null, null); } if (query.Metadata.IndexFieldNames.Contains(QueryFieldName.DocumentId) == false) { return(null, null); } IDisposable releaseServerContext = null; IDisposable closeServerTransaction = null; TransactionOperationContext serverContext = null; try { if (query.Metadata.HasCmpXchg) { releaseServerContext = context.DocumentDatabase.ServerStore.ContextPool.AllocateOperationContext(out serverContext); closeServerTransaction = serverContext.OpenReadTransaction(); } using (closeServerTransaction) { var idsRetriever = new RetrieveDocumentIdsVisitor(serverContext, context, query.Metadata, _context.Allocator); idsRetriever.Visit(query.Metadata.Query.Where, query.QueryParameters); return(idsRetriever.Ids?.OrderBy(x => x, SliceComparer.Instance).ToList(), idsRetriever.StartsWith); } } finally { releaseServerContext?.Dispose(); } }
public PutOperationResults PutDocument(DocumentsOperationContext context, string id, string expectedChangeVector, BlittableJsonReaderObject document, long?lastModifiedTicks = null, string changeVector = null, DocumentFlags flags = DocumentFlags.None, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (context.Transaction == null) { ThrowRequiresTransaction(); return(default(PutOperationResults)); // never hit } #if DEBUG var documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); #endif var newEtag = _documentsStorage.GenerateNextEtag(); var modifiedTicks = _documentsStorage.GetOrCreateLastModifiedTicks(lastModifiedTicks); id = BuildDocumentId(id, newEtag, out bool knownNewId); using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var collectionName = _documentsStorage.ExtractCollectionName(context, document); var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); var oldValue = default(TableValueReader); if (knownNewId == false) { // delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size); table.ReadByKey(lowerId, out oldValue); } BlittableJsonReaderObject oldDoc = null; if (oldValue.Pointer == null) { // expectedChangeVector being null means we don't care, // and empty means that it must be new if (string.IsNullOrEmpty(expectedChangeVector) == false) { ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector); } } else { // expectedChangeVector has special meaning here // null - means, don't care, don't check // "" / empty - means, must be new // anything else - must match exactly if (expectedChangeVector != null) { var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue); if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0) { ThrowConcurrentException(id, expectedChangeVector, oldChangeVector); } } oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context); var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc); if (oldCollectionName != collectionName) { ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName); } var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue); if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate && (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication) { if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments) { flags |= DocumentFlags.HasAttachments; } } } var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue); if (string.IsNullOrEmpty(result.ChangeVector)) { ChangeVectorUtils.ThrowConflictingEtag(id, changeVector, newEtag, _documentsStorage.Environment.Base64Id, _documentDatabase.ServerStore.NodeTag); } changeVector = result.ChangeVector; nonPersistentFlags |= result.NonPersistentFlags; if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved)) { flags |= DocumentFlags.Resolved; } if (collectionName.IsHiLo == false && (flags & DocumentFlags.Artificial) != DocumentFlags.Artificial) { if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags)) { #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } #endif document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); #if DEBUG documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif } if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false && (flags.Contain(DocumentFlags.Resolved) || _documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null )) { var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document, ref flags, out RevisionsCollectionConfiguration configuration); if (shouldVersion) { _documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags, changeVector, modifiedTicks, configuration, collectionName); } } } using (Slice.From(context.Allocator, changeVector, out var cv)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(Bits.SwapBytes(newEtag)); tvb.Add(idPtr); tvb.Add(document.BasePointer, document.Size); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(modifiedTicks); tvb.Add((int)flags); tvb.Add(context.GetTransactionMarker()); if (oldValue.Pointer == null) { table.Insert(tvb); } else { table.Update(oldValue.Id, tvb); } } if (collectionName.IsHiLo == false) { _documentsStorage.ExpirationStorage.Put(context, lowerId, document); } context.LastDatabaseChangeVector = changeVector; _documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1); _documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size); context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = changeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Put, }); #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif return(new PutOperationResults { Etag = newEtag, Id = id, Collection = collectionName, ChangeVector = changeVector, Flags = flags, LastModified = new DateTime(modifiedTicks) }); } }
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand) { throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e); }
public void Should_be_able_to_read_index_stats_even_if_corruption_happened() { UseNewLocalServer(); using (var db = CreateDocumentDatabase()) { using (var index = MapIndex.CreateNew(new IndexDefinition() { Name = "Users_ByName", Maps = { "from user in docs.Users select new { user.Name }" }, Type = IndexType.Map }, db)) { PutUser(db); index._indexStorage.SimulateCorruption = true; db.ServerStore.DatabasesLandlord.CatastrophicFailureHandler.MaxDatabaseUnloads = 0; var mre = new ManualResetEventSlim(); db.Changes.OnIndexChange += change => { if (change.Type == IndexChangeTypes.IndexMarkedAsErrored) { mre.Set(); } }; index.Start(); Assert.True(mre.Wait(TimeSpan.FromMinutes(1))); Assert.Equal(IndexState.Error, index.State); long errorCount = 0; for (int i = 0; i < 20; i++) { errorCount = index.GetErrorCount(); if (errorCount > 0) { break; } Thread.Sleep(500); // errors are updated in next tx when we update the stats } Assert.Equal(1, errorCount); using (var context = DocumentsOperationContext.ShortTermSingleUse(db)) { using (context.OpenReadTransaction()) { var indexStats = index.GetIndexStats(context); Assert.True(indexStats.IsStale); } } var indexingErrors = index.GetErrors(); Assert.Equal(1, indexingErrors.Count); } } }
private void GetDocumentsById(DocumentsOperationContext context, StringValues ids, Transformer transformer, bool metadataOnly) { var includePaths = HttpContext.Request.Query["include"]; var documents = new List <Document>(ids.Count); List <long> etags = null; var includes = new List <Document>(includePaths.Count * ids.Count); var includeDocs = new IncludeDocumentsCommand(Database.DocumentsStorage, context, includePaths); foreach (var id in ids) { var document = Database.DocumentsStorage.Get(context, id); documents.Add(document); includeDocs.Gather(document); } IEnumerable <Document> documentsToWrite; if (transformer != null) { var transformerParameters = GetTransformerParameters(context); using (var scope = transformer.OpenTransformationScope(transformerParameters, includeDocs, Database.DocumentsStorage, Database.TransformerStore, context)) { documentsToWrite = scope.Transform(documents).ToList(); etags = scope.LoadedDocumentEtags; } } else { documentsToWrite = documents; } includeDocs.Fill(includes); var actualEtag = ComputeEtagsFor(documents, includes, etags); if (transformer != null) { actualEtag ^= transformer.Hash; } var etag = GetLongFromHeaders("If-None-Match"); if (etag == actualEtag) { HttpContext.Response.StatusCode = 304; return; } HttpContext.Response.Headers[Constants.MetadataEtagField] = "\"" + actualEtag + "\""; if (HttpContext.Request.Query["blittable"] == "true") { WriteDocumentsBlittable(context, documentsToWrite, includes); } else { WriteDocumentsJson(context, metadataOnly, documentsToWrite, includeDocs, includes); } }
public SqlDocumentTransformer(Transformation transformation, DocumentDatabase database, DocumentsOperationContext context, SqlEtlConfiguration config) : base(database, context, new PatchRequest(transformation.Script, PatchRequestType.SqlEtl), null) { _transformation = transformation; _config = config; var destinationTables = transformation.GetCollectionsFromScript(); LoadToDestinations = destinationTables; _tables = new Dictionary <string, SqlTableWithRecords>(destinationTables.Length); _tablesForScript = new List <SqlEtlTable>(destinationTables.Length); // ReSharper disable once ForCanBeConvertedToForeach for (var i = 0; i < _config.SqlTables.Count; i++) { var table = _config.SqlTables[i]; if (destinationTables.Contains(table.TableName, StringComparer.OrdinalIgnoreCase)) { _tablesForScript.Add(table); } } if (_transformation.IsLoadingAttachments) { _loadedAttachments = new Dictionary <string, Queue <Attachment> >(StringComparer.OrdinalIgnoreCase); } }
public override int Execute(DocumentsOperationContext context) { if (_database.ServerStore.Server.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable) { FeaturesAvailabilityException.Throw("Counters"); } foreach (var kvp in _dictionary) { Document doc = null; var docId = kvp.Key; string docCollection = null; foreach (var operation in kvp.Value) { switch (operation.Type) { case CounterOperationType.Increment: case CounterOperationType.Delete: case CounterOperationType.Put: LoadDocument(); if (doc != null) { docCollection = CollectionName.GetCollectionName(doc.Data); } break; } switch (operation.Type) { case CounterOperationType.Increment: LastChangeVector = _database.DocumentsStorage.CountersStorage.IncrementCounter(context, docId, docCollection, operation.CounterName, operation.Delta); GetCounterValue(context, _database, docId, operation.CounterName, _replyWithAllNodesValues, CountersDetail); break; case CounterOperationType.Delete: if (_fromEtl && doc == null) { break; } LastChangeVector = _database.DocumentsStorage.CountersStorage.DeleteCounter(context, docId, docCollection, operation.CounterName); break; case CounterOperationType.Put: if (_fromEtl && doc == null) { break; } // intentionally not setting LastChangeVector, we never use it for // etl / import and it isn't meaningful in those scenarios if (_fromEtl) { _database.DocumentsStorage.CountersStorage.PutCounter(context, docId, docCollection, operation.CounterName, operation.Delta); } else { _database.DocumentsStorage.CountersStorage.PutCounter(context, docId, docCollection, operation.CounterName, operation.ChangeVector, operation.Delta); } break; case CounterOperationType.None: break; case CounterOperationType.Get: GetCounterValue(context, _database, docId, operation.CounterName, _replyWithAllNodesValues, CountersDetail); break; default: ThrowInvalidBatchOperationType(operation); break; } } if (doc != null) { _database.DocumentsStorage.CountersStorage.UpdateDocumentCounters(context, doc.Data, docId, kvp.Value); } void LoadDocument() { if (doc != null) { return; } try { doc = _database.DocumentsStorage.Get(context, docId, throwOnConflict: true); if (doc == null) { if (_fromEtl) { return; } ThrowMissingDocument(docId); return; // never hit } if (doc.Flags.HasFlag(DocumentFlags.Artificial)) { ThrowArtificialDocument(doc); } } catch (DocumentConflictException) { if (_fromEtl) { return; } // this is fine, we explicitly support // setting the flag if we are in conflicted state is // done by the conflict resolver // avoid loading same document again, we validate write using the metadata instance doc = new Document(); } } } return(CountersDetail.Counters.Count); }
public abstract Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token);