protected Task <IOperationResult> ExecuteDelete(IndexQueryServerSide query, Index index, QueryOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(query, index, options, context, onProgress, (key, retrieveDetails) =>
{
var command = new DeleteDocumentCommand(key, null, Database);
return new BulkOperationCommand <DeleteDocumentCommand>(command, retrieveDetails, x => new BulkOperationResult.DeleteDetails
{
Id = key,
Etag = x.DeleteResult?.Etag
});
}, token));
}
private void GetDocuments(DocumentsOperationContext context, Transformer transformer, bool metadataOnly)
{
// everything here operates on all docs
var actualEtag = ComputeAllDocumentsEtag(context);
if (GetLongFromHeaders("If-None-Match") == actualEtag)
{
HttpContext.Response.StatusCode = 304;
return;
}
HttpContext.Response.Headers["ETag"] = "\"" + actualEtag + "\"";
var etag = GetLongQueryString("etag", false);
IEnumerable <Document> documents;
bool isLoadStartingWith = false;
if (etag != null)
{
documents = Database.DocumentsStorage.GetDocumentsFrom(context, etag.Value, GetStart(), GetPageSize());
}
else if (HttpContext.Request.Query.ContainsKey("startsWith"))
{
isLoadStartingWith = true;
documents = Database.DocumentsStorage.GetDocumentsStartingWith(context,
HttpContext.Request.Query["startsWith"],
HttpContext.Request.Query["matches"],
HttpContext.Request.Query["excludes"],
GetStart(),
GetPageSize()
);
}
else // recent docs
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, GetStart(), GetPageSize());
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
if (isLoadStartingWith)
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
}
if (transformer != null)
{
var transformerParameters = GetTransformerParameters(context);
using (var scope = transformer.OpenTransformationScope(transformerParameters, null, Database.DocumentsStorage,
Database.TransformerStore, context))
{
writer.WriteDocuments(context, scope.Transform(documents).ToList(), metadataOnly);
}
}
else
{
writer.WriteDocuments(context, documents, metadataOnly);
}
if (isLoadStartingWith)
{
writer.WriteEndObject();
}
}
}
protected override long ExecuteCmd(DocumentsOperationContext context)
{
throw new NotSupportedException("Should only call Execute() here");
}
public RetrieveDocumentIdsVisitor(TransactionOperationContext serverContext, DocumentsOperationContext context, QueryMetadata metadata, ByteStringContext allocator) : base(metadata.Query.QueryText)
{
_query = metadata.Query;
_serverContext = serverContext;
_context = context;
_metadata = metadata;
_allocator = allocator;
}
public StreamDestination(Stream stream, DocumentsOperationContext context, DatabaseSource source)
{
_stream = stream;
_context = context;
_source = source;
}
public TermsQueryResultServerSide ExecuteGetTermsQuery(string indexName, string field, string fromValue, long?existingResultEtag, int pageSize, DocumentsOperationContext context, OperationCancelToken token, out Index index)
{
Exception lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
index = GetIndex(indexName);
var etag = index.GetIndexEtag(null);
if (etag == existingResultEtag)
{
return(TermsQueryResultServerSide.NotModifiedResult);
}
return(index.GetTerms(field, fromValue, pageSize, context, token));
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
WaitForIndexBeingLikelyReplacedDuringQuery().GetAwaiter().GetResult();
}
catch (OperationCanceledException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
if (token.Token.IsCancellationRequested)
{
throw;
}
lastException = e;
WaitForIndexBeingLikelyReplacedDuringQuery().GetAwaiter().GetResult();
}
}
throw CreateRetriesFailedException(lastException);
}
public override async Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
Exception lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
return(await GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, context, onProgress, token));
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
catch (OperationCanceledException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
if (token.Token.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
}
throw CreateRetriesFailedException(lastException);
}
public override async Task <DocumentQueryResult> ExecuteQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token)
{
var sw = Stopwatch.StartNew();
var result = await GetRunner(query).ExecuteQuery(query, documentsContext, existingResultEtag, token);
result.DurationInMs = (long)sw.Elapsed.TotalMilliseconds;
return(result);
}
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamDocumentQueryResultWriter writer, OperationCancelToken token)
{
return(GetRunner(query).ExecuteStreamQuery(query, documentsContext, response, writer, token));
}
public override Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, context, onProgress, token));
}
public List <DynamicQueryToIndexMatcher.Explanation> ExplainDynamicIndexSelection(IndexQueryServerSide query, DocumentsOperationContext context)
{
if (query.Metadata.IsDynamic == false)
{
throw new InvalidOperationException("Explain can only work on dynamic indexes");
}
if (_dynamic is DynamicQueryRunner d)
{
return(d.ExplainIndexSelection(query, context));
}
throw new NotSupportedException(InvalidQueryRunner.ErrorMessage);
}
public override Task <IOperationResult> ExecuteDeleteQuery(IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(GetRunner(query).ExecuteDeleteQuery(query, options, context, onProgress, token));
}
public override Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
return(GetRunner(query).ExecuteIndexEntriesQuery(query, context, existingResultEtag, token));
}
protected Task <IOperationResult> ExecutePatch(IndexQueryServerSide query, Index index, QueryOperationOptions options, PatchRequest patch,
BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(query, index, options, context, onProgress, (key, retrieveDetails) =>
{
var command = new PatchDocumentCommand(context, key,
expectedChangeVector: null,
skipPatchIfChangeVectorMismatch: false,
patch: (patch, patchArgs),
patchIfMissing: (null, null),
database: Database,
debugMode: false,
isTest: false);
return new BulkOperationCommand <PatchDocumentCommand>(command, retrieveDetails, x => new BulkOperationResult.PatchDetails
{
Id = key,
ChangeVector = x.PatchResult.ChangeVector,
Status = x.PatchResult.Status
});
}, token));
public override async Task ExecuteStreamIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response,
IStreamQueryResultWriter <BlittableJsonReaderObject> writer, OperationCancelToken token)
{
Exception lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
documentsContext.CloseTransaction();
await GetRunner(query).ExecuteStreamIndexEntriesQuery(query, documentsContext, response, writer, token);
return;
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
catch (OperationCanceledException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
if (token.Token.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
}
throw CreateRetriesFailedException(lastException);
}
public async Task <FacetedQueryResult> ExecuteFacetedQuery(IndexQueryServerSide query, long?existingResultEtag, DocumentsOperationContext documentsContext, OperationCancelToken token)
{
if (query.Metadata.IsDynamic)
{
throw new InvalidQueryException("Facet query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters);
}
var sw = Stopwatch.StartNew();
var result = await _static.ExecuteFacetedQuery(query, existingResultEtag, documentsContext, token);
result.DurationInMs = (long)sw.Elapsed.TotalMilliseconds;
return(result);
}
public async Task <FacetedQueryResult> ExecuteFacetedQuery(IndexQueryServerSide query, long?existingResultEtag, DocumentsOperationContext documentsContext, OperationCancelToken token)
{
if (query.Metadata.IsDynamic)
{
throw new InvalidQueryException("Facet query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters);
}
Exception lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
var sw = Stopwatch.StartNew();
var result = await _static.ExecuteFacetedQuery(query, existingResultEtag, documentsContext, token);
result.DurationInMs = (long)sw.Elapsed.TotalMilliseconds;
return(result);
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
catch (OperationCanceledException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
if (token.Token.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
}
throw CreateRetriesFailedException(lastException);
}
public TermsQueryResultServerSide ExecuteGetTermsQuery(Index index, string field, string fromValue, long?existingResultEtag, int pageSize, DocumentsOperationContext context, OperationCancelToken token)
{
var etag = index.GetIndexEtag();
if (etag == existingResultEtag)
{
return(TermsQueryResultServerSide.NotModifiedResult);
}
return(index.GetTerms(field, fromValue, pageSize, context, token));
}
public override async Task <IndexEntriesQueryResult> ExecuteIndexEntriesQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
Exception lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
return(await GetRunner(query).ExecuteIndexEntriesQuery(query, context, existingResultEtag, token));
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
catch (OperationCanceledException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
if (token.Token.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
}
throw CreateRetriesFailedException(lastException);
}
public async Task <SuggestionQueryResult> ExecuteSuggestionQuery(IndexQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
if (query.Metadata.IsDynamic)
{
throw new InvalidQueryException("Suggestion query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters);
}
var sw = Stopwatch.StartNew();
if (query.Metadata.SelectFields.Length != 1 || query.Metadata.SelectFields[0].IsSuggest == false)
{
throw new InvalidQueryException("Suggestion query must have one suggest token in SELECT.", query.Metadata.QueryText, query.QueryParameters);
}
var selectField = (SuggestionField)query.Metadata.SelectFields[0];
var index = GetIndex(query.Metadata.IndexName);
var indexDefinition = index.GetIndexDefinition();
if (indexDefinition.Fields.TryGetValue(selectField.Name, out IndexFieldOptions field) == false)
{
throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' does not have a field '{selectField.Name}'.");
}
if (field.Suggestions == null)
{
throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' does not have suggestions configured for field '{selectField.Name}'.");
}
if (field.Suggestions.Value == false)
{
throw new InvalidOperationException($"Index '{query.Metadata.IndexName}' have suggestions explicitly disabled for field '{selectField.Name}'.");
}
if (existingResultEtag.HasValue)
{
var etag = index.GetIndexEtag();
if (etag == existingResultEtag.Value)
{
return(SuggestionQueryResult.NotModifiedResult);
}
}
var result = await index.SuggestionQuery(query, context, token);
result.DurationInMs = (int)sw.Elapsed.TotalMilliseconds;
return(result);
}
public override async Task <DocumentQueryResult> ExecuteQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token)
{
Exception lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
Stopwatch sw = null;
QueryTimingsScope scope;
DocumentQueryResult result;
using (scope = query.Timings?.Start())
{
if (scope == null)
{
sw = Stopwatch.StartNew();
}
result = await GetRunner(query).ExecuteQuery(query, documentsContext, existingResultEtag, token);
}
result.DurationInMs = sw != null ? (long)sw.Elapsed.TotalMilliseconds : (long)scope.Duration.TotalMilliseconds;
return(result);
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
catch (OperationCanceledException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
if (token.Token.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
}
throw CreateRetriesFailedException(lastException);
}
public (BlittableJsonReaderObject Document, string Id) Test(MigrationTestSettings settings, DatabaseSchema dbSchema, DocumentsOperationContext context)
{
using (var enumerationConnection = OpenConnection())
using (var referencesConnection = OpenConnection())
{
var collectionToImport = settings.Collection;
var tableSchema = dbSchema.GetTable(collectionToImport.SourceTableSchema, collectionToImport.SourceTableName);
var specialColumns = dbSchema.FindSpecialColumns(collectionToImport.SourceTableSchema, collectionToImport.SourceTableName);
string CollectionNameProvider(string schema, string name) => settings.CollectionsMapping.Single(x => x.TableSchema == schema && x.TableName == name).CollectionName;
using (var patcher = new JsPatcher(collectionToImport, context))
{
var references = ResolveReferences(collectionToImport, dbSchema, CollectionNameProvider);
InitializeDataProviders(references, referencesConnection);
try
{
Dictionary <string, object> queryParameters = null;
var queryToUse = settings.Mode == MigrationTestMode.First
? GetQueryForCollection(collectionToImport) // we limit rows count internally by passing rowsLimit: 1
: GetQueryByPrimaryKey(collectionToImport, tableSchema, settings.PrimaryKeyValues, out queryParameters);
foreach (var doc in EnumerateTable(queryToUse, collectionToImport.ColumnsMapping, specialColumns,
collectionToImport.AttachmentNameMapping, enumerationConnection, rowsLimit: 1, queryParameters: queryParameters))
{
doc.SetCollection(collectionToImport.Name);
var id = GenerateDocumentId(doc.Collection, GetColumns(doc.SpecialColumnsValues, tableSchema.PrimaryKeyColumns));
FillDocumentFields(doc.Object, doc.SpecialColumnsValues, references, "", doc.Attachments);
return(patcher.Patch(doc.ToBlittable(context)), id);
}
}
catch (JavaScriptException e)
{
if (e.InnerException is Jint.Runtime.JavaScriptException innerException && string.Equals(innerException.Message, "skip", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Document was skipped", e);
}
private (List <Slice>, string) ExtractIdsFromQuery(IndexQueryServerSide query, DocumentsOperationContext context)
{
if (string.IsNullOrWhiteSpace(query.Query))
{
return(null, null);
}
if (query.Metadata.Query.Where == null)
{
return(null, null);
}
if (query.Metadata.IndexFieldNames.Contains(QueryFieldName.DocumentId) == false)
{
return(null, null);
}
IDisposable releaseServerContext = null;
IDisposable closeServerTransaction = null;
TransactionOperationContext serverContext = null;
try
{
if (query.Metadata.HasCmpXchg)
{
releaseServerContext = context.DocumentDatabase.ServerStore.ContextPool.AllocateOperationContext(out serverContext);
closeServerTransaction = serverContext.OpenReadTransaction();
}
using (closeServerTransaction)
{
var idsRetriever = new RetrieveDocumentIdsVisitor(serverContext, context, query.Metadata, _context.Allocator);
idsRetriever.Visit(query.Metadata.Query.Where, query.QueryParameters);
return(idsRetriever.Ids?.OrderBy(x => x, SliceComparer.Instance).ToList(), idsRetriever.StartsWith);
}
}
finally
{
releaseServerContext?.Dispose();
}
}
public PutOperationResults PutDocument(DocumentsOperationContext context, string id,
string expectedChangeVector,
BlittableJsonReaderObject document,
long?lastModifiedTicks = null,
string changeVector = null,
DocumentFlags flags = DocumentFlags.None,
NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
{
if (context.Transaction == null)
{
ThrowRequiresTransaction();
return(default(PutOperationResults)); // never hit
}
#if DEBUG
var documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
#endif
var newEtag = _documentsStorage.GenerateNextEtag();
var modifiedTicks = _documentsStorage.GetOrCreateLastModifiedTicks(lastModifiedTicks);
id = BuildDocumentId(id, newEtag, out bool knownNewId);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var collectionName = _documentsStorage.ExtractCollectionName(context, document);
var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
var oldValue = default(TableValueReader);
if (knownNewId == false)
{
// delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it
DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size);
table.ReadByKey(lowerId, out oldValue);
}
BlittableJsonReaderObject oldDoc = null;
if (oldValue.Pointer == null)
{
// expectedChangeVector being null means we don't care,
// and empty means that it must be new
if (string.IsNullOrEmpty(expectedChangeVector) == false)
{
ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector);
}
}
else
{
// expectedChangeVector has special meaning here
// null - means, don't care, don't check
// "" / empty - means, must be new
// anything else - must match exactly
if (expectedChangeVector != null)
{
var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue);
if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0)
{
ThrowConcurrentException(id, expectedChangeVector, oldChangeVector);
}
}
oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context);
var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc);
if (oldCollectionName != collectionName)
{
ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName);
}
var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue);
if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
(nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
{
if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
flags |= DocumentFlags.HasAttachments;
}
}
}
var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue);
if (string.IsNullOrEmpty(result.ChangeVector))
{
ChangeVectorUtils.ThrowConflictingEtag(id, changeVector, newEtag, _documentsStorage.Environment.Base64Id, _documentDatabase.ServerStore.NodeTag);
}
changeVector = result.ChangeVector;
nonPersistentFlags |= result.NonPersistentFlags;
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved))
{
flags |= DocumentFlags.Resolved;
}
if (collectionName.IsHiLo == false &&
(flags & DocumentFlags.Artificial) != DocumentFlags.Artificial)
{
if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags))
{
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
#endif
document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
#if DEBUG
documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
}
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false &&
(flags.Contain(DocumentFlags.Resolved) ||
_documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null
))
{
var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document,
ref flags, out RevisionsCollectionConfiguration configuration);
if (shouldVersion)
{
_documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags,
changeVector, modifiedTicks, configuration, collectionName);
}
}
}
using (Slice.From(context.Allocator, changeVector, out var cv))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(Bits.SwapBytes(newEtag));
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(modifiedTicks);
tvb.Add((int)flags);
tvb.Add(context.GetTransactionMarker());
if (oldValue.Pointer == null)
{
table.Insert(tvb);
}
else
{
table.Update(oldValue.Id, tvb);
}
}
if (collectionName.IsHiLo == false)
{
_documentsStorage.ExpirationStorage.Put(context, lowerId, document);
}
context.LastDatabaseChangeVector = changeVector;
_documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1);
_documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size);
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = changeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Put,
});
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
return(new PutOperationResults
{
Etag = newEtag,
Id = id,
Collection = collectionName,
ChangeVector = changeVector,
Flags = flags,
LastModified = new DateTime(modifiedTicks)
});
}
}
public override IQueryResultRetriever GetQueryResultRetriever(IndexQueryServerSide query, QueryTimingsScope queryTimings, DocumentsOperationContext documentsContext, FieldsToFetch fieldsToFetch, IncludeDocumentsCommand includeDocumentsCommand, IncludeCompareExchangeValuesCommand includeCompareExchangeValuesCommand)
{
throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e);
}
public void Should_be_able_to_read_index_stats_even_if_corruption_happened()
{
UseNewLocalServer();
using (var db = CreateDocumentDatabase())
{
using (var index = MapIndex.CreateNew(new IndexDefinition()
{
Name = "Users_ByName",
Maps =
{
"from user in docs.Users select new { user.Name }"
},
Type = IndexType.Map
}, db))
{
PutUser(db);
index._indexStorage.SimulateCorruption = true;
db.ServerStore.DatabasesLandlord.CatastrophicFailureHandler.MaxDatabaseUnloads = 0;
var mre = new ManualResetEventSlim();
db.Changes.OnIndexChange += change =>
{
if (change.Type == IndexChangeTypes.IndexMarkedAsErrored)
{
mre.Set();
}
};
index.Start();
Assert.True(mre.Wait(TimeSpan.FromMinutes(1)));
Assert.Equal(IndexState.Error, index.State);
long errorCount = 0;
for (int i = 0; i < 20; i++)
{
errorCount = index.GetErrorCount();
if (errorCount > 0)
{
break;
}
Thread.Sleep(500); // errors are updated in next tx when we update the stats
}
Assert.Equal(1, errorCount);
using (var context = DocumentsOperationContext.ShortTermSingleUse(db))
{
using (context.OpenReadTransaction())
{
var indexStats = index.GetIndexStats(context);
Assert.True(indexStats.IsStale);
}
}
var indexingErrors = index.GetErrors();
Assert.Equal(1, indexingErrors.Count);
}
}
}
private void GetDocumentsById(DocumentsOperationContext context, StringValues ids, Transformer transformer, bool metadataOnly)
{
var includePaths = HttpContext.Request.Query["include"];
var documents = new List <Document>(ids.Count);
List <long> etags = null;
var includes = new List <Document>(includePaths.Count * ids.Count);
var includeDocs = new IncludeDocumentsCommand(Database.DocumentsStorage, context, includePaths);
foreach (var id in ids)
{
var document = Database.DocumentsStorage.Get(context, id);
documents.Add(document);
includeDocs.Gather(document);
}
IEnumerable <Document> documentsToWrite;
if (transformer != null)
{
var transformerParameters = GetTransformerParameters(context);
using (var scope = transformer.OpenTransformationScope(transformerParameters, includeDocs, Database.DocumentsStorage, Database.TransformerStore, context))
{
documentsToWrite = scope.Transform(documents).ToList();
etags = scope.LoadedDocumentEtags;
}
}
else
{
documentsToWrite = documents;
}
includeDocs.Fill(includes);
var actualEtag = ComputeEtagsFor(documents, includes, etags);
if (transformer != null)
{
actualEtag ^= transformer.Hash;
}
var etag = GetLongFromHeaders("If-None-Match");
if (etag == actualEtag)
{
HttpContext.Response.StatusCode = 304;
return;
}
HttpContext.Response.Headers[Constants.MetadataEtagField] = "\"" + actualEtag + "\"";
if (HttpContext.Request.Query["blittable"] == "true")
{
WriteDocumentsBlittable(context, documentsToWrite, includes);
}
else
{
WriteDocumentsJson(context, metadataOnly, documentsToWrite, includeDocs, includes);
}
}
public SqlDocumentTransformer(Transformation transformation, DocumentDatabase database, DocumentsOperationContext context, SqlEtlConfiguration config)
: base(database, context, new PatchRequest(transformation.Script, PatchRequestType.SqlEtl), null)
{
_transformation = transformation;
_config = config;
var destinationTables = transformation.GetCollectionsFromScript();
LoadToDestinations = destinationTables;
_tables = new Dictionary <string, SqlTableWithRecords>(destinationTables.Length);
_tablesForScript = new List <SqlEtlTable>(destinationTables.Length);
// ReSharper disable once ForCanBeConvertedToForeach
for (var i = 0; i < _config.SqlTables.Count; i++)
{
var table = _config.SqlTables[i];
if (destinationTables.Contains(table.TableName, StringComparer.OrdinalIgnoreCase))
{
_tablesForScript.Add(table);
}
}
if (_transformation.IsLoadingAttachments)
{
_loadedAttachments = new Dictionary <string, Queue <Attachment> >(StringComparer.OrdinalIgnoreCase);
}
}
public override int Execute(DocumentsOperationContext context)
{
if (_database.ServerStore.Server.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable)
{
FeaturesAvailabilityException.Throw("Counters");
}
foreach (var kvp in _dictionary)
{
Document doc = null;
var docId = kvp.Key;
string docCollection = null;
foreach (var operation in kvp.Value)
{
switch (operation.Type)
{
case CounterOperationType.Increment:
case CounterOperationType.Delete:
case CounterOperationType.Put:
LoadDocument();
if (doc != null)
{
docCollection = CollectionName.GetCollectionName(doc.Data);
}
break;
}
switch (operation.Type)
{
case CounterOperationType.Increment:
LastChangeVector =
_database.DocumentsStorage.CountersStorage.IncrementCounter(context, docId, docCollection, operation.CounterName, operation.Delta);
GetCounterValue(context, _database, docId, operation.CounterName, _replyWithAllNodesValues, CountersDetail);
break;
case CounterOperationType.Delete:
if (_fromEtl && doc == null)
{
break;
}
LastChangeVector = _database.DocumentsStorage.CountersStorage.DeleteCounter(context, docId, docCollection, operation.CounterName);
break;
case CounterOperationType.Put:
if (_fromEtl && doc == null)
{
break;
}
// intentionally not setting LastChangeVector, we never use it for
// etl / import and it isn't meaningful in those scenarios
if (_fromEtl)
{
_database.DocumentsStorage.CountersStorage.PutCounter(context, docId, docCollection,
operation.CounterName, operation.Delta);
}
else
{
_database.DocumentsStorage.CountersStorage.PutCounter(context, docId, docCollection,
operation.CounterName, operation.ChangeVector, operation.Delta);
}
break;
case CounterOperationType.None:
break;
case CounterOperationType.Get:
GetCounterValue(context, _database, docId, operation.CounterName, _replyWithAllNodesValues, CountersDetail);
break;
default:
ThrowInvalidBatchOperationType(operation);
break;
}
}
if (doc != null)
{
_database.DocumentsStorage.CountersStorage.UpdateDocumentCounters(context, doc.Data, docId, kvp.Value);
}
void LoadDocument()
{
if (doc != null)
{
return;
}
try
{
doc = _database.DocumentsStorage.Get(context, docId,
throwOnConflict: true);
if (doc == null)
{
if (_fromEtl)
{
return;
}
ThrowMissingDocument(docId);
return; // never hit
}
if (doc.Flags.HasFlag(DocumentFlags.Artificial))
{
ThrowArtificialDocument(doc);
}
}
catch (DocumentConflictException)
{
if (_fromEtl)
{
return;
}
// this is fine, we explicitly support
// setting the flag if we are in conflicted state is
// done by the conflict resolver
// avoid loading same document again, we validate write using the metadata instance
doc = new Document();
}
}
}
return(CountersDetail.Counters.Count);
}
public abstract Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch,
BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token);
