public IEnumerable <CounterDetail> GetCountersFrom(DocumentsOperationContext context, string collection, long etag, int skip, int take)
{
var collectionName = _documentsStorage.GetCollection(collection, throwIfDoesNotExist: false);
if (collectionName == null)
{
yield break;
}
var table = GetCountersTable(context.Transaction.InnerTransaction, collectionName);
if (table == null)
{
yield break;
}
foreach (var result in table.SeekForwardFrom(CountersSchema.FixedSizeIndexes[CollectionCountersEtagsSlice], etag, skip))
{
if (take-- <= 0)
{
yield break;
}
yield return(TableValueToCounterDetail(context, result.Reader));
}
}
public bool IsEmpty()
{
if (CollectionName == string.Empty)
{
return(_documentStorage.GetNumberOfDocuments() == 0);
}
return(_documentStorage.GetCollection(CollectionName, _context).Count == 0);
}
private IEnumerable <Document> GetDocuments()
{
IEnumerable <Document> documents;
if (_startsWith != null)
{
if (_isAllDocsCollection)
{
documents = _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, null, _start, _query.PageSize);
}
else
{
documents = _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, null, _start, _query.PageSize, _collection);
}
}
else if (_ids != null)
{
if (_ids.Count == 0)
{
documents = Enumerable.Empty <Document>();
}
else
{
documents = _isAllDocsCollection
? _documents.GetDocuments(_context, _ids, _start, _query.PageSize, _totalResults)
: _documents.GetDocuments(_context, _ids, _collection, _start, _query.PageSize, _totalResults);
}
}
else if (_isAllDocsCollection)
{
documents = _documents.GetDocumentsFrom(_context, 0, _start, _query.PageSize);
_totalResults.Value = (int)_documents.GetNumberOfDocuments(_context);
}
else
{
documents = _documents.GetDocumentsFrom(_context, _collection, 0, _start, _query.PageSize);
_totalResults.Value = (int)_documents.GetCollection(_collection, _context).Count;
}
return(documents);
}
private IEnumerable <Document> GetDocuments()
{
IEnumerable <Document> documents;
if (_ids != null && _ids.Count > 0)
{
documents = _documents.GetDocuments(_context, _ids, _start, _query.PageSize, _totalResults);
}
else if (_isAllDocsCollection)
{
documents = _documents.GetDocumentsFrom(_context, 0, _start, _query.PageSize);
_totalResults.Value = (int)_documents.GetNumberOfDocuments(_context);
}
else
{
documents = _documents.GetDocumentsFrom(_context, _collection, 0, _start, _query.PageSize);
_totalResults.Value = (int)_documents.GetCollection(_collection, _context).Count;
}
return(ApplySorting(documents));
}
private unsafe void FillCountOfResultsAndIndexEtag(QueryResultServerSide resultToFill, string collection)
{
var buffer = stackalloc long[3];
if (collection == Constants.Documents.Collections.AllDocumentsCollection)
{
var numberOfDocuments = _documents.GetNumberOfDocuments(_context);
buffer[0] = DocumentsStorage.ReadLastDocumentEtag(_context.Transaction.InnerTransaction);
buffer[1] = DocumentsStorage.ReadLastTombstoneEtag(_context.Transaction.InnerTransaction);
buffer[2] = numberOfDocuments;
resultToFill.TotalResults = (int)numberOfDocuments;
}
else
{
var collectionStats = _documents.GetCollection(collection, _context);
buffer[0] = _documents.GetLastDocumentEtag(_context, collection);
buffer[1] = _documents.GetLastTombstoneEtag(_context, collection);
buffer[2] = collectionStats.Count;
}
resultToFill.ResultEtag = (long)Hashing.XXHash64.Calculate((byte *)buffer, sizeof(long) * 3);
}
public void AddConflict(
DocumentsOperationContext context,
string id,
long lastModifiedTicks,
BlittableJsonReaderObject incomingDoc,
string incomingChangeVector,
string incomingTombstoneCollection,
DocumentFlags flags,
NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Adding conflict to {id} (Incoming change vector {incomingChangeVector})");
}
var tx = context.Transaction.InnerTransaction;
var conflictsTable = tx.OpenTable(ConflictsSchema, ConflictsSlice);
var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler;
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
CollectionName collectionName;
// ReSharper disable once ArgumentsStyleLiteral
var existing = _documentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false);
if (existing.Document != null)
{
var existingDoc = existing.Document;
if (fromSmuggler == false)
{
using (Slice.From(context.Allocator, existingDoc.ChangeVector, out Slice cv))
using (DocumentIdWorker.GetStringPreserveCase(context, CollectionName.GetLazyCollectionNameFrom(context, existingDoc.Data), out Slice collectionSlice))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv);
tvb.Add(idPtr);
tvb.Add(existingDoc.Data.BasePointer, existingDoc.Data.Size);
tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag()));
tvb.Add(collectionSlice);
tvb.Add(existingDoc.LastModified.Ticks);
tvb.Add((int)existingDoc.Flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
}
// we delete the data directly, without generating a tombstone, because we have a
// conflict instead
_documentsStorage.EnsureLastEtagIsPersisted(context, existingDoc.Etag);
collectionName = _documentsStorage.ExtractCollectionName(context, existingDoc.Data);
//make sure that the relevant collection tree exists
var table = tx.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
table.Delete(existingDoc.StorageId);
}
else if (existing.Tombstone != null)
{
var existingTombstone = existing.Tombstone;
if (fromSmuggler == false)
{
using (Slice.From(context.Allocator, existingTombstone.ChangeVector, out var cv))
using (DocumentIdWorker.GetStringPreserveCase(context, existingTombstone.Collection, out Slice collectionSlice))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv);
tvb.Add(idPtr);
tvb.Add(null, 0);
tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag()));
tvb.Add(collectionSlice);
tvb.Add(existingTombstone.LastModified.Ticks);
tvb.Add((int)existingTombstone.Flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
}
// we delete the data directly, without generating a tombstone, because we have a
// conflict instead
_documentsStorage.EnsureLastEtagIsPersisted(context, existingTombstone.Etag);
collectionName = _documentsStorage.GetCollection(existingTombstone.Collection, throwIfDoesNotExist: true);
var table = tx.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones));
table.Delete(existingTombstone.StorageId);
}
else // has existing conflicts
{
collectionName = _documentsStorage.ExtractCollectionName(context, incomingDoc);
using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice))
{
var conflicts = GetConflictsFor(context, prefixSlice);
foreach (var conflict in conflicts)
{
var conflictStatus = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, conflict.ChangeVector);
switch (conflictStatus)
{
case ConflictStatus.Update:
DeleteConflictsFor(context, conflict.ChangeVector); // delete this, it has been subsumed
break;
case ConflictStatus.Conflict:
if (fromSmuggler &&
DocumentCompare.IsEqualTo(conflict.Doc, incomingDoc, false) == DocumentCompareResult.Equal)
{
return; // we already have a conflict with equal content, no need to create another one
}
break; // we'll add this conflict if no one else also includes it
case ConflictStatus.AlreadyMerged:
return; // we already have a conflict that includes this version
default:
throw new ArgumentOutOfRangeException("Invalid conflict status " + conflictStatus);
}
}
}
}
var etag = _documentsStorage.GenerateNextEtag();
if (context.LastDatabaseChangeVector == null)
{
context.LastDatabaseChangeVector = GetDatabaseChangeVector(context);
}
var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbBase64Id, etag, context.LastDatabaseChangeVector);
if (result.IsValid)
{
context.LastDatabaseChangeVector = result.ChangeVector;
}
byte * doc = null;
var docSize = 0;
string collection;
if (incomingDoc != null) // can be null if it is a tombstone
{
doc = incomingDoc.BasePointer;
docSize = incomingDoc.Size;
collection = CollectionName.GetLazyCollectionNameFrom(context, incomingDoc);
}
else
{
collection = incomingTombstoneCollection;
}
using (Slice.From(context.Allocator, incomingChangeVector, out var cv))
using (DocumentIdWorker.GetStringPreserveCase(context, collection, out Slice collectionSlice))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv);
tvb.Add(idPtr);
tvb.Add(doc, docSize);
tvb.Add(Bits.SwapBytes(etag));
tvb.Add(collectionSlice);
tvb.Add(lastModifiedTicks);
tvb.Add((int)flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = incomingChangeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Conflict,
});
}
}
private IEnumerable <Document> GetDocuments()
{
IEnumerable <Document> documents;
if (_startsWith != null)
{
var countQuery = false;
if (_query.PageSize == 0)
{
countQuery = true;
_query.PageSize = int.MaxValue;
}
if (_isAllDocsCollection)
{
documents = _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, null, _start, _query.PageSize);
}
else
{
documents = _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, null, _start, _query.PageSize, _collection);
}
if (countQuery)
{
foreach (var document in documents)
{
using (document.Data)
_totalResults.Value++;
}
documents = Enumerable.Empty <Document>();
_query.PageSize = 0;
}
}
else if (_ids != null)
{
if (_ids.Count == 0)
{
documents = Enumerable.Empty <Document>();
}
else
{
documents = _isAllDocsCollection
? _documents.GetDocuments(_context, _ids, _start, _query.PageSize, _totalResults)
: _documents.GetDocuments(_context, _ids, _collection, _start, _query.PageSize, _totalResults);
}
}
else if (_isAllDocsCollection)
{
documents = _documents.GetDocumentsFrom(_context, 0, _start, _query.PageSize);
_totalResults.Value = (int)_documents.GetNumberOfDocuments(_context);
}
else
{
documents = _documents.GetDocumentsFrom(_context, _collection, 0, _start, _query.PageSize);
_totalResults.Value = (int)_documents.GetCollection(_collection, _context).Count;
}
return(documents);
}
private void ExecuteCollectionQuery(QueryResultServerSide resultToFill, IndexQueryServerSide query, string collection)
{
var isAllDocsCollection = collection == Constants.Indexing.AllDocumentsCollection;
// we optimize for empty queries without sorting options
resultToFill.IndexName = isAllDocsCollection ? "AllDocs" : collection;
resultToFill.IsStale = false;
resultToFill.ResultEtag = Environment.TickCount;
resultToFill.LastQueryTime = DateTime.MinValue;
resultToFill.IndexTimestamp = DateTime.MinValue;
_context.OpenReadTransaction();
if (isAllDocsCollection)
{
resultToFill.TotalResults = (int)_documents.GetNumberOfDocuments(_context);
}
else
{
var collectionStats = _documents.GetCollection(collection, _context);
resultToFill.TotalResults = (int)collectionStats.Count;
}
var includeDocumentsCommand = new IncludeDocumentsCommand(_documents, _context, query.Includes);
Transformer transformer = null;
if (string.IsNullOrEmpty(query.Transformer) == false)
{
transformer = _transformerStore.GetTransformer(query.Transformer);
if (transformer == null)
{
throw new InvalidOperationException($"The transformer '{query.Transformer}' was not found.");
}
}
using (var scope = transformer?.OpenTransformationScope(query.TransformerParameters, includeDocumentsCommand, _documents, _transformerStore, _context))
{
var fieldsToFetch = new FieldsToFetch(query, null, transformer);
var documents = new CollectionQueryEnumerable(_documents, fieldsToFetch, collection, query, _context);
var results = scope != null?scope.Transform(documents) : documents;
try
{
foreach (var document in results)
{
_token.Token.ThrowIfCancellationRequested();
resultToFill.AddResult(document);
includeDocumentsCommand.Gather(document);
}
}
catch (Exception e)
{
if (resultToFill.SupportsExceptionHandling == false)
{
throw;
}
resultToFill.HandleException(e);
}
}
includeDocumentsCommand.Fill(resultToFill.Includes);
}
private IEnumerable <Document> GetDocuments()
{
IEnumerable <Document> documents;
if (_startsWith != null)
{
var countQuery = false;
if (_query.PageSize == 0)
{
countQuery = true;
_query.PageSize = int.MaxValue;
}
documents = _isAllDocsCollection
? _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, _startAfterId, _start, _query.PageSize, fields: _fields)
: _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, _startAfterId, _start, _query.PageSize, _collection, _fields);
if (countQuery)
{
foreach (var document in documents)
{
using (document.Data)
_totalResults.Value++;
}
documents = Enumerable.Empty <Document>();
_query.PageSize = 0;
}
}
else if (_ids != null)
{
if (_ids.Count == 0)
{
documents = Enumerable.Empty <Document>();
}
else if (_alreadySeenIdsCount != null)
{
var idsLeft = _ids.Count - _alreadySeenIdsCount.Value;
if (idsLeft == 0)
{
documents = Enumerable.Empty <Document>();
}
else
{
var count = idsLeft >= _query.PageSize ? _query.PageSize : idsLeft;
var ids = _ids.Skip((int)_alreadySeenIdsCount.Value).Take((int)count);
_alreadySeenIdsCount.Value += count;
documents = _isAllDocsCollection
? _documents.GetDocuments(_context, ids, 0, _query.PageSize, _totalResults)
: _documents.GetDocuments(_context, ids, _collection, 0, _query.PageSize, _totalResults);
}
}
else
{
documents = _isAllDocsCollection
? _documents.GetDocuments(_context, _ids, _start, _query.PageSize, _totalResults)
: _documents.GetDocuments(_context, _ids, _collection, _start, _query.PageSize, _totalResults);
}
}
else if (_isAllDocsCollection)
{
documents = _documents.GetDocumentsFrom(_context, 0, _start, _query.PageSize);
_totalResults.Value = (int)_documents.GetNumberOfDocuments(_context);
}
else
{
documents = _documents.GetDocumentsFrom(_context, _collection, 0, _start, _query.PageSize);
_totalResults.Value = (int)_documents.GetCollection(_collection, _context).Count;
}
return(documents);
}
