public IEnumerable <CounterDetail> GetCountersFrom(DocumentsOperationContext context, string collection, long etag, int skip, int take) { var collectionName = _documentsStorage.GetCollection(collection, throwIfDoesNotExist: false); if (collectionName == null) { yield break; } var table = GetCountersTable(context.Transaction.InnerTransaction, collectionName); if (table == null) { yield break; } foreach (var result in table.SeekForwardFrom(CountersSchema.FixedSizeIndexes[CollectionCountersEtagsSlice], etag, skip)) { if (take-- <= 0) { yield break; } yield return(TableValueToCounterDetail(context, result.Reader)); } }
public bool IsEmpty() { if (CollectionName == string.Empty) { return(_documentStorage.GetNumberOfDocuments() == 0); } return(_documentStorage.GetCollection(CollectionName, _context).Count == 0); }
private IEnumerable <Document> GetDocuments() { IEnumerable <Document> documents; if (_startsWith != null) { if (_isAllDocsCollection) { documents = _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, null, _start, _query.PageSize); } else { documents = _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, null, _start, _query.PageSize, _collection); } } else if (_ids != null) { if (_ids.Count == 0) { documents = Enumerable.Empty <Document>(); } else { documents = _isAllDocsCollection ? _documents.GetDocuments(_context, _ids, _start, _query.PageSize, _totalResults) : _documents.GetDocuments(_context, _ids, _collection, _start, _query.PageSize, _totalResults); } } else if (_isAllDocsCollection) { documents = _documents.GetDocumentsFrom(_context, 0, _start, _query.PageSize); _totalResults.Value = (int)_documents.GetNumberOfDocuments(_context); } else { documents = _documents.GetDocumentsFrom(_context, _collection, 0, _start, _query.PageSize); _totalResults.Value = (int)_documents.GetCollection(_collection, _context).Count; } return(documents); }
private IEnumerable <Document> GetDocuments() { IEnumerable <Document> documents; if (_ids != null && _ids.Count > 0) { documents = _documents.GetDocuments(_context, _ids, _start, _query.PageSize, _totalResults); } else if (_isAllDocsCollection) { documents = _documents.GetDocumentsFrom(_context, 0, _start, _query.PageSize); _totalResults.Value = (int)_documents.GetNumberOfDocuments(_context); } else { documents = _documents.GetDocumentsFrom(_context, _collection, 0, _start, _query.PageSize); _totalResults.Value = (int)_documents.GetCollection(_collection, _context).Count; } return(ApplySorting(documents)); }
private unsafe void FillCountOfResultsAndIndexEtag(QueryResultServerSide resultToFill, string collection) { var buffer = stackalloc long[3]; if (collection == Constants.Documents.Collections.AllDocumentsCollection) { var numberOfDocuments = _documents.GetNumberOfDocuments(_context); buffer[0] = DocumentsStorage.ReadLastDocumentEtag(_context.Transaction.InnerTransaction); buffer[1] = DocumentsStorage.ReadLastTombstoneEtag(_context.Transaction.InnerTransaction); buffer[2] = numberOfDocuments; resultToFill.TotalResults = (int)numberOfDocuments; } else { var collectionStats = _documents.GetCollection(collection, _context); buffer[0] = _documents.GetLastDocumentEtag(_context, collection); buffer[1] = _documents.GetLastTombstoneEtag(_context, collection); buffer[2] = collectionStats.Count; } resultToFill.ResultEtag = (long)Hashing.XXHash64.Calculate((byte *)buffer, sizeof(long) * 3); }
public void AddConflict( DocumentsOperationContext context, string id, long lastModifiedTicks, BlittableJsonReaderObject incomingDoc, string incomingChangeVector, string incomingTombstoneCollection, DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (_logger.IsInfoEnabled) { _logger.Info($"Adding conflict to {id} (Incoming change vector {incomingChangeVector})"); } var tx = context.Transaction.InnerTransaction; var conflictsTable = tx.OpenTable(ConflictsSchema, ConflictsSlice); var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler; using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { CollectionName collectionName; // ReSharper disable once ArgumentsStyleLiteral var existing = _documentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false); if (existing.Document != null) { var existingDoc = existing.Document; if (fromSmuggler == false) { using (Slice.From(context.Allocator, existingDoc.ChangeVector, out Slice cv)) using (DocumentIdWorker.GetStringPreserveCase(context, CollectionName.GetLazyCollectionNameFrom(context, existingDoc.Data), out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(existingDoc.Data.BasePointer, existingDoc.Data.Size); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(collectionSlice); tvb.Add(existingDoc.LastModified.Ticks); tvb.Add((int)existingDoc.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingDoc.Etag); collectionName = _documentsStorage.ExtractCollectionName(context, existingDoc.Data); //make sure that the relevant collection tree exists var table = tx.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); table.Delete(existingDoc.StorageId); } else if (existing.Tombstone != null) { var existingTombstone = existing.Tombstone; if (fromSmuggler == false) { using (Slice.From(context.Allocator, existingTombstone.ChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, existingTombstone.Collection, out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(null, 0); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(collectionSlice); tvb.Add(existingTombstone.LastModified.Ticks); tvb.Add((int)existingTombstone.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingTombstone.Etag); collectionName = _documentsStorage.GetCollection(existingTombstone.Collection, throwIfDoesNotExist: true); var table = tx.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones)); table.Delete(existingTombstone.StorageId); } else // has existing conflicts { collectionName = _documentsStorage.ExtractCollectionName(context, incomingDoc); using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice)) { var conflicts = GetConflictsFor(context, prefixSlice); foreach (var conflict in conflicts) { var conflictStatus = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, conflict.ChangeVector); switch (conflictStatus) { case ConflictStatus.Update: DeleteConflictsFor(context, conflict.ChangeVector); // delete this, it has been subsumed break; case ConflictStatus.Conflict: if (fromSmuggler && DocumentCompare.IsEqualTo(conflict.Doc, incomingDoc, false) == DocumentCompareResult.Equal) { return; // we already have a conflict with equal content, no need to create another one } break; // we'll add this conflict if no one else also includes it case ConflictStatus.AlreadyMerged: return; // we already have a conflict that includes this version default: throw new ArgumentOutOfRangeException("Invalid conflict status " + conflictStatus); } } } } var etag = _documentsStorage.GenerateNextEtag(); if (context.LastDatabaseChangeVector == null) { context.LastDatabaseChangeVector = GetDatabaseChangeVector(context); } var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbBase64Id, etag, context.LastDatabaseChangeVector); if (result.IsValid) { context.LastDatabaseChangeVector = result.ChangeVector; } byte * doc = null; var docSize = 0; string collection; if (incomingDoc != null) // can be null if it is a tombstone { doc = incomingDoc.BasePointer; docSize = incomingDoc.Size; collection = CollectionName.GetLazyCollectionNameFrom(context, incomingDoc); } else { collection = incomingTombstoneCollection; } using (Slice.From(context.Allocator, incomingChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collection, out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(doc, docSize); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(collectionSlice); tvb.Add(lastModifiedTicks); tvb.Add((int)flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = incomingChangeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Conflict, }); } }
private IEnumerable <Document> GetDocuments() { IEnumerable <Document> documents; if (_startsWith != null) { var countQuery = false; if (_query.PageSize == 0) { countQuery = true; _query.PageSize = int.MaxValue; } if (_isAllDocsCollection) { documents = _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, null, _start, _query.PageSize); } else { documents = _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, null, _start, _query.PageSize, _collection); } if (countQuery) { foreach (var document in documents) { using (document.Data) _totalResults.Value++; } documents = Enumerable.Empty <Document>(); _query.PageSize = 0; } } else if (_ids != null) { if (_ids.Count == 0) { documents = Enumerable.Empty <Document>(); } else { documents = _isAllDocsCollection ? _documents.GetDocuments(_context, _ids, _start, _query.PageSize, _totalResults) : _documents.GetDocuments(_context, _ids, _collection, _start, _query.PageSize, _totalResults); } } else if (_isAllDocsCollection) { documents = _documents.GetDocumentsFrom(_context, 0, _start, _query.PageSize); _totalResults.Value = (int)_documents.GetNumberOfDocuments(_context); } else { documents = _documents.GetDocumentsFrom(_context, _collection, 0, _start, _query.PageSize); _totalResults.Value = (int)_documents.GetCollection(_collection, _context).Count; } return(documents); }
private void ExecuteCollectionQuery(QueryResultServerSide resultToFill, IndexQueryServerSide query, string collection) { var isAllDocsCollection = collection == Constants.Indexing.AllDocumentsCollection; // we optimize for empty queries without sorting options resultToFill.IndexName = isAllDocsCollection ? "AllDocs" : collection; resultToFill.IsStale = false; resultToFill.ResultEtag = Environment.TickCount; resultToFill.LastQueryTime = DateTime.MinValue; resultToFill.IndexTimestamp = DateTime.MinValue; _context.OpenReadTransaction(); if (isAllDocsCollection) { resultToFill.TotalResults = (int)_documents.GetNumberOfDocuments(_context); } else { var collectionStats = _documents.GetCollection(collection, _context); resultToFill.TotalResults = (int)collectionStats.Count; } var includeDocumentsCommand = new IncludeDocumentsCommand(_documents, _context, query.Includes); Transformer transformer = null; if (string.IsNullOrEmpty(query.Transformer) == false) { transformer = _transformerStore.GetTransformer(query.Transformer); if (transformer == null) { throw new InvalidOperationException($"The transformer '{query.Transformer}' was not found."); } } using (var scope = transformer?.OpenTransformationScope(query.TransformerParameters, includeDocumentsCommand, _documents, _transformerStore, _context)) { var fieldsToFetch = new FieldsToFetch(query, null, transformer); var documents = new CollectionQueryEnumerable(_documents, fieldsToFetch, collection, query, _context); var results = scope != null?scope.Transform(documents) : documents; try { foreach (var document in results) { _token.Token.ThrowIfCancellationRequested(); resultToFill.AddResult(document); includeDocumentsCommand.Gather(document); } } catch (Exception e) { if (resultToFill.SupportsExceptionHandling == false) { throw; } resultToFill.HandleException(e); } } includeDocumentsCommand.Fill(resultToFill.Includes); }
private IEnumerable <Document> GetDocuments() { IEnumerable <Document> documents; if (_startsWith != null) { var countQuery = false; if (_query.PageSize == 0) { countQuery = true; _query.PageSize = int.MaxValue; } documents = _isAllDocsCollection ? _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, _startAfterId, _start, _query.PageSize, fields: _fields) : _documents.GetDocumentsStartingWith(_context, _startsWith, null, null, _startAfterId, _start, _query.PageSize, _collection, _fields); if (countQuery) { foreach (var document in documents) { using (document.Data) _totalResults.Value++; } documents = Enumerable.Empty <Document>(); _query.PageSize = 0; } } else if (_ids != null) { if (_ids.Count == 0) { documents = Enumerable.Empty <Document>(); } else if (_alreadySeenIdsCount != null) { var idsLeft = _ids.Count - _alreadySeenIdsCount.Value; if (idsLeft == 0) { documents = Enumerable.Empty <Document>(); } else { var count = idsLeft >= _query.PageSize ? _query.PageSize : idsLeft; var ids = _ids.Skip((int)_alreadySeenIdsCount.Value).Take((int)count); _alreadySeenIdsCount.Value += count; documents = _isAllDocsCollection ? _documents.GetDocuments(_context, ids, 0, _query.PageSize, _totalResults) : _documents.GetDocuments(_context, ids, _collection, 0, _query.PageSize, _totalResults); } } else { documents = _isAllDocsCollection ? _documents.GetDocuments(_context, _ids, _start, _query.PageSize, _totalResults) : _documents.GetDocuments(_context, _ids, _collection, _start, _query.PageSize, _totalResults); } } else if (_isAllDocsCollection) { documents = _documents.GetDocumentsFrom(_context, 0, _start, _query.PageSize); _totalResults.Value = (int)_documents.GetNumberOfDocuments(_context); } else { documents = _documents.GetDocumentsFrom(_context, _collection, 0, _start, _query.PageSize); _totalResults.Value = (int)_documents.GetCollection(_collection, _context).Count; } return(documents); }