private void ApplyMetadataModifications(LazyStringValue id, DocumentInfo documentInfo) { if (documentInfo.Metadata.Modifications == null) { return; } documentInfo.MetadataInstance = null; using (documentInfo.Document) using (documentInfo.Metadata) { documentInfo.Metadata = _session.Context.ReadObject(documentInfo.Metadata, id); documentInfo.Metadata.Modifications = null; documentInfo.Document.Modifications = new DynamicJsonValue(documentInfo.Document) { [Constants.Documents.Metadata.Key] = documentInfo.Metadata }; documentInfo.Document = _session.Context.ReadObject(documentInfo.Document, id); documentInfo.Document.Modifications = null; } }
private (bool AllExpired, LazyStringValue Id) GetConflictedExpiration(DocumentsOperationContext context, DateTime currentTime, Slice clonedId) { LazyStringValue id = null; var allExpired = true; var conflicts = _database.DocumentsStorage.ConflictsStorage.GetConflictsFor(context, clonedId); if (conflicts.Count > 0) { foreach (var conflict in conflicts) { id = conflict.Id; if (HasPassed(conflict.Doc, currentTime)) { continue; } allExpired = false; break; } } return(allExpired, id); }
public IEnumerable <string> GetCountersForDocument(DocumentsOperationContext context, string docId) { var table = new Table(CountersSchema, context.Transaction.InnerTransaction); using (GetCounterPartialKey(context, docId, out var key)) { LazyStringValue prev = null; foreach (var result in table.SeekByPrimaryKeyPrefix(key, Slices.Empty, 0)) { var current = ExtractCounterName(context, result.Value.Reader); if (prev?.Equals(current) == true) { // already seen this one, skip it continue; } yield return(current); prev?.Dispose(); prev = current; } } }
private void HandleTreeReduction(TransactionOperationContext indexContext, IndexingStatsScope stats, MapReduceResultsStore modifiedStore, LowLevelTransaction lowLevelTransaction, IndexWriteOperation writer, LazyStringValue reduceKeyHash, Table table, CancellationToken token) { EnsureValidTreeReductionStats(stats); var tree = modifiedStore.Tree; var branchesToAggregate = new HashSet <long>(); var parentPagesToAggregate = new HashSet <long>(); var page = new TreePage(null, Constants.Storage.PageSize); HashSet <long> compressedEmptyLeafs = null; Dictionary <long, Exception> failedAggregatedLeafs = null; foreach (var modifiedPage in modifiedStore.ModifiedPages) { token.ThrowIfCancellationRequested(); page.Base = lowLevelTransaction.GetPage(modifiedPage).Pointer; stats.RecordReduceTreePageModified(page.IsLeaf); if (page.IsLeaf == false) { Debug.Assert(page.IsBranch); branchesToAggregate.Add(modifiedPage); continue; } var leafPage = page; var compressed = leafPage.IsCompressed; if (compressed) { stats.RecordCompressedLeafPage(); } using (compressed ? (DecompressedLeafPage)(leafPage = tree.DecompressPage(leafPage, skipCache: true)) : null) { if (leafPage.NumberOfEntries == 0) { if (leafPage.PageNumber == tree.State.RootPageNumber) { writer.DeleteReduceResult(reduceKeyHash, stats); var emptyPageNumber = Bits.SwapBytes(leafPage.PageNumber); using (Slice.External(indexContext.Allocator, (byte *)&emptyPageNumber, sizeof(long), out Slice pageNumSlice)) table.DeleteByKey(pageNumSlice); continue; } if (compressed) { // it doesn't have any entries after decompression because // each compressed entry has the delete tombstone if (compressedEmptyLeafs == null) { compressedEmptyLeafs = new HashSet <long>(); } compressedEmptyLeafs.Add(leafPage.PageNumber); continue; } throw new UnexpectedReduceTreePageException( $"Encountered empty page which isn't a root. Page {leafPage} in '{tree.Name}' tree."); } var parentPage = tree.GetParentPageOf(leafPage); stats.RecordReduceAttempts(leafPage.NumberOfEntries); try { using (var result = AggregateLeafPage(leafPage, lowLevelTransaction, indexContext, token)) { if (parentPage == -1) { writer.DeleteReduceResult(reduceKeyHash, stats); foreach (var output in result.GetOutputs()) { writer.IndexDocument(reduceKeyHash, output, stats, indexContext); } } else { StoreAggregationResult(leafPage, table, result); parentPagesToAggregate.Add(parentPage); } _metrics.MapReduceIndexes.ReducedPerSec.Mark(leafPage.NumberOfEntries); stats.RecordReduceSuccesses(leafPage.NumberOfEntries); } } catch (Exception e) when(e is OperationCanceledException == false) { if (failedAggregatedLeafs == null) { failedAggregatedLeafs = new Dictionary <long, Exception>(); } failedAggregatedLeafs.Add(leafPage.PageNumber, e); _index.ErrorIndexIfCriticalException(e); HandleReductionError(e, reduceKeyHash, writer, stats, updateStats: parentPage == -1, page: leafPage); } } } long tmp = 0; using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice)) { foreach (var freedPage in modifiedStore.FreedPages) { tmp = Bits.SwapBytes(freedPage); table.DeleteByKey(pageNumberSlice); } } while (parentPagesToAggregate.Count > 0 || branchesToAggregate.Count > 0) { token.ThrowIfCancellationRequested(); var branchPages = parentPagesToAggregate; parentPagesToAggregate = new HashSet <long>(); foreach (var pageNumber in branchPages) { page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer; try { if (page.IsBranch == false) { throw new UnexpectedReduceTreePageException("Parent page was found that wasn't a branch, error at " + page); } stats.RecordReduceAttempts(page.NumberOfEntries); var parentPage = tree.GetParentPageOf(page); using (var result = AggregateBranchPage(page, table, indexContext, branchesToAggregate, compressedEmptyLeafs, failedAggregatedLeafs, token)) { if (parentPage == -1) { writer.DeleteReduceResult(reduceKeyHash, stats); foreach (var output in result.GetOutputs()) { writer.IndexDocument(reduceKeyHash, output, stats, indexContext); } } else { parentPagesToAggregate.Add(parentPage); StoreAggregationResult(page, table, result); } _metrics.MapReduceIndexes.ReducedPerSec.Mark(page.NumberOfEntries); stats.RecordReduceSuccesses(page.NumberOfEntries); } } catch (Exception e) when(e is OperationCanceledException == false) { _index.ErrorIndexIfCriticalException(e); HandleReductionError(e, reduceKeyHash, writer, stats, updateStats: true, page: page); } finally { branchesToAggregate.Remove(pageNumber); } } if (parentPagesToAggregate.Count == 0 && branchesToAggregate.Count > 0) { // we still have unaggregated branches which were modified but their children were not modified (branch page splitting) so we missed them parentPagesToAggregate.Add(branchesToAggregate.First()); } } if (compressedEmptyLeafs != null && compressedEmptyLeafs.Count > 0) { // we had some compressed pages that are empty after decompression // let's remove them and reduce the tree once again modifiedStore.ModifiedPages.Clear(); modifiedStore.FreedPages.Clear(); foreach (var pageNumber in compressedEmptyLeafs) { page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer; using (var emptyPage = tree.DecompressPage(page, skipCache: true)) { if (emptyPage.NumberOfEntries > 0) // could be changed meanwhile { continue; } modifiedStore.Tree.RemoveEmptyDecompressedPage(emptyPage); } } HandleTreeReduction(indexContext, stats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token); } }
public static ByteStringContext.InternalScope From(ByteStringContext context, LazyStringValue value, out Slice str) { return(From(context, value.Buffer, value.Size, ByteStringType.Immutable, out str)); }
public PatchDocumentCommand( JsonOperationContext context, string id, LazyStringValue expectedChangeVector, bool skipPatchIfChangeVectorMismatch, (PatchRequest run, BlittableJsonReaderObject args) patch,
public static ByteStringContext.InternalScope GetLower(ByteStringContext byteStringContext, LazyStringValue str, out Slice loweredKey) { return(GetLower(byteStringContext, str.Buffer, str.Size, out loweredKey)); }
protected abstract int GetFields<T>(T instance, LazyStringValue key, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer) where T : ILuceneDocumentWrapper;
public static ConflictStatus GetConflictStatusForDocument(DocumentsOperationContext context, string id, LazyStringValue remote, out string conflictingVector) { //tombstones also can be a conflict entry conflictingVector = null; var conflicts = context.DocumentDatabase.DocumentsStorage.ConflictsStorage.GetConflictsFor(context, id); if (conflicts.Count > 0) { foreach (var existingConflict in conflicts) { if (ChangeVectorUtils.GetConflictStatus(remote, existingConflict.ChangeVector) == ConflictStatus.Conflict) { conflictingVector = existingConflict.ChangeVector; return(ConflictStatus.Conflict); } } // this document will resolve the conflicts when putted return(ConflictStatus.Update); } var result = context.DocumentDatabase.DocumentsStorage.GetDocumentOrTombstone(context, id); string local; if (result.Document != null) { local = result.Document.ChangeVector; } else if (result.Tombstone != null) { local = result.Tombstone.ChangeVector; } else { return(ConflictStatus.Update); //document with 'id' doesn't exist locally, so just do PUT } var status = ChangeVectorUtils.GetConflictStatus(remote, local); if (status == ConflictStatus.Conflict) { conflictingVector = local; } return(status); }
public IEnumerable <Slice> GetDocumentKeysFromCollectionThatReference(string collection, LazyStringValue referenceKey, RavenTransaction tx) { var collectionTree = tx.InnerTransaction.ReadTree("#" + collection); if (collectionTree == null) { yield break; } using (DocumentIdWorker.GetLower(tx.InnerTransaction.Allocator, referenceKey, out var k)) using (var it = collectionTree.MultiRead(k)) { if (it.Seek(Slices.BeforeAllKeys) == false) { yield break; } do { yield return(it.CurrentKey); } while (it.MoveNext()); } }
public static (LazyStringValue DocId, string CounterName) ExtractDocIdAndCounterNameFromTombstone(JsonOperationContext context, LazyStringValue counterTombstoneId) { var p = counterTombstoneId.Buffer; var size = counterTombstoneId.Size; int sizeOfDocId = 0; for (; sizeOfDocId < size; sizeOfDocId++) { if (p[sizeOfDocId] == SpecialChars.RecordSeparator) { break; } } var doc = context.AllocateStringValue(null, p, sizeOfDocId); var name = Encoding.UTF8.GetString(p + sizeOfDocId + 1, size - (sizeOfDocId + 2)); return(doc, name); }
private static bool CompareBlittableArray(string fieldPath, string id, BlittableJsonReaderArray oldArray, BlittableJsonReaderArray newArray, IDictionary <string, DocumentsChanges[]> changes, List <DocumentsChanges> docChanges, LazyStringValue propName) { // if we don't care about the changes if (oldArray.Length != newArray.Length && changes == null) { return(true); } var position = 0; var changed = false; while (position < oldArray.Length && position < newArray.Length && (changed == false || changes != null)) { switch (oldArray[position]) { case BlittableJsonReaderObject bjro1: if (newArray[position] is BlittableJsonReaderObject bjro2) { changed |= CompareBlittable(AddIndexFieldPath(fieldPath, position), id, bjro1, bjro2, changes, docChanges); } else { changed = true; if (changes != null) { NewChange(AddIndexFieldPath(fieldPath, position), propName, newArray[position], oldArray[position], docChanges, DocumentsChanges.ChangeType.ArrayValueChanged); } } break; case BlittableJsonReaderArray bjra1: if (newArray[position] is BlittableJsonReaderArray bjra2) { changed |= CompareBlittableArray(AddIndexFieldPath(fieldPath, position), id, bjra1, bjra2, changes, docChanges, propName); } else { changed = true; if (changes != null) { NewChange(AddIndexFieldPath(fieldPath, position), propName, newArray[position], oldArray[position], docChanges, DocumentsChanges.ChangeType.ArrayValueChanged); } } break; case null: if (newArray[position] != null) { changed = true; if (changes != null) { NewChange(AddIndexFieldPath(fieldPath, position), propName, newArray[position], oldArray[position], docChanges, DocumentsChanges.ChangeType.ArrayValueChanged); } } break; default: if (oldArray[position].Equals(newArray[position]) == false) { if (changes != null) { NewChange(AddIndexFieldPath(fieldPath, position), propName, newArray[position], oldArray[position], docChanges, DocumentsChanges.ChangeType.ArrayValueChanged); } changed = true; } break; } position++; } if (changes == null) { return(changed); } // if one of the arrays is larger than the other while (position < oldArray.Length) { NewChange(fieldPath, propName, null, oldArray[position], docChanges, DocumentsChanges.ChangeType.ArrayValueRemoved); position++; } while (position < newArray.Length) { NewChange(fieldPath, propName, newArray[position], null, docChanges, DocumentsChanges.ChangeType.ArrayValueAdded); position++; } return(changed); }
public async Task <ImportResult> Import(DocumentsOperationContext context, Stream stream, Action <IOperationProgress> onProgress = null) { var result = new ImportResult(); var progress = new IndeterminateProgress(); var state = new JsonParserState(); JsonOperationContext.ManagedPinnedBuffer buffer; using (context.GetManagedBuffer(out buffer)) using (var parser = new UnmanagedJsonParser(context, state, "fileName")) { var operateOnType = "__top_start_object"; var buildVersion = 0L; var identities = new Dictionary <string, long>(); VersioningStorage versioningStorage = null; while (true) { if (parser.Read() == false) { var read = await stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length); if (read == 0) { if (state.CurrentTokenType != JsonParserToken.EndObject) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } break; } parser.SetBuffer(buffer, read); continue; } switch (state.CurrentTokenType) { case JsonParserToken.String: unsafe { operateOnType = new LazyStringValue(null, state.StringBuffer, state.StringSize, context).ToString(); } break; case JsonParserToken.Integer: switch (operateOnType) { case "BuildVersion": buildVersion = state.Long; break; } break; case JsonParserToken.StartObject: if (operateOnType == "__top_start_object") { operateOnType = null; break; } context.CachedProperties.NewDocument(); var builder = new BlittableJsonDocumentBuilder(_batchPutCommand.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "ImportObject", parser, state); builder.ReadNestedObject(); while (builder.Read() == false) { var read = await stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } parser.SetBuffer(buffer, read); } builder.FinalizeDocument(); if (operateOnType == "Docs" && Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents)) { progress.Progress = "Importing Documents"; onProgress?.Invoke(progress); PatchDocument patch = null; PatchRequest patchRequest = null; if (string.IsNullOrWhiteSpace(Options.TransformScript) == false) { patch = new PatchDocument(context.DocumentDatabase); patchRequest = new PatchRequest { Script = Options.TransformScript }; } result.DocumentsCount++; var reader = builder.CreateReader(); var document = new Document { Data = reader, }; if (Options.IncludeExpired == false && document.Expired(_database.Time.GetUtcNow())) { continue; } TransformScriptOrDisableVersioningIfNeeded(context, patch, reader, document, patchRequest); _batchPutCommand.Add(document.Data); if (result.DocumentsCount % 1000 == 0) { progress.Progress = $"Imported {result.DocumentsCount} documents"; onProgress?.Invoke(progress); } await HandleBatchOfDocuments(context, parser, buildVersion).ConfigureAwait(false); } else if (operateOnType == "RevisionDocuments" && Options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments)) { if (versioningStorage == null) { break; } result.RevisionDocumentsCount++; var reader = builder.CreateReader(); _batchPutCommand.Add(reader); await HandleBatchOfDocuments(context, parser, buildVersion).ConfigureAwait(false);; } else { using (builder) { switch (operateOnType) { case "Attachments": result.Warnings.Add("Attachments are not supported anymore. Use RavenFS isntead. Skipping."); break; case "Indexes": if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes) == false) { continue; } result.IndexesCount++; progress.Progress = "importing Indexes"; onProgress?.Invoke(progress); try { IndexProcessor.Import(builder, _database, buildVersion, Options.RemoveAnalyzers); } catch (Exception e) { result.Warnings.Add($"Could not import index. Message: {e.Message}"); } break; case "Transformers": if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Transformers) == false) { continue; } result.TransformersCount++; progress.Progress = "Importing Transformers"; onProgress?.Invoke(progress); try { TransformerProcessor.Import(builder, _database, buildVersion); } catch (Exception e) { result.Warnings.Add($"Could not import transformer. Message: {e.Message}"); } break; case "Identities": if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Identities)) { result.IdentitiesCount++; progress.Progress = "Importing Identities"; onProgress?.Invoke(progress); using (var reader = builder.CreateReader()) { try { string identityKey, identityValueString; long identityValue; if (reader.TryGet("Key", out identityKey) == false || reader.TryGet("Value", out identityValueString) == false || long.TryParse(identityValueString, out identityValue) == false) { result.Warnings.Add($"Cannot import the following identity: '{reader}'. Skipping."); } else { identities[identityKey] = identityValue; } } catch (Exception e) { result.Warnings.Add($"Cannot import the following identity: '{reader}'. Error: {e}. Skipping."); } } } break; default: result.Warnings.Add( $"The following type is not recognized: '{operateOnType}'. Skipping."); break; } } } break; case JsonParserToken.StartArray: switch (operateOnType) { case "RevisionDocuments": // We are taking a reference here since the documents import can activate or disable the versioning. // We hold a local copy because the user can disable the bundle during the import process, exteranly. // In this case we want to continue to import the revisions documents. versioningStorage = _database.BundleLoader.VersioningStorage; _batchPutCommand.IsRevision = true; break; } break; case JsonParserToken.EndArray: switch (operateOnType) { case "Docs": await FinishBatchOfDocuments(); _batchPutCommand = new MergedBatchPutCommand(_database, buildVersion); break; case "RevisionDocuments": await FinishBatchOfDocuments(); break; case "Identities": if (identities.Count > 0) { using (var tx = context.OpenWriteTransaction()) { _database.DocumentsStorage.UpdateIdentities(context, identities); tx.Commit(); } } identities = null; break; } break; } } } return(result); }
public override void DeleteReduceResult(LazyStringValue reduceKeyHash, IndexingStatsScope stats) { base.DeleteReduceResult(reduceKeyHash, stats); _outputReduceToCollectionCommandBatcher.DeleteReduce(reduceKeyHash); }
public override void Delete(LazyStringValue key, IndexingStatsScope stats) { throw new NotSupportedException("Deleting index entries by id() field isn't supported by map-reduce indexes"); }
protected unsafe void SetLazyStringValueFromString(DocumentsOperationContext context, out LazyStringValue prop) { prop = null; var size = *(int *)Reader.ReadExactly(sizeof(int)); if (size < 0) { return; } // This is a special (broken) case. // On the source it is stored as Slice in LSV format which is wrong(?) unlike the normal LSV the escaping position is kept before the value itself. // and therefore the escaping doesn't include in the LSV size. // Additionally, we are over-allocating so writing this value doesn't cause a failure (we look for the escaping after the value) // this also work, because we don't pass those values between contexts, if we need to do so, we convert it to string first. // TODO: this is inefficient, can skip string allocation prop = context.GetLazyString(Encoding.UTF8.GetString(Reader.ReadExactly(size), size)); }
public override int HandleMap(LazyStringValue lowerId, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats) { throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e); }
private string GetPrefixedId(LazyStringValue documentId, string loadCollectionName) { return($"{documentId}/{_script.IdPrefixForCollection[loadCollectionName]}/"); }
protected unsafe int PutMapResults(LazyStringValue lowerId, IEnumerable <MapResult> mappedResults, TransactionOperationContext indexContext, IndexingStatsScope stats) { EnsureValidStats(stats); using (Slice.External(indexContext.Allocator, lowerId.Buffer, lowerId.Length, out Slice docIdAsSlice)) { Queue <MapEntry> existingEntries = null; using (_stats.GetMapEntriesTree.Start()) MapReduceWorkContext.DocumentMapEntries.RepurposeInstance(docIdAsSlice, clone: false); if (MapReduceWorkContext.DocumentMapEntries.NumberOfEntries > 0) { using (_stats.GetMapEntries.Start()) existingEntries = GetMapEntries(MapReduceWorkContext.DocumentMapEntries); } int resultsCount = 0; foreach (var mapResult in mappedResults) { using (mapResult.Data) { resultsCount++; var reduceKeyHash = mapResult.ReduceKeyHash; long id = -1; if (existingEntries?.Count > 0) { var existing = existingEntries.Dequeue(); var storeOfExisting = GetResultsStore(existing.ReduceKeyHash, indexContext, false); if (reduceKeyHash == existing.ReduceKeyHash) { using (var existingResult = storeOfExisting.Get(existing.Id)) { if (ResultsBinaryEqual(mapResult.Data, existingResult.Data)) { continue; } } id = existing.Id; } else { using (_stats.RemoveResult.Start()) { MapReduceWorkContext.DocumentMapEntries.Delete(existing.Id); storeOfExisting.Delete(existing.Id); } } } using (_stats.PutResult.Start()) { if (id == -1) { id = MapReduceWorkContext.NextMapResultId++; using (Slice.External(indexContext.Allocator, (byte *)&reduceKeyHash, sizeof(ulong), out Slice val)) MapReduceWorkContext.DocumentMapEntries.Add(id, val); } GetResultsStore(reduceKeyHash, indexContext, create: true).Add(id, mapResult.Data); } } } HandleIndexOutputsPerDocument(lowerId, resultsCount, stats); DocumentDatabase.Metrics.MapReduceIndexes.MappedPerSec.Mark(resultsCount); while (existingEntries?.Count > 0) { // need to remove remaining old entries var oldResult = existingEntries.Dequeue(); var oldState = GetResultsStore(oldResult.ReduceKeyHash, indexContext, create: false); using (_stats.RemoveResult.Start()) { oldState.Delete(oldResult.Id); MapReduceWorkContext.DocumentMapEntries.Delete(oldResult.Id); } } return(resultsCount); } }
internal async Task CleanupExpiredDocs() { var currentTime = _database.Time.GetUtcNow(); var currentTicks = currentTime.Ticks; try { if (Logger.IsInfoEnabled) { Logger.Info("Trying to find expired documents to delete"); } using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { using (var tx = context.OpenReadTransaction()) { var expirationTree = tx.InnerTransaction.CreateTree(DocumentsByExpiration); Dictionary <Slice, List <(Slice LowerId, LazyStringValue Id)> > expired; Stopwatch duration; using (var it = expirationTree.Iterate(false)) { if (it.Seek(Slices.BeforeAllKeys) == false) { return; } expired = new Dictionary <Slice, List <(Slice LowerId, LazyStringValue Id)> >(); duration = Stopwatch.StartNew(); do { var entryTicks = it.CurrentKey.CreateReader().ReadBigEndianInt64(); if (entryTicks >= currentTicks) { break; } var ticksAsSlice = it.CurrentKey.Clone(tx.InnerTransaction.Allocator); var expiredDocs = new List <(Slice LowerId, LazyStringValue Id)>(); expired.Add(ticksAsSlice, expiredDocs); using (var multiIt = expirationTree.MultiRead(it.CurrentKey)) { if (multiIt.Seek(Slices.BeforeAllKeys)) { do { if (CancellationToken.IsCancellationRequested) { return; } var clonedId = multiIt.CurrentKey.Clone(tx.InnerTransaction.Allocator); try { var document = _database.DocumentsStorage.Get(context, clonedId); if (document == null) { expiredDocs.Add((clonedId, null)); continue; } if (HasExpired(document.Data, currentTime) == false) { continue; } expiredDocs.Add((clonedId, document.Id)); } catch (DocumentConflictException) { LazyStringValue id = null; var allExpired = true; var conflicts = _database.DocumentsStorage.ConflictsStorage.GetConflictsFor(context, clonedId); if (conflicts.Count == 0) { continue; } foreach (var conflict in conflicts) { id = conflict.Id; if (HasExpired(conflict.Doc, currentTime)) { continue; } allExpired = false; break; } if (allExpired) { expiredDocs.Add((clonedId, id)); } } } while (multiIt.MoveNext()); } } } while (it.MoveNext()); } var command = new DeleteExpiredDocumentsCommand(expired, _database, Logger); await _database.TxMerger.Enqueue(command); if (Logger.IsInfoEnabled) { Logger.Info($"Successfully deleted {command.DeletionCount:#,#;;0} documents in {duration.ElapsedMilliseconds:#,#;;0} ms."); } } } } catch (Exception e) { if (Logger.IsOperationsEnabled) { Logger.Operations($"Failed to delete expired documents on {_database.Name} which are older than {currentTime}", e); } } }
protected Field GetOrCreateField(string name, string value, LazyStringValue lazyValue, BlittableJsonReaderObject blittableValue, Field.Store store, Field.Index index, Field.TermVector termVector) { int cacheKey = FieldCacheKey.GetHashCode(name, index, store, termVector, _multipleItemsSameFieldCount); Field field; if (_fieldsCache.TryGetValue(cacheKey, out CachedFieldItem<Field> cached) == false || !cached.Key.IsSame(name, index, store, termVector, _multipleItemsSameFieldCount)) { LazyStringReader stringReader = null; BlittableObjectReader blittableReader = null; if ((lazyValue != null || blittableValue != null) && store.IsStored() == false && index.IsIndexed() && index.IsAnalyzed()) { TextReader reader; if (lazyValue != null) { stringReader = new LazyStringReader(); reader = stringReader.GetTextReaderFor(lazyValue); } else { blittableReader = Scope.GetBlittableReader(); reader = blittableReader.GetTextReaderFor(blittableValue); } field = new Field(name, reader, termVector); } else { if (value == null && lazyValue == null) blittableReader = Scope.GetBlittableReader(); field = new Field(name, value ?? LazyStringReader.GetStringFor(lazyValue) ?? blittableReader.GetStringFor(blittableValue), store, index, termVector); } field.Boost = 1; field.OmitNorms = true; _fieldsCache[cacheKey] = new CachedFieldItem<Field> { Key = new FieldCacheKey(name, index, store, termVector, _multipleItemsSameFieldCount.ToArray()), Field = field, LazyStringReader = stringReader }; } else { BlittableObjectReader blittableReader = null; field = cached.Field; if (lazyValue != null && cached.LazyStringReader == null) cached.LazyStringReader = new LazyStringReader(); if (blittableValue != null) blittableReader = Scope.GetBlittableReader(); if ((lazyValue != null || blittableValue != null) && store.IsStored() == false && index.IsIndexed() && index.IsAnalyzed()) { field.SetValue(lazyValue != null ? cached.LazyStringReader.GetTextReaderFor(lazyValue) : blittableReader.GetTextReaderFor(blittableValue)); } else { field.SetValue(value ?? LazyStringReader.GetStringFor(lazyValue) ?? blittableReader.GetStringFor(blittableValue)); } } return field; }
private static ulong CalculateSecondaryHash(LazyStringValue key) { return(Hashing.XXHash64.CalculateInline(key.Buffer, (ulong)key.Size, seed: 1337)); }
protected override int GetFields <T>(T instance, LazyStringValue key, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer) { if (!(document is ObjectInstance documentToProcess)) { return(0); } int newFields = 0; if (key != null) { instance.Add(GetOrCreateKeyField(key)); newFields++; } if (_reduceOutput) { var reduceResult = JsBlittableBridge.Translate(indexContext, documentToProcess.Engine, documentToProcess); instance.Add(GetReduceResultValueField(reduceResult, writeBuffer)); newFields++; } foreach (var(property, propertyDescriptor) in documentToProcess.GetOwnProperties()) { if (_fields.TryGetValue(property, out var field) == false) { field = _fields[property] = IndexField.Create(property, new IndexFieldOptions(), _allFields); } object value; var actualValue = propertyDescriptor.Value; if (actualValue.IsObject() && actualValue.IsArray() == false) { //In case TryDetectDynamicFieldCreation finds a dynamic field it will populate 'field.Name' with the actual property name //so we must use field.Name and not property from this point on. var val = TryDetectDynamicFieldCreation(property, actualValue.AsObject(), field); if (val != null) { if (val.IsObject() && val.AsObject().TryGetValue("$spatial", out _)) { actualValue = val; //Here we populate the dynamic spatial field that will be handled below. } else { value = TypeConverter.ToBlittableSupportedType(val, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext); newFields += GetRegularFields(instance, field, value, indexContext, out _); continue; } } var objectValue = actualValue.AsObject(); if (objectValue.HasOwnProperty("$spatial") && objectValue.TryGetValue("$spatial", out var inner)) { SpatialField spatialField; IEnumerable <AbstractField> spatial; if (inner.IsString()) { spatialField = StaticIndexBase.GetOrCreateSpatialField(field.Name); spatial = StaticIndexBase.CreateSpatialField(spatialField, inner.AsString()); } else if (inner.IsObject()) { var innerObject = inner.AsObject(); if (innerObject.HasOwnProperty("Lat") && innerObject.HasOwnProperty("Lng") && innerObject.TryGetValue("Lat", out var lat) && lat.IsNumber() && innerObject.TryGetValue("Lng", out var lng) && lng.IsNumber()) { spatialField = StaticIndexBase.GetOrCreateSpatialField(field.Name); spatial = StaticIndexBase.CreateSpatialField(spatialField, lat.AsNumber(), lng.AsNumber()); } else { continue; //Ignoring bad spatial field } } else { continue; //Ignoring bad spatial field } newFields += GetRegularFields(instance, field, spatial, indexContext, out _); continue; } } value = TypeConverter.ToBlittableSupportedType(propertyDescriptor.Value, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext); newFields += GetRegularFields(instance, field, value, indexContext, out _); if (value is IDisposable toDispose) { // the value was converted to a lucene field and isn't needed anymore toDispose.Dispose(); } } return(newFields); }
public MetadataFieldCache(LazyStringValue id, LazyStringValue key, LazyStringValue collection) { Id = id; Key = key; Collection = collection; }
protected override int GetFields <T>(T instance, LazyStringValue key, object document, JsonOperationContext indexContext) { int newFields = 0; if (key != null) { instance.Add(GetOrCreateKeyField(key)); newFields++; } var boostedValue = document as BoostedValue; var documentToProcess = boostedValue == null ? document : boostedValue.Value; IPropertyAccessor accessor; if (_isMultiMap == false) { accessor = _propertyAccessor ?? (_propertyAccessor = PropertyAccessor.Create(documentToProcess.GetType(), documentToProcess)); } else { accessor = TypeConverter.GetPropertyAccessor(documentToProcess); } var reduceResult = _reduceOutput ? new DynamicJsonValue() : null; foreach (var property in accessor.GetPropertiesInOrder(documentToProcess)) { var value = property.Value; IndexField field; try { field = _fields[property.Key]; } catch (KeyNotFoundException e) { throw new InvalidOperationException($"Field '{property.Key}' is not defined. Available fields: {string.Join(", ", _fields.Keys)}.", e); } var numberOfCreatedFields = GetRegularFields(instance, field, value, indexContext); newFields += numberOfCreatedFields; if (boostedValue != null) { var fields = instance.GetFields(); for (int idx = fields.Count - 1; numberOfCreatedFields > 0; numberOfCreatedFields--, idx--) { var luceneField = fields[idx]; luceneField.Boost = boostedValue.Boost; luceneField.OmitNorms = false; } } if (reduceResult != null && numberOfCreatedFields > 0) { reduceResult[property.Key] = TypeConverter.ToBlittableSupportedType(value, flattenArrays: true); } } if (_reduceOutput) { instance.Add(GetReduceResultValueField(Scope.CreateJson(reduceResult, indexContext))); newFields++; } return(newFields); }
public override int HandleMap(LazyStringValue lowerId, LazyStringValue id, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats) { throw new NotImplementedException(); }
public static ByteStringContext.ExternalScope External(ByteStringContext context, LazyStringValue value, out Slice slice) { return(External(context, value.Buffer, value.Size, ByteStringType.Mutable | ByteStringType.External, out slice)); }
private void HandleNestedValuesReduction(TransactionOperationContext indexContext, IndexingStatsScope stats, CancellationToken token, MapReduceResultsStore modifiedStore, IndexWriteOperation writer, LazyStringValue reduceKeyHash) { EnsureValidNestedValuesReductionStats(stats); var numberOfEntriesToReduce = 0; try { var section = modifiedStore.GetNestedResultsSection(); if (section.IsModified == false) { return; } using (_nestedValuesReductionStats.NestedValuesRead.Start()) { numberOfEntriesToReduce += section.GetResults(indexContext, _aggregationBatch); } stats.RecordReduceAttempts(numberOfEntriesToReduce); AggregationResult result; using (_nestedValuesReductionStats.NestedValuesAggregation.Start()) { result = AggregateOn(_aggregationBatch, indexContext, token); } if (section.IsNew == false) { writer.DeleteReduceResult(reduceKeyHash, stats); } foreach (var output in result.GetOutputs()) { writer.IndexDocument(reduceKeyHash, output, stats, indexContext); } _index.ReducesPerSec.Mark(numberOfEntriesToReduce); _metrics.MapReduceReducedPerSecond.Mark(numberOfEntriesToReduce); stats.RecordReduceSuccesses(numberOfEntriesToReduce); } catch (Exception e) { _index.HandleError(e); foreach (var item in _aggregationBatch) { item.Dispose(); } var message = $"Failed to execute reduce function for reduce key '{reduceKeyHash}' on nested values of '{_indexDefinition.Name}' index."; if (_logger.IsInfoEnabled) { _logger.Info(message, e); } stats.RecordReduceErrors(numberOfEntriesToReduce); stats.AddReduceError(message + $" Exception: {e}"); } finally { _aggregationBatch.Clear(); } }
private void HandleReductionError(Exception error, LazyStringValue reduceKeyHash, IndexWriteOperation writer, IndexingStatsScope stats, bool updateStats, TreePage page, int numberOfNestedValues = -1) { var builder = new StringBuilder("Failed to execute reduce function on "); if (page != null) { builder.Append($"page {page} "); } else { builder.Append("nested values "); } builder.Append($"of '{_indexDefinition.Name}' index. The relevant reduce result is going to be removed from the index "); builder.Append($"as it would be incorrect due to encountered errors (reduce key hash: {reduceKeyHash}"); var sampleItem = _aggregationBatch?.Items?.FirstOrDefault(); if (sampleItem != null) { builder.Append($", sample item to reduce: {sampleItem}"); } builder.Append(")"); var message = builder.ToString(); if (_logger.IsInfoEnabled) { _logger.Info(message, error); } try { writer.DeleteReduceResult(reduceKeyHash, stats); } catch (Exception e) { if (_logger.IsInfoEnabled) { _logger.Info($"Failed to delete an index result from '${_indexDefinition.Name}' index on reduce error (reduce key hash: ${reduceKeyHash})", e); } } if (updateStats) { var numberOfEntries = page?.NumberOfEntries ?? numberOfNestedValues; Debug.Assert(numberOfEntries != -1); // we'll only want to record exceptions on some of these, to give the // user information about what is going on, otherwise we'll have to wait a lot if we // are processing a big batch, and this can be a perf killer. See: RavenDB-11038 stats.RecordReduceErrors(numberOfEntries); if (stats.NumberOfKeptReduceErrors < IndexStorage.MaxNumberOfKeptErrors) { stats.AddReduceError(message + $" Exception: {error}"); } var failureInfo = new IndexFailureInformation { Name = _index.Name, MapErrors = stats.MapErrors, MapAttempts = stats.MapAttempts, ReduceErrors = stats.ReduceErrors, ReduceAttempts = stats.ReduceAttempts }; if (failureInfo.IsInvalidIndex(_isStaleBecauseOfRunningReduction)) { throw new ExcessiveNumberOfReduceErrorsException("Excessive number of errors during the reduce phase for the current batch. Failure info: " + failureInfo.GetErrorMessage()); } } }
private void HandleTreeReduction(TransactionOperationContext indexContext, IndexingStatsScope stats, CancellationToken token, MapReduceResultsStore modifiedStore, LowLevelTransaction lowLevelTransaction, IndexWriteOperation writer, LazyStringValue reduceKeyHash, Table table) { EnsureValidTreeReductionStats(stats); var tree = modifiedStore.Tree; var branchesToAggregate = new HashSet <long>(); var parentPagesToAggregate = new HashSet <long>(); var page = new TreePage(null, Constants.Storage.PageSize); foreach (var modifiedPage in modifiedStore.ModifiedPages) { token.ThrowIfCancellationRequested(); page.Base = lowLevelTransaction.GetPage(modifiedPage).Pointer; stats.RecordReduceTreePageModified(page.IsLeaf); if (page.IsLeaf == false) { Debug.Assert(page.IsBranch); branchesToAggregate.Add(modifiedPage); continue; } var leafPage = page; var compressed = leafPage.IsCompressed; if (compressed) { stats.RecordCompressedLeafPage(); } using (compressed ? (DecompressedLeafPage)(leafPage = tree.DecompressPage(leafPage, skipCache: true)) : null) { if (leafPage.NumberOfEntries == 0) { if (leafPage.PageNumber != tree.State.RootPageNumber) { throw new InvalidOperationException( $"Encountered empty page which isn't a root. Page #{leafPage.PageNumber} in '{tree.Name}' tree."); } writer.DeleteReduceResult(reduceKeyHash, stats); var emptyPageNumber = Bits.SwapBytes(leafPage.PageNumber); using (Slice.External(indexContext.Allocator, (byte *)&emptyPageNumber, sizeof(long), out Slice pageNumSlice)) table.DeleteByKey(pageNumSlice); continue; } var parentPage = tree.GetParentPageOf(leafPage); stats.RecordReduceAttempts(leafPage.NumberOfEntries); try { using (var result = AggregateLeafPage(leafPage, lowLevelTransaction, indexContext, token)) { if (parentPage == -1) { writer.DeleteReduceResult(reduceKeyHash, stats); foreach (var output in result.GetOutputs()) { writer.IndexDocument(reduceKeyHash, output, stats, indexContext); } } else { StoreAggregationResult(leafPage.PageNumber, leafPage.NumberOfEntries, table, result); parentPagesToAggregate.Add(parentPage); } _metrics.MapReduceReducedPerSecond.Mark(leafPage.NumberOfEntries); stats.RecordReduceSuccesses(leafPage.NumberOfEntries); } } catch (Exception e) { _index.HandleError(e); var message = $"Failed to execute reduce function for reduce key '{tree.Name}' on a leaf page #{leafPage} of '{_indexDefinition.Name}' index."; if (_logger.IsInfoEnabled) { _logger.Info(message, e); } if (parentPage == -1) { stats.RecordReduceErrors(leafPage.NumberOfEntries); stats.AddReduceError(message + $" Exception: {e}"); } } } } long tmp = 0; using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice)) { foreach (var freedPage in modifiedStore.FreedPages) { tmp = Bits.SwapBytes(freedPage); table.DeleteByKey(pageNumberSlice); } } while (parentPagesToAggregate.Count > 0 || branchesToAggregate.Count > 0) { token.ThrowIfCancellationRequested(); var branchPages = parentPagesToAggregate; parentPagesToAggregate = new HashSet <long>(); foreach (var pageNumber in branchPages) { page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer; try { if (page.IsBranch == false) { throw new InvalidOperationException("Parent page was found that wasn't a branch, error at " + page.PageNumber); } stats.RecordReduceAttempts(page.NumberOfEntries); var parentPage = tree.GetParentPageOf(page); using (var result = AggregateBranchPage(page, table, indexContext, branchesToAggregate, token)) { if (parentPage == -1) { writer.DeleteReduceResult(reduceKeyHash, stats); foreach (var output in result.GetOutputs()) { writer.IndexDocument(reduceKeyHash, output, stats, indexContext); } } else { parentPagesToAggregate.Add(parentPage); StoreAggregationResult(page.PageNumber, page.NumberOfEntries, table, result); } _metrics.MapReduceReducedPerSecond.Mark(page.NumberOfEntries); stats.RecordReduceSuccesses(page.NumberOfEntries); } } catch (Exception e) { _index.HandleError(e); var message = $"Failed to execute reduce function for reduce key '{tree.Name}' on a branch page #{page} of '{_indexDefinition.Name}' index."; if (_logger.IsInfoEnabled) { _logger.Info(message, e); } stats.RecordReduceErrors(page.NumberOfEntries); stats.AddReduceError(message + $" Exception: {e}"); } finally { branchesToAggregate.Remove(pageNumber); } } if (parentPagesToAggregate.Count == 0 && branchesToAggregate.Count > 0) { // we still have unaggregated branches which were modified but their children were not modified (branch page splitting) so we missed them parentPagesToAggregate.Add(branchesToAggregate.First()); } } }