public IDisposable OpenReadTransaction() { var documentsTx = Documents.OpenReadTransaction(); RavenTransaction serverTx = null; if (Server != null) { serverTx = Server.OpenReadTransaction(); } return(new DisposeTransactions(documentsTx, serverTx)); }
public IndexState ReadState(RavenTransaction tx) { var statsTree = tx.InnerTransaction.ReadTree(IndexSchema.StatsTree); var state = statsTree.Read(IndexSchema.StateSlice); if (state == null) { return(IndexState.Normal); } return((IndexState)state.Reader.ReadLittleEndianInt32()); }
public DateTime?ReadLastIndexingTime(RavenTransaction tx) { var statsTree = tx.InnerTransaction.ReadTree(IndexSchema.StatsTree); var lastIndexingTime = statsTree.Read(IndexSchema.LastIndexingTimeSlice); if (lastIndexingTime == null) { return(null); } return(DateTime.FromBinary(lastIndexingTime.Reader.ReadLittleEndianInt64())); }
public long ReadLastProcessedReferenceTombstoneEtag(RavenTransaction tx, string collection, CollectionName referencedCollection) { var tree = tx.InnerTransaction.ReadTree("%" + collection); var result = tree?.Read(referencedCollection.Name); if (result == null) { return(0); } return(result.Reader.ReadLittleEndianInt64()); }
public int ReadMaxNumberOfOutputsPerDocument(RavenTransaction tx) { var statsTree = tx.InnerTransaction.ReadTree(IndexSchema.StatsTree); var lastIndexingTime = statsTree.Read(IndexSchema.LastIndexingTimeSlice); if (lastIndexingTime != null) { return(statsTree.Read(IndexSchema.MaxNumberOfOutputsPerDocument).Reader.ReadLittleEndianInt32()); } return(0); }
private static long ReadLastEtag(RavenTransaction tx, string tree, Slice collection) { var statsTree = tx.InnerTransaction.CreateTree(tree); var readResult = statsTree.Read(collection); long lastEtag = 0; if (readResult != null) { lastEtag = readResult.Reader.ReadLittleEndianInt64(); } return(lastEtag); }
public void Dismiss(string id, RavenTransaction existingTransaction = null, bool sendNotificationEvenIfDoesntExist = true) { var deleted = _notificationsStorage.Delete(id, existingTransaction); if (deleted == false && sendNotificationEvenIfDoesntExist == false) { return; } // send this notification even when notification doesn't exist // we don't persist all notifications Add(NotificationUpdated.Create(id, NotificationUpdateType.Dismissed)); }
private NotificationTableValue Get(string id, JsonOperationContext context, RavenTransaction tx) { var table = tx.InnerTransaction.OpenTable(_actionsSchema, NotificationsSchema.NotificationsTree); using (Slice.From(tx.InnerTransaction.Allocator, id, out Slice slice)) { if (table.ReadByKey(slice, out TableValueReader tvr) == false) { return(null); } return(Read(context, ref tvr)); } }
public override void Execute(DocumentsOperationContext context, RavenTransaction tx) { foreach (var document in Documents) { BlittableJsonReaderObject metadata; if (document.TryGet(Constants.Metadata.Key, out metadata) == false) { throw new InvalidOperationException("A document must have a metadata"); } // We are using the id term here and not key in order to be backward compatiable with old export files. string key; if (metadata.TryGet(Constants.Metadata.Id, out key) == false) { throw new InvalidOperationException("Document's metadata must include the document's key."); } DynamicJsonValue mutatedMetadata; metadata.Modifications = mutatedMetadata = new DynamicJsonValue(metadata); mutatedMetadata.Remove(Constants.Metadata.Id); mutatedMetadata.Remove(Constants.Metadata.Etag); if (IsRevision) { long etag; if (metadata.TryGet(Constants.Metadata.Etag, out etag) == false) { throw new InvalidOperationException("Document's metadata must include the document's key."); } _database.BundleLoader.VersioningStorage.PutDirect(context, key, etag, document); } else if (_buildVersion < 4000 && key.Contains("/revisions/")) { long etag; if (metadata.TryGet(Constants.Metadata.Etag, out etag) == false) { throw new InvalidOperationException("Document's metadata must include the document's key."); } var endIndex = key.IndexOf("/revisions/", StringComparison.OrdinalIgnoreCase); key = key.Substring(0, endIndex); _database.BundleLoader.VersioningStorage.PutDirect(context, key, etag, document); } else { _database.DocumentsStorage.Put(context, key, null, document); } } }
public void AddAfterTransactionCommit(Notification notification, RavenTransaction tx) { var llt = tx.InnerTransaction.LowLevelTransaction; llt.OnDispose += _ => { if (llt.Committed == false) { return; } Add(notification); }; }
public IndexStats ReadStats(RavenTransaction tx) { var statsTree = tx.InnerTransaction.ReadTree(IndexSchema.StatsTree); var table = tx.InnerTransaction.OpenTable(_errorsSchema, "Errors"); var stats = new IndexStats { CreatedTimestamp = DateTime.FromBinary(statsTree.Read(IndexSchema.CreatedTimestampSlice).Reader.ReadLittleEndianInt64()), ErrorsCount = (int)(table?.NumberOfEntries ?? 0) }; var lastIndexingTime = statsTree.Read(IndexSchema.LastIndexingTimeSlice); stats.Collections = new Dictionary <string, IndexStats.CollectionStats>(); foreach (var collection in _index.Definition.Collections) { stats.Collections[collection] = new IndexStats.CollectionStats { LastProcessedDocumentEtag = ReadLastIndexedEtag(tx, collection), LastProcessedTombstoneEtag = ReadLastProcessedTombstoneEtag(tx, collection) }; } if (lastIndexingTime != null) { stats.LastIndexingTime = DateTime.FromBinary(lastIndexingTime.Reader.ReadLittleEndianInt64()); stats.MapAttempts = statsTree.Read(IndexSchema.MapAttemptsSlice).Reader.ReadLittleEndianInt32(); stats.MapErrors = statsTree.Read(IndexSchema.MapErrorsSlice).Reader.ReadLittleEndianInt32(); stats.MapSuccesses = statsTree.Read(IndexSchema.MapSuccessesSlice).Reader.ReadLittleEndianInt32(); stats.MaxNumberOfOutputsPerDocument = statsTree.Read(IndexSchema.MaxNumberOfOutputsPerDocument).Reader.ReadLittleEndianInt32(); if (_index.Type.IsMapReduce()) { stats.ReduceAttempts = statsTree.Read(IndexSchema.ReduceAttemptsSlice)?.Reader.ReadLittleEndianInt32() ?? 0; stats.ReduceSuccesses = statsTree.Read(IndexSchema.ReduceSuccessesSlice)?.Reader.ReadLittleEndianInt32() ?? 0; stats.ReduceErrors = statsTree.Read(IndexSchema.ReduceErrorsSlice)?.Reader.ReadLittleEndianInt32() ?? 0; } if (_index.GetReferencedCollections()?.Count > 0) { stats.MapReferenceAttempts = statsTree.Read(IndexSchema.MapReferencedAttemptsSlice)?.Reader.ReadLittleEndianInt32() ?? 0; stats.MapReferenceSuccesses = statsTree.Read(IndexSchema.MapReferenceSuccessesSlice)?.Reader.ReadLittleEndianInt32() ?? 0; stats.MapReferenceErrors = statsTree.Read(IndexSchema.MapReferenceErrorsSlice)?.Reader.ReadLittleEndianInt32() ?? 0; } } return(stats); }
private unsafe void WriteLastEtag(RavenTransaction tx, string tree, Slice collection, long etag) { if (SimulateCorruption) { SimulateCorruptionError(); } if (_logger.IsInfoEnabled) { _logger.Info($"Writing last etag for '{_index.Name}'. Tree: {tree}. Collection: {collection}. Etag: {etag}."); } var statsTree = tx.InnerTransaction.CreateTree(tree); using (Slice.External(tx.InnerTransaction.Allocator, (byte *)&etag, sizeof(long), out Slice etagSlice)) statsTree.Add(collection, etagSlice); }
private unsafe void WriteLastEtag(RavenTransaction tx, string tree, Slice collection, long etag) { if (_simulateCorruption) { throw new SimulatedVoronUnrecoverableErrorException("Simulated corruption."); } if (_logger.IsInfoEnabled) { _logger.Info($"Writing last etag for '{_index.Name} ({_index.IndexId})'. Tree: {tree}. Collection: {collection}. Etag: {etag}."); } var statsTree = tx.InnerTransaction.CreateTree(tree); Slice etagSlice; using (Slice.External(tx.InnerTransaction.Allocator, (byte *)&etag, sizeof(long), out etagSlice)) statsTree.Add(collection, etagSlice); }
public long ReadLastIndexedEtag(RavenTransaction tx, string collection) { var txi = tx.InnerTransaction; if (txi.IsWriteTransaction == false) { if (txi.LowLevelTransaction.ImmutableExternalState is IndexTransactionCache cache) { if (cache.Collections.TryGetValue(collection, out var val)) { return(val.LastIndexedEtag); } } } using (Slice.From(txi.Allocator, collection, out Slice collectionSlice)) { return(ReadLastEtag(txi, IndexSchema.EtagsTree, collectionSlice)); } }
public unsafe void WriteReferences(CurrentIndexingScope indexingScope, RavenTransaction tx) { // IndexSchema: // having 'Users' and 'Addresses' we will end up with // // #Users (tree) - splitted by collection so we can easily return all items of same collection to the indexing function // |- addresses/1 (key) -> [ users/1, users/2 ] // |- addresses/2 (key) -> [ users/3 ] // // References (tree) - used in delete operations // |- users/1 -> [ addresses/1 ] // |- users/2 -> [ addresses/1 ] // |- users/3 -> [ addresses/2 ] // // $Users (tree) - holding highest visible etag of 'referenced collection' per collection, so we will have a starting point for references processing // |- Addresses (key) -> 5 if (indexingScope.ReferencesByCollection != null) { var referencesTree = tx.InnerTransaction.ReadTree(IndexSchema.References); foreach (var collections in indexingScope.ReferencesByCollection) { var collectionTree = tx.InnerTransaction.CreateTree("#" + collections.Key); // #collection foreach (var keys in collections.Value) { using (Slice.From(tx.InnerTransaction.Allocator, keys.Key, ByteStringType.Immutable, out Slice key)) { foreach (var referenceKey in keys.Value) { collectionTree.MultiAdd(referenceKey, key); referencesTree.MultiAdd(key, referenceKey); } RemoveReferences(key, collections.Key, keys.Value, tx); } } } } }
private void CompletePreviousTransaction( DocumentsOperationContext context, RavenTransaction previous, CommitStats commitStats, ref List <MergedTransactionCommand> previousPendingOps, bool throwOnError) { try { _recording.State?.Record(context, TxInstruction.EndAsyncCommit); previous.EndAsyncCommit(); //not sure about this 'if' if (commitStats != null) { SlowWriteNotification.Notify(commitStats, _parent); } if (_log.IsInfoEnabled) { _log.Info($"EndAsyncCommit on {previous.InnerTransaction.LowLevelTransaction.Id}"); } NotifyOnThreadPool(previousPendingOps); } catch (Exception e) { foreach (var op in previousPendingOps) { op.Exception = e; } NotifyOnThreadPool(previousPendingOps); previousPendingOps = null; // RavenDB-7417 if (throwOnError) { throw; } } }
public void RemoveReferences(Slice key, string collection, HashSet <Slice> referenceKeysToSkip, RavenTransaction tx) { var referencesTree = tx.InnerTransaction.ReadTree(IndexSchema.References); List <Slice> referenceKeys; using (var it = referencesTree.MultiRead(key)) { if (it.Seek(Slices.BeforeAllKeys) == false) { return; } referenceKeys = new List <Slice>(); do { if (referenceKeysToSkip == null || referenceKeysToSkip.Contains(it.CurrentKey) == false) { referenceKeys.Add(it.CurrentKey.Clone(tx.InnerTransaction.Allocator, ByteStringType.Immutable)); } } while (it.MoveNext()); } if (referenceKeys.Count == 0) { return; } var collectionTree = tx.InnerTransaction.ReadTree("#" + collection); foreach (var referenceKey in referenceKeys) { referencesTree.MultiDelete(key, referenceKey); collectionTree?.MultiDelete(referenceKey, key); referenceKey.Release(tx.InnerTransaction.Allocator); } }
public unsafe void WriteReferences(CurrentIndexingScope indexingScope, RavenTransaction tx) { // IndexSchema: // having 'Users' and 'Addresses' we will end up with // // #Users (tree) - splitted by collection so we can easily return all items of same collection to the indexing function // |- addresses/1 (key) -> [ users/1, users/2 ] // |- addresses/2 (key) -> [ users/3 ] // // References (tree) - used in delete operations // |- users/1 -> [ addresses/1 ] // |- users/2 -> [ addresses/1 ] // |- users/3 -> [ addresses/2 ] // // $Users (tree) - holding highest visible etag of 'referenced collection' per collection, so we will have a starting point for references processing // |- Addresses (key) -> 5 if (indexingScope.ReferencesByCollection != null) { var referencesTree = tx.InnerTransaction.ReadTree(IndexSchema.References); foreach (var collections in indexingScope.ReferencesByCollection) { var collectionTree = tx.InnerTransaction.CreateTree("#" + collections.Key); // #collection foreach (var keys in collections.Value) { using (Slice.From(tx.InnerTransaction.Allocator, keys.Key, ByteStringType.Immutable, out Slice key)) { foreach (var referenceKey in keys.Value) { collectionTree.MultiAdd(referenceKey, key); referencesTree.MultiAdd(key, referenceKey); } RemoveReferences(key, collections.Key, keys.Value, tx); } } } } if (indexingScope.ReferenceEtagsByCollection != null) { foreach (var kvp in indexingScope.ReferenceEtagsByCollection) { var collectionEtagTree = tx.InnerTransaction.CreateTree("$" + kvp.Key); // $collection foreach (var collections in kvp.Value) { if (_referencedCollections.TryGetValue(collections.Key, out CollectionName collectionName) == false) { throw new InvalidOperationException( $"Could not find collection {collections.Key} in the index storage collections. Should not happen ever!"); } using (Slice.From(tx.InnerTransaction.Allocator, collectionName.Name, ByteStringType.Immutable, out Slice collectionKey)) { var etag = collections.Value; var result = collectionEtagTree.Read(collectionKey); var oldEtag = result?.Reader.ReadLittleEndianInt64(); if (oldEtag >= etag) { continue; } using (Slice.External(tx.InnerTransaction.Allocator, (byte *)&etag, sizeof(long), out Slice etagSlice)) collectionEtagTree.Add(collectionKey, etagSlice); } } } } }
public IEnumerable <Slice> GetDocumentKeysFromCollectionThatReference(string collection, LazyStringValue referenceKey, RavenTransaction tx) { var collectionTree = tx.InnerTransaction.ReadTree("#" + collection); if (collectionTree == null) { yield break; } using (DocumentIdWorker.GetLower(tx.InnerTransaction.Allocator, referenceKey, out var k)) using (var it = collectionTree.MultiRead(k)) { if (it.Seek(Slices.BeforeAllKeys) == false) { yield break; } do { yield return(it.CurrentKey); } while (it.MoveNext()); } }
private static unsafe ByteStringContext.ExternalScope CreateKey(RavenTransaction tx, LazyStringValue key, out Slice keySlice) { return(Slice.External(tx.InnerTransaction.Allocator, key.Buffer, key.Size, out keySlice)); }
private async Task <IOperationResult> ExecuteOperation(string indexName, IndexQueryServerSide query, QueryOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Action <string> action, OperationCancelToken token) { var index = GetIndex(indexName); if (index.Type.IsMapReduce()) { throw new InvalidOperationException("Cannot execute bulk operation on Map-Reduce indexes."); } query = ConvertToOperationQuery(query, options); const int BatchSize = 1024; RavenTransaction tx = null; var operationsInCurrentBatch = 0; List <string> resultKeys; try { var results = await index.Query(query, context, token).ConfigureAwait(false); if (options.AllowStale == false && results.IsStale) { throw new InvalidOperationException("Cannot perform bulk operation. Query is stale."); } resultKeys = new List <string>(results.Results.Count); foreach (var document in results.Results) { resultKeys.Add(document.Key.ToString()); } } finally //make sure to close tx if DocumentConflictException is thrown { context.CloseTransaction(); } var progress = new DeterminateProgress { Total = resultKeys.Count, Processed = 0 }; onProgress(progress); using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { foreach (var document in resultKeys) { if (rateGate != null && rateGate.WaitToProceed(0) == false) { using (tx) { tx?.Commit(); } tx = null; rateGate.WaitToProceed(); } if (tx == null) { operationsInCurrentBatch = 0; tx = context.OpenWriteTransaction(); } action(document); operationsInCurrentBatch++; progress.Processed++; if (progress.Processed % 128 == 0) { onProgress(progress); } if (operationsInCurrentBatch < BatchSize) { continue; } using (tx) { tx.Commit(); } tx = null; } } using (tx) { tx?.Commit(); } return(new BulkOperationResult { Total = progress.Total }); }
public override void Execute(DocumentsOperationContext context, RavenTransaction tx) { for (int i = 0; i < ParsedCommands.Length; i++) { var cmd = ParsedCommands[i]; switch (cmd.Method) { case "PUT": var putResult = Database.DocumentsStorage.Put(context, cmd.Key, cmd.Etag, cmd.Document); context.DocumentDatabase.HugeDocuments.AddIfDocIsHuge(cmd.Key, cmd.Document.Size); BlittableJsonReaderObject metadata; cmd.Document.TryGet(Constants.Metadata.Key, out metadata); LastEtag = putResult.Etag; ModifiedCollections?.Add(putResult.Collection.Name); Reply.Add(new DynamicJsonValue { ["Key"] = putResult.Key, ["Etag"] = putResult.Etag, ["Method"] = "PUT", ["AdditionalData"] = cmd.AdditionalData, ["Metadata"] = metadata }); break; case "PATCH": // TODO: Move this code out of the merged transaction // TODO: We should have an object that handles this externally, // TODO: and apply it there var patchResult = Database.Patch.Apply(context, cmd.Key, cmd.Etag, cmd.Patch, null, cmd.IsDebugMode); var additionalData = new DynamicJsonValue { ["Debug"] = patchResult.DebugInfo, }; if (cmd.Document != null) { context.DocumentDatabase.HugeDocuments.AddIfDocIsHuge(cmd.Key, cmd.Document.Size); } if (cmd.IsDebugMode) { additionalData["Document"] = patchResult.ModifiedDocument; additionalData["Actions"] = patchResult.DebugActions; } if (patchResult.Etag != null) { LastEtag = patchResult.Etag.Value; } if (patchResult.Collection != null) { ModifiedCollections?.Add(patchResult.Collection.Name); } Reply.Add(new DynamicJsonValue { ["Key"] = cmd.Key, ["Etag"] = patchResult.Etag, ["Method"] = "PATCH", ["AdditionalData"] = additionalData, ["PatchResult"] = patchResult.PatchResult.ToString(), }); break; case "DELETE": var deleted = Database.DocumentsStorage.Delete(context, cmd.Key, cmd.Etag); if (deleted != null) { LastEtag = deleted.Value.Etag; ModifiedCollections?.Add(deleted.Value.Collection.Name); } Reply.Add(new DynamicJsonValue { ["Key"] = cmd.Key, ["Method"] = "DELETE", ["AdditionalData"] = cmd.AdditionalData, ["Deleted"] = deleted != null }); break; } } }
public IEnumerable <Slice> GetDocumentKeysFromCollectionThatReference(string collection, LazyStringValue referenceKey, RavenTransaction tx) { var collectionTree = tx.InnerTransaction.ReadTree("#" + collection); if (collectionTree == null) { yield break; } Slice referenceKeyAsSlice; using (CreateKey(tx, referenceKey, out referenceKeyAsSlice)) { using (var it = collectionTree.MultiRead(referenceKeyAsSlice)) { if (it.Seek(Slices.BeforeAllKeys) == false) { yield break; } do { yield return(it.CurrentKey); } while (it.MoveNext()); } } }
private void Store(LazyStringValue id, DateTime createdAt, DateTime?postponedUntil, BlittableJsonReaderObject action, RavenTransaction tx) { var table = tx.InnerTransaction.OpenTable(_actionsSchema, NotificationsSchema.NotificationsTree); var createdAtTicks = Bits.SwapBytes(createdAt.Ticks); var postponedUntilTicks = postponedUntil != null ? Bits.SwapBytes(postponedUntil.Value.Ticks) : _postponeDateNotSpecified; using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(id.Buffer, id.Size); tvb.Add((byte *)&createdAtTicks, sizeof(long)); tvb.Add((byte *)&postponedUntilTicks, sizeof(long)); tvb.Add(action.BasePointer, action.Size); table.Set(tvb); } }
internal (long ReferenceTableCount, long CollectionTableCount) GetReferenceTablesCount(string collection, RavenTransaction tx) { var referencesTree = tx.InnerTransaction.ReadTree(_referenceTreeName); var referencesCount = referencesTree.State.NumberOfEntries; var collectionTree = tx.InnerTransaction.ReadTree(_referenceCollectionPrefix + collection); if (collectionTree != null) { return(referencesCount, collectionTree.State.NumberOfEntries); } return(referencesCount, 0); }
public abstract void Execute(DocumentsOperationContext context, RavenTransaction tx);
public void RemoveReferencesByPrefix(Slice prefixKey, string collection, HashSet <Slice> referenceKeysToSkip, RavenTransaction tx) { var referencesTree = tx.InnerTransaction.ReadTree(_referenceTreeName); while (true) { using (var it = referencesTree.Iterate(false)) { it.SetRequiredPrefix(prefixKey); if (it.Seek(prefixKey) == false) { return; } var key = it.CurrentKey.Clone(tx.InnerTransaction.Allocator); try { RemoveReferences(key, collection, referenceKeysToSkip, tx); } finally { key.Release(tx.InnerTransaction.Allocator); } } } }
public unsafe void AddAlert(Alert alert, TransactionOperationContext context, RavenTransaction tx) { _store?.TrackChangeAfterTransactionCommit(context, "AlertRaised", alert.Key); var table = tx.InnerTransaction.OpenTable(_alertsSchema, AlertsSchema.AlertsTree); var alertId = alert.Id; var alertAsJson = alert.ToJson(); // if previous alert has dismissed until value pass this value to newly saved alert Slice slice; using (Slice.From(tx.InnerTransaction.Allocator, alertId, out slice)) { var existingTvr = table.ReadByKey(slice); if (existingTvr != null) { var existingAlert = Read(context, existingTvr); object dismissedUntilValue; existingAlert.TryGetMember(nameof(alert.DismissedUntil), out dismissedUntilValue); if (dismissedUntilValue != null) { var dismissedUntil = (LazyStringValue)dismissedUntilValue; alertAsJson[nameof(alert.DismissedUntil)] = dismissedUntil; } } } using (var id = context.GetLazyString(alertId)) using (var json = context.ReadObject(alertAsJson, "Alert", BlittableJsonDocumentBuilder.UsageMode.ToDisk)) { var tvb = new TableValueBuilder { { id.Buffer, id.Size }, { json.BasePointer, json.Size } }; table.Set(tvb); } }
public void WriteReferences(Dictionary <string, Dictionary <Slice, HashSet <Slice> > > referencesByCollection, RavenTransaction tx) { var referencesTree = tx.InnerTransaction.ReadTree(_referenceTreeName); foreach (var collections in referencesByCollection) { var collectionTree = tx.InnerTransaction.CreateTree(_referenceCollectionPrefix + collections.Key); // #collection foreach (var keys in collections.Value) { var key = keys.Key; foreach (var referenceKey in keys.Value) { collectionTree.MultiAdd(referenceKey, key); referencesTree.MultiAdd(key, referenceKey); } RemoveReferences(key, collections.Key, keys.Value, tx); } } }
public void Initialize(RavenTransaction tx, RavenConfiguration configuration, Logger log) { _logHistoryMaxEntries = configuration.Cluster.LogHistoryMaxEntries; LogHistoryTable.Create(tx.InnerTransaction, RachisLogHistory.LogHistorySlice, 16); _log = log; }