public bool Update(UpdateStep step) { var ids = new HashSet <long>(); var minimal = long.MaxValue; const string dbKey = "db/"; var continueAfterCommit = true; var skip = 0; while (continueAfterCommit) { continueAfterCommit = false; var fixedItems = 0; var items = step.WriteTx.OpenTable(ClusterStateMachine.ItemsSchema, ClusterStateMachine.Items); using (Slice.From(step.WriteTx.Allocator, dbKey, out Slice loweredPrefix)) { foreach (var result in items.SeekByPrimaryKeyPrefix(loweredPrefix, Slices.Empty, 0)) { var databaseName = ClusterStateMachine.GetCurrentItemKey(result.Value).Substring(3); using (Slice.From(step.WriteTx.Allocator, dbKey + databaseName.ToLowerInvariant(), out var key)) { if (items.VerifyKeyExists(key) == false) { continue; } } using (Slice.From(step.WriteTx.Allocator, SubscriptionState.SubscriptionPrefix(databaseName), out var startWith)) using (var ctx = JsonOperationContext.ShortTermSingleUse()) { foreach (var holder in items.SeekByPrimaryKeyPrefix(startWith, Slices.Empty, skip)) { skip++; var reader = holder.Value.Reader; var ptr = reader.Read(2, out int size); using (var doc = new BlittableJsonReaderObject(ptr, size, ctx)) { if (doc.TryGet(nameof(SubscriptionState.SubscriptionId), out long id) == false) { continue; } if (minimal > id) { minimal = id; } if (ids.Add(id)) { continue; } minimal--; ids.Add(minimal); var subscriptionState = JsonDeserializationClient.SubscriptionState(doc); subscriptionState.SubscriptionId = minimal; var subscriptionItemName = SubscriptionState.GenerateSubscriptionItemKeyName(databaseName, subscriptionState.SubscriptionName); using (Slice.From(step.WriteTx.Allocator, subscriptionItemName, out Slice valueName)) using (Slice.From(step.WriteTx.Allocator, subscriptionItemName.ToLowerInvariant(), out Slice valueNameLowered)) using (var receivedSubscriptionState = ctx.ReadObject(subscriptionState.ToJson(), subscriptionState.SubscriptionName)) { ClusterStateMachine.UpdateValue(0, items, valueNameLowered, valueName, receivedSubscriptionState); } } fixedItems++; if (fixedItems < 1024) { continue; } continueAfterCommit = true; break; } } if (continueAfterCommit) { break; } } } if (continueAfterCommit) { step.Commit(null); step.RenewTransactions(); } } return(true); }
public bool Update(UpdateStep step) { UpdateSchemaForDocumentsAndRevisions(step); var readTable = new Table(CountersSchema, step.WriteTx); var countersTree = readTable.GetTree(CountersSchema.Key); if (countersTree == null) { return(true); } step.DocumentsStorage.CountersStorage = new CountersStorage(step.DocumentsStorage.DocumentDatabase, step.WriteTx); // this schema update uses DocumentsStorage.GenerateNextEtag() so we need to read and set LastEtag in storage _dbId = From41016.ReadDbId(step); step.DocumentsStorage.InitializeLastEtag(step.ReadTx); var entriesToDelete = new List <long>(); var batch = new CounterBatchUpdate(); string currentDocId = null; string currentPrefix = null; CollectionName currentCollection = null; var done = false; // 4.2 counters processing while (done == false) { readTable = new Table(CountersSchema, step.ReadTx); var processedInCurrentTx = 0; using (step.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { context.TransactionMarkerOffset = (short)step.WriteTx.LowLevelTransaction.Id; var commit = false; foreach (var counterGroup in GetCounters(readTable, context, currentPrefix)) { if (currentDocId == counterGroup.Id) { batch.Counters.Add(counterGroup); } else { if (currentDocId != null) { // finished processing all counter groups for current document DeleteProcessedEntries(step, entriesToDelete, currentCollection); PutCounterGroups(step, batch, context, currentCollection); UpdateDocumentCounters(step, context, currentDocId, currentCollection); if (processedInCurrentTx >= NumberOfCounterGroupsToMigrateInSingleTransaction || context.AllocatedMemory >= MaxSizeToMigrateInSingleTransaction) { commit = true; currentPrefix = counterGroup.Id; break; } } // start new batch currentDocId = counterGroup.Id; currentCollection = new CollectionName(counterGroup.Collection); batch.Counters.Add(counterGroup); } // add table etag to delete-list entriesToDelete.Add(counterGroup.Etag); processedInCurrentTx++; } if (commit) { step.Commit(context); step.RenewTransactions(); currentDocId = null; continue; } if (batch.Counters.Count > 0) { DeleteProcessedEntries(step, entriesToDelete, currentCollection); PutCounterGroups(step, batch, context, currentCollection); UpdateDocumentCounters(step, context, currentDocId, currentCollection); } done = true; } } return(true); }
public bool Update(UpdateStep step) { var legacyCounterCollectionTables = new List <string>(); using (var it = step.ReadTx.LowLevelTransaction.RootObjects.Iterate(prefetch: false)) { if (it.Seek(Slices.BeforeAllKeys) == false) { return(true); } var legacyCounterCollectionTablePrefix = CollectionName.GetTablePrefix(CollectionTableType.Counters); do { var current = it.CurrentKey; var currentAsString = current.ToString(); if (currentAsString.StartsWith(legacyCounterCollectionTablePrefix, StringComparison.OrdinalIgnoreCase)) { var type = step.ReadTx.GetRootObjectType(current); if (type != RootObjectType.Table) { continue; // precaution } legacyCounterCollectionTables.Add(currentAsString); } } while (it.MoveNext()); } // this schema update uses DocumentsStorage.GenerateNextEtag() so we need to read and set LastEtag in storage step.DocumentsStorage.InitializeLastEtag(step.ReadTx); step.DocumentsStorage.CountersStorage = new CountersStorage(step.DocumentsStorage.DocumentDatabase, step.WriteTx); _dbId = ReadDbId(step); // legacy counters processing string currentDocId = null; var batch = new CounterBatchUpdate(); var dbIds = new HashSet <string>(); var done = false; while (done == false) { var readTable = new Table(LegacyCountersSchema, step.WriteTx); var countersTree = readTable.GetTree(LegacyCountersSchema.Key); if (countersTree == null) { return(true); } var processedInCurrentTx = 0; var toDeleteCounterEtagsByCollection = new Dictionary <CollectionName, List <long> >(); var toDeleteEtagsForCurrentDocumentId = new List <long>(); using (step.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { context.TransactionMarkerOffset = (short)step.WriteTx.LowLevelTransaction.Id; var commit = false; foreach (var item in GetCounters(readTable, context)) { var counterDetail = item.Counter; if (currentDocId == counterDetail.DocumentId) { if (batch.Counters.TryGetValue(counterDetail.CounterName, out var list) == false) { list = new List <CounterDetail>(); batch.Counters.Add(counterDetail.CounterName, list); } list.Add(counterDetail); } else { if (currentDocId != null) { var docCollection = PutCounters(step, context, dbIds, batch.Counters, currentDocId); if (docCollection != null) { if (toDeleteCounterEtagsByCollection.TryGetValue(docCollection, out var toDeleteList) == false) { toDeleteList = new List <long>(); toDeleteCounterEtagsByCollection.Add(docCollection, toDeleteList); } toDeleteList.AddRange(toDeleteEtagsForCurrentDocumentId); toDeleteEtagsForCurrentDocumentId.Clear(); } batch.Clear(); if (processedInCurrentTx >= NumberOfCountersToMigrateInSingleTransaction) { foreach (var toDeleteForCollection in toDeleteCounterEtagsByCollection) { var table = step.WriteTx.OpenTable(LegacyCountersSchema, toDeleteForCollection.Key.GetTableName(CollectionTableType.Counters)); DeleteMigratedLegacyCounters(table, toDeleteForCollection.Value); } toDeleteCounterEtagsByCollection.Clear(); commit = true; break; } } batch.Clear(); currentDocId = counterDetail.DocumentId; batch.Counters.Add(counterDetail.CounterName, new List <CounterDetail> { counterDetail }); } toDeleteEtagsForCurrentDocumentId.Add(item.Counter.Etag); using (var dbId = ExtractDbId(context, counterDetail.CounterKey)) { dbIds.Add(dbId.ToString()); } processedInCurrentTx++; } if (commit) { step.Commit(context); step.RenewTransactions(); step.DocumentsStorage.CountersStorage = new CountersStorage(step.DocumentsStorage.DocumentDatabase, step.WriteTx); currentDocId = null; continue; } if (batch.Counters.Count > 0) { PutCounters(step, context, dbIds, batch.Counters, currentDocId); batch.Clear(); } if (toDeleteCounterEtagsByCollection.Count > 0) { foreach (var toDeleteForCollection in toDeleteCounterEtagsByCollection) { var table = step.WriteTx.OpenTable(LegacyCountersSchema, toDeleteForCollection.Key.GetTableName(CollectionTableType.Counters)); DeleteMigratedLegacyCounters(table, toDeleteForCollection.Value); } } done = true; } // we must delete tables first before deleting any global index trees from the root that can be in use by tables foreach (var tableName in legacyCounterCollectionTables) { step.WriteTx.DeleteTable(tableName); } // let's remove remaining counter trees from the root if (step.WriteTx.LowLevelTransaction.RootObjects.Read(CounterKeysSlice) != null) { step.WriteTx.DeleteTree(CounterKeysSlice); } if (step.WriteTx.LowLevelTransaction.RootObjects.Read(AllCountersEtagSlice) != null) { step.WriteTx.DeleteFixedTree(AllCountersEtagSlice.ToString()); } } // legacy counter tombstones processing var counterTombstones = step.ReadTx.OpenTable(TombstonesSchema, CountersTombstonesSlice); if (counterTombstones != null) { // for each counter tombstone, delete the matching // counter from the new table (if exists) using (step.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { foreach (var result in counterTombstones.SeekByPrimaryKeyPrefix(Slices.BeforeAllKeys, Slices.Empty, 0)) { var t = new Tombstone { LowerId = TableValueToString(context, (int)TombstoneTable.LowerId, ref result.Value.Reader), Type = *(Tombstone.TombstoneType *)result.Value.Reader.Read((int)TombstoneTable.Type, out _), }; if (t.Type != Tombstone.TombstoneType.Counter) { continue; } DeleteCounter(step, t.LowerId, context); t.LowerId.Dispose(); } // delete counter-tombstones from Tombstones table var countersTombstoneTable = step.WriteTx.OpenTable(TombstonesSchema, CountersTombstonesSlice); DeleteFromTable(context, countersTombstoneTable, TombstonesSchema.Key, tvh => { var type = *(Tombstone.TombstoneType *)tvh.Reader.Read((int)TombstoneTable.Type, out _); return(type != Tombstone.TombstoneType.Counter); }); } step.WriteTx.DeleteTable(CountersTombstonesSlice.ToString()); } return(true); }