private static DocumentConflict TableValueToConflictDocument(DocumentsOperationContext context, ref TableValueReader tvr) { var result = new DocumentConflict { StorageId = tvr.Id, LowerId = TableValueToString(context, (int)ConflictsTable.LowerId, ref tvr), Id = TableValueToId(context, (int)ConflictsTable.Id, ref tvr), ChangeVector = TableValueToChangeVector(context, (int)ConflictsTable.ChangeVector, ref tvr), Etag = TableValueToEtag((int)ConflictsTable.Etag, ref tvr), Collection = TableValueToId(context, (int)ConflictsTable.Collection, ref tvr), LastModified = TableValueToDateTime((int)ConflictsTable.LastModified, ref tvr), Flags = TableValueToFlags((int)ConflictsTable.Flags, ref tvr) }; var read = tvr.Read((int)ConflictsTable.Data, out int size); if (size > 0) { //otherwise this is a tombstone conflict and should be treated as such result.Doc = new BlittableJsonReaderObject(read, size, context); Transaction.DebugDisposeReaderAfterTransaction(context.Transaction.InnerTransaction, result.Doc); } return(result); }
public static DocumentConflict ParseRawDataSectionConflictWithValidation(JsonOperationContext context, ref TableValueReader tvr, int expectedSize, out long etag) { var read = tvr.Read((int)ConflictsTable.Data, out var size); if (size > expectedSize || size <= 0) { throw new ArgumentException("Document size is invalid, possible corruption when parsing BlittableJsonReaderObject", nameof(size)); } var result = new DocumentConflict { StorageId = tvr.Id, LowerId = TableValueToString(context, (int)ConflictsTable.LowerId, ref tvr), Id = TableValueToId(context, (int)ConflictsTable.Id, ref tvr), ChangeVector = TableValueToChangeVector(context, (int)ConflictsTable.ChangeVector, ref tvr), Etag = etag = TableValueToEtag((int)ConflictsTable.Etag, ref tvr), Doc = new BlittableJsonReaderObject(read, size, context), Collection = TableValueToId(context, (int)ConflictsTable.Collection, ref tvr), LastModified = TableValueToDateTime((int)ConflictsTable.LastModified, ref tvr), Flags = TableValueToFlags((int)ConflictsTable.Flags, ref tvr) }; return(result); }
public static CounterDetail TableValueToCounterDetail(JsonOperationContext context, TableValueReader tvr) { var(doc, name) = ExtractDocIdAndName(context, tvr); return(new CounterDetail { DocumentId = doc, LazyDocumentId = doc, CounterName = name, ChangeVector = TableValueToString(context, (int)CountersTable.ChangeVector, ref tvr), TotalValue = TableValueToLong((int)CountersTable.Value, ref tvr), Etag = TableValueToEtag((int)CountersTable.Etag, ref tvr), }); }
private static (LazyStringValue Doc, LazyStringValue Name) ExtractDocIdAndName(JsonOperationContext context, TableValueReader tvr) { var p = tvr.Read((int)CountersTable.CounterKey, out var size); Debug.Assert(size > DbIdAsBase64Size + 2 /* record separators */); int sizeOfDocId = 0; for (; sizeOfDocId < size; sizeOfDocId++) { if (p[sizeOfDocId] == SpecialChars.RecordSeparator) { break; } } var doc = context.AllocateStringValue(null, p, sizeOfDocId); var name = ExtractCounterName(context, tvr); return(doc, name); }
private static unsafe SingleClusterDatabaseCommand ReadCommand(TransactionOperationContext context, TableValueReader reader) { var ptr = reader.Read((int)TransactionCommandsColumn.Commands, out var size); if (ptr == null) { return(null); } var blittable = new BlittableJsonReaderObject(ptr, size, context); blittable.TryGet(nameof(DatabaseCommands), out BlittableJsonReaderArray array); ClusterTransactionOptions options = null; if (blittable.TryGet(nameof(Options), out BlittableJsonReaderObject blittableOptions)) { options = JsonDeserializationServer.ClusterTransactionOptions(blittableOptions); } var index = *(long *)reader.Read((int)TransactionCommandsColumn.RaftIndex, out _); var keyPtr = reader.Read((int)TransactionCommandsColumn.Key, out size); var database = Encoding.UTF8.GetString(keyPtr, size - sizeof(long) - 1); return(new SingleClusterDatabaseCommand { Options = options, Commands = array, Index = index, PreviousCount = Bits.SwapBytes(*(long *)(keyPtr + size - sizeof(long))), Database = database }); }
private unsafe bool InstallSnapshot(TransactionOperationContext context) { var txw = context.Transaction.InnerTransaction; var sp = Stopwatch.StartNew(); var reader = _connection.CreateReader(); while (true) { var type = reader.ReadInt32(); if (type == -1) { return(false); } int size; long entries; switch ((RootObjectType)type) { case RootObjectType.None: return(true); case RootObjectType.VariableSizeTree: size = reader.ReadInt32(); reader.ReadExactly(size); Slice treeName; // will be freed on context close Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out treeName); txw.DeleteTree(treeName); var tree = txw.CreateTree(treeName); entries = reader.ReadInt64(); for (long i = 0; i < entries; i++) { MaybeNotifyLeaderThatWeAreStillAlive(context, sp); size = reader.ReadInt32(); reader.ReadExactly(size); using (Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out Slice valKey)) { size = reader.ReadInt32(); reader.ReadExactly(size); using (tree.DirectAdd(valKey, size, out byte *ptr)) { fixed(byte *pBuffer = reader.Buffer) { Memory.Copy(ptr, pBuffer, size); } } } } break; case RootObjectType.Table: size = reader.ReadInt32(); reader.ReadExactly(size); Slice tableName; // will be freed on context close Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out tableName); var tableTree = txw.ReadTree(tableName, RootObjectType.Table); // Get the table schema var schemaSize = tableTree.GetDataSize(TableSchema.SchemasSlice); var schemaPtr = tableTree.DirectRead(TableSchema.SchemasSlice); if (schemaPtr == null) { throw new InvalidOperationException( "When trying to install snapshot, found missing table " + tableName); } var schema = TableSchema.ReadFrom(txw.Allocator, schemaPtr, schemaSize); var table = txw.OpenTable(schema, tableName); // delete the table TableValueReader tvr; while (true) { if (table.SeekOnePrimaryKey(Slices.AfterAllKeys, out tvr) == false) { break; } table.Delete(tvr.Id); MaybeNotifyLeaderThatWeAreStillAlive(context, sp); } entries = reader.ReadInt64(); for (long i = 0; i < entries; i++) { MaybeNotifyLeaderThatWeAreStillAlive(context, sp); size = reader.ReadInt32(); reader.ReadExactly(size); fixed(byte *pBuffer = reader.Buffer) { tvr = new TableValueReader(pBuffer, size); table.Insert(ref tvr); } } break; default: throw new ArgumentOutOfRangeException(nameof(type), type.ToString()); } } }
private unsafe bool ReadSnapshot(SnapshotReader reader, ClusterOperationContext context, Transaction txw, bool dryRun, CancellationToken token) { var type = reader.ReadInt32(); if (type == -1) { return(false); } while (true) { token.ThrowIfCancellationRequested(); int size; long entries; switch ((RootObjectType)type) { case RootObjectType.None: return(true); case RootObjectType.VariableSizeTree: size = reader.ReadInt32(); reader.ReadExactly(size); Tree tree = null; if (dryRun == false) { Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out Slice treeName); // The Slice will be freed on context close txw.DeleteTree(treeName); tree = txw.CreateTree(treeName); } entries = reader.ReadInt64(); for (long i = 0; i < entries; i++) { token.ThrowIfCancellationRequested(); size = reader.ReadInt32(); reader.ReadExactly(size); using (Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out Slice valKey)) { size = reader.ReadInt32(); reader.ReadExactly(size); if (dryRun == false) { using (tree.DirectAdd(valKey, size, out byte *ptr)) { fixed(byte *pBuffer = reader.Buffer) { Memory.Copy(ptr, pBuffer, size); } } } } } break; case RootObjectType.Table: size = reader.ReadInt32(); reader.ReadExactly(size); TableValueReader tvr; Table table = null; if (dryRun == false) { Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out Slice tableName);//The Slice will be freed on context close var tableTree = txw.ReadTree(tableName, RootObjectType.Table); // Get the table schema var schemaSize = tableTree.GetDataSize(TableSchema.SchemasSlice); var schemaPtr = tableTree.DirectRead(TableSchema.SchemasSlice); if (schemaPtr == null) { throw new InvalidOperationException( "When trying to install snapshot, found missing table " + tableName); } var schema = TableSchema.ReadFrom(txw.Allocator, schemaPtr, schemaSize); table = txw.OpenTable(schema, tableName); // delete the table while (true) { token.ThrowIfCancellationRequested(); if (table.SeekOnePrimaryKey(Slices.AfterAllKeys, out tvr) == false) { break; } table.Delete(tvr.Id); } } entries = reader.ReadInt64(); for (long i = 0; i < entries; i++) { token.ThrowIfCancellationRequested(); size = reader.ReadInt32(); reader.ReadExactly(size); if (dryRun == false) { fixed(byte *pBuffer = reader.Buffer) { tvr = new TableValueReader(pBuffer, size); table.Insert(ref tvr); } } } break; default: throw new ArgumentOutOfRangeException(nameof(type), type.ToString()); } type = reader.ReadInt32(); } }
public static Document ParseRawDataSectionRevisionWithValidation(JsonOperationContext context, ref TableValueReader tvr, int expectedSize, out long etag) { var ptr = tvr.Read((int)Columns.Document, out var size); if (size > expectedSize || size <= 0) { throw new ArgumentException("Data size is invalid, possible corruption when parsing BlittableJsonReaderObject", nameof(size)); } var result = new Document { StorageId = tvr.Id, LowerId = TableValueToString(context, (int)Columns.LowerId, ref tvr), Id = TableValueToId(context, (int)Columns.Id, ref tvr), Etag = etag = TableValueToEtag((int)Columns.Etag, ref tvr), Data = new BlittableJsonReaderObject(ptr, size, context), LastModified = TableValueToDateTime((int)Columns.LastModified, ref tvr), Flags = TableValueToFlags((int)Columns.Flags, ref tvr), TransactionMarker = *(short *)tvr.Read((int)Columns.TransactionMarker, out size), ChangeVector = TableValueToChangeVector(context, (int)Columns.ChangeVector, ref tvr) }; if (size != sizeof(short)) { throw new ArgumentException("TransactionMarker size is invalid, possible corruption when parsing BlittableJsonReaderObject", nameof(size)); } return(result); }
private static unsafe BlittableJsonReaderObject Read(JsonOperationContext context, ref TableValueReader reader) { var ptr = reader.Read(DatabaseInfoSchema.DatabaseInfoTable.JsonIndex, out int size); return(new BlittableJsonReaderObject(ptr, size, context)); }
private (string ChangeVector, NonPersistentDocumentFlags NonPersistentFlags) BuildChangeVectorAndResolveConflicts( DocumentsOperationContext context, string id, Slice lowerId, long newEtag, BlittableJsonReaderObject document, string changeVector, string expectedChangeVector, DocumentFlags flags, TableValueReader oldValue) { var nonPersistentFlags = NonPersistentDocumentFlags.None; var fromReplication = (flags & DocumentFlags.FromReplication) == DocumentFlags.FromReplication; if (_documentsStorage.ConflictsStorage.ConflictsCount != 0) { // Since this document resolve the conflict we don't need to alter the change vector. // This way we avoid another replication back to the source _documentsStorage.ConflictsStorage.ThrowConcurrencyExceptionOnConflictIfNeeded(context, lowerId, expectedChangeVector); if (fromReplication) { nonPersistentFlags = _documentsStorage.ConflictsStorage.DeleteConflictsFor(context, id, document).NonPersistentFlags; } else { var result = _documentsStorage.ConflictsStorage.MergeConflictChangeVectorIfNeededAndDeleteConflicts(changeVector, context, id, newEtag, document); changeVector = result.ChangeVector; nonPersistentFlags = result.NonPersistentFlags; } } if (string.IsNullOrEmpty(changeVector) == false) { return(changeVector, nonPersistentFlags); } string oldChangeVector; if (fromReplication == false) { if (context.LastDatabaseChangeVector == null) { context.LastDatabaseChangeVector = GetDatabaseChangeVector(context); } oldChangeVector = context.LastDatabaseChangeVector; } else { oldChangeVector = oldValue.Pointer != null?TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue) : null; } changeVector = SetDocumentChangeVectorForLocalChange(context, lowerId, oldChangeVector, newEtag); return(changeVector, nonPersistentFlags); }
private static CounterDetail TableValueToCounterDetail(JsonOperationContext context, TableValueReader tvr) { var(doc, name) = ExtractDocIdAndNameFromLegacyCounter(context, ref tvr); using (name) using (doc) { return(new CounterDetail { CounterKey = TableValueToString(context, (int)LegacyCountersTable.CounterKey, ref tvr), DocumentId = doc.ToString(), CounterName = name.ToString(), TotalValue = TableValueToLong((int)LegacyCountersTable.Value, ref tvr), Etag = TableValueToEtag((int)LegacyCountersTable.Etag, ref tvr), }); } }
private ChangeVectorEntry[] SetIndexTransformerChangeVectorForLocalChange(Transaction tx, TransactionOperationContext context, Slice loweredName, TableValueReader oldValue, long newEtag, ChangeVectorEntry[] vector) { if (oldValue != null) { var changeVector = ReplicationUtils.GetChangeVectorEntriesFromTableValueReader(oldValue, (int)MetadataFields.ChangeVector); return(ReplicationUtils.UpdateChangeVectorWithNewEtag(_environment.DbId, newEtag, changeVector)); } return(GetMergedConflictChangeVectorsAndDeleteConflicts(tx, context, loweredName, newEtag)); }
public static unsafe long GetValue(TableValueReader tvr) { return(*(long *)tvr.Read((int)ClusterStateMachine.IdentitiesTable.Value, out _)); }