private static DocumentConflict TableValueToConflictDocument(DocumentsOperationContext context, ref TableValueReader tvr)
{
var result = new DocumentConflict
{
StorageId = tvr.Id,
LowerId = TableValueToString(context, (int)ConflictsTable.LowerId, ref tvr),
Id = TableValueToId(context, (int)ConflictsTable.Id, ref tvr),
ChangeVector = TableValueToChangeVector(context, (int)ConflictsTable.ChangeVector, ref tvr),
Etag = TableValueToEtag((int)ConflictsTable.Etag, ref tvr),
Collection = TableValueToId(context, (int)ConflictsTable.Collection, ref tvr),
LastModified = TableValueToDateTime((int)ConflictsTable.LastModified, ref tvr),
Flags = TableValueToFlags((int)ConflictsTable.Flags, ref tvr)
};
var read = tvr.Read((int)ConflictsTable.Data, out int size);
if (size > 0)
{
//otherwise this is a tombstone conflict and should be treated as such
result.Doc = new BlittableJsonReaderObject(read, size, context);
Transaction.DebugDisposeReaderAfterTransaction(context.Transaction.InnerTransaction, result.Doc);
}
return(result);
}
public static DocumentConflict ParseRawDataSectionConflictWithValidation(JsonOperationContext context, ref TableValueReader tvr, int expectedSize, out long etag)
{
var read = tvr.Read((int)ConflictsTable.Data, out var size);
if (size > expectedSize || size <= 0)
{
throw new ArgumentException("Document size is invalid, possible corruption when parsing BlittableJsonReaderObject", nameof(size));
}
var result = new DocumentConflict
{
StorageId = tvr.Id,
LowerId = TableValueToString(context, (int)ConflictsTable.LowerId, ref tvr),
Id = TableValueToId(context, (int)ConflictsTable.Id, ref tvr),
ChangeVector = TableValueToChangeVector(context, (int)ConflictsTable.ChangeVector, ref tvr),
Etag = etag = TableValueToEtag((int)ConflictsTable.Etag, ref tvr),
Doc = new BlittableJsonReaderObject(read, size, context),
Collection = TableValueToId(context, (int)ConflictsTable.Collection, ref tvr),
LastModified = TableValueToDateTime((int)ConflictsTable.LastModified, ref tvr),
Flags = TableValueToFlags((int)ConflictsTable.Flags, ref tvr)
};
return(result);
}
public static CounterDetail TableValueToCounterDetail(JsonOperationContext context, TableValueReader tvr)
{
var(doc, name) = ExtractDocIdAndName(context, tvr);
return(new CounterDetail
{
DocumentId = doc,
LazyDocumentId = doc,
CounterName = name,
ChangeVector = TableValueToString(context, (int)CountersTable.ChangeVector, ref tvr),
TotalValue = TableValueToLong((int)CountersTable.Value, ref tvr),
Etag = TableValueToEtag((int)CountersTable.Etag, ref tvr),
});
}
private static (LazyStringValue Doc, LazyStringValue Name) ExtractDocIdAndName(JsonOperationContext context, TableValueReader tvr)
{
var p = tvr.Read((int)CountersTable.CounterKey, out var size);
Debug.Assert(size > DbIdAsBase64Size + 2 /* record separators */);
int sizeOfDocId = 0;
for (; sizeOfDocId < size; sizeOfDocId++)
{
if (p[sizeOfDocId] == SpecialChars.RecordSeparator)
{
break;
}
}
var doc = context.AllocateStringValue(null, p, sizeOfDocId);
var name = ExtractCounterName(context, tvr);
return(doc, name);
}
private static unsafe SingleClusterDatabaseCommand ReadCommand(TransactionOperationContext context, TableValueReader reader)
{
var ptr = reader.Read((int)TransactionCommandsColumn.Commands, out var size);
if (ptr == null)
{
return(null);
}
var blittable = new BlittableJsonReaderObject(ptr, size, context);
blittable.TryGet(nameof(DatabaseCommands), out BlittableJsonReaderArray array);
ClusterTransactionOptions options = null;
if (blittable.TryGet(nameof(Options), out BlittableJsonReaderObject blittableOptions))
{
options = JsonDeserializationServer.ClusterTransactionOptions(blittableOptions);
}
var index = *(long *)reader.Read((int)TransactionCommandsColumn.RaftIndex, out _);
var keyPtr = reader.Read((int)TransactionCommandsColumn.Key, out size);
var database = Encoding.UTF8.GetString(keyPtr, size - sizeof(long) - 1);
return(new SingleClusterDatabaseCommand
{
Options = options,
Commands = array,
Index = index,
PreviousCount = Bits.SwapBytes(*(long *)(keyPtr + size - sizeof(long))),
Database = database
});
}
private unsafe bool InstallSnapshot(TransactionOperationContext context)
{
var txw = context.Transaction.InnerTransaction;
var sp = Stopwatch.StartNew();
var reader = _connection.CreateReader();
while (true)
{
var type = reader.ReadInt32();
if (type == -1)
{
return(false);
}
int size;
long entries;
switch ((RootObjectType)type)
{
case RootObjectType.None:
return(true);
case RootObjectType.VariableSizeTree:
size = reader.ReadInt32();
reader.ReadExactly(size);
Slice treeName; // will be freed on context close
Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out treeName);
txw.DeleteTree(treeName);
var tree = txw.CreateTree(treeName);
entries = reader.ReadInt64();
for (long i = 0; i < entries; i++)
{
MaybeNotifyLeaderThatWeAreStillAlive(context, sp);
size = reader.ReadInt32();
reader.ReadExactly(size);
using (Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out Slice valKey))
{
size = reader.ReadInt32();
reader.ReadExactly(size);
using (tree.DirectAdd(valKey, size, out byte *ptr))
{
fixed(byte *pBuffer = reader.Buffer)
{
Memory.Copy(ptr, pBuffer, size);
}
}
}
}
break;
case RootObjectType.Table:
size = reader.ReadInt32();
reader.ReadExactly(size);
Slice tableName; // will be freed on context close
Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable,
out tableName);
var tableTree = txw.ReadTree(tableName, RootObjectType.Table);
// Get the table schema
var schemaSize = tableTree.GetDataSize(TableSchema.SchemasSlice);
var schemaPtr = tableTree.DirectRead(TableSchema.SchemasSlice);
if (schemaPtr == null)
{
throw new InvalidOperationException(
"When trying to install snapshot, found missing table " + tableName);
}
var schema = TableSchema.ReadFrom(txw.Allocator, schemaPtr, schemaSize);
var table = txw.OpenTable(schema, tableName);
// delete the table
TableValueReader tvr;
while (true)
{
if (table.SeekOnePrimaryKey(Slices.AfterAllKeys, out tvr) == false)
{
break;
}
table.Delete(tvr.Id);
MaybeNotifyLeaderThatWeAreStillAlive(context, sp);
}
entries = reader.ReadInt64();
for (long i = 0; i < entries; i++)
{
MaybeNotifyLeaderThatWeAreStillAlive(context, sp);
size = reader.ReadInt32();
reader.ReadExactly(size);
fixed(byte *pBuffer = reader.Buffer)
{
tvr = new TableValueReader(pBuffer, size);
table.Insert(ref tvr);
}
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type.ToString());
}
}
}
private unsafe bool ReadSnapshot(SnapshotReader reader, ClusterOperationContext context, Transaction txw, bool dryRun, CancellationToken token)
{
var type = reader.ReadInt32();
if (type == -1)
{
return(false);
}
while (true)
{
token.ThrowIfCancellationRequested();
int size;
long entries;
switch ((RootObjectType)type)
{
case RootObjectType.None:
return(true);
case RootObjectType.VariableSizeTree:
size = reader.ReadInt32();
reader.ReadExactly(size);
Tree tree = null;
if (dryRun == false)
{
Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out Slice treeName); // The Slice will be freed on context close
txw.DeleteTree(treeName);
tree = txw.CreateTree(treeName);
}
entries = reader.ReadInt64();
for (long i = 0; i < entries; i++)
{
token.ThrowIfCancellationRequested();
size = reader.ReadInt32();
reader.ReadExactly(size);
using (Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable, out Slice valKey))
{
size = reader.ReadInt32();
reader.ReadExactly(size);
if (dryRun == false)
{
using (tree.DirectAdd(valKey, size, out byte *ptr))
{
fixed(byte *pBuffer = reader.Buffer)
{
Memory.Copy(ptr, pBuffer, size);
}
}
}
}
}
break;
case RootObjectType.Table:
size = reader.ReadInt32();
reader.ReadExactly(size);
TableValueReader tvr;
Table table = null;
if (dryRun == false)
{
Slice.From(context.Allocator, reader.Buffer, 0, size, ByteStringType.Immutable,
out Slice tableName);//The Slice will be freed on context close
var tableTree = txw.ReadTree(tableName, RootObjectType.Table);
// Get the table schema
var schemaSize = tableTree.GetDataSize(TableSchema.SchemasSlice);
var schemaPtr = tableTree.DirectRead(TableSchema.SchemasSlice);
if (schemaPtr == null)
{
throw new InvalidOperationException(
"When trying to install snapshot, found missing table " + tableName);
}
var schema = TableSchema.ReadFrom(txw.Allocator, schemaPtr, schemaSize);
table = txw.OpenTable(schema, tableName);
// delete the table
while (true)
{
token.ThrowIfCancellationRequested();
if (table.SeekOnePrimaryKey(Slices.AfterAllKeys, out tvr) == false)
{
break;
}
table.Delete(tvr.Id);
}
}
entries = reader.ReadInt64();
for (long i = 0; i < entries; i++)
{
token.ThrowIfCancellationRequested();
size = reader.ReadInt32();
reader.ReadExactly(size);
if (dryRun == false)
{
fixed(byte *pBuffer = reader.Buffer)
{
tvr = new TableValueReader(pBuffer, size);
table.Insert(ref tvr);
}
}
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type.ToString());
}
type = reader.ReadInt32();
}
}
public static Document ParseRawDataSectionRevisionWithValidation(JsonOperationContext context, ref TableValueReader tvr, int expectedSize, out long etag)
{
var ptr = tvr.Read((int)Columns.Document, out var size);
if (size > expectedSize || size <= 0)
{
throw new ArgumentException("Data size is invalid, possible corruption when parsing BlittableJsonReaderObject", nameof(size));
}
var result = new Document
{
StorageId = tvr.Id,
LowerId = TableValueToString(context, (int)Columns.LowerId, ref tvr),
Id = TableValueToId(context, (int)Columns.Id, ref tvr),
Etag = etag = TableValueToEtag((int)Columns.Etag, ref tvr),
Data = new BlittableJsonReaderObject(ptr, size, context),
LastModified = TableValueToDateTime((int)Columns.LastModified, ref tvr),
Flags = TableValueToFlags((int)Columns.Flags, ref tvr),
TransactionMarker = *(short *)tvr.Read((int)Columns.TransactionMarker, out size),
ChangeVector = TableValueToChangeVector(context, (int)Columns.ChangeVector, ref tvr)
};
if (size != sizeof(short))
{
throw new ArgumentException("TransactionMarker size is invalid, possible corruption when parsing BlittableJsonReaderObject", nameof(size));
}
return(result);
}
private static unsafe BlittableJsonReaderObject Read(JsonOperationContext context, ref TableValueReader reader)
{
var ptr = reader.Read(DatabaseInfoSchema.DatabaseInfoTable.JsonIndex, out int size);
return(new BlittableJsonReaderObject(ptr, size, context));
}
private (string ChangeVector, NonPersistentDocumentFlags NonPersistentFlags) BuildChangeVectorAndResolveConflicts(
DocumentsOperationContext context, string id, Slice lowerId, long newEtag,
BlittableJsonReaderObject document, string changeVector, string expectedChangeVector, DocumentFlags flags, TableValueReader oldValue)
{
var nonPersistentFlags = NonPersistentDocumentFlags.None;
var fromReplication = (flags & DocumentFlags.FromReplication) == DocumentFlags.FromReplication;
if (_documentsStorage.ConflictsStorage.ConflictsCount != 0)
{
// Since this document resolve the conflict we don't need to alter the change vector.
// This way we avoid another replication back to the source
_documentsStorage.ConflictsStorage.ThrowConcurrencyExceptionOnConflictIfNeeded(context, lowerId, expectedChangeVector);
if (fromReplication)
{
nonPersistentFlags = _documentsStorage.ConflictsStorage.DeleteConflictsFor(context, id, document).NonPersistentFlags;
}
else
{
var result = _documentsStorage.ConflictsStorage.MergeConflictChangeVectorIfNeededAndDeleteConflicts(changeVector, context, id, newEtag, document);
changeVector = result.ChangeVector;
nonPersistentFlags = result.NonPersistentFlags;
}
}
if (string.IsNullOrEmpty(changeVector) == false)
{
return(changeVector, nonPersistentFlags);
}
string oldChangeVector;
if (fromReplication == false)
{
if (context.LastDatabaseChangeVector == null)
{
context.LastDatabaseChangeVector = GetDatabaseChangeVector(context);
}
oldChangeVector = context.LastDatabaseChangeVector;
}
else
{
oldChangeVector = oldValue.Pointer != null?TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue) : null;
}
changeVector = SetDocumentChangeVectorForLocalChange(context, lowerId, oldChangeVector, newEtag);
return(changeVector, nonPersistentFlags);
}
private static CounterDetail TableValueToCounterDetail(JsonOperationContext context, TableValueReader tvr)
{
var(doc, name) = ExtractDocIdAndNameFromLegacyCounter(context, ref tvr);
using (name)
using (doc)
{
return(new CounterDetail
{
CounterKey = TableValueToString(context, (int)LegacyCountersTable.CounterKey, ref tvr),
DocumentId = doc.ToString(),
CounterName = name.ToString(),
TotalValue = TableValueToLong((int)LegacyCountersTable.Value, ref tvr),
Etag = TableValueToEtag((int)LegacyCountersTable.Etag, ref tvr),
});
}
}
private ChangeVectorEntry[] SetIndexTransformerChangeVectorForLocalChange(Transaction tx, TransactionOperationContext context, Slice loweredName, TableValueReader oldValue, long newEtag, ChangeVectorEntry[] vector)
{
if (oldValue != null)
{
var changeVector = ReplicationUtils.GetChangeVectorEntriesFromTableValueReader(oldValue, (int)MetadataFields.ChangeVector);
return(ReplicationUtils.UpdateChangeVectorWithNewEtag(_environment.DbId, newEtag, changeVector));
}
return(GetMergedConflictChangeVectorsAndDeleteConflicts(tx, context, loweredName, newEtag));
}
public static unsafe long GetValue(TableValueReader tvr)
{
return(*(long *)tvr.Read((int)ClusterStateMachine.IdentitiesTable.Value, out _));
}
