/// <summary>
/// Read the given property value.
/// </summary>
/// <param name="reader">Stream to read from.</param>
/// <param name="property">Property to read.</param>
/// <param name="valueSize">The size in bytes of the value.</param>
protected override void ReadProperty(BinaryReader reader, int property, int valueSize)
{
ByteAlignedReadWriteHelper.ReadPaddingUntilAligned(reader, true);
switch ((PropertyId)property)
{
case PropertyId.ValuesHandle:
this.ValuesHandle = BlockHandle.Read(reader);
break;
case PropertyId.ValueCount:
this.ValueCount = reader.ReadInt64();
break;
case PropertyId.FileId:
this.FileId = reader.ReadUInt32();
ByteAlignedReadWriteHelper.ReadPaddingUntilAligned(reader, true);
break;
default:
base.ReadProperty(reader, property, valueSize);
ByteAlignedReadWriteHelper.ReadPaddingUntilAligned(reader, true);
break;
}
ByteAlignedReadWriteHelper.ReadPaddingUntilAligned(reader, true);
}
public EngineFlowController(ModuleGlobalInfo globalInfo)
{
_globalInfo = globalInfo;
this._blockHandle = new BlockHandle();
// TODO 暂时写死，只是用本地测试生成实体
_testsMaintainer = new LocalTestEntityMaintainer(_globalInfo, _blockHandle);
}
/// <summary>
/// Deserializes the metadata (footer, properties, etc.) for this checkpoint file.
/// </summary>
private async Task ReadMetadataAsync()
{
Stream filestream = null;
try
{
filestream = this.ReaderPool.AcquireStream();
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(filestream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
this.Footer =
await FileBlock.ReadBlockAsync(filestream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);
// Verify we know how to deserialize this version of the checkpoint file.
if (this.Footer.Version != FileVersion)
{
throw new InvalidDataException(SR.Error_KeyCheckpointFile_Deserialized);
}
// Read and validate the Properties section.
var propertiesHandle = this.Footer.PropertiesHandle;
this.Properties =
await
FileBlock.ReadBlockAsync(
filestream,
propertiesHandle,
(sectionReader, sectionHandle) => FilePropertySection.Read <KeyCheckpointFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);
}
finally
{
this.ReaderPool.ReleaseStream(filestream, true);
}
}
/// <summary>
/// Convert to initialization context.
/// </summary>
/// <param name="bytes">Serialized bytes.</param>
/// <returns>Deserialized initialization context.</returns>
public static StoreInitializationParameters FromByteArray(byte[] bytes)
{
if (bytes == null)
{
throw new ArgumentNullException(SR.Error_BytesNull);
}
using (var stream = new MemoryStream(bytes))
using (var reader = new BinaryReader(stream))
{
// Read and check the version first. This is for future proofing - if the format dramatically changes,
// we can change the serialization version and modify the serialization/deserialization code.
// For now, we verify the version matches exactly the expected version.
var version = reader.ReadInt32();
if (version != SerializationVersion)
{
throw new InvalidDataException(
string.Format(
System.Globalization.CultureInfo.CurrentCulture,
SR.Error_StoreInitParameters_Deserialized,
version,
SerializationVersion));
}
// Currently, the rest of the properties are serialized as [string, value] pairs (see ReadProperty and Write methods).
var handle = new BlockHandle(offset: sizeof(int), size: bytes.Length - sizeof(int));
var initializationParameters = Read <StoreInitializationParameters>(reader, handle);
initializationParameters.Version = version;
return(initializationParameters);
}
}
/// <summary>
///
/// </summary>
/// <remarks>Exposed for testability.</remarks>
internal static async Task PopulateMetadataAsync(MetadataTable metadataTable, Stream stream, string traceType)
{
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(stream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
var footer = await FileBlock.ReadBlockAsync(stream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);
// Verify we know how to deserialize this version of the file.
if (footer.Version != FileVersion)
{
throw new InvalidDataException(SR.Error_MetadataManager_Deserialized);
}
// Read and validate the Properties section.
var propertiesHandle = footer.PropertiesHandle;
var properties =
await
FileBlock.ReadBlockAsync(
stream,
propertiesHandle,
(sectionReader, sectionHandle) => FilePropertySection.Read <MetadataManagerFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);
// Read disk metadata into memory.
await ReadDiskMetadataAsync(metadataTable.Table, stream, properties, traceType).ConfigureAwait(false);
metadataTable.CheckpointLSN = properties.CheckpointLSN;
}
public void Abort(int sessionId)
{
ControlMessage abortMessage = new ControlMessage(MessageNames.CtrlAbort, CommonConst.BroadcastSession);
abortMessage.AddParam("IsRequest", true.ToString());
abortMessage.Id = sessionId;
_globalInfo.MessageTransceiver.Send(abortMessage);
AbortEventInfo abortEventInfo = new AbortEventInfo(sessionId, true, false);
_globalInfo.EventQueue.Enqueue(abortEventInfo);
if (null == _abortBlocker)
{
_abortBlocker = new BlockHandle();
}
// 目前使用同步Abort，每次只能释放一个
_abortBlocker.Timeout = _globalInfo.ConfigData.GetProperty <int>("AbortTimeout");
bool isNotTimeout = _abortBlocker.Wait(Constants.AbortState);
if (!isNotTimeout)
{
_globalInfo.LogService.Print(LogLevel.Warn, CommonConst.PlatformLogSession,
$"Session {sessionId} abort timeout.");
}
}
internal static extern unsafe ReturnCode terab_utxo_set_coins(
SafeConnectionHandle connection,
BlockHandle context,
int coinLength,
Coin *coins,
int storageLength,
byte *storage
);
public unsafe void SetCoins(BlockHandle context, Span <Coin> coins, Span <byte> storage)
{
if (coins.Length > MaxCoinsPerCall)
{
throw new ArgumentOutOfRangeException(nameof(coins), "Too many coins.");
}
ref var coinRef = ref coins.GetPinnableReference();
public void CommitBlock(BlockHandle handle, CommittedBlockId blockId)
{
var returnCode = PInvokes.terab_utxo_commit_block(_connection, handle, ref blockId);
if (returnCode != ReturnCode.SUCCESS)
{
throw new TerabException("CommitBlock failed.", returnCode);
}
}
public void GenesisParentHandle()
{
var rand = new Random(42);
var randInt = rand.Next();
var handle1 = new BlockHandle(unchecked ((uint)randInt) ^ BlockHandle.DefaultMask);
var handle2 = BlockAlias.GenesisParent.ConvertToBlockHandle(new BlockHandleMask(unchecked ((uint)randInt)));
Assert.Equal(handle1, handle2);
}
// ----------------
// HELPER FUNCTIONS
// ----------------
/// Converts a BlockHandle to a Block prefab.
static GameObject ToPrefab(BlockHandle blockHandle)
{
switch (blockHandle)
{
case BlockHandle.WallSimple: return(Prefabs.Instance.wallSimple);
case BlockHandle.Spawn: return(Prefabs.Instance.spawn);
default: return(Prefabs.Instance.empty);
}
}
public (Coin[] coins, byte[] storage) GetProductionCoins(BlockHandle handle)
{
var coins = new Coin[3];
coins[0] = new Coin
{
Outpoint = GetOutpoint(123),
Production = handle,
Consumption = BlockHandle.Undefined,
Satoshis = 42,
NLockTime = 11,
ScriptOffset = 0,
ScriptLength = 10,
Flags = CoinFlags.Coinbase,
Status = CoinStatus.None,
};
coins[1] = new Coin
{
Outpoint = GetOutpoint(124),
Production = handle,
Consumption = BlockHandle.Undefined,
Satoshis = 43,
NLockTime = 12,
ScriptOffset = 10,
ScriptLength = 20,
Flags = CoinFlags.None,
Status = CoinStatus.None,
};
coins[2] = new Coin
{
Outpoint = GetOutpoint(125),
Production = handle,
Consumption = BlockHandle.Undefined,
Satoshis = 44,
NLockTime = 13,
ScriptOffset = 30,
ScriptLength = 40,
Flags = CoinFlags.None,
Status = CoinStatus.None,
};
var storage = new byte[70];
for (var i = 0; i < storage.Length; i++)
{
storage[i] = (byte)i;
}
return(coins, storage);
}
public BlockInfo GetBlockInfo(BlockHandle handle)
{
var info = new BlockInfo();
var returnCode = PInvokes.terab_utxo_get_blockinfo(
_connection, handle, ref info);
if (returnCode != ReturnCode.SUCCESS)
{
throw new TerabException("GetBlockInfo failed.", returnCode);
}
return(info);
}
public EngineFlowController(ModuleGlobalInfo globalInfo)
{
_globalInfo = globalInfo;
// TODO 暂时写死，只是用本地测试生成实体
_testsMaintainer = new LocalTestEntityMaintainer(_globalInfo, _blockHandle);
if (EnableDebug)
{
_debugManager = new DebugManager(_globalInfo);
}
this._blockHandle = new BlockHandle()
{
Timeout = _globalInfo.ConfigData.GetProperty <int>("TestGenTimeout")
};
}
public void Reset()
{
_blockIndexEnum.Reset();
BlockEntry entry = _blockIndexEnum.Current;
if (entry == null)
{
throw new Exception("Error");
}
byte[] dataBlock = _table.GetBlock(BlockHandle.ReadBlockHandle(entry.Data.Span));
_currentBlockEnum = new BlockEnumerator(dataBlock);
Current = _currentBlockEnum.Current;
}
/// <summary> Client-side perspective. </summary>
public GetCoinRequest(
RequestId requestId,
ref Outpoint outpoint,
BlockHandle context,
SpanPool <byte> pool)
{
_buffer = pool.GetSpan(Header.SizeInBytes);
_mask = default;
AsHeader.RequestHeader.MessageSizeInBytes = Header.SizeInBytes;
AsHeader.RequestHeader.RequestId = requestId;
AsHeader.RequestHeader.MessageKind = MessageKind.GetCoin;
Outpoint = outpoint;
HandleContext = context;
}
/// <summary>
/// Allocate an array. Intended for testing purposes only.
/// </summary>
internal static GetBlockInfoRequest From(
RequestId requestId, ClientId clientId, BlockHandle handle)
{
var request = new GetBlockInfoRequest {
_buffer = new Span <byte>(new byte[sizeof(Header)])
};
request.MessageHeader.MessageSizeInBytes = Header.SizeInBytes;
request.MessageHeader.RequestId = requestId;
request.MessageHeader.ClientId = clientId;
request.MessageHeader.MessageKind = MessageKind.GetBlockInfo;
request.AsHeader.Handle = handle;
return(request);
}
private static void CommitBlock(BlockHandle handle, CommittedBlockId blockId, ISocketLike socket)
{
var request = CommitBlockRequest.From(
RequestId.MinRequestId, ClientId.MinClientId, handle, blockId);
socket.Send(request.Span);
var response = new CommitBlockResponse(new byte[CommitBlockResponse.SizeInBytes]);
socket.Receive(response.Span);
if (response.Status != CommitBlockStatus.Success)
{
throw new InvalidOperationException("Failed to commit block.");
}
}
public static async Task <RecoveryPointMetadataFile> OpenAsync(
Stream filestream,
string recoveryPointMetadataFilePath,
CancellationToken cancellationToken)
{
var recoveryPointMetadataFile = new RecoveryPointMetadataFile(recoveryPointMetadataFilePath);
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(
filestream.Length - FileFooter.SerializedSize - sizeof(ulong),
FileFooter.SerializedSize);
recoveryPointMetadataFile.footer =
await
FileBlock.ReadBlockAsync(
filestream,
footerHandle,
(reader, handle) => FileFooter.Read(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
// Verify we know how to deserialize this version of the backup log file.
if (recoveryPointMetadataFile.footer.Version != Version)
{
throw new InvalidDataException(
string.Format(
CultureInfo.CurrentCulture,
SR.Error_BackupMetadata_Deserialized,
recoveryPointMetadataFile.footer.Version,
Version));
}
// Read and validate the properties section.
var propertiesHandle = recoveryPointMetadataFile.footer.PropertiesHandle;
recoveryPointMetadataFile.properties =
await
FileBlock.ReadBlockAsync(
filestream,
propertiesHandle,
(reader, handle) => FileProperties.Read <RecoveryPointMetadataFileProperties>(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
return(recoveryPointMetadataFile);
}
/// <summary>
/// Allocate an array. Intended for testing purposes only.
/// </summary>
internal static CommitBlockRequest From(
RequestId requestId, ClientId clientId, BlockHandle handle, CommittedBlockId blockId)
{
var request = new CommitBlockRequest {
_buffer = new Span <byte>(new byte[sizeof(Header)])
};
request.MessageHeader.MessageSizeInBytes = Header.SizeInBytes;
request.MessageHeader.RequestId = requestId;
request.MessageHeader.ClientId = clientId;
request.MessageHeader.MessageKind = MessageKind.CommitBlock;
request.AsHeader.Handle = handle;
request.AsHeader.BlockId = blockId;
return(request);
}
/// <summary>
/// Deserialize the <see cref="FilePropertySection"/> from the given stream.
/// </summary>
/// <param name="reader">The stream to read from.</param>
/// <param name="handle">Starting offset and size within the stream for the file properties.</param>
/// <returns>The deserialized <see cref="FilePropertySection"/> section.</returns>
public static TProperties Read <TProperties>(BinaryReader reader, BlockHandle handle) where TProperties : FilePropertySection, new()
{
var properties = new TProperties();
if (handle.Offset < 0)
{
throw new InvalidDataException(SR.Error_FilePropertySection_Missing);
}
if (handle.Size == 0)
{
return(properties);
}
// The FileProperties start at the given 'offset' within the stream, and contains properties up to 'offset' + 'size'.
reader.BaseStream.Position = handle.Offset;
while (reader.BaseStream.Position < handle.EndOffset)
{
// Read the property name.
var property = reader.ReadInt32();
// Read the size in bytes of the property's value.
var valueSize = VarInt.ReadInt32(reader);
if (valueSize < 0)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FilePropertySection_CorruptSizeNegative_OneArgs, valueSize));
}
if (reader.BaseStream.Position + valueSize > handle.EndOffset)
{
throw new InvalidDataException(SR.Error_FilePropertySection_CorruptSizeExtend);
}
// Read the property's value.
properties.ReadProperty(reader, property, valueSize);
}
// Validate the properties section ended exactly where expected.
if (reader.BaseStream.Position != handle.EndOffset)
{
throw new InvalidDataException(SR.Error_FilePropertySection_CorruptSizeIncorrect);
}
return(properties);
}
/// <summary> Coin removal. Client-side perspective. </summary>
public RemoveCoinRequest(
Span <byte> buffer,
RequestId requestId,
ref Outpoint outpoint,
BlockHandle context,
bool removeProduction,
bool removeConsumption)
{
_buffer = buffer;
_mask = default;
AsHeader.RequestHeader.MessageSizeInBytes = Header.SizeInBytes;
AsHeader.RequestHeader.RequestId = requestId;
AsHeader.RequestHeader.MessageKind = MessageKind.RemoveCoin;
Outpoint = outpoint;
HandleContext = context;
RemoveProduction = removeProduction;
RemoveConsumption = removeConsumption;
}
/// <summary>
/// Read the given property value.
/// </summary>
/// <param name="reader">Stream to read from.</param>
/// <param name="property">Property to read.</param>
/// <param name="valueSize">The size in bytes of the value.</param>
protected override void ReadProperty(BinaryReader reader, int property, int valueSize)
{
switch ((PropertyId)property)
{
case PropertyId.MetadataHandle:
this.MetadataHandle = BlockHandle.Read(reader);
break;
case PropertyId.FileCount:
this.FileCount = reader.ReadInt32();
break;
case PropertyId.CheckpointLSN:
this.checkpointLSN = reader.ReadInt64();
break;
default:
base.ReadProperty(reader, property, valueSize);
break;
}
}
/// <summary> Coin production. Client-side perspective. </summary>
public ProduceCoinRequest(
RequestId requestId,
ref Outpoint outpoint,
OutpointFlags flags,
BlockHandle context,
ulong satoshis,
uint nLockTime,
int scriptSizeInBytes,
SpanPool <byte> pool)
{
_buffer = pool.GetSpan(Header.SizeInBytes + scriptSizeInBytes);
_mask = default;
AsHeader.RequestHeader.MessageSizeInBytes = Header.SizeInBytes + scriptSizeInBytes;
AsHeader.RequestHeader.RequestId = requestId;
AsHeader.RequestHeader.MessageKind = MessageKind.ProduceCoin;
Outpoint = outpoint;
HandleContext = context;
Flags = flags;
Satoshis = satoshis;
NLockTime = nLockTime;
}
/// <summary>
/// Deserializes the metadata (footer, properties, etc.) for this checkpoint file.
/// </summary>
private async Task ReadMetadataAsync()
{
Stream filestream = null;
try
{
filestream = this.ReaderPool.AcquireStream();
var snapFileStream = filestream as FileStream;
Diagnostics.Assert(snapFileStream != null, this.traceType, "fileStream must be a FileStream");
Microsoft.ServiceFabric.Replicator.Utility.SetIoPriorityHint(snapFileStream.SafeFileHandle, this.priorityHint);
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(filestream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
this.Footer =
await FileBlock.ReadBlockAsync(filestream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);
// Verify we know how to deserialize this version of the checkpoint file.
if (this.Footer.Version != FileVersion)
{
throw new InvalidDataException(SR.Error_ValueCheckpoint_Deserialized);
}
// Read and validate the Properties section.
var propertiesHandle = this.Footer.PropertiesHandle;
this.Properties =
await
FileBlock.ReadBlockAsync(
filestream,
propertiesHandle,
(sectionReader, sectionHandle) => FilePropertySection.Read <ValueCheckpointFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);
}
finally
{
this.ReaderPool.ReleaseStream(filestream, true);
}
}
public bool MoveNext()
{
if (!_currentBlockEnum.MoveNext())
{
if (!_blockIndexEnum.MoveNext())
{
return(false);
}
BlockEntry entry = _blockIndexEnum.Current;
if (entry == null)
{
Log.Warn($"Unexpected empty index entry");
return(false);
}
byte[] dataBlock = _table.GetBlock(BlockHandle.ReadBlockHandle(entry.Data.Span));
_currentBlockEnum = new BlockEnumerator(dataBlock);
}
Current = _currentBlockEnum.Current;
return(Current != null);
}
internal static async Task ValidateAsync(string path, string traceType)
{
// MCoskun: Open of metadata file's IO priority is not set.
// Reason: USed during CompleteCheckpoint which can block copy and backup.
using (var stream = FabricFile.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous))
{
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(stream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
var footer = await FileBlock.ReadBlockAsync(stream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);
// Verify we know how to deserialize this version of the file.
if (footer.Version != FileVersion)
{
throw new InvalidDataException(SR.Error_MetadataManager_Deserialized);
}
// Read and validate the Properties section.
var propertiesHandle = footer.PropertiesHandle;
var properties = await FileBlock.ReadBlockAsync(
stream,
propertiesHandle,
(sectionReader, sectionHandle) => FilePropertySection.Read <MetadataManagerFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);
}
}
internal static extern ReturnCode terab_utxo_get_uncommitted_block(
SafeConnectionHandle connection,
ref UncommittedBlockId blockUcid,
out BlockHandle handle);
internal static extern ReturnCode terab_utxo_get_blockinfo(
SafeConnectionHandle connection,
BlockHandle handle,
ref BlockInfo info);
public LocalTestEntityMaintainer(ModuleGlobalInfo globalInfo, BlockHandle blockHandle)
{
_globalInfo = globalInfo;
this._runtimeContainers = new Dictionary <int, RuntimeContainer>(Constants.DefaultRuntimeSize);
this._blockHandle = blockHandle;
}
