public void OverFlowVolatile()
{
var sozuTable = PrepareTests(2, /*firstLayerVolatile*/ true);
var whatToWrite = new TxoPack(new Outpoint[120], new byte[8192]);
var wa = whatToWrite.GetResetWriter();
for (var i = 0UL; i < 20; i++)
{
var txid = new TxId(new Hash256(0xF111111111111122UL, 0x22222222UL, 0x33333333UL, 0x44444444UL + i));
var outpoint = new Outpoint(txid, 0, 200);
var payload = new byte[200];
payload[8] = 8;
payload[16 + i] = 42;
wa.Write(in outpoint, payload);
}
sozuTable.Write(whatToWrite);
var txoRead = new TxoPack(new Outpoint[120], new byte[8192]);
sozuTable.Read(whatToWrite, txoRead);
Assert.Equal(txoRead.Count, whatToWrite.Count);
for (var i = 0; i < txoRead.Count; ++i)
{
Assert.True(txoRead.OutPoints[i].Equals(whatToWrite.OutPoints[i]));
Assert.True(txoRead.Payloads.SequenceEqual(whatToWrite.Payloads));
}
}
public void Read()
{
var path = @"Read-Test.store";
File.Delete(path);
File.Delete(Path.Combine(Path.GetDirectoryName(path), @"hot-" + Path.GetFileName(path)));
using (var fw = File.OpenWrite(path))
{
fw.Write(new byte[]
{
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x01, 0x00, 0x00, 0x00, 0xf1, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x22, 0x00, 0x00, 0x00, 0x00,
0x22, 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x33, 0x33, 0x33, 0x33, 0x00, 0x00, 0x00, 0x00,
0x44, 0x44, 0x44, 0x44, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0xb8, 0x02, 0xb4, 0xb6, 0x09, 0x57, 0x75,
0x10, 0x92, 0xd2, 0x04, 0x1b, 0xac, 0xae, 0x6e, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xd6, 0x17, 0xe6, 0x61, 0x8f, 0xa0, 0xb9, 0x49, 0x99, 0xf9, 0xb2, 0x89, 0xe1, 0xb1, 0x72, 0xac,
0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5a, 0x92, 0x65, 0xbe, 0xf7, 0x46, 0xc3, 0x12,
0xd1, 0xa2, 0x27, 0xfa, 0x1b, 0x2f, 0x3e, 0x4c, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xda, 0x26, 0x9a, 0x14, 0x9f, 0x07, 0xc9, 0xa8, 0x7d, 0xeb, 0xd7, 0xf7, 0x4f, 0x94, 0x9b, 0x7e
});
}
var startHash = new Hash256(0xC000000000000000UL);
var endHash = new Hash256(0xF400000000000000UL);
var sozuFactoryConfigs = new SozuFactoryConfig[1];
sozuFactoryConfigs[0] = new SozuFactoryConfig(2, SozuFactory.TwoPhaseStoreType, 88);
var whatToWrite = new TxoPack(new Outpoint[1], new byte[2048]);
var firstHash = new TxId(new Hash256(0xF111111111111122UL, 0x22222222UL, 0x33333333UL, 0x44444444UL));
var wa = whatToWrite.GetResetWriter();
var outpoint = new Outpoint(firstHash, 0, 8);
var payload = new byte[8];
payload[0] = 42;
wa.Write(in outpoint, payload);
var txoRead = new TxoPack(new Outpoint[1], new byte[2048]);
var sozuTable = new SozuFactory(null, startHash, endHash, sozuFactoryConfigs, path).Create();
sozuTable.Read(whatToWrite, txoRead);
Assert.Equal(txoRead.Count, whatToWrite.Count);
Assert.True(txoRead.OutPoints[0].Equals(whatToWrite.OutPoints[0]));
Assert.True(payload.AsSpan().SequenceEqual(txoRead.Payloads.Slice(0, 8)));
}
private void Work()
{
var txoIn = TxoPack.Create();
var txoOut = TxoPack.Create();
// local array for capturing outpoints from inbox then sorting
var outpoints = new Outpoint[Constants.MaxOutpointsCount];
// array of inbox messages headers
var messageHeaders = new byte[Constants.MaxOutpointsCount][];
// array of inbox messages payloads
// var payloads = new byte[MaxTxoCount][]; // not used in read request
// provide index array for argsort
var argsort = new int[Constants.MaxOutpointsCount];
int txoCount = 0;
while (!_source.Token.IsCancellationRequested)
{
var message = _inbox.Peek();
if (message.Length > 0)
{
// parse current message and add txo to txopack
var outpoint =
Outpoint.CreateNaked(
message.Slice(ClientServerMessage.PayloadStart, Outpoint.KeySizeInBytes));
outpoints[txoCount] = outpoint;
messageHeaders[txoCount] = message.Slice(0, ClientServerMessage.PayloadStart).ToArray();
// payloads[messageCount] = message.Slice(ClientServerMessage.PayloadStart).ToArray();
// not used in read request
argsort[txoCount] = txoCount;
++txoCount;
Debug.Assert(txoCount <= Constants.MaxOutpointsCount,
"MaxTxoCount reached when looping inbox messages.");
_inbox.Next(message.Length);
}
else
{
// current round of inbox message picking is over
Array.Sort(outpoints, argsort, 0, txoCount);
// sort txopack
ref var writer = ref txoIn.GetResetWriter();
// identify identical outpoint, and write querying txopack
ref var previous = ref outpoints[argsort[0]];
writer.Write(in previous, ReadOnlySpan <byte> .Empty);
for (ushort index = 1; index < txoCount; ++index)
{
ref var current = ref outpoints[argsort[index]];
if (previous != current)
{
previous = current;
writer.Write(in previous, ReadOnlySpan <byte> .Empty);
}
}
/// <summary>
/// Deserialize a TxoPack from Sector.
/// </summary>
/// <param name="sector">Sector to deserialize</param>
/// <param name="pack">TxoPack to fill</param>
/// <param name="childSectorIndex">sectorIndex of child</param>
/// <remarks>
/// Life-time of the response lasts until the next call to 'Read'.
/// Size of TxoPack should be large enough to not overflow.
/// </remarks>
public static void Deserialize(this Sector sector, TxoPack pack, out SectorIndex childSectorIndex)
{
var sectorContent = sector.Contents;
childSectorIndex =
new SectorIndex(BitConverter.ToInt32(sectorContent.Slice(ChildSectorIndexStart, ChildSectorIndexSizeInBytes)));
var numberOfTxo = BitConverter.ToInt32(sectorContent.Slice(TxoCountStart, TxoCountSizeInBytes));
var payloadLengthAccumulator = 0;
ref var writer = ref pack.GetResetWriter();
public void Write(TxoPack txos, SectorIndex sectorIndex, Hash256 sectorHash)
{
if (_config.ImplicitSectors)
{
throw new InvalidOperationException();
}
var partition = PartitionByExplicitSector(sectorIndex, sectorHash, txos.GetForwardIterator());
Write(partition, txos);
}
public void Write(TxoPack txos)
{
if (!_config.ImplicitSectors)
{
throw new InvalidOperationException();
}
var partition = PartitionByImplicitSector(txos.GetForwardIterator());
Write(partition, txos);
}
/// <summary>
/// Reading layers with explicit outpoint ranges.
/// </summary>
/// <param name="txoIn">txopack providing outpoints requesting a read</param>
/// <param name="sectorIndex">Identifies the first sector to be read.</param>
/// <param name="sectorHash">Identifies the hash range of the sector.</param>
/// <param name="txoOut">output txoPack to collect read results</param>
private void Read(TxoPack txoIn, SectorIndex sectorIndex, Hash256 sectorHash, TxoPack txoOut)
{
if (_config.ImplicitSectors)
{
throw new InvalidOperationException();
}
var partition = PartitionByExplicitSector(sectorIndex, sectorHash, txoIn.GetForwardIterator());
Read(partition, txoOut);
}
public void Read(TxoPack txoIn, TxoPack txoOut)
{
if (!_config.ImplicitSectors)
{
throw new InvalidOperationException();
}
txoOut.Clear();
var partition = PartitionByImplicitSector(txoIn.GetForwardIterator());
Read(partition, txoOut);
}
private void Write(IEnumerable <SectorRange> partition, TxoPack txos)
{
// 'partition' is expected to be aligned with local sectors
foreach (var range in partition)
{
var childSectorIndex = SectorIndex.Unknown;
if (_store.TryRead(range.Index, out var sector))
{
sector.Deserialize(_deserialized, out childSectorIndex);
}
else
{
_deserialized.Clear(); // make empty TxoPack
}
_deserialized.AddOrMerge(txos.GetForwardIterator(), _merged, _lineage);
// extra TXOs fit locally
if (_merged.IsOverflowing(sector.Length))
{
if (_subTable == null)
{
// Normally everything should have been set up during initialization
throw new SozuTableOverflowException();
}
// TODO: [tandabany] WIP for the overflowing part
_merged.Split((int)(sector.Length * .5), _kept, _flowDown);
if (childSectorIndex.IsUnknown())
{
var deltaBitDepth = _subTable._config.SectorBitDepth - _config.SectorBitDepth;
childSectorIndex = _subTable._store.AllocateSectors(1 << deltaBitDepth);
}
sector.Serialize(_kept, childSectorIndex);
_store.Write(range.Index, sector);
_subTable.Write(_flowDown, childSectorIndex, range.LeftTxid);
}
else
{
sector.Serialize(_merged, childSectorIndex);
_store.Write(range.Index, sector);
}
}
}
public void AddWithoutMergeTest()
{
const int testSize = 5;
var txoPack = MakeRandomTxoPack(testSize);
var anotherTxoPack = MakeRandomTxoPack(testSize);
var resultTxoPack = TxoPack.Create(2);
txoPack.AddOrMerge(anotherTxoPack.GetForwardIterator(), resultTxoPack, null);
// ensure number of outpoints
Assert.True(testSize * 2 == resultTxoPack.Count, "Merged outpoints total number mismatch.");
// ensure that merged TxoPack has sorted keys;
var iter = resultTxoPack.GetForwardIterator();
ref var op = ref iter.Current;
public void WriteVolatile()
{
var sozuTable = PrepareTests(1, /*firstLayerVolatile*/ true);
var whatToWrite = new TxoPack(new Outpoint[1], new byte[2048]);
var firstHash = new TxId(new Hash256(0xF111111111111122UL, 0x22222222UL, 0x33333333UL, 0x44444444UL));
var wa = whatToWrite.GetResetWriter();
var outpoint = new Outpoint(firstHash, 0, 100);
var payload = new byte[100];
payload[0] = 42;
wa.Write(in outpoint, payload);
sozuTable.Write(whatToWrite);
var txoRead = new TxoPack(new Outpoint[1], new byte[2048]);
sozuTable.Read(whatToWrite, txoRead);
Assert.Equal(txoRead.Count, whatToWrite.Count);
Assert.True(txoRead.OutPoints[0].Equals(whatToWrite.OutPoints[0]));
Assert.True(txoRead.Payloads.SequenceEqual(whatToWrite.Payloads));
}
/// <param name="config"></param>
/// <param name="lineage"></param>
/// <param name="store"></param>
/// <param name="subTable"></param>
public SozuTable(SozuConfig config, ILineage lineage, ISectorStore store, SozuTable subTable = null)
{
_config = config;
_lineage = lineage;
_store = store;
_subTable = subTable;
_residue = TxoPack.Create();
_rangeOutputs = TxoPack.Create();
_deserialized = TxoPack.Create();
_complement = TxoPack.Create();
_kept = TxoPack.Create();
_flowDown = TxoPack.Create();
_merged = TxoPack.Create(2);
store.Initialize();
if (config.ImplicitSectors && !store.HasSectors())
{
store.AllocateSectors(1 << config.SectorBitDepth);
EnsurePersistence();
}
}
private void Read(IEnumerable <SectorRange> partition, TxoPack outputs)
{
foreach (var range in partition)
{
var sector = _store.Read(range.Index);
sector.Deserialize(_deserialized, out var childSectorIndex);
// We need not to overwrite txos, but put inside the payload found
_deserialized.Filter(range.Outpoints, _kept, _residue, _complement);
if (_residue.Count == 0 || childSectorIndex.IsUnknown() || _subTable == null)
{
_flowDown.Clear();
}
else
{
// Recursive exploration
_subTable.Read(_residue, childSectorIndex, range.LeftTxid, _flowDown);
}
_flowDown.AddOrMerge(_kept.GetForwardIterator(), outputs, null);
} // foreach (var range in partition)
}
public SectorRange(SectorIndex index, Hash256 leftTxid, TxoPack outpoints)
{
Index = index;
LeftTxid = leftTxid;
Outpoints = outpoints;
}
