public void Test_ReedSolomon8()
{
for (int count = 128 - 1; count >= 0; count--)
{
int blockLength = _random.Next(32, 1024);
ReedSolomon8 reedSolomon8 = new ReedSolomon8(128, 256, 2, _bufferManager);
var buffList = new ArraySegment <byte> [128];
for (int i = 0; i < 128; i++)
{
var buffer = new byte[blockLength];
_random.NextBytes(buffer);
buffList[i] = new ArraySegment <byte>(buffer, 0, buffer.Length);
}
var buffList2 = new ArraySegment <byte> [256];
for (int i = 0; i < 256; i++)
{
var buffer = new byte[blockLength];
buffList2[i] = new ArraySegment <byte>(buffer, 0, buffer.Length);
}
var intList = new int[256];
for (int i = 0; i < 256; i++)
{
intList[i] = i;
}
reedSolomon8.Encode(buffList, buffList2, intList, blockLength);
var buffList3 = new ArraySegment <byte> [128];
{
int i = 0;
for (int j = 0; i < 64; i++, j++)
{
buffList3[i] = buffList2[i];
}
for (int j = 0; i < 128; i++, j++)
{
buffList3[i] = buffList2[128 + j];
}
}
var intList2 = new int[128];
{
int i = 0;
for (int j = 0; i < 64; i++, j++)
{
intList2[i] = i;
}
for (int j = 0; i < 128; i++, j++)
{
intList2[i] = 128 + j;
}
}
{
int n = buffList3.Length;
while (n > 1)
{
int k = _random.Next(n--);
var temp = buffList3[n];
buffList3[n] = buffList3[k];
buffList3[k] = temp;
var temp2 = intList2[n];
intList2[n] = intList2[k];
intList2[k] = temp2;
}
}
reedSolomon8.Decode(buffList3, intList2, blockLength);
for (int i = 0; i < buffList.Length; i++)
{
Assert.IsTrue(CollectionUtilities.Equals(buffList[i].Array, buffList[i].Offset, buffList3[i].Array, buffList3[i].Offset, blockLength), "ReedSolomon");
}
}
{
ReedSolomon8 reedSolomon8 = new ReedSolomon8(128, 256, 4, _bufferManager);
var buffList = new ArraySegment <byte> [128];
for (int i = 0; i < 128; i++)
{
var buffer = new byte[1024 * 32];
_random.NextBytes(buffer);
buffList[i] = new ArraySegment <byte>(buffer, 0, buffer.Length);
}
var buffList2 = new ArraySegment <byte> [128];
for (int i = 0; i < 128; i++)
{
var buffer = new byte[1024 * 32];
buffList2[i] = new ArraySegment <byte>(buffer, 0, buffer.Length);
}
var intList = new int[128];
for (int i = 0; i < 128; i++)
{
intList[i] = i + 128;
}
{
Stopwatch sw = new Stopwatch();
sw.Start();
var task1 = Task.Factory.StartNew(() =>
{
reedSolomon8.Encode(buffList, buffList2, intList, 1024 * 32);
});
Thread.Sleep(1000 * 1);
var task2 = Task.Factory.StartNew(() =>
{
reedSolomon8.Cancel();
});
Task.WaitAll(task1, task2);
sw.Stop();
Assert.IsTrue(sw.Elapsed.TotalSeconds < 3);
}
}
{
ReedSolomon8 pc = new ReedSolomon8(128, 256, 4, _bufferManager);
var buffList = new ArraySegment <byte> [128];
for (int i = 0; i < 128; i++)
{
var buffer = new byte[1024 * 32];
_random.NextBytes(buffer);
buffList[i] = new ArraySegment <byte>(buffer, 0, buffer.Length);
}
var buffList2 = new ArraySegment <byte> [128];
for (int i = 0; i < 128; i++)
{
var buffer = new byte[1024 * 32];
buffList2[i] = new ArraySegment <byte>(buffer, 0, buffer.Length);
}
var intList = new int[128];
for (int i = 0; i < 128; i++)
{
intList[i] = i + 128;
}
pc.Encode(buffList, buffList2, intList, 1024 * 32);
{
Stopwatch sw = new Stopwatch();
sw.Start();
var task1 = Task.Factory.StartNew(() =>
{
pc.Decode(buffList2.ToArray(), intList, 1024 * 32);
});
Thread.Sleep(1000 * 1);
var task2 = Task.Factory.StartNew(() =>
{
pc.Cancel();
});
Task.WaitAll(task1, task2);
sw.Stop();
Assert.IsTrue(sw.Elapsed.TotalSeconds < 3);
}
}
}
private IEnumerable <Hash> ParityEncoding(IEnumerable <ArraySegment <byte> > buffers, HashAlgorithm hashAlgorithm, CorrectionAlgorithm correctionAlgorithm, CancellationToken token)
{
if (correctionAlgorithm == CorrectionAlgorithm.ReedSolomon8)
{
if (buffers.Count() > 128)
{
throw new ArgumentOutOfRangeException(nameof(buffers));
}
var createBuffers = new List <ArraySegment <byte> >();
try
{
var targetBuffers = new ArraySegment <byte> [buffers.Count()];
var parityBuffers = new ArraySegment <byte> [buffers.Count()];
int blockLength = buffers.Max(n => n.Count);
// Normalize
{
int index = 0;
foreach (var buffer in buffers)
{
token.ThrowIfCancellationRequested();
if (buffer.Count < blockLength)
{
var tempBuffer = new ArraySegment <byte>(_bufferManager.TakeBuffer(blockLength), 0, blockLength);
Unsafe.Copy(buffer.Array, buffer.Offset, tempBuffer.Array, tempBuffer.Offset, buffer.Count);
Unsafe.Zero(tempBuffer.Array, tempBuffer.Offset + buffer.Count, tempBuffer.Count - buffer.Count);
createBuffers.Add(tempBuffer);
targetBuffers[index] = tempBuffer;
}
else
{
targetBuffers[index] = buffer;
}
index++;
}
}
for (int i = 0; i < parityBuffers.Length; i++)
{
parityBuffers[i] = new ArraySegment <byte>(_bufferManager.TakeBuffer(blockLength), 0, blockLength);
}
var indexes = new int[parityBuffers.Length];
for (int i = 0; i < parityBuffers.Length; i++)
{
indexes[i] = targetBuffers.Length + i;
}
using (var reedSolomon = new ReedSolomon8(targetBuffers.Length, targetBuffers.Length + parityBuffers.Length, _threadCount, _bufferManager))
{
reedSolomon.Encode(targetBuffers, parityBuffers, indexes, blockLength, token).Wait();
}
token.ThrowIfCancellationRequested();
var parityHashes = new List <Hash>();
for (int i = 0; i < parityBuffers.Length; i++)
{
Hash hash;
if (hashAlgorithm == HashAlgorithm.Sha256)
{
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(parityBuffers[i]));
}
else
{
throw new NotSupportedException();
}
_blocksManager.Lock(hash);
_blocksManager.Set(hash, parityBuffers[i]);
parityHashes.Add(hash);
}
return(parityHashes);
}
finally
{
foreach (var buffer in createBuffers)
{
if (buffer.Array == null)
{
continue;
}
_bufferManager.ReturnBuffer(buffer.Array);
}
}
}
else
{
throw new NotSupportedException();
}
}
public Task <IEnumerable <Hash> > ParityDecoding(Group group, CancellationToken token)
{
return(Task.Run <IEnumerable <Hash> >(() =>
{
if (group.CorrectionAlgorithm == CorrectionAlgorithm.ReedSolomon8)
{
int blockLength = group.Hashes.Max(n => this.GetLength(n));
int informationCount = group.Hashes.Count / 2;
if (group.Hashes.Take(informationCount).All(n => this.Contains(n)))
{
return group.Hashes.Take(informationCount).ToList();
}
var buffers = new ArraySegment <byte> [informationCount];
var indexes = new int[informationCount];
try
{
// Load
{
int count = 0;
for (int i = 0; i < group.Hashes.Count; i++)
{
token.ThrowIfCancellationRequested();
if (!this.Contains(group.Hashes[i]))
{
continue;
}
var buffer = new ArraySegment <byte>();
try
{
buffer = this.GetBlock(group.Hashes[i]);
if (buffer.Count < blockLength)
{
var tempBuffer = new ArraySegment <byte>(_bufferManager.TakeBuffer(blockLength), 0, blockLength);
Unsafe.Copy(buffer.Array, buffer.Offset, tempBuffer.Array, tempBuffer.Offset, buffer.Count);
Unsafe.Zero(tempBuffer.Array, tempBuffer.Offset + buffer.Count, tempBuffer.Count - buffer.Count);
_bufferManager.ReturnBuffer(buffer.Array);
buffer = tempBuffer;
}
}
catch (Exception)
{
if (buffer.Array != null)
{
_bufferManager.ReturnBuffer(buffer.Array);
}
throw;
}
indexes[count] = i;
buffers[count] = buffer;
count++;
if (count >= informationCount)
{
break;
}
}
if (count < informationCount)
{
throw new BlockNotFoundException();
}
}
using (var reedSolomon = new ReedSolomon8(informationCount, informationCount * 2, _threadCount, _bufferManager))
{
reedSolomon.Decode(buffers, indexes, blockLength, token).Wait();
}
// Set
{
long length = group.Length;
for (int i = 0; i < informationCount; length -= blockLength, i++)
{
_blocksManager.Set(group.Hashes[i], new ArraySegment <byte>(buffers[i].Array, buffers[i].Offset, (int)Math.Min(buffers[i].Count, length)));
}
}
}
finally
{
foreach (var buffer in buffers)
{
if (buffer.Array == null)
{
continue;
}
_bufferManager.ReturnBuffer(buffer.Array);
}
}
return group.Hashes.Take(informationCount).ToList();
}
else
{
throw new NotSupportedException();
}
}));
}
public async ValueTask <OmniHash[]> ParityDecode(MerkleTreeSection merkleTreeSection, CancellationToken token = default)
{
return(await Task.Run(async() =>
{
if (merkleTreeSection.CorrectionAlgorithmType == CorrectionAlgorithmType.ReedSolomon8)
{
// 実ブロック数
int informationCount = merkleTreeSection.Hashes.Count - 128;
// デコード可能かチェックする
if (merkleTreeSection.Hashes.Count(n => this.Contains(n)) < informationCount)
{
throw new ParityDecodeFailed();
}
// デコードする必要がない場合
if (merkleTreeSection.Hashes.Take(informationCount).All(n => this.Contains(n)))
{
return merkleTreeSection.Hashes.Take(informationCount).ToArray();
}
// 最大ブロック長
uint blockLength = merkleTreeSection.Hashes.Max(n => this.GetLength(n));
var memoryOwners = new List <IMemoryOwner <byte> >();
try
{
var targetBuffers = new Memory <byte> [128];
var indexes = new int[128];
// Load
{
int count = 0;
int index = 0;
int position = 0;
for (; index < informationCount && count < informationCount; index++, position++)
{
token.ThrowIfCancellationRequested();
if (!this.TryGetBlock(merkleTreeSection.Hashes[position], out var blockMemoryOwner))
{
continue;
}
// 実ブロックの長さが足りない場合、0byteでpaddingを行う
if (blockMemoryOwner.Memory.Length < blockLength)
{
var tempMemoryOwner = _bufferPool.Rent((int)blockLength);
BytesOperations.Copy(blockMemoryOwner.Memory.Span, tempMemoryOwner.Memory.Span, blockMemoryOwner.Memory.Length);
BytesOperations.Zero(tempMemoryOwner.Memory.Span.Slice(blockMemoryOwner.Memory.Length));
blockMemoryOwner.Dispose();
blockMemoryOwner = tempMemoryOwner;
}
memoryOwners.Add(blockMemoryOwner);
indexes[count] = index;
targetBuffers[count] = blockMemoryOwner.Memory;
count++;
}
// 実ブロック数が128に満たない場合、0byte配列でPaddingを行う。
for (; index < 128 && count < informationCount; index++)
{
token.ThrowIfCancellationRequested();
var tempMemoryOwner = _bufferPool.Rent((int)blockLength);
BytesOperations.Zero(tempMemoryOwner.Memory.Span);
memoryOwners.Add(tempMemoryOwner);
indexes[count] = index;
targetBuffers[count] = tempMemoryOwner.Memory;
count++;
}
for (; index < 256 && count < informationCount; index++, position++)
{
token.ThrowIfCancellationRequested();
if (!this.TryGetBlock(merkleTreeSection.Hashes[position], out var blockMemoryOwner))
{
continue;
}
memoryOwners.Add(blockMemoryOwner);
indexes[count] = index;
targetBuffers[count] = blockMemoryOwner.Memory;
count++;
}
if (count < informationCount)
{
throw new BlockNotFoundException();
}
}
var reedSolomon = new ReedSolomon8(informationCount, _threadCount, _bufferPool);
await reedSolomon.Decode(targetBuffers, indexes, (int)blockLength, informationCount * 2, token);
// Set
{
ulong length = merkleTreeSection.Length;
for (int i = 0; i < informationCount; length -= blockLength, i++)
{
bool result = _blockStorage.TrySet(merkleTreeSection.Hashes[i], memoryOwners[i].Memory.Span.Slice(0, (int)Math.Min(length, blockLength)));
if (!result)
{
throw new ParityDecodeFailed("Failed to save Block.");
}
}
}
}
finally
{
foreach (var memoryOwner in memoryOwners)
{
memoryOwner.Dispose();
}
}
return merkleTreeSection.Hashes.Take(informationCount).ToArray();
}
else
{
throw new NotSupportedException();
}
}));
}
private async ValueTask <OmniHash[]> ParityEncode(IEnumerable <Memory <byte> > blocks, OmniHashAlgorithmType hashAlgorithmType, CorrectionAlgorithmType correctionAlgorithmType, CancellationToken token = default)
{
if (correctionAlgorithmType == CorrectionAlgorithmType.ReedSolomon8)
{
var blockList = blocks.ToList();
// blocksの要素数をチェックする
if (blockList.Count <= 0 || blockList.Count > 128)
{
throw new ArgumentOutOfRangeException(nameof(blocks));
}
// 最大ブロック長
int blockLength = blockList[0].Length;
// 各ブロックの長さをチェックする
for (int i = 1; i < blockList.Count; i++)
{
// 末尾以外
if (i < (blockList.Count - 1))
{
// 先頭ブロックと同一の長さでなければならない
if (!(blockList[i].Length == blockLength))
{
throw new ArgumentOutOfRangeException(nameof(blocks), $"{nameof(blocks)}[{i}].Length");
}
}
// 末尾
else
{
// 先頭ブロックと同一かそれ以下の長さでなければならない
if (!(blockList[i].Length <= blockLength))
{
throw new ArgumentOutOfRangeException(nameof(blocks), $"{nameof(blocks)}[{i}].Length");
}
}
}
return(await Task.Run(async() =>
{
var memoryOwners = new List <IMemoryOwner <byte> >();
try
{
var targetBuffers = new ReadOnlyMemory <byte> [128];
var parityBuffers = new Memory <byte> [128];
// Load
{
int index = 0;
foreach (var buffer in blocks)
{
token.ThrowIfCancellationRequested();
// 実ブロックの長さが足りない場合、0byteでpaddingを行う
if (buffer.Length < blockLength)
{
var tempMemoryOwner = _bufferPool.Rent(blockLength);
BytesOperations.Copy(buffer.Span, tempMemoryOwner.Memory.Span, buffer.Length);
BytesOperations.Zero(tempMemoryOwner.Memory.Span.Slice(buffer.Length));
memoryOwners.Add(tempMemoryOwner);
targetBuffers[index++] = tempMemoryOwner.Memory;
}
else
{
targetBuffers[index++] = buffer;
}
}
// 実ブロック数が128に満たない場合、0byte配列でPaddingを行う。
for (int i = (128 - blocks.Count()) - 1; i >= 0; i--)
{
var tempMemoryOwner = _bufferPool.Rent(blockLength);
BytesOperations.Zero(tempMemoryOwner.Memory.Span);
memoryOwners.Add(tempMemoryOwner);
targetBuffers[index++] = tempMemoryOwner.Memory;
}
}
for (int i = 0; i < parityBuffers.Length; i++)
{
var tempMemoryOwner = _bufferPool.Rent(blockLength);
BytesOperations.Zero(tempMemoryOwner.Memory.Span);
memoryOwners.Add(tempMemoryOwner);
parityBuffers[i] = tempMemoryOwner.Memory;
}
var indexes = new int[parityBuffers.Length];
for (int i = 0; i < parityBuffers.Length; i++)
{
indexes[i] = targetBuffers.Length + i;
}
var reedSolomon = new ReedSolomon8(targetBuffers.Length, targetBuffers.Length + parityBuffers.Length, _bufferPool);
await reedSolomon.Encode(targetBuffers, indexes, parityBuffers, blockLength, _threadCount, token);
token.ThrowIfCancellationRequested();
var parityHashes = new List <OmniHash>();
for (int i = 0; i < parityBuffers.Length; i++)
{
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(parityBuffers[i].Span));
}
else
{
throw new NotSupportedException();
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, parityBuffers[i].Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
parityHashes.Add(hash);
}
return parityHashes.ToArray();
}
finally
{
foreach (var memoryOwner in memoryOwners)
{
memoryOwner.Dispose();
}
}
}));
}
else
{
throw new NotSupportedException();
}
}
public void ReedSolomon8()
{
Random random = new Random();
const int blockSize = 1024 * 1024;
for (int c = 1; c <= 4; c++)
{
ReedSolomon8 pc = new ReedSolomon8(128, 256, c, _bufferManager);
ArraySegment <byte>[] buffList = new ArraySegment <byte> [128];
for (int i = 0; i < 128; i++)
{
var buffer = _bufferManager.TakeBuffer(blockSize);
random.NextBytes(buffer);
buffList[i] = new ArraySegment <byte>(buffer);
}
ArraySegment <byte>[] buffList2 = new ArraySegment <byte> [128];
for (int i = 0; i < 128; i++)
{
var buffer = _bufferManager.TakeBuffer(blockSize);
buffList2[i] = new ArraySegment <byte>(buffer);
}
int[] intList = new int[128];
for (int i = 0; i < 128; i++)
{
intList[i] = i + 128;
}
Stopwatch sw = new Stopwatch();
sw.Start();
pc.Encode(buffList, buffList2, intList, blockSize);
pc.Decode(buffList2, intList, blockSize);
sw.Stop();
for (int i = 0; i < buffList.Length; i++)
{
Assert.IsTrue(CollectionUtilities.Equals(buffList[i].Array, buffList[i].Offset, buffList2[i].Array, buffList2[i].Offset, blockSize), "ReedSolomon");
}
for (int i = 0; i < 128; i++)
{
_bufferManager.ReturnBuffer(buffList[i].Array);
}
for (int i = 0; i < 128; i++)
{
_bufferManager.ReturnBuffer(buffList2[i].Array);
}
Console.WriteLine(string.Format("ReedSolomon8 Parallel {0}: ", c) + sw.Elapsed.ToString());
}
Console.Write(Environment.NewLine);
}
