public static OmniHash Create(OmniHashAlgorithmType algorithmType, ReadOnlySpan <byte> message)
{
return(algorithmType switch
{
OmniHashAlgorithmType.Sha2_256 => new OmniHash(algorithmType, Sha2_256.ComputeHash(message)),
_ => throw new NotSupportedException(),
});
public OmniHash Deserialize(Omnix.Serialization.RocketPack.RocketPackReader r, int rank)
{
if (rank > 256)
{
throw new System.FormatException();
}
OmniHashAlgorithmType p_algorithmType = (OmniHashAlgorithmType)0;
System.ReadOnlyMemory <byte> p_value = System.ReadOnlyMemory <byte> .Empty;
{
p_algorithmType = (OmniHashAlgorithmType)r.GetUInt64();
}
{
p_value = r.GetMemory(256);
}
return(new OmniHash(p_algorithmType, p_value));
}
public OmniHash(OmniHashAlgorithmType algorithmType, System.ReadOnlyMemory <byte> value)
{
if (value.Length > 256)
{
throw new System.ArgumentOutOfRangeException("value");
}
this.AlgorithmType = algorithmType;
this.Value = value;
{
var __h = new System.HashCode();
if (this.AlgorithmType != default)
{
__h.Add(this.AlgorithmType.GetHashCode());
}
if (!this.Value.IsEmpty)
{
__h.Add(Omnix.Base.Helpers.ObjectHelper.GetHashCode(this.Value.Span));
}
__hashCode = __h.ToHashCode();
}
}
private static OmniHash CreateOmniHash(string name, ReadOnlySpan <byte> publicKey, OmniHashAlgorithmType hashAlgorithmType)
{
if (name == null)
{
throw new ArgumentNullException(nameof(name));
}
using (var hub = new Hub())
{
{
var writer = new RocketPackWriter(hub.Writer, BufferPool.Shared);
writer.Write(name);
writer.Write(publicKey);
hub.Writer.Complete();
}
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
var result = new OmniHash(hashAlgorithmType, Sha2_256.ComputeHash(hub.Reader.GetSequence()));
hub.Reader.Complete();
return(result);
}
else
{
throw new NotSupportedException();
}
}
}
public async ValueTask <XeusClue> Import(string path, CancellationToken token = default)
{
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
return(await Task.Run(async() =>
{
// Check
lock (_lockObject)
{
var info = _contentMetadataStorage.GetFileContentMetadata(path);
if (info != null)
{
return info.Clue;
}
}
XeusClue?clue = null;
var lockedHashes = new HashSet <OmniHash>();
SharedBlocksMetadata?sharedBlocksInfo = null;
{
const int blockLength = 1024 * 1024;
const OmniHashAlgorithmType hashAlgorithmType = OmniHashAlgorithmType.Sha2_256;
const CorrectionAlgorithmType correctionAlgorithmType = CorrectionAlgorithmType.ReedSolomon8;
byte depth = 0;
var merkleTreeSectionList = new List <MerkleTreeSection>();
// File
using (var stream = new UnbufferedFileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferPool))
{
if (stream.Length <= blockLength)
{
OmniHash hash;
using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
{
stream.Read(bufferMemoryOwner.Memory.Span);
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
}
sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, (uint)stream.Length, new OmniHash[] { hash });
clue = new XeusClue(hash, depth);
}
else
{
var sharedHashes = new List <OmniHash>();
for (; ;)
{
var targetHashes = new List <OmniHash>();
var targetMemoryOwners = new List <IMemoryOwner <byte> >();
ulong sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
var bufferMemoryOwner = _bufferPool.Rent((int)length);
try
{
stream.Read(bufferMemoryOwner.Memory.Span);
sumLength += length;
}
catch (Exception e)
{
bufferMemoryOwner.Dispose();
throw e;
}
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
sharedHashes.Add(hash);
targetHashes.Add(hash);
targetMemoryOwners.Add(bufferMemoryOwner);
if (targetMemoryOwners.Count >= 128)
{
break;
}
}
var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
lockedHashes.UnionWith(parityHashes);
merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var memoryOwner in targetMemoryOwners)
{
memoryOwner.Dispose();
}
}
if (stream.Position == stream.Length)
{
break;
}
}
sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, blockLength, sharedHashes.ToArray());
depth++;
}
}
while (merkleTreeSectionList.Count > 0)
{
// Index
using (var stream = new RecyclableMemoryStream(_bufferPool))
{
RocketPackHelper.MessageToStream(new MerkleTreeNode(merkleTreeSectionList.ToArray()), stream);
stream.Seek(0, SeekOrigin.Begin);
merkleTreeSectionList.Clear();
if (stream.Length <= blockLength)
{
OmniHash hash;
using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
{
stream.Read(bufferMemoryOwner.Memory.Span);
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
lockedHashes.Add(hash);
}
clue = new XeusClue(hash, depth);
}
else
{
for (; ;)
{
var targetHashes = new List <OmniHash>();
var targetMemoryOwners = new List <IMemoryOwner <byte> >();
ulong sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
var bufferMemoryOwner = _bufferPool.Rent((int)length);
try
{
stream.Read(bufferMemoryOwner.Memory.Span);
sumLength += length;
}
catch (Exception e)
{
bufferMemoryOwner.Dispose();
throw e;
}
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
lockedHashes.Add(hash);
targetHashes.Add(hash);
targetMemoryOwners.Add(bufferMemoryOwner);
if (targetMemoryOwners.Count >= 128)
{
break;
}
}
var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
lockedHashes.UnionWith(parityHashes);
merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var memoryOwner in targetMemoryOwners)
{
memoryOwner.Dispose();
}
}
if (stream.Position == stream.Length)
{
break;
}
}
depth++;
}
}
}
}
if (clue == null)
{
throw new ImportFailed("clue is null");
}
lock (_lockObject)
{
if (!_contentMetadataStorage.ContainsFileContentMetadata(path))
{
_contentMetadataStorage.Add(new ContentMetadata(clue, lockedHashes.ToArray(), sharedBlocksInfo));
foreach (var hash in lockedHashes)
{
_blockStorage.Lock(hash);
}
}
}
return clue;
}, token));
}
private async ValueTask <OmniHash[]> ParityEncode(IEnumerable <Memory <byte> > blocks, OmniHashAlgorithmType hashAlgorithmType, CorrectionAlgorithmType correctionAlgorithmType, CancellationToken token = default)
{
if (correctionAlgorithmType == CorrectionAlgorithmType.ReedSolomon8)
{
var blockList = blocks.ToList();
// blocksの要素数をチェックする
if (blockList.Count <= 0 || blockList.Count > 128)
{
throw new ArgumentOutOfRangeException(nameof(blocks));
}
// 最大ブロック長
int blockLength = blockList[0].Length;
// 各ブロックの長さをチェックする
for (int i = 1; i < blockList.Count; i++)
{
// 末尾以外
if (i < (blockList.Count - 1))
{
// 先頭ブロックと同一の長さでなければならない
if (!(blockList[i].Length == blockLength))
{
throw new ArgumentOutOfRangeException(nameof(blocks), $"{nameof(blocks)}[{i}].Length");
}
}
// 末尾
else
{
// 先頭ブロックと同一かそれ以下の長さでなければならない
if (!(blockList[i].Length <= blockLength))
{
throw new ArgumentOutOfRangeException(nameof(blocks), $"{nameof(blocks)}[{i}].Length");
}
}
}
return(await Task.Run(async() =>
{
var memoryOwners = new List <IMemoryOwner <byte> >();
try
{
var targetBuffers = new ReadOnlyMemory <byte> [128];
var parityBuffers = new Memory <byte> [128];
// Load
{
int index = 0;
foreach (var buffer in blocks)
{
token.ThrowIfCancellationRequested();
// 実ブロックの長さが足りない場合、0byteでpaddingを行う
if (buffer.Length < blockLength)
{
var tempMemoryOwner = _bufferPool.Rent(blockLength);
BytesOperations.Copy(buffer.Span, tempMemoryOwner.Memory.Span, buffer.Length);
BytesOperations.Zero(tempMemoryOwner.Memory.Span.Slice(buffer.Length));
memoryOwners.Add(tempMemoryOwner);
targetBuffers[index++] = tempMemoryOwner.Memory;
}
else
{
targetBuffers[index++] = buffer;
}
}
// 実ブロック数が128に満たない場合、0byte配列でPaddingを行う。
for (int i = (128 - blocks.Count()) - 1; i >= 0; i--)
{
var tempMemoryOwner = _bufferPool.Rent(blockLength);
BytesOperations.Zero(tempMemoryOwner.Memory.Span);
memoryOwners.Add(tempMemoryOwner);
targetBuffers[index++] = tempMemoryOwner.Memory;
}
}
for (int i = 0; i < parityBuffers.Length; i++)
{
var tempMemoryOwner = _bufferPool.Rent(blockLength);
BytesOperations.Zero(tempMemoryOwner.Memory.Span);
memoryOwners.Add(tempMemoryOwner);
parityBuffers[i] = tempMemoryOwner.Memory;
}
var indexes = new int[parityBuffers.Length];
for (int i = 0; i < parityBuffers.Length; i++)
{
indexes[i] = targetBuffers.Length + i;
}
var reedSolomon = new ReedSolomon8(targetBuffers.Length, targetBuffers.Length + parityBuffers.Length, _bufferPool);
await reedSolomon.Encode(targetBuffers, indexes, parityBuffers, blockLength, _threadCount, token);
token.ThrowIfCancellationRequested();
var parityHashes = new List <OmniHash>();
for (int i = 0; i < parityBuffers.Length; i++)
{
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(parityBuffers[i].Span));
}
else
{
throw new NotSupportedException();
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, parityBuffers[i].Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
parityHashes.Add(hash);
}
return parityHashes.ToArray();
}
finally
{
foreach (var memoryOwner in memoryOwners)
{
memoryOwner.Dispose();
}
}
}));
}
else
{
throw new NotSupportedException();
}
}
private static OmniHash CreateOmniHash(string name, ReadOnlySpan <byte> publicKey, OmniHashAlgorithmType hashAlgorithmType)
{
if (name == null)
{
throw new ArgumentNullException(nameof(name));
}
using var bytesPipe = new BytesPipe();
{
var writer = new RocketMessageWriter(bytesPipe.Writer, BytesPool.Shared);
writer.Write(name);
writer.Write(publicKey);
}
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
var result = new OmniHash(hashAlgorithmType, Sha2_256.ComputeHash(bytesPipe.Reader.GetSequence()));
return(result);
}
else
{
throw new NotSupportedException();
}
}
