public static OmniHash Create(OmniHashAlgorithmType algorithmType, ReadOnlySpan <byte> message)
{
return(algorithmType switch
{
OmniHashAlgorithmType.Sha2_256 => new OmniHash(algorithmType, Sha2_256.ComputeHash(message)),
_ => throw new NotSupportedException(),
});
private IEnumerable <(ReadOnlyMemory <byte>, string)> GetHashesV1(V1.Internal.ProfileMessage profileMessage, OmniAgreementPublicKey agreementPublicKey, V1.Internal.HashAlgorithm hashAlgorithm)
{
var results = new Dictionary <ReadOnlyMemory <byte>, string>();
byte[] verificationMessageHash;
{
var verificationMessage = new V1.Internal.VerificationMessage(profileMessage, agreementPublicKey);
if (hashAlgorithm == V1.Internal.HashAlgorithm.Sha2_256)
{
var hub = new Hub();
verificationMessage.Export(hub.Writer, _bufferPool);
verificationMessageHash = Sha2_256.ComputeHash(hub.Reader.GetSequence());
}
else
{
throw new NotSupportedException(nameof(hashAlgorithm));
}
}
foreach (var password in _passwords)
{
if (hashAlgorithm.HasFlag(V1.Internal.HashAlgorithm.Sha2_256))
{
results.Add(Hmac_Sha2_256.ComputeHash(verificationMessageHash, Sha2_256.ComputeHash(password)), password);
}
}
return(results.Select(item => (item.Key, item.Value)));
}
private static OmniHash CreateOmniHash(string name, ReadOnlySpan <byte> publicKey, OmniHashAlgorithmType hashAlgorithmType)
{
if (name == null)
{
throw new ArgumentNullException(nameof(name));
}
using (var hub = new Hub())
{
{
var writer = new RocketPackWriter(hub.Writer, BufferPool.Shared);
writer.Write(name);
writer.Write(publicKey);
hub.Writer.Complete();
}
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
var result = new OmniHash(hashAlgorithmType, Sha2_256.ComputeHash(hub.Reader.GetSequence()));
hub.Reader.Complete();
return(result);
}
else
{
throw new NotSupportedException();
}
}
}
private static OmniHash CreateOmniHash(string name, ReadOnlySpan <byte> publicKey, OmniHashAlgorithmType hashAlgorithmType)
{
if (name == null)
{
throw new ArgumentNullException(nameof(name));
}
using var bytesPipe = new BytesPipe();
{
var writer = new RocketMessageWriter(bytesPipe.Writer, BytesPool.Shared);
writer.Write(name);
writer.Write(publicKey);
}
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
var result = new OmniHash(hashAlgorithmType, Sha2_256.ComputeHash(bytesPipe.Reader.GetSequence()));
return(result);
}
else
{
throw new NotSupportedException();
}
}
public static string SignatureToString(OmniSignature signature)
{
using var bytesPipe = new BytesPipe(BytesPool.Shared);
signature.Export(bytesPipe.Writer, BytesPool.Shared);
var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bytesPipe.Reader.GetSequence()));
return(hash.ToString(ConvertStringType.Base16));
}
private byte[] ComputeHash(ProfileMessage profileMessage, OmniAgreementPublicKey agreementPublicKey, HashAlgorithmType hashAlgorithm)
{
var verificationMessage = new VerificationMessage(profileMessage, agreementPublicKey);
if (hashAlgorithm == HashAlgorithmType.Sha2_256)
{
using var bytesPipe = new BytesPipe();
verificationMessage.Export(bytesPipe.Writer, _bytesPool);
return(Sha2_256.ComputeHash(bytesPipe.Reader.GetSequence()));
}
throw new NotSupportedException(nameof(hashAlgorithm));
}
public static uint Verify_Simple_Sha2_256(ReadOnlySpan <byte> key, ReadOnlySpan <byte> value)
{
if (key == null)
{
throw new ArgumentNullException(nameof(key));
}
if (key.Length != 32)
{
throw new ArgumentOutOfRangeException(nameof(key));
}
if (value == null)
{
throw new ArgumentNullException(nameof(value));
}
if (value.Length != 32)
{
throw new ArgumentOutOfRangeException(nameof(value));
}
Span <byte> buffer = stackalloc byte[64];
byte[] hash;
{
BytesOperations.Copy(key, buffer, key.Length);
BytesOperations.Copy(value, buffer.Slice(key.Length), value.Length);
hash = Sha2_256.ComputeHash(buffer);
}
uint count = 0;
for (int i = 0; i < 32; i++)
{
for (int j = 0; j < 8; j++)
{
if (((hash[i] << j) & 0x80) == 0)
{
count++;
}
else
{
goto End;
}
}
}
End:
return(count);
}
public static bool TryComputeHash(ReadOnlySequence <byte> sequence, ReadOnlySpan <byte> key, Span <byte> destination)
{
if (destination.Length < 32)
{
throw new ArgumentOutOfRangeException(nameof(destination));
}
Span <byte> extendedKey = stackalloc byte[_blockLength];
if (key.Length > _blockLength)
{
Sha2_256.TryComputeHash(key, extendedKey);
}
else
{
BytesOperations.Copy(key, extendedKey, Math.Min(key.Length, extendedKey.Length));
}
Span <byte> ixor = stackalloc byte[_blockLength];
BytesOperations.Xor(_ipad, extendedKey, ixor);
Span <byte> oxor = stackalloc byte[_blockLength];
BytesOperations.Xor(_opad, extendedKey, oxor);
Span <byte> ihash = stackalloc byte[32];
using (var incrementalHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256))
{
incrementalHash.AppendData(ixor);
foreach (var segment in sequence)
{
incrementalHash.AppendData(segment.Span);
}
incrementalHash.TryGetHashAndReset(ihash, out _);
}
using (var incrementalHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256))
{
incrementalHash.AppendData(oxor);
incrementalHash.AppendData(ihash);
return(incrementalHash.TryGetHashAndReset(destination, out _));
}
}
public bool Equals(BlobFingerprint other)
{
if (ReferenceEquals(this, other))
{
return(true);
}
if (ReferenceEquals(null, other))
{
return(false);
}
return(Size == other.Size &&
Sha3_512.SequenceEqual(other.Sha3_512) &&
Sha2_256.SequenceEqual(other.Sha2_256) &&
Md5.SequenceEqual(other.Md5));
}
public static bool Verify(ReadOnlyMemory <byte> publicKey, ReadOnlyMemory <byte> signature, ReadOnlySequence <byte> sequence)
{
ECParameters ecParameters;
{
var plist = SerializeHelper.Decode(publicKey).ToArray();
ecParameters = new ECParameters()
{
Q = new ECPoint()
{
X = plist[0].ToArray(),
Y = plist[1].ToArray(),
},
};
}
using (var ecdsa = ECDsa.Create())
{
ecdsa.ImportParameters(ecParameters);
return(ecdsa.VerifyHash(Sha2_256.ComputeHash(sequence).AsSpan(), signature.Span));
}
}
public async Task Test()
{
var random = new Random();
using var blockStorage = new BlockStorage(UnitTestEnvironment.TempDirectoryPath, BufferPool.Shared);
var sizeList = new List <int>(new[] { 0, 1, 10, 100, 1000, 10000 });
for (int i = 0; i < 32; i++)
{
sizeList.Add(random.Next(0, 10));
sizeList.Add(random.Next(0, 256));
sizeList.Add(random.Next(0, 1024 * 32));
}
await blockStorage.LoadAsync();
blockStorage.Resize(1024 * 1024 * 1024);
var hashList = new List <OmniHash>();
foreach (var size in sizeList)
{
using var memoryOwner = BufferPool.Shared.Rent(size);
random.NextBytes(memoryOwner.Memory.Span);
var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(memoryOwner.Memory.Span));
Assert.True(blockStorage.TrySet(hash, memoryOwner.Memory.Span));
hashList.Add(hash);
}
foreach (var hash in hashList)
{
Assert.True(blockStorage.TryGet(hash, out var memoryOwner));
Assert.True(BytesOperations.SequenceEqual(hash.Value.Span, Sha2_256.ComputeHash(memoryOwner.Memory.Span)));
}
}
public static byte[] Sign(ReadOnlyMemory <byte> privateKey, ReadOnlySequence <byte> sequence)
{
ECParameters ecParameters;
{
var plist = SerializeHelper.Decode(privateKey).ToArray();
ecParameters = new ECParameters()
{
Q = new ECPoint()
{
X = plist[0].ToArray(),
Y = plist[1].ToArray(),
},
D = plist[2].ToArray(),
};
}
using (var ecdsa = ECDsa.Create())
{
ecdsa.ImportParameters(ecParameters);
return(ecdsa.SignHash(Sha2_256.ComputeHash(sequence)));
}
}
private async ValueTask <OmniHash[]> ParityEncode(IEnumerable <Memory <byte> > blocks, OmniHashAlgorithmType hashAlgorithmType, CorrectionAlgorithmType correctionAlgorithmType, CancellationToken token = default)
{
if (correctionAlgorithmType == CorrectionAlgorithmType.ReedSolomon8)
{
var blockList = blocks.ToList();
// blocksの要素数をチェックする
if (blockList.Count <= 0 || blockList.Count > 128)
{
throw new ArgumentOutOfRangeException(nameof(blocks));
}
// 最大ブロック長
int blockLength = blockList[0].Length;
// 各ブロックの長さをチェックする
for (int i = 1; i < blockList.Count; i++)
{
// 末尾以外
if (i < (blockList.Count - 1))
{
// 先頭ブロックと同一の長さでなければならない
if (!(blockList[i].Length == blockLength))
{
throw new ArgumentOutOfRangeException(nameof(blocks), $"{nameof(blocks)}[{i}].Length");
}
}
// 末尾
else
{
// 先頭ブロックと同一かそれ以下の長さでなければならない
if (!(blockList[i].Length <= blockLength))
{
throw new ArgumentOutOfRangeException(nameof(blocks), $"{nameof(blocks)}[{i}].Length");
}
}
}
return(await Task.Run(async() =>
{
var memoryOwners = new List <IMemoryOwner <byte> >();
try
{
var targetBuffers = new ReadOnlyMemory <byte> [128];
var parityBuffers = new Memory <byte> [128];
// Load
{
int index = 0;
foreach (var buffer in blocks)
{
token.ThrowIfCancellationRequested();
// 実ブロックの長さが足りない場合、0byteでpaddingを行う
if (buffer.Length < blockLength)
{
var tempMemoryOwner = _bufferPool.Rent(blockLength);
BytesOperations.Copy(buffer.Span, tempMemoryOwner.Memory.Span, buffer.Length);
BytesOperations.Zero(tempMemoryOwner.Memory.Span.Slice(buffer.Length));
memoryOwners.Add(tempMemoryOwner);
targetBuffers[index++] = tempMemoryOwner.Memory;
}
else
{
targetBuffers[index++] = buffer;
}
}
// 実ブロック数が128に満たない場合、0byte配列でPaddingを行う。
for (int i = (128 - blocks.Count()) - 1; i >= 0; i--)
{
var tempMemoryOwner = _bufferPool.Rent(blockLength);
BytesOperations.Zero(tempMemoryOwner.Memory.Span);
memoryOwners.Add(tempMemoryOwner);
targetBuffers[index++] = tempMemoryOwner.Memory;
}
}
for (int i = 0; i < parityBuffers.Length; i++)
{
var tempMemoryOwner = _bufferPool.Rent(blockLength);
BytesOperations.Zero(tempMemoryOwner.Memory.Span);
memoryOwners.Add(tempMemoryOwner);
parityBuffers[i] = tempMemoryOwner.Memory;
}
var indexes = new int[parityBuffers.Length];
for (int i = 0; i < parityBuffers.Length; i++)
{
indexes[i] = targetBuffers.Length + i;
}
var reedSolomon = new ReedSolomon8(targetBuffers.Length, targetBuffers.Length + parityBuffers.Length, _bufferPool);
await reedSolomon.Encode(targetBuffers, indexes, parityBuffers, blockLength, _threadCount, token);
token.ThrowIfCancellationRequested();
var parityHashes = new List <OmniHash>();
for (int i = 0; i < parityBuffers.Length; i++)
{
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(parityBuffers[i].Span));
}
else
{
throw new NotSupportedException();
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, parityBuffers[i].Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
parityHashes.Add(hash);
}
return parityHashes.ToArray();
}
finally
{
foreach (var memoryOwner in memoryOwners)
{
memoryOwner.Dispose();
}
}
}));
}
else
{
throw new NotSupportedException();
}
}
public static bool Verify(ReadOnlyMemory <byte> publicKey, ReadOnlyMemory <byte> signature, ReadOnlySequence <byte> sequence)
{
using var ecdsa = ECDsa.Create();
ecdsa.ImportSubjectPublicKeyInfo(publicKey.Span, out var _);
return(ecdsa.VerifyHash(Sha2_256.ComputeHash(sequence).AsSpan(), signature.Span));
}
public static byte[] Sign(ReadOnlyMemory <byte> privateKey, ReadOnlySequence <byte> sequence)
{
using var ecdsa = ECDsa.Create();
ecdsa.ImportPkcs8PrivateKey(privateKey.Span, out var _);
return(ecdsa.SignHash(Sha2_256.ComputeHash(sequence)));
}
public bool TryGet(OmniHash hash, out IMemoryOwner <byte>?memoryOwner)
{
if (!EnumHelper.IsValid(hash.AlgorithmType))
{
throw new ArgumentException($"Incorrect HashAlgorithmType: {hash.AlgorithmType}");
}
memoryOwner = null;
bool success = false;
try
{
lock (_lockObject)
{
if (_clusterMetadataMap.TryGetValue(hash, out var clusterInfo))
{
clusterInfo = new ClusterMetadata(clusterInfo.Sectors.ToArray(), clusterInfo.Length, Timestamp.FromDateTime(DateTime.UtcNow));
_clusterMetadataMap[hash] = clusterInfo;
}
if (clusterInfo == null)
{
return(false);
}
memoryOwner = _bufferPool.Rent((int)clusterInfo.Length);
try
{
uint remain = clusterInfo.Length;
for (int i = 0; i < clusterInfo.Sectors.Count; i++, remain -= SectorSize)
{
ulong position = clusterInfo.Sectors[i] * SectorSize;
if (position > (ulong)_fileStream.Length)
{
_logger.Debug($"position too large: {position}");
return(false);
}
if ((ulong)_fileStream.Position != position)
{
_fileStream.Seek((long)position, SeekOrigin.Begin);
}
uint length = Math.Min(remain, SectorSize);
_fileStream.Read(_sectorBuffer, 0, _sectorBuffer.Length);
BytesOperations.Copy(_sectorBuffer, memoryOwner.Memory.Span.Slice((int)(SectorSize * i)), (int)length);
}
}
catch (Exception e)
{
_logger.Debug(e);
return(false);
}
}
if (hash.AlgorithmType == OmniHashAlgorithmType.Sha2_256 &&
BytesOperations.SequenceEqual(Sha2_256.ComputeHash(memoryOwner.Memory.Span), hash.Value.Span))
{
success = true;
return(true);
}
else
{
_logger.Debug("Broken block.");
return(false);
}
}
finally
{
if (!success)
{
if (memoryOwner != null)
{
memoryOwner.Dispose();
memoryOwner = null;
}
this.Remove(hash);
}
}
}
public async ValueTask <XeusClue> Import(string path, CancellationToken token = default)
{
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
return(await Task.Run(async() =>
{
// Check
lock (_lockObject)
{
var info = _contentMetadataStorage.GetFileContentMetadata(path);
if (info != null)
{
return info.Clue;
}
}
XeusClue?clue = null;
var lockedHashes = new HashSet <OmniHash>();
SharedBlocksMetadata?sharedBlocksInfo = null;
{
const int blockLength = 1024 * 1024;
const OmniHashAlgorithmType hashAlgorithmType = OmniHashAlgorithmType.Sha2_256;
const CorrectionAlgorithmType correctionAlgorithmType = CorrectionAlgorithmType.ReedSolomon8;
byte depth = 0;
var merkleTreeSectionList = new List <MerkleTreeSection>();
// File
using (var stream = new UnbufferedFileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferPool))
{
if (stream.Length <= blockLength)
{
OmniHash hash;
using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
{
stream.Read(bufferMemoryOwner.Memory.Span);
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
}
sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, (uint)stream.Length, new OmniHash[] { hash });
clue = new XeusClue(hash, depth);
}
else
{
var sharedHashes = new List <OmniHash>();
for (; ;)
{
var targetHashes = new List <OmniHash>();
var targetMemoryOwners = new List <IMemoryOwner <byte> >();
ulong sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
var bufferMemoryOwner = _bufferPool.Rent((int)length);
try
{
stream.Read(bufferMemoryOwner.Memory.Span);
sumLength += length;
}
catch (Exception e)
{
bufferMemoryOwner.Dispose();
throw e;
}
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
sharedHashes.Add(hash);
targetHashes.Add(hash);
targetMemoryOwners.Add(bufferMemoryOwner);
if (targetMemoryOwners.Count >= 128)
{
break;
}
}
var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
lockedHashes.UnionWith(parityHashes);
merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var memoryOwner in targetMemoryOwners)
{
memoryOwner.Dispose();
}
}
if (stream.Position == stream.Length)
{
break;
}
}
sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, blockLength, sharedHashes.ToArray());
depth++;
}
}
while (merkleTreeSectionList.Count > 0)
{
// Index
using (var stream = new RecyclableMemoryStream(_bufferPool))
{
RocketPackHelper.MessageToStream(new MerkleTreeNode(merkleTreeSectionList.ToArray()), stream);
stream.Seek(0, SeekOrigin.Begin);
merkleTreeSectionList.Clear();
if (stream.Length <= blockLength)
{
OmniHash hash;
using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
{
stream.Read(bufferMemoryOwner.Memory.Span);
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
lockedHashes.Add(hash);
}
clue = new XeusClue(hash, depth);
}
else
{
for (; ;)
{
var targetHashes = new List <OmniHash>();
var targetMemoryOwners = new List <IMemoryOwner <byte> >();
ulong sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
var bufferMemoryOwner = _bufferPool.Rent((int)length);
try
{
stream.Read(bufferMemoryOwner.Memory.Span);
sumLength += length;
}
catch (Exception e)
{
bufferMemoryOwner.Dispose();
throw e;
}
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
lockedHashes.Add(hash);
targetHashes.Add(hash);
targetMemoryOwners.Add(bufferMemoryOwner);
if (targetMemoryOwners.Count >= 128)
{
break;
}
}
var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
lockedHashes.UnionWith(parityHashes);
merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var memoryOwner in targetMemoryOwners)
{
memoryOwner.Dispose();
}
}
if (stream.Position == stream.Length)
{
break;
}
}
depth++;
}
}
}
}
if (clue == null)
{
throw new ImportFailed("clue is null");
}
lock (_lockObject)
{
if (!_contentMetadataStorage.ContainsFileContentMetadata(path))
{
_contentMetadataStorage.Add(new ContentMetadata(clue, lockedHashes.ToArray(), sharedBlocksInfo));
foreach (var hash in lockedHashes)
{
_blockStorage.Lock(hash);
}
}
}
return clue;
}, token));
}
public async ValueTask <OmniHash> AddPublishFileAsync(string filePath, CancellationToken cancellationToken = default)
{
using (await _asyncLock.LockAsync())
{
// 既にエンコード済みの場合の処理
{
if (_publishFilePathToRootHashMap.TryGetValue(filePath, out var rootHash))
{
return(rootHash);
}
}
// エンコード処理
{
var tempPath = Path.Combine(_configPath, "_temp_");
var merkleTreeSections = new Stack <MerkleTreeSection>();
// ファイルからハッシュ値を算出する
using (var inStream = new FileStream(filePath, FileMode.Open))
{
var hashList = new List <OmniHash>();
using (var memoryOwner = _bytesPool.Memory.Rent(MaxBlockLength))
{
var remain = inStream.Length;
while (remain > 0)
{
var blockLength = (int)Math.Min(remain, MaxBlockLength);
remain -= blockLength;
var memory = memoryOwner.Memory.Slice(0, blockLength);
inStream.Read(memory.Span);
var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(memory.Span));
hashList.Add(hash);
}
}
merkleTreeSections.Push(new MerkleTreeSection(0, (ulong)inStream.Length, hashList.ToArray()));
}
OmniHash rootHash;
// ハッシュ値からMerkle treeを作成する
for (; ;)
{
using var hub = new BytesHub(_bytesPool);
var lastMerkleTreeSection = merkleTreeSections.Peek();
lastMerkleTreeSection.Export(hub.Writer, _bytesPool);
if (hub.Writer.WrittenBytes > MaxBlockLength)
{
var hashList = new List <OmniHash>();
using (var memoryOwner = _bytesPool.Memory.Rent(MaxBlockLength))
{
var sequence = hub.Reader.GetSequence();
var remain = sequence.Length;
while (remain > 0)
{
var blockLength = (int)Math.Min(remain, MaxBlockLength);
remain -= blockLength;
var memory = memoryOwner.Memory.Slice(0, blockLength);
sequence.CopyTo(memory.Span);
var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(memory.Span));
hashList.Add(hash);
await this.WriteAsync(tempPath, hash, memory);
sequence = sequence.Slice(blockLength);
}
}
merkleTreeSections.Push(new MerkleTreeSection(merkleTreeSections.Count, (ulong)hub.Writer.WrittenBytes, hashList.ToArray()));
}
else
{
using (var memoryOwner = _bytesPool.Memory.Rent(MaxBlockLength))
{
var sequence = hub.Reader.GetSequence();
var memory = memoryOwner.Memory.Slice(0, (int)sequence.Length);
sequence.CopyTo(memory.Span);
var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(memory.Span));
await this.WriteAsync(tempPath, hash, memory);
rootHash = hash;
}
break;
}
}
// 一時フォルダからキャッシュフォルダへ移動させる
{
var cachePath = Path.Combine(_configPath, this.OmniHashToFilePath(rootHash));
Directory.Move(tempPath, cachePath);
}
var status = new PublishFileStatus(rootHash, filePath, merkleTreeSections.ToArray());
_publishFileStatusMap.Add(rootHash, status);
_publishFilePathToRootHashMap[filePath] = rootHash;
return(rootHash);
}
}
}
public bool TryGetBlock(OmniHash hash, [NotNullWhen(true)] out IMemoryOwner <byte>?memoryOwner)
{
if (!EnumHelper.IsValid(hash.AlgorithmType))
{
throw new ArgumentException($"Incorrect HashAlgorithmType: {hash.AlgorithmType}");
}
// Cache
{
var result = _blockStorage.TryGet(hash, out memoryOwner);
if (result)
{
return(true);
}
}
bool success = false;
string?path = null;
// Share
try
{
lock (_lockObject)
{
var sharedBlocksInfo = _contentMetadataStorage.GetSharedBlocksInfo(hash);
if (sharedBlocksInfo != null)
{
ulong position = (ulong)sharedBlocksInfo.GetIndex(hash) * sharedBlocksInfo.BlockLength;
uint length = (uint)Math.Min(sharedBlocksInfo.Length - position, sharedBlocksInfo.BlockLength);
memoryOwner = _bufferPool.Rent((int)length);
try
{
using (var stream = new UnbufferedFileStream(sharedBlocksInfo.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferPool))
{
stream.Seek((long)position, SeekOrigin.Begin);
stream.Read(memoryOwner.Memory.Span);
}
path = sharedBlocksInfo.Path;
}
catch (Exception e)
{
_logger.Debug(e);
return(false);
}
}
}
if (memoryOwner == null)
{
return(false);
}
if (hash.AlgorithmType == OmniHashAlgorithmType.Sha2_256 &&
BytesOperations.SequenceEqual(Sha2_256.ComputeHash(memoryOwner.Memory.Span), hash.Value.Span))
{
success = true;
return(true);
}
else
{
_logger.Warn("Broken block.");
return(false);
}
}
finally
{
if (!success)
{
if (memoryOwner != null)
{
memoryOwner.Dispose();
memoryOwner = null;
}
if (path != null)
{
this.RemoveContent(path);
}
}
}
}
public bool TrySet(OmniHash hash, ReadOnlySpan <byte> value)
{
if (!EnumHelper.IsValid(hash.AlgorithmType))
{
throw new ArgumentException($"Incorrect HashAlgorithmType: {hash.AlgorithmType}");
}
if (value.Length > 1024 * 1024 * 32)
{
_logger.Debug($"{nameof(value)} too large.");
return(false);
}
if (hash.AlgorithmType == OmniHashAlgorithmType.Sha2_256 &&
!BytesOperations.SequenceEqual(Sha2_256.ComputeHash(value), hash.Value.Span))
{
_logger.Debug("Broken block.");
return(false);
}
lock (_lockObject)
{
if (this.Contains(hash))
{
_logger.Debug($"Already exist.");
return(true);
}
if (!this.TryGetFreeSectors((int)((value.Length + (SectorSize - 1)) / SectorSize), out var sectors))
{
_errorReportEventQueue.Enqueue(new ErrorReport(Timestamp.FromDateTime(DateTime.UtcNow), ErrorReportType.SpaceNotFound));
_logger.Debug("Space not found.");
return(false);
}
try
{
uint remain = (uint)value.Length;
for (int i = 0; i < sectors.Length && 0 < remain; i++, remain -= SectorSize)
{
ulong position = sectors[i] * SectorSize;
if ((ulong)_fileStream.Length < position + SectorSize)
{
const uint unit = 1024 * 1024 * 256; // 256MB
ulong size = MathHelper.Roundup((position + SectorSize), unit);
_fileStream.SetLength((long)Math.Min(size, this.Size));
}
if ((ulong)_fileStream.Position != position)
{
_fileStream.Seek((long)position, SeekOrigin.Begin);
}
uint length = Math.Min(remain, SectorSize);
BytesOperations.Copy(value.Slice((int)(SectorSize * i)), _sectorBuffer, (int)length);
BytesOperations.Zero(_sectorBuffer.AsSpan((int)length, (int)(_sectorBuffer.Length - length)));
_fileStream.Write(_sectorBuffer, 0, _sectorBuffer.Length);
}
_fileStream.Flush();
}
catch (Exception e)
{
_logger.Debug(e);
return(false);
}
_clusterMetadataMap[hash] = new ClusterMetadata(sectors, (uint)value.Length, Timestamp.FromDateTime(DateTime.UtcNow));
// Event
_addedBlockEventQueue.Enqueue(hash);
return(true);
}
}
