Ejemplo n.º 1
0
 public static OmniHash Create(OmniHashAlgorithmType algorithmType, ReadOnlySpan <byte> message)
 {
     return(algorithmType switch
     {
         OmniHashAlgorithmType.Sha2_256 => new OmniHash(algorithmType, Sha2_256.ComputeHash(message)),
         _ => throw new NotSupportedException(),
     });
Ejemplo n.º 2
0
        private IEnumerable <(ReadOnlyMemory <byte>, string)> GetHashesV1(V1.Internal.ProfileMessage profileMessage, OmniAgreementPublicKey agreementPublicKey, V1.Internal.HashAlgorithm hashAlgorithm)
        {
            var results = new Dictionary <ReadOnlyMemory <byte>, string>();

            byte[] verificationMessageHash;
            {
                var verificationMessage = new V1.Internal.VerificationMessage(profileMessage, agreementPublicKey);

                if (hashAlgorithm == V1.Internal.HashAlgorithm.Sha2_256)
                {
                    var hub = new Hub();

                    verificationMessage.Export(hub.Writer, _bufferPool);
                    verificationMessageHash = Sha2_256.ComputeHash(hub.Reader.GetSequence());
                }
                else
                {
                    throw new NotSupportedException(nameof(hashAlgorithm));
                }
            }

            foreach (var password in _passwords)
            {
                if (hashAlgorithm.HasFlag(V1.Internal.HashAlgorithm.Sha2_256))
                {
                    results.Add(Hmac_Sha2_256.ComputeHash(verificationMessageHash, Sha2_256.ComputeHash(password)), password);
                }
            }

            return(results.Select(item => (item.Key, item.Value)));
        }
Ejemplo n.º 3
0
        private static OmniHash CreateOmniHash(string name, ReadOnlySpan <byte> publicKey, OmniHashAlgorithmType hashAlgorithmType)
        {
            if (name == null)
            {
                throw new ArgumentNullException(nameof(name));
            }

            using (var hub = new Hub())
            {
                {
                    var writer = new RocketPackWriter(hub.Writer, BufferPool.Shared);

                    writer.Write(name);
                    writer.Write(publicKey);

                    hub.Writer.Complete();
                }

                if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                {
                    var result = new OmniHash(hashAlgorithmType, Sha2_256.ComputeHash(hub.Reader.GetSequence()));
                    hub.Reader.Complete();

                    return(result);
                }
                else
                {
                    throw new NotSupportedException();
                }
            }
        }
Ejemplo n.º 4
0
    private static OmniHash CreateOmniHash(string name, ReadOnlySpan <byte> publicKey, OmniHashAlgorithmType hashAlgorithmType)
    {
        if (name == null)
        {
            throw new ArgumentNullException(nameof(name));
        }

        using var bytesPipe = new BytesPipe();
        {
            var writer = new RocketMessageWriter(bytesPipe.Writer, BytesPool.Shared);

            writer.Write(name);
            writer.Write(publicKey);
        }

        if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
        {
            var result = new OmniHash(hashAlgorithmType, Sha2_256.ComputeHash(bytesPipe.Reader.GetSequence()));

            return(result);
        }
        else
        {
            throw new NotSupportedException();
        }
    }
Ejemplo n.º 5
0
    public static string SignatureToString(OmniSignature signature)
    {
        using var bytesPipe = new BytesPipe(BytesPool.Shared);
        signature.Export(bytesPipe.Writer, BytesPool.Shared);
        var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bytesPipe.Reader.GetSequence()));

        return(hash.ToString(ConvertStringType.Base16));
    }
Ejemplo n.º 6
0
        private byte[] ComputeHash(ProfileMessage profileMessage, OmniAgreementPublicKey agreementPublicKey, HashAlgorithmType hashAlgorithm)
        {
            var verificationMessage = new VerificationMessage(profileMessage, agreementPublicKey);

            if (hashAlgorithm == HashAlgorithmType.Sha2_256)
            {
                using var bytesPipe = new BytesPipe();
                verificationMessage.Export(bytesPipe.Writer, _bytesPool);

                return(Sha2_256.ComputeHash(bytesPipe.Reader.GetSequence()));
            }

            throw new NotSupportedException(nameof(hashAlgorithm));
        }
Ejemplo n.º 7
0
            public static uint Verify_Simple_Sha2_256(ReadOnlySpan <byte> key, ReadOnlySpan <byte> value)
            {
                if (key == null)
                {
                    throw new ArgumentNullException(nameof(key));
                }
                if (key.Length != 32)
                {
                    throw new ArgumentOutOfRangeException(nameof(key));
                }
                if (value == null)
                {
                    throw new ArgumentNullException(nameof(value));
                }
                if (value.Length != 32)
                {
                    throw new ArgumentOutOfRangeException(nameof(value));
                }

                Span <byte> buffer = stackalloc byte[64];

                byte[] hash;
                {
                    BytesOperations.Copy(key, buffer, key.Length);
                    BytesOperations.Copy(value, buffer.Slice(key.Length), value.Length);
                    hash = Sha2_256.ComputeHash(buffer);
                }

                uint count = 0;

                for (int i = 0; i < 32; i++)
                {
                    for (int j = 0; j < 8; j++)
                    {
                        if (((hash[i] << j) & 0x80) == 0)
                        {
                            count++;
                        }
                        else
                        {
                            goto End;
                        }
                    }
                }

End:
                return(count);
            }
Ejemplo n.º 8
0
    public static bool TryComputeHash(ReadOnlySequence <byte> sequence, ReadOnlySpan <byte> key, Span <byte> destination)
    {
        if (destination.Length < 32)
        {
            throw new ArgumentOutOfRangeException(nameof(destination));
        }

        Span <byte> extendedKey = stackalloc byte[_blockLength];

        if (key.Length > _blockLength)
        {
            Sha2_256.TryComputeHash(key, extendedKey);
        }
        else
        {
            BytesOperations.Copy(key, extendedKey, Math.Min(key.Length, extendedKey.Length));
        }

        Span <byte> ixor = stackalloc byte[_blockLength];

        BytesOperations.Xor(_ipad, extendedKey, ixor);

        Span <byte> oxor = stackalloc byte[_blockLength];

        BytesOperations.Xor(_opad, extendedKey, oxor);

        Span <byte> ihash = stackalloc byte[32];

        using (var incrementalHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256))
        {
            incrementalHash.AppendData(ixor);

            foreach (var segment in sequence)
            {
                incrementalHash.AppendData(segment.Span);
            }

            incrementalHash.TryGetHashAndReset(ihash, out _);
        }

        using (var incrementalHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256))
        {
            incrementalHash.AppendData(oxor);
            incrementalHash.AppendData(ihash);

            return(incrementalHash.TryGetHashAndReset(destination, out _));
        }
    }
Ejemplo n.º 9
0
        public bool Equals(BlobFingerprint other)
        {
            if (ReferenceEquals(this, other))
            {
                return(true);
            }

            if (ReferenceEquals(null, other))
            {
                return(false);
            }

            return(Size == other.Size &&
                   Sha3_512.SequenceEqual(other.Sha3_512) &&
                   Sha2_256.SequenceEqual(other.Sha2_256) &&
                   Md5.SequenceEqual(other.Md5));
        }
Ejemplo n.º 10
0
        public static bool Verify(ReadOnlyMemory <byte> publicKey, ReadOnlyMemory <byte> signature, ReadOnlySequence <byte> sequence)
        {
            ECParameters ecParameters;

            {
                var plist = SerializeHelper.Decode(publicKey).ToArray();

                ecParameters = new ECParameters()
                {
                    Q = new ECPoint()
                    {
                        X = plist[0].ToArray(),
                        Y = plist[1].ToArray(),
                    },
                };
            }

            using (var ecdsa = ECDsa.Create())
            {
                ecdsa.ImportParameters(ecParameters);
                return(ecdsa.VerifyHash(Sha2_256.ComputeHash(sequence).AsSpan(), signature.Span));
            }
        }
Ejemplo n.º 11
0
        public async Task Test()
        {
            var random = new Random();

            using var blockStorage = new BlockStorage(UnitTestEnvironment.TempDirectoryPath, BufferPool.Shared);
            var sizeList = new List <int>(new[] { 0, 1, 10, 100, 1000, 10000 });

            for (int i = 0; i < 32; i++)
            {
                sizeList.Add(random.Next(0, 10));
                sizeList.Add(random.Next(0, 256));
                sizeList.Add(random.Next(0, 1024 * 32));
            }

            await blockStorage.LoadAsync();

            blockStorage.Resize(1024 * 1024 * 1024);

            var hashList = new List <OmniHash>();

            foreach (var size in sizeList)
            {
                using var memoryOwner = BufferPool.Shared.Rent(size);
                random.NextBytes(memoryOwner.Memory.Span);
                var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(memoryOwner.Memory.Span));

                Assert.True(blockStorage.TrySet(hash, memoryOwner.Memory.Span));

                hashList.Add(hash);
            }

            foreach (var hash in hashList)
            {
                Assert.True(blockStorage.TryGet(hash, out var memoryOwner));
                Assert.True(BytesOperations.SequenceEqual(hash.Value.Span, Sha2_256.ComputeHash(memoryOwner.Memory.Span)));
            }
        }
Ejemplo n.º 12
0
        public static byte[] Sign(ReadOnlyMemory <byte> privateKey, ReadOnlySequence <byte> sequence)
        {
            ECParameters ecParameters;

            {
                var plist = SerializeHelper.Decode(privateKey).ToArray();

                ecParameters = new ECParameters()
                {
                    Q = new ECPoint()
                    {
                        X = plist[0].ToArray(),
                        Y = plist[1].ToArray(),
                    },
                    D = plist[2].ToArray(),
                };
            }

            using (var ecdsa = ECDsa.Create())
            {
                ecdsa.ImportParameters(ecParameters);
                return(ecdsa.SignHash(Sha2_256.ComputeHash(sequence)));
            }
        }
Ejemplo n.º 13
0
        private async ValueTask <OmniHash[]> ParityEncode(IEnumerable <Memory <byte> > blocks, OmniHashAlgorithmType hashAlgorithmType, CorrectionAlgorithmType correctionAlgorithmType, CancellationToken token = default)
        {
            if (correctionAlgorithmType == CorrectionAlgorithmType.ReedSolomon8)
            {
                var blockList = blocks.ToList();

                // blocksの要素数をチェックする
                if (blockList.Count <= 0 || blockList.Count > 128)
                {
                    throw new ArgumentOutOfRangeException(nameof(blocks));
                }

                // 最大ブロック長
                int blockLength = blockList[0].Length;

                // 各ブロックの長さをチェックする
                for (int i = 1; i < blockList.Count; i++)
                {
                    // 末尾以外
                    if (i < (blockList.Count - 1))
                    {
                        // 先頭ブロックと同一の長さでなければならない
                        if (!(blockList[i].Length == blockLength))
                        {
                            throw new ArgumentOutOfRangeException(nameof(blocks), $"{nameof(blocks)}[{i}].Length");
                        }
                    }
                    // 末尾
                    else
                    {
                        // 先頭ブロックと同一かそれ以下の長さでなければならない
                        if (!(blockList[i].Length <= blockLength))
                        {
                            throw new ArgumentOutOfRangeException(nameof(blocks), $"{nameof(blocks)}[{i}].Length");
                        }
                    }
                }

                return(await Task.Run(async() =>
                {
                    var memoryOwners = new List <IMemoryOwner <byte> >();

                    try
                    {
                        var targetBuffers = new ReadOnlyMemory <byte> [128];
                        var parityBuffers = new Memory <byte> [128];

                        // Load
                        {
                            int index = 0;

                            foreach (var buffer in blocks)
                            {
                                token.ThrowIfCancellationRequested();

                                // 実ブロックの長さが足りない場合、0byteでpaddingを行う
                                if (buffer.Length < blockLength)
                                {
                                    var tempMemoryOwner = _bufferPool.Rent(blockLength);

                                    BytesOperations.Copy(buffer.Span, tempMemoryOwner.Memory.Span, buffer.Length);
                                    BytesOperations.Zero(tempMemoryOwner.Memory.Span.Slice(buffer.Length));

                                    memoryOwners.Add(tempMemoryOwner);

                                    targetBuffers[index++] = tempMemoryOwner.Memory;
                                }
                                else
                                {
                                    targetBuffers[index++] = buffer;
                                }
                            }

                            // 実ブロック数が128に満たない場合、0byte配列でPaddingを行う。
                            for (int i = (128 - blocks.Count()) - 1; i >= 0; i--)
                            {
                                var tempMemoryOwner = _bufferPool.Rent(blockLength);

                                BytesOperations.Zero(tempMemoryOwner.Memory.Span);

                                memoryOwners.Add(tempMemoryOwner);

                                targetBuffers[index++] = tempMemoryOwner.Memory;
                            }
                        }

                        for (int i = 0; i < parityBuffers.Length; i++)
                        {
                            var tempMemoryOwner = _bufferPool.Rent(blockLength);

                            BytesOperations.Zero(tempMemoryOwner.Memory.Span);

                            memoryOwners.Add(tempMemoryOwner);

                            parityBuffers[i] = tempMemoryOwner.Memory;
                        }

                        var indexes = new int[parityBuffers.Length];

                        for (int i = 0; i < parityBuffers.Length; i++)
                        {
                            indexes[i] = targetBuffers.Length + i;
                        }

                        var reedSolomon = new ReedSolomon8(targetBuffers.Length, targetBuffers.Length + parityBuffers.Length, _bufferPool);
                        await reedSolomon.Encode(targetBuffers, indexes, parityBuffers, blockLength, _threadCount, token);

                        token.ThrowIfCancellationRequested();

                        var parityHashes = new List <OmniHash>();

                        for (int i = 0; i < parityBuffers.Length; i++)
                        {
                            OmniHash hash;

                            if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                            {
                                hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(parityBuffers[i].Span));
                            }
                            else
                            {
                                throw new NotSupportedException();
                            }

                            _blockStorage.Lock(hash);

                            bool result = _blockStorage.TrySet(hash, parityBuffers[i].Span);

                            if (!result)
                            {
                                throw new ImportFailed("Failed to save Block.");
                            }

                            parityHashes.Add(hash);
                        }

                        return parityHashes.ToArray();
                    }
                    finally
                    {
                        foreach (var memoryOwner in memoryOwners)
                        {
                            memoryOwner.Dispose();
                        }
                    }
                }));
            }
            else
            {
                throw new NotSupportedException();
            }
        }
Ejemplo n.º 14
0
 public static bool Verify(ReadOnlyMemory <byte> publicKey, ReadOnlyMemory <byte> signature, ReadOnlySequence <byte> sequence)
 {
     using var ecdsa = ECDsa.Create();
     ecdsa.ImportSubjectPublicKeyInfo(publicKey.Span, out var _);
     return(ecdsa.VerifyHash(Sha2_256.ComputeHash(sequence).AsSpan(), signature.Span));
 }
Ejemplo n.º 15
0
 public static byte[] Sign(ReadOnlyMemory <byte> privateKey, ReadOnlySequence <byte> sequence)
 {
     using var ecdsa = ECDsa.Create();
     ecdsa.ImportPkcs8PrivateKey(privateKey.Span, out var _);
     return(ecdsa.SignHash(Sha2_256.ComputeHash(sequence)));
 }
Ejemplo n.º 16
0
        public bool TryGet(OmniHash hash, out IMemoryOwner <byte>?memoryOwner)
        {
            if (!EnumHelper.IsValid(hash.AlgorithmType))
            {
                throw new ArgumentException($"Incorrect HashAlgorithmType: {hash.AlgorithmType}");
            }

            memoryOwner = null;
            bool success = false;

            try
            {
                lock (_lockObject)
                {
                    if (_clusterMetadataMap.TryGetValue(hash, out var clusterInfo))
                    {
                        clusterInfo = new ClusterMetadata(clusterInfo.Sectors.ToArray(), clusterInfo.Length, Timestamp.FromDateTime(DateTime.UtcNow));
                        _clusterMetadataMap[hash] = clusterInfo;
                    }

                    if (clusterInfo == null)
                    {
                        return(false);
                    }

                    memoryOwner = _bufferPool.Rent((int)clusterInfo.Length);

                    try
                    {
                        uint remain = clusterInfo.Length;

                        for (int i = 0; i < clusterInfo.Sectors.Count; i++, remain -= SectorSize)
                        {
                            ulong position = clusterInfo.Sectors[i] * SectorSize;

                            if (position > (ulong)_fileStream.Length)
                            {
                                _logger.Debug($"position too large: {position}");

                                return(false);
                            }

                            if ((ulong)_fileStream.Position != position)
                            {
                                _fileStream.Seek((long)position, SeekOrigin.Begin);
                            }

                            uint length = Math.Min(remain, SectorSize);

                            _fileStream.Read(_sectorBuffer, 0, _sectorBuffer.Length);
                            BytesOperations.Copy(_sectorBuffer, memoryOwner.Memory.Span.Slice((int)(SectorSize * i)), (int)length);
                        }
                    }
                    catch (Exception e)
                    {
                        _logger.Debug(e);

                        return(false);
                    }
                }

                if (hash.AlgorithmType == OmniHashAlgorithmType.Sha2_256 &&
                    BytesOperations.SequenceEqual(Sha2_256.ComputeHash(memoryOwner.Memory.Span), hash.Value.Span))
                {
                    success = true;

                    return(true);
                }
                else
                {
                    _logger.Debug("Broken block.");

                    return(false);
                }
            }
            finally
            {
                if (!success)
                {
                    if (memoryOwner != null)
                    {
                        memoryOwner.Dispose();
                        memoryOwner = null;
                    }

                    this.Remove(hash);
                }
            }
        }
Ejemplo n.º 17
0
        public async ValueTask <XeusClue> Import(string path, CancellationToken token = default)
        {
            if (path == null)
            {
                throw new ArgumentNullException(nameof(path));
            }

            return(await Task.Run(async() =>
            {
                // Check
                lock (_lockObject)
                {
                    var info = _contentMetadataStorage.GetFileContentMetadata(path);
                    if (info != null)
                    {
                        return info.Clue;
                    }
                }

                XeusClue?clue = null;
                var lockedHashes = new HashSet <OmniHash>();
                SharedBlocksMetadata?sharedBlocksInfo = null;

                {
                    const int blockLength = 1024 * 1024;
                    const OmniHashAlgorithmType hashAlgorithmType = OmniHashAlgorithmType.Sha2_256;
                    const CorrectionAlgorithmType correctionAlgorithmType = CorrectionAlgorithmType.ReedSolomon8;

                    byte depth = 0;

                    var merkleTreeSectionList = new List <MerkleTreeSection>();

                    // File
                    using (var stream = new UnbufferedFileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferPool))
                    {
                        if (stream.Length <= blockLength)
                        {
                            OmniHash hash;

                            using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
                            {
                                stream.Read(bufferMemoryOwner.Memory.Span);

                                if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                                {
                                    hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
                                }
                            }

                            sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, (uint)stream.Length, new OmniHash[] { hash });
                            clue = new XeusClue(hash, depth);
                        }
                        else
                        {
                            var sharedHashes = new List <OmniHash>();

                            for (; ;)
                            {
                                var targetHashes = new List <OmniHash>();
                                var targetMemoryOwners = new List <IMemoryOwner <byte> >();
                                ulong sumLength = 0;

                                try
                                {
                                    for (int i = 0; stream.Position < stream.Length; i++)
                                    {
                                        token.ThrowIfCancellationRequested();

                                        uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
                                        var bufferMemoryOwner = _bufferPool.Rent((int)length);

                                        try
                                        {
                                            stream.Read(bufferMemoryOwner.Memory.Span);

                                            sumLength += length;
                                        }
                                        catch (Exception e)
                                        {
                                            bufferMemoryOwner.Dispose();

                                            throw e;
                                        }

                                        OmniHash hash;

                                        if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                                        {
                                            hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
                                        }

                                        sharedHashes.Add(hash);

                                        targetHashes.Add(hash);
                                        targetMemoryOwners.Add(bufferMemoryOwner);

                                        if (targetMemoryOwners.Count >= 128)
                                        {
                                            break;
                                        }
                                    }

                                    var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
                                    lockedHashes.UnionWith(parityHashes);

                                    merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
                                }
                                finally
                                {
                                    foreach (var memoryOwner in targetMemoryOwners)
                                    {
                                        memoryOwner.Dispose();
                                    }
                                }

                                if (stream.Position == stream.Length)
                                {
                                    break;
                                }
                            }

                            sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, blockLength, sharedHashes.ToArray());

                            depth++;
                        }
                    }

                    while (merkleTreeSectionList.Count > 0)
                    {
                        // Index
                        using (var stream = new RecyclableMemoryStream(_bufferPool))
                        {
                            RocketPackHelper.MessageToStream(new MerkleTreeNode(merkleTreeSectionList.ToArray()), stream);
                            stream.Seek(0, SeekOrigin.Begin);

                            merkleTreeSectionList.Clear();

                            if (stream.Length <= blockLength)
                            {
                                OmniHash hash;

                                using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
                                {
                                    stream.Read(bufferMemoryOwner.Memory.Span);

                                    if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                                    {
                                        hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
                                    }

                                    _blockStorage.Lock(hash);

                                    bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);

                                    if (!result)
                                    {
                                        throw new ImportFailed("Failed to save Block.");
                                    }

                                    lockedHashes.Add(hash);
                                }

                                clue = new XeusClue(hash, depth);
                            }
                            else
                            {
                                for (; ;)
                                {
                                    var targetHashes = new List <OmniHash>();
                                    var targetMemoryOwners = new List <IMemoryOwner <byte> >();
                                    ulong sumLength = 0;

                                    try
                                    {
                                        for (int i = 0; stream.Position < stream.Length; i++)
                                        {
                                            token.ThrowIfCancellationRequested();

                                            uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
                                            var bufferMemoryOwner = _bufferPool.Rent((int)length);

                                            try
                                            {
                                                stream.Read(bufferMemoryOwner.Memory.Span);

                                                sumLength += length;
                                            }
                                            catch (Exception e)
                                            {
                                                bufferMemoryOwner.Dispose();

                                                throw e;
                                            }

                                            OmniHash hash;

                                            if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                                            {
                                                hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
                                            }

                                            _blockStorage.Lock(hash);

                                            bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);

                                            if (!result)
                                            {
                                                throw new ImportFailed("Failed to save Block.");
                                            }

                                            lockedHashes.Add(hash);

                                            targetHashes.Add(hash);
                                            targetMemoryOwners.Add(bufferMemoryOwner);

                                            if (targetMemoryOwners.Count >= 128)
                                            {
                                                break;
                                            }
                                        }

                                        var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
                                        lockedHashes.UnionWith(parityHashes);

                                        merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
                                    }
                                    finally
                                    {
                                        foreach (var memoryOwner in targetMemoryOwners)
                                        {
                                            memoryOwner.Dispose();
                                        }
                                    }

                                    if (stream.Position == stream.Length)
                                    {
                                        break;
                                    }
                                }

                                depth++;
                            }
                        }
                    }
                }

                if (clue == null)
                {
                    throw new ImportFailed("clue is null");
                }

                lock (_lockObject)
                {
                    if (!_contentMetadataStorage.ContainsFileContentMetadata(path))
                    {
                        _contentMetadataStorage.Add(new ContentMetadata(clue, lockedHashes.ToArray(), sharedBlocksInfo));

                        foreach (var hash in lockedHashes)
                        {
                            _blockStorage.Lock(hash);
                        }
                    }
                }

                return clue;
            }, token));
        }
Ejemplo n.º 18
0
        public async ValueTask <OmniHash> AddPublishFileAsync(string filePath, CancellationToken cancellationToken = default)
        {
            using (await _asyncLock.LockAsync())
            {
                // 既にエンコード済みの場合の処理
                {
                    if (_publishFilePathToRootHashMap.TryGetValue(filePath, out var rootHash))
                    {
                        return(rootHash);
                    }
                }

                // エンコード処理
                {
                    var tempPath = Path.Combine(_configPath, "_temp_");

                    var merkleTreeSections = new Stack <MerkleTreeSection>();

                    // ファイルからハッシュ値を算出する
                    using (var inStream = new FileStream(filePath, FileMode.Open))
                    {
                        var hashList = new List <OmniHash>();

                        using (var memoryOwner = _bytesPool.Memory.Rent(MaxBlockLength))
                        {
                            var remain = inStream.Length;

                            while (remain > 0)
                            {
                                var blockLength = (int)Math.Min(remain, MaxBlockLength);
                                remain -= blockLength;

                                var memory = memoryOwner.Memory.Slice(0, blockLength);
                                inStream.Read(memory.Span);

                                var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(memory.Span));
                                hashList.Add(hash);
                            }
                        }

                        merkleTreeSections.Push(new MerkleTreeSection(0, (ulong)inStream.Length, hashList.ToArray()));
                    }

                    OmniHash rootHash;

                    // ハッシュ値からMerkle treeを作成する
                    for (; ;)
                    {
                        using var hub = new BytesHub(_bytesPool);

                        var lastMerkleTreeSection = merkleTreeSections.Peek();
                        lastMerkleTreeSection.Export(hub.Writer, _bytesPool);

                        if (hub.Writer.WrittenBytes > MaxBlockLength)
                        {
                            var hashList = new List <OmniHash>();

                            using (var memoryOwner = _bytesPool.Memory.Rent(MaxBlockLength))
                            {
                                var sequence = hub.Reader.GetSequence();
                                var remain   = sequence.Length;

                                while (remain > 0)
                                {
                                    var blockLength = (int)Math.Min(remain, MaxBlockLength);
                                    remain -= blockLength;

                                    var memory = memoryOwner.Memory.Slice(0, blockLength);
                                    sequence.CopyTo(memory.Span);

                                    var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(memory.Span));
                                    hashList.Add(hash);

                                    await this.WriteAsync(tempPath, hash, memory);

                                    sequence = sequence.Slice(blockLength);
                                }
                            }

                            merkleTreeSections.Push(new MerkleTreeSection(merkleTreeSections.Count, (ulong)hub.Writer.WrittenBytes, hashList.ToArray()));
                        }
                        else
                        {
                            using (var memoryOwner = _bytesPool.Memory.Rent(MaxBlockLength))
                            {
                                var sequence = hub.Reader.GetSequence();

                                var memory = memoryOwner.Memory.Slice(0, (int)sequence.Length);
                                sequence.CopyTo(memory.Span);

                                var hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(memory.Span));

                                await this.WriteAsync(tempPath, hash, memory);

                                rootHash = hash;
                            }

                            break;
                        }
                    }

                    // 一時フォルダからキャッシュフォルダへ移動させる
                    {
                        var cachePath = Path.Combine(_configPath, this.OmniHashToFilePath(rootHash));
                        Directory.Move(tempPath, cachePath);
                    }

                    var status = new PublishFileStatus(rootHash, filePath, merkleTreeSections.ToArray());
                    _publishFileStatusMap.Add(rootHash, status);

                    _publishFilePathToRootHashMap[filePath] = rootHash;

                    return(rootHash);
                }
            }
        }
Ejemplo n.º 19
0
        public bool TryGetBlock(OmniHash hash, [NotNullWhen(true)] out IMemoryOwner <byte>?memoryOwner)
        {
            if (!EnumHelper.IsValid(hash.AlgorithmType))
            {
                throw new ArgumentException($"Incorrect HashAlgorithmType: {hash.AlgorithmType}");
            }

            // Cache
            {
                var result = _blockStorage.TryGet(hash, out memoryOwner);

                if (result)
                {
                    return(true);
                }
            }

            bool   success = false;
            string?path    = null;

            // Share
            try
            {
                lock (_lockObject)
                {
                    var sharedBlocksInfo = _contentMetadataStorage.GetSharedBlocksInfo(hash);

                    if (sharedBlocksInfo != null)
                    {
                        ulong position = (ulong)sharedBlocksInfo.GetIndex(hash) * sharedBlocksInfo.BlockLength;
                        uint  length   = (uint)Math.Min(sharedBlocksInfo.Length - position, sharedBlocksInfo.BlockLength);

                        memoryOwner = _bufferPool.Rent((int)length);

                        try
                        {
                            using (var stream = new UnbufferedFileStream(sharedBlocksInfo.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferPool))
                            {
                                stream.Seek((long)position, SeekOrigin.Begin);
                                stream.Read(memoryOwner.Memory.Span);
                            }

                            path = sharedBlocksInfo.Path;
                        }
                        catch (Exception e)
                        {
                            _logger.Debug(e);

                            return(false);
                        }
                    }
                }

                if (memoryOwner == null)
                {
                    return(false);
                }

                if (hash.AlgorithmType == OmniHashAlgorithmType.Sha2_256 &&
                    BytesOperations.SequenceEqual(Sha2_256.ComputeHash(memoryOwner.Memory.Span), hash.Value.Span))
                {
                    success = true;

                    return(true);
                }
                else
                {
                    _logger.Warn("Broken block.");

                    return(false);
                }
            }
            finally
            {
                if (!success)
                {
                    if (memoryOwner != null)
                    {
                        memoryOwner.Dispose();
                        memoryOwner = null;
                    }

                    if (path != null)
                    {
                        this.RemoveContent(path);
                    }
                }
            }
        }
Ejemplo n.º 20
0
        public bool TrySet(OmniHash hash, ReadOnlySpan <byte> value)
        {
            if (!EnumHelper.IsValid(hash.AlgorithmType))
            {
                throw new ArgumentException($"Incorrect HashAlgorithmType: {hash.AlgorithmType}");
            }

            if (value.Length > 1024 * 1024 * 32)
            {
                _logger.Debug($"{nameof(value)} too large.");

                return(false);
            }

            if (hash.AlgorithmType == OmniHashAlgorithmType.Sha2_256 &&
                !BytesOperations.SequenceEqual(Sha2_256.ComputeHash(value), hash.Value.Span))
            {
                _logger.Debug("Broken block.");

                return(false);
            }

            lock (_lockObject)
            {
                if (this.Contains(hash))
                {
                    _logger.Debug($"Already exist.");

                    return(true);
                }


                if (!this.TryGetFreeSectors((int)((value.Length + (SectorSize - 1)) / SectorSize), out var sectors))
                {
                    _errorReportEventQueue.Enqueue(new ErrorReport(Timestamp.FromDateTime(DateTime.UtcNow), ErrorReportType.SpaceNotFound));

                    _logger.Debug("Space not found.");

                    return(false);
                }

                try
                {
                    uint remain = (uint)value.Length;

                    for (int i = 0; i < sectors.Length && 0 < remain; i++, remain -= SectorSize)
                    {
                        ulong position = sectors[i] * SectorSize;

                        if ((ulong)_fileStream.Length < position + SectorSize)
                        {
                            const uint unit = 1024 * 1024 * 256; // 256MB
                            ulong      size = MathHelper.Roundup((position + SectorSize), unit);

                            _fileStream.SetLength((long)Math.Min(size, this.Size));
                        }

                        if ((ulong)_fileStream.Position != position)
                        {
                            _fileStream.Seek((long)position, SeekOrigin.Begin);
                        }

                        uint length = Math.Min(remain, SectorSize);

                        BytesOperations.Copy(value.Slice((int)(SectorSize * i)), _sectorBuffer, (int)length);
                        BytesOperations.Zero(_sectorBuffer.AsSpan((int)length, (int)(_sectorBuffer.Length - length)));

                        _fileStream.Write(_sectorBuffer, 0, _sectorBuffer.Length);
                    }

                    _fileStream.Flush();
                }
                catch (Exception e)
                {
                    _logger.Debug(e);

                    return(false);
                }

                _clusterMetadataMap[hash] = new ClusterMetadata(sectors, (uint)value.Length, Timestamp.FromDateTime(DateTime.UtcNow));

                // Event
                _addedBlockEventQueue.Enqueue(hash);

                return(true);
            }
        }