public Task <Metadata> Import(string path, DateTime creationTime, CancellationToken token)
{
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
return(Task.Run(() =>
{
// Check
lock (_lockObject)
{
var info = _contentInfoManager.GetFileContentInfo(path);
if (info != null)
{
return info.Metadata;
}
}
Metadata metadata = null;
var lockedHashes = new HashSet <Hash>();
ShareInfo shareInfo = null;
{
const int blockLength = 1024 * 1024;
const HashAlgorithm hashAlgorithm = HashAlgorithm.Sha256;
const CorrectionAlgorithm correctionAlgorithm = CorrectionAlgorithm.ReedSolomon8;
int depth = 0;
var groupList = new List <Group>();
// File
using (var stream = new UnbufferedFileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferManager))
{
if (stream.Length <= blockLength)
{
Hash hash;
using (var safeBuffer = _bufferManager.CreateSafeBuffer(blockLength))
{
int length = (int)stream.Length;
stream.Read(safeBuffer.Value, 0, length);
if (hashAlgorithm == HashAlgorithm.Sha256)
{
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(safeBuffer.Value, 0, length));
}
}
shareInfo = new ShareInfo(path, stream.Length, (int)stream.Length, new Hash[] { hash });
metadata = new Metadata(depth, hash);
}
else
{
var sharedHashes = new List <Hash>();
for (; ;)
{
var targetHashes = new List <Hash>();
var targetBuffers = new List <ArraySegment <byte> >();
long sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
var buffer = new ArraySegment <byte>();
try
{
int length = (int)Math.Min(stream.Length - stream.Position, blockLength);
buffer = new ArraySegment <byte>(_bufferManager.TakeBuffer(length), 0, length);
stream.Read(buffer.Array, 0, length);
sumLength += length;
}
catch (Exception)
{
if (buffer.Array != null)
{
_bufferManager.ReturnBuffer(buffer.Array);
}
throw;
}
Hash hash;
if (hashAlgorithm == HashAlgorithm.Sha256)
{
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(buffer));
}
sharedHashes.Add(hash);
targetHashes.Add(hash);
targetBuffers.Add(buffer);
if (targetBuffers.Count >= 128)
{
break;
}
}
var parityHashes = this.ParityEncoding(targetBuffers, hashAlgorithm, correctionAlgorithm, token);
lockedHashes.UnionWith(parityHashes);
groupList.Add(new Group(correctionAlgorithm, sumLength, CollectionUtils.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var buffer in targetBuffers)
{
if (buffer.Array == null)
{
continue;
}
_bufferManager.ReturnBuffer(buffer.Array);
}
}
if (stream.Position == stream.Length)
{
break;
}
}
shareInfo = new ShareInfo(path, stream.Length, blockLength, sharedHashes);
depth++;
}
}
while (groupList.Count > 0)
{
// Index
using (var stream = (new Index(groupList)).Export(_bufferManager))
{
groupList.Clear();
if (stream.Length <= blockLength)
{
Hash hash;
using (var safeBuffer = _bufferManager.CreateSafeBuffer(blockLength))
{
int length = (int)stream.Length;
stream.Read(safeBuffer.Value, 0, length);
if (hashAlgorithm == HashAlgorithm.Sha256)
{
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(safeBuffer.Value, 0, length));
}
_blocksManager.Lock(hash);
_blocksManager.Set(hash, new ArraySegment <byte>(safeBuffer.Value, 0, length));
lockedHashes.Add(hash);
}
metadata = new Metadata(depth, hash);
}
else
{
for (; ;)
{
var targetHashes = new List <Hash>();
var targetBuffers = new List <ArraySegment <byte> >();
long sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
var buffer = new ArraySegment <byte>();
try
{
int length = (int)Math.Min(stream.Length - stream.Position, blockLength);
buffer = new ArraySegment <byte>(_bufferManager.TakeBuffer(length), 0, length);
stream.Read(buffer.Array, 0, length);
sumLength += length;
}
catch (Exception)
{
if (buffer.Array != null)
{
_bufferManager.ReturnBuffer(buffer.Array);
}
throw;
}
Hash hash;
if (hashAlgorithm == HashAlgorithm.Sha256)
{
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(buffer));
}
_blocksManager.Lock(hash);
_blocksManager.Set(hash, buffer);
lockedHashes.Add(hash);
targetHashes.Add(hash);
targetBuffers.Add(buffer);
if (targetBuffers.Count >= 128)
{
break;
}
}
var parityHashes = this.ParityEncoding(targetBuffers, hashAlgorithm, correctionAlgorithm, token);
lockedHashes.UnionWith(parityHashes);
groupList.Add(new Group(correctionAlgorithm, sumLength, CollectionUtils.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var buffer in targetBuffers)
{
if (buffer.Array == null)
{
continue;
}
_bufferManager.ReturnBuffer(buffer.Array);
}
}
if (stream.Position == stream.Length)
{
break;
}
}
depth++;
}
}
}
}
lock (_lockObject)
{
if (!_contentInfoManager.ContainsFileContentInfo(path))
{
_contentInfoManager.Add(new ContentInfo(creationTime, Timeout.InfiniteTimeSpan, metadata, lockedHashes, shareInfo));
foreach (var hash in lockedHashes)
{
_blocksManager.Lock(hash);
}
}
}
return metadata;
}, token));
}
public async ValueTask <XeusClue> Import(string path, CancellationToken token = default)
{
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
return(await Task.Run(async() =>
{
// Check
lock (_lockObject)
{
var info = _contentMetadataStorage.GetFileContentMetadata(path);
if (info != null)
{
return info.Clue;
}
}
XeusClue?clue = null;
var lockedHashes = new HashSet <OmniHash>();
SharedBlocksMetadata?sharedBlocksInfo = null;
{
const int blockLength = 1024 * 1024;
const OmniHashAlgorithmType hashAlgorithmType = OmniHashAlgorithmType.Sha2_256;
const CorrectionAlgorithmType correctionAlgorithmType = CorrectionAlgorithmType.ReedSolomon8;
byte depth = 0;
var merkleTreeSectionList = new List <MerkleTreeSection>();
// File
using (var stream = new UnbufferedFileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferPool))
{
if (stream.Length <= blockLength)
{
OmniHash hash;
using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
{
stream.Read(bufferMemoryOwner.Memory.Span);
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
}
sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, (uint)stream.Length, new OmniHash[] { hash });
clue = new XeusClue(hash, depth);
}
else
{
var sharedHashes = new List <OmniHash>();
for (; ;)
{
var targetHashes = new List <OmniHash>();
var targetMemoryOwners = new List <IMemoryOwner <byte> >();
ulong sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
var bufferMemoryOwner = _bufferPool.Rent((int)length);
try
{
stream.Read(bufferMemoryOwner.Memory.Span);
sumLength += length;
}
catch (Exception e)
{
bufferMemoryOwner.Dispose();
throw e;
}
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
sharedHashes.Add(hash);
targetHashes.Add(hash);
targetMemoryOwners.Add(bufferMemoryOwner);
if (targetMemoryOwners.Count >= 128)
{
break;
}
}
var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
lockedHashes.UnionWith(parityHashes);
merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var memoryOwner in targetMemoryOwners)
{
memoryOwner.Dispose();
}
}
if (stream.Position == stream.Length)
{
break;
}
}
sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, blockLength, sharedHashes.ToArray());
depth++;
}
}
while (merkleTreeSectionList.Count > 0)
{
// Index
using (var stream = new RecyclableMemoryStream(_bufferPool))
{
RocketPackHelper.MessageToStream(new MerkleTreeNode(merkleTreeSectionList.ToArray()), stream);
stream.Seek(0, SeekOrigin.Begin);
merkleTreeSectionList.Clear();
if (stream.Length <= blockLength)
{
OmniHash hash;
using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
{
stream.Read(bufferMemoryOwner.Memory.Span);
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
lockedHashes.Add(hash);
}
clue = new XeusClue(hash, depth);
}
else
{
for (; ;)
{
var targetHashes = new List <OmniHash>();
var targetMemoryOwners = new List <IMemoryOwner <byte> >();
ulong sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
var bufferMemoryOwner = _bufferPool.Rent((int)length);
try
{
stream.Read(bufferMemoryOwner.Memory.Span);
sumLength += length;
}
catch (Exception e)
{
bufferMemoryOwner.Dispose();
throw e;
}
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
lockedHashes.Add(hash);
targetHashes.Add(hash);
targetMemoryOwners.Add(bufferMemoryOwner);
if (targetMemoryOwners.Count >= 128)
{
break;
}
}
var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
lockedHashes.UnionWith(parityHashes);
merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var memoryOwner in targetMemoryOwners)
{
memoryOwner.Dispose();
}
}
if (stream.Position == stream.Length)
{
break;
}
}
depth++;
}
}
}
}
if (clue == null)
{
throw new ImportFailed("clue is null");
}
lock (_lockObject)
{
if (!_contentMetadataStorage.ContainsFileContentMetadata(path))
{
_contentMetadataStorage.Add(new ContentMetadata(clue, lockedHashes.ToArray(), sharedBlocksInfo));
foreach (var hash in lockedHashes)
{
_blockStorage.Lock(hash);
}
}
}
return clue;
}, token));
}
public ArraySegment <byte> GetBlock(Hash hash)
{
// Cache
{
var result = _blocksManager.Get(hash);
if (result != null)
{
return(result.Value);
}
}
// Share
{
ArraySegment <byte>?result = null;
string path = null;
lock (_lockObject)
{
var shareInfo = _contentInfoManager.GetShareInfo(hash);
if (shareInfo != null)
{
var buffer = _bufferManager.TakeBuffer(shareInfo.BlockLength);
try
{
int length;
try
{
using (var stream = new UnbufferedFileStream(shareInfo.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferManager))
{
stream.Seek((long)shareInfo.GetIndex(hash) * shareInfo.BlockLength, SeekOrigin.Begin);
length = (int)Math.Min(stream.Length - stream.Position, shareInfo.BlockLength);
stream.Read(buffer, 0, length);
}
}
catch (ArgumentOutOfRangeException)
{
throw new BlockNotFoundException();
}
catch (IOException)
{
throw new BlockNotFoundException();
}
result = new ArraySegment <byte>(buffer, 0, length);
path = shareInfo.Path;
}
catch (Exception)
{
_bufferManager.ReturnBuffer(buffer);
throw;
}
}
}
if (result != null)
{
if (hash.Algorithm == HashAlgorithm.Sha256 &&
Unsafe.Equals(Sha256.Compute(result.Value), hash.Value))
{
return(result.Value);
}
else
{
_bufferManager.ReturnBuffer(result.Value.Array);
result = null;
this.RemoveContent(path);
}
}
}
throw new BlockNotFoundException();
}
public bool TryGetBlock(OmniHash hash, [NotNullWhen(true)] out IMemoryOwner <byte>?memoryOwner)
{
if (!EnumHelper.IsValid(hash.AlgorithmType))
{
throw new ArgumentException($"Incorrect HashAlgorithmType: {hash.AlgorithmType}");
}
// Cache
{
var result = _blockStorage.TryGet(hash, out memoryOwner);
if (result)
{
return(true);
}
}
bool success = false;
string?path = null;
// Share
try
{
lock (_lockObject)
{
var sharedBlocksInfo = _contentMetadataStorage.GetSharedBlocksInfo(hash);
if (sharedBlocksInfo != null)
{
ulong position = (ulong)sharedBlocksInfo.GetIndex(hash) * sharedBlocksInfo.BlockLength;
uint length = (uint)Math.Min(sharedBlocksInfo.Length - position, sharedBlocksInfo.BlockLength);
memoryOwner = _bufferPool.Rent((int)length);
try
{
using (var stream = new UnbufferedFileStream(sharedBlocksInfo.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferPool))
{
stream.Seek((long)position, SeekOrigin.Begin);
stream.Read(memoryOwner.Memory.Span);
}
path = sharedBlocksInfo.Path;
}
catch (Exception e)
{
_logger.Debug(e);
return(false);
}
}
}
if (memoryOwner == null)
{
return(false);
}
if (hash.AlgorithmType == OmniHashAlgorithmType.Sha2_256 &&
BytesOperations.SequenceEqual(Sha2_256.ComputeHash(memoryOwner.Memory.Span), hash.Value.Span))
{
success = true;
return(true);
}
else
{
_logger.Warn("Broken block.");
return(false);
}
}
finally
{
if (!success)
{
if (memoryOwner != null)
{
memoryOwner.Dispose();
memoryOwner = null;
}
if (path != null)
{
this.RemoveContent(path);
}
}
}
}