public ContentMetadata?GetMessageContentMetadata(XeusClue clue)
{
if (!_messageContentMetadatas.TryGetValue(clue, out var contentInfo))
{
return(null);
}
return(contentInfo);
}
public ContentMetadata Deserialize(global::Omnix.Serialization.RocketPack.RocketPackReader r, int rank)
{
if (rank > 256)
{
throw new global::System.FormatException();
}
uint propertyCount = r.GetUInt32();
XeusClue p_clue = XeusClue.Empty;
OmniHash[] p_lockedHashes = global::System.Array.Empty <OmniHash>();
SharedBlocksMetadata?p_sharedBlocksMetadata = null;
for (; propertyCount > 0; propertyCount--)
{
uint id = r.GetUInt32();
switch (id)
{
case 0:
{
p_clue = XeusClue.Formatter.Deserialize(r, rank + 1);
break;
}
case 1:
{
var length = r.GetUInt32();
p_lockedHashes = new OmniHash[length];
for (int i = 0; i < p_lockedHashes.Length; i++)
{
p_lockedHashes[i] = OmniHash.Formatter.Deserialize(r, rank + 1);
}
break;
}
case 2:
{
p_sharedBlocksMetadata = SharedBlocksMetadata.Formatter.Deserialize(r, rank + 1);
break;
}
}
}
return(new ContentMetadata(p_clue, p_lockedHashes, p_sharedBlocksMetadata));
}
public ContentMetadata(XeusClue clue, OmniHash[] lockedHashes, SharedBlocksMetadata?sharedBlocksMetadata)
{
if (clue is null)
{
throw new global::System.ArgumentNullException("clue");
}
if (lockedHashes is null)
{
throw new global::System.ArgumentNullException("lockedHashes");
}
if (lockedHashes.Length > 1073741824)
{
throw new global::System.ArgumentOutOfRangeException("lockedHashes");
}
this.Clue = clue;
this.LockedHashes = new global::Omnix.DataStructures.ReadOnlyListSlim <OmniHash>(lockedHashes);
this.SharedBlocksMetadata = sharedBlocksMetadata;
{
var __h = new global::System.HashCode();
if (this.Clue != default)
{
__h.Add(this.Clue.GetHashCode());
}
foreach (var n in this.LockedHashes)
{
if (n != default)
{
__h.Add(n.GetHashCode());
}
}
if (this.SharedBlocksMetadata != default)
{
__h.Add(this.SharedBlocksMetadata.GetHashCode());
}
__hashCode = __h.ToHashCode();
}
}
public void RemoveMessage(XeusClue clue)
{
lock (_lockObject)
{
var contentInfo = _contentMetadataStorage.GetMessageContentMetadata(clue);
if (contentInfo == null)
{
return;
}
_contentMetadataStorage.RemoveMessageContentMetadata(clue);
foreach (var hash in contentInfo.LockedHashes)
{
_blockStorage.Unlock(hash);
}
if (contentInfo.SharedBlocksMetadata != null)
{
// Event
_removedBlockEventQueue.Enqueue(contentInfo.SharedBlocksMetadata.Hashes.Where(n => !this.Contains(n)).ToArray());
}
}
}
public bool ContainsMessageContentMetadata(XeusClue clue)
{
return(_messageContentMetadatas.ContainsKey(clue));
}
public void RemoveMessageContentMetadata(XeusClue clue)
{
_messageContentMetadatas.Remove(clue);
}
public async ValueTask <XeusClue> Import(string path, CancellationToken token = default)
{
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
return(await Task.Run(async() =>
{
// Check
lock (_lockObject)
{
var info = _contentMetadataStorage.GetFileContentMetadata(path);
if (info != null)
{
return info.Clue;
}
}
XeusClue?clue = null;
var lockedHashes = new HashSet <OmniHash>();
SharedBlocksMetadata?sharedBlocksInfo = null;
{
const int blockLength = 1024 * 1024;
const OmniHashAlgorithmType hashAlgorithmType = OmniHashAlgorithmType.Sha2_256;
const CorrectionAlgorithmType correctionAlgorithmType = CorrectionAlgorithmType.ReedSolomon8;
byte depth = 0;
var merkleTreeSectionList = new List <MerkleTreeSection>();
// File
using (var stream = new UnbufferedFileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferPool))
{
if (stream.Length <= blockLength)
{
OmniHash hash;
using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
{
stream.Read(bufferMemoryOwner.Memory.Span);
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
}
sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, (uint)stream.Length, new OmniHash[] { hash });
clue = new XeusClue(hash, depth);
}
else
{
var sharedHashes = new List <OmniHash>();
for (; ;)
{
var targetHashes = new List <OmniHash>();
var targetMemoryOwners = new List <IMemoryOwner <byte> >();
ulong sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
var bufferMemoryOwner = _bufferPool.Rent((int)length);
try
{
stream.Read(bufferMemoryOwner.Memory.Span);
sumLength += length;
}
catch (Exception e)
{
bufferMemoryOwner.Dispose();
throw e;
}
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
sharedHashes.Add(hash);
targetHashes.Add(hash);
targetMemoryOwners.Add(bufferMemoryOwner);
if (targetMemoryOwners.Count >= 128)
{
break;
}
}
var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
lockedHashes.UnionWith(parityHashes);
merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var memoryOwner in targetMemoryOwners)
{
memoryOwner.Dispose();
}
}
if (stream.Position == stream.Length)
{
break;
}
}
sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, blockLength, sharedHashes.ToArray());
depth++;
}
}
while (merkleTreeSectionList.Count > 0)
{
// Index
using (var stream = new RecyclableMemoryStream(_bufferPool))
{
RocketPackHelper.MessageToStream(new MerkleTreeNode(merkleTreeSectionList.ToArray()), stream);
stream.Seek(0, SeekOrigin.Begin);
merkleTreeSectionList.Clear();
if (stream.Length <= blockLength)
{
OmniHash hash;
using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
{
stream.Read(bufferMemoryOwner.Memory.Span);
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
lockedHashes.Add(hash);
}
clue = new XeusClue(hash, depth);
}
else
{
for (; ;)
{
var targetHashes = new List <OmniHash>();
var targetMemoryOwners = new List <IMemoryOwner <byte> >();
ulong sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
var bufferMemoryOwner = _bufferPool.Rent((int)length);
try
{
stream.Read(bufferMemoryOwner.Memory.Span);
sumLength += length;
}
catch (Exception e)
{
bufferMemoryOwner.Dispose();
throw e;
}
OmniHash hash;
if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
{
hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
}
_blockStorage.Lock(hash);
bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);
if (!result)
{
throw new ImportFailed("Failed to save Block.");
}
lockedHashes.Add(hash);
targetHashes.Add(hash);
targetMemoryOwners.Add(bufferMemoryOwner);
if (targetMemoryOwners.Count >= 128)
{
break;
}
}
var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
lockedHashes.UnionWith(parityHashes);
merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var memoryOwner in targetMemoryOwners)
{
memoryOwner.Dispose();
}
}
if (stream.Position == stream.Length)
{
break;
}
}
depth++;
}
}
}
}
if (clue == null)
{
throw new ImportFailed("clue is null");
}
lock (_lockObject)
{
if (!_contentMetadataStorage.ContainsFileContentMetadata(path))
{
_contentMetadataStorage.Add(new ContentMetadata(clue, lockedHashes.ToArray(), sharedBlocksInfo));
foreach (var hash in lockedHashes)
{
_blockStorage.Lock(hash);
}
}
}
return clue;
}, token));
}
