private IEnumerable <long> GetFreeSectors(int count)
{
using (_lockManager.WriteLock())
{
if (_sectorsManager.FreeSectorCount >= count)
{
return(_sectorsManager.TakeFreeSectors(count));
}
else
{
var removePairs = _clusterIndex
.Where(n => !_protectionManager.Contains(n.Key))
.ToList();
removePairs.Sort((x, y) =>
{
return(x.Value.UpdateTime.CompareTo(y.Value.UpdateTime));
});
foreach (var hash in removePairs.Select(n => n.Key))
{
this.Remove(hash);
if (_sectorsManager.FreeSectorCount >= 1024 * 4)
{
break;
}
}
if (_sectorsManager.FreeSectorCount < count)
{
throw new SpaceNotFoundException();
}
return(_sectorsManager.TakeFreeSectors(count));
}
}
}
public Task <Metadata> Import(Stream stream, TimeSpan lifeSpan, CancellationToken token)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
return(Task.Run(() =>
{
Metadata metadata = null;
var lockedHashes = new HashSet <Hash>();
try
{
const int blockLength = 1024 * 1024;
const HashAlgorithm hashAlgorithm = HashAlgorithm.Sha256;
const CorrectionAlgorithm correctionAlgorithm = CorrectionAlgorithm.ReedSolomon8;
int depth = 0;
var creationTime = DateTime.UtcNow;
var groupList = new List <Group>();
for (; ;)
{
if (stream.Length <= blockLength)
{
Hash hash;
using (var safeBuffer = _bufferManager.CreateSafeBuffer(blockLength))
{
int length = (int)stream.Length;
stream.Read(safeBuffer.Value, 0, length);
if (hashAlgorithm == HashAlgorithm.Sha256)
{
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(safeBuffer.Value, 0, length));
}
_blocksManager.Lock(hash);
_blocksManager.Set(hash, new ArraySegment <byte>(safeBuffer.Value, 0, length));
lockedHashes.Add(hash);
}
// Stream Dispose
{
stream.Dispose();
stream = null;
}
metadata = new Metadata(depth, hash);
break;
}
else
{
for (; ;)
{
var targetHashes = new List <Hash>();
var targetBuffers = new List <ArraySegment <byte> >();
long sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
var buffer = new ArraySegment <byte>();
try
{
int length = (int)Math.Min(stream.Length - stream.Position, blockLength);
buffer = new ArraySegment <byte>(_bufferManager.TakeBuffer(length), 0, length);
stream.Read(buffer.Array, 0, length);
sumLength += length;
}
catch (Exception)
{
if (buffer.Array != null)
{
_bufferManager.ReturnBuffer(buffer.Array);
}
throw;
}
Hash hash;
if (hashAlgorithm == HashAlgorithm.Sha256)
{
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(buffer));
}
_blocksManager.Lock(hash);
_blocksManager.Set(hash, buffer);
lockedHashes.Add(hash);
targetHashes.Add(hash);
targetBuffers.Add(buffer);
if (targetBuffers.Count >= 128)
{
break;
}
}
var parityHashes = this.ParityEncoding(targetBuffers, hashAlgorithm, correctionAlgorithm, token);
lockedHashes.UnionWith(parityHashes);
groupList.Add(new Group(correctionAlgorithm, sumLength, CollectionUtils.Unite(targetHashes, parityHashes)));
}
finally
{
foreach (var buffer in targetBuffers)
{
if (buffer.Array == null)
{
continue;
}
_bufferManager.ReturnBuffer(buffer.Array);
}
}
if (stream.Position == stream.Length)
{
break;
}
}
depth++;
// Stream Dispose
{
stream.Dispose();
stream = null;
}
stream = (new Index(groupList)).Export(_bufferManager);
}
}
}
finally
{
if (stream != null)
{
stream.Dispose();
stream = null;
}
}
using (_lockManager.WriteLock())
{
if (!_contentInfoManager.ContainsMessageContentInfo(metadata))
{
_contentInfoManager.Add(new ContentInfo(DateTime.UtcNow, lifeSpan, metadata, lockedHashes, null));
foreach (var hash in lockedHashes)
{
_blocksManager.Lock(hash);
}
}
}
return metadata;
}, token));
}
public void Refresh()
{
using (_lockManager.WriteLock())
{
var lockSignatures = new HashSet <Signature>(this.OnGetLockSignaturesEvent());
var lockTags = new HashSet <Tag>(_aliveTags);
_aliveTags.Update();
// Broadcast
{
{
var removeTypes = new HashSet <string>(_broadcastTypes.OrderBy(n => n.Value).Select(n => n.Key).Take(_broadcastTypes.Count - 32));
foreach (string type in removeTypes)
{
_broadcastTypes.Remove(type);
}
foreach (string type in _broadcastMetadatas.Keys.ToArray())
{
if (!removeTypes.Contains(type))
{
continue;
}
_broadcastMetadatas.Remove(type);
}
}
foreach (var dic in _broadcastMetadatas.Values)
{
var keys = dic.Keys.Where(n => !lockSignatures.Contains(n)).ToList();
foreach (var key in keys.Randomize().Take(keys.Count - 1024))
{
dic.Remove(key);
}
}
}
// Unicast
{
{
var removeTypes = new HashSet <string>(_unicastTypes.OrderBy(n => n.Value).Select(n => n.Key).Take(_unicastTypes.Count - 32));
foreach (string type in removeTypes)
{
_unicastTypes.Remove(type);
}
foreach (string type in _unicastMetadatas.Keys.ToArray())
{
if (!removeTypes.Contains(type))
{
continue;
}
_unicastMetadatas.Remove(type);
}
}
foreach (var dic in _unicastMetadatas.Values)
{
var keys = dic.Keys.Where(n => !lockSignatures.Contains(n)).ToList();
foreach (var key in keys.Randomize().Take(keys.Count - 1024))
{
dic.Remove(key);
}
}
foreach (var dic in _unicastMetadatas.Values.SelectMany(n => n.Values))
{
var keys = dic.Keys.Where(n => !lockSignatures.Contains(n)).ToList();
foreach (var key in keys.Randomize().Take(keys.Count - 32))
{
dic.Remove(key);
}
}
foreach (var hashset in _unicastMetadatas.Values.SelectMany(n => n.Values.SelectMany(m => m.Values)).ToArray())
{
if (hashset.Count <= 32)
{
continue;
}
var list = hashset.ToList();
list.Sort((x, y) => x.CreationTime.CompareTo(y.CreationTime));
foreach (var value in list.Take(list.Count - 32))
{
hashset.Remove(value);
}
}
}
// Multicast
{
{
var removeTypes = new HashSet <string>(_multicastTypes.OrderBy(n => n.Value).Select(n => n.Key).Take(_multicastTypes.Count - 32));
foreach (string type in removeTypes)
{
_multicastTypes.Remove(type);
}
foreach (string type in _multicastMetadatas.Keys.ToArray())
{
if (!removeTypes.Contains(type))
{
continue;
}
_multicastMetadatas.Remove(type);
}
}
foreach (var dic in _multicastMetadatas.Values)
{
var keys = dic.Keys.Where(n => !lockTags.Contains(n)).ToList();
foreach (var key in keys.Randomize().Take(keys.Count - 1024))
{
dic.Remove(key);
}
}
foreach (var dic in _multicastMetadatas.Values.SelectMany(n => n.Values))
{
var keys = dic.Keys.Where(n => !lockSignatures.Contains(n)).ToList();
foreach (var key in keys.Randomize().Take(keys.Count - 32))
{
dic.Remove(key);
}
}
foreach (var hashset in _multicastMetadatas.Values.SelectMany(n => n.Values.SelectMany(m => m.Values)).ToArray())
{
if (hashset.Count <= 32)
{
continue;
}
var list = hashset.ToList();
list.Sort((x, y) => x.CreationTime.CompareTo(y.CreationTime));
foreach (var value in list.Take(list.Count - 32))
{
hashset.Remove(value);
}
}
}
}
}
