private IEnumerable <long> GetFreeSectors(int count) { using (_lockManager.WriteLock()) { if (_sectorsManager.FreeSectorCount >= count) { return(_sectorsManager.TakeFreeSectors(count)); } else { var removePairs = _clusterIndex .Where(n => !_protectionManager.Contains(n.Key)) .ToList(); removePairs.Sort((x, y) => { return(x.Value.UpdateTime.CompareTo(y.Value.UpdateTime)); }); foreach (var hash in removePairs.Select(n => n.Key)) { this.Remove(hash); if (_sectorsManager.FreeSectorCount >= 1024 * 4) { break; } } if (_sectorsManager.FreeSectorCount < count) { throw new SpaceNotFoundException(); } return(_sectorsManager.TakeFreeSectors(count)); } } }
public Task <Metadata> Import(Stream stream, TimeSpan lifeSpan, CancellationToken token) { if (stream == null) { throw new ArgumentNullException(nameof(stream)); } return(Task.Run(() => { Metadata metadata = null; var lockedHashes = new HashSet <Hash>(); try { const int blockLength = 1024 * 1024; const HashAlgorithm hashAlgorithm = HashAlgorithm.Sha256; const CorrectionAlgorithm correctionAlgorithm = CorrectionAlgorithm.ReedSolomon8; int depth = 0; var creationTime = DateTime.UtcNow; var groupList = new List <Group>(); for (; ;) { if (stream.Length <= blockLength) { Hash hash; using (var safeBuffer = _bufferManager.CreateSafeBuffer(blockLength)) { int length = (int)stream.Length; stream.Read(safeBuffer.Value, 0, length); if (hashAlgorithm == HashAlgorithm.Sha256) { hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(safeBuffer.Value, 0, length)); } _blocksManager.Lock(hash); _blocksManager.Set(hash, new ArraySegment <byte>(safeBuffer.Value, 0, length)); lockedHashes.Add(hash); } // Stream Dispose { stream.Dispose(); stream = null; } metadata = new Metadata(depth, hash); break; } else { for (; ;) { var targetHashes = new List <Hash>(); var targetBuffers = new List <ArraySegment <byte> >(); long sumLength = 0; try { for (int i = 0; stream.Position < stream.Length; i++) { token.ThrowIfCancellationRequested(); var buffer = new ArraySegment <byte>(); try { int length = (int)Math.Min(stream.Length - stream.Position, blockLength); buffer = new ArraySegment <byte>(_bufferManager.TakeBuffer(length), 0, length); stream.Read(buffer.Array, 0, length); sumLength += length; } catch (Exception) { if (buffer.Array != null) { _bufferManager.ReturnBuffer(buffer.Array); } throw; } Hash hash; if (hashAlgorithm == HashAlgorithm.Sha256) { hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(buffer)); } _blocksManager.Lock(hash); _blocksManager.Set(hash, buffer); lockedHashes.Add(hash); targetHashes.Add(hash); targetBuffers.Add(buffer); if (targetBuffers.Count >= 128) { break; } } var parityHashes = this.ParityEncoding(targetBuffers, hashAlgorithm, correctionAlgorithm, token); lockedHashes.UnionWith(parityHashes); groupList.Add(new Group(correctionAlgorithm, sumLength, CollectionUtils.Unite(targetHashes, parityHashes))); } finally { foreach (var buffer in targetBuffers) { if (buffer.Array == null) { continue; } _bufferManager.ReturnBuffer(buffer.Array); } } if (stream.Position == stream.Length) { break; } } depth++; // Stream Dispose { stream.Dispose(); stream = null; } stream = (new Index(groupList)).Export(_bufferManager); } } } finally { if (stream != null) { stream.Dispose(); stream = null; } } using (_lockManager.WriteLock()) { if (!_contentInfoManager.ContainsMessageContentInfo(metadata)) { _contentInfoManager.Add(new ContentInfo(DateTime.UtcNow, lifeSpan, metadata, lockedHashes, null)); foreach (var hash in lockedHashes) { _blocksManager.Lock(hash); } } } return metadata; }, token)); }
public void Refresh() { using (_lockManager.WriteLock()) { var lockSignatures = new HashSet <Signature>(this.OnGetLockSignaturesEvent()); var lockTags = new HashSet <Tag>(_aliveTags); _aliveTags.Update(); // Broadcast { { var removeTypes = new HashSet <string>(_broadcastTypes.OrderBy(n => n.Value).Select(n => n.Key).Take(_broadcastTypes.Count - 32)); foreach (string type in removeTypes) { _broadcastTypes.Remove(type); } foreach (string type in _broadcastMetadatas.Keys.ToArray()) { if (!removeTypes.Contains(type)) { continue; } _broadcastMetadatas.Remove(type); } } foreach (var dic in _broadcastMetadatas.Values) { var keys = dic.Keys.Where(n => !lockSignatures.Contains(n)).ToList(); foreach (var key in keys.Randomize().Take(keys.Count - 1024)) { dic.Remove(key); } } } // Unicast { { var removeTypes = new HashSet <string>(_unicastTypes.OrderBy(n => n.Value).Select(n => n.Key).Take(_unicastTypes.Count - 32)); foreach (string type in removeTypes) { _unicastTypes.Remove(type); } foreach (string type in _unicastMetadatas.Keys.ToArray()) { if (!removeTypes.Contains(type)) { continue; } _unicastMetadatas.Remove(type); } } foreach (var dic in _unicastMetadatas.Values) { var keys = dic.Keys.Where(n => !lockSignatures.Contains(n)).ToList(); foreach (var key in keys.Randomize().Take(keys.Count - 1024)) { dic.Remove(key); } } foreach (var dic in _unicastMetadatas.Values.SelectMany(n => n.Values)) { var keys = dic.Keys.Where(n => !lockSignatures.Contains(n)).ToList(); foreach (var key in keys.Randomize().Take(keys.Count - 32)) { dic.Remove(key); } } foreach (var hashset in _unicastMetadatas.Values.SelectMany(n => n.Values.SelectMany(m => m.Values)).ToArray()) { if (hashset.Count <= 32) { continue; } var list = hashset.ToList(); list.Sort((x, y) => x.CreationTime.CompareTo(y.CreationTime)); foreach (var value in list.Take(list.Count - 32)) { hashset.Remove(value); } } } // Multicast { { var removeTypes = new HashSet <string>(_multicastTypes.OrderBy(n => n.Value).Select(n => n.Key).Take(_multicastTypes.Count - 32)); foreach (string type in removeTypes) { _multicastTypes.Remove(type); } foreach (string type in _multicastMetadatas.Keys.ToArray()) { if (!removeTypes.Contains(type)) { continue; } _multicastMetadatas.Remove(type); } } foreach (var dic in _multicastMetadatas.Values) { var keys = dic.Keys.Where(n => !lockTags.Contains(n)).ToList(); foreach (var key in keys.Randomize().Take(keys.Count - 1024)) { dic.Remove(key); } } foreach (var dic in _multicastMetadatas.Values.SelectMany(n => n.Values)) { var keys = dic.Keys.Where(n => !lockSignatures.Contains(n)).ToList(); foreach (var key in keys.Randomize().Take(keys.Count - 32)) { dic.Remove(key); } } foreach (var hashset in _multicastMetadatas.Values.SelectMany(n => n.Values.SelectMany(m => m.Values)).ToArray()) { if (hashset.Count <= 32) { continue; } var list = hashset.ToList(); list.Sort((x, y) => x.CreationTime.CompareTo(y.CreationTime)); foreach (var value in list.Take(list.Count - 32)) { hashset.Remove(value); } } } } }