private void EventLoop(CancellationToken token) { try { for (; ;) { token.ThrowIfCancellationRequested(); long length = (long)Varint.GetUInt64(_stream); if (length == 0) { continue; } Stream resultSteram = null; try { resultSteram = new RecyclableMemoryStream(_bufferManager); using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 32)) { long remain = length; while (remain > 0) { int readLength = _stream.Read(safeBuffer.Value, 0, (int)Math.Min(remain, safeBuffer.Value.Length)); resultSteram.Write(safeBuffer.Value, 0, readLength); remain -= readLength; } } resultSteram.Seek(0, SeekOrigin.Begin); this.OnReceive(resultSteram); } catch (Exception) { if (resultSteram != null) { resultSteram.Dispose(); resultSteram = null; } } } } catch (Exception) { } }
public void MessageUpload() { _callback.Invoke("----- CoreManager Message Upload Test (Start) -----"); _callback.Invoke(""); var random = RandomProvider.GetThreadRandom(); _coreManager.Resize((long)1024 * 1024 * 1024 * 32).Wait(); Metadata metadata = null; Hash hash; using (var stream = new RecyclableMemoryStream(_bufferManager)) { using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4)) { for (long remain = (long)1024 * 1024 * 256; 0 < remain; remain -= safeBuffer.Value.Length) { int length = (int)Math.Min(remain, safeBuffer.Value.Length); random.NextBytes(safeBuffer.Value); stream.Write(safeBuffer.Value, 0, length); } } stream.Seek(0, SeekOrigin.Begin); hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(new WrapperStream(stream))); stream.Seek(0, SeekOrigin.Begin); metadata = _coreManager.VolatileSetStream(stream, new TimeSpan(1, 0, 0, 0), CancellationToken.None).Result; } using (var stream = metadata.Export(_bufferManager)) using (var safeBuffer = _bufferManager.CreateSafeBuffer((int)stream.Length)) { stream.Read(safeBuffer.Value, 0, (int)stream.Length); Console.WriteLine(NetworkConverter.ToBase64UrlString(safeBuffer.Value, 0, (int)stream.Length)); } using (var stream = hash.Export(_bufferManager)) using (var safeBuffer = _bufferManager.CreateSafeBuffer((int)stream.Length)) { stream.Read(safeBuffer.Value, 0, (int)stream.Length); Console.WriteLine(NetworkConverter.ToBase64UrlString(safeBuffer.Value, 0, (int)stream.Length)); } _callback.Invoke("----- CoreManager Message Upload Test (End) -----"); }
public void SetLength(long length) { lock (_lockObject) { { long size = BitmapManager.Roundup(length, 8); _bitmapStream.SetLength(size); _bitmapStream.Seek(0, SeekOrigin.Begin); { using (var safeBuffer = _bufferManager.CreateSafeBuffer(4096)) { Unsafe.Zero(safeBuffer.Value); for (long i = (size / safeBuffer.Value.Length), remain = size; i >= 0; i--, remain -= safeBuffer.Value.Length) { _bitmapStream.Write(safeBuffer.Value, 0, (int)Math.Min(remain, safeBuffer.Value.Length)); _bitmapStream.Flush(); } } } } _length = length; { _cacheChanged = false; _cacheSector = -1; _cacheBufferLength = 0; } } }
private static Stream GetStream(string url, IWebProxy proxy) { BufferManager bufferManager = BufferManager.Instance; for (int i = 0; i < 10; i++) { var bufferStream = new BufferStream(bufferManager); try { var request = (HttpWebRequest)WebRequest.Create(url); request.AllowAutoRedirect = true; request.Proxy = proxy; request.Headers.Add("Pragma", "no-cache"); request.Headers.Add("Cache-Control", "no-cache"); request.Timeout = 1000 * 60 * 5; request.ReadWriteTimeout = 1000 * 60 * 5; using (WebResponse response = request.GetResponse()) { if (response.ContentLength > 1024 * 1024 * 32) { throw new Exception("too large"); } using (Stream stream = response.GetResponseStream()) using (var safeBuffer = bufferManager.CreateSafeBuffer(1024 * 4)) { int length; while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0) { bufferStream.Write(safeBuffer.Value, 0, length); if (bufferStream.Length > 1024 * 1024 * 32) { throw new Exception("too large"); } } } if (response.ContentLength != -1 && bufferStream.Length != response.ContentLength) { continue; } bufferStream.Seek(0, SeekOrigin.Begin); return(bufferStream); } } catch (Exception) { bufferStream.Dispose(); } } throw new Exception(string.Format("not found: {0}", url)); }
public void ReadWriteTest() { for (int i = 0; i < 256; i++) { int size = _random.Next(1, 1024 * 1024 * 4); using (var safeBuffer = _bufferManager.CreateSafeBuffer(size)) { _random.NextBytes(safeBuffer.Value); var block = new ArraySegment <byte>(safeBuffer.Value, 0, size); var hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(block)); _cacheManager.Set(hash, block); var result = _cacheManager.GetBlock(hash); Assert.True(Unsafe.Equals(block.Array, block.Offset, result.Array, result.Offset, size)); _bufferManager.ReturnBuffer(result.Array); } } }
private Stream GetStream(string url) { var recyclableMemoryStream = new RecyclableMemoryStream(_bufferManager); try { using (var client = new HttpClient()) { using (var stream = client.GetStreamAsync(url).Result) { if (stream.Length > 1024 * 1024 * 32) { throw new Exception("too large"); } using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4)) { int length; while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0) { recyclableMemoryStream.Write(safeBuffer.Value, 0, length); if (recyclableMemoryStream.Length > 1024 * 1024 * 32) { throw new Exception("too large"); } } } recyclableMemoryStream.Seek(0, SeekOrigin.Begin); return(recyclableMemoryStream); } } } catch (Exception) { recyclableMemoryStream.Dispose(); throw; } }
private static Stream Compress(Stream stream) { if (stream == null) { throw new ArgumentNullException(nameof(stream)); } try { var dic = new Dictionary <byte, Stream>(); try { stream.Seek(0, SeekOrigin.Begin); RecyclableMemoryStream deflateBufferStream = null; try { deflateBufferStream = new RecyclableMemoryStream(_bufferManager); using (var deflateStream = new DeflateStream(deflateBufferStream, CompressionMode.Compress, true)) using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4)) { int length; while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0) { deflateStream.Write(safeBuffer.Value, 0, length); } } deflateBufferStream.Seek(0, SeekOrigin.Begin); dic.Add((byte)ConvertCompressionAlgorithm.Deflate, deflateBufferStream); } catch (Exception) { if (deflateBufferStream != null) { deflateBufferStream.Dispose(); } } } catch (Exception) { } dic.Add((byte)ConvertCompressionAlgorithm.None, stream); var list = dic.ToList(); list.Sort((x, y) => { int c = x.Value.Length.CompareTo(y.Value.Length); if (c != 0) { return(c); } return(x.Key.CompareTo(y.Key)); }); for (int i = 1; i < list.Count; i++) { list[i].Value.Dispose(); } var headerStream = new RecyclableMemoryStream(_bufferManager); Varint.SetUInt64(headerStream, list[0].Key); return(new UniteStream(headerStream, list[0].Value)); } catch (Exception ex) { if (stream != null) { stream.Dispose(); } throw new ArgumentException(ex.Message, ex); } }
private static Stream ToStream <T>(int version, ItemBase <T> item) where T : ItemBase <T> { Stream stream = null; try { stream = new RangeStream(item.Export(_bufferManager)); var dic = new Dictionary <byte, Stream>(); try { stream.Seek(0, SeekOrigin.Begin); BufferStream deflateBufferStream = null; try { deflateBufferStream = new BufferStream(_bufferManager); using (var deflateStream = new DeflateStream(deflateBufferStream, CompressionMode.Compress, true)) using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4)) { int length; while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0) { deflateStream.Write(safeBuffer.Value, 0, length); } } deflateBufferStream.Seek(0, SeekOrigin.Begin); dic.Add((byte)ConvertCompressionAlgorithm.Deflate, deflateBufferStream); } catch (Exception) { if (deflateBufferStream != null) { deflateBufferStream.Dispose(); } } } catch (Exception) { } dic.Add((byte)ConvertCompressionAlgorithm.None, stream); var list = dic.ToList(); list.Sort((x, y) => { int c = x.Value.Length.CompareTo(y.Value.Length); if (c != 0) { return(c); } return(x.Key.CompareTo(y.Key)); }); for (int i = 1; i < list.Count; i++) { list[i].Value.Dispose(); } var headerStream = new BufferStream(_bufferManager); VintUtils.SetUInt64(headerStream, (uint)version); VintUtils.SetUInt64(headerStream, list[0].Key); var dataStream = new UniteStream(headerStream, list[0].Value); var crcStream = new MemoryStream(Crc32_Castagnoli.ComputeHash(new WrapperStream(dataStream, true))); return(new UniteStream(dataStream, crcStream)); } catch (Exception ex) { if (stream != null) { stream.Dispose(); } throw new ArgumentException(ex.Message, ex); } }
public Task <Metadata> Import(Stream stream, TimeSpan lifeSpan, CancellationToken token) { if (stream == null) { throw new ArgumentNullException(nameof(stream)); } return(Task.Run(() => { Metadata metadata = null; var lockedHashes = new HashSet <Hash>(); try { const int blockLength = 1024 * 1024; const HashAlgorithm hashAlgorithm = HashAlgorithm.Sha256; const CorrectionAlgorithm correctionAlgorithm = CorrectionAlgorithm.ReedSolomon8; int depth = 0; var creationTime = DateTime.UtcNow; var groupList = new List <Group>(); for (; ;) { if (stream.Length <= blockLength) { Hash hash; using (var safeBuffer = _bufferManager.CreateSafeBuffer(blockLength)) { int length = (int)stream.Length; stream.Read(safeBuffer.Value, 0, length); if (hashAlgorithm == HashAlgorithm.Sha256) { hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(safeBuffer.Value, 0, length)); } _blocksManager.Lock(hash); _blocksManager.Set(hash, new ArraySegment <byte>(safeBuffer.Value, 0, length)); lockedHashes.Add(hash); } // Stream Dispose { stream.Dispose(); stream = null; } metadata = new Metadata(depth, hash); break; } else { for (; ;) { var targetHashes = new List <Hash>(); var targetBuffers = new List <ArraySegment <byte> >(); long sumLength = 0; try { for (int i = 0; stream.Position < stream.Length; i++) { token.ThrowIfCancellationRequested(); var buffer = new ArraySegment <byte>(); try { int length = (int)Math.Min(stream.Length - stream.Position, blockLength); buffer = new ArraySegment <byte>(_bufferManager.TakeBuffer(length), 0, length); stream.Read(buffer.Array, 0, length); sumLength += length; } catch (Exception) { if (buffer.Array != null) { _bufferManager.ReturnBuffer(buffer.Array); } throw; } Hash hash; if (hashAlgorithm == HashAlgorithm.Sha256) { hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(buffer)); } _blocksManager.Lock(hash); _blocksManager.Set(hash, buffer); lockedHashes.Add(hash); targetHashes.Add(hash); targetBuffers.Add(buffer); if (targetBuffers.Count >= 128) { break; } } var parityHashes = this.ParityEncoding(targetBuffers, hashAlgorithm, correctionAlgorithm, token); lockedHashes.UnionWith(parityHashes); groupList.Add(new Group(correctionAlgorithm, sumLength, CollectionUtils.Unite(targetHashes, parityHashes).ToArray())); } finally { foreach (var buffer in targetBuffers) { if (buffer.Array == null) { continue; } _bufferManager.ReturnBuffer(buffer.Array); } } if (stream.Position == stream.Length) { break; } } depth++; // Stream Dispose { stream.Dispose(); stream = null; } stream = (new Index(groupList)).Export(_bufferManager); } } } finally { if (stream != null) { stream.Dispose(); stream = null; } } lock (_lockObject) { if (!_contentInfoManager.ContainsMessageContentInfo(metadata)) { _contentInfoManager.Add(new ContentInfo(DateTime.UtcNow, lifeSpan, metadata, lockedHashes, null)); foreach (var hash in lockedHashes) { _blocksManager.Lock(hash); } } } return metadata; }, token)); }
private void DecodingThread(CancellationToken token) { for (; ;) { if (token.WaitHandle.WaitOne(1000)) { return; } DownloadItemInfo item = null; lock (_lockObject) { item = CollectionUtils.Unite(_volatileDownloadItemInfoManager, _downloadItemInfoManager) .Where(n => !_workingItems.Contains(n)) .Where(n => n.State == DownloadState.Decoding || n.State == DownloadState.ParityDecoding) .OrderBy(n => (n.Depth == n.Metadata.Depth) ? 0 : 1) .OrderBy(n => (n.State == DownloadState.Decoding) ? 0 : 1) .FirstOrDefault(); if (item != null) { _workingItems.Add(item); } } if (item == null) { continue; } try { if ((item.Depth == 0 && !_cacheManager.Contains(item.Metadata.Hash)) || (item.Depth > 0 && !item.Index.Groups.All(n => _existManager.GetCount(n, true) >= n.Hashes.Count() / 2))) { item.State = DownloadState.Downloading; } else { var hashes = new HashCollection(); var totalHashes = new HashCollection(); if (item.Depth == 0) { hashes.Add(item.Metadata.Hash); totalHashes.Add(item.Metadata.Hash); } else { try { foreach (var group in item.Index.Groups) { if (item.State == DownloadState.Error) { throw new OperationCanceledException(); } hashes.AddRange(_cacheManager.ParityDecoding(group, token).Result); } totalHashes.AddRange(item.Index.Groups.SelectMany(n => n.Hashes)); } catch (OperationCanceledException) { continue; } item.State = DownloadState.Decoding; } if (item.Depth < item.Metadata.Depth) { Index index; try { using (var stream = _cacheManager.Decoding(hashes)) using (var progressStream = new ProgressStream(stream, null, 1024 * 1024, token)) { if (item.State == DownloadState.Error) { throw new OperationCanceledException(); } if (progressStream.Length > item.MaxLength) { throw new ArgumentException(); } index = Index.Import(progressStream, _bufferManager); } } catch (OperationCanceledException) { continue; } lock (_lockObject) { if (item.Path != null) { _protectCacheInfoManager.Add(new ProtectedCacheInfo(DateTime.UtcNow, totalHashes)); } this.CheckState(index); this.UncheckState(item.Index); item.Index = index; item.Depth++; item.State = DownloadState.Downloading; } } else { if (item.Path != null) { string filePath = null; try { token.ThrowIfCancellationRequested(); string targetPath; if (Path.IsPathRooted(item.Path)) { targetPath = item.Path; } else { targetPath = Path.GetFullPath(Path.Combine(_basePath, item.Path)); // ディレクトリトラバーサル対策 if (!targetPath.StartsWith(Path.GetFullPath(_basePath))) { targetPath = Path.GetFullPath(Path.Combine(_basePath, Path.GetFileName(item.Path))); } } Directory.CreateDirectory(Path.GetDirectoryName(targetPath)); using (var inStream = _cacheManager.Decoding(hashes)) using (var outStream = DownloadManager.GetUniqueFileStream(targetPath + ".tmp")) using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 1024)) { filePath = outStream.Name; int readLength; while ((readLength = inStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0) { if (item.State == DownloadState.Error) { throw new OperationCanceledException(); } token.ThrowIfCancellationRequested(); outStream.Write(safeBuffer.Value, 0, readLength); } } File.Move(filePath, DownloadManager.GetUniqueFilePath(targetPath)); } catch (OperationCanceledException) { if (filePath != null) { File.Delete(filePath); } continue; } } lock (_lockObject) { if (item.Path != null) { _protectCacheInfoManager.Add(new ProtectedCacheInfo(DateTime.UtcNow, totalHashes)); } item.ResultHashes.AddRange(hashes); item.State = DownloadState.Completed; } } } } catch (Exception e) { item.State = DownloadState.Error; Log.Error(e); } finally { _workingItems.Remove(item); } } }
public static void Compress(Stream inStream, Stream outStream, BufferManager bufferManager) { var info = new ProcessStartInfo(_path); info.CreateNoWindow = true; info.UseShellExecute = false; info.RedirectStandardInput = true; info.RedirectStandardOutput = true; info.RedirectStandardError = true; info.Arguments = "--compress --format=xz -0 --threads=1 --stdout"; using (var inCacheStream = new BufferedStream(inStream, 1024 * 32)) using (var outCacheStream = new BufferedStream(outStream, 1024 * 32)) { using (Process process = Process.Start(info)) { process.PriorityClass = ProcessPriorityClass.Idle; process.ErrorDataReceived += (object sender, DataReceivedEventArgs e) => { Log.Error(e.Data); }; Exception threadException = null; var thread = new Thread(() => { try { using (var standardOutputStream = process.StandardOutput.BaseStream) using (var safeBuffer = bufferManager.CreateSafeBuffer(1024 * 32)) { int length; while ((length = standardOutputStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0) { outCacheStream.Write(safeBuffer.Value, 0, length); } } } catch (Exception e) { threadException = e; } }); thread.IsBackground = true; thread.Start(); try { using (var standardInputStream = process.StandardInput.BaseStream) using (var safeBuffer = bufferManager.CreateSafeBuffer(1024 * 32)) { int length; while ((length = inCacheStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0) { standardInputStream.Write(safeBuffer.Value, 0, length); } } } catch (Exception) { throw; } thread.Join(); if (threadException != null) throw threadException; process.WaitForExit(); } } }
// http://blogs.msdn.com/b/feroze_daud/archive/2004/03/30/104440.aspx private static string Decode(WebResponse w) { BufferManager bufferManager = BufferManager.Instance; // // first see if content length header has charset = calue // string charset = null; { string content_type = w.Headers["content-type"]; if (content_type != null) { int index = content_type.IndexOf("charset="); if (index != -1) { charset = content_type.Substring(index + 8); } } } // save data to a rawStream var rawStream = new BufferStream(bufferManager); using (Stream stream = w.GetResponseStream()) using (var safeBuffer = bufferManager.CreateSafeBuffer(1024 * 4)) { int length; while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0) { rawStream.Write(safeBuffer.Value, 0, length); if (rawStream.Length > 1024 * 1024 * 32) { throw new Exception("too large"); } } } // // if ContentType is null, or did not contain charset, we search in body // if (charset == null) { rawStream.Seek(0, SeekOrigin.Begin); using (StreamReader reader = new StreamReader(new WrapperStream(rawStream, true), Encoding.ASCII)) { string meta = reader.ReadToEnd(); if (!string.IsNullOrWhiteSpace(meta)) { int start_index = meta.IndexOf("charset="); if (start_index != -1) { int end_index = meta.IndexOf("\"", start_index); if (end_index != -1) { start_index += 8; charset = meta.Substring(start_index, end_index - start_index); } } } } } Encoding encoding = null; if (charset == null) { encoding = Encoding.UTF8; //default encoding } else { try { encoding = Encoding.GetEncoding(charset); } catch (Exception) { encoding = Encoding.UTF8; } } rawStream.Seek(0, SeekOrigin.Begin); using (StreamReader reader = new StreamReader(rawStream, encoding)) { return(reader.ReadToEnd()); } }
private void MessageUploadAndDownload(IEnumerable <CoreManager> coreManagers) { _callback.Invoke("----- CoreManager Message Send and Receive Test -----"); _callback.Invoke(""); var coreManagerList = coreManagers.ToList(); Parallel.ForEach(coreManagerList, coreManager => { coreManager.Resize((long)1024 * 1024 * 1024 * 32).Wait(); }); var hashList = new LockedHashSet <Hash>(); var metadataList = new LockedList <Metadata>(); Parallel.For(0, 30, new ParallelOptions() { MaxDegreeOfParallelism = 3 }, i => { var random = RandomProvider.GetThreadRandom(); using (var stream = new BufferStream(_bufferManager)) { using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4)) { for (long remain = (long)1024 * 1024 * 32; remain > 0; remain = Math.Max(0, remain - safeBuffer.Value.Length)) { int length = (int)Math.Min(remain, safeBuffer.Value.Length); random.NextBytes(safeBuffer.Value); stream.Write(safeBuffer.Value, 0, length); } } stream.Seek(0, SeekOrigin.Begin); hashList.Add(new Hash(HashAlgorithm.Sha256, Sha256.ComputeHash(new WrapperStream(stream, true)))); stream.Seek(0, SeekOrigin.Begin); using (var tokenSource = new CancellationTokenSource()) { metadataList.Add(coreManagerList[i].VolatileSetStream(stream, new TimeSpan(1, 0, 0), tokenSource.Token).Result); } } }); var sw = Stopwatch.StartNew(); Parallel.ForEach(metadataList, metadata => { for (; ;) { Thread.Sleep(1000); Stream stream = null; try { stream = coreManagerList[0].VolatileGetStream(metadata, 1024 * 1024 * 256); if (stream == null) { continue; } var hash = new Hash(HashAlgorithm.Sha256, Sha256.ComputeHash(stream)); if (!hashList.Contains(hash)) { throw new ArgumentException("Broken"); } _callback.Invoke($"{sw.Elapsed.ToString("hh\\:mm\\:ss")}: Success {NetworkConverter.ToBase64UrlString(metadata.Hash.Value)}"); return; } finally { if (stream != null) { stream.Dispose(); } } } }); _callback.Invoke("----- End -----"); }