private void EventLoop(CancellationToken token)
{
try
{
for (; ;)
{
token.ThrowIfCancellationRequested();
long length = (long)Varint.GetUInt64(_stream);
if (length == 0)
{
continue;
}
Stream resultSteram = null;
try
{
resultSteram = new RecyclableMemoryStream(_bufferManager);
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 32))
{
long remain = length;
while (remain > 0)
{
int readLength = _stream.Read(safeBuffer.Value, 0, (int)Math.Min(remain, safeBuffer.Value.Length));
resultSteram.Write(safeBuffer.Value, 0, readLength);
remain -= readLength;
}
}
resultSteram.Seek(0, SeekOrigin.Begin);
this.OnReceive(resultSteram);
}
catch (Exception)
{
if (resultSteram != null)
{
resultSteram.Dispose();
resultSteram = null;
}
}
}
}
catch (Exception)
{
}
}
public void MessageUpload()
{
_callback.Invoke("----- CoreManager Message Upload Test (Start) -----");
_callback.Invoke("");
var random = RandomProvider.GetThreadRandom();
_coreManager.Resize((long)1024 * 1024 * 1024 * 32).Wait();
Metadata metadata = null;
Hash hash;
using (var stream = new RecyclableMemoryStream(_bufferManager))
{
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
for (long remain = (long)1024 * 1024 * 256; 0 < remain; remain -= safeBuffer.Value.Length)
{
int length = (int)Math.Min(remain, safeBuffer.Value.Length);
random.NextBytes(safeBuffer.Value);
stream.Write(safeBuffer.Value, 0, length);
}
}
stream.Seek(0, SeekOrigin.Begin);
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(new WrapperStream(stream)));
stream.Seek(0, SeekOrigin.Begin);
metadata = _coreManager.VolatileSetStream(stream, new TimeSpan(1, 0, 0, 0), CancellationToken.None).Result;
}
using (var stream = metadata.Export(_bufferManager))
using (var safeBuffer = _bufferManager.CreateSafeBuffer((int)stream.Length))
{
stream.Read(safeBuffer.Value, 0, (int)stream.Length);
Console.WriteLine(NetworkConverter.ToBase64UrlString(safeBuffer.Value, 0, (int)stream.Length));
}
using (var stream = hash.Export(_bufferManager))
using (var safeBuffer = _bufferManager.CreateSafeBuffer((int)stream.Length))
{
stream.Read(safeBuffer.Value, 0, (int)stream.Length);
Console.WriteLine(NetworkConverter.ToBase64UrlString(safeBuffer.Value, 0, (int)stream.Length));
}
_callback.Invoke("----- CoreManager Message Upload Test (End) -----");
}
public void SetLength(long length)
{
lock (_lockObject)
{
{
long size = BitmapManager.Roundup(length, 8);
_bitmapStream.SetLength(size);
_bitmapStream.Seek(0, SeekOrigin.Begin);
{
using (var safeBuffer = _bufferManager.CreateSafeBuffer(4096))
{
Unsafe.Zero(safeBuffer.Value);
for (long i = (size / safeBuffer.Value.Length), remain = size; i >= 0; i--, remain -= safeBuffer.Value.Length)
{
_bitmapStream.Write(safeBuffer.Value, 0, (int)Math.Min(remain, safeBuffer.Value.Length));
_bitmapStream.Flush();
}
}
}
}
_length = length;
{
_cacheChanged = false;
_cacheSector = -1;
_cacheBufferLength = 0;
}
}
}
private static Stream GetStream(string url, IWebProxy proxy)
{
BufferManager bufferManager = BufferManager.Instance;
for (int i = 0; i < 10; i++)
{
var bufferStream = new BufferStream(bufferManager);
try
{
var request = (HttpWebRequest)WebRequest.Create(url);
request.AllowAutoRedirect = true;
request.Proxy = proxy;
request.Headers.Add("Pragma", "no-cache");
request.Headers.Add("Cache-Control", "no-cache");
request.Timeout = 1000 * 60 * 5;
request.ReadWriteTimeout = 1000 * 60 * 5;
using (WebResponse response = request.GetResponse())
{
if (response.ContentLength > 1024 * 1024 * 32)
{
throw new Exception("too large");
}
using (Stream stream = response.GetResponseStream())
using (var safeBuffer = bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
bufferStream.Write(safeBuffer.Value, 0, length);
if (bufferStream.Length > 1024 * 1024 * 32)
{
throw new Exception("too large");
}
}
}
if (response.ContentLength != -1 && bufferStream.Length != response.ContentLength)
{
continue;
}
bufferStream.Seek(0, SeekOrigin.Begin);
return(bufferStream);
}
}
catch (Exception)
{
bufferStream.Dispose();
}
}
throw new Exception(string.Format("not found: {0}", url));
}
public void ReadWriteTest()
{
for (int i = 0; i < 256; i++)
{
int size = _random.Next(1, 1024 * 1024 * 4);
using (var safeBuffer = _bufferManager.CreateSafeBuffer(size))
{
_random.NextBytes(safeBuffer.Value);
var block = new ArraySegment <byte>(safeBuffer.Value, 0, size);
var hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(block));
_cacheManager.Set(hash, block);
var result = _cacheManager.GetBlock(hash);
Assert.True(Unsafe.Equals(block.Array, block.Offset, result.Array, result.Offset, size));
_bufferManager.ReturnBuffer(result.Array);
}
}
}
private Stream GetStream(string url)
{
var recyclableMemoryStream = new RecyclableMemoryStream(_bufferManager);
try
{
using (var client = new HttpClient())
{
using (var stream = client.GetStreamAsync(url).Result)
{
if (stream.Length > 1024 * 1024 * 32)
{
throw new Exception("too large");
}
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
recyclableMemoryStream.Write(safeBuffer.Value, 0, length);
if (recyclableMemoryStream.Length > 1024 * 1024 * 32)
{
throw new Exception("too large");
}
}
}
recyclableMemoryStream.Seek(0, SeekOrigin.Begin);
return(recyclableMemoryStream);
}
}
}
catch (Exception)
{
recyclableMemoryStream.Dispose();
throw;
}
}
private static Stream Compress(Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
try
{
var dic = new Dictionary <byte, Stream>();
try
{
stream.Seek(0, SeekOrigin.Begin);
RecyclableMemoryStream deflateBufferStream = null;
try
{
deflateBufferStream = new RecyclableMemoryStream(_bufferManager);
using (var deflateStream = new DeflateStream(deflateBufferStream, CompressionMode.Compress, true))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
deflateStream.Write(safeBuffer.Value, 0, length);
}
}
deflateBufferStream.Seek(0, SeekOrigin.Begin);
dic.Add((byte)ConvertCompressionAlgorithm.Deflate, deflateBufferStream);
}
catch (Exception)
{
if (deflateBufferStream != null)
{
deflateBufferStream.Dispose();
}
}
}
catch (Exception)
{
}
dic.Add((byte)ConvertCompressionAlgorithm.None, stream);
var list = dic.ToList();
list.Sort((x, y) =>
{
int c = x.Value.Length.CompareTo(y.Value.Length);
if (c != 0)
{
return(c);
}
return(x.Key.CompareTo(y.Key));
});
for (int i = 1; i < list.Count; i++)
{
list[i].Value.Dispose();
}
var headerStream = new RecyclableMemoryStream(_bufferManager);
Varint.SetUInt64(headerStream, list[0].Key);
return(new UniteStream(headerStream, list[0].Value));
}
catch (Exception ex)
{
if (stream != null)
{
stream.Dispose();
}
throw new ArgumentException(ex.Message, ex);
}
}
private static Stream ToStream <T>(int version, ItemBase <T> item)
where T : ItemBase <T>
{
Stream stream = null;
try
{
stream = new RangeStream(item.Export(_bufferManager));
var dic = new Dictionary <byte, Stream>();
try
{
stream.Seek(0, SeekOrigin.Begin);
BufferStream deflateBufferStream = null;
try
{
deflateBufferStream = new BufferStream(_bufferManager);
using (var deflateStream = new DeflateStream(deflateBufferStream, CompressionMode.Compress, true))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
deflateStream.Write(safeBuffer.Value, 0, length);
}
}
deflateBufferStream.Seek(0, SeekOrigin.Begin);
dic.Add((byte)ConvertCompressionAlgorithm.Deflate, deflateBufferStream);
}
catch (Exception)
{
if (deflateBufferStream != null)
{
deflateBufferStream.Dispose();
}
}
}
catch (Exception)
{
}
dic.Add((byte)ConvertCompressionAlgorithm.None, stream);
var list = dic.ToList();
list.Sort((x, y) =>
{
int c = x.Value.Length.CompareTo(y.Value.Length);
if (c != 0)
{
return(c);
}
return(x.Key.CompareTo(y.Key));
});
for (int i = 1; i < list.Count; i++)
{
list[i].Value.Dispose();
}
var headerStream = new BufferStream(_bufferManager);
VintUtils.SetUInt64(headerStream, (uint)version);
VintUtils.SetUInt64(headerStream, list[0].Key);
var dataStream = new UniteStream(headerStream, list[0].Value);
var crcStream = new MemoryStream(Crc32_Castagnoli.ComputeHash(new WrapperStream(dataStream, true)));
return(new UniteStream(dataStream, crcStream));
}
catch (Exception ex)
{
if (stream != null)
{
stream.Dispose();
}
throw new ArgumentException(ex.Message, ex);
}
}
public Task <Metadata> Import(Stream stream, TimeSpan lifeSpan, CancellationToken token)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
return(Task.Run(() =>
{
Metadata metadata = null;
var lockedHashes = new HashSet <Hash>();
try
{
const int blockLength = 1024 * 1024;
const HashAlgorithm hashAlgorithm = HashAlgorithm.Sha256;
const CorrectionAlgorithm correctionAlgorithm = CorrectionAlgorithm.ReedSolomon8;
int depth = 0;
var creationTime = DateTime.UtcNow;
var groupList = new List <Group>();
for (; ;)
{
if (stream.Length <= blockLength)
{
Hash hash;
using (var safeBuffer = _bufferManager.CreateSafeBuffer(blockLength))
{
int length = (int)stream.Length;
stream.Read(safeBuffer.Value, 0, length);
if (hashAlgorithm == HashAlgorithm.Sha256)
{
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(safeBuffer.Value, 0, length));
}
_blocksManager.Lock(hash);
_blocksManager.Set(hash, new ArraySegment <byte>(safeBuffer.Value, 0, length));
lockedHashes.Add(hash);
}
// Stream Dispose
{
stream.Dispose();
stream = null;
}
metadata = new Metadata(depth, hash);
break;
}
else
{
for (; ;)
{
var targetHashes = new List <Hash>();
var targetBuffers = new List <ArraySegment <byte> >();
long sumLength = 0;
try
{
for (int i = 0; stream.Position < stream.Length; i++)
{
token.ThrowIfCancellationRequested();
var buffer = new ArraySegment <byte>();
try
{
int length = (int)Math.Min(stream.Length - stream.Position, blockLength);
buffer = new ArraySegment <byte>(_bufferManager.TakeBuffer(length), 0, length);
stream.Read(buffer.Array, 0, length);
sumLength += length;
}
catch (Exception)
{
if (buffer.Array != null)
{
_bufferManager.ReturnBuffer(buffer.Array);
}
throw;
}
Hash hash;
if (hashAlgorithm == HashAlgorithm.Sha256)
{
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(buffer));
}
_blocksManager.Lock(hash);
_blocksManager.Set(hash, buffer);
lockedHashes.Add(hash);
targetHashes.Add(hash);
targetBuffers.Add(buffer);
if (targetBuffers.Count >= 128)
{
break;
}
}
var parityHashes = this.ParityEncoding(targetBuffers, hashAlgorithm, correctionAlgorithm, token);
lockedHashes.UnionWith(parityHashes);
groupList.Add(new Group(correctionAlgorithm, sumLength, CollectionUtils.Unite(targetHashes, parityHashes).ToArray()));
}
finally
{
foreach (var buffer in targetBuffers)
{
if (buffer.Array == null)
{
continue;
}
_bufferManager.ReturnBuffer(buffer.Array);
}
}
if (stream.Position == stream.Length)
{
break;
}
}
depth++;
// Stream Dispose
{
stream.Dispose();
stream = null;
}
stream = (new Index(groupList)).Export(_bufferManager);
}
}
}
finally
{
if (stream != null)
{
stream.Dispose();
stream = null;
}
}
lock (_lockObject)
{
if (!_contentInfoManager.ContainsMessageContentInfo(metadata))
{
_contentInfoManager.Add(new ContentInfo(DateTime.UtcNow, lifeSpan, metadata, lockedHashes, null));
foreach (var hash in lockedHashes)
{
_blocksManager.Lock(hash);
}
}
}
return metadata;
}, token));
}
private void DecodingThread(CancellationToken token)
{
for (; ;)
{
if (token.WaitHandle.WaitOne(1000))
{
return;
}
DownloadItemInfo item = null;
lock (_lockObject)
{
item = CollectionUtils.Unite(_volatileDownloadItemInfoManager, _downloadItemInfoManager)
.Where(n => !_workingItems.Contains(n))
.Where(n => n.State == DownloadState.Decoding || n.State == DownloadState.ParityDecoding)
.OrderBy(n => (n.Depth == n.Metadata.Depth) ? 0 : 1)
.OrderBy(n => (n.State == DownloadState.Decoding) ? 0 : 1)
.FirstOrDefault();
if (item != null)
{
_workingItems.Add(item);
}
}
if (item == null)
{
continue;
}
try
{
if ((item.Depth == 0 && !_cacheManager.Contains(item.Metadata.Hash)) ||
(item.Depth > 0 && !item.Index.Groups.All(n => _existManager.GetCount(n, true) >= n.Hashes.Count() / 2)))
{
item.State = DownloadState.Downloading;
}
else
{
var hashes = new HashCollection();
var totalHashes = new HashCollection();
if (item.Depth == 0)
{
hashes.Add(item.Metadata.Hash);
totalHashes.Add(item.Metadata.Hash);
}
else
{
try
{
foreach (var group in item.Index.Groups)
{
if (item.State == DownloadState.Error)
{
throw new OperationCanceledException();
}
hashes.AddRange(_cacheManager.ParityDecoding(group, token).Result);
}
totalHashes.AddRange(item.Index.Groups.SelectMany(n => n.Hashes));
}
catch (OperationCanceledException)
{
continue;
}
item.State = DownloadState.Decoding;
}
if (item.Depth < item.Metadata.Depth)
{
Index index;
try
{
using (var stream = _cacheManager.Decoding(hashes))
using (var progressStream = new ProgressStream(stream, null, 1024 * 1024, token))
{
if (item.State == DownloadState.Error)
{
throw new OperationCanceledException();
}
if (progressStream.Length > item.MaxLength)
{
throw new ArgumentException();
}
index = Index.Import(progressStream, _bufferManager);
}
}
catch (OperationCanceledException)
{
continue;
}
lock (_lockObject)
{
if (item.Path != null)
{
_protectCacheInfoManager.Add(new ProtectedCacheInfo(DateTime.UtcNow, totalHashes));
}
this.CheckState(index);
this.UncheckState(item.Index);
item.Index = index;
item.Depth++;
item.State = DownloadState.Downloading;
}
}
else
{
if (item.Path != null)
{
string filePath = null;
try
{
token.ThrowIfCancellationRequested();
string targetPath;
if (Path.IsPathRooted(item.Path))
{
targetPath = item.Path;
}
else
{
targetPath = Path.GetFullPath(Path.Combine(_basePath, item.Path));
// ディレクトリトラバーサル対策
if (!targetPath.StartsWith(Path.GetFullPath(_basePath)))
{
targetPath = Path.GetFullPath(Path.Combine(_basePath, Path.GetFileName(item.Path)));
}
}
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
using (var inStream = _cacheManager.Decoding(hashes))
using (var outStream = DownloadManager.GetUniqueFileStream(targetPath + ".tmp"))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 1024))
{
filePath = outStream.Name;
int readLength;
while ((readLength = inStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
if (item.State == DownloadState.Error)
{
throw new OperationCanceledException();
}
token.ThrowIfCancellationRequested();
outStream.Write(safeBuffer.Value, 0, readLength);
}
}
File.Move(filePath, DownloadManager.GetUniqueFilePath(targetPath));
}
catch (OperationCanceledException)
{
if (filePath != null)
{
File.Delete(filePath);
}
continue;
}
}
lock (_lockObject)
{
if (item.Path != null)
{
_protectCacheInfoManager.Add(new ProtectedCacheInfo(DateTime.UtcNow, totalHashes));
}
item.ResultHashes.AddRange(hashes);
item.State = DownloadState.Completed;
}
}
}
}
catch (Exception e)
{
item.State = DownloadState.Error;
Log.Error(e);
}
finally
{
_workingItems.Remove(item);
}
}
}
public static void Compress(Stream inStream, Stream outStream, BufferManager bufferManager)
{
var info = new ProcessStartInfo(_path);
info.CreateNoWindow = true;
info.UseShellExecute = false;
info.RedirectStandardInput = true;
info.RedirectStandardOutput = true;
info.RedirectStandardError = true;
info.Arguments = "--compress --format=xz -0 --threads=1 --stdout";
using (var inCacheStream = new BufferedStream(inStream, 1024 * 32))
using (var outCacheStream = new BufferedStream(outStream, 1024 * 32))
{
using (Process process = Process.Start(info))
{
process.PriorityClass = ProcessPriorityClass.Idle;
process.ErrorDataReceived += (object sender, DataReceivedEventArgs e) =>
{
Log.Error(e.Data);
};
Exception threadException = null;
var thread = new Thread(() =>
{
try
{
using (var standardOutputStream = process.StandardOutput.BaseStream)
using (var safeBuffer = bufferManager.CreateSafeBuffer(1024 * 32))
{
int length;
while ((length = standardOutputStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
outCacheStream.Write(safeBuffer.Value, 0, length);
}
}
}
catch (Exception e)
{
threadException = e;
}
});
thread.IsBackground = true;
thread.Start();
try
{
using (var standardInputStream = process.StandardInput.BaseStream)
using (var safeBuffer = bufferManager.CreateSafeBuffer(1024 * 32))
{
int length;
while ((length = inCacheStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
standardInputStream.Write(safeBuffer.Value, 0, length);
}
}
}
catch (Exception)
{
throw;
}
thread.Join();
if (threadException != null) throw threadException;
process.WaitForExit();
}
}
}
// http://blogs.msdn.com/b/feroze_daud/archive/2004/03/30/104440.aspx
private static string Decode(WebResponse w)
{
BufferManager bufferManager = BufferManager.Instance;
//
// first see if content length header has charset = calue
//
string charset = null;
{
string content_type = w.Headers["content-type"];
if (content_type != null)
{
int index = content_type.IndexOf("charset=");
if (index != -1)
{
charset = content_type.Substring(index + 8);
}
}
}
// save data to a rawStream
var rawStream = new BufferStream(bufferManager);
using (Stream stream = w.GetResponseStream())
using (var safeBuffer = bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
rawStream.Write(safeBuffer.Value, 0, length);
if (rawStream.Length > 1024 * 1024 * 32)
{
throw new Exception("too large");
}
}
}
//
// if ContentType is null, or did not contain charset, we search in body
//
if (charset == null)
{
rawStream.Seek(0, SeekOrigin.Begin);
using (StreamReader reader = new StreamReader(new WrapperStream(rawStream, true), Encoding.ASCII))
{
string meta = reader.ReadToEnd();
if (!string.IsNullOrWhiteSpace(meta))
{
int start_index = meta.IndexOf("charset=");
if (start_index != -1)
{
int end_index = meta.IndexOf("\"", start_index);
if (end_index != -1)
{
start_index += 8;
charset = meta.Substring(start_index, end_index - start_index);
}
}
}
}
}
Encoding encoding = null;
if (charset == null)
{
encoding = Encoding.UTF8; //default encoding
}
else
{
try
{
encoding = Encoding.GetEncoding(charset);
}
catch (Exception)
{
encoding = Encoding.UTF8;
}
}
rawStream.Seek(0, SeekOrigin.Begin);
using (StreamReader reader = new StreamReader(rawStream, encoding))
{
return(reader.ReadToEnd());
}
}
private void MessageUploadAndDownload(IEnumerable <CoreManager> coreManagers)
{
_callback.Invoke("----- CoreManager Message Send and Receive Test -----");
_callback.Invoke("");
var coreManagerList = coreManagers.ToList();
Parallel.ForEach(coreManagerList, coreManager =>
{
coreManager.Resize((long)1024 * 1024 * 1024 * 32).Wait();
});
var hashList = new LockedHashSet <Hash>();
var metadataList = new LockedList <Metadata>();
Parallel.For(0, 30, new ParallelOptions()
{
MaxDegreeOfParallelism = 3
}, i =>
{
var random = RandomProvider.GetThreadRandom();
using (var stream = new BufferStream(_bufferManager))
{
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
for (long remain = (long)1024 * 1024 * 32; remain > 0; remain = Math.Max(0, remain - safeBuffer.Value.Length))
{
int length = (int)Math.Min(remain, safeBuffer.Value.Length);
random.NextBytes(safeBuffer.Value);
stream.Write(safeBuffer.Value, 0, length);
}
}
stream.Seek(0, SeekOrigin.Begin);
hashList.Add(new Hash(HashAlgorithm.Sha256, Sha256.ComputeHash(new WrapperStream(stream, true))));
stream.Seek(0, SeekOrigin.Begin);
using (var tokenSource = new CancellationTokenSource())
{
metadataList.Add(coreManagerList[i].VolatileSetStream(stream, new TimeSpan(1, 0, 0), tokenSource.Token).Result);
}
}
});
var sw = Stopwatch.StartNew();
Parallel.ForEach(metadataList, metadata =>
{
for (; ;)
{
Thread.Sleep(1000);
Stream stream = null;
try
{
stream = coreManagerList[0].VolatileGetStream(metadata, 1024 * 1024 * 256);
if (stream == null)
{
continue;
}
var hash = new Hash(HashAlgorithm.Sha256, Sha256.ComputeHash(stream));
if (!hashList.Contains(hash))
{
throw new ArgumentException("Broken");
}
_callback.Invoke($"{sw.Elapsed.ToString("hh\\:mm\\:ss")}: Success {NetworkConverter.ToBase64UrlString(metadata.Hash.Value)}");
return;
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
});
_callback.Invoke("----- End -----");
}
