private void DecodingThread(CancellationToken token)
{
for (; ;)
{
if (token.WaitHandle.WaitOne(1000))
{
return;
}
DownloadItemInfo item = null;
lock (_lockObject)
{
item = CollectionUtils.Unite(_volatileDownloadItemInfoManager, _downloadItemInfoManager)
.Where(n => !_workingItems.Contains(n))
.Where(n => n.State == DownloadState.Decoding || n.State == DownloadState.ParityDecoding)
.OrderBy(n => (n.Depth == n.Metadata.Depth) ? 0 : 1)
.OrderBy(n => (n.State == DownloadState.Decoding) ? 0 : 1)
.FirstOrDefault();
if (item != null)
{
_workingItems.Add(item);
}
}
if (item == null)
{
continue;
}
try
{
if ((item.Depth == 0 && !_cacheManager.Contains(item.Metadata.Hash)) ||
(item.Depth > 0 && !item.Index.Groups.All(n => _existManager.GetCount(n, true) >= n.Hashes.Count() / 2)))
{
item.State = DownloadState.Downloading;
}
else
{
var hashes = new HashCollection();
var totalHashes = new HashCollection();
if (item.Depth == 0)
{
hashes.Add(item.Metadata.Hash);
totalHashes.Add(item.Metadata.Hash);
}
else
{
try
{
foreach (var group in item.Index.Groups)
{
if (item.State == DownloadState.Error)
{
throw new OperationCanceledException();
}
hashes.AddRange(_cacheManager.ParityDecoding(group, token).Result);
}
totalHashes.AddRange(item.Index.Groups.SelectMany(n => n.Hashes));
}
catch (OperationCanceledException)
{
continue;
}
item.State = DownloadState.Decoding;
}
if (item.Depth < item.Metadata.Depth)
{
Index index;
try
{
using (var stream = _cacheManager.Decoding(hashes))
using (var progressStream = new ProgressStream(stream, null, 1024 * 1024, token))
{
if (item.State == DownloadState.Error)
{
throw new OperationCanceledException();
}
if (progressStream.Length > item.MaxLength)
{
throw new ArgumentException();
}
index = Index.Import(progressStream, _bufferManager);
}
}
catch (OperationCanceledException)
{
continue;
}
lock (_lockObject)
{
if (item.Path != null)
{
_protectCacheInfoManager.Add(new ProtectedCacheInfo(DateTime.UtcNow, totalHashes));
}
this.CheckState(index);
this.UncheckState(item.Index);
item.Index = index;
item.Depth++;
item.State = DownloadState.Downloading;
}
}
else
{
if (item.Path != null)
{
string filePath = null;
try
{
token.ThrowIfCancellationRequested();
string targetPath;
if (Path.IsPathRooted(item.Path))
{
targetPath = item.Path;
}
else
{
targetPath = Path.GetFullPath(Path.Combine(_basePath, item.Path));
// ディレクトリトラバーサル対策
if (!targetPath.StartsWith(Path.GetFullPath(_basePath)))
{
targetPath = Path.GetFullPath(Path.Combine(_basePath, Path.GetFileName(item.Path)));
}
}
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
using (var inStream = _cacheManager.Decoding(hashes))
using (var outStream = DownloadManager.GetUniqueFileStream(targetPath + ".tmp"))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 1024))
{
filePath = outStream.Name;
int readLength;
while ((readLength = inStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
if (item.State == DownloadState.Error)
{
throw new OperationCanceledException();
}
token.ThrowIfCancellationRequested();
outStream.Write(safeBuffer.Value, 0, readLength);
}
}
File.Move(filePath, DownloadManager.GetUniqueFilePath(targetPath));
}
catch (OperationCanceledException)
{
if (filePath != null)
{
File.Delete(filePath);
}
continue;
}
}
lock (_lockObject)
{
if (item.Path != null)
{
_protectCacheInfoManager.Add(new ProtectedCacheInfo(DateTime.UtcNow, totalHashes));
}
item.ResultHashes.AddRange(hashes);
item.State = DownloadState.Completed;
}
}
}
}
catch (Exception e)
{
item.State = DownloadState.Error;
Log.Error(e);
}
finally
{
_workingItems.Remove(item);
}
}
}
private void EncodeManagerThread()
{
for (; ;)
{
Thread.Sleep(1000 * 1);
if (this.EncodeState == ManagerState.Stop)
{
return;
}
UploadItem item = null;
try
{
lock (this.ThisLock)
{
if (_settings.UploadItems.Count > 0)
{
item = _settings.UploadItems
.Where(n => n.State == UploadState.ComputeHash || n.State == UploadState.Encoding || n.State == UploadState.ParityEncoding)
.Where(n => n.Priority != 0)
.OrderBy(n => - n.Priority)
.Where(n => !_workingPaths.Contains(n.FilePath))
.FirstOrDefault();
if (item != null)
{
_workingPaths.Add(item.FilePath);
}
}
}
}
catch (Exception)
{
return;
}
if (item == null)
{
continue;
}
try
{
if (item.Type == UploadType.Upload)
{
if (item.Groups.Count == 0 && item.Keys.Count == 0)
{
item.State = UploadState.Encoding;
KeyCollection keys = null;
byte[] cryptoKey = null;
try
{
using (FileStream stream = new FileStream(item.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
using (ProgressStream encodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
{
item.EncodeBytes = stream.Length;
item.Seed.Length = stream.Length;
if (item.Seed.Length == 0)
{
throw new InvalidOperationException("Stream Length");
}
item.State = UploadState.ComputeHash;
if (item.HashAlgorithm == HashAlgorithm.Sha256)
{
cryptoKey = Sha256.ComputeHash(hashProgressStream);
}
stream.Seek(0, SeekOrigin.Begin);
item.EncodingBytes = 0;
item.State = UploadState.Encoding;
keys = _cacheManager.Encoding(encodingProgressStream, item.CompressionAlgorithm, item.CryptoAlgorithm, cryptoKey, item.BlockLength, item.HashAlgorithm);
}
}
catch (StopIoException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in keys)
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.EncodingBytes = 0;
item.EncodeBytes = 0;
item.CryptoKey = cryptoKey;
item.Keys.AddRange(keys);
}
}
else if (item.Groups.Count == 0 && item.Keys.Count == 1)
{
lock (this.ThisLock)
{
item.Seed.Rank = item.Rank;
item.Seed.Key = item.Keys[0];
item.Keys.Clear();
item.Seed.CompressionAlgorithm = item.CompressionAlgorithm;
item.Seed.CryptoAlgorithm = item.CryptoAlgorithm;
item.Seed.CryptoKey = item.CryptoKey;
if (item.DigitalSignature != null)
{
item.Seed.CreateCertificate(item.DigitalSignature);
}
item.UploadKeys.Add(item.Seed.Key);
foreach (var key in item.UploadKeys)
{
_connectionsManager.Upload(key);
}
this.CheckState(item);
_cacheManager.SetSeed(item.Seed.Clone(), item.Indexes.Select(n => n.Clone()).ToArray());
item.Indexes.Clear();
foreach (var key in item.LockedKeys)
{
_cacheManager.Unlock(key);
}
item.LockedKeys.Clear();
item.State = UploadState.Uploading;
}
}
else if (item.Keys.Count > 0)
{
item.State = UploadState.ParityEncoding;
item.EncodeBytes = item.Groups.Sum(n =>
{
long sumLength = 0;
for (int i = 0; i < n.InformationLength; i++)
{
if (_cacheManager.Contains(n.Keys[i]))
{
sumLength += (long)_cacheManager.GetLength(n.Keys[i]);
}
}
return(sumLength);
}) + item.Keys.Sum(n =>
{
if (_cacheManager.Contains(n))
{
return((long)_cacheManager.GetLength(n));
}
return(0);
});
var length = Math.Min(item.Keys.Count, 128);
var keys = new KeyCollection(item.Keys.Take(length));
Group group = null;
try
{
group = _cacheManager.ParityEncoding(keys, item.HashAlgorithm, item.BlockLength, item.CorrectionAlgorithm, (object state2) =>
{
return(this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
});
}
catch (StopException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in group.Keys.Skip(group.InformationLength))
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.Groups.Add(group);
item.EncodingBytes = item.Groups.Sum(n =>
{
long sumLength = 0;
for (int i = 0; i < n.InformationLength; i++)
{
if (_cacheManager.Contains(n.Keys[i]))
{
sumLength += (long)_cacheManager.GetLength(n.Keys[i]);
}
}
return(sumLength);
});
item.Keys.RemoveRange(0, length);
}
}
else if (item.Groups.Count > 0 && item.Keys.Count == 0)
{
item.State = UploadState.Encoding;
var index = new Index();
index.Groups.AddRange(item.Groups);
index.CompressionAlgorithm = item.CompressionAlgorithm;
index.CryptoAlgorithm = item.CryptoAlgorithm;
index.CryptoKey = item.CryptoKey;
item.Indexes.Add(index);
byte[] cryptoKey = null;
KeyCollection keys = null;
try
{
using (var stream = index.Export(_bufferManager))
using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
using (ProgressStream encodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
{
item.EncodeBytes = stream.Length;
item.State = UploadState.ComputeHash;
if (item.HashAlgorithm == HashAlgorithm.Sha256)
{
cryptoKey = Sha256.ComputeHash(hashProgressStream);
}
stream.Seek(0, SeekOrigin.Begin);
item.EncodingBytes = 0;
item.State = UploadState.Encoding;
keys = _cacheManager.Encoding(encodingProgressStream, item.CompressionAlgorithm, item.CryptoAlgorithm, cryptoKey, item.BlockLength, item.HashAlgorithm);
}
}
catch (StopIoException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in keys)
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.EncodingBytes = 0;
item.EncodeBytes = 0;
item.CryptoKey = cryptoKey;
item.Keys.AddRange(keys);
item.Rank++;
item.Groups.Clear();
}
}
}
else if (item.Type == UploadType.Share)
{
if (item.Groups.Count == 0 && item.Keys.Count == 0)
{
item.State = UploadState.ComputeHash;
KeyCollection keys = null;
try
{
using (FileStream stream = new FileStream(item.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
{
item.EncodeBytes = stream.Length;
item.Seed.Length = stream.Length;
if (item.Seed.Length == 0)
{
throw new InvalidOperationException("Stream Length");
}
keys = _cacheManager.Share(hashProgressStream, stream.Name, item.HashAlgorithm, item.BlockLength);
}
}
catch (StopIoException)
{
continue;
}
if (keys.Count == 1)
{
lock (this.ThisLock)
{
item.EncodingBytes = 0;
item.EncodeBytes = 0;
item.Keys.Add(keys[0]);
item.State = UploadState.Encoding;
}
}
else
{
var groups = new List <Group>();
for (int i = 0, remain = keys.Count; 0 < remain; i++, remain -= 256)
{
var tempKeys = keys.GetRange(i * 256, Math.Min(remain, 256));
Group group = new Group();
group.CorrectionAlgorithm = CorrectionAlgorithm.None;
group.InformationLength = tempKeys.Count;
group.BlockLength = item.BlockLength;
group.Length = tempKeys.Sum(n => (long)_cacheManager.GetLength(n));
group.Keys.AddRange(tempKeys);
groups.Add(group);
}
lock (this.ThisLock)
{
item.EncodingBytes = 0;
item.EncodeBytes = 0;
foreach (var key in keys)
{
item.UploadKeys.Add(key);
}
item.Groups.AddRange(groups);
item.State = UploadState.Encoding;
}
}
}
else if (item.Groups.Count == 0 && item.Keys.Count == 1)
{
lock (this.ThisLock)
{
item.Seed.Rank = item.Rank;
item.Seed.Key = item.Keys[0];
item.Keys.Clear();
if (item.Rank != 1)
{
item.Seed.CompressionAlgorithm = item.CompressionAlgorithm;
item.Seed.CryptoAlgorithm = item.CryptoAlgorithm;
item.Seed.CryptoKey = item.CryptoKey;
}
if (item.DigitalSignature != null)
{
item.Seed.CreateCertificate(item.DigitalSignature);
}
item.UploadKeys.Add(item.Seed.Key);
foreach (var key in item.UploadKeys)
{
_connectionsManager.Upload(key);
}
this.CheckState(item);
_cacheManager.SetSeed(item.Seed.Clone(), item.FilePath, item.Indexes.Select(n => n.Clone()).ToArray());
item.Indexes.Clear();
foreach (var key in item.LockedKeys)
{
_cacheManager.Unlock(key);
}
item.LockedKeys.Clear();
item.State = UploadState.Uploading;
}
}
else if (item.Keys.Count > 0)
{
item.State = UploadState.ParityEncoding;
item.EncodeBytes = item.Groups.Sum(n =>
{
long sumLength = 0;
for (int i = 0; i < n.InformationLength; i++)
{
if (_cacheManager.Contains(n.Keys[i]))
{
sumLength += (long)_cacheManager.GetLength(n.Keys[i]);
}
}
return(sumLength);
}) + item.Keys.Sum(n =>
{
if (_cacheManager.Contains(n))
{
return((long)_cacheManager.GetLength(n));
}
return(0);
});
var length = Math.Min(item.Keys.Count, 128);
var keys = new KeyCollection(item.Keys.Take(length));
Group group = null;
try
{
group = _cacheManager.ParityEncoding(keys, item.HashAlgorithm, item.BlockLength, item.CorrectionAlgorithm, (object state2) =>
{
return(this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
});
}
catch (StopException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in group.Keys.Skip(group.InformationLength))
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.Groups.Add(group);
item.EncodingBytes = item.Groups.Sum(n =>
{
long sumLength = 0;
for (int i = 0; i < n.InformationLength; i++)
{
if (_cacheManager.Contains(n.Keys[i]))
{
sumLength += (long)_cacheManager.GetLength(n.Keys[i]);
}
}
return(sumLength);
});
item.Keys.RemoveRange(0, length);
}
}
else if (item.Groups.Count > 0 && item.Keys.Count == 0)
{
item.State = UploadState.Encoding;
var index = new Index();
index.Groups.AddRange(item.Groups);
if (item.Rank != 1)
{
index.CompressionAlgorithm = item.CompressionAlgorithm;
index.CryptoAlgorithm = item.CryptoAlgorithm;
index.CryptoKey = item.CryptoKey;
item.Indexes.Add(index);
}
byte[] cryptoKey = null;
KeyCollection keys = null;
try
{
using (var stream = index.Export(_bufferManager))
using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
using (ProgressStream encodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
{
item.EncodeBytes = stream.Length;
item.State = UploadState.ComputeHash;
if (item.HashAlgorithm == HashAlgorithm.Sha256)
{
cryptoKey = Sha256.ComputeHash(hashProgressStream);
}
stream.Seek(0, SeekOrigin.Begin);
item.EncodingBytes = 0;
item.State = UploadState.Encoding;
keys = _cacheManager.Encoding(encodingProgressStream, item.CompressionAlgorithm, item.CryptoAlgorithm, cryptoKey, item.BlockLength, item.HashAlgorithm);
}
}
catch (StopIoException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in keys)
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.EncodingBytes = 0;
item.EncodeBytes = 0;
item.CryptoKey = cryptoKey;
item.Keys.AddRange(keys);
item.Rank++;
item.Groups.Clear();
}
}
}
}
catch (Exception e)
{
item.State = UploadState.Error;
Log.Error(e);
}
finally
{
_workingPaths.Remove(item.FilePath);
}
}
}
