private void EncodeThread() { for (;;) { Thread.Sleep(1000 * 1); if (this.EncodeState == ManagerState.Stop) return; UploadItem item = null; try { lock (_thisLock) { if (_settings.UploadItems.Count > 0) { item = _settings.UploadItems .Where(n => n.State == UploadState.ComputeHash || n.State == UploadState.Encoding || n.State == UploadState.ParityEncoding) .Where(n => n.Priority != 0) .OrderBy(n => -n.Priority) .Where(n => !_workingPaths.Contains(n.FilePath)) .FirstOrDefault(); if (item != null) { _workingPaths.Add(item.FilePath); } } } } catch (Exception) { return; } if (item == null) continue; try { if (item.Groups.Count == 0 && item.Keys.Count == 0) { if (item.Type == UploadType.Upload) { item.State = UploadState.Encoding; KeyCollection keys = null; byte[] cryptoKey = null; try { using (var stream = new UnbufferedFileStream(item.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferManager)) using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) => { isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item)); item.EncodeOffset = Math.Min(readSize, stream.Length); }, 1024 * 1024, true)) using (ProgressStream encodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) => { isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item)); item.EncodeOffset = Math.Min(readSize, stream.Length); }, 1024 * 1024, true)) { if (stream.Length == 0) throw new InvalidOperationException("Stream Length"); item.Length = stream.Length; item.EncodeLength = stream.Length; item.State = UploadState.ComputeHash; if (item.HashAlgorithm == HashAlgorithm.Sha256) { cryptoKey = Sha256.ComputeHash(hashProgressStream); } stream.Seek(0, SeekOrigin.Begin); item.EncodeOffset = 0; item.State = UploadState.Encoding; keys = _cacheManager.Encoding(encodingProgressStream, item.CompressionAlgorithm, item.CryptoAlgorithm, cryptoKey, item.BlockLength, item.HashAlgorithm); } } catch (StopIoException) { continue; } lock (_thisLock) { if (!_settings.UploadItems.Contains(item)) { foreach (var key in keys) { _cacheManager.Unlock(key); } continue; } foreach (var key in keys) { item.UploadKeys.Add(key); item.LockedKeys.Add(key); } item.EncodeOffset = 0; item.EncodeLength = 0; item.CryptoKey = cryptoKey; item.Keys.AddRange(keys); } } else if (item.Type == UploadType.Share) { item.State = UploadState.ComputeHash; KeyCollection keys = null; try { using (var stream = new UnbufferedFileStream(item.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferManager)) using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) => { isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item)); item.EncodeOffset = Math.Min(readSize, stream.Length); }, 1024 * 1024, true)) { if (stream.Length == 0) throw new InvalidOperationException("Stream Length"); item.Length = stream.Length; item.EncodeLength = stream.Length; keys = _cacheManager.Share(hashProgressStream, stream.Name, item.HashAlgorithm, item.BlockLength); } } catch (StopIoException) { continue; } if (keys.Count == 1) { lock (_thisLock) { if (!_settings.UploadItems.Contains(item)) { foreach (var key in keys) { _cacheManager.Unlock(key); } continue; } item.EncodeOffset = 0; item.EncodeLength = 0; item.UploadKeys.Add(keys[0]); item.Keys.Add(keys[0]); item.State = UploadState.Encoding; } } else { var groups = new List<Group>(); for (int i = 0, remain = keys.Count; 0 < remain; i++, remain -= 256) { var tempKeys = keys.GetRange(i * 256, Math.Min(remain, 256)); var group = new Group(); group.CorrectionAlgorithm = CorrectionAlgorithm.None; group.InformationLength = tempKeys.Count; group.BlockLength = item.BlockLength; group.Length = tempKeys.Sum(n => (long)_cacheManager.GetLength(n)); group.Keys.AddRange(tempKeys); groups.Add(group); } lock (_thisLock) { if (!_settings.UploadItems.Contains(item)) { foreach (var key in keys) { _cacheManager.Unlock(key); } continue; } item.EncodeOffset = 0; item.EncodeLength = 0; foreach (var key in keys) { item.UploadKeys.Add(key); } item.Groups.AddRange(groups); item.State = UploadState.Encoding; } } } } else if (item.Groups.Count == 0 && item.Keys.Count == 1) { lock (_thisLock) { if (!_settings.UploadItems.Contains(item)) continue; Metadata metadata = null; { if (item.Type == UploadType.Upload) { metadata = new Metadata(item.Depth, item.Keys[0], item.CompressionAlgorithm, item.CryptoAlgorithm, item.CryptoKey); } else if (item.Type == UploadType.Share) { if (item.Depth == 1) { metadata = new Metadata(item.Depth, item.Keys[0], CompressionAlgorithm.None, CryptoAlgorithm.None, null); } else { metadata = new Metadata(item.Depth, item.Keys[0], item.CompressionAlgorithm, item.CryptoAlgorithm, item.CryptoKey); } } item.Keys.Clear(); } item.Seed = new Seed(metadata); item.Seed.Name = item.Name; item.Seed.Length = item.Length; item.Seed.CreationTime = item.CreationTime; item.Seed.Keywords.AddRange(item.Keywords); if (item.DigitalSignature != null) { item.Seed.CreateCertificate(item.DigitalSignature); } foreach (var key in item.UploadKeys) { _connectionsManager.Upload(key); } { if (item.Type == UploadType.Upload) { _cacheManager.SetSeed(item.Seed.Clone(), item.RetainKeys.ToArray()); } else if (item.Type == UploadType.Share) { _cacheManager.SetSeed(item.Seed.Clone(), item.FilePath, item.RetainKeys.ToArray()); } item.RetainKeys.Clear(); } foreach (var key in item.LockedKeys) { _cacheManager.Unlock(key); } item.LockedKeys.Clear(); item.State = UploadState.Uploading; this.CheckState(item); } } else if (item.Keys.Count > 0) { item.State = UploadState.ParityEncoding; item.EncodeLength = item.Groups.Sum(n => { long sumLength = 0; for (int i = 0; i < n.InformationLength; i++) { if (_cacheManager.Contains(n.Keys[i])) { sumLength += (long)_cacheManager.GetLength(n.Keys[i]); } } return sumLength; }) + item.Keys.Sum(n => { if (_cacheManager.Contains(n)) { return (long)_cacheManager.GetLength(n); } return 0; }); var length = Math.Min(item.Keys.Count, 128); var keys = new KeyCollection(item.Keys.Take(length)); Group group = null; try { using (var tokenSource = new CancellationTokenSource()) { var task = _cacheManager.ParityEncoding(keys, item.HashAlgorithm, item.BlockLength, item.CorrectionAlgorithm, tokenSource.Token); while (!task.IsCompleted) { if ((this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item))) tokenSource.Cancel(); Thread.Sleep(1000); } group = task.Result; } } catch (Exception) { continue; } lock (_thisLock) { if (!_settings.UploadItems.Contains(item)) { foreach (var key in group.Keys.Skip(group.InformationLength)) { _cacheManager.Unlock(key); } continue; } foreach (var key in group.Keys.Skip(group.InformationLength)) { item.UploadKeys.Add(key); item.LockedKeys.Add(key); } foreach (var key in group.Keys.Skip(group.Keys.Count - group.InformationLength)) { item.RetainKeys.Add(key); } item.Groups.Add(group); item.EncodeOffset = item.Groups.Sum(n => { long sumLength = 0; for (int i = 0; i < n.InformationLength; i++) { if (_cacheManager.Contains(n.Keys[i])) { sumLength += (long)_cacheManager.GetLength(n.Keys[i]); } } return sumLength; }); item.Keys.RemoveRange(0, length); } } else if (item.Groups.Count > 0 && item.Keys.Count == 0) { item.State = UploadState.Encoding; var index = new Index(); if (item.Type == UploadType.Upload) { index.Groups.AddRange(item.Groups); index.CompressionAlgorithm = item.CompressionAlgorithm; index.CryptoAlgorithm = item.CryptoAlgorithm; index.CryptoKey = item.CryptoKey; } else if (item.Type == UploadType.Share) { index.Groups.AddRange(item.Groups); if (item.Depth != 1) { index.CompressionAlgorithm = item.CompressionAlgorithm; index.CryptoAlgorithm = item.CryptoAlgorithm; index.CryptoKey = item.CryptoKey; } } byte[] cryptoKey = null; KeyCollection keys = null; try { using (var stream = index.Export(_bufferManager)) using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) => { isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item)); item.EncodeOffset = Math.Min(readSize, stream.Length); }, 1024 * 1024, true)) using (ProgressStream encodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) => { isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item)); item.EncodeOffset = Math.Min(readSize, stream.Length); }, 1024 * 1024, true)) { item.EncodeLength = stream.Length; item.State = UploadState.ComputeHash; if (item.HashAlgorithm == HashAlgorithm.Sha256) { cryptoKey = Sha256.ComputeHash(hashProgressStream); } stream.Seek(0, SeekOrigin.Begin); item.EncodeOffset = 0; item.State = UploadState.Encoding; keys = _cacheManager.Encoding(encodingProgressStream, item.CompressionAlgorithm, item.CryptoAlgorithm, cryptoKey, item.BlockLength, item.HashAlgorithm); } } catch (StopIoException) { continue; } lock (_thisLock) { if (!_settings.UploadItems.Contains(item)) { foreach (var key in keys) { _cacheManager.Unlock(key); } continue; } foreach (var key in keys) { item.UploadKeys.Add(key); item.LockedKeys.Add(key); } item.EncodeOffset = 0; item.EncodeLength = 0; item.CryptoKey = cryptoKey; item.Keys.AddRange(keys); item.Depth++; item.Groups.Clear(); } } } catch (Exception e) { item.State = UploadState.Error; Log.Error(e); } finally { _workingPaths.Remove(item.FilePath); } } }
private void DecodeThread() { var random = new Random(); for (;;) { Thread.Sleep(1000 * 3); if (this.DecodeState == ManagerState.Stop) return; DownloadItem item = null; try { lock (_thisLock) { if (_settings.DownloadItems.Count > 0) { item = _settings.DownloadItems .Where(n => !_workingSeeds.Contains(n.Seed)) .Where(n => n.State == DownloadState.Decoding || n.State == DownloadState.ParityDecoding) .Where(n => n.Priority != 0) .OrderBy(n => (n.Depth != n.Seed.Metadata.Depth) ? 0 : 1) .OrderBy(n => (n.State == DownloadState.Decoding) ? 0 : 1) .FirstOrDefault(); if (item != null) { _workingSeeds.Add(item.Seed); } } } } catch (Exception) { return; } if (item == null) continue; try { { if ((item.Depth == 1 && !_cacheManager.Contains(item.Seed.Metadata.Key)) || (item.Depth > 1 && !item.Index.Groups.All(n => _existManager.GetCount(n) >= n.InformationLength))) { item.State = DownloadState.Downloading; } else { var keys = new KeyCollection(); var compressionAlgorithm = CompressionAlgorithm.None; var cryptoAlgorithm = CryptoAlgorithm.None; byte[] cryptoKey = null; if (item.Depth == 1) { keys.Add(item.Seed.Metadata.Key); compressionAlgorithm = item.Seed.Metadata.CompressionAlgorithm; cryptoAlgorithm = item.Seed.Metadata.CryptoAlgorithm; cryptoKey = item.Seed.Metadata.CryptoKey; } else { item.State = DownloadState.ParityDecoding; item.DecodeOffset = 0; item.DecodeLength = item.Index.Groups.Sum(n => n.Length); try { foreach (var group in item.Index.Groups.ToArray()) { using (var tokenSource = new CancellationTokenSource()) { var task = _cacheManager.ParityDecoding(group, tokenSource.Token); while (!task.IsCompleted) { if (this.DecodeState == ManagerState.Stop || !_settings.DownloadItems.Contains(item)) tokenSource.Cancel(); Thread.Sleep(1000); } keys.AddRange(task.Result); } item.DecodeOffset += group.Length; } } catch (Exception) { continue; } compressionAlgorithm = item.Index.CompressionAlgorithm; cryptoAlgorithm = item.Index.CryptoAlgorithm; cryptoKey = item.Index.CryptoKey; } item.State = DownloadState.Decoding; if (item.Depth < item.Seed.Metadata.Depth) { string fileName = null; bool largeFlag = false; try { item.DecodeOffset = 0; item.DecodeLength = keys.Sum(n => (long)_cacheManager.GetLength(n)); using (var stream = DownloadManager.GetUniqueFileStream(Path.Combine(_workDirectory, "index"))) using (ProgressStream decodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) => { isStop = (this.DecodeState == ManagerState.Stop || !_settings.DownloadItems.Contains(item)); if (!isStop && (stream.Length > item.Seed.Length)) { isStop = true; largeFlag = true; } item.DecodeOffset = writeSize; }, 1024 * 1024, true)) { fileName = stream.Name; _cacheManager.Decoding(decodingProgressStream, compressionAlgorithm, cryptoAlgorithm, cryptoKey, keys); } } catch (StopIoException) { if (File.Exists(fileName)) { File.Delete(fileName); } if (largeFlag) { throw new Exception("size too large."); } continue; } catch (Exception) { if (File.Exists(fileName)) { File.Delete(fileName); } throw; } Index index; using (var stream = new UnbufferedFileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferManager)) { index = Index.Import(stream, _bufferManager); } File.Delete(fileName); lock (_thisLock) { if (!_settings.DownloadItems.Contains(item)) continue; item.DecodeOffset = 0; item.DecodeLength = 0; this.UncheckState(item.Index); item.Index = index; this.CheckState(item.Index); foreach (var group in item.Index.Groups) { foreach (var key in group.Keys) { _cacheManager.Lock(key); } } item.Indexes.Add(index); item.Depth++; item.State = DownloadState.Downloading; } } else { item.State = DownloadState.Decoding; string fileName = null; bool largeFlag = false; string downloadDirectory; if (item.Path == null) { downloadDirectory = this.BaseDirectory; } else { if (Path.IsPathRooted(item.Path)) { downloadDirectory = item.Path; } else { downloadDirectory = Path.Combine(this.BaseDirectory, item.Path); } } Directory.CreateDirectory(downloadDirectory); try { item.DecodeOffset = 0; item.DecodeLength = keys.Sum(n => (long)_cacheManager.GetLength(n)); using (var stream = DownloadManager.GetUniqueFileStream(Path.Combine(downloadDirectory, string.Format("{0}.tmp", DownloadManager.GetNormalizedPath(item.Seed.Name))))) using (ProgressStream decodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) => { isStop = (this.DecodeState == ManagerState.Stop || !_settings.DownloadItems.Contains(item)); if (!isStop && (stream.Length > item.Seed.Length)) { isStop = true; largeFlag = true; } item.DecodeOffset = writeSize; }, 1024 * 1024, true)) { fileName = stream.Name; _cacheManager.Decoding(decodingProgressStream, compressionAlgorithm, cryptoAlgorithm, cryptoKey, keys); if (stream.Length != item.Seed.Length) throw new Exception("Stream.Length != Seed.Length"); } } catch (StopIoException) { if (File.Exists(fileName)) { File.Delete(fileName); } if (largeFlag) { throw new Exception("size too large."); } continue; } catch (Exception) { if (File.Exists(fileName)) { File.Delete(fileName); } throw; } File.Move(fileName, DownloadManager.GetUniqueFilePath(Path.Combine(downloadDirectory, DownloadManager.GetNormalizedPath(item.Seed.Name)))); lock (_thisLock) { if (!_settings.DownloadItems.Contains(item)) continue; item.DecodeOffset = 0; item.DecodeLength = 0; { var usingKeys = new HashSet<Key>(); foreach (var index in item.Indexes) { foreach (var group in index.Groups) { usingKeys.UnionWith(group.Keys .Where(n => _cacheManager.Contains(n)) .Reverse() .Take(group.InformationLength)); } } _cacheManager.SetSeed(item.Seed.Clone(), usingKeys.ToArray()); } _settings.DownloadedSeeds.Add(item.Seed.Clone()); _cacheManager.Unlock(item.Seed.Metadata.Key); foreach (var index in item.Indexes) { foreach (var group in index.Groups) { foreach (var key in group.Keys) { _cacheManager.Unlock(key); } } } item.Indexes.Clear(); item.State = DownloadState.Completed; } } } } } catch (Exception e) { // Check { var list = new List<Key>(); list.Add(item.Seed.Metadata.Key); foreach (var index in item.Indexes) { foreach (var group in index.Groups) { foreach (var key in group.Keys) { list.Add(key); } } } foreach (var key in list) { if (this.DecodeState == ManagerState.Stop) return; if (!_cacheManager.Contains(key)) continue; var buffer = new ArraySegment<byte>(); try { buffer = _cacheManager[key]; } catch (Exception) { } finally { if (buffer.Array != null) { _bufferManager.ReturnBuffer(buffer.Array); } } } } item.State = DownloadState.Error; Log.Error(e); } finally { _workingSeeds.Remove(item.Seed); } } }