public virtual void Load(string directoryPath)
{
Stopwatch sw = new Stopwatch();
sw.Start();
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
// Tempファイルを削除する。
{
foreach (var path in Directory.GetFiles(directoryPath, "*", SearchOption.TopDirectoryOnly))
{
var ext = Path.GetExtension(path);
if (ext == ".tmp")
{
try
{
File.Delete(path);
}
catch (Exception)
{
}
}
}
}
LockedHashSet <string> successNames = new LockedHashSet <string>();
// DataContractSerializerのBinaryバージョン
foreach (var extension in new string[] { ".v2", ".v2.bak" })
{
Parallel.ForEach(Directory.GetFiles(directoryPath), new ParallelOptions()
{
MaxDegreeOfParallelism = 8
}, configPath =>
{
if (!configPath.EndsWith(extension))
{
return;
}
var name = Path.GetFileName(configPath.Substring(0, configPath.Length - extension.Length));
if (successNames.Contains(name))
{
return;
}
Content content = null;
if (!_dic.TryGetValue(name, out content))
{
return;
}
try
{
using (FileStream stream = new FileStream(configPath, FileMode.Open))
using (CacheStream cacheStream = new CacheStream(stream, _cacheSize, BufferManager.Instance))
using (GZipStream decompressStream = new GZipStream(cacheStream, CompressionMode.Decompress))
{
//using (XmlDictionaryReader xmlDictionaryReader = XmlDictionaryReader.CreateTextReader(decompressStream, XmlDictionaryReaderQuotas.Max))
using (XmlDictionaryReader xml = XmlDictionaryReader.CreateBinaryReader(decompressStream, XmlDictionaryReaderQuotas.Max))
{
var deserializer = new DataContractSerializer(content.Type);
content.Value = deserializer.ReadObject(xml);
}
}
successNames.Add(name);
}
catch (Exception e)
{
Log.Warning(e);
}
});
}
// DataContractSerializerのTextバージョン
// 互換性は高いが処理速度が遅い。
foreach (var extension in new string[] { ".gz", ".gz.bak" })
{
foreach (var configPath in Directory.GetFiles(directoryPath))
{
if (!configPath.EndsWith(extension))
{
continue;
}
var name = Path.GetFileName(configPath.Substring(0, configPath.Length - extension.Length));
if (successNames.Contains(name))
{
continue;
}
Content content = null;
if (!_dic.TryGetValue(name, out content))
{
continue;
}
try
{
using (FileStream stream = new FileStream(configPath, FileMode.Open))
using (CacheStream cacheStream = new CacheStream(stream, _cacheSize, BufferManager.Instance))
using (GZipStream decompressStream = new GZipStream(cacheStream, CompressionMode.Decompress))
{
using (XmlDictionaryReader xml = XmlDictionaryReader.CreateTextReader(decompressStream, XmlDictionaryReaderQuotas.Max))
{
var deserializer = new DataContractSerializer(content.Type);
content.Value = deserializer.ReadObject(xml);
}
}
successNames.Add(name);
}
catch (Exception e)
{
Log.Warning(e);
}
}
}
sw.Stop();
Debug.WriteLine("Settings Load {0} {1}", Path.GetFileName(directoryPath), sw.ElapsedMilliseconds);
}
private void EncodeManagerThread()
{
for (; ;)
{
Thread.Sleep(1000 * 1);
if (this.EncodeState == ManagerState.Stop)
{
return;
}
UploadItem item = null;
try
{
lock (this.ThisLock)
{
if (_settings.UploadItems.Count > 0)
{
item = _settings.UploadItems
.Where(n => n.State == UploadState.ComputeHash || n.State == UploadState.Encoding || n.State == UploadState.ParityEncoding)
.Where(n => n.Priority != 0)
.OrderBy(n => - n.Priority)
.Where(n => !_workingPaths.Contains(n.FilePath))
.FirstOrDefault();
if (item != null)
{
_workingPaths.Add(item.FilePath);
}
}
}
}
catch (Exception)
{
return;
}
if (item == null)
{
continue;
}
try
{
if (item.Type == UploadType.Upload)
{
if (item.Groups.Count == 0 && item.Keys.Count == 0)
{
item.State = UploadState.Encoding;
KeyCollection keys = null;
byte[] cryptoKey = null;
try
{
using (FileStream stream = new FileStream(item.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
using (ProgressStream encodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
{
item.EncodeBytes = stream.Length;
item.Seed.Length = stream.Length;
if (item.Seed.Length == 0)
{
throw new InvalidOperationException("Stream Length");
}
item.State = UploadState.ComputeHash;
if (item.HashAlgorithm == HashAlgorithm.Sha256)
{
cryptoKey = Sha256.ComputeHash(hashProgressStream);
}
stream.Seek(0, SeekOrigin.Begin);
item.EncodingBytes = 0;
item.State = UploadState.Encoding;
keys = _cacheManager.Encoding(encodingProgressStream, item.CompressionAlgorithm, item.CryptoAlgorithm, cryptoKey, item.BlockLength, item.HashAlgorithm);
}
}
catch (StopIoException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in keys)
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.EncodingBytes = 0;
item.EncodeBytes = 0;
item.CryptoKey = cryptoKey;
item.Keys.AddRange(keys);
}
}
else if (item.Groups.Count == 0 && item.Keys.Count == 1)
{
lock (this.ThisLock)
{
item.Seed.Rank = item.Rank;
item.Seed.Key = item.Keys[0];
item.Keys.Clear();
item.Seed.CompressionAlgorithm = item.CompressionAlgorithm;
item.Seed.CryptoAlgorithm = item.CryptoAlgorithm;
item.Seed.CryptoKey = item.CryptoKey;
if (item.DigitalSignature != null)
{
item.Seed.CreateCertificate(item.DigitalSignature);
}
item.UploadKeys.Add(item.Seed.Key);
foreach (var key in item.UploadKeys)
{
_connectionsManager.Upload(key);
}
this.CheckState(item);
_cacheManager.SetSeed(item.Seed.Clone(), item.Indexes.Select(n => n.Clone()).ToArray());
item.Indexes.Clear();
foreach (var key in item.LockedKeys)
{
_cacheManager.Unlock(key);
}
item.LockedKeys.Clear();
item.State = UploadState.Uploading;
}
}
else if (item.Keys.Count > 0)
{
item.State = UploadState.ParityEncoding;
item.EncodeBytes = item.Groups.Sum(n =>
{
long sumLength = 0;
for (int i = 0; i < n.InformationLength; i++)
{
if (_cacheManager.Contains(n.Keys[i]))
{
sumLength += (long)_cacheManager.GetLength(n.Keys[i]);
}
}
return(sumLength);
}) + item.Keys.Sum(n =>
{
if (_cacheManager.Contains(n))
{
return((long)_cacheManager.GetLength(n));
}
return(0);
});
var length = Math.Min(item.Keys.Count, 128);
var keys = new KeyCollection(item.Keys.Take(length));
Group group = null;
try
{
group = _cacheManager.ParityEncoding(keys, item.HashAlgorithm, item.BlockLength, item.CorrectionAlgorithm, (object state2) =>
{
return(this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
});
}
catch (StopException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in group.Keys.Skip(group.InformationLength))
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.Groups.Add(group);
item.EncodingBytes = item.Groups.Sum(n =>
{
long sumLength = 0;
for (int i = 0; i < n.InformationLength; i++)
{
if (_cacheManager.Contains(n.Keys[i]))
{
sumLength += (long)_cacheManager.GetLength(n.Keys[i]);
}
}
return(sumLength);
});
item.Keys.RemoveRange(0, length);
}
}
else if (item.Groups.Count > 0 && item.Keys.Count == 0)
{
item.State = UploadState.Encoding;
var index = new Index();
index.Groups.AddRange(item.Groups);
index.CompressionAlgorithm = item.CompressionAlgorithm;
index.CryptoAlgorithm = item.CryptoAlgorithm;
index.CryptoKey = item.CryptoKey;
item.Indexes.Add(index);
byte[] cryptoKey = null;
KeyCollection keys = null;
try
{
using (var stream = index.Export(_bufferManager))
using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
using (ProgressStream encodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
{
item.EncodeBytes = stream.Length;
item.State = UploadState.ComputeHash;
if (item.HashAlgorithm == HashAlgorithm.Sha256)
{
cryptoKey = Sha256.ComputeHash(hashProgressStream);
}
stream.Seek(0, SeekOrigin.Begin);
item.EncodingBytes = 0;
item.State = UploadState.Encoding;
keys = _cacheManager.Encoding(encodingProgressStream, item.CompressionAlgorithm, item.CryptoAlgorithm, cryptoKey, item.BlockLength, item.HashAlgorithm);
}
}
catch (StopIoException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in keys)
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.EncodingBytes = 0;
item.EncodeBytes = 0;
item.CryptoKey = cryptoKey;
item.Keys.AddRange(keys);
item.Rank++;
item.Groups.Clear();
}
}
}
else if (item.Type == UploadType.Share)
{
if (item.Groups.Count == 0 && item.Keys.Count == 0)
{
item.State = UploadState.ComputeHash;
KeyCollection keys = null;
try
{
using (FileStream stream = new FileStream(item.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
{
item.EncodeBytes = stream.Length;
item.Seed.Length = stream.Length;
if (item.Seed.Length == 0)
{
throw new InvalidOperationException("Stream Length");
}
keys = _cacheManager.Share(hashProgressStream, stream.Name, item.HashAlgorithm, item.BlockLength);
}
}
catch (StopIoException)
{
continue;
}
if (keys.Count == 1)
{
lock (this.ThisLock)
{
item.EncodingBytes = 0;
item.EncodeBytes = 0;
item.Keys.Add(keys[0]);
item.State = UploadState.Encoding;
}
}
else
{
var groups = new List <Group>();
for (int i = 0, remain = keys.Count; 0 < remain; i++, remain -= 256)
{
var tempKeys = keys.GetRange(i * 256, Math.Min(remain, 256));
Group group = new Group();
group.CorrectionAlgorithm = CorrectionAlgorithm.None;
group.InformationLength = tempKeys.Count;
group.BlockLength = item.BlockLength;
group.Length = tempKeys.Sum(n => (long)_cacheManager.GetLength(n));
group.Keys.AddRange(tempKeys);
groups.Add(group);
}
lock (this.ThisLock)
{
item.EncodingBytes = 0;
item.EncodeBytes = 0;
foreach (var key in keys)
{
item.UploadKeys.Add(key);
}
item.Groups.AddRange(groups);
item.State = UploadState.Encoding;
}
}
}
else if (item.Groups.Count == 0 && item.Keys.Count == 1)
{
lock (this.ThisLock)
{
item.Seed.Rank = item.Rank;
item.Seed.Key = item.Keys[0];
item.Keys.Clear();
if (item.Rank != 1)
{
item.Seed.CompressionAlgorithm = item.CompressionAlgorithm;
item.Seed.CryptoAlgorithm = item.CryptoAlgorithm;
item.Seed.CryptoKey = item.CryptoKey;
}
if (item.DigitalSignature != null)
{
item.Seed.CreateCertificate(item.DigitalSignature);
}
item.UploadKeys.Add(item.Seed.Key);
foreach (var key in item.UploadKeys)
{
_connectionsManager.Upload(key);
}
this.CheckState(item);
_cacheManager.SetSeed(item.Seed.Clone(), item.FilePath, item.Indexes.Select(n => n.Clone()).ToArray());
item.Indexes.Clear();
foreach (var key in item.LockedKeys)
{
_cacheManager.Unlock(key);
}
item.LockedKeys.Clear();
item.State = UploadState.Uploading;
}
}
else if (item.Keys.Count > 0)
{
item.State = UploadState.ParityEncoding;
item.EncodeBytes = item.Groups.Sum(n =>
{
long sumLength = 0;
for (int i = 0; i < n.InformationLength; i++)
{
if (_cacheManager.Contains(n.Keys[i]))
{
sumLength += (long)_cacheManager.GetLength(n.Keys[i]);
}
}
return(sumLength);
}) + item.Keys.Sum(n =>
{
if (_cacheManager.Contains(n))
{
return((long)_cacheManager.GetLength(n));
}
return(0);
});
var length = Math.Min(item.Keys.Count, 128);
var keys = new KeyCollection(item.Keys.Take(length));
Group group = null;
try
{
group = _cacheManager.ParityEncoding(keys, item.HashAlgorithm, item.BlockLength, item.CorrectionAlgorithm, (object state2) =>
{
return(this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
});
}
catch (StopException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in group.Keys.Skip(group.InformationLength))
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.Groups.Add(group);
item.EncodingBytes = item.Groups.Sum(n =>
{
long sumLength = 0;
for (int i = 0; i < n.InformationLength; i++)
{
if (_cacheManager.Contains(n.Keys[i]))
{
sumLength += (long)_cacheManager.GetLength(n.Keys[i]);
}
}
return(sumLength);
});
item.Keys.RemoveRange(0, length);
}
}
else if (item.Groups.Count > 0 && item.Keys.Count == 0)
{
item.State = UploadState.Encoding;
var index = new Index();
index.Groups.AddRange(item.Groups);
if (item.Rank != 1)
{
index.CompressionAlgorithm = item.CompressionAlgorithm;
index.CryptoAlgorithm = item.CryptoAlgorithm;
index.CryptoKey = item.CryptoKey;
item.Indexes.Add(index);
}
byte[] cryptoKey = null;
KeyCollection keys = null;
try
{
using (var stream = index.Export(_bufferManager))
using (ProgressStream hashProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
using (ProgressStream encodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.EncodeState == ManagerState.Stop || !_settings.UploadItems.Contains(item));
item.EncodingBytes = Math.Min(readSize, stream.Length);
}, 1024 * 1024, true))
{
item.EncodeBytes = stream.Length;
item.State = UploadState.ComputeHash;
if (item.HashAlgorithm == HashAlgorithm.Sha256)
{
cryptoKey = Sha256.ComputeHash(hashProgressStream);
}
stream.Seek(0, SeekOrigin.Begin);
item.EncodingBytes = 0;
item.State = UploadState.Encoding;
keys = _cacheManager.Encoding(encodingProgressStream, item.CompressionAlgorithm, item.CryptoAlgorithm, cryptoKey, item.BlockLength, item.HashAlgorithm);
}
}
catch (StopIoException)
{
continue;
}
lock (this.ThisLock)
{
foreach (var key in keys)
{
item.UploadKeys.Add(key);
item.LockedKeys.Add(key);
}
item.EncodingBytes = 0;
item.EncodeBytes = 0;
item.CryptoKey = cryptoKey;
item.Keys.AddRange(keys);
item.Rank++;
item.Groups.Clear();
}
}
}
}
catch (Exception e)
{
item.State = UploadState.Error;
Log.Error(e);
}
finally
{
_workingPaths.Remove(item.FilePath);
}
}
}
private void DecodingThread(CancellationToken token)
{
for (; ;)
{
if (token.WaitHandle.WaitOne(1000))
{
return;
}
DownloadItemInfo item = null;
lock (_lockObject)
{
item = CollectionUtils.Unite(_volatileDownloadItemInfoManager, _downloadItemInfoManager)
.Where(n => !_workingItems.Contains(n))
.Where(n => n.State == DownloadState.Decoding || n.State == DownloadState.ParityDecoding)
.OrderBy(n => (n.Depth == n.Metadata.Depth) ? 0 : 1)
.OrderBy(n => (n.State == DownloadState.Decoding) ? 0 : 1)
.FirstOrDefault();
if (item != null)
{
_workingItems.Add(item);
}
}
if (item == null)
{
continue;
}
try
{
if ((item.Depth == 0 && !_cacheManager.Contains(item.Metadata.Hash)) ||
(item.Depth > 0 && !item.Index.Groups.All(n => _existManager.GetCount(n, true) >= n.Hashes.Count() / 2)))
{
item.State = DownloadState.Downloading;
}
else
{
var hashes = new HashCollection();
var totalHashes = new HashCollection();
if (item.Depth == 0)
{
hashes.Add(item.Metadata.Hash);
totalHashes.Add(item.Metadata.Hash);
}
else
{
try
{
foreach (var group in item.Index.Groups)
{
if (item.State == DownloadState.Error)
{
throw new OperationCanceledException();
}
hashes.AddRange(_cacheManager.ParityDecoding(group, token).Result);
}
totalHashes.AddRange(item.Index.Groups.SelectMany(n => n.Hashes));
}
catch (OperationCanceledException)
{
continue;
}
item.State = DownloadState.Decoding;
}
if (item.Depth < item.Metadata.Depth)
{
Index index;
try
{
using (var stream = _cacheManager.Decoding(hashes))
using (var progressStream = new ProgressStream(stream, null, 1024 * 1024, token))
{
if (item.State == DownloadState.Error)
{
throw new OperationCanceledException();
}
if (progressStream.Length > item.MaxLength)
{
throw new ArgumentException();
}
index = Index.Import(progressStream, _bufferManager);
}
}
catch (OperationCanceledException)
{
continue;
}
lock (_lockObject)
{
if (item.Path != null)
{
_protectCacheInfoManager.Add(new ProtectedCacheInfo(DateTime.UtcNow, totalHashes));
}
this.CheckState(index);
this.UncheckState(item.Index);
item.Index = index;
item.Depth++;
item.State = DownloadState.Downloading;
}
}
else
{
if (item.Path != null)
{
string filePath = null;
try
{
token.ThrowIfCancellationRequested();
string targetPath;
if (Path.IsPathRooted(item.Path))
{
targetPath = item.Path;
}
else
{
targetPath = Path.GetFullPath(Path.Combine(_basePath, item.Path));
// ディレクトリトラバーサル対策
if (!targetPath.StartsWith(Path.GetFullPath(_basePath)))
{
targetPath = Path.GetFullPath(Path.Combine(_basePath, Path.GetFileName(item.Path)));
}
}
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
using (var inStream = _cacheManager.Decoding(hashes))
using (var outStream = DownloadManager.GetUniqueFileStream(targetPath + ".tmp"))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 1024))
{
filePath = outStream.Name;
int readLength;
while ((readLength = inStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
if (item.State == DownloadState.Error)
{
throw new OperationCanceledException();
}
token.ThrowIfCancellationRequested();
outStream.Write(safeBuffer.Value, 0, readLength);
}
}
File.Move(filePath, DownloadManager.GetUniqueFilePath(targetPath));
}
catch (OperationCanceledException)
{
if (filePath != null)
{
File.Delete(filePath);
}
continue;
}
}
lock (_lockObject)
{
if (item.Path != null)
{
_protectCacheInfoManager.Add(new ProtectedCacheInfo(DateTime.UtcNow, totalHashes));
}
item.ResultHashes.AddRange(hashes);
item.State = DownloadState.Completed;
}
}
}
}
catch (Exception e)
{
item.State = DownloadState.Error;
Log.Error(e);
}
finally
{
_workingItems.Remove(item);
}
}
}
private void MessageUploadAndDownload(IEnumerable <CoreManager> coreManagers)
{
_callback.Invoke("----- CoreManager Message Send and Receive Test -----");
_callback.Invoke("");
var coreManagerList = coreManagers.ToList();
Parallel.ForEach(coreManagerList, coreManager =>
{
coreManager.Resize((long)1024 * 1024 * 1024 * 32).Wait();
});
var hashList = new LockedHashSet <Hash>();
var metadataList = new LockedList <Metadata>();
Parallel.For(0, 30, new ParallelOptions()
{
MaxDegreeOfParallelism = 3
}, i =>
{
var random = RandomProvider.GetThreadRandom();
using (var stream = new BufferStream(_bufferManager))
{
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
for (long remain = (long)1024 * 1024 * 32; remain > 0; remain = Math.Max(0, remain - safeBuffer.Value.Length))
{
int length = (int)Math.Min(remain, safeBuffer.Value.Length);
random.NextBytes(safeBuffer.Value);
stream.Write(safeBuffer.Value, 0, length);
}
}
stream.Seek(0, SeekOrigin.Begin);
hashList.Add(new Hash(HashAlgorithm.Sha256, Sha256.ComputeHash(new WrapperStream(stream, true))));
stream.Seek(0, SeekOrigin.Begin);
using (var tokenSource = new CancellationTokenSource())
{
metadataList.Add(coreManagerList[i].VolatileSetStream(stream, new TimeSpan(1, 0, 0), tokenSource.Token).Result);
}
}
});
var sw = Stopwatch.StartNew();
Parallel.ForEach(metadataList, metadata =>
{
for (; ;)
{
Thread.Sleep(1000);
Stream stream = null;
try
{
stream = coreManagerList[0].VolatileGetStream(metadata, 1024 * 1024 * 256);
if (stream == null)
{
continue;
}
var hash = new Hash(HashAlgorithm.Sha256, Sha256.ComputeHash(stream));
if (!hashList.Contains(hash))
{
throw new ArgumentException("Broken");
}
_callback.Invoke($"{sw.Elapsed.ToString("hh\\:mm\\:ss")}: Success {NetworkConverter.ToBase64UrlString(metadata.Hash.Value)}");
return;
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
});
_callback.Invoke("----- End -----");
}