/// <summary>
/// 继续镜像
/// </summary>
public void Continue(IAsyncProgress asyn)
{
if (null != MirrorService && MirrorService.EnableSuspend)
{
MirrorService.Continue(asyn);
}
}
/// <summary>
///
/// </summary>
/// <param name="script"></param>
/// <param name="asyn"></param>
/// <param name="argrument"></param>
/// <param name="paramValues"></param>
/// <param name="isThrowExeception"></param>
/// <returns></returns>
public object Execute(string script, IAsyncProgress asyn, object[] argrument = null, Dictionary <string, object> paramValues = null, bool isThrowExeception = true)
{
try
{
if (!FileHelper.IsValid(script))
{
throw new Exception("Python File '" + script + "' is not exist!");
}
InitEnvriment(script);
dynamic py = LoadPythonScript(script);
var result = py.main(argrument, asyn);
return(result);
}
catch (Exception e)
{
if (isThrowExeception)
{
throw e;
}
return(null);
}
finally
{
Dispose();
}
}
/// <summary>
/// 采用备份方式拷贝用户数据 支持加密备份 支持IOS8.3及其以上系统
/// </summary>
/// <param name="device">设备</param>
/// <param name="targetPath">目标路径</param>
/// <param name="asyn"></param>
/// <param name="InputPassword">回调方法，如果有密码则调用，在方法内返回密码</param>
/// <returns></returns>
public string CopyUserData(Device device, string targetPath, IAsyncProgress asyn, Func <string> InputPassword)
{
//1.初始化相关参数
InitCopyUserData(device, 100, asyn);
//2.文件拷贝
var res = IOSDeviceCoreDll.CopyUserDataPWD(targetPath, device.ID, CopyUserDataCallback, (b) =>
{
var password = InputPassword();
var pS = Marshal.StringToHGlobalAnsi(password);
Marshal.WriteIntPtr(b, pS);
return(0);
});
if (0 != res)
{
IsCopying = false;
}
else
{
while (IsCopying)
{
Thread.Sleep(500);
}
}
//3.清理
ClearCopyUserData();
return(Path.Combine(targetPath, device.ID));
}
public override void Execute(Mirror mirror, IAsyncProgress asyn)
{
var device = mirror.Source as Device;
DeviceManager = device.DeviceManager as IOSDeviceManager;
//数据缓存路径
var tempSavePath = FileHelper.ConnectPath(mirror.Target, "temp");
FileHelper.CreateExitsDirectorySafe(tempSavePath);
//数据备份
var resPath = DeviceManager.CopyUserData(device, tempSavePath, asyn);
if (!FileHelper.IsValidDictory(resPath))
{//数据拷贝失败！
}
//打包
mirror.Local = FileHelper.ConnectPath(mirror.Target, mirror.TargetFile);
ZipFile.CreateFromDirectory(resPath, mirror.Local);
if (!FileHelper.IsValid(mirror.Local))
{//打包失败！
}
//删除缓存文件
FileHelper.DeleteDirectorySafe(tempSavePath);
}
/// <summary>
/// 采用备份方式拷贝用户数据 支持IOS8.3及其以上系统
/// </summary>
/// <param name="device">设备</param>
/// <param name="targetPath">目标路径</param>
/// <param name="asyn"></param>
/// <returns></returns>
public string CopyUserData(Device device, string targetPath, IAsyncProgress asyn)
{
//1.初始化相关参数
InitCopyUserData(device, 100, asyn);
//2.文件拷贝
var res = IOSDeviceCoreDll.CopyUserData(device.ID, targetPath, _CopyUserDataCallback);
if (0 != res)
{
IsCopying = false;
}
else
{
while (IsCopying)
{
Thread.Sleep(500);
}
}
//3.清理
ClearCopyUserData();
return(Path.Combine(targetPath, device.ID));
}
protected override void Initialize(DataReportPluginArgument arg, IAsyncProgress progress)
{
if (!Directory.Exists(arg.ReportPath))
{
Directory.CreateDirectory(arg.ReportPath);
}
}
/// <summary>
/// 执行数据镜像
/// </summary>
/// <param name="task">任务</param>
/// <param name="mirror">镜像源信息</param>
/// <param name="asyn">异步通知</param>
public void Execute(SPFTask task, Mirror mirror, IAsyncProgress asyn)
{
//生成保存目录
FileHelper.CreateExitsDirectorySafe(mirror.Target);
//镜像
MirrorService = SingleWrapperHelper <MirrorServiceFactory> .GetInstance().GetInstance(mirror);
MirrorService.Execute(mirror, asyn);
if (asyn.IsSuccess && FileHelper.IsValid(mirror.Local))
{//镜像成功
mirror.VerifyCode = FileHelper.MD5FromFileUpper(mirror.Local);
//生成MD5文件
var md5File = mirror.Local.TrimEnd(SPFTask.EXT_MIRROR) + SPFTask.EXT_VERIFYCODE_FILE;
FileHelper.CreateFile(md5File, mirror.VerifyCode, Encoding.UTF8);
//生成设备信息文件
var deviceFile = mirror.Local.TrimEnd(SPFTask.EXT_MIRROR) + SPFTask.EXT_DEVICE;
Serializer.SerializeToBinary(task.Device, deviceFile);
task.TaskMirrorFilePath = mirror.Local;
task.VerifyCodes.Add(new FileVerifyCode()
{
FilePath = mirror.Local, VerifyCode = mirror.VerifyCode
});
}
}
/// <summary>
/// 暂停镜像
/// </summary>
public void Pause(IAsyncProgress asyn)
{
if (null != MirrorService && MirrorService.EnableSuspend)
{
MirrorService.Suspend(asyn);
}
}
/// <summary>
/// 执行JavaScript脚本，该脚本同源SPF脚本格式，在后续只兼容执行，不再添加新脚本
/// </summary>
/// <param name="content">脚本内容（替换了$source之后的内容)</param>
/// <param name="asyn">消息通知</param>
/// <param name="argrument">传入main函数的参数，JavaScript脚本应该为空</param>
/// <param name="paramValues">其它需要设置到脚本的动态参数</param>
/// <param name="isThrowExeception">如果执行出现错误，是否抛出异常</param>
/// <returns></returns>
public object Execute(string content, IAsyncProgress asyn, object[] argrument = null, Dictionary <string, object> paramValues = null, bool isThrowExeception = true)
{
try
{
using (JavascriptContext context = new JavascriptContext())
{
context.SetParameter("XLY", SingleWrapperHelper <XLYEngine> .Instance);
var ac = new Action <object>(s => { SingleWrapperHelper <XLYEngine> .Instance.Debug.Write(s); });
context.SetParameter("log", ac);
if (paramValues != null)
{
foreach (var pk in paramValues)
{
context.SetParameter(pk.Key, pk.Value);
}
}
//所有脚本均从main函数开始执行。
//content += string.Format(@"
// var ___result = main({0});
// ___result;
//", argrument);
var obj = context.Run(content);
return(obj);
}
}
catch (Exception ex)
{
if (isThrowExeception)
{
throw ex;
}
return(null);
}
}
/// <summary>
/// The constructor.
/// </summary>
/// <param name="list">The collection with which to create this instance of the ReadOnlyObservableAsyncCollection class.
/// The object must also implement IAsyncProgress, INotifyCollectionChanged and INotifyPropertyChanged.</param>
public ReadOnlyObservableAsyncCollection(IList <T> list)
: base(list)
{
this.asyncProgress = list as IAsyncProgress;
((INotifyCollectionChanged)this.Items).CollectionChanged += new NotifyCollectionChangedEventHandler(this.HandleCollectionChanged);
((INotifyPropertyChanged)this.Items).PropertyChanged += new PropertyChangedEventHandler(this.HandlePropertyChanged);
}
public static async Task ReportAsync(this IAsyncProgress <IFormattable> progress, string value)
{
if (progress is null)
{
throw new ArgumentNullException(paramName: nameof(progress));
}
//
await progress.ReportAsync(value : value.ToFormattableString());
}
public object Execute(object arg, IAsyncProgress progress)
{
var p = arg as DataReportPluginArgument;
Initialize(p, progress);
ExportData(p, progress);
ExportFile(p, progress);
return(ExportCompleted(p, progress));
}
/// <summary>
/// 加载插件
/// </summary>
/// <param name="asyn"></param>
/// <returns></returns>
public IEnumerable <IPlugin> Load(IAsyncProgress asyn)
{
if (!IsLoaded)
{
this.LoadPlugin(asyn);
this.IsLoaded = true;
}
return(this.Plugins);
}
/// <summary>
/// 数据提取控制器构造器
/// </summary>
/// <param name="task">当前任务实例</param>
/// <param name="source">提取对象</param>
/// <param name="extractItems">提取项集合</param>
/// <param name="asyn">异步通知</param>
/// <param name="workMode">工作模式</param>
public DataExtractControler(SPFTask task, Pump source, ExtractItemCollection extractItems, IAsyncProgress asyn, EnumDataExtractWorkMode workMode)
{
OwnerTask = task;
SourcePump = source;
Asyn = asyn;
WorkMode = workMode;
ExtractItems = extractItems.GetAllCheckedExtractItem();
MainWorkThread = new SingleThread();
}
/// <summary>
/// 获取文件系统
/// </summary>
/// <param name="device"></param>
/// <param name="iAsync"></param>
/// <returns></returns>
public FNodeX GetFileSystem(IFileSystemDevice device, IAsyncProgress iAsync)
{
IAsync = iAsync;
CreateFileServiceAbstractX(device, iAsync);
if (fileServiceX == null)
{
return(null);
}
_systemTree = fileServiceX.GetFileSystem();
return(_systemTree);
}
/// <summary>
/// 执行插件
/// </summary>
/// <param name="plugin">要执行的插件</param>
/// <param name="asyn">异步通知</param>
/// <param name="callback">插件执行完回调</param>
public void ExecutePlugin(DataParsePluginInfo plugin, IAsyncProgress asyn, Action <IDataSource> callback)
{
var pl = Plugins[plugin] as AbstractDataParsePlugin;
if (null != pl)
{
pl.StartTime = DateTime.Now;
var ds = pl.Execute(null, asyn) as IDataSource;
pl.EndTime = DateTime.Now;
callback?.Invoke(ds);
}
}
public override object Execute(object arg, IAsyncProgress progress)
{
var context = GetContext();
if (context == null)
{
throw new System.Exception("Dont find context!");
}
DataParsePluginInfo p = PluginInfo as DataParsePluginInfo;
return(context?.Execute(p.ScriptObject, progress));
}
/// <summary>
/// 构造函数
/// </summary>
/// <param name="device">当前处理的设备</param>
/// <param name="iAsyn">异步消息通知</param>
protected FileServiceAbstractX(IFileSystemDevice device, IAsyncProgress iAsyn)
{
Asyn = iAsyn;
Device = device;
if (AllFileNodeX == null)
{
AllFileNodeX = new List <FNodeX>();
}
if (GetUserPartitionFiles == null)
{
GetUserPartitionFiles = new NodeCollection();
}
}
protected override void Initialize(DataReportPluginArgument arg, IAsyncProgress progress)
{
DataReportPluginInfo plugin = (DataReportPluginInfo)PluginInfo;
if (plugin.Modules.Count == 0)
{
throw new Exception("Modules is null");
}
_module = plugin.Modules.FirstOrDefault(m => m.Name == arg.ReportModuleName) ?? plugin.Modules[0]; //默认使用第一个模板
if (!Directory.Exists(arg.ReportPath))
{
Directory.CreateDirectory(arg.ReportPath);
}
BaseUtility.Helper.FileHelper.CopyDirectory(_module.ZipTempDirectory, arg.ReportPath); //拷贝模板文件目录
}
public void Extract(SourceFileItem item, IAsyncProgress asyn)
{
if (Context == null)
{
return;
}
Context.UnsafeSource = item;
DataPumpControllableExecutionContext contextEx = Context as DataPumpControllableExecutionContext;
if (contextEx != null)
{
contextEx.Reporter = asyn;
}
DataPump.Execute(Context);
}
public override object Execute(object arg, IAsyncProgress progress)
{
var task = arg as SPFTask;
DataParsePluginInfo p = PluginInfo as DataParsePluginInfo;
var files = task.SourceFiles.Where(s => p.SourcePath.Any(k => k.Config == s.Config));
var str = string.Empty;
if (files.IsValid())
{
str = Serializer.JsonSerilize(files.Select(s => s.Local));
}
var js = p.ScriptObject.Replace("$source", str);
var obj = ExecuteJs(js, progress);
return(null);
}
public static Task DownloadAsync(this ILargeFileDownloadParameters parameters,
CancellationToken?cancellationToken = null,
IAsyncProgress <LargeFileDownloadProgressChangedEventArgs> progress = null,
Action <string> logger = null, BufferManager bufferManager = null)
{
CancellationToken ct = (cancellationToken != null) ? cancellationToken.Value : CancellationToken.None;
FailureToken ft = new FailureToken();
Task task = Task.Factory.StartNew(() =>
{
// we put this logic on a new thread to avoid the risk of
// running into thread starvation on the threadpool / deadlocks
var t = new Thread(() => Downloader.StartDownloading(ct, ft, parameters, progress, logger));
t.Start();
t.Join();
}, ct);
return(task);
}
/// <summary>
/// 初始化
/// </summary>
public void Initialization(IAsyncProgress asyn)
{
Plugins = new Dictionary <AbstractPluginInfo, IPlugin>();
var pluginLoaders = IocManagerSingle.Instance.GetParts <IPluginLoader>(PluginExportKeys.PluginLoaderKey);
foreach (var loader in pluginLoaders)
{
var pls = loader.Value.Load(asyn);
foreach (var pl in pls)
{
if (null != pl.PluginInfo)
{
Plugins.Add(pl.PluginInfo as AbstractPluginInfo, pl);
}
}
}
}
private void CreateFileServiceAbstractX(IFileSystemDevice device, IAsyncProgress iAsync)
{
if (device is MirrorDevice)
{
fileServiceX = new MirrorDeviceService(device, iAsync);
}
else if (device is SDCardDevice)
{
fileServiceX = new SDCardDeviceService(device, iAsync);
}
else if (device is CellbriteDevice)
{
fileServiceX = new CellbriteDeviceService(device, iAsync);
}
else if (device is CottageDevice)
{
fileServiceX = new CottageMirrorDeviceService(device, iAsync);
}
}
/// <summary>
/// 批量执行插件集
/// </summary>
/// <param name="plugins">要执行的插件列表</param>
/// <param name="asyn">异步通知</param>
/// <param name="callback">插件执行完回调</param>
public void ExecutePluginList(List <DataParsePluginInfo> plugins, IAsyncProgress asyn, Action <IDataSource> callback)
{
if (plugins != null)
{
return;
}
var pls = plugins.ToList();
try
{
foreach (var p in pls)
{
ExecutePlugin(p, asyn, callback);
}
}
catch (OperationCanceledException oe)
{
LoggerManagerSingle.Instance.Warn("Exit analysis task:" + oe.Message);
return;
}
}
protected override void LoadPlugin(IAsyncProgress asyn)
{
List <IPlugin> pluginList = new List <IPlugin>();
string dir = SystemContext.Instance.CurLanguage == LanguageType.Cn ? FileHelper.GetPhysicalPath("\\Script\\cn")
: FileHelper.GetPhysicalPath("\\Script\\en");
var res = System.Threading.Tasks.Parallel.ForEach(FileHelper.GetFiles(dir, new[] { JS_EXT, RELEASE_JS_EXT }), (s) =>
{
var plug = this.LoadFile(s.FullName);
lock (pluginList)
{
if (plug != null)
{
pluginList.Add(plug);
}
}
System.Threading.Thread.Sleep(20);
});
Plugins = pluginList;
}
private object ExecuteJs(string jsCode, IAsyncProgress progress)
{
if (!System.IO.File.Exists(_ScriptContextRunName))
{
var context = GetContext();
return(context?.Execute(jsCode, progress));// new ScriptContext().Execute(jsCode);
}
else
{
string res = "";
try
{
string jsFile = System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, String.Format("JS_{0}.jscode", Guid.NewGuid().ToString()));
System.IO.File.WriteAllText(jsFile, jsCode, System.Text.Encoding.UTF8);
Process pro = new Process();
pro.StartInfo.FileName = _ScriptContextRunName;
pro.StartInfo.Arguments = string.Format("\"{0}\"", jsFile);;
pro.StartInfo.UseShellExecute = false;
pro.StartInfo.CreateNoWindow = true;
pro.Start();
pro.WaitForExit();
if (System.IO.File.Exists(jsFile))
{
res = System.IO.File.ReadAllText(jsFile, System.Text.Encoding.UTF8);
System.IO.File.Delete(jsFile);
}
}
catch (Exception ex)
{
LoggerManagerSingle.Instance.Error("执行脚本失败!", ex);
}
return(res);
}
}
/// <summary>
/// 从黑莓系统拷贝一个文件到Windows系统中。
/// </summary>
/// <param name="device">任务设备</param>
/// <param name="source">文件</param>
/// <param name="targetPath">Windows目标路径。</param>
/// <param name="asyn">异步通知</param>
/// <returns>返回Windows路径。</returns>
public string CopyFile(Device device, string source, string targetPath, IAsyncProgress asyn)
{
int blackberryHadnle = 0;
var service = X86DLLClientSingle.Instance.BlackBerryDeviceAPIChannel;
try
{
blackberryHadnle = service.BlackBerry_Mount(device.ID);
if (0 == blackberryHadnle)
{
return(string.Empty);
}
X86DLLClientSingle.Instance.ClientCallback._BlackBerryBackupCallBack += ImageDataCallBack;
int imageResult = service.BlackBerry_ImageAppData(blackberryHadnle, targetPath);
if (imageResult != 0)
{
return(string.Empty);
}
}
catch
{
}
finally
{
X86DLLClientSingle.Instance.ClientCallback._BlackBerryBackupCallBack -= ImageDataCallBack;
if (0 != blackberryHadnle)
{
service.BlackBerry_Close(blackberryHadnle);
}
}
return(targetPath);
}
private static readonly string DefaultUnrarPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); //默认的临时解压缩路径
/* 脚本文件结构:
*
* --Android_QQ_5.1.0.zip
|--plugin.config (主配置文件，名称固定)
|--main.py (主插件，名称固定)
|--icon.png (图标，名称固定)
|--chalib (数据恢复特征库文件夹)
|--其它文件及文件夹
*
*/
protected override void LoadPlugin(IAsyncProgress asyn)
{
List <IPlugin> plugins = new List <IPlugin>();
//string dir = SystemContext.Instance.CurLanguage == LanguageType.Cn ? FileHelper.GetPhysicalPath("\\Script\\cn")
// : FileHelper.GetPhysicalPath("\\Script\\en");
string dir = FileHelper.GetPhysicalPath("\\Script\\cn");
foreach (var file in FileHelper.GetFiles(dir, new[] { DebugScriptExtension, ReleaseScriptExtension }))
{
try
{
bool isPassword = file.Extension.Equals(ReleaseScriptExtension, StringComparison.OrdinalIgnoreCase);
//解压缩文件到临时目录
string tmpDir = UnRarFile(file, isPassword);
//读取配置文件
AbstractZipPluginInfo pluginInfo = ReadPluginInfo(Path.Combine(tmpDir, PluginConfigFileName));
//读取脚本文件内容
pluginInfo.ZipTempDirectory = tmpDir;
ReadScriptContent(pluginInfo, isPassword);
//生成插件实例
IPlugin plugin = GetPlugin(pluginInfo);
plugin.PluginInfo = pluginInfo;
plugins.Add(plugin);
}
catch (Exception ex)
{
LoggerManagerSingle.Instance.Warn(ex, string.Format("解析脚本发生异常！脚本文件：{0}", file.FullName));
}
}
Plugins = plugins;
}
private void InitCopyUserData(Device device, double totalProgress, IAsyncProgress asyn)
{
_CopyUserDataCallback = CopyUserDataCallback;
CurrentDeviceName = device.Model;
IsCopying = true;
IsStop = false;
Asyn = asyn;
// 内置应用包虚拟进度条额外参数(第一步）
_OneAllProgress = 0;
_OneStepLastProgress = 0;
// 内置应用包虚拟进度条额外参数(第二步）
_TwoAllProgress = 0;
_TwoCumulativeProgress = 0;
// 内置应用包虚拟进度条额外参数(第三步）
_ThreeSetpLastProgress = 0;
_ThreeAllProgress = 0;
_ThreeCumulativeProgress = 0;
_OneAllProgress = 0.2 * totalProgress;
_TwoAllProgress = 0.6 * totalProgress;
_ThreeAllProgress = 0.2 * totalProgress;
}
// needed for Unit Testing
internal static void StartDownloading(CancellationToken ct, FailureToken ft,
ILargeFileDownloadParameters parameters,
IAsyncProgress<LargeFileDownloadProgressChangedEventArgs> progress = null,
Action<string> logger = null,
BufferManager bufferManager = null)
{
//create the file
Stream stream = parameters.GetOutputStream();
if (parameters.FileSize == 0) // Terminate Zero size files
{
if (progress != null)
{
progress.Report(new LargeFileDownloadProgressChangedEventArgs(100, 0, 0,
parameters.FileSize,
parameters.FileSize, "",
"",
null));
}
if (parameters.AutoCloseStream)
stream.Close();
return;
}
//figure out number of chunks
int chunkCount = GetChunkCount(parameters.FileSize, parameters.MaxChunkSize);
int numberOfThreads = Math.Min(parameters.MaxThreads, chunkCount);
logger = logger ?? ((s) => { });
var downloadWorkers = new List<Downloader>(numberOfThreads);
var chunksWritten = new Dictionary<int, bool>();
bool isFailed = false;
long totalBytesWritten = 0;
double byteWriteRate = 0.0;
try
{
var readStack = new ConcurrentStack<int>();
//add all of the chunks to the stack
var rangeArray = Enumerable.Range(0, chunkCount).Reverse().ToArray();
readStack.PushRange(rangeArray);
chunksWritten = readStack.ToDictionary(k => k, v => false);
var writeQueue = new ConcurrentQueue<ChunkedFilePart>();
// ReSharper disable AccessToModifiedClosure
Func<int, bool> downloadThrottle = (int c) => writeQueue.Count > 30;
// ReSharper restore AccessToModifiedClosure
if (bufferManager == null)
{
bufferManager = new BufferManager(new[]
{
new BufferQueueSetting(SimpleHttpGetByRangeClient.BUFFER_SIZE, (uint) numberOfThreads),
new BufferQueueSetting((uint) parameters.MaxChunkSize, (uint) numberOfThreads)
});
}
int expectedChunkDownloadTime = ExpectedDownloadTimeInSeconds(parameters.MaxChunkSize);
for (int i = 0; i < numberOfThreads; i++)
{
downloadWorkers.Add(new Downloader(bufferManager, parameters, writeQueue, readStack,
downloadThrottle, expectedChunkDownloadTime, ft, logger, ct));
}
//start all the download threads
downloadWorkers.ForEach(x => x.Start());
var watch = new System.Diagnostics.Stopwatch();
watch.Start();
long oldElapsedMilliSeconds = watch.ElapsedMilliseconds;
DateTime lastWriteTime = DateTime.MaxValue;
long lastPointInFile = 0;
int kc = 0;
//start the write loop
while (chunksWritten.Any(kvp => !kvp.Value) && !ct.IsCancellationRequested && !ft.FailureDetected)
{
ChunkedFilePart part;
while (writeQueue.TryDequeue(out part) && !ft.FailureDetected)
{
//retry?
logger(string.Format("[{1}] writing chunk: {0}", part.Chunk, parameters.Id));
stream.Position = part.FileOffset;
stream.Write(part.Content, 0, part.Length);
totalBytesWritten += part.Length;
bufferManager.FreeBuffer(part.Content);
chunksWritten[part.Chunk] = true;
lastWriteTime = DateTime.Now;
if (progress != null)
{
var elapsed = watch.ElapsedMilliseconds;
var diff = elapsed - oldElapsedMilliSeconds;
if (diff > 2000)
{
long bytesDownloaded = (long)chunksWritten.Count(kvp => kvp.Value) * parameters.MaxChunkSize;
long interimReads = bytesDownloaded + part.Length - lastPointInFile;
byteWriteRate = (interimReads / (diff / (double)1000));
lastPointInFile += interimReads;
oldElapsedMilliSeconds = elapsed;
progress.Report(new LargeFileDownloadProgressChangedEventArgs(ComputeProgressIndicator(totalBytesWritten, parameters.FileSize),
byteWriteRate, byteWriteRate, totalBytesWritten, totalBytesWritten, "", "", null));
}
}
}
// kill hanged workers
var timedOutWorkers = downloadWorkers
.Where(w => w.Status == ThreadState.Running || w.Status == ThreadState.WaitSleepJoin)
.Where((w) =>
{
if (w.SimulateTimedOut)
return true;
return w.HeartBeat.AddSeconds(expectedChunkDownloadTime) < DateTime.Now;
})
.ToList();
if (timedOutWorkers.Any())
{
foreach (var worker in timedOutWorkers)
{
try
{
worker.DownloadWorkerThread.Abort(); // this has a minute chance of throwing
logger(string.Format("[{1}] killing thread as it timed out {0}", kc++, parameters.Id));
if (worker.SimulateTimedOut)
Thread.Sleep(3000); // introduce delay for unit test to pick-up the condition
}
catch (Exception)
{ }
}
}
var activeWorkers = downloadWorkers.Where(x => x != null &&
(x.Status == ThreadState.Running
|| x.Status == ThreadState.WaitSleepJoin)).ToList();
// re-spawn the missing workers if some had too many retries or were killed
if (NeedToCheckForUnwrittenChunks(readStack, lastWriteTime, STALE_WRITE_CHECK_MINUTES))
{
// if there are any parts remaining to be written, AND the read stack is empty
var unreadParts = chunksWritten.Where(kvp => !kvp.Value);
if (readStack.IsEmpty && unreadParts.Any() && !ft.FailureDetected)
{
logger(string.Format("read stack is empty, but there remains unwritten parts! Adding {0} parts back to read stack.", unreadParts.Count()));
readStack.Push(unreadParts.Select(kvp => kvp.Key).First());
}
lastWriteTime = DateTime.Now; // don't check again for a while
}
//wait for something that was added
Thread.Sleep(100);
if (activeWorkers.Count() < numberOfThreads)
{
for (int i = 0; i < numberOfThreads; i++)
{
if (downloadWorkers[i] == null)
{
logger(string.Format("[{0}] reviving killed thread", parameters.Id));
downloadWorkers[i] = new Downloader(bufferManager, parameters, writeQueue, readStack,
downloadThrottle, expectedChunkDownloadTime, ft, logger, ct);
downloadWorkers[i].Start();
continue;
}
if (downloadWorkers[i].Status == ThreadState.Running
|| downloadWorkers[i].Status == ThreadState.WaitSleepJoin
|| downloadWorkers[i].Status == ThreadState.Background
|| downloadWorkers[i].Status == ThreadState.Stopped) continue;
logger(string.Format("[{0}] reviving killed thread", parameters.Id));
downloadWorkers[i] = new Downloader(bufferManager, parameters, writeQueue, readStack,
downloadThrottle, expectedChunkDownloadTime, ft, logger, ct);
downloadWorkers[i].Start();
}
}
}
if (ft.FailureDetected)
{
throw new Exception(String.Format("[{0}]A Non Retry-able Failure was reported by one or more of the download workers.", parameters.Id));
}
}
catch (Exception e)
{
// Report Failure
isFailed = true;
logger(string.Format("[{0}] Exception: TerminalVelocity Downloading failed.", parameters.Id));
logger(string.Format("[{0}] Message: {1} ", parameters.Id, e.Message));
logger(string.Format("[{0}] StackTrace: {1}", parameters.Id, e.StackTrace));
if (progress != null)
{
progress.Report(new LargeFileDownloadProgressChangedEventArgs(ComputeProgressIndicator(totalBytesWritten, parameters.FileSize), 0, 0, totalBytesWritten, totalBytesWritten, "", "", null, isFailed, e.Message));
}
}
finally
{
//kill all the tasks if exist
if (downloadWorkers != null)
{
downloadWorkers.ForEach(x =>
{
if (x == null) return;
ExecuteAndSquash(x.Dispose);
});
}
if (parameters.AutoCloseStream)
{
if (progress != null)
{
progress.Report(new LargeFileDownloadProgressChangedEventArgs(ComputeProgressIndicator(totalBytesWritten, parameters.FileSize), byteWriteRate, byteWriteRate, totalBytesWritten, totalBytesWritten, "", "", null, isFailed));
}
logger(string.Format("[{0}] AutoClosing stream", parameters.Id));
stream.Close();
}
}
}
public static Task DownloadAsync(this ILargeFileDownloadParameters parameters,
CancellationToken? cancellationToken = null,
IAsyncProgress<LargeFileDownloadProgressChangedEventArgs> progress = null,
Action<string> logger = null, BufferManager bufferManager = null)
{
CancellationToken ct = (cancellationToken != null) ? cancellationToken.Value : CancellationToken.None;
FailureToken ft = new FailureToken();
Task task = Task.Factory.StartNew(() =>
{
// we put this logic on a new thread to avoid the risk of
// running into thread starvation on the threadpool / deadlocks
var t = new Thread(() => Downloader.StartDownloading(ct, ft, parameters, progress, logger));
t.Start();
t.Join();
} , ct);
return task;
}
public void ProgressObj (IAsyncProgress _p)
{
_progress = _p;
}
