private async Task DownloadFile(ResourceTask task, string url)
{
DateTime started = DateTime.Now;
List<Tuple<DateTime, long>> points = new List<Tuple<DateTime, long>>();
using (WebClient client = new WebClient())
{
client.DownloadProgressChanged += (sender, args) =>
{
task.OnProgress(args.BytesReceived, args.TotalBytesToReceive);
double elapsed = (DateTime.Now - started).TotalSeconds;
long downloaded = args.BytesReceived;
long left = (args.TotalBytesToReceive - args.BytesReceived);
task.OnEstimation(TimeSpan.FromSeconds(elapsed * left / downloaded));
lock (points)
{
points.Add(Tuple.Create(DateTime.Now, args.BytesReceived));
if (points.Count == 256)
{
points.RemoveAt(0);
}
if (points.Count >= 2)
{
var lowest = points[0];
var highest = points[points.Count - 1];
if ((highest.Item1 - lowest.Item1).TotalSeconds > 0)
{
task.OnSpeed(Convert.ToInt64(Math.Round((highest.Item2 - lowest.Item2) / (highest.Item1 - lowest.Item1).TotalSeconds)));
}
}
}
};
task.OnStatus.Invoke("downloading");
task.OnLog.Information("Downloading file.");
await client.DownloadFileTaskAsync(url, task.Destination);
task.Cancellation.Register(client.CancelAsync);
task.OnStatus("completed");
}
}
private async Task<PhantomResponse> CallPhantom(ResourceTask task)
{
Regex regex = new Regex(@"to wait (?<minutes>[0-9]+) minutes");
PhantomResponse response = new PhantomResponse
{
Lines = new List<string>()
};
task.OnStatus.Invoke("starting");
task.OnLog.Information("Starting PhantomJS.");
ProcessStartInfo info = new ProcessStartInfo
{
FileName = GetPhantomPath(),
Arguments = GetScriptPath(task.Hosting) + " download " + task.Url.ToString(),
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardInput = true,
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden,
WorkingDirectory = GetDataPath()
};
using (Process process = Process.Start(info))
{
PhantomCallback callback = new PhantomCallback
{
OnDownload = url =>
{
response.DownloadUrl = url;
return false;
},
OnMessage = message =>
{
Match match = regex.Match(message);
if (match.Success == true)
{
if (match.Groups["minutes"].Success == true)
{
response.Waiting = TimeSpan.FromMinutes(Int32.Parse(match.Groups["minutes"].Value));
}
}
return true;
},
OnDebug = text =>
{
task.OnLog.Debug(text);
return true;
},
OnFatal = text =>
{
task.OnLog.Debug(text);
return true;
},
OnDumpImage = base64 =>
{
task.OnLog.Debug("PhantomJS dumped an image.", Convert.FromBase64String(base64), "image");
return true;
},
OnDumpHtml = base64 =>
{
task.OnLog.Debug("PhantomJS dumped an html content.", Convert.FromBase64String(base64), "text");
return true;
},
OnFileName = text => true,
OnFileSize = text => true,
OnFileStatus = text => true,
OnFallback = text => true,
OnRaw = line => { },
};
callback.OnCaptcha = async url =>
{
string solution;
task.Cancellation.ThrowIfCancellationRequested();
task.OnLog.Information("Handling captcha.");
using (WebClient client = new WebClient())
{
task.OnStatus("decaptching");
TimeSpan timeout = TimeSpan.FromMinutes(3);
CancellationTokenSource source = CancellationTokenSource.CreateLinkedTokenSource(new CancellationTokenSource(timeout).Token, task.Cancellation);
Captcha captcha = new Captcha
{
Type = "image",
Data = client.DownloadData(url),
Cancellation = source.Token
};
Action debug = () =>
{
switch (captcha.Type)
{
case "image":
task.OnLog.Debug("Got captcha image data.", captcha.Data, "image");
break;
case "audio":
task.OnLog.Debug("Got captcha audio data.", captcha.Data, "audio");
break;
}
};
PhantomCallback local = callback.Override(new PhantomCallback
{
OnCaptcha = async reloadUrl =>
{
source = CancellationTokenSource.CreateLinkedTokenSource(new CancellationTokenSource(timeout).Token, task.Cancellation);
captcha.Cancellation = source.Token;
captcha.Data = await client.DownloadDataTaskAsync(reloadUrl);
debug.Invoke();
return false;
}
});
debug.Invoke();
captcha.Reload = async () =>
{
await process.StandardInput.WriteLineAsync("::reload::");
task.OnLog.Information("Reloading captcha.");
await this.HandleInThread(local, task.Cancellation, process);
};
captcha.ToAudio = async () =>
{
await process.StandardInput.WriteLineAsync("::audio::");
task.OnLog.Information("Switching to audio.");
captcha.Type = "audio";
await this.HandleInThread(local, task.Cancellation, process);
};
captcha.ToImage = async () =>
{
await process.StandardInput.WriteLineAsync("::image::");
task.OnLog.Information("Switching to image.");
captcha.Type = "image";
await this.HandleInThread(local, task.Cancellation, process);
};
solution = await task.OnCaptcha.Invoke(captcha);
task.OnStatus("working");
}
task.Cancellation.ThrowIfCancellationRequested();
task.OnLog.Information("Sending captcha.");
await process.StandardInput.WriteLineAsync(solution);
return true;
};
try
{
task.OnStatus("working");
await this.Handle(callback, task.Cancellation, process);
process.WaitForExit();
return response;
}
finally
{
if (process.HasExited == false)
{
process.Kill();
}
}
}
}
private async Task Wait(ResourceTask task, TimeSpan waiting)
{
TimeSpan counter = waiting;
if (counter.TotalMinutes > 20)
{
counter = TimeSpan.FromMinutes(20);
}
task.OnLog.Information("Waiting.");
while (counter.TotalMinutes > 0)
{
task.OnStatus(String.Format("{0} / {1}", Math.Round(waiting.TotalMinutes), Math.Round(counter.TotalMinutes)));
await Task.Delay(TimeSpan.FromMinutes(1), task.Cancellation);
waiting = waiting - TimeSpan.FromMinutes(1);
counter = counter - TimeSpan.FromMinutes(1);
}
}
public void Download(ResourceTask task)
{
Task.Run(async () =>
{
try
{
await this.AcquireSlot(task);
while (true)
{
PhantomResponse response = await CallPhantom(task);
if (response.Waiting != null)
{
await this.Wait(task, response.Waiting.Value + TimeSpan.FromMinutes(3));
continue;
}
if (response.DownloadUrl != null)
{
await this.DownloadFile(task, response.DownloadUrl);
task.OnCompleted.Invoke(true);
task.OnLog.Information("Completed.");
break;
}
task.OnStatus("terminated");
task.OnCompleted.Invoke(false);
task.OnLog.Warning("Terminated without downloading.");
break;
}
}
catch (TaskCanceledException)
{
task.OnStatus("timeout");
task.OnCompleted.Invoke(false);
task.OnLog.Warning("Solving captcha timed out.");
}
catch (OperationCanceledException)
{
task.OnStatus("cancelled");
task.OnCompleted.Invoke(false);
task.OnLog.Warning("Downloading was cancelled.");
}
catch (Exception ex)
{
task.OnStatus("failed");
task.OnCompleted.Invoke(false);
task.OnLog.Warning("Downloading failed. " + ex.Message);
}
finally
{
this.ReleaseSlot(task);
}
});
}