public void Download(string url, string targetDir, string targetFname)
{
string fname = targetFname.RemoveColon();
string filepath = Path.Combine(targetDir, fname);
//ensure it respects mppl
filepath = Utilities.TrimPathPart(filepath, _futureleanCourse.Max_path_part_len);
WebHeaderCollection responseHeaders = _futureleanCourse._client.ResponseHeaders;
int contentLength = GetContentLength(responseHeaders);
bool isFileNeeded = IsFileNeeded(filepath, contentLength, fname);
if (isFileNeeded)
{
if (Path.GetExtension(filepath) == ".html")
{
string content = _futureleanCourse._client.DownloadString(url);
NReadabilityTranscoder transcoder = new NReadabilityTranscoder();
TranscodingInput tiInput = new TranscodingInput(content);
TranscodingResult transcodedContent = transcoder.Transcode(tiInput);
//.Transcode(content, out success);
File.WriteAllText(filepath, transcodedContent.ExtractedContent);
}
else
{
_futureleanCourse._client.DownloadFile(url, filepath);
}
}
}
private Response MergeResponses(Response original, Response append)
{
if (original == null)
{
return(append);
}
if (append == null)
{
return(original);
}
TranscodingResult mergedResult = original.TranscodingResult;
mergedResult.ExtractedContent += String.Format("<div class=\"readability-nextpage\" data-page=\"{0}\"></div>{1}", (original.PageCount + 1).ToString(), append.TranscodingResult.ExtractedContent);
if (mergedResult.Images == null || mergedResult.Images.Count() == 0)
{
mergedResult.Images = append.TranscodingResult.Images;
}
else if (append.TranscodingResult.Images != null && append.TranscodingResult.Images.Count() > 0)
{
List <XElement> images = mergedResult.Images.ToList();
images.AddRange(append.TranscodingResult.Images);
mergedResult.Images = images;
}
mergedResult.NextPageUrl = append.TranscodingResult.NextPageUrl;
return(new Response()
{
PageCount = original.PageCount + 1,
Encoding = original.Encoding,
TranscodingResult = mergedResult
});
}
