public void Download(string url, string targetDir, string targetFname) { string fname = targetFname.RemoveColon(); string filepath = Path.Combine(targetDir, fname); //ensure it respects mppl filepath = Utilities.TrimPathPart(filepath, _futureleanCourse.Max_path_part_len); WebHeaderCollection responseHeaders = _futureleanCourse._client.ResponseHeaders; int contentLength = GetContentLength(responseHeaders); bool isFileNeeded = IsFileNeeded(filepath, contentLength, fname); if (isFileNeeded) { if (Path.GetExtension(filepath) == ".html") { string content = _futureleanCourse._client.DownloadString(url); NReadabilityTranscoder transcoder = new NReadabilityTranscoder(); TranscodingInput tiInput = new TranscodingInput(content); TranscodingResult transcodedContent = transcoder.Transcode(tiInput); //.Transcode(content, out success); File.WriteAllText(filepath, transcodedContent.ExtractedContent); } else { _futureleanCourse._client.DownloadFile(url, filepath); } } }
private Response MergeResponses(Response original, Response append) { if (original == null) { return(append); } if (append == null) { return(original); } TranscodingResult mergedResult = original.TranscodingResult; mergedResult.ExtractedContent += String.Format("<div class=\"readability-nextpage\" data-page=\"{0}\"></div>{1}", (original.PageCount + 1).ToString(), append.TranscodingResult.ExtractedContent); if (mergedResult.Images == null || mergedResult.Images.Count() == 0) { mergedResult.Images = append.TranscodingResult.Images; } else if (append.TranscodingResult.Images != null && append.TranscodingResult.Images.Count() > 0) { List <XElement> images = mergedResult.Images.ToList(); images.AddRange(append.TranscodingResult.Images); mergedResult.Images = images; } mergedResult.NextPageUrl = append.TranscodingResult.NextPageUrl; return(new Response() { PageCount = original.PageCount + 1, Encoding = original.Encoding, TranscodingResult = mergedResult }); }