/// <summary> /// /// </summary> /// <param name="courseUrl"></param> /// <param name="headers"></param> /// <returns></returns> public virtual string get_page(string courseUrl, Dictionary <string, string> headers = null) { HttpWebResponse r = WebConnectionStuff.GetResponse(url: courseUrl, headers: headers); Stream responseStream = r.GetResponseStream(); //Encoding encoding = System.Text.Encoding.GetEncoding(r.ContentEncoding); StreamReader reader = new StreamReader(responseStream); string page = reader.ReadToEnd(); reader.Close(); responseStream.Close(); r.Close(); return(page); }
/// <summary> /// Get the json data /// </summary> private static JObject GetJson(string url) { JObject jObject; using (HttpWebResponse response = WebConnectionStuff.GetResponse(url)) { using (StreamReader reader = new StreamReader(response.GetResponseStream())) { string readToEnd = reader.ReadToEnd(); jObject = JObject.Parse(readToEnd); reader.Close(); } response.Close(); } return(jObject); }
/// <summary> /// Download the url to the given filename /// </summary> public void Download(string url, string targetDir = ".", string targetFname = null) { using (HttpWebResponse response = WebConnectionStuff.GetResponse(url, stream: true)) { WebHeaderCollection responseHeaders = response.Headers; int contentLength = GetContentLength(responseHeaders); string filepath = GetFilePath(url, targetDir, responseHeaders); string fname = Path.GetFileName(filepath); bool dl = IsFileNeeded(filepath, contentLength, fname); filepath = Path.Combine(targetDir, fname); //ensure it respects mppl filepath = Utilities.TrimPathPart(filepath, _courseraCourse.Max_path_part_len); if (dl) { try { Console.WriteLine(" - Downloading {0}", fname); int full_size = contentLength; int done_size = 0; int slice_size = 524288; //512 kB buffer DateTime last_time = DateTime.Now; using (StreamReader reader = new StreamReader(response.GetResponseStream())) { using (Stream s = File.Create(filepath)) { reader.BaseStream.CopyTo(s); } reader.Close(); } } catch (Exception e) { Console.WriteLine("Failed to download url {0} to {1}: {2}", url, filepath, e.Message); } } response.Close(); } }