protected override WebResponse GetWebResponse(WebRequest request)
{
WebResponse response = base.GetWebResponse(request);
String setCookieHeader = response.Headers[HttpResponseHeader.SetCookie];
CookieCollection allCookiesFromHeader = WebConnectionStuff.GetAllCookiesFromHeader(setCookieHeader, null);
CookieContainer.Add(allCookiesFromHeader);
//if (setCookieHeader != null)
//{
// //do something if needed to parse out the cookie.
// try
// {
// Cookie cookie = new Cookie();
// //create cookie
// this.CookieContainer.Add(cookie);
// }
// catch (Exception)
// {
// }
//}
return(response);
}
/// <summary>
///
/// </summary>
/// <param name="courseUrl"></param>
/// <param name="headers"></param>
/// <returns></returns>
public virtual string get_page(string courseUrl, Dictionary <string, string> headers = null)
{
HttpWebResponse r = WebConnectionStuff.GetResponse(url: courseUrl, headers: headers);
Stream responseStream = r.GetResponseStream();
//Encoding encoding = System.Text.Encoding.GetEncoding(r.ContentEncoding);
StreamReader reader = new StreamReader(responseStream);
string page = reader.ReadToEnd();
reader.Close();
responseStream.Close();
r.Close();
return(page);
}
/// <summary>
/// Get the json data
/// </summary>
private static JObject GetJson(string url)
{
JObject jObject;
using (HttpWebResponse response = WebConnectionStuff.GetResponse(url))
{
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
string readToEnd = reader.ReadToEnd();
jObject = JObject.Parse(readToEnd);
reader.Close();
}
response.Close();
}
return(jObject);
}
/// <summary>
/// Download the url to the given filename
/// </summary>
public void Download(string url, string targetDir = ".", string targetFname = null)
{
using (HttpWebResponse response = WebConnectionStuff.GetResponse(url, stream: true))
{
WebHeaderCollection responseHeaders = response.Headers;
int contentLength = GetContentLength(responseHeaders);
string filepath = GetFilePath(url, targetDir, responseHeaders);
string fname = Path.GetFileName(filepath);
bool dl = IsFileNeeded(filepath, contentLength, fname);
filepath = Path.Combine(targetDir, fname);
//ensure it respects mppl
filepath = Utilities.TrimPathPart(filepath, _courseraCourse.Max_path_part_len);
if (dl)
{
try
{
Console.WriteLine(" - Downloading {0}", fname);
int full_size = contentLength;
int done_size = 0;
int slice_size = 524288; //512 kB buffer
DateTime last_time = DateTime.Now;
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
using (Stream s = File.Create(filepath))
{
reader.BaseStream.CopyTo(s);
}
reader.Close();
}
}
catch (Exception e)
{
Console.WriteLine("Failed to download url {0} to {1}: {2}", url, filepath, e.Message);
}
}
response.Close();
}
}
public CourseraBkup(string username, string password, string proxy, string parser, string ignorefiles, int mppl, bool gzipCourses, string wkfilter)
{
AUTH_URL = BASE_URL + "/auth/auth_redirector?type=login&subtype=normal";
QUIZ_URL = BASE_URL + "/quiz/index";
Username = username;
Password = password;
Parser = parser;
// Split "ignorefiles" argument on commas, strip, remove prefixing dot
// if there is one, and filter out empty tokens.
char[] charsToTrim = { '.', ' ' };
if (ignorefiles != null)
{
Ignorefiles = ignorefiles.Split(',').Select(s => s.Trim(charsToTrim));
}
Session = null;
Proxy = proxy;
Max_path_part_len = mppl;
Gzip_courses = gzipCourses;
try
{
if (string.IsNullOrEmpty(wkfilter))
{
Wk_filter = null;
}
else
{
Wk_filter = wkfilter.Split(',');
}
_webConnectionStuff = new WebConnectionStuff();
}
catch (Exception e)
{
Console.WriteLine("Invalid week filter, should be a comma separated list of integers.");
Console.WriteLine(e.Message);
}
Courses = new List <Course>();
}
public FutureLearn(string username, string password, string proxy, string parser, string ignorefiles, int mppl, bool gzipCourses, string wkfilter)
{
AUTH_URL = BASE_URL + "/auth/auth_redirector?type=login&subtype=normal";
QUIZ_URL = BASE_URL + "/quiz/index";
Username = username;
Password = password;
Parser = parser;
// Split "ignorefiles" argument on commas, strip, remove prefixing dot
// if there is one, and filter out empty tokens.
char[] charsToTrim = { '.', ' ' };
if (ignorefiles != null) Ignorefiles = ignorefiles.Split(',').Select(s => s.Trim(charsToTrim));
Session = null;
Proxy = proxy;
Max_path_part_len = mppl;
Gzip_courses = gzipCourses;
try
{
if (string.IsNullOrEmpty(wkfilter))
{
Wk_filter = null;
}
else
{
Wk_filter = wkfilter.Split(',');
}
_webConnectionStuff = new WebConnectionStuff();
}
catch (Exception e)
{
Console.WriteLine("Invalid week filter, should be a comma separated list of integers.");
Console.WriteLine(e.Message);
}
Courses = new List<Course>();
}