예제 #1
0
        protected override WebResponse GetWebResponse(WebRequest request)
        {
            WebResponse response = base.GetWebResponse(request);

            String setCookieHeader = response.Headers[HttpResponseHeader.SetCookie];

            CookieCollection allCookiesFromHeader = WebConnectionStuff.GetAllCookiesFromHeader(setCookieHeader, null);

            CookieContainer.Add(allCookiesFromHeader);
            //if (setCookieHeader != null)
            //{
            //    //do something if needed to parse out the cookie.
            //    try
            //    {

            //        Cookie cookie = new Cookie();
            //        //create cookie
            //        this.CookieContainer.Add(cookie);

            //    }
            //    catch (Exception)
            //    {

            //    }
            //}
            return(response);
        }
예제 #2
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="courseUrl"></param>
        /// <param name="headers"></param>
        /// <returns></returns>
        public virtual string get_page(string courseUrl, Dictionary <string, string> headers = null)
        {
            HttpWebResponse r = WebConnectionStuff.GetResponse(url: courseUrl, headers: headers);
            Stream          responseStream = r.GetResponseStream();
            //Encoding encoding = System.Text.Encoding.GetEncoding(r.ContentEncoding);
            StreamReader reader = new StreamReader(responseStream);
            string       page   = reader.ReadToEnd();

            reader.Close();
            responseStream.Close();
            r.Close();
            return(page);
        }
        /// <summary>
        /// Get the json data
        /// </summary>
        private static JObject GetJson(string url)
        {
            JObject jObject;

            using (HttpWebResponse response = WebConnectionStuff.GetResponse(url))
            {
                using (StreamReader reader = new StreamReader(response.GetResponseStream()))
                {
                    string readToEnd = reader.ReadToEnd();
                    jObject = JObject.Parse(readToEnd);
                    reader.Close();
                }
                response.Close();
            }

            return(jObject);
        }
        /// <summary>
        /// Download the url to the given filename
        /// </summary>
        public void Download(string url, string targetDir = ".", string targetFname = null)
        {
            using (HttpWebResponse response = WebConnectionStuff.GetResponse(url, stream: true))
            {
                WebHeaderCollection responseHeaders = response.Headers;

                int    contentLength = GetContentLength(responseHeaders);
                string filepath      = GetFilePath(url, targetDir, responseHeaders);

                string fname = Path.GetFileName(filepath);

                bool dl = IsFileNeeded(filepath, contentLength, fname);

                filepath = Path.Combine(targetDir, fname);
                //ensure it respects mppl
                filepath = Utilities.TrimPathPart(filepath, _courseraCourse.Max_path_part_len);

                if (dl)
                {
                    try
                    {
                        Console.WriteLine("     - Downloading {0}", fname);
                        int      full_size  = contentLength;
                        int      done_size  = 0;
                        int      slice_size = 524288; //512 kB buffer
                        DateTime last_time  = DateTime.Now;

                        using (StreamReader reader = new StreamReader(response.GetResponseStream()))
                        {
                            using (Stream s = File.Create(filepath))
                            {
                                reader.BaseStream.CopyTo(s);
                            }
                            reader.Close();
                        }
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine("Failed to download url {0} to {1}: {2}", url, filepath, e.Message);
                    }
                }
                response.Close();
            }
        }
        public CourseraBkup(string username, string password, string proxy, string parser, string ignorefiles, int mppl, bool gzipCourses, string wkfilter)
        {
            AUTH_URL = BASE_URL + "/auth/auth_redirector?type=login&subtype=normal";
            QUIZ_URL = BASE_URL + "/quiz/index";

            Username = username;
            Password = password;
            Parser   = parser;

            // Split "ignorefiles" argument on commas, strip, remove prefixing dot
            // if there is one, and filter out empty tokens.
            char[] charsToTrim = { '.', ' ' };
            if (ignorefiles != null)
            {
                Ignorefiles = ignorefiles.Split(',').Select(s => s.Trim(charsToTrim));
            }

            Session           = null;
            Proxy             = proxy;
            Max_path_part_len = mppl;
            Gzip_courses      = gzipCourses;
            try
            {
                if (string.IsNullOrEmpty(wkfilter))
                {
                    Wk_filter = null;
                }
                else
                {
                    Wk_filter = wkfilter.Split(',');
                }
                _webConnectionStuff = new WebConnectionStuff();
            }
            catch (Exception e)
            {
                Console.WriteLine("Invalid week filter, should be a comma separated list of integers.");
                Console.WriteLine(e.Message);
            }

            Courses = new List <Course>();
        }
예제 #6
0
        public FutureLearn(string username, string password, string proxy, string parser, string ignorefiles, int mppl, bool gzipCourses, string wkfilter)
        {
            AUTH_URL = BASE_URL + "/auth/auth_redirector?type=login&subtype=normal";
            QUIZ_URL = BASE_URL + "/quiz/index";

            Username = username;
            Password = password;
            Parser = parser;

            // Split "ignorefiles" argument on commas, strip, remove prefixing dot
            // if there is one, and filter out empty tokens.
            char[] charsToTrim = { '.', ' ' };
            if (ignorefiles != null) Ignorefiles = ignorefiles.Split(',').Select(s => s.Trim(charsToTrim));

            Session = null;
            Proxy = proxy;
            Max_path_part_len = mppl;
            Gzip_courses = gzipCourses;
            try
            {
                if (string.IsNullOrEmpty(wkfilter))
                {
                    Wk_filter = null;
                }
                else
                {
                    Wk_filter = wkfilter.Split(',');
                }
                _webConnectionStuff = new WebConnectionStuff();
            }
            catch (Exception e)
            {
                Console.WriteLine("Invalid week filter, should be a comma separated list of integers.");
                Console.WriteLine(e.Message);
            }

            Courses = new List<Course>();
        }