private static void Login(WebClientEx client) { var loginValues = new NameValueCollection { {"EmailAddress", "*****@*****.**"}, {"Password", "wishing"}, }; Console.WriteLine("Logging in..."); client.UploadValues(LOGIN_URL, loginValues); }
public List<ContentList> Run(string contentDirectory, bool fastMode) { this.contentDir = contentDirectory; this.inFastMode = fastMode; List<ContentList> content; using (var client = new WebClientEx()) { Login(client); content = RetrieveContent(client); } return content; }
private List<ContentList> RetrieveContent(WebClientEx client) { var listOfLists = new List<ContentList>(); string listOfListsSource = GetListOfLists(client); int startIndex = 0; int totalItems = 0; while (true) { startIndex = listOfListsSource.IndexOf(EACH_LIST_BEGINS, startIndex); if (startIndex < 0) break; startIndex += EACH_LIST_BEGINS.Length; var contentList = GetListInfo(listOfListsSource, startIndex); Console.WriteLine("Retrieving content for " + contentList.Name + "..."); List<ContentItem> pageContent; int pageNumber = 1; do { string pageURL = contentList.URL + "&pg=" + pageNumber++; string pageSource = client.DownloadString(pageURL); pageContent = GetContentItemsOnPage(pageURL, pageSource, contentList); contentList.ContentItems.AddRange(pageContent); foreach (var ci in pageContent) { foreach (var tag in ci.Tags) { contentList.Tags.Add(tag); } } } while (pageContent.Count > 0 && !inFastMode); // just grab the 1st page if in fast mode totalItems += contentList.ContentItems.Count; Console.WriteLine("Found " + contentList.ContentItems.Count + " items for " + contentList.Name); listOfLists.Add(contentList); } Console.WriteLine("Read " + totalItems + " items (total)."); return listOfLists; }
private string GetListOfLists(WebClientEx client) { var listOfListsSource = client.DownloadString(CONTENT_URL); int listStartIdx = listOfListsSource.GetIndexAfter(LISTS_START_WITH); if (listStartIdx < 0) throw new Exception("Unable to find lists"); return listOfListsSource.Substring(listStartIdx); }