// DoWork event handler for the file download BackgroundWorker. // The selected TreeNode (either a module, folder, or file) is passed in the DoWorkEventArgs e private void DownloadBW_DoWork(object sender, DoWorkEventArgs e) { BackgroundWorker worker = sender as BackgroundWorker; TreeNode selectedNode = e.Argument as TreeNode; scraper.DownloadProgress.BeginJob(worker); // -- DETERMINE TYPE OF NODE SELECTED -- // Module selected if (selectedNode.Tag.GetType() == typeof(BbModule)) { BbModule module = selectedNode.Tag as BbModule; scraper.DownloadProgress.TotalWork = module.Content.CountAllFiles(); scraper.DownloadModuleFiles(module); } // Single folder selected else if (selectedNode.Tag.GetType() == typeof(BbContentDirectory)) { BbContentDirectory folder = selectedNode.Tag as BbContentDirectory; scraper.DownloadProgress.TotalWork = folder.CountAllFiles(); scraper.DownloadFolder(folder, scraper.OutputDirectory + BbUtils.CleanDirectory(folder.Name) + @"\"); } // Single file selected else if (selectedNode.Tag.GetType() == typeof(BbContentItem)) { BbContentItem file = selectedNode.Tag as BbContentItem; scraper.DownloadProgress.TotalWork = 1; scraper.DownloadFile(file); } }
// Returns true if other BbContentItem has the same url public bool Equals(BbContentItem other) { if (other == null) { return(false); } return(this.url.Equals(other.Url)); }
// Adds given BbContentItem file to the content treeview. private void AddTreeFile(TreeNode parent, BbContentItem file) { TreeNode fileNode = new TreeNode(file.Name); fileNode.Tag = file; fileNode.ImageIndex = 4; fileNode.SelectedImageIndex = 4; parent.Nodes.Add(fileNode); }
public void CreateUrlShortcut(BbContentItem file, string directory) { Directory.CreateDirectory(directory); using (StreamWriter w = new StreamWriter(directory + "\\" + file.Filename + ".url")) { w.WriteLine("[InternetShortcut]"); w.WriteLine("URL=" + file.Url.AbsoluteUri); } }
// Event Handler for the content treeview. Changes information displayed private void contentTree_AfterSelect(object sender, TreeViewEventArgs e) { ClearInfoBox(); // Change what's displayed in the Information Box depending on type of node selected // File selected if (contentTree.SelectedNode.Tag.GetType() == typeof(BbContentItem)) { BbContentItem file = contentTree.SelectedNode.Tag as BbContentItem; infoLabel1.Text = "Name"; infoText1.Text = file.Name; infoLabel2.Text = "Filename"; infoText2.Text = file.Filename; infoLabel3.Text = "Link Type"; infoText3.Text = file.LinkType; infoLabel4.Text = "URL"; infoTextLink.Text = file.Url.AbsoluteUri; } // Folder selected else if (contentTree.SelectedNode.Tag.GetType() == typeof(BbContentDirectory)) { BbContentDirectory folder = contentTree.SelectedNode.Tag as BbContentDirectory; infoLabel1.Text = "Name"; infoText1.Text = folder.Name; infoLabel2.Text = "Files"; infoText2.Text = folder.CountAllFiles().ToString(); infoLabel3.Text = "Subfolders"; infoText3.Text = folder.SubFolders.Count.ToString(); infoLabel4.Text = "URL"; infoTextLink.Text = folder.Url.AbsoluteUri; } // Module selected else if (contentTree.SelectedNode.Tag.GetType() == typeof(BbModule)) { BbModule module = contentTree.SelectedNode.Tag as BbModule; infoLabel1.Text = "Name"; infoText1.Text = module.Name; infoLabel2.Text = "Files"; infoText2.Text = module.Content.CountAllFiles().ToString(); infoLabel3.Text = "Subfolders"; infoText3.Text = module.Content.SubFolders.Count.ToString(); infoLabel4.Text = "URL"; infoTextLink.Text = module.Url.AbsoluteUri; } }
public static void ViewFile(BbContentItem file) { Console.WriteLine(); Console.WriteLine("VIEWING FILE " + file.Name); Console.WriteLine("------------------------------------------------------------------"); Console.WriteLine(file); Console.WriteLine("------------------------------------------------------------------"); Console.WriteLine(); Console.Write("Download? [Y/N]: "); string choice = Console.ReadLine(); if (choice.ToUpper().StartsWith("Y")) { scraper.DownloadFile(file); } ViewDirectory(file.Folder); }
// Determines filename public void DetectFileName(BbContentItem file) { // if direct link to file or dropbox link, get filename from path if (file.LinkType == "directlink" || file.LinkType == "dropbox") { string filename = Path.GetFileName(file.Url.AbsoluteUri); file.Filename = filename; } else { try { // Sends a request to file URL and inspects Location header of response for filename HttpWebRequest request = (HttpWebRequest)WebRequest.Create(file.Url); request.AllowAutoRedirect = false; request.Headers.Add("Cookie", cookieHeader); using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) { Uri fileURL = new Uri(response.Headers["Location"]); string filename = Path.GetFileName(fileURL.LocalPath); file.Filename = filename; } } catch (System.ArgumentNullException e) { log.Write("ERROR: Could not detect filename for " + file); log.WriteException(e); downloadProgress.ReportError("ERROR: Could not detect filename for " + file.Name + " - No location header"); } catch (WebException e) { log.Write("ERROR: Could not detect filename for " + file); log.WriteException(e); downloadProgress.ReportError("ERROR: Could not detect filename for " + file.Name + " - WebException"); } catch (NotSupportedException e) { log.Write("ERROR: Could not detect filename for " + file); log.WriteException(e); downloadProgress.ReportError("ERROR: Could not detect filename for " + file.Name + " - URL format issue"); } } }
// Used recursively to populate all subfolders of a module public void PopulateContentDirectory(BbContentDirectory folder) { string pageSource = http.DownloadString(folder.Url.AbsoluteUri); List <HtmlNode> contentLinks = HTMLParser.GetContentLinks(pageSource); if (contentLinks == null) { return; } foreach (HtmlNode link in contentLinks) { //Console.WriteLine("Adding " + folder.Name + ": " + link.InnerText); populateProgress.AppendStatus("."); Uri linkURL = new Uri(folder.Url, link.Attributes["href"].Value); if (HTMLParser.IsSubFolder(link)) // content is a subfolder { BbContentDirectory subFolder = new BbContentDirectory(link.InnerText, linkURL, folder); if (!folder.SubFolders.Contains(subFolder)) { folder.AddSubFolder(subFolder); } PopulateContentDirectory(subFolder); } else if (HTMLParser.IsLearningUnit(link)) //content is a learning unit { BbContentDirectory subFolder = new BbContentDirectory(link.InnerText, linkURL, folder); if (!folder.SubFolders.Contains(subFolder)) { folder.AddSubFolder(subFolder); } PopulateLearningUnit(subFolder); } else // content is a file { string linkType = HTMLParser.GetLinkType(linkURL); BbContentItem newFile = new BbContentItem(link.InnerText, linkURL, folder, linkType); if (!folder.Files.Contains(newFile)) { folder.AddFile(newFile); } } } }
// Downloads a BbContentItem file and saves it to directory. public void DownloadFile(BbContentItem file, string directory) { string shortDir = directory.Substring(outputDirectory.Length, directory.Length - outputDirectory.Length); try { if (file.LinkType == "onedrive") { file.Url = OneDriveURL(file.Url); } if (file.LinkType == "dropbox") { file.Url = DropboxURL(file.Url); } else if (file.LinkType == "website") { log.Write("Linking to website " + file.Url.AbsoluteUri); downloadProgress.ReportStatus("Creating shortcut: " + shortDir + file.Name + file.Url.AbsoluteUri); CreateUrlShortcut(file, directory); return; } else if (file.LinkType == "email") { return; // Don't try to download e-mail } downloadProgress.ReportStatus("Downloading file (" + file.LinkType + "): " + shortDir + file.Name); Directory.CreateDirectory(directory); //Create directory if it doesn't exist already DetectFileName(file); http.DownloadFile(file.Url.AbsoluteUri, directory + file.Filename); } catch (WebException e) { log.WriteException(e); log.Write("ERROR: Cannot download file " + file); log.Write(shortDir); downloadProgress.ReportError("ERROR: Cannot download file " + file); } downloadProgress.IncWorkCounter(); }
// Add a BbContentItem to the directory public void AddFile(BbContentItem f) { files.Add(f); }
// Used recursively to populate all subfolders of a module public void PopulateContentDirectory(BbContentDirectory folder) { string pageSource = http.DownloadString(folder.Url.AbsoluteUri); List<HtmlNode> contentLinks = HTMLParser.GetContentLinks(pageSource); if (contentLinks == null) return; foreach (HtmlNode link in contentLinks) { //Console.WriteLine("Adding " + folder.Name + ": " + link.InnerText); populateProgress.AppendStatus("."); Uri linkURL = new Uri(folder.Url, link.Attributes["href"].Value); if (HTMLParser.IsSubFolder(link)) // content is a subfolder { BbContentDirectory subFolder = new BbContentDirectory(link.InnerText, linkURL, folder); if (!folder.SubFolders.Contains(subFolder)) { folder.AddSubFolder(subFolder); } PopulateContentDirectory(subFolder); } else if (HTMLParser.IsLearningUnit(link)) //content is a learning unit { BbContentDirectory subFolder = new BbContentDirectory(link.InnerText, linkURL, folder); if (!folder.SubFolders.Contains(subFolder)) { folder.AddSubFolder(subFolder); } PopulateLearningUnit(subFolder); } else // content is a file { string linkType = HTMLParser.GetLinkType(linkURL); BbContentItem newFile = new BbContentItem(link.InnerText, linkURL, folder, linkType); if (!folder.Files.Contains(newFile)) { folder.AddFile(newFile); } } } }
// Returns true if other BbContentItem has the same url public bool Equals(BbContentItem other) { if (other == null) { return false; } return this.url.Equals(other.Url); }
public void DownloadFile(BbContentItem file) { DownloadFile(file, outputDirectory); }