// DoWork event handler for the file download BackgroundWorker.
// The selected TreeNode (either a module, folder, or file) is passed in the DoWorkEventArgs e
private void DownloadBW_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker worker = sender as BackgroundWorker;
TreeNode selectedNode = e.Argument as TreeNode;
scraper.DownloadProgress.BeginJob(worker);
// -- DETERMINE TYPE OF NODE SELECTED --
// Module selected
if (selectedNode.Tag.GetType() == typeof(BbModule))
{
BbModule module = selectedNode.Tag as BbModule;
scraper.DownloadProgress.TotalWork = module.Content.CountAllFiles();
scraper.DownloadModuleFiles(module);
}
// Single folder selected
else if (selectedNode.Tag.GetType() == typeof(BbContentDirectory))
{
BbContentDirectory folder = selectedNode.Tag as BbContentDirectory;
scraper.DownloadProgress.TotalWork = folder.CountAllFiles();
scraper.DownloadFolder(folder, scraper.OutputDirectory + BbUtils.CleanDirectory(folder.Name) + @"\");
}
// Single file selected
else if (selectedNode.Tag.GetType() == typeof(BbContentItem))
{
BbContentItem file = selectedNode.Tag as BbContentItem;
scraper.DownloadProgress.TotalWork = 1;
scraper.DownloadFile(file);
}
}
// Returns true if other BbContentItem has the same url
public bool Equals(BbContentItem other)
{
if (other == null)
{
return(false);
}
return(this.url.Equals(other.Url));
}
// Adds given BbContentItem file to the content treeview.
private void AddTreeFile(TreeNode parent, BbContentItem file)
{
TreeNode fileNode = new TreeNode(file.Name);
fileNode.Tag = file;
fileNode.ImageIndex = 4;
fileNode.SelectedImageIndex = 4;
parent.Nodes.Add(fileNode);
}
public void CreateUrlShortcut(BbContentItem file, string directory)
{
Directory.CreateDirectory(directory);
using (StreamWriter w = new StreamWriter(directory + "\\" + file.Filename + ".url"))
{
w.WriteLine("[InternetShortcut]");
w.WriteLine("URL=" + file.Url.AbsoluteUri);
}
}
// Event Handler for the content treeview. Changes information displayed
private void contentTree_AfterSelect(object sender, TreeViewEventArgs e)
{
ClearInfoBox();
// Change what's displayed in the Information Box depending on type of node selected
// File selected
if (contentTree.SelectedNode.Tag.GetType() == typeof(BbContentItem))
{
BbContentItem file = contentTree.SelectedNode.Tag as BbContentItem;
infoLabel1.Text = "Name";
infoText1.Text = file.Name;
infoLabel2.Text = "Filename";
infoText2.Text = file.Filename;
infoLabel3.Text = "Link Type";
infoText3.Text = file.LinkType;
infoLabel4.Text = "URL";
infoTextLink.Text = file.Url.AbsoluteUri;
}
// Folder selected
else if (contentTree.SelectedNode.Tag.GetType() == typeof(BbContentDirectory))
{
BbContentDirectory folder = contentTree.SelectedNode.Tag as BbContentDirectory;
infoLabel1.Text = "Name";
infoText1.Text = folder.Name;
infoLabel2.Text = "Files";
infoText2.Text = folder.CountAllFiles().ToString();
infoLabel3.Text = "Subfolders";
infoText3.Text = folder.SubFolders.Count.ToString();
infoLabel4.Text = "URL";
infoTextLink.Text = folder.Url.AbsoluteUri;
}
// Module selected
else if (contentTree.SelectedNode.Tag.GetType() == typeof(BbModule))
{
BbModule module = contentTree.SelectedNode.Tag as BbModule;
infoLabel1.Text = "Name";
infoText1.Text = module.Name;
infoLabel2.Text = "Files";
infoText2.Text = module.Content.CountAllFiles().ToString();
infoLabel3.Text = "Subfolders";
infoText3.Text = module.Content.SubFolders.Count.ToString();
infoLabel4.Text = "URL";
infoTextLink.Text = module.Url.AbsoluteUri;
}
}
public static void ViewFile(BbContentItem file)
{
Console.WriteLine();
Console.WriteLine("VIEWING FILE " + file.Name);
Console.WriteLine("------------------------------------------------------------------");
Console.WriteLine(file);
Console.WriteLine("------------------------------------------------------------------");
Console.WriteLine();
Console.Write("Download? [Y/N]: ");
string choice = Console.ReadLine();
if (choice.ToUpper().StartsWith("Y"))
{
scraper.DownloadFile(file);
}
ViewDirectory(file.Folder);
}
// Determines filename
public void DetectFileName(BbContentItem file)
{
// if direct link to file or dropbox link, get filename from path
if (file.LinkType == "directlink" || file.LinkType == "dropbox")
{
string filename = Path.GetFileName(file.Url.AbsoluteUri);
file.Filename = filename;
}
else
{
try
{
// Sends a request to file URL and inspects Location header of response for filename
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(file.Url);
request.AllowAutoRedirect = false;
request.Headers.Add("Cookie", cookieHeader);
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
Uri fileURL = new Uri(response.Headers["Location"]);
string filename = Path.GetFileName(fileURL.LocalPath);
file.Filename = filename;
}
}
catch (System.ArgumentNullException e)
{
log.Write("ERROR: Could not detect filename for " + file);
log.WriteException(e);
downloadProgress.ReportError("ERROR: Could not detect filename for " + file.Name + " - No location header");
}
catch (WebException e)
{
log.Write("ERROR: Could not detect filename for " + file);
log.WriteException(e);
downloadProgress.ReportError("ERROR: Could not detect filename for " + file.Name + " - WebException");
}
catch (NotSupportedException e)
{
log.Write("ERROR: Could not detect filename for " + file);
log.WriteException(e);
downloadProgress.ReportError("ERROR: Could not detect filename for " + file.Name + " - URL format issue");
}
}
}
// Used recursively to populate all subfolders of a module
public void PopulateContentDirectory(BbContentDirectory folder)
{
string pageSource = http.DownloadString(folder.Url.AbsoluteUri);
List <HtmlNode> contentLinks = HTMLParser.GetContentLinks(pageSource);
if (contentLinks == null)
{
return;
}
foreach (HtmlNode link in contentLinks)
{
//Console.WriteLine("Adding " + folder.Name + ": " + link.InnerText);
populateProgress.AppendStatus(".");
Uri linkURL = new Uri(folder.Url, link.Attributes["href"].Value);
if (HTMLParser.IsSubFolder(link)) // content is a subfolder
{
BbContentDirectory subFolder = new BbContentDirectory(link.InnerText, linkURL, folder);
if (!folder.SubFolders.Contains(subFolder))
{
folder.AddSubFolder(subFolder);
}
PopulateContentDirectory(subFolder);
}
else if (HTMLParser.IsLearningUnit(link)) //content is a learning unit
{
BbContentDirectory subFolder = new BbContentDirectory(link.InnerText, linkURL, folder);
if (!folder.SubFolders.Contains(subFolder))
{
folder.AddSubFolder(subFolder);
}
PopulateLearningUnit(subFolder);
}
else // content is a file
{
string linkType = HTMLParser.GetLinkType(linkURL);
BbContentItem newFile = new BbContentItem(link.InnerText, linkURL, folder, linkType);
if (!folder.Files.Contains(newFile))
{
folder.AddFile(newFile);
}
}
}
}
// Downloads a BbContentItem file and saves it to directory.
public void DownloadFile(BbContentItem file, string directory)
{
string shortDir = directory.Substring(outputDirectory.Length, directory.Length - outputDirectory.Length);
try
{
if (file.LinkType == "onedrive")
{
file.Url = OneDriveURL(file.Url);
}
if (file.LinkType == "dropbox")
{
file.Url = DropboxURL(file.Url);
}
else if (file.LinkType == "website")
{
log.Write("Linking to website " + file.Url.AbsoluteUri);
downloadProgress.ReportStatus("Creating shortcut: " + shortDir + file.Name + file.Url.AbsoluteUri);
CreateUrlShortcut(file, directory);
return;
}
else if (file.LinkType == "email")
{
return; // Don't try to download e-mail
}
downloadProgress.ReportStatus("Downloading file (" + file.LinkType + "): " + shortDir + file.Name);
Directory.CreateDirectory(directory); //Create directory if it doesn't exist already
DetectFileName(file);
http.DownloadFile(file.Url.AbsoluteUri, directory + file.Filename);
}
catch (WebException e)
{
log.WriteException(e);
log.Write("ERROR: Cannot download file " + file);
log.Write(shortDir);
downloadProgress.ReportError("ERROR: Cannot download file " + file);
}
downloadProgress.IncWorkCounter();
}
// Add a BbContentItem to the directory
public void AddFile(BbContentItem f)
{
files.Add(f);
}
// Used recursively to populate all subfolders of a module
public void PopulateContentDirectory(BbContentDirectory folder)
{
string pageSource = http.DownloadString(folder.Url.AbsoluteUri);
List<HtmlNode> contentLinks = HTMLParser.GetContentLinks(pageSource);
if (contentLinks == null) return;
foreach (HtmlNode link in contentLinks)
{
//Console.WriteLine("Adding " + folder.Name + ": " + link.InnerText);
populateProgress.AppendStatus(".");
Uri linkURL = new Uri(folder.Url, link.Attributes["href"].Value);
if (HTMLParser.IsSubFolder(link)) // content is a subfolder
{
BbContentDirectory subFolder = new BbContentDirectory(link.InnerText, linkURL, folder);
if (!folder.SubFolders.Contains(subFolder))
{
folder.AddSubFolder(subFolder);
}
PopulateContentDirectory(subFolder);
}
else if (HTMLParser.IsLearningUnit(link)) //content is a learning unit
{
BbContentDirectory subFolder = new BbContentDirectory(link.InnerText, linkURL, folder);
if (!folder.SubFolders.Contains(subFolder))
{
folder.AddSubFolder(subFolder);
}
PopulateLearningUnit(subFolder);
}
else // content is a file
{
string linkType = HTMLParser.GetLinkType(linkURL);
BbContentItem newFile = new BbContentItem(link.InnerText, linkURL, folder, linkType);
if (!folder.Files.Contains(newFile))
{
folder.AddFile(newFile);
}
}
}
}
// Determines filename
public void DetectFileName(BbContentItem file)
{
// if direct link to file or dropbox link, get filename from path
if (file.LinkType == "directlink" || file.LinkType == "dropbox")
{
string filename = Path.GetFileName(file.Url.AbsoluteUri);
file.Filename = filename;
}
else
{
try
{
// Sends a request to file URL and inspects Location header of response for filename
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(file.Url);
request.AllowAutoRedirect = false;
request.Headers.Add("Cookie", cookieHeader);
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
Uri fileURL = new Uri(response.Headers["Location"]);
string filename = Path.GetFileName(fileURL.LocalPath);
file.Filename = filename;
}
}
catch (System.ArgumentNullException e)
{
log.Write("ERROR: Could not detect filename for " + file);
log.WriteException(e);
downloadProgress.ReportError("ERROR: Could not detect filename for " + file.Name + " - No location header");
}
catch (WebException e)
{
log.Write("ERROR: Could not detect filename for " + file);
log.WriteException(e);
downloadProgress.ReportError("ERROR: Could not detect filename for " + file.Name + " - WebException");
}
catch (NotSupportedException e)
{
log.Write("ERROR: Could not detect filename for " + file);
log.WriteException(e);
downloadProgress.ReportError("ERROR: Could not detect filename for " + file.Name + " - URL format issue");
}
}
}
public void CreateUrlShortcut(BbContentItem file, string directory)
{
Directory.CreateDirectory(directory);
using (StreamWriter w = new StreamWriter(directory + "\\" + file.Filename + ".url"))
{
w.WriteLine("[InternetShortcut]");
w.WriteLine("URL=" + file.Url.AbsoluteUri);
}
}
// Returns true if other BbContentItem has the same url
public bool Equals(BbContentItem other)
{
if (other == null)
{
return false;
}
return this.url.Equals(other.Url);
}
public void DownloadFile(BbContentItem file)
{
DownloadFile(file, outputDirectory);
}
public static void ViewFile(BbContentItem file)
{
Console.WriteLine();
Console.WriteLine("VIEWING FILE " + file.Name);
Console.WriteLine("------------------------------------------------------------------");
Console.WriteLine(file);
Console.WriteLine("------------------------------------------------------------------");
Console.WriteLine();
Console.Write("Download? [Y/N]: ");
string choice = Console.ReadLine();
if (choice.ToUpper().StartsWith("Y"))
{
scraper.DownloadFile(file);
}
ViewDirectory(file.Folder);
}
// Downloads a BbContentItem file and saves it to directory.
public void DownloadFile(BbContentItem file, string directory)
{
string shortDir = directory.Substring(outputDirectory.Length, directory.Length - outputDirectory.Length);
try
{
if (file.LinkType == "onedrive")
{
file.Url = OneDriveURL(file.Url);
}
if (file.LinkType == "dropbox")
{
file.Url = DropboxURL(file.Url);
}
else if (file.LinkType == "website")
{
log.Write("Linking to website " + file.Url.AbsoluteUri);
downloadProgress.ReportStatus("Creating shortcut: " + shortDir + file.Name + file.Url.AbsoluteUri);
CreateUrlShortcut(file, directory);
return;
}
else if (file.LinkType == "email")
{
return; // Don't try to download e-mail
}
downloadProgress.ReportStatus("Downloading file (" + file.LinkType + "): " + shortDir + file.Name);
Directory.CreateDirectory(directory); //Create directory if it doesn't exist already
DetectFileName(file);
http.DownloadFile(file.Url.AbsoluteUri, directory + file.Filename);
}
catch (WebException e)
{
log.WriteException(e);
log.Write("ERROR: Cannot download file " + file);
log.Write(shortDir);
downloadProgress.ReportError("ERROR: Cannot download file " + file);
}
downloadProgress.IncWorkCounter();
}
// Add a BbContentItem to the directory
public void AddFile(BbContentItem f)
{
files.Add(f);
}
public void DownloadFile(BbContentItem file)
{
DownloadFile(file, outputDirectory);
}
