private Fic FicFromTb(TextBlock tb)
{
var fic_id = tb.Tag.ToString();
Fic current = currentDisplay[fic_id];
return(current);
}
private Fic GetFic(String url)
{
Fic fic = new Fic();
url += "?view_adult=true";
if (new Uri(url).Host != new Uri(Constants.BaseUrl).Host)
{
Log($"{url} is from an unsupported domain, this application only supports pages from AO3.");
return(null);
}
if (url.Contains("series"))
{
Log($"{url} is a series, this application currently does not support downloading entire series. Skipping...");
return(null);
}
Log("Fetching " + url);
try
{
using (WebClient w = new WebClient())
{
w.Encoding = System.Text.Encoding.UTF8;
if (this.userCancel)
{
return(null);
}
string htmlCode;
try
{
htmlCode = w.DownloadString(url);
}
catch
{
Log($"Failed to get data for {url}");
return(null);
}
try
{
fic = AO3LinkHelper.GetFicData(htmlCode);
}
catch
{
htmlCode = w.DownloadString(AO3LinkHelper.ProceedUrl(url));
fic = AO3LinkHelper.GetFicData(htmlCode);
}
}
fic.Url = url;
return(fic);
}
catch (Exception)
{
Log($"Unable to get download links for {url}");
return(null);
}
}
public static void StoreFic(Fic fic)
{
using (var db = new LiteDatabase(Constants.LiteDbPath))
{
var store = db.GetCollection <Fic>("Works");
store.Upsert(fic);
}
}
public static Fic Merge(Fic one, Fic two)
{
// This might need an update later
DateTime dateOne = DateTime.Parse(one.Updated);
DateTime dateTwo = DateTime.Parse(two.Updated);
if (dateOne < dateTwo)
{
foreach (KeyValuePair <string, string> pair in one.LocalFiles)
{
if (!two.LocalFiles.ContainsKey(pair.Key))
{
two.LocalFiles[pair.Key] = pair.Value;
}
}
}
return(two);
}
private void Start_Click(object sender, RoutedEventArgs e)
{
if (pasteBox.Text == Constants.PasteBoxDefText)
{
Log("There are no URLs to download!");
return;
}
List <String> urlEntries = pasteBox.Text.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries).ToList();
urlEntries = urlEntries.Distinct().ToList();
this.pendingDownloads = new List <WebClient>();
this.activeDownloads = true;
UpdateControls(true);
var fics = new SerializableDictionary <string, Fic>();
var dlLinks = new List <String>();
// NOTE: These are also set when using the browse dialog
userSettings.DownloadLocation = saveLocation.Text;
userSettings.DevicePath = deviceLocation.Text;
Task.Factory.StartNew(() =>
{
foreach (string urlEntry in urlEntries)
{
Fic fic = GetFic(urlEntry);
if (fic != null)
{
fics[fic.ID] = fic;
IncrementProgress(labelProgressedFetched);
BeGentle(linksTotal: urlEntries.Count);
}
}
}).ContinueWith(x =>
{
// TODO create flag
//foreach (Fic fic in fics)
//{
// foreach (KeyValuePair<string, string> entry in fic.Downloads)
// {
// dlLinks.Add(entry.Value);
// }
//}
//Task[] tasks = new Task[dlLinks.Count];
//for (int i = 0; i < dlLinks.Count; i++)
//{
// string link = dlLinks[i];
// tasks[i] = Task.Factory.StartNew(() => DownloadEbook(link)); // THIS IS IMPORTANT
//}
//Task.WaitAll(tasks); // SO IS THIS
//}).ContinueWith(x =>
//{
// Both download methods are kept for now, for performance testing purposes
// New method, to hopefully make connecting metadata and file easier
List <string> formats = getSelectedFormats();
int totalDl = fics.Count * formats.Count;
int currentDl = 0;
Task[] tasks = new Task[totalDl];
foreach (KeyValuePair <string, Fic> pair in fics)
{
var fic = pair.Value;
tasks[currentDl] = Task.Factory.StartNew(() =>
{
foreach (KeyValuePair <string, string> entry in fic.Downloads)
{
if (formats.Contains(entry.Key))
{
string success = DownloadEbook(entry.Value);
if (!String.IsNullOrEmpty(success))
{
fic.LocalFiles[entry.Key] = success;
IncrementProgress(labelProgressDownloaded);
Log($"Downloaded {success}...");
}
BeGentle(linksTotal: totalDl);
}
}
});
currentDl++;
}
Task.WaitAll(tasks);
}).ContinueWith(x =>
{
if (this.userCancel)
{
Log("Cancelled by user.");
}
}).ContinueWith(x =>
{
this.activeDownloads = false;
Dictionary <string, Fic> allFics = new Dictionary <string, Fic>();
try
{
allFics = MiddleDude.GetFics();
//allFics = XmlOperator.DeserializeFile<SerializableDictionary<string, Fic>>(Constants.WorksRef);
}
catch (System.Xml.XmlException)
{
}
if (allFics != null)
{
foreach (KeyValuePair <string, Fic> pair in fics)
{
if (!allFics.ContainsKey(pair.Key))
{
allFics[pair.Key] = pair.Value;
}
else
{
Fic mergedFic = Fic.Merge(allFics[pair.Key], pair.Value);
allFics[pair.Key] = mergedFic;
}
}
}
else
{
allFics = fics;
}
MiddleDude.StoreFics(allFics);
UpdateControls(false);
Log("Downloads complete!");
});
}
public static void AddWork(Fic work)
{
;
}
// A LOT OF THIS IS NOT VERY GOOD PLEASE CLOSE YOUR EYES THANK YOU
public static Fic GetFicData(string htmlCode)
{
// TODO look into more efficient value retrieval and assignment
var fic = new Fic();
List <string> formats = new List <string>();
foreach (string fmt in Constants.AllowedExt)
{
formats.Add(fmt.ToUpper()); // Not sure if this is necessary, but it worked with uppercase before so nyeh
}
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(htmlCode);
// Single-value attributes
string title = GetElementValue(htmlDoc, Xpaths.Title).Replace("\n", String.Empty).TrimStart(' ').TrimEnd(' ');
string lengthString = GetElementValue(htmlDoc, Xpaths.Words);
int length = int.Parse(lengthString);
string chaptersString = GetElementValue(htmlDoc, Xpaths.Chapters);
int chapters;
try
{
chapters = int.Parse(chaptersString.Split('/')[1]);
}
catch (Exception)
{
chapters = int.Parse(chaptersString.Split('/')[0]);
}
string published = GetElementValue(htmlDoc, Xpaths.Published);
string updated;
try
{
updated = GetElementValue(htmlDoc, Xpaths.Updated);
}
catch (Exception)
{
updated = published;
}
string rating = GetElementValue(htmlDoc, Xpaths.Rating);
// Multi-value attributes
var authorElements = htmlDoc.DocumentNode.SelectNodes(Xpaths.Author);
List <String> authors = new List <String>();
if (authorElements != null)
{
foreach (HtmlNode authorNode in authorElements)
{
string authorUrl;
try
{
authorUrl = authorNode.Attributes["href"].Value;
}
catch (Exception)
{
authorUrl = "";
}
string authorName;
try
{
authorName = authorNode.InnerText;
}
catch (Exception)
{
authorName = "Anonymous";
}
authors.Add(authorName + " : " + authorUrl);
}
}
else
{
authors.Add("Anonymous");
}
SerializableDictionary <string, string> downloadLinks = new SerializableDictionary <string, string>();
foreach (string format in formats)
{
string formatPath = string.Format(Xpaths.Download, format);
string formatUrl = htmlDoc.DocumentNode.SelectSingleNode(formatPath).Attributes["href"].Value;
downloadLinks.Add(format, Constants.BaseUrl + formatUrl);
}
List <String> categories = new List <String>();
try
{
var categoryElements = htmlDoc.DocumentNode.SelectNodes(Xpaths.Category);
foreach (HtmlNode catNode in categoryElements)
{
string catName = catNode.InnerText;
categories.Add(catName);
}
}
catch (Exception)
{
}
List <String> fandoms = new List <String>();
try
{
var fandomElements = htmlDoc.DocumentNode.SelectNodes(Xpaths.Fandom);
foreach (HtmlNode fanNode in fandomElements)
{
string fandomName = fanNode.InnerText;
fandoms.Add(fandomName);
}
}
catch (Exception)
{
}
List <String> relationships = new List <String>();
try
{
var relElements = htmlDoc.DocumentNode.SelectNodes(Xpaths.Relationship);
foreach (HtmlNode relNode in relElements)
{
string relName = relNode.InnerText;
relationships.Add(relName);
}
}
catch (Exception)
{
}
List <String> characters = new List <String>();
try
{
var charElements = htmlDoc.DocumentNode.SelectNodes(Xpaths.Characters);
foreach (HtmlNode charNode in charElements)
{
string charName = charNode.InnerText;
characters.Add(charName);
}
}
catch (Exception)
{
}
List <String> tags = new List <String>();
try
{
var tagElements = htmlDoc.DocumentNode.SelectNodes(Xpaths.Tags);
foreach (HtmlNode tagNode in tagElements)
{
string tagName = tagNode.InnerText;
tags.Add(tagName);
}
}
catch (Exception)
{
}
List <string> summary = new List <string>();
try
{
var parElements = htmlDoc.DocumentNode.SelectNodes(Xpaths.Summary);
foreach (HtmlNode parNode in parElements)
{
string line = parNode.InnerText;
summary.Add(line);
}
}
catch (Exception)
{
}
// Property assignment
fic.Title = title;
fic.Words = length;
fic.Chapters = chapters;
fic.Downloads = downloadLinks;
fic.Published = published;
fic.Updated = updated;
fic.Rating = rating;
fic.Author = authors.ToArray();
fic.Category = categories.ToArray();
fic.Fandom = fandoms.ToArray();
fic.Relationship = relationships.ToArray();
fic.Characters = characters.ToArray();
fic.AdditionalTags = tags.ToArray();
fic.Summary = String.Join(Environment.NewLine, summary);
return(fic);
}
