/// <summary>
/// Read the chapters or search for them and apply them to the given <param name="xray"></param>
/// </summary>
// TODO Do something about unattended/dialog stuff
public void HandleChapters(XRay xray, string asin, long mlLen, HtmlDocument doc, string rawMl, bool overwriteChapters, SafeShowDelegate safeShow, bool unattended, bool enableEdit)
{
//Similar to aliases, if chapters definition exists, load it. Otherwise, attempt to build it from the book
var chapterFile = $@"{Environment.CurrentDirectory}\ext\{asin}.chapters";
if (File.Exists(chapterFile) && !overwriteChapters)
{
try
{
xray.Chapters = LoadChapters(asin).ToList();
_logger.Log($"Chapters read from {chapterFile}.\r\nDelete this file if you want chapters built automatically.");
}
catch (Exception ex)
{
_logger.Log($"An error occurred reading chapters from {chapterFile}: {ex.Message}");
}
}
else
{
try
{
var chapters = SearchChapters(doc, rawMl);
if (chapters != null)
{
xray.Chapters = chapters.ToList();
}
}
catch (Exception ex)
{
_logger.Log("Error searching for chapters: " + ex.Message);
}
//Built chapters list is saved for manual editing
if (xray.Chapters.Count > 0)
{
SaveChapters(xray);
_logger.Log($"Chapters exported to {chapterFile} for manual editing.");
}
else
{
_logger.Log($"No chapters detected.\r\nYou can create a file at {chapterFile} if you want to define chapters manually.");
}
}
if (!unattended && enableEdit)
{
if (DialogResult.Yes ==
safeShow("Would you like to open the chapters file in notepad for editing?", "Chapters",
MessageBoxButtons.YesNo, MessageBoxIcon.Question, MessageBoxDefaultButton.Button2))
{
Functions.RunNotepad(chapterFile);
xray.Chapters.Clear();
try
{
xray.Chapters = LoadChapters(asin).ToList();
_logger.Log("Reloaded chapters from edited file.");
}
catch (Exception ex)
{
_logger.Log($"An error occurred reading chapters from {chapterFile}: {ex.Message}");
}
}
}
//If no chapters were found, add a default chapter that spans the entire book
//Define srl and erl so "progress bar" shows up correctly
if (xray.Chapters.Count == 0)
{
xray.Chapters.Add(new Chapter
{
Name = "",
Start = 1,
End = mlLen
});
xray.Srl = 1;
xray.Erl = mlLen;
}
else
{
//Run through all chapters and take the highest value, in case some chapters can be defined in individual chapters and parts.
//EG. Part 1 includes chapters 1-6, Part 2 includes chapters 7-12.
xray.Srl = xray.Chapters[0].Start;
_logger.Log("Found chapters:");
foreach (var c in xray.Chapters)
{
if (c.End > xray.Erl)
{
xray.Erl = c.End;
}
_logger.Log($"{c.Name} | start: {c.Start} | end: {c.End}");
}
}
}
// TODO split this up, possible return a result instead of modifying xray
public void ExpandFromRawMl(
XRay xray,
IMetadata metadata,
Stream rawMlStream,
bool enableEdit,
bool useNewVersion,
bool skipNoLikes,
int minClipLen,
bool overwriteChapters,
SafeShowDelegate safeShow,
IProgressBar progress,
CancellationToken token,
bool ignoreSoftHypen = false,
bool shortEx = true)
{
var locOffset = metadata.IsAzw3 ? -16 : 0;
// If there is an apostrophe, attempt to match 's at the end of the term
// Match end of word, then search for any lingering punctuation
var apostrophes = _encoding.GetString(Encoding.UTF8.GetBytes("('|\u2019|\u0060|\u00B4)")); // '\u2019\u0060\u00B4
var quotes = _encoding.GetString(Encoding.UTF8.GetBytes("(\"|\u2018|\u2019|\u201A|\u201B|\u201C|\u201D|\u201E|\u201F)"));
var dashesEllipsis = _encoding.GetString(Encoding.UTF8.GetBytes("(-|\u2010|\u2011|\u2012|\u2013|\u2014|\u2015|\u2026|–|—|’|‚|…)")); //U+2010 to U+2015 and U+2026
var punctuationMarks = string.Format(@"({0}s|{0})?{1}?[!\.?,""\);:]*{0}*{1}*{2}*", apostrophes, quotes, dashesEllipsis);
var excerptId = 0;
var web = new HtmlDocument();
web.Load(rawMlStream, _encoding);
// Only load chapters when building the old format
if (!useNewVersion)
{
rawMlStream.Seek(0, SeekOrigin.Begin);
// TODO: passing stream, doc, and contents probably not necessary)
using var streamReader = new StreamReader(rawMlStream, Encoding.UTF8);
var readContents = streamReader.ReadToEnd();
var utf8Doc = new HtmlDocument();
utf8Doc.LoadHtml(readContents);
_chaptersService.HandleChapters(xray, xray.Asin, rawMlStream.Length, utf8Doc, readContents, overwriteChapters, safeShow, xray.Unattended, enableEdit);
}
else
{
// set default ERL to prevent filtering
xray.Srl = 1;
xray.Erl = rawMlStream.Length;
}
_logger.Log("Scanning book content...");
var timer = new System.Diagnostics.Stopwatch();
timer.Start();
//Iterate over all paragraphs in book
var nodes = web.DocumentNode.SelectNodes("//p")
?? web.DocumentNode.SelectNodes("//div[@class='paragraph']")
?? web.DocumentNode.SelectNodes("//div[@class='p-indent']");
if (nodes == null)
{
nodes = web.DocumentNode.SelectNodes("//div");
_logger.Log("Warning: Could not locate paragraphs normally (p elements or divs of class 'paragraph').\r\n" +
"Searching all book contents (all divs), which may produce odd results.");
}
if (nodes == null)
{
throw new Exception("Could not locate any paragraphs in this book.\r\n" +
"Report this error along with a copy of the book to improve parsing.");
}
progress?.Set(0, nodes.Count);
for (var i = 0; i < nodes.Count; i++)
{
token.ThrowIfCancellationRequested();
var node = nodes[i];
if (node.FirstChild == null)
{
continue; //If the inner HTML is just empty, skip the paragraph!
}
var lenQuote = node.InnerHtml.Length;
var location = node.FirstChild.StreamPosition;
if (location < 0)
{
throw new Exception($"Unable to locate paragraph {i} within the book content.");
}
//Skip paragraph if outside chapter range
if (location < xray.Srl || location > xray.Erl)
{
continue;
}
var noSoftHypen = "";
if (ignoreSoftHypen)
{
noSoftHypen = node.InnerText;
noSoftHypen = noSoftHypen.Replace("\u00C2\u00AD", "");
noSoftHypen = noSoftHypen.Replace("­", "");
noSoftHypen = noSoftHypen.Replace("­", "");
noSoftHypen = noSoftHypen.Replace("­", "");
noSoftHypen = noSoftHypen.Replace("­", "");
}
foreach (var character in xray.Terms)
{
//Search for character name and aliases in the html-less text. If failed, try in the HTML for rare situations.
//TODO: Improve location searching as IndexOf will not work if book length exceeds 2,147,483,647...
//If soft hyphen ignoring is turned on, also search hyphen-less text.
if (!character.Match)
{
continue;
}
var termFound = false;
// Convert from UTF8 string to default-encoded representation
var search = character.Aliases.Select(alias => _encoding.GetString(Encoding.UTF8.GetBytes(alias)))
.ToList();
if (character.RegexAliases)
{
if (search.Any(r => Regex.Match(node.InnerText, r).Success) ||
search.Any(r => Regex.Match(node.InnerHtml, r).Success) ||
(ignoreSoftHypen && search.Any(r => Regex.Match(noSoftHypen, r).Success)))
{
termFound = true;
}
}
else
{
// Search for character name and aliases
// If there is an apostrophe, attempt to match 's at the end of the term
// Match end of word, then search for any lingering punctuation
search.Add(character.TermName);
// Search list should be in descending order by length, even the term name itself
search = search.OrderByDescending(s => s.Length).ToList();
if ((character.MatchCase && (search.Any(node.InnerText.Contains) || search.Any(node.InnerHtml.Contains))) ||
(!character.MatchCase && (search.Any(node.InnerText.ContainsIgnorecase) || search.Any(node.InnerHtml.ContainsIgnorecase))) ||
(ignoreSoftHypen && (character.MatchCase && search.Any(noSoftHypen.Contains)) ||
(!character.MatchCase && search.Any(noSoftHypen.ContainsIgnorecase))))
{
termFound = true;
}
}
if (!termFound)
{
continue;
}
var locHighlight = new List <int>();
var lenHighlight = new List <int>();
//Search html for character name and aliases
foreach (var s in search)
{
var matches = Regex.Matches(node.InnerHtml, $@"{quotes}?\b{s}{punctuationMarks}", character.MatchCase || character.RegexAliases ? RegexOptions.None : RegexOptions.IgnoreCase);
foreach (Match match in matches)
{
if (locHighlight.Contains(match.Index) && lenHighlight.Contains(match.Length))
{
continue;
}
locHighlight.Add(match.Index);
lenHighlight.Add(match.Length);
}
}
//If normal search fails, use regexp to search in case there is some wacky html nested in term
//Regexp may be less than ideal for parsing HTML but seems to work ok so far in these small paragraphs
//Also search in soft hyphen-less text if option is set to do so
if (locHighlight.Count == 0)
{
foreach (var s in search)
{
var patterns = new List <string>();
const string patternHtml = "(?:<[^>]*>)*";
//Match HTML tags -- provided there's nothing malformed
const string patternSoftHypen = "(\u00C2\u00AD|­|­|­|­|­)*";
var pattern = string.Format("{0}{1}{0}{2}",
patternHtml,
string.Join(patternHtml + patternSoftHypen, character.RegexAliases ? s.ToCharArray() : Regex.Unescape(s).ToCharArray()),
punctuationMarks);
patterns.Add(pattern);
foreach (var pat in patterns)
{
MatchCollection matches;
if (character.MatchCase || character.RegexAliases)
{
matches = Regex.Matches(node.InnerHtml, pat);
}
else
{
matches = Regex.Matches(node.InnerHtml, pat, RegexOptions.IgnoreCase);
}
foreach (Match match in matches)
{
if (locHighlight.Contains(match.Index) && lenHighlight.Contains(match.Length))
{
continue;
}
locHighlight.Add(match.Index);
lenHighlight.Add(match.Length);
}
}
}
}
if (locHighlight.Count == 0 || locHighlight.Count != lenHighlight.Count) //something went wrong
{
// _logger.Log($"An error occurred while searching for start of highlight.\r\nWas looking for (or one of the aliases of): {character.TermName}\r\nSearching in: {node.InnerHtml}");
continue;
}
//If an excerpt is too long, the X-Ray reader cuts it off.
//If the location of the highlighted word (character name) within the excerpt is far enough in to get cut off,
//this section attempts to shorted the excerpt by locating the start of a sentence that is just far enough away from the highlight.
//The length is determined by the space the excerpt takes up rather than its actual length... so 135 is just a guess based on what I've seen.
const int lengthLimit = 135;
for (var j = 0; j < locHighlight.Count; j++)
{
if (!shortEx || locHighlight[j] + lenHighlight[j] <= lengthLimit)
{
continue;
}
var start = locHighlight[j];
long newLoc = -1;
var newLenQuote = 0;
var newLocHighlight = 0;
while (start > -1)
{
var at = node.InnerHtml.LastIndexOfAny(new[] { '.', '?', '!' }, start);
if (at > -1)
{
start = at - 1;
if (locHighlight[j] + lenHighlight[j] + 1 - at - 2 <= lengthLimit)
{
newLoc = location + at + 2;
newLenQuote = lenQuote - at - 2;
newLocHighlight = locHighlight[j] - at - 2;
}
else
{
break;
}
}
else
{
break;
}
}
//Only add new locs if shorter excerpt was found
if (newLoc >= 0)
{
character.Locs.Add(new []
{
newLoc + locOffset,
newLenQuote,
newLocHighlight,
lenHighlight[j]
});
locHighlight.RemoveAt(j);
lenHighlight.RemoveAt(j--);
}
}
for (var j = 0; j < locHighlight.Count; j++)
{
// For old format
character.Locs.Add(new long[]
{
location + locOffset,
lenQuote,
locHighlight[j],
lenHighlight[j]
});
// For new format
character.Occurrences.Add(new[] { location + locOffset + locHighlight[j], lenHighlight[j] });
}
var exCheck = xray.Excerpts.Where(t => t.Start.Equals(location + locOffset)).ToArray();
if (exCheck.Length > 0)
{
if (!exCheck[0].RelatedEntities.Contains(character.Id))
{
exCheck[0].RelatedEntities.Add(character.Id);
}
}
else
{
var newExcerpt = new Excerpt
{
Id = excerptId++,
Start = location + locOffset,
Length = lenQuote
};
newExcerpt.RelatedEntities.Add(character.Id);
xray.Excerpts.Add(newExcerpt);
}
}
// Attempt to match downloaded notable clips, not worried if no matches occur as some will be added later anyway
if (useNewVersion && xray.NotableClips != null)
{
foreach (var quote in xray.NotableClips)
{
var index = node.InnerText.IndexOf(quote.Text, StringComparison.Ordinal);
if (index > -1)
{
// See if an excerpt already exists at this location
var excerpt = xray.Excerpts.FirstOrDefault(e => e.Start == index);
if (excerpt == null)
{
if (skipNoLikes && quote.Likes == 0 ||
quote.Text.Length < minClipLen)
{
continue;
}
excerpt = new Excerpt
{
Id = excerptId++,
Start = location,
Length = node.InnerHtml.Length,
Notable = true,
Highlights = quote.Likes
};
excerpt.RelatedEntities.Add(0); // Mark the excerpt as notable
// TODO: also add other related entities
xray.Excerpts.Add(excerpt);
}
else
{
excerpt.RelatedEntities.Add(0);
}
xray.FoundNotables++;
}
}
}
progress?.Add(1);
}
timer.Stop();
_logger.Log($"Scan time: {timer.Elapsed}");
//output list of terms with no locs
foreach (var t in xray.Terms.Where(t => t.Match && t.Locs.Count == 0))
{
_logger.Log($"No locations were found for the term \"{t.TermName}\".\r\nYou should add aliases for this term using the book or rawml as a reference.");
}
}
