private List <TextToken> GetSearchAfterContext(BookTokenIterator bookTokenIterator, int endTokenId, int count = 8)
{
var result = new List <TextToken>();
var tokenId = endTokenId;
while (++tokenId < bookTokenIterator.Count && result.Count < count)
{
bookTokenIterator.MoveTo(tokenId);
bookTokenIterator.MoveNext();
if (bookTokenIterator.Current is NewPageToken)
{
break;
}
var textToken = bookTokenIterator.Current as TextToken;
if (textToken == null)
{
continue;
}
result.Add(textToken);
}
bookTokenIterator.MoveTo(endTokenId);
bookTokenIterator.MoveNext();
return(result);
}
private List <TextToken> GetSearchBeforeContext(BookTokenIterator bookTokenIterator, int startTokenId, int count = 8)
{
var result = new List <TextToken>();
var tokenId = startTokenId;
while (--tokenId >= 0 && result.Count < count)
{
bookTokenIterator.MoveTo(tokenId);
bookTokenIterator.MoveNext();
if (bookTokenIterator.Current is NewPageToken)
{
break;
}
var textToken = bookTokenIterator.Current as TextToken;
if (textToken == null)
{
continue;
}
result.Insert(0, textToken);
}
bookTokenIterator.MoveTo(startTokenId);
bookTokenIterator.MoveNext();
return(result);
}
public string GetText(BookModel book, int tokenOffset, int wordsCount, out int lastTokenId)
{
lastTokenId = -1;
var result = new List <string>();
using (var tokenIterator = new BookTokenIterator(book.GetTokensPath(), TokensTool.GetTokens(book.BookID)))
{
int words = 0;
tokenIterator.MoveTo(tokenOffset);
while (tokenIterator.MoveNext() && words < wordsCount)
{
if (tokenIterator.Current is NewPageToken && result.Count > 0)
{
break;
}
var textToken = tokenIterator.Current as TextToken;
if (textToken == null)
{
continue;
}
lastTokenId = textToken.ID;
result.Add(textToken.Text);
words++;
}
}
return(string.Join(" ", result));
}
public void Init()
{
if (_bookTokenIterator != null && _book != null)
{
int tokenId = _bookTokenIterator.Current.ID;
_bookTokenIterator.Dispose();
_bookTokenIterator = new BookTokenIterator(_book.GetTokensPath(), TokensTool.GetTokens(_book.BookID));
_bookTokenIterator.MoveTo(tokenId);
_bookTokenIterator.MoveNext();
}
}
public void Init()
{
if (_bookTokenIterator != null && _book != null)
{
int tokenId = _bookTokenIterator.Current.ID;
_bookTokenIterator.Dispose();
_bookTokenIterator = new BookTokenIterator(_book.GetTokensPath(), TokensTool.GetTokens(_book.BookID));
_bookTokenIterator.MoveTo(tokenId);
_bookTokenIterator.MoveNext();
}
}
public void Init()
{
if (_bookTokenIterator == null || _book == null) return;
var tokenId = 0;
try
{
tokenId = _bookTokenIterator.Current.ID;
}
catch (Exception)
{
// ignored
}
_bookTokenIterator.Dispose();
_bookTokenIterator = new BookTokenIterator(_book.GetTokensPath(), TokensTool.GetTokens(_book.BookID));
_bookTokenIterator.MoveTo(tokenId);
_bookTokenIterator.MoveNext();
}
public string GetLastParagraphByToken(BookModel book, int tokenOffset, out string pointer)
{
var result = new List<string>();
using (var tokenIterator = new BookTokenIterator(book.GetTokensPath(), TokensTool.GetTokens(book.BookID)))
{
var token = FindNewToken(tokenOffset, tokenIterator);
pointer = token.Pointer;
var tokenId = token.ID;
tokenIterator.MoveTo(tokenId);
while (tokenIterator.MoveNext())
{
if (tokenIterator.Current is TagOpenToken && result.Count > 0)
break;
var textToken = tokenIterator.Current as TextToken;
if (textToken == null)
continue;
result.Add(textToken.Text);
}
}
return string.Join(" ", result);
}
public string GetText(BookModel book, int tokenOffset, int wordsCount, out int lastTokenId)
{
lastTokenId = -1;
var result = new List<string>();
using (var tokenIterator = new BookTokenIterator(book.GetTokensPath(), TokensTool.GetTokens(book.BookID)))
{
int words = 0;
tokenIterator.MoveTo(tokenOffset);
while (tokenIterator.MoveNext() && words < wordsCount)
{
if(tokenIterator.Current is NewPageToken && result.Count > 0)
break;
var textToken = tokenIterator.Current as TextToken;
if(textToken == null)
continue;
lastTokenId = textToken.ID;
result.Add(textToken.Text);
words++;
}
}
return string.Join(" ", result);
}
private List<TextToken> GetSearchAfterContext(BookTokenIterator bookTokenIterator, int endTokenId, int count = 8)
{
var result = new List<TextToken>();
var tokenId = endTokenId;
while (++tokenId < bookTokenIterator.Count && result.Count < count)
{
bookTokenIterator.MoveTo(tokenId);
bookTokenIterator.MoveNext();
if (bookTokenIterator.Current is NewPageToken)
break;
var textToken = bookTokenIterator.Current as TextToken;
if (textToken == null)
continue;
result.Add(textToken);
}
bookTokenIterator.MoveTo(endTokenId);
bookTokenIterator.MoveNext();
return result;
}
private List<TextToken> GetSearchBeforeContext(BookTokenIterator bookTokenIterator, int startTokenId, int count = 8)
{
var result = new List<TextToken>();
var tokenId = startTokenId;
while (--tokenId >= 0 && result.Count < count)
{
bookTokenIterator.MoveTo(tokenId);
bookTokenIterator.MoveNext();
if (bookTokenIterator.Current is NewPageToken)
break;
var textToken = bookTokenIterator.Current as TextToken;
if (textToken == null)
continue;
result.Insert(0, textToken);
}
bookTokenIterator.MoveTo(startTokenId);
bookTokenIterator.MoveNext();
return result;
}
private static TagOpenToken FindNewToken(int tokenOffset, BookTokenIterator tokenIterator)
{
List<TagOpenToken> tokens = new List<TagOpenToken>();
var offset = 40;
var searchToken = tokenOffset - offset;
var idx = searchToken;
if (searchToken < 0) searchToken = 0;
tokenIterator.MoveTo(searchToken);
while (tokenIterator.MoveNext() && idx < tokenOffset)
{
if (!(tokenIterator.Current is TagOpenToken)) { idx++; continue;}
var tagToken = tokenIterator.Current as TagOpenToken;
if (tagToken.Name.Contains("p"))
{
tokens.Add(tagToken);
}
idx++;
}
var result = tokens.Count > 0 ? tokens.Last() : FindNewToken(searchToken, tokenIterator);
return result;
}
