public override sealed bool IncrementToken()
{
if (m_tokens.Count > 0)
{
Debug.Assert(current != null);
CompoundToken token = m_tokens.First.Value;
m_tokens.Remove(token);
RestoreState(current); // keep all other attributes untouched
m_termAtt.SetEmpty().Append(token.Text);
m_offsetAtt.SetOffset(token.StartOffset, token.EndOffset);
posIncAtt.PositionIncrement = 0;
return(true);
}
current = null; // not really needed, but for safety
if (m_input.IncrementToken())
{
// Only words longer than minWordSize get processed
if (m_termAtt.Length >= this.m_minWordSize)
{
Decompose();
// only capture the state if we really need it for producing new tokens
if (m_tokens.Count > 0)
{
current = CaptureState();
}
}
// return original token:
return(true);
}
else
{
return(false);
}
}
protected override void Decompose()
{
// get the hyphenation points
Hyphenation.Hyphenation hyphens = hyphenator.Hyphenate(m_termAtt.Buffer, 0, m_termAtt.Length, 1, 1);
// No hyphen points found -> exit
if (hyphens is null)
{
return;
}
int[] hyp = hyphens.HyphenationPoints;
for (int i = 0; i < hyp.Length; ++i)
{
int remaining = hyp.Length - i;
int start = hyp[i];
CompoundToken longestMatchToken = null;
for (int j = 1; j < remaining; j++)
{
int partLength = hyp[i + j] - start;
// if the part is longer than maxSubwordSize we
// are done with this round
if (partLength > this.m_maxSubwordSize)
{
break;
}
// we only put subwords to the token stream
// that are longer than minPartSize
if (partLength < this.m_minSubwordSize)
{
// BOGUS/BROKEN/FUNKY/WACKO: somehow we have negative 'parts' according to the
// calculation above, and we rely upon minSubwordSize being >=0 to filter them out...
continue;
}
// check the dictionary
if (m_dictionary is null || m_dictionary.Contains(m_termAtt.Buffer, start, partLength))
{
if (this.m_onlyLongestMatch)
{
if (longestMatchToken != null)
{
if (longestMatchToken.Text.Length < partLength)
{
longestMatchToken = new CompoundToken(this, start, partLength);
}
}
else
{
longestMatchToken = new CompoundToken(this, start, partLength);
}
}
else
{
m_tokens.Enqueue(new CompoundToken(this, start, partLength));
}
}
public string RenderToHtml(string markdown)
{
if (markdown == null)
{
throw new ArgumentNullException(nameof(markdown));
}
var reader = new TokenReader(new MdLanguage(), markdown);
var token = new CompoundToken(reader.ReadTokens());
var renderer = new HtmlTokenRenderer();
return(renderer.Render(token));
}
protected override void Decompose()
{
int len = m_termAtt.Length;
for (int i = 0; i <= len - this.m_minSubwordSize; ++i)
{
CompoundToken longestMatchToken = null;
for (int j = this.m_minSubwordSize; j <= this.m_maxSubwordSize; ++j)
{
if (i + j > len)
{
break;
}
if (m_dictionary.Contains(m_termAtt.Buffer, i, j))
{
if (this.m_onlyLongestMatch)
{
if (longestMatchToken != null)
{
if (longestMatchToken.Text.Length < j)
{
longestMatchToken = new CompoundToken(this, i, j);
}
}
else
{
longestMatchToken = new CompoundToken(this, i, j);
}
}
else
{
m_tokens.Enqueue(new CompoundToken(this, i, j));
}
}
}
if (this.m_onlyLongestMatch && longestMatchToken != null)
{
m_tokens.Enqueue(longestMatchToken);
}
}
}
protected internal override void Decompose()
{
int len = termAtt.Length;
for (int i = 0; i <= len - this.minSubwordSize; ++i)
{
CompoundToken longestMatchToken = null;
for (int j = this.minSubwordSize; j <= this.maxSubwordSize; ++j)
{
if (i + j > len)
{
break;
}
if (dictionary.Contains(termAtt.Buffer(), i, j))
{
if (this.onlyLongestMatch)
{
if (longestMatchToken != null)
{
if (longestMatchToken.txt.Length < j)
{
longestMatchToken = new CompoundToken(this, i, j);
}
}
else
{
longestMatchToken = new CompoundToken(this, i, j);
}
}
else
{
tokens.AddLast(new CompoundToken(this, i, j));
}
}
}
if (this.onlyLongestMatch && longestMatchToken != null)
{
tokens.AddLast(longestMatchToken);
}
}
}
protected internal override void Decompose()
{
int len = termAtt.Length;
for (int i = 0; i <= len - this.minSubwordSize; ++i)
{
CompoundToken longestMatchToken = null;
for (int j = this.minSubwordSize; j <= this.maxSubwordSize; ++j)
{
if (i + j > len)
{
break;
}
if (dictionary.Contains(termAtt.Buffer(), i, j))
{
if (this.onlyLongestMatch)
{
if (longestMatchToken != null)
{
if (longestMatchToken.txt.Length < j)
{
longestMatchToken = new CompoundToken(this, i, j);
}
}
else
{
longestMatchToken = new CompoundToken(this, i, j);
}
}
else
{
tokens.AddLast(new CompoundToken(this, i, j));
}
}
}
if (this.onlyLongestMatch && longestMatchToken != null)
{
tokens.AddLast(longestMatchToken);
}
}
}
protected internal override void decompose()
{
// get the hyphenation points
Hyphenation hyphens = hyphenator.hyphenate(termAtt.Buffer(), 0, termAtt.Length(), 1, 1);
// No hyphen points found -> exit
if (hyphens == null)
{
return;
}
int[] hyp = hyphens.HyphenationPoints;
for (int i = 0; i < hyp.Length; ++i)
{
int remaining = hyp.Length - i;
int start = hyp[i];
CompoundToken longestMatchToken = null;
for (int j = 1; j < remaining; j++)
{
int partLength = hyp[i + j] - start;
// if the part is longer than maxSubwordSize we
// are done with this round
if (partLength > this.maxSubwordSize)
{
break;
}
// we only put subwords to the token stream
// that are longer than minPartSize
if (partLength < this.minSubwordSize)
{
// BOGUS/BROKEN/FUNKY/WACKO: somehow we have negative 'parts' according to the
// calculation above, and we rely upon minSubwordSize being >=0 to filter them out...
continue;
}
// check the dictionary
if (dictionary == null || dictionary.Contains(termAtt.Buffer(), start, partLength))
{
if (this.onlyLongestMatch)
{
if (longestMatchToken != null)
{
if (longestMatchToken.txt.Length() < partLength)
{
longestMatchToken = new CompoundToken(this, start, partLength);
}
}
else
{
longestMatchToken = new CompoundToken(this, start, partLength);
}
}
else
{
tokens.AddLast(new CompoundToken(this, start, partLength));
}
}
else if (dictionary.contains(termAtt.buffer(), start, partLength - 1))
{
// check the dictionary again with a word that is one character
// shorter
// to avoid problems with genitive 's characters and other binding
// characters
if (this.onlyLongestMatch)
{
if (longestMatchToken != null)
{
if (longestMatchToken.txt.Length() < partLength - 1)
{
longestMatchToken = new CompoundToken(this, start, partLength - 1);
}
}
else
{
longestMatchToken = new CompoundToken(this, start, partLength - 1);
}
}
else
{
tokens.AddLast(new CompoundToken(this, start, partLength - 1));
}
}
}
if (this.onlyLongestMatch && longestMatchToken != null)
{
tokens.AddLast(longestMatchToken);
}
}
}
protected internal override void decompose()
{
// get the hyphenation points
Hyphenation hyphens = hyphenator.hyphenate(termAtt.Buffer(), 0, termAtt.Length(), 1, 1);
// No hyphen points found -> exit
if (hyphens == null)
{
return;
}
int[] hyp = hyphens.HyphenationPoints;
for (int i = 0; i < hyp.Length; ++i)
{
int remaining = hyp.Length - i;
int start = hyp[i];
CompoundToken longestMatchToken = null;
for (int j = 1; j < remaining; j++)
{
int partLength = hyp[i + j] - start;
// if the part is longer than maxSubwordSize we
// are done with this round
if (partLength > this.maxSubwordSize)
{
break;
}
// we only put subwords to the token stream
// that are longer than minPartSize
if (partLength < this.minSubwordSize)
{
// BOGUS/BROKEN/FUNKY/WACKO: somehow we have negative 'parts' according to the
// calculation above, and we rely upon minSubwordSize being >=0 to filter them out...
continue;
}
// check the dictionary
if (dictionary == null || dictionary.Contains(termAtt.Buffer(), start, partLength))
{
if (this.onlyLongestMatch)
{
if (longestMatchToken != null)
{
if (longestMatchToken.txt.Length() < partLength)
{
longestMatchToken = new CompoundToken(this, start, partLength);
}
}
else
{
longestMatchToken = new CompoundToken(this, start, partLength);
}
}
else
{
tokens.AddLast(new CompoundToken(this, start, partLength));
}
}
else if (dictionary.contains(termAtt.buffer(), start, partLength - 1))
{
// check the dictionary again with a word that is one character
// shorter
// to avoid problems with genitive 's characters and other binding
// characters
if (this.onlyLongestMatch)
{
if (longestMatchToken != null)
{
if (longestMatchToken.txt.Length() < partLength - 1)
{
longestMatchToken = new CompoundToken(this, start, partLength - 1);
}
}
else
{
longestMatchToken = new CompoundToken(this, start, partLength - 1);
}
}
else
{
tokens.AddLast(new CompoundToken(this, start, partLength - 1));
}
}
}
if (this.onlyLongestMatch && longestMatchToken != null)
{
tokens.AddLast(longestMatchToken);
}
}
}
private void parseLexemes()
{
var children = new List <Token>();
lexInd = 0;
/*
* <module> ::= { <import> } /\n+/
* <export> [ <implement> ] /\n+/
* { <definition> /\n+/ }
*/
// Get all the imports
while (lexInd < lexemes.Length &&
(string)lexemes[lexInd].Source() == "imports")
{
var import = parseImport();
if (lexInd >= lexemes.Length)
{
throw new UnexpectedEOFException(import.Line());
}
else if (lexemes[lexInd].Type() != TokenType.NewLine)
{
throw new UnexpectedTokenException(
lexemes[lexInd].Type(),
new TokenType[] { TokenType.NewLine },
import.Line()
);
}
var newLine = lexemes[lexInd++];
eatNewLines();
children.Add(import);
children.Add(newLine);
}
// Get the exports and implements line
var export = parseExport();
children.Add(export);
if (lexInd >= lexemes.Length)
{
throw new UnexpectedEOFException(export.Line());
}
else if ((string)lexemes[lexInd].Source() == "implements")
{
children.Add(parseImplement()); // Optional, hence if stmt
}
if (lexInd >= lexemes.Length)
{
throw new UnexpectedEOFException(export.Line());
}
else if (lexemes[lexInd].Type() != TokenType.NewLine)
{
throw new UnexpectedTokenException(
lexemes[lexInd].Type(),
new TokenType[] { TokenType.NewLine },
export.Line()
);
}
children.Add(lexemes[lexInd++]);
eatNewLines();
// Finally, parse definitions
while (lexInd < lexemes.Length)
{
var definition = parseDefinition();
if (lexInd >= lexemes.Length)
{
throw new UnexpectedEOFException(definition.Line());
}
else if (lexemes[lexInd].Type() != TokenType.NewLine)
{
throw new UnexpectedTokenException(
lexemes[lexInd].Type(),
new TokenType[] { TokenType.NewLine },
definition.Line()
);
}
var newLine3 = lexemes[lexInd++];
eatNewLines();
children.Add(definition);
children.Add(newLine3);
}
ast = new CompoundToken(TokenType.Module, children, 1);
}