IList <WikiTextToken> IWikiTextTokenizer.TokenizeWikiText(string wikiText, WikiTokenizationSettings settings)
{
Contract.Requires(wikiText != null);
Contract.Requires(settings != null);
Contract.Ensures(Contract.Result <IList <WikiTextToken> >() != null);
Contract.Ensures(Contract.ForAll(Contract.Result <IList <WikiTextToken> >(), x => x != null));
throw new System.NotImplementedException();
}
private void ProcessLine(
DocumentDefBuilder docBuilder,
ParsingContext context,
string lineText)
{
Contract.Requires(docBuilder != null);
Contract.Requires(lineText != null);
WikiTokenizationSettings tokenizationSettings = new WikiTokenizationSettings();
tokenizationSettings.IsWholeLine = true;
IList <WikiTextToken> tokens = tokenizer.TokenizeWikiText(lineText, tokenizationSettings);
if (tokens.Count == 0)
{
throw new NotImplementedException("todo next:");
}
TokenBuffer tokenBuffer = new TokenBuffer(tokens);
WikiTextToken firstToken = tokenBuffer.Token;
switch (firstToken.Type)
{
case WikiTextToken.TokenType.Heading1Start:
case WikiTextToken.TokenType.Heading2Start:
case WikiTextToken.TokenType.Heading3Start:
case WikiTextToken.TokenType.Heading4Start:
case WikiTextToken.TokenType.Heading5Start:
case WikiTextToken.TokenType.Heading6Start:
docBuilder.FinalizeCurrentParagraph();
HandleHeadingLine(docBuilder, context, tokenBuffer);
break;
case WikiTextToken.TokenType.Text:
case WikiTextToken.TokenType.DoubleApostrophe:
case WikiTextToken.TokenType.TripleApostrophe:
case WikiTextToken.TokenType.DoubleSquareBracketsOpen:
case WikiTextToken.TokenType.SingleSquareBracketsOpen:
case WikiTextToken.TokenType.BulletList:
case WikiTextToken.TokenType.NumberedList:
case WikiTextToken.TokenType.Indent:
HandleText(docBuilder, context, tokenBuffer);
break;
default:
throw new NotImplementedException("todo next: {0}".Fmt(firstToken.Type));
}
}
public IList <WikiTextToken> TokenizeWikiText(string wikiText, WikiTokenizationSettings settings)
{
List <WikiTextToken> tokens = new List <WikiTextToken> ();
int index = 0;
int?textTokenStart = null;
int scope = settings.IsWholeLine ? (int)WikiTextTokenScopes.LineStart : (int)WikiTextTokenScopes.InnerText;
while (true)
{
if (index >= wikiText.Length)
{
break;
}
int tokenEndingIndex;
WikiTextTokenDef tokenFound = LookForMatchingToken(
wikiText, ref scope, index, out tokenEndingIndex);
if (tokenFound != null)
{
AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope);
string tokenText = wikiText.Substring(index, tokenEndingIndex - index);
tokens.Add(new WikiTextToken(tokenFound.TokenType, tokenText, (WikiTextTokenScopes)scope));
index = tokenEndingIndex;
}
else
{
AddToTextToken(index, ref textTokenStart);
// be sure to remove any LineStart scopes
if ((scope & (int)WikiTextTokenScopes.LineStart) != 0)
{
scope = (scope | (int)WikiTextTokenScopes.InnerText) & ~(int)WikiTextTokenScopes.LineStart;
}
index++;
}
}
AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope);
return(tokens);
}