private ITokenizer MakeTokenizer(bool markup, SeekableTextReader seekableTextReader)
{
if (markup)
{
return(MarkupTokenizerFactory(seekableTextReader));
}
else
{
return(CodeTokenizerFactory(seekableTextReader));
}
}
public virtual IEnumerable <SyntaxToken> TokenizeString(SourceLocation start, string input)
{
using (var reader = new SeekableTextReader(input, start.FilePath))
{
var tok = CreateTokenizer(reader);
SyntaxToken token;
while ((token = tok.NextToken()) != null)
{
yield return(token);
}
}
}
public virtual IEnumerable <TSymbol> TokenizeString(SourceLocation start, string input)
{
using (var reader = new SeekableTextReader(input, start.FilePath))
{
var tok = CreateTokenizer(reader);
TSymbol sym;
while ((sym = tok.NextSymbol()) != null)
{
yield return(sym);
}
}
}
internal void TestTokenizer <TSymbol, TSymbolType>(string input, params TSymbol[] expectedSymbols)
where TSymbolType : struct
where TSymbol : TokenBase <TSymbolType>
{
// Arrange
var success = true;
var output = new StringBuilder();
using (var source = new SeekableTextReader(input, filePath: null))
{
var tokenizer = (Tokenizer <TSymbol, TSymbolType>)CreateTokenizer(source);
var counter = 0;
TSymbol current = null;
while ((current = tokenizer.NextToken()) != null)
{
if (counter >= expectedSymbols.Length)
{
output.AppendLine(string.Format("F: Expected: << Nothing >>; Actual: {0}", current));
success = false;
}
else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
{
output.AppendLine(string.Format("P: Ignored {0}", current));
}
else
{
if (!Equals(expectedSymbols[counter], current))
{
output.AppendLine(string.Format("F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current));
success = false;
}
else
{
output.AppendLine(string.Format("P: Expected: {0}", expectedSymbols[counter]));
}
counter++;
}
}
if (counter < expectedSymbols.Length && !ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
{
success = false;
for (; counter < expectedSymbols.Length; counter++)
{
output.AppendLine(string.Format("F: Expected: {0}; Actual: << None >>", expectedSymbols[counter]));
}
}
}
Assert.True(success, Environment.NewLine + output.ToString());
WriteTraceLine(output.Replace("{", "{{").Replace("}", "}}").ToString());
}
internal void TestTokenizer(string input, params SyntaxToken[] expectedSymbols)
{
// Arrange
var success = true;
var output = new StringBuilder();
using (var source = new SeekableTextReader(input, filePath: null))
{
var tokenizer = (Tokenizer)CreateTokenizer(source);
var counter = 0;
SyntaxToken current = null;
while ((current = tokenizer.NextToken()) != null)
{
if (counter >= expectedSymbols.Length)
{
output.AppendLine(string.Format(CultureInfo.InvariantCulture, "F: Expected: << Nothing >>; Actual: {0}", current));
success = false;
}
else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
{
output.AppendLine(string.Format(CultureInfo.InvariantCulture, "P: Ignored |{0}|", current));
}
else
{
if (!expectedSymbols[counter].IsEquivalentTo(current))
{
output.AppendLine(string.Format(CultureInfo.InvariantCulture, "F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current));
success = false;
}
else
{
output.AppendLine(string.Format(CultureInfo.InvariantCulture, "P: Expected: {0}", expectedSymbols[counter]));
}
counter++;
}
}
if (counter < expectedSymbols.Length && !ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
{
success = false;
for (; counter < expectedSymbols.Length; counter++)
{
output.AppendLine(string.Format(CultureInfo.InvariantCulture, "F: Expected: {0}; Actual: << None >>", expectedSymbols[counter]));
}
}
}
Assert.True(success, Environment.NewLine + output.ToString());
WriteTraceLine(output.Replace("{", "{{").Replace("}", "}}").ToString());
}
public ParserContext(RazorSourceDocument source, RazorParserOptions options)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
SourceDocument = source;
Source = new SeekableTextReader(SourceDocument);
DesignTimeMode = options.DesignTime;
FeatureFlags = options.FeatureFlags;
ParseLeadingDirectives = options.ParseLeadingDirectives;
ErrorSink = new ErrorSink();
SeenDirectives = new HashSet <string>(StringComparer.Ordinal);
}
public ParserContext(RazorSourceDocument source, RazorParserOptions options)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
SourceDocument = source;
var chars = new char[source.Length];
source.CopyTo(0, chars, 0, source.Length);
Source = new SeekableTextReader(chars, source.FilePath);
DesignTimeMode = options.DesignTime;
ParseLeadingDirectives = options.ParseLeadingDirectives;
Builder = new SyntaxTreeBuilder();
ErrorSink = new ErrorSink();
SeenDirectives = new HashSet <string>(StringComparer.Ordinal);
}
