private ITokenizer MakeTokenizer(bool markup, SeekableTextReader seekableTextReader) { if (markup) { return(MarkupTokenizerFactory(seekableTextReader)); } else { return(CodeTokenizerFactory(seekableTextReader)); } }
public virtual IEnumerable <SyntaxToken> TokenizeString(SourceLocation start, string input) { using (var reader = new SeekableTextReader(input, start.FilePath)) { var tok = CreateTokenizer(reader); SyntaxToken token; while ((token = tok.NextToken()) != null) { yield return(token); } } }
public virtual IEnumerable <TSymbol> TokenizeString(SourceLocation start, string input) { using (var reader = new SeekableTextReader(input, start.FilePath)) { var tok = CreateTokenizer(reader); TSymbol sym; while ((sym = tok.NextSymbol()) != null) { yield return(sym); } } }
internal void TestTokenizer <TSymbol, TSymbolType>(string input, params TSymbol[] expectedSymbols) where TSymbolType : struct where TSymbol : TokenBase <TSymbolType> { // Arrange var success = true; var output = new StringBuilder(); using (var source = new SeekableTextReader(input, filePath: null)) { var tokenizer = (Tokenizer <TSymbol, TSymbolType>)CreateTokenizer(source); var counter = 0; TSymbol current = null; while ((current = tokenizer.NextToken()) != null) { if (counter >= expectedSymbols.Length) { output.AppendLine(string.Format("F: Expected: << Nothing >>; Actual: {0}", current)); success = false; } else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining)) { output.AppendLine(string.Format("P: Ignored {0}", current)); } else { if (!Equals(expectedSymbols[counter], current)) { output.AppendLine(string.Format("F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current)); success = false; } else { output.AppendLine(string.Format("P: Expected: {0}", expectedSymbols[counter])); } counter++; } } if (counter < expectedSymbols.Length && !ReferenceEquals(expectedSymbols[counter], IgnoreRemaining)) { success = false; for (; counter < expectedSymbols.Length; counter++) { output.AppendLine(string.Format("F: Expected: {0}; Actual: << None >>", expectedSymbols[counter])); } } } Assert.True(success, Environment.NewLine + output.ToString()); WriteTraceLine(output.Replace("{", "{{").Replace("}", "}}").ToString()); }
internal void TestTokenizer(string input, params SyntaxToken[] expectedSymbols) { // Arrange var success = true; var output = new StringBuilder(); using (var source = new SeekableTextReader(input, filePath: null)) { var tokenizer = (Tokenizer)CreateTokenizer(source); var counter = 0; SyntaxToken current = null; while ((current = tokenizer.NextToken()) != null) { if (counter >= expectedSymbols.Length) { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "F: Expected: << Nothing >>; Actual: {0}", current)); success = false; } else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining)) { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "P: Ignored |{0}|", current)); } else { if (!expectedSymbols[counter].IsEquivalentTo(current)) { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current)); success = false; } else { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "P: Expected: {0}", expectedSymbols[counter])); } counter++; } } if (counter < expectedSymbols.Length && !ReferenceEquals(expectedSymbols[counter], IgnoreRemaining)) { success = false; for (; counter < expectedSymbols.Length; counter++) { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "F: Expected: {0}; Actual: << None >>", expectedSymbols[counter])); } } } Assert.True(success, Environment.NewLine + output.ToString()); WriteTraceLine(output.Replace("{", "{{").Replace("}", "}}").ToString()); }
public ParserContext(RazorSourceDocument source, RazorParserOptions options) { if (source == null) { throw new ArgumentNullException(nameof(source)); } SourceDocument = source; Source = new SeekableTextReader(SourceDocument); DesignTimeMode = options.DesignTime; FeatureFlags = options.FeatureFlags; ParseLeadingDirectives = options.ParseLeadingDirectives; ErrorSink = new ErrorSink(); SeenDirectives = new HashSet <string>(StringComparer.Ordinal); }
public ParserContext(RazorSourceDocument source, RazorParserOptions options) { if (source == null) { throw new ArgumentNullException(nameof(source)); } SourceDocument = source; var chars = new char[source.Length]; source.CopyTo(0, chars, 0, source.Length); Source = new SeekableTextReader(chars, source.FilePath); DesignTimeMode = options.DesignTime; ParseLeadingDirectives = options.ParseLeadingDirectives; Builder = new SyntaxTreeBuilder(); ErrorSink = new ErrorSink(); SeenDirectives = new HashSet <string>(StringComparer.Ordinal); }