private void ParserTest(string content, Block document, TagProvidersCollection providers = null) { providers = providers ?? TagProvidersCollection.Default; var output = new StringBuilder(); using (var reader = new StringReader(content)) { using (var source = new SeekableTextReader(reader)) { var errors = new ParserErrorSink(); var parser = new HandlebarsParser(); var context = new ParserContext(source, parser, errors, providers); parser.Context = context; parser.ParseDocument(); var results = context.CompleteParse(); var comparer = new EquivalanceComparer(output, 0); Assert.True(comparer.Equals(document, results.Document), output.ToString()); } } }
private static List <Span> ParseDocument( string documentContents, List <RazorError> errors = null, List <LineMapping> lineMappings = null) { errors = errors ?? new List <RazorError>(); var markupParser = new HtmlMarkupParser(); var codeParser = new TestMvcCSharpRazorCodeParser(); var reader = new SeekableTextReader(documentContents); var context = new ParserContext( reader, codeParser, markupParser, markupParser, new ErrorSink()); codeParser.Context = context; markupParser.Context = context; markupParser.ParseDocument(); var results = context.CompleteParse(); errors.AddRange(results.ParserErrors); return(results.Document.Flatten().ToList()); }
/// <summary> /// Creates a document <see cref="Block"/> from the given template. /// </summary> /// <param name="template">The template.</param> /// <returns>The document <see cref="Block"/></returns> private Block CreateDocument(string template) { using (var reader = new StringReader(template)) { using (var source = new SeekableTextReader(reader)) { var errors = new ParserErrorSink(); var parser = new HandlebarsParser(); var context = new ParserContext(source, parser, errors, TagProviders); parser.Context = context; parser.ParseDocument(); var results = context.CompleteParse(); if (results.Success) { return(results.Document); } throw new InvalidOperationException( string.Join("\n", results.Errors.Select(e => string.Format("{0}: {1}", e.Location, e.Message)))); } } }
public override ParsedDocument Parse(bool storeAst, string fileName, System.IO.TextReader content, Projects.Project project = null) { currentDocument = openDocuments.FirstOrDefault(d => d != null && d.FileName == fileName); // We need document and project to be loaded to correctly initialize Razor Host. this.project = project as DotNetProject; if (currentDocument == null && !TryAddDocument(fileName)) { return(new RazorCSharpParsedDocument(fileName, new RazorCSharpPageInfo())); } this.aspProject = project as AspMvcProject; EnsureParserInitializedFor(fileName); var errors = new List <Error> (); using (var source = new SeekableTextReader(content)) { var textChange = CreateTextChange(source); var parseResult = editorParser.CheckForStructureChanges(textChange); if (parseResult == PartialParseResult.Rejected) { parseComplete.WaitOne(); if (!capturedArgs.GeneratorResults.Success) { GetRazorErrors(errors); } } } ParseHtmlDocument(errors); CreateCSharpParsedDocument(); ClearLastChange(); RazorHostKind kind = RazorHostKind.WebPage; if (editorParser.Host is WebCodeRazorHost) { kind = RazorHostKind.WebCode; } else if (editorParser.Host is MonoDevelop.RazorGenerator.RazorHost) { kind = RazorHostKind.Template; } var pageInfo = new RazorCSharpPageInfo() { HtmlRoot = htmlParsedDocument, GeneratorResults = capturedArgs.GeneratorResults, Spans = editorParser.CurrentParseTree.Flatten(), CSharpParsedFile = parsedCodeFile, CSharpCode = csharpCode, Errors = errors, FoldingRegions = GetFoldingRegions(), Comments = comments, Compilation = CreateCompilation(), HostKind = kind, }; return(new RazorCSharpParsedDocument(fileName, pageInfo)); }
protected virtual ParserResults RunParse(string document, Func <ParserBase, Action> parserActionSelector, bool designTimeParser, Func <ParserContext, ParserBase> parserSelector = null) { parserSelector = parserSelector ?? (c => c.ActiveParser); // Create the source ParserResults results = null; using (SeekableTextReader reader = new SeekableTextReader(document)) { try { ParserBase codeParser = CreateCodeParser(); ParserBase markupParser = CreateMarkupParser(); ParserContext context = CreateParserContext(reader, codeParser, markupParser); context.DesignTimeMode = designTimeParser; codeParser.Context = context; markupParser.Context = context; // Run the parser parserActionSelector(parserSelector(context))(); results = context.CompleteParse(); } finally { if (results != null && results.Document != null) { WriteTraceLine(String.Empty); WriteTraceLine("Actual Parse Tree:"); WriteNode(0, results.Document); } } } return(results); }
/// <summary> /// Parses the contents specified by the <paramref name="inputStream"/> and returns the generated code. /// </summary> /// <param name="inputStream">A <see cref="Stream"/> that represents the contents to be parsed.</param> /// <param name="className">The name of the generated class. When <c>null</c>, defaults to /// <see cref="RazorEngineHost.DefaultClassName"/> (<c>Host.DefaultClassName</c>).</param> /// <param name="rootNamespace">The namespace in which the generated class will reside. When <c>null</c>, /// defaults to <see cref="RazorEngineHost.DefaultNamespace"/> (<c>Host.DefaultNamespace</c>).</param> /// <param name="sourceFileName"> /// The file name to use in line pragmas, usually the original Razor file. /// </param> /// <returns>A <see cref="GeneratorResults"/> that represents the results of parsing the content.</returns> /// <remarks> /// This overload calculates the checksum of the contents of <paramref name="inputStream"/> prior to code /// generation. The checksum is used for producing the <c>#pragma checksum</c> line pragma required for /// debugging. /// </remarks> public GeneratorResults GenerateCode( Stream inputStream, string className, string rootNamespace, string sourceFileName) { if (inputStream == null) { throw new ArgumentNullException(nameof(inputStream)); } MemoryStream memoryStream = null; string checksum = null; try { if (!Host.DesignTimeMode) { // We don't need to calculate the checksum in design time. if (!inputStream.CanSeek) { memoryStream = new MemoryStream(); inputStream.CopyTo(memoryStream); // We don't have to dispose the input stream since it is owned externally. inputStream = memoryStream; } inputStream.Position = 0; checksum = ComputeChecksum(inputStream); inputStream.Position = 0; } using (var reader = new StreamReader( inputStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: BufferSize, leaveOpen: true)) { var seekableStream = new SeekableTextReader(reader); return(GenerateCodeCore( seekableStream, className, rootNamespace, sourceFileName, checksum, cancelToken: null)); } } finally { if (memoryStream != null) { memoryStream.Dispose(); } } }
System.Threading.Tasks.Task <ParsedDocument> Parse(RazorCSharpParserContext context, CancellationToken cancellationToken) { EnsureParserInitializedFor(context); var errors = new List <Error> (); using (var source = new SeekableTextReader(context.Content.CreateReader())) { var textChange = CreateTextChange(context, source); var parseResult = context.EditorParser.CheckForStructureChanges(textChange); if (parseResult == PartialParseResult.Rejected) { context.RazorDocument.ParseComplete.WaitOne(); if (!context.CapturedArgs.GeneratorResults.Success) { GetRazorErrors(context, errors); } } } ParseHtmlDocument(context, errors); CreateCSharpParsedDocument(context); context.ClearLastTextChange(); RazorHostKind kind = RazorHostKind.WebPage; if (context.EditorParser.Host is WebCodeRazorHost) { kind = RazorHostKind.WebCode; } else if (context.EditorParser.Host is MonoDevelop.AspNet.Razor.Generator.PreprocessedRazorHost) { kind = RazorHostKind.Template; } var model = context.AnalysisDocument.GetSemanticModelAsync(cancellationToken).Result; var pageInfo = new RazorCSharpPageInfo() { HtmlRoot = context.HtmlParsedDocument, GeneratorResults = context.CapturedArgs.GeneratorResults, Spans = context.EditorParser.CurrentParseTree.Flatten(), CSharpSyntaxTree = context.ParsedSyntaxTree, ParsedDocument = new DefaultParsedDocument("generated.cs") { Ast = model }, AnalysisDocument = context.AnalysisDocument, CSharpCode = context.CSharpCode, Errors = errors, FoldingRegions = GetFoldingRegions(context), Comments = context.Comments, HostKind = kind, }; return(System.Threading.Tasks.Task.FromResult((ParsedDocument) new RazorCSharpParsedDocument(context.FileName, pageInfo))); }
/// <summary> /// Tests the tokenizer. /// </summary> /// <param name="input">The input.</param> /// <param name="expected">The expected symbols.</param> protected void TestTokenizerSymbols(string input, params TSymbolType[] expected) { bool success = true; var output = new StringBuilder(); using (var reader = new StringReader(input)) { using (var source = new SeekableTextReader(reader)) { var tokenizer = CreateTokenizer(source); int counter = 0; TSymbol current = null; while ((current = tokenizer.NextSymbol()) != null) { if (counter > expected.Length) { output.AppendFormat("F: Expected: << Nothing >>; Actual: {0}\n", current.Type); success = false; } else if (Equals(current.Type, IgnoreRemaining.Type)) { output.AppendFormat("P: Ignored {0}\n", current); } else { if (!Equals(expected[counter], current.Type)) { output.AppendFormat("F: Expected: {0}; Actual: {1}\n", expected[counter], current.Type); success = false; } else { output.AppendFormat("P: Expected: {0}\n", current.Type); } counter++; } } if (counter < expected.Length && !Equals(expected[counter], IgnoreRemaining.Type)) { success = false; for (; counter < expected.Length; counter++) { output.AppendFormat("F: Expected: {0}; Actual: << None >>\n", expected[counter]); } } } } Assert.True(success, "\r\n" + output.ToString()); }
private ITokenizer MakeTokenizer(bool markup, SeekableTextReader seekableTextReader) { if (markup) { return(MarkupTokenizerFactory(seekableTextReader)); } else { return(CodeTokenizerFactory(seekableTextReader)); } }
/// <summary> /// Static method for dumping the tokenization of a string content. /// </summary> /// <param name="content">The string content to dump the tokenization</param> public static void DumpTokens(string content) { SeekableTextReader reader = new SeekableTextReader(new StringReader(content)); CSharpRazorTokenizer tokenizer = new CSharpRazorTokenizer(reader); CSharpSymbol csSym = null; while ((csSym = tokenizer.NextSymbol()) != null) { System.Diagnostics.Debug.Write(csSym.Content); } }
public virtual IEnumerable <SyntaxToken> TokenizeString(SourceLocation start, string input) { using (var reader = new SeekableTextReader(input, start.FilePath)) { var tok = CreateTokenizer(reader); SyntaxToken token; while ((token = tok.NextToken()) != null) { yield return(token); } } }
internal static IEnumerable <HtmlSymbol> Tokenize(string content) { using (SeekableTextReader reader = new SeekableTextReader(content)) { var tokenizer = new HtmlTokenizer(reader); HtmlSymbol symbol; while ((symbol = tokenizer.NextSymbol()) != null) { yield return(symbol); } } }
internal static IEnumerable <VBSymbol> Tokenize(string content) { using (SeekableTextReader reader = new SeekableTextReader(content)) { VBTokenizer tok = new VBTokenizer(reader); VBSymbol sym; while ((sym = tok.NextSymbol()) != null) { yield return(sym); } } }
public virtual IEnumerable <TSymbol> TokenizeString(SourceLocation start, string input) { using (SeekableTextReader reader = new SeekableTextReader(input)) { var tok = CreateTokenizer(reader); TSymbol sym; while ((sym = tok.NextSymbol()) != null) { sym.OffsetStart(start); yield return(sym); } } }
protected void TestTokenizer(string input, params TSymbol[] expectedSymbols) { // Arrange var success = true; var output = new StringBuilder(); using (StringReader reader = new StringReader(input)) { using (SeekableTextReader source = new SeekableTextReader(reader)) { Tokenizer <TSymbol, TSymbolType> tokenizer = CreateTokenizer(source); var counter = 0; TSymbol current = null; while ((current = tokenizer.NextSymbol()) != null) { if (counter >= expectedSymbols.Length) { output.AppendLine(string.Format("F: Expected: << Nothing >>; Actual: {0}", current)); success = false; } else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining)) { output.AppendLine(string.Format("P: Ignored {0}", current)); } else { if (!Equals(expectedSymbols[counter], current)) { output.AppendLine(string.Format("F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current)); success = false; } else { output.AppendLine(string.Format("P: Expected: {0}", expectedSymbols[counter])); } counter++; } } if (counter < expectedSymbols.Length && !ReferenceEquals(expectedSymbols[counter], IgnoreRemaining)) { success = false; for (; counter < expectedSymbols.Length; counter++) { output.AppendLine(string.Format("F: Expected: {0}; Actual: << None >>", expectedSymbols[counter])); } } } } Assert.True(success, Environment.NewLine + output.ToString()); WriteTraceLine(output.Replace("{", "{{").Replace("}", "}}").ToString()); }
TextChange CreateTextChange(SeekableTextReader source) { if (lastChange == null) { return(new TextChange(0, 0, new SeekableTextReader(String.Empty), 0, source.Length, source)); } if (lastChange.DeleteChange) { return(new TextChange(lastChange.StartOffset, lastChange.AbsoluteLength, lastChange.Buffer, lastChange.StartOffset, 0, source)); } return(new TextChange(lastChange.StartOffset, 0, lastChange.Buffer, lastChange.StartOffset, lastChange.AbsoluteLength, source)); }
internal void TestTokenizer(string input, params SyntaxToken[] expectedSymbols) { // Arrange var success = true; var output = new StringBuilder(); using (var source = new SeekableTextReader(input, filePath: null)) { var tokenizer = (Tokenizer)CreateTokenizer(source); var counter = 0; SyntaxToken current = null; while ((current = tokenizer.NextToken()) != null) { if (counter >= expectedSymbols.Length) { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "F: Expected: << Nothing >>; Actual: {0}", current)); success = false; } else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining)) { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "P: Ignored |{0}|", current)); } else { if (!expectedSymbols[counter].IsEquivalentTo(current)) { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current)); success = false; } else { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "P: Expected: {0}", expectedSymbols[counter])); } counter++; } } if (counter < expectedSymbols.Length && !ReferenceEquals(expectedSymbols[counter], IgnoreRemaining)) { success = false; for (; counter < expectedSymbols.Length; counter++) { output.AppendLine(string.Format(CultureInfo.InvariantCulture, "F: Expected: {0}; Actual: << None >>", expectedSymbols[counter])); } } } Assert.True(success, Environment.NewLine + output.ToString()); WriteTraceLine(output.Replace("{", "{{").Replace("}", "}}").ToString()); }
static TextChange CreateTextChange(RazorCSharpParserContext context, SeekableTextReader source) { ChangeInfo lastChange = context.GetLastTextChange(); if (lastChange == null) { return(new TextChange(0, 0, new SeekableTextReader(String.Empty), 0, source.Length, source)); } if (lastChange.DeleteChange) { return(new TextChange(lastChange.StartOffset, lastChange.AbsoluteLength, lastChange.Buffer, lastChange.StartOffset, 0, source)); } return(new TextChange(lastChange.StartOffset, 0, lastChange.Buffer, lastChange.StartOffset, lastChange.AbsoluteLength, source)); }
public void ConstructorInitializesProperties() { // Arrange var expectedBuffer = new SeekableTextReader(TextReader.Null); var expectedCodeParser = new CSharpCodeParser(); var expectedMarkupParser = new HtmlMarkupParser(); // Act var context = new ParserContext(expectedBuffer, expectedCodeParser, expectedMarkupParser, expectedCodeParser); // Assert Assert.NotNull(context.Source); Assert.Same(expectedCodeParser, context.CodeParser); Assert.Same(expectedMarkupParser, context.MarkupParser); Assert.Same(expectedCodeParser, context.ActiveParser); }
public ParserContext(RazorSourceDocument source, RazorParserOptions options) { if (source == null) { throw new ArgumentNullException(nameof(source)); } SourceDocument = source; Source = new SeekableTextReader(SourceDocument); DesignTimeMode = options.DesignTime; FeatureFlags = options.FeatureFlags; ParseLeadingDirectives = options.ParseLeadingDirectives; ErrorSink = new ErrorSink(); SeenDirectives = new HashSet <string>(StringComparer.Ordinal); }
// Creates spans when the parsed document isn't available yet. void CreateSpans() { using (SeekableTextReader source = new SeekableTextReader(doc.Text)) { var markupParser = new HtmlMarkupParser(); var codeParser = new MvcCSharpRazorCodeParser(); var context = new ParserContext(source, codeParser, markupParser, markupParser) { DesignTimeMode = true }; codeParser.Context = context; markupParser.Context = context; context.ActiveParser.ParseDocument(); var results = context.CompleteParse(); currentSpans = results.Document.Flatten().ToList(); } }
public void ConstructorInitializesProperties() { // Arrange var expectedBuffer = new SeekableTextReader(TextReader.Null); var expectedCodeParser = new CSharpCodeParser(); var expectedMarkupParser = new HtmlMarkupParser(); // Act var context = new ParserContext(expectedBuffer, expectedCodeParser, expectedMarkupParser, expectedCodeParser, new ErrorSink()); // Assert Assert.NotNull(context.Source); Assert.Same(expectedCodeParser, context.CodeParser); Assert.Same(expectedMarkupParser, context.MarkupParser); Assert.Same(expectedCodeParser, context.ActiveParser); }
/// <summary> /// Parses the contents specified by the <paramref name="inputStream"/> and returns the generated code. /// </summary> /// <param name="inputStream">A <see cref="Stream"/> that represents the contents to be parsed.</param> /// <param name="className">The name of the generated class. When <c>null</c>, defaults to /// <see cref="RazorEngineHost.DefaultClassName"/> (<c>Host.DefaultClassName</c>).</param> /// <param name="rootNamespace">The namespace in which the generated class will reside. When <c>null</c>, /// defaults to <see cref="RazorEngineHost.DefaultNamespace"/> (<c>Host.DefaultNamespace</c>).</param> /// <param name="sourceFileName"> /// The file name to use in line pragmas, usually the original Razor file. /// </param> /// <returns>A <see cref="GeneratorResults"/> that represents the results of parsing the content.</returns> /// <remarks> /// This overload calculates the checksum of the contents of <paramref name="inputStream"/> prior to code /// generation. The checksum is used for producing the <c>#pragma checksum</c> line pragma required for /// debugging. /// </remarks> public GeneratorResults GenerateCode( Stream inputStream, string className, string rootNamespace, string sourceFileName) { if (inputStream == null) { throw new ArgumentNullException(nameof(inputStream)); } MemoryStream memoryStream = null; string checksum = null; try { if (!Host.DesignTimeMode) { // We don't need to calculate the checksum in design time. if (!inputStream.CanSeek) { memoryStream = new MemoryStream(); inputStream.CopyTo(memoryStream); // We don't have to dispose the input stream since it is owned externally. inputStream = memoryStream; } inputStream.Position = 0; checksum = ComputeChecksum(inputStream); inputStream.Position = 0; } using (var reader = new StreamReader( inputStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: BufferSize, leaveOpen: true)) { var seekableStream = new SeekableTextReader(reader); return GenerateCodeCore( seekableStream, className, rootNamespace, sourceFileName, checksum, cancelToken: null); } } finally { if (memoryStream != null) { memoryStream.Dispose(); } } }
protected virtual ParserResults RunParse(string document, Func<ParserBase, Action> parserActionSelector, bool designTimeParser, Func<ParserContext, ParserBase> parserSelector = null, ErrorSink errorSink = null) { parserSelector = parserSelector ?? (c => c.ActiveParser); errorSink = errorSink ?? new ErrorSink(); // Create the source ParserResults results = null; using (var reader = new SeekableTextReader(document)) { try { var codeParser = CreateCodeParser(); var markupParser = CreateMarkupParser(); var context = CreateParserContext(reader, codeParser, markupParser, errorSink); context.DesignTimeMode = designTimeParser; codeParser.Context = context; markupParser.Context = context; // Run the parser parserActionSelector(parserSelector(context))(); results = context.CompleteParse(); } finally { if (results != null && results.Document != null) { WriteTraceLine(string.Empty); WriteTraceLine("Actual Parse Tree:"); WriteNode(0, results.Document); } } } return results; }
private static List<Span> ParseDocument( string documentContents, List<RazorError> errors = null, List<LineMapping> lineMappings = null) { errors = errors ?? new List<RazorError>(); var markupParser = new HtmlMarkupParser(); var codeParser = new TestMvcCSharpRazorCodeParser(); var reader = new SeekableTextReader(documentContents); var context = new ParserContext( reader, codeParser, markupParser, markupParser, new ErrorSink()); codeParser.Context = context; markupParser.Context = context; markupParser.ParseDocument(); var results = context.CompleteParse(); errors.AddRange(results.ParserErrors); return results.Document.Flatten().ToList(); }
public ChangeInfo(int off, SeekableTextReader buffer) { offset = off; Length = 0; Buffer = buffer; }
// Token: 0x06000024 RID: 36 RVA: 0x00002B4E File Offset: 0x00000D4E internal void Reset(SeekableTextReader reader, FASTClassificationItem.AbortCallback abortCallback) { this.charactersProcessed = 0L; this.reader = reader; this.abortCallback = abortCallback; }
private ITokenizer MakeTokenizer(bool markup, SeekableTextReader seekableTextReader) { if (markup) { return MarkupTokenizerFactory(seekableTextReader); } else { return CodeTokenizerFactory(seekableTextReader); } }