private StateResult QuotedLiteral(char quote, Func <char, bool> isEndQuotedLiteral, SyntaxKind literalType) { TakeUntil(isEndQuotedLiteral); if (CurrentCharacter == '\\') { TakeCurrent(); // Take the '\' // If the next char is the same quote that started this if (CurrentCharacter == quote || CurrentCharacter == '\\') { TakeCurrent(); // Take it so that we don't prematurely end the literal. } return(Stay()); } else if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter)) { CurrentErrors.Add( RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral( new SourceSpan(CurrentStart, contentLength: 1 /* " */))); } else { TakeCurrent(); // No-op if at EOF } return(Transition(CSharpTokenizerState.Data, EndToken(literalType))); }
private bool ParseModelStatement(CodeBlockInfo block) { var currentLocation = this.CurrentLocation; var acceptedCharacters = this.RequireSingleWhiteSpace() ? AcceptedCharacters.None : AcceptedCharacters.Any; this.End(MetaCodeSpan.Create(this.Context, false, acceptedCharacters)); if (this.modelStatementFound) { this.OnError(currentLocation, string.Format(CultureInfo.CurrentCulture, "Only one @model statement is allowed.")); } this.modelStatementFound = true; this.Context.AcceptWhiteSpace(false); string modelTypeName = null; if (ParserHelpers.IsIdentifierStart(this.CurrentCharacter)) { using (this.Context.StartTemporaryBuffer()) { this.Context.AcceptUntil(c => ParserHelpers.IsNewLine(c)); modelTypeName = this.Context.ContentBuffer.ToString(); this.Context.AcceptTemporaryBuffer(); } this.Context.AcceptNewLine(); } else { this.OnError(currentLocation, string.Format(CultureInfo.CurrentCulture, "@model must be followed by a type name.")); } this.CheckForInheritsAndModelStatements(); this.End(new ModelSpan(this.Context, modelTypeName)); return(false); }
// Internal for testing internal static bool TryCreateIndentationContext(int changePosition, int changeLength, string finalText, VisualStudioDocumentTracker documentTracker, out BraceIndentationContext context) { var focusedTextView = documentTracker.GetFocusedTextView(); if (focusedTextView != null && ParserHelpers.IsNewLine(finalText)) { var currentSnapshot = documentTracker.TextBuffer.CurrentSnapshot; var preChangeLineSnapshot = currentSnapshot.GetLineFromPosition(changePosition); // Handle the case where the \n comes through separately from the \r and the position // on the line is beyond what the GetText call above gives back. var linePosition = Math.Min(preChangeLineSnapshot.Length, changePosition - preChangeLineSnapshot.Start) - 1; if (AfterOpeningBrace(linePosition, preChangeLineSnapshot)) { var afterChangePosition = changePosition + changeLength; var afterChangeLineSnapshot = currentSnapshot.GetLineFromPosition(afterChangePosition); var afterChangeLinePosition = afterChangePosition - afterChangeLineSnapshot.Start; if (BeforeClosingBrace(afterChangeLinePosition, afterChangeLineSnapshot)) { context = new BraceIndentationContext(focusedTextView, changePosition); return(true); } } } context = null; return(false); }
public static SourceLocation GetSourceLocation(this SyntaxNode node, RazorSourceDocument source) { try { if (source.Length == 0) { // Just a marker symbol return(new SourceLocation(source.FilePath, 0, 0, 0)); } if (node.Position == source.Length) { // E.g. Marker symbol at the end of the document var lastPosition = source.Length - 1; var endsWithLineBreak = ParserHelpers.IsNewLine(source[lastPosition]); var lastLocation = source.Lines.GetLocation(lastPosition); return(new SourceLocation( source.FilePath, // GetLocation prefers RelativePath but we want FilePath. lastLocation.AbsoluteIndex + 1, lastLocation.LineIndex + (endsWithLineBreak ? 1 : 0), endsWithLineBreak ? 0 : lastLocation.CharacterIndex + 1)); } var location = source.Lines.GetLocation(node.Position); return(new SourceLocation( source.FilePath, // GetLocation prefers RelativePath but we want FilePath. location.AbsoluteIndex, location.LineIndex, location.CharacterIndex)); } catch (IndexOutOfRangeException) { Debug.Assert(false, "Node position should stay within document length."); return(new SourceLocation(source.FilePath, node.Position, 0, 0)); } }
private StateResult Text() { var prev = '\0'; while (!EndOfFile && !(ParserHelpers.IsWhitespace(CurrentCharacter) || ParserHelpers.IsNewLine(CurrentCharacter)) && !AtToken()) { prev = CurrentCharacter; TakeCurrent(); } if (CurrentCharacter == '@') { var next = Peek(); if ((ParserHelpers.IsLetter(prev) || ParserHelpers.IsDecimalDigit(prev)) && (ParserHelpers.IsLetter(next) || ParserHelpers.IsDecimalDigit(next))) { TakeCurrent(); // Take the "@" return(Stay()); // Stay in the Text state } } // Output the Text token and return to the Data state to tokenize the next character (if there is one) return(Transition(HtmlTokenizerState.Data, EndToken(SyntaxKind.Text))); }
/// <summary> /// Tokenizes a quoted literal. /// </summary> /// <param name="quote">The quote character.</param> /// <returns>The state result.</returns> private StateResult QuotedLiteral(char quote) { TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c)); if (CurrentCharacter == '\\') { TakeCurrent(); if (CurrentCharacter == quote || CurrentCharacter == '\\') { TakeCurrent(); } return(Stay()); } if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter)) { CurrentErrors.Add(new Error("Untermined string literal", CurrentStart)); } else { TakeCurrent(); } return(Stay(EndSymbol(HandlebarsSymbolType.StringLiteral))); }
private StateResult QuotedLiteral(char quote, CSharpSymbolType literalType) { TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c)); if (CurrentCharacter == '\\') { TakeCurrent(); // Take the '\' // If the next char is the same quote that started this if (CurrentCharacter == quote || CurrentCharacter == '\\') { TakeCurrent(); // Take it so that we don't prematurely end the literal. } return(Stay()); } else if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter)) { CurrentErrors.Add( new RazorError( RazorResources.ParseError_Unterminated_String_Literal, CurrentStart, length: 1 /* " */)); } else { TakeCurrent(); // No-op if at EOF } return(Transition(EndSymbol(literalType), Data)); }
protected override PartialParseResult CanAcceptChange(TextChange change) { if (IsEndInsertion(change) && ParserHelpers.IsNewLine(change.NewText) && AutoCompleteString != null) { return(PartialParseResult.Rejected | PartialParseResult.AutoCompleteBlock); } return(PartialParseResult.Rejected); }
/// <summary> /// Tokenizes a block of whitespace. /// </summary> /// <returns>The state result.</returns> private StateResult WhiteSpace() { TakeUntil(c => !ParserHelpers.IsWhiteSpace(c) && !ParserHelpers.IsNewLine(c)); if (HaveContent) { return(Transition(EndSymbol(HandlebarsSymbolType.WhiteSpace), Data)); } return(Transition(Data)); }
private StateResult DateLiteral() { AssertCurrent('#'); TakeCurrent(); TakeUntil(c => c == '#' || ParserHelpers.IsNewLine(c)); if (CurrentCharacter == '#') { TakeCurrent(); } return(Stay(EndSymbol(VBSymbolType.DateLiteral))); }
protected override PartialParseResultInternal CanAcceptChange(SyntaxNode target, SourceChange change) { if (((AutoCompleteAtEndOfSpan && IsAtEndOfSpan(target, change)) || IsAtEndOfFirstLine(target, change)) && change.IsInsert && ParserHelpers.IsNewLine(change.NewText) && AutoCompleteString != null) { return(PartialParseResultInternal.Rejected | PartialParseResultInternal.AutoCompleteBlock); } return(PartialParseResultInternal.Rejected); }
protected override PartialParseResult CanAcceptChange(Span target, TextChange normalizedChange) { if (((AutoCompleteAtEndOfSpan && IsAtEndOfSpan(target, normalizedChange)) || IsAtEndOfFirstLine(target, normalizedChange)) && normalizedChange.IsInsert && ParserHelpers.IsNewLine(normalizedChange.NewText) && AutoCompleteString != null) { return(PartialParseResult.Rejected | PartialParseResult.AutoCompleteBlock); } return(PartialParseResult.Rejected); }
/// <summary> /// Represents the default state of the tokenizer. /// </summary> /// <returns>The state result.</returns> private StateResult Data() { if (EndOfFile) { if (HaveContent) { return(Transition(EndSymbol(HandlebarsSymbolType.Text), Stop)); } return(Stop()); } if (ParserHelpers.IsWhiteSpace(CurrentCharacter) || ParserHelpers.IsNewLine(CurrentCharacter)) { if (HaveContent) { return(Transition(EndSymbol(HandlebarsSymbolType.Text), WhiteSpace)); } return(Transition(WhiteSpace)); } TakeUntil(c => c == '{' || ParserHelpers.IsWhiteSpace(c)); if (ParserHelpers.IsWhiteSpace(CurrentCharacter)) { return(Stay()); } if (HaveContent && CurrentCharacter == '{') { if (Buffer[Buffer.Length - 1] == '\\') { // The { character is being escaped, so move on. TakeCurrent(); return(Stay()); } if (Peek() == '{') { // We're at the start of a tag. return(Transition(EndSymbol(HandlebarsSymbolType.Text), BeginTag)); } } if (Peek() == '{') { // We're at the start of a tag. return(Transition(BeginTag)); } TakeCurrent(); return(Stay()); }
/// <summary> /// Appends the specified content to the buffer. /// </summary> /// <param name="content">The content.</param> public void Append(string content) { for (int i = 0; i < content.Length; i++) { AppendCore(content[i]); if ((content[i] == '\r' && (i + 1 == content.Length || content[i + 1] != '\n')) || (content[i] != '\r' && ParserHelpers.IsNewLine(content[i]))) { PushNewLine(); } } }
private SyntaxToken Newline() { Debug.Assert(ParserHelpers.IsNewLine(CurrentCharacter)); // CSharp Spec §2.3.1 var checkTwoCharNewline = CurrentCharacter == '\r'; TakeCurrent(); if (checkTwoCharNewline && CurrentCharacter == '\n') { TakeCurrent(); } return(EndToken(SyntaxKind.NewLine)); }
private HtmlSymbol Newline() { Debug.Assert(ParserHelpers.IsNewLine(CurrentCharacter)); // CSharp Spec §2.3.1 bool checkTwoCharNewline = CurrentCharacter == '\r'; TakeCurrent(); if (checkTwoCharNewline && CurrentCharacter == '\n') { TakeCurrent(); } return(EndSymbol(HtmlSymbolType.NewLine)); }
/// <summary> /// Updates the character and line indexes based on the changes. /// </summary> /// <param name="read">The read.</param> /// <param name="next">The next.</param> private void UpdateCharacterCore(char read, char next) { _absolute++; if (ParserHelpers.IsNewLine(read) && (read != '\r' || next != '\n')) { _line++; _character = 0; } else { _character++; } }
private void UpdateCharacterCore(char characterRead, char nextCharacter) { _absoluteIndex++; if (ParserHelpers.IsNewLine(characterRead) && (characterRead != '\r' || nextCharacter != '\n')) { _lineIndex++; _characterIndex = 0; } else { _characterIndex++; } }
private void UpdateCharacterCore(char characterRead, char nextCharacter) { _absoluteIndex++; if (Environment.NewLine.Length == 1 && characterRead == Environment.NewLine[0] || ParserHelpers.IsNewLine(characterRead) && (characterRead != '\r' || nextCharacter != '\n')) { _lineIndex++; _characterIndex = 0; } else { _characterIndex++; } }
internal static void UpdateCharacterCore(char characterRead, char nextCharacter, ref int absoluteIndex, ref int lineIndex, ref int characterIndex) { absoluteIndex++; if (Environment.NewLine.Length == 1 && characterRead == Environment.NewLine[0] || ParserHelpers.IsNewLine(characterRead) && (characterRead != '\r' || nextCharacter != '\n')) { lineIndex++; characterIndex = 0; } else { characterIndex++; } }
public void UpdateLocation(char characterRead, char nextCharacter) { _absoluteIndex++; if (ParserHelpers.IsNewLine(characterRead) && (characterRead != '\r' || nextCharacter != '\n')) { _lineIndex++; _characterIndex = 0; } else { _characterIndex++; } UpdateLocation(); }
public void Append(string content) { for (int i = 0; i < content.Length; i++) { AppendCore(content[i]); // \r on it's own: Start a new line, otherwise wait for \n // Other Newline: Start a new line if ( (content[i] == '\r' && (i + 1 == content.Length || content[i + 1] != '\n')) || (content[i] != '\r' && ParserHelpers.IsNewLine(content[i])) ) { PushNewLine(); } } }
private StateResult QuotedLiteral(char quote, CSharpSymbolType literalType) { TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c)); if (CurrentCharacter == '\\') { TakeCurrent(); // Take the '\' TakeCurrent(); // Take the next char as well (multi-char escapes don't matter) return(Stay()); } else if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter)) { CurrentErrors.Add(new RazorError(RazorResources.ParseError_Unterminated_String_Literal, CurrentStart)); } else { TakeCurrent(); // No-op if at EOF } return(Transition(EndSymbol(literalType), Data)); }
public SnapshotLine(string contentWithLineBreak, int start, ITextSnapshot owner) { _contentWithLineBreak = contentWithLineBreak; _content = contentWithLineBreak; if (_content.EndsWith("\r\n", StringComparison.Ordinal)) { _content = _content.Substring(0, _content.Length - 2); } else if (_content.Length > 0 && ParserHelpers.IsNewLine(_content[_content.Length - 1])) { _content = _content.Substring(0, _content.Length - 1); } Start = new SnapshotPoint(owner, start); End = new SnapshotPoint(owner, start + _content.Length); Snapshot = owner; LineNumber = (owner as StringTextSnapshot)._lines.Count; }
// http://dev.w3.org/html5/spec/Overview.html#data-state private StateResult Data() { if (ParserHelpers.IsWhitespace(CurrentCharacter)) { return(Stay(Whitespace())); } else if (ParserHelpers.IsNewLine(CurrentCharacter)) { return(Stay(Newline())); } else if (CurrentCharacter == '@') { TakeCurrent(); if (CurrentCharacter == '*') { return(Transition( EndSymbol(HtmlSymbolType.RazorCommentTransition), AfterRazorCommentTransition )); } else if (CurrentCharacter == '@') { // Could be escaped comment transition return(Transition( EndSymbol(HtmlSymbolType.Transition), () => { TakeCurrent(); return Transition(EndSymbol(HtmlSymbolType.Transition), Data); } )); } return(Stay(EndSymbol(HtmlSymbolType.Transition))); } else if (AtSymbol()) { return(Stay(Symbol())); } else { return(Transition(Text)); } }
public void Append(string content) { var previousIndex = 0; for (var i = 0; i < content.Length; i++) { // \r on it's own: Start a new line, otherwise wait for \n // Other Newline: Start a new line if ((content[i] == '\r' && (i + 1 == content.Length || content[i + 1] != '\n')) || (content[i] != '\r' && ParserHelpers.IsNewLine(content[i]))) { AppendCore(content, previousIndex, i - previousIndex + 1); previousIndex = i + 1; PushNewLine(); } } if (previousIndex < content.Length) { AppendCore(content, previousIndex, content.Length - previousIndex); } }
private bool ParseModelStatement(CodeBlockInfo block) { using (StartBlock(BlockType.Directive)) { block.ResumeSpans(Context); SourceLocation endModelLocation = CurrentLocation; bool readWhitespace = RequireSingleWhiteSpace(); End(MetaCodeSpan.Create(Context, hidden: false, acceptedCharacters: readWhitespace ? AcceptedCharacters.None : AcceptedCharacters.Any)); if (_modelStatementFound) { OnError(endModelLocation, String.Format(CultureInfo.CurrentCulture, MvcResources.MvcRazorCodeParser_OnlyOneModelStatementIsAllowed, ModelTypeKeyword)); } _modelStatementFound = true; // Accept Whitespace up to the new line or non-whitespace character Context.AcceptWhiteSpace(includeNewLines: false); string typeName = null; if (ParserHelpers.IsIdentifierStart(CurrentCharacter)) { using (Context.StartTemporaryBuffer()) { Context.AcceptUntil(c => ParserHelpers.IsNewLine(c)); typeName = Context.ContentBuffer.ToString(); Context.AcceptTemporaryBuffer(); } Context.AcceptNewLine(); } else { OnError(endModelLocation, String.Format(CultureInfo.CurrentCulture, MvcResources.MvcRazorCodeParser_ModelKeywordMustBeFollowedByTypeName, ModelTypeKeyword)); } CheckForInheritsAndModelStatements(); End(new ModelSpan(Context, typeName)); } return(false); }
// http://dev.w3.org/html5/spec/Overview.html#data-state private StateResult Data() { if (ParserHelpers.IsWhitespace(CurrentCharacter)) { return(Stay(Whitespace())); } else if (ParserHelpers.IsNewLine(CurrentCharacter)) { return(Stay(Newline())); } else if (CurrentCharacter == '@') { TakeCurrent(); if (CurrentCharacter == '*') { return(Transition( HtmlTokenizerState.AfterRazorCommentTransition, EndToken(SyntaxKind.RazorCommentTransition))); } else if (CurrentCharacter == '@') { // Could be escaped comment transition return(Transition( HtmlTokenizerState.EscapedRazorCommentTransition, EndToken(SyntaxKind.Transition))); } return(Stay(EndToken(SyntaxKind.Transition))); } else if (AtToken()) { return(Stay(Token())); } else { return(Transition(HtmlTokenizerState.Text)); } }
private StateResult QuotedLiteral() { TakeUntil(c => VBHelpers.IsDoubleQuote(c) || ParserHelpers.IsNewLine(c)); if (VBHelpers.IsDoubleQuote(CurrentCharacter)) { TakeCurrent(); if (VBHelpers.IsDoubleQuote(CurrentCharacter)) { // Escape sequence, remain in the string TakeCurrent(); return(Stay()); } } VBSymbolType type = VBSymbolType.StringLiteral; if (Char.ToLowerInvariant(CurrentCharacter) == 'c') { TakeCurrent(); type = VBSymbolType.CharacterLiteral; } return(Transition(EndSymbol(type), Data)); }
public StringTextSnapshot(string content, int versionNumber) { Content = content; _lines = new List <ITextSnapshotLine>(); var start = 0; var delimiterIndex = 0; while (delimiterIndex != -1) { var delimiterLength = 2; delimiterIndex = Content.IndexOf("\r\n", start, StringComparison.Ordinal); if (delimiterIndex == -1) { delimiterLength = 1; for (var i = start; i < Content.Length; i++) { if (ParserHelpers.IsNewLine(content[i])) { delimiterIndex = i; break; } } } var nextLineStartIndex = delimiterIndex != -1 ? delimiterIndex + delimiterLength : Content.Length; var lineText = Content.Substring(start, nextLineStartIndex - start); _lines.Add(new SnapshotLine(lineText, start, this)); start = nextLineStartIndex; Version = new TextVersion(versionNumber); } }