private SyntaxToken QuickScanSyntaxToken() { this.Start(); var state = QuickScanState.Initial; int i = TextWindow.Offset; int n = TextWindow.CharacterWindowCount; n = Math.Min(n, i + MaxCachedTokenSize); int hashCode = Hash.FnvOffsetBias; //localize frequently accessed fields var charWindow = TextWindow.CharacterWindow; var charPropLength = s_charProperties.Length; for (; i < n; i++) { char c = charWindow[i]; int uc = unchecked ((int)c); var flags = uc < charPropLength ? (CharFlags)s_charProperties[uc] : CharFlags.Complex; state = (QuickScanState)s_stateTransitions[(int)state, (int)flags]; // NOTE: that Bad > Done and it is the only state like that // as a result, we will exit the loop on either Bad or Done. // the assert below will validate that these are the only states on which we exit // Also note that we must exit on Done or Bad // since the state machine does not have transitions for these states // and will promptly fail if we do not exit. if (state >= QuickScanState.Done) { goto exitWhile; } hashCode = unchecked ((hashCode ^ uc) * Hash.FnvPrime); } state = QuickScanState.Bad; // ran out of characters in window exitWhile: TextWindow.AdvanceChar(i - TextWindow.Offset); Debug.Assert(state == QuickScanState.Bad || state == QuickScanState.Done, "can only exit with Bad or Done"); if (state == QuickScanState.Done) { // this is a good token! var token = _cache.LookupToken( TextWindow.CharacterWindow, TextWindow.LexemeRelativeStart, i - TextWindow.LexemeRelativeStart, hashCode, _createQuickTokenFunction); return(token); } else { TextWindow.Reset(TextWindow.LexemeStartPosition); return(null); } }
private void InitializeEnergy() { GameObject energy = GetElement("Energy"); if (energy != null) { energyWidget = new TextWindow(energy, "Text", ""); energyWidget.Reset(); } }
private SyntaxToken QuickScanSyntaxToken() { this.Start(); var state = QuickScanState.Initial; int i = TextWindow.Offset; int n = TextWindow.CharacterWindowCount; n = Math.Min(n, i + MaxCachedTokenSize); int hashCode = Hash.FnvOffsetBias; //localize frequently accessed fields var charWindow = TextWindow.CharacterWindow; var charPropLength = charProperties.Length; for (; i < n; i++) { char c = charWindow[i]; int uc = unchecked ((int)c); var flags = uc < charPropLength ? (CharFlags)charProperties[uc] : CharFlags.Complex; state = (QuickScanState)stateTransitions[(int)state, (int)flags]; if (state == QuickScanState.Done || state == QuickScanState.Bad) { goto exitWhile; } hashCode = unchecked ((hashCode ^ uc) * Hash.FnvPrime); } state = QuickScanState.Bad; // ran out of characters in window exitWhile: TextWindow.AdvanceChar(i - TextWindow.Offset); Debug.Assert(state == QuickScanState.Bad || state == QuickScanState.Done); if (state == QuickScanState.Done) { // this is a good token! var token = this._cache.LookupToken( TextWindow.CharacterWindow, TextWindow.LexemeRelativeStart, i - TextWindow.LexemeRelativeStart, hashCode, createQuickTokenFunction); return(token); } else { TextWindow.Reset(TextWindow.LexemeStartPosition); return(null); } }
private SyntaxToken CreateQuickToken() { #if DEBUG var quickWidth = TextWindow.Width; #endif TextWindow.Reset(TextWindow.LexemeStartPosition); var token = this.LexSyntaxToken(); #if DEBUG Debug.Assert(quickWidth == token.FullWidth); #endif return(token); }
private char ScanEscapeSequence(out char surrogateCharacter) { var start = TextWindow.Position; surrogateCharacter = SlidingTextWindow.InvalidCharacter; char ch = TextWindow.NextChar(); Debug.Assert(ch == '\\'); ch = TextWindow.NextChar(); switch (ch) { // escaped characters that translate to themselves case '\'': case '"': case '\\': break; // translate escapes as per C# spec 2.4.4.4 case '0': ch = '\u0000'; break; case 'a': ch = '\u0007'; break; case 'b': ch = '\u0008'; break; case 'f': ch = '\u000c'; break; case 'n': ch = '\u000a'; break; case 'r': ch = '\u000d'; break; case 't': ch = '\u0009'; break; case 'v': ch = '\u000b'; break; case 'x': case 'u': case 'U': TextWindow.Reset(start); SyntaxDiagnosticInfo error; ch = TextWindow.NextUnicodeEscape(surrogateCharacter: out surrogateCharacter, info: out error); AddError(error); break; default: this.AddError(start, TextWindow.Position - start, ErrorCode.ERR_IllegalEscape); break; } return(ch); }
private void ScanMultiLineRawStringLiteral(ref TokenInfo info, int startingQuoteCount) { info.Kind = SyntaxKind.MultiLineRawStringLiteralToken; // The indentation-whitespace computed from the very last line of the raw string literal var indentationWhitespace = PooledStringBuilder.GetInstance(); // The leading whitespace of whatever line we are currently on. var currentLineWhitespace = PooledStringBuilder.GetInstance(); try { // Do the first pass, finding the end of the raw string, and determining the 'indentation whitespace' // that must be complimentary with all content lines of the raw string literal. var afterStartDelimiter = TextWindow.Position; Debug.Assert(SyntaxFacts.IsNewLine(TextWindow.PeekChar())); var contentLineCount = 0; while (ScanMultiLineRawStringLiteralLine(startingQuoteCount, indentationWhitespace.Builder)) { contentLineCount++; } // If the initial scan failed then just bail out without a constant value. if (this.HasErrors) { return; } // The trivial raw string literal is not legal in the language. if (contentLineCount == 0) { this.AddError( position: TextWindow.Position - startingQuoteCount, width: startingQuoteCount, ErrorCode.ERR_RawStringMustContainContent); return; } // Now, do the second pass, building up the literal value. This may produce an error as well if the // indentation whitespace of the lines isn't complimentary. // Reset us to right after the starting delimiter. Note: if we fail to generate a constant value we'll // ensure that we reset back to the original end we scanned to above. var tokenEnd = TextWindow.Position; TextWindow.Reset(afterStartDelimiter); Debug.Assert(SyntaxFacts.IsNewLine(TextWindow.PeekChar())); for (var currentLine = 0; currentLine < contentLineCount; currentLine++) { AddMultiLineRawStringLiteralLineContents( indentationWhitespace.Builder, currentLineWhitespace.Builder, firstContentLine: currentLine == 0); // If processing the line produced errors, then bail out from continued processing. if (this.HasErrors) { break; } } info.StringValue = this.HasErrors ? "" : TextWindow.Intern(_builder); // Make sure that even if we fail to determine the constant content value of the string that // we still consume all the way to original end that we computed. TextWindow.Reset(tokenEnd); } finally { indentationWhitespace.Free(); currentLineWhitespace.Free(); } }
/// <summary> /// This method is essentially the same as ScanIdentifier_SlowPath, /// except that it can handle XML entities. Since ScanIdentifier /// is hot code and since this method does extra work, it seem /// worthwhile to separate it from the common case. /// </summary> /// <param name="info"></param> /// <returns></returns> private bool ScanIdentifier_CrefSlowPath(ref TokenInfo info) { Debug.Assert(InXmlCrefOrNameAttributeValue); int start = TextWindow.Position; this.ResetIdentBuffer(); if (AdvanceIfMatches('@')) { // In xml name _annotation values, the '@' is part of the value text of the identifier // (to match dev11). if (InXmlNameAttributeValue) { AddIdentChar('@'); } else { info.IsVerbatim = true; } } while (true) { int beforeConsumed = TextWindow.Position; char consumedChar; char consumedSurrogate; if (TextWindow.PeekChar() == '&') { if (!TextWindow.TryScanXmlEntity(out consumedChar, out consumedSurrogate)) { // If it's not a valid entity, then it's not part of the identifier. TextWindow.Reset(beforeConsumed); goto LoopExit; } } else { consumedChar = TextWindow.NextChar(); consumedSurrogate = SlidingTextWindow.InvalidCharacter; } // NOTE: If the surrogate is non-zero, then consumedChar won't match // any of the cases below (UTF-16 guarantees that members of surrogate // pairs aren't separately valid). bool isEscaped = false; top: switch (consumedChar) { case '\\': // NOTE: For completeness, we should allow xml entities in unicode escape // sequences (DevDiv #16321). Since it is not currently a priority, we will // try to make the interim behavior sensible: we will only attempt to scan // a unicode escape if NONE of the characters are XML entities (including // the backslash, which we have already consumed). // When we're ready to implement this behavior, we can drop the position // check and use AdvanceIfMatches instead of PeekChar. if (!isEscaped && (TextWindow.Position == beforeConsumed + 1) && (TextWindow.PeekChar() == 'u' || TextWindow.PeekChar() == 'U')) { Debug.Assert(consumedSurrogate == SlidingTextWindow.InvalidCharacter, "Since consumedChar == '\\'"); info.HasIdentifierEscapeSequence = true; TextWindow.Reset(beforeConsumed); // ^^^^^^^ otherwise \u005Cu1234 looks just like \u1234! (i.e. escape within escape) isEscaped = true; SyntaxDiagnosticInfo error; consumedChar = TextWindow.NextUnicodeEscape(out consumedSurrogate, out error); AddCrefError(error); goto top; } goto default; case '_': case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': { // Again, these are the 'common' identifier characters... break; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { if (this._identLen == 0) { TextWindow.Reset(beforeConsumed); goto LoopExit; } // Again, these are the 'common' identifier characters... break; } case ' ': case '$': case '\t': case '.': case ';': case '(': case ')': case ',': case '<': // ...and these are the 'common' stop characters. TextWindow.Reset(beforeConsumed); goto LoopExit; case SlidingTextWindow.InvalidCharacter: if (!TextWindow.IsReallyAtEnd()) { goto default; } TextWindow.Reset(beforeConsumed); goto LoopExit; default: { // This is the 'expensive' call if (this._identLen == 0 && consumedChar > 127 && SyntaxKindFacts.IsIdentifierStartCharacter(consumedChar)) { break; } else if (this._identLen > 0 && consumedChar > 127 && SyntaxKindFacts.IsIdentifierPartCharacter(consumedChar)) { //// BUG 424819 : Handle identifier chars > 0xFFFF via surrogate pairs if (SyntaxKindFacts.IsFormattingChar(consumedChar)) { continue; // Ignore formatting characters } break; } else { // Not a valid identifier character, so bail. TextWindow.Reset(beforeConsumed); goto LoopExit; } } } this.AddIdentChar(consumedChar); if (consumedSurrogate != SlidingTextWindow.InvalidCharacter) { this.AddIdentChar(consumedSurrogate); } } LoopExit: if (this._identLen > 0) { // NOTE: If we don't intern the string value, then we won't get a hit // in the keyword dictionary! (It searches for a key using identity.) // The text does not have to be interned (and probalbly shouldn't be // if it contains entities (else-case). var width = TextWindow.Width; // exact size of input characters // id buffer is identical to width in input if (this._identLen == width) { info.StringValue = TextWindow.GetInternedText(); info.Text = info.StringValue; } else { info.StringValue = TextWindow.Intern(this._identBuffer, 0, this._identLen); info.Text = TextWindow.GetText(intern: false); } return(true); } else { info.Text = null; info.StringValue = null; TextWindow.Reset(start); return(false); } }
private bool ScanIdentifier_SlowPath(ref TokenInfo info) { int start = TextWindow.Position; this.ResetIdentBuffer(); info.IsVerbatim = TextWindow.PeekChar() == '@' && TextWindow.PeekChar(1) == '"'; var isAnnoStart = !info.IsVerbatim && TextWindow.PeekChar() == '@'; if (info.IsVerbatim || isAnnoStart) { TextWindow.AdvanceChar(); } while (true) { char surrogateCharacter = SlidingTextWindow.InvalidCharacter; bool isEscaped = false; char ch = TextWindow.PeekChar(); top: switch (ch) { case '\\': if (!isEscaped && TextWindow.IsUnicodeEscape()) { // ^^^^^^^ otherwise \u005Cu1234 looks just like \u1234! (i.e. escape within escape) info.HasIdentifierEscapeSequence = true; isEscaped = true; ch = TextWindow.PeekUnicodeEscape(out surrogateCharacter); goto top; } goto default; case '$': goto LoopExit; case SlidingTextWindow.InvalidCharacter: if (!TextWindow.IsReallyAtEnd()) { goto default; } goto LoopExit; case '_': case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': { // Again, these are the 'common' identifier characters... break; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { if (this._identLen == 0) { goto LoopExit; } // Again, these are the 'common' identifier characters... break; } case ' ': case '\t': case '.': case ';': case '(': case ')': case ',': // ...and these are the 'common' stop characters. goto LoopExit; case '<': if (this._identLen == 0 && this.ModeIs(LexerMode.DebuggerSyntax) && TextWindow.PeekChar(1) == '>') { // In DebuggerSyntax mode, identifiers are allowed to begin with <>. TextWindow.AdvanceChar(2); this.AddIdentChar('<'); this.AddIdentChar('>'); continue; } goto LoopExit; default: { // This is the 'expensive' call if (this._identLen == 0 && ch > 127 && SyntaxKindFacts.IsIdentifierStartCharacter(ch)) { break; } else if (this._identLen > 0 && ch > 127 && SyntaxKindFacts.IsIdentifierPartCharacter(ch)) { //// BUG 424819 : Handle identifier chars > 0xFFFF via surrogate pairs if (SyntaxKindFacts.IsFormattingChar(ch)) { if (isEscaped) { SyntaxDiagnosticInfo error; TextWindow.NextCharOrUnicodeEscape(out surrogateCharacter, out error); AddError(error); } else { TextWindow.AdvanceChar(); } continue; // Ignore formatting characters } break; } else { // Not a valid identifier character, so bail. goto LoopExit; } } } if (isEscaped) { SyntaxDiagnosticInfo error; TextWindow.NextCharOrUnicodeEscape(out surrogateCharacter, out error); AddError(error); } else { TextWindow.AdvanceChar(); } this.AddIdentChar(ch); if (surrogateCharacter != SlidingTextWindow.InvalidCharacter) { this.AddIdentChar(surrogateCharacter); } } LoopExit: var width = TextWindow.Width; // exact size of input characters if (this._identLen > 0) { info.Text = TextWindow.GetInternedText(); // id buffer is identical to width in input if (this._identLen == width) { info.StringValue = info.Text; } else { info.StringValue = TextWindow.Intern(this._identBuffer, 0, this._identLen); } return(true); } else { info.Text = null; info.StringValue = null; TextWindow.Reset(start); return(false); } }