/* Function: BuildFullType * Creates a new <Tokenizer> for the variable type, including all modifiers, even if they are not continuous. This is a support * function for <BuildFullType(TokenIterator, TokenIterator, Tokenizer)> and it always builds a new <Tokenizer>. */ protected Tokenizer BuildFullType() { TokenIterator iterator = start; TypeBuilder typeBuilder = new TypeBuilder(end.RawTextIndex - start.RawTextIndex, end.TokenIndex - start.TokenIndex); while (iterator < end) { if (iterator.PrototypeParsingType == PrototypeParsingType.Type || iterator.PrototypeParsingType == PrototypeParsingType.TypeModifier || iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier || iterator.PrototypeParsingType == PrototypeParsingType.ParamModifier || iterator.PrototypeParsingType == PrototypeParsingType.StartOfTuple || iterator.PrototypeParsingType == PrototypeParsingType.EndOfTuple || iterator.PrototypeParsingType == PrototypeParsingType.TupleMemberSeparator || iterator.PrototypeParsingType == PrototypeParsingType.TupleMemberName) { typeBuilder.AddToken(iterator); } else if (iterator.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier || iterator.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier) { TokenIterator closingModifier; GetClosingModifier(iterator, out closingModifier); typeBuilder.AddModifierBlock(iterator, closingModifier); iterator = closingModifier; } iterator.Next(); } return(typeBuilder.ToTokenizer()); }
// Group: Protected Functions // __________________________________________________________________________ /* Function: IsStandaloneWord * Returns whether the iterator is on a text token and the tokens immediately before and after it are not underscores. */ protected bool IsStandaloneWord(TokenIterator iterator) { if (iterator.FundamentalType != FundamentalType.Text) { return(false); } TokenIterator lookahead = iterator; lookahead.Next(); if (lookahead.FundamentalType == FundamentalType.Text || lookahead.Character == '_') { return(false); } TokenIterator lookbehind = iterator; lookbehind.Previous(); if (lookbehind.FundamentalType == FundamentalType.Text || lookbehind.Character == '_') { return(false); } return(true); }
// Group: Parsing Functions // __________________________________________________________________________ /* Function: GenericSkip * * Moves the iterator ahead one code element, which could be a single token, whitespace, an entire comment, or an entire * string. The important part is that this skips comments and strings all in one step so that anything appearing inside them * will not be misinterpreted as code. * * It is virtual so you can extend it to handle things like regular expressions. The default implementation handles strings * and comments based on LineCommentStrings, BlockCommentStringPairs, and <QuoteCharacters>. */ protected virtual void GenericSkip(ref TokenIterator iterator) { if (!TryToSkipComment(ref iterator) && !TryToSkipString(ref iterator)) { iterator.Next(); } }
/* Function: TryToSkipOpeningBlockCommentSymbol * * If the iterator is on an opening block comment symbol, moves it past it and returns true and the closing symbol. Otherwise * leaves the iterator alone and returns false and null. This handles --[[ comments as well as forms with equals signs such as * --[==[. */ protected bool TryToSkipOpeningBlockCommentSymbol(ref TokenIterator iterator, out string closingSymbol) { if (iterator.MatchesAcrossTokens("--[") == false) { closingSymbol = null; return(false); } TokenIterator lookahead = iterator; lookahead.Next(3); int equals = 0; while (lookahead.Character == '=') { lookahead.Next(); equals++; } if (lookahead.Character != '[') { closingSymbol = null; return(false); } lookahead.Next(); iterator = lookahead; if (equals == 0) { closingSymbol = "]]"; } else { StringBuilder closingSymbolBuilder = new StringBuilder(2 + equals); closingSymbolBuilder.Append(']'); closingSymbolBuilder.Append('=', equals); closingSymbolBuilder.Append(']'); closingSymbol = closingSymbolBuilder.ToString(); } return(true); }
public void TestBadTwoNext() { List <Token> tokens = new List <Token>(); tokens.Add(new Token(TokenType.AtIdentifier, "asd", 0, 0)); TokenIterator tokenIterator = new TokenIterator(tokens); try { tokenIterator.Next(); } catch (ArgumentOutOfRangeException) { // First one is valid, it shouldn't fail here. Assert.Fail(); } tokenIterator.Next(); }
/* Function: TryToSkipLocaleSubstitutionIdentifier * If the iterator is on a valid locale substitution identifier, advances it past it, returns it, and returns true. * Otherwise the iterator will be left alone and it will return false. */ protected bool TryToSkipLocaleSubstitutionIdentifier(ref TokenIterator iterator, out string identifier, out string localeIdentifier) { if (iterator.Character != '$' && iterator.Character != '@') { identifier = null; localeIdentifier = null; return(false); } TokenIterator lookahead = iterator; lookahead.Next(); if (lookahead.MatchesAcrossTokens("Locale{") == false) { identifier = null; localeIdentifier = null; return(false); } lookahead.NextByCharacters(7); TokenIterator startOfLocaleIdentifier = lookahead; while (lookahead.IsInBounds && lookahead.Character != '}') { lookahead.Next(); } if (lookahead.Character != '}') { identifier = null; localeIdentifier = null; return(false); } localeIdentifier = startOfLocaleIdentifier.TextBetween(lookahead); lookahead.Next(); identifier = iterator.TextBetween(lookahead); iterator = lookahead; return(true); }
/* Function: TryToSkipSubstitutionDefinition * * If the iterator is on a valid substitution definition, advances it past it, determines its properties, and returns true. Otherwise the * iterator will be left alone and it will return false. * * identifier - "$identifier" in "$identifier = value;" * value - "value" in "$identifier = value;" * declaration - "$identifier = value;" in "$identifier = value;" */ protected bool TryToSkipSubstitutionDefinition(ref TokenIterator iterator, out string identifier, out string value, out string declaration) { identifier = null; value = null; declaration = null; TokenIterator lookahead = iterator; if (TryToSkipSubstitutionIdentifier(ref lookahead, out identifier) == false) { return(false); } lookahead.NextPastWhitespace(); if (lookahead.Character != ':' && lookahead.Character != '=') { identifier = null; return(false); } lookahead.Next(); lookahead.NextPastWhitespace(); TokenIterator startOfValue = lookahead; while (lookahead.IsInBounds && lookahead.Character != ';' && lookahead.FundamentalType != FundamentalType.LineBreak) { GenericSkip(ref lookahead); } value = startOfValue.TextBetween(lookahead); if (lookahead.Character == ';') { lookahead.Next(); } declaration = iterator.TextBetween(lookahead); iterator = lookahead; return(true); }
/* Function: TryToSkipClassParent * * Tries to move the iterator past a single class parent declaration. * * Supported Modes: * * - <ParseMode.IterateOnly> * - <ParseMode.ParseClassPrototype> * - Everything else is treated as <ParseMode.IterateOnly>. */ protected bool TryToSkipClassParent(ref TokenIterator iterator, ParseMode mode = ParseMode.IterateOnly) { TokenIterator lookahead = iterator; if (lookahead.MatchesToken("metaclass")) { lookahead.Next(); TryToSkipWhitespace(ref lookahead); if (lookahead.Character == '=') { if (mode == ParseMode.ParseClassPrototype) { iterator.ClassPrototypeParsingType = ClassPrototypeParsingType.Modifier; } lookahead.Next(); TryToSkipWhitespace(ref lookahead); } else { // Nevermind, reset lookahead = iterator; } } TokenIterator startOfIdentifier = lookahead; if (TryToSkipIdentifier(ref lookahead) == false) { ResetTokensBetween(iterator, lookahead, mode); return(false); } if (mode == ParseMode.ParseClassPrototype) { lookahead.Tokenizer.SetClassPrototypeParsingTypeBetween(startOfIdentifier, lookahead, ClassPrototypeParsingType.Name); } iterator = lookahead; return(true); }
void AppendLinkCandidates(TokenIterator start, TokenIterator end, StringBuilder output) { TokenIterator iterator = start; TokenIterator linkableTypeStart = start; int linkableTypes = 0; for (;;) { while (iterator < end && iterator.PrototypeParsingType != PrototypeParsingType.TypeQualifier && iterator.PrototypeParsingType != PrototypeParsingType.Type) { iterator.Next(); } if (iterator >= end) { break; } linkableTypeStart = iterator; while (iterator < end && (iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier || iterator.PrototypeParsingType == PrototypeParsingType.Type)) { iterator.Next(); } if (linkableTypes > 0) { output.Append(", "); } linkableTypeStart.AppendTextBetweenTo(iterator, output); linkableTypes++; } if (linkableTypes == 0) { output.Append("(none)"); } }
/* Function: GetCompletePrototype * Returns the bounds of the complete prototype, minus whitespace. This does NOT include pre-prototype lines. */ public void GetCompletePrototype(out TokenIterator start, out TokenIterator end) { Section beforeParameters = FindSection(SectionType.BeforeParameters); start = tokenizer.FirstToken; start.Next(beforeParameters.StartIndex); Section afterParameters = FindSection(SectionType.AfterParameters); end = start; if (afterParameters == null) { end.Next(beforeParameters.EndIndex - beforeParameters.StartIndex); } else { end.Next(afterParameters.EndIndex - beforeParameters.StartIndex); } }
public void TestHasNext() { List <Token> tokens = new List <Token>(); tokens.Add(new Token(TokenType.DollarIdentifier, "a", 5, 2)); tokens.Add(new Token(TokenType.Word, "a", 5, 3)); tokens.Add(new Token(TokenType.AtIdentifier, "@b", 12, 0)); tokens.Add(new Token(TokenType.Word, "c", 14, 42)); TokenIterator tokenIterator = new TokenIterator(tokens); Assert.IsTrue(tokenIterator.HasNextType(TokenType.DollarIdentifier)); Assert.IsTrue(tokenIterator.HasNextType(TokenType.DollarIdentifier, TokenType.Pipe)); Assert.IsTrue(tokenIterator.HasNextType(TokenType.Period, TokenType.DollarIdentifier, TokenType.Pipe)); Assert.IsFalse(tokenIterator.HasNextType(TokenType.Number)); tokenIterator.Next(); tokenIterator.Next(); tokenIterator.Next(); tokenIterator.Next(); Assert.IsFalse(tokenIterator.HasNextType(TokenType.Word)); }
/* Function: ExtractTypeLinks * Goes through the prototype of the passed <Topic> and adds any type links it finds to <LinkSet>. */ protected void ExtractTypeLinks(Topic topic, LinkSet linkSet) { if (topic.Prototype == null) { return; } Language language = EngineInstance.Languages.FromID(topic.LanguageID); TokenIterator symbolStart = topic.ParsedPrototype.Tokenizer.FirstToken; TokenIterator symbolEnd; while (symbolStart.IsInBounds) { if (symbolStart.PrototypeParsingType == PrototypeParsingType.Type || symbolStart.PrototypeParsingType == PrototypeParsingType.TypeQualifier) { symbolEnd = symbolStart; do { symbolEnd.Next(); }while (symbolEnd.PrototypeParsingType == PrototypeParsingType.Type || symbolEnd.PrototypeParsingType == PrototypeParsingType.TypeQualifier); if (language.IsBuiltInType(symbolStart, symbolEnd) == false) { Link link = new Link(); // ignore LinkID link.Type = LinkType.Type; link.Symbol = SymbolString.FromPlainText_NoParameters(symbolStart.Tokenizer.TextBetween(symbolStart, symbolEnd)); link.Context = topic.PrototypeContext; // ignore contextID link.FileID = topic.FileID; link.ClassString = topic.ClassString; // ignore classID link.LanguageID = topic.LanguageID; // ignore EndingSymbol // ignore TargetTopicID // ignore TargetScore linkSet.Add(link); } symbolStart = symbolEnd; } else { symbolStart.Next(); } } }
/* Function: CountSymbols * * An internal function that detects whether the <TokenIterator> is on a stretch of symbols, and if so, returns * true along with what the symbol is and how many there are. It will leave the start iterator equal to end or at * the first token after the stretch. * * If the start iterator wasn't on a symbol it returns false, sets the symbol to null, the count to zero, and does * not move the start iterator. */ private static bool CountSymbols(ref TokenIterator start, TokenIterator end, out char symbol, out int count) { if (start >= end || start.FundamentalType != FundamentalType.Symbol) { symbol = '\0'; count = 0; return(false); } symbol = start.Character; count = 1; start.Next(); while (start < end && start.Character == symbol) { count++; start.Next(); } return(true); }
/* Function: GetDefaultValue * Returns the bounds of the default value as marked by <PrototypeParsingType.DefaultValueSeparator> and * <PrototypeParsingType.DefaultValue, or false if it couldn't find it. */ virtual public bool GetDefaultValue(out TokenIterator defaultValueStart, out TokenIterator defaultValueEnd) { TokenIterator iterator = start; while (iterator < end && iterator.PrototypeParsingType != PrototypeParsingType.DefaultValue) { iterator.Next(); } defaultValueStart = iterator; while (iterator.PrototypeParsingType == PrototypeParsingType.DefaultValue) { iterator.Next(); } defaultValueEnd = iterator; return(defaultValueStart != defaultValueEnd); }
// Group: Protected Functions // __________________________________________________________________________ /* Function: TryToSkipModifierBlock * If the iterator is on a <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier> * token, moves it past the entire block including any nested blocks. */ protected static bool TryToSkipModifierBlock(ref TokenIterator iterator) { if (iterator.PrototypeParsingType != PrototypeParsingType.OpeningTypeModifier && iterator.PrototypeParsingType != PrototypeParsingType.OpeningParamModifier) { return(false); } TokenIterator lookahead = iterator; lookahead.Next(); int level = 1; // We're going to cheat and assume all blocks are balanced and nested in a way that makes sense. This lets us handle // both in a simple loop. while (lookahead.IsInBounds) { if (lookahead.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier || lookahead.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier) { level++; } else if (lookahead.PrototypeParsingType == PrototypeParsingType.ClosingTypeModifier || lookahead.PrototypeParsingType == PrototypeParsingType.ClosingParamModifier) { level--; if (level == 0) { lookahead.Next(); iterator = lookahead; return(true); } } lookahead.Next(); } return(false); }
/* Function: TryToSkipLineCommentSymbol * * If the iterator is on a line comment, moves it past it and returns true. Otherwise leaves the iterator alone and * returns false. This handles the fact that block comments also start with -- and will not move past them. */ protected bool TryToSkipLineCommentSymbol(ref TokenIterator iterator) { if (iterator.MatchesAcrossTokens("--") && !IsOnOpeningBlockCommentSymbol(iterator)) { iterator.Next(2); return(true); } else { return(false); } }
/* Function: AddModifierBlock * Adds a full modifier block marked with <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier> * to the type builder. */ public void AddModifierBlock(TokenIterator openingToken, TokenIterator closingToken) { if ((openingToken.PrototypeParsingType != PrototypeParsingType.OpeningTypeModifier && openingToken.PrototypeParsingType != PrototypeParsingType.OpeningParamModifier) || (closingToken.PrototypeParsingType != PrototypeParsingType.ClosingTypeModifier && closingToken.PrototypeParsingType != PrototypeParsingType.ClosingParamModifier)) { throw new InvalidOperationException(); } TokenIterator iterator = openingToken; TokenIterator end = closingToken; end.Next(); while (iterator < end) { FundamentalType thisTokenType = (iterator.Character == '_' ? FundamentalType.Text : iterator.FundamentalType); // Do we need to add a space? if ((thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Text && (iterator.Tokenizer != lastTokenIterator.Tokenizer || iterator.TokenIndex != lastTokenIterator.TokenIndex + 1)) || (thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Symbol && !dontAddSpaceAfterSymbol && (pastFirstText || lastSymbolWasBlock))) { rawText.Append(' '); prototypeParsingTypes.Add(PrototypeParsingType.Null); syntaxHighlightingTypes.Add(SyntaxHighlightingType.Null); } while (iterator < end) { iterator.AppendTokenTo(rawText); prototypeParsingTypes.Add(iterator.PrototypeParsingType); syntaxHighlightingTypes.Add(iterator.SyntaxHighlightingType); iterator.Next(); } lastTokenIterator = closingToken; lastTokenType = thisTokenType; lastSymbolWasBlock = true; dontAddSpaceAfterSymbol = false; if (thisTokenType == FundamentalType.Text) { pastFirstText = true; } } }
/* Function: TryToSkipLineComment * If the iterator is on the opening symbol of a line comment, skips over it and returns true. Otherwise leaves the iterator * alone and returns false. */ protected bool TryToSkipLineComment(ref TokenIterator iterator, string symbol) { if (iterator.MatchesAcrossTokens(symbol)) { iterator.NextByCharacters(symbol.Length); while (iterator.IsInBounds && iterator.FundamentalType != FundamentalType.LineBreak) { iterator.Next(); } if (iterator.FundamentalType == FundamentalType.LineBreak) { iterator.Next(); } return(true); } else { return(false); } }
/* Function: GetBaseType * Returns the bounds of the base type if one is marked by <PrototypeParsingType.Type> tokens, or false if it couldn't find it. * It will also include type qualifiers ("Package.Class") but exclude modifiers (so "unsigned int*[]" would just be "int".) */ virtual public bool GetBaseType(out TokenIterator baseTypeStart, out TokenIterator baseTypeEnd) { TokenIterator iterator = start; while (iterator < end && iterator.PrototypeParsingType != PrototypeParsingType.Type && iterator.PrototypeParsingType != PrototypeParsingType.TypeQualifier) { iterator.Next(); } baseTypeStart = iterator; while (iterator.PrototypeParsingType == PrototypeParsingType.Type || iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier) { iterator.Next(); } baseTypeEnd = iterator; return(baseTypeStart != baseTypeEnd); }
/* Function: ToTokenizer * Creates a <Tokenizer> from everything added to the type builder. */ public Tokenizer ToTokenizer() { Tokenizer tokenizer = new Tokenizer(rawText.ToString()); TokenIterator iterator = tokenizer.FirstToken; while (iterator.IsInBounds) { iterator.PrototypeParsingType = prototypeParsingTypes[iterator.TokenIndex]; iterator.SyntaxHighlightingType = syntaxHighlightingTypes[iterator.TokenIndex]; iterator.Next(); } return(tokenizer); }
/* Function: GetAccessLevel * Returns the <Languages.AccessLevel> if it can be determined. This should only be used with basic language support * as it's not as reliable as the results from the dedicated language parsers. */ virtual public Languages.AccessLevel GetAccessLevel() { Languages.AccessLevel accessLevel = Languages.AccessLevel.Unknown; TokenIterator iterator = start; while (iterator < end) { if (iterator.FundamentalType == FundamentalType.Text && iterator.PrototypeParsingType == PrototypeParsingType.TypeModifier && IsStandaloneWord(iterator)) { if (iterator.MatchesToken("public")) { accessLevel = Languages.AccessLevel.Public; } else if (iterator.MatchesToken("private")) { accessLevel = Languages.AccessLevel.Private; } else if (iterator.MatchesToken("protected")) { if (accessLevel == Languages.AccessLevel.Internal) { accessLevel = Languages.AccessLevel.ProtectedInternal; } else { accessLevel = Languages.AccessLevel.Protected; } } else if (iterator.MatchesToken("internal")) { if (accessLevel == Languages.AccessLevel.Protected) { accessLevel = Languages.AccessLevel.ProtectedInternal; } else { accessLevel = Languages.AccessLevel.Internal; } } } iterator.Next(); } return(accessLevel); }
/* Function: RecalculateParameterStyle * Determines whether the parameters are C-style ("int x") or Pascal-style ("x: int"). If it has no parameters or * no types this will return C. Tokens must be marked with <PrototypeParsingType.Name>, <PrototypeParsingType.Type>, * and <PrototypeParsingType.NameTypeSeparator> for this to work. */ public void RecalculateParameterStyle() { if (parameters == null) { parameterStyle = ParsedPrototype.ParameterStyle.C; return; } foreach (var parameter in parameters) { bool foundName = false; bool foundType = false; TokenIterator iterator = parameter.Start; while (iterator < parameter.End) { if (iterator.PrototypeParsingType == PrototypeParsingType.Name) { if (foundType) { parameterStyle = ParsedPrototype.ParameterStyle.C; return; } else { foundName = true; } } else if (iterator.PrototypeParsingType == PrototypeParsingType.Type) { if (foundName) { parameterStyle = ParsedPrototype.ParameterStyle.Pascal; return; } else { foundType = true; } } iterator.Next(); } } // If we hit the end without anything definitive, treat it as C parameterStyle = ParsedPrototype.ParameterStyle.C; }
/* Function: TryToSkipModifierBlock * If the iterator is on a <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier> * token, moves the iterator past the entire block including any nested blocks. */ protected bool TryToSkipModifierBlock(ref TokenIterator iterator) { TokenIterator closingModifier; if (GetClosingModifier(iterator, out closingModifier)) { iterator = closingModifier; iterator.Next(); return(true); } else { return(false); } }
private void SelectItem() { var listBoxItem = (ListBoxItem <IObjectName>)ListBox.SelectedItem; if (listBoxItem != null) { var selectedItem = listBoxItem.Item.UnquotedName; var startIndex = _response.StartPosition; var tokenIterator = new TokenIterator(_textBox.Text.Substring(startIndex)); var token = tokenIterator.Next(); int length; if (token != null && token.StartPosition == 0) { length = token.EndPosition - token.StartPosition + 1; } else { length = 0; } var originalText = _textBox.Text.Substring(startIndex, length); var originalItems = originalText.Split('.'); var newItems = selectedItem.Split('.'); var sb = new StringBuilder(); for (var i = 0; i < originalItems.Length - newItems.Length; i++) { if (sb.Length > 0) { sb.Append('.'); } sb.Append(originalItems[i]); } for (var i = 0; i < newItems.Length; i++) { if (sb.Length > 0) { sb.Append('.'); } sb.Append(newItems[i]); } var newText = sb.ToString(); // TODO _completionForm.SelectItem(startIndex, length, listBoxItem.Item); } }
/* Function: TryToSkipDecorator * * Tries to move the iterator past a single decorator. Note that there may be more than one decorator in a row, so use <TryToSkipDecorators()> * if you need to move past all of them. * * Supported Modes: * * - <ParseMode.IterateOnly> * - <ParseMode.ParsePrototype> * - Each decorator will create a new prototype section. * - <ParseMode.ParseClassPrototype> * - Will mark the first token with <ClassPrototypeParsingType.StartOfPrePrototypeLine> and the rest with <ClassPrototypeParsingType.PrePrototypeLine>. * - Everything else is treated as <ParseMode.IterateOnly>. */ protected bool TryToSkipDecorator(ref TokenIterator iterator, ParseMode mode = ParseMode.IterateOnly) { if (iterator.Character != '@') { return(false); } TokenIterator lookahead = iterator; lookahead.Next(); if (TryToSkipIdentifier(ref lookahead) == false) { return(false); } TokenIterator decoratorStart = iterator; TokenIterator decoratorEnd = lookahead; if (mode == ParseMode.SyntaxHighlight) { decoratorStart.SetSyntaxHighlightingTypeBetween(decoratorEnd, SyntaxHighlightingType.Metadata); } TryToSkipWhitespace(ref lookahead); if (TryToSkipDecoratorParameters(ref lookahead, mode)) { decoratorEnd = lookahead; } if (mode == ParseMode.ParsePrototype) { decoratorStart.PrototypeParsingType = PrototypeParsingType.StartOfPrototypeSection; decoratorEnd.PrototypeParsingType = PrototypeParsingType.EndOfPrototypeSection; } else if (mode == ParseMode.ParseClassPrototype) { iterator.SetClassPrototypeParsingTypeBetween(lookahead, ClassPrototypeParsingType.PrePrototypeLine); iterator.ClassPrototypeParsingType = ClassPrototypeParsingType.StartOfPrePrototypeLine; } iterator = decoratorEnd; return(true); }
/* Function: TryToSkipAnnotation * * Tries to move the iterator past a single annotation, like "@Preliminary" or "@Copynight("String")". * * Supported Modes: * * - <ParseMode.IterateOnly> * - <ParseMode.SyntaxHighlight> * - <ParseMode.ParsePrototype> * - Each annotation will create a new prototype section. * - Everything else is treated as <ParseMode.IterateOnly>. */ protected bool TryToSkipAnnotation(ref TokenIterator iterator, ParseMode mode = ParseMode.IterateOnly) { if (iterator.Character != '@') { return(false); } TokenIterator lookahead = iterator; lookahead.Next(); // Whitespace is allowed between the @ and the identifier, though it's not recommended TryToSkipWhitespace(ref lookahead, true, mode); if (!TryToSkipIdentifier(ref lookahead, mode)) { return(false); } // That's all we need to be successful. Try to find parameters though. TokenIterator annotationStart = iterator; TokenIterator annotationEnd = lookahead; if (mode == ParseMode.SyntaxHighlight) { annotationStart.SetSyntaxHighlightingTypeBetween(annotationEnd, SyntaxHighlightingType.Metadata); } TryToSkipWhitespace(ref lookahead, true, mode); if (TryToSkipAnnotationParameters(ref lookahead, mode)) { annotationEnd = lookahead; } if (mode == ParseMode.ParsePrototype) { annotationStart.PrototypeParsingType = PrototypeParsingType.StartOfPrototypeSection; annotationEnd.PrototypeParsingType = PrototypeParsingType.EndOfPrototypeSection; } iterator = annotationEnd; return(true); }
public static List <Token> Tokenize(string text) { var tokens = new List <Token>(); var iterator = new TokenIterator(text); while (true) { var token = iterator.Next(); if (token == null) { break; } tokens.Add(token); } return(tokens); }
public void TestTokenizeFileWithIterator() { Tokenizer tokenizer = Tokenizer.FromFile(RESOURCE_TEST_FILE); TokenIterator it = tokenizer.GetTokenIterator(); Token token; Token actualToken; int index = 0; Assert.IsTrue(FILE_EXPECTED_TOKEN_ELEMENTS.Length == it.Count * 4); while (it.HasNext()) { token = it.Next(); actualToken = new Token((TokenType)FILE_EXPECTED_TOKEN_ELEMENTS[index, 0], (string)FILE_EXPECTED_TOKEN_ELEMENTS[index, 1], (int)FILE_EXPECTED_TOKEN_ELEMENTS[index, 2], (int)FILE_EXPECTED_TOKEN_ELEMENTS[index, 3]); Assert.AreEqual(actualToken, token); index++; } }
/* Function: GetClosingModifier * If the iterator is on a <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier> * token, returns a reference to the closing token. It will handle any nested blocks. If the iterator isn't on an appropriate * token or it couldn't find the end of the block, returns false. */ protected bool GetClosingModifier(TokenIterator openingModifier, out TokenIterator closingModifier) { if (openingModifier.PrototypeParsingType != PrototypeParsingType.OpeningTypeModifier && openingModifier.PrototypeParsingType != PrototypeParsingType.OpeningParamModifier) { closingModifier = openingModifier; return(false); } closingModifier = openingModifier; closingModifier.Next(); int level = 1; // We're going to cheat and assume all blocks are balanced and nested in a way that makes sense. This lets us handle // both in a simple loop. for (;;) { if (closingModifier >= end) { closingModifier = openingModifier; return(false); } else if (closingModifier.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier || closingModifier.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier) { level++; } else if (closingModifier.PrototypeParsingType == PrototypeParsingType.ClosingTypeModifier || closingModifier.PrototypeParsingType == PrototypeParsingType.ClosingParamModifier) { level--; if (level == 0) { return(true); } } closingModifier.Next(); } }
/* Function: GetSectionBounds * Returns the bounds of the passed section and whether it exists. An index of zero represents the first section of that * type, 1 represents the second, etc. */ protected bool GetSectionBounds(SectionType type, int index, out TokenIterator start, out TokenIterator end) { Section section = FindSection(type, index); if (section == null) { start = tokenizer.LastToken; end = start; return(false); } else { start = tokenizer.FirstToken; start.Next(section.StartIndex); end = start; end.Next(section.EndIndex - section.StartIndex); return(true); } }