/* Function: BuildFullType
* Creates a new <Tokenizer> for the variable type, including all modifiers, even if they are not continuous. This is a support
* function for <BuildFullType(TokenIterator, TokenIterator, Tokenizer)> and it always builds a new <Tokenizer>.
*/
protected Tokenizer BuildFullType()
{
TokenIterator iterator = start;
TypeBuilder typeBuilder = new TypeBuilder(end.RawTextIndex - start.RawTextIndex, end.TokenIndex - start.TokenIndex);
while (iterator < end)
{
if (iterator.PrototypeParsingType == PrototypeParsingType.Type ||
iterator.PrototypeParsingType == PrototypeParsingType.TypeModifier ||
iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier ||
iterator.PrototypeParsingType == PrototypeParsingType.ParamModifier ||
iterator.PrototypeParsingType == PrototypeParsingType.StartOfTuple ||
iterator.PrototypeParsingType == PrototypeParsingType.EndOfTuple ||
iterator.PrototypeParsingType == PrototypeParsingType.TupleMemberSeparator ||
iterator.PrototypeParsingType == PrototypeParsingType.TupleMemberName)
{
typeBuilder.AddToken(iterator);
}
else if (iterator.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier ||
iterator.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier)
{
TokenIterator closingModifier;
GetClosingModifier(iterator, out closingModifier);
typeBuilder.AddModifierBlock(iterator, closingModifier);
iterator = closingModifier;
}
iterator.Next();
}
return(typeBuilder.ToTokenizer());
}
// Group: Protected Functions
// __________________________________________________________________________
/* Function: IsStandaloneWord
* Returns whether the iterator is on a text token and the tokens immediately before and after it are not underscores.
*/
protected bool IsStandaloneWord(TokenIterator iterator)
{
if (iterator.FundamentalType != FundamentalType.Text)
{
return(false);
}
TokenIterator lookahead = iterator;
lookahead.Next();
if (lookahead.FundamentalType == FundamentalType.Text ||
lookahead.Character == '_')
{
return(false);
}
TokenIterator lookbehind = iterator;
lookbehind.Previous();
if (lookbehind.FundamentalType == FundamentalType.Text ||
lookbehind.Character == '_')
{
return(false);
}
return(true);
}
// Group: Parsing Functions
// __________________________________________________________________________
/* Function: GenericSkip
*
* Moves the iterator ahead one code element, which could be a single token, whitespace, an entire comment, or an entire
* string. The important part is that this skips comments and strings all in one step so that anything appearing inside them
* will not be misinterpreted as code.
*
* It is virtual so you can extend it to handle things like regular expressions. The default implementation handles strings
* and comments based on LineCommentStrings, BlockCommentStringPairs, and <QuoteCharacters>.
*/
protected virtual void GenericSkip(ref TokenIterator iterator)
{
if (!TryToSkipComment(ref iterator) &&
!TryToSkipString(ref iterator))
{
iterator.Next();
}
}
/* Function: TryToSkipOpeningBlockCommentSymbol
*
* If the iterator is on an opening block comment symbol, moves it past it and returns true and the closing symbol. Otherwise
* leaves the iterator alone and returns false and null. This handles --[[ comments as well as forms with equals signs such as
* --[==[.
*/
protected bool TryToSkipOpeningBlockCommentSymbol(ref TokenIterator iterator, out string closingSymbol)
{
if (iterator.MatchesAcrossTokens("--[") == false)
{
closingSymbol = null;
return(false);
}
TokenIterator lookahead = iterator;
lookahead.Next(3);
int equals = 0;
while (lookahead.Character == '=')
{
lookahead.Next();
equals++;
}
if (lookahead.Character != '[')
{
closingSymbol = null;
return(false);
}
lookahead.Next();
iterator = lookahead;
if (equals == 0)
{
closingSymbol = "]]";
}
else
{
StringBuilder closingSymbolBuilder = new StringBuilder(2 + equals);
closingSymbolBuilder.Append(']');
closingSymbolBuilder.Append('=', equals);
closingSymbolBuilder.Append(']');
closingSymbol = closingSymbolBuilder.ToString();
}
return(true);
}
public void TestBadTwoNext()
{
List <Token> tokens = new List <Token>();
tokens.Add(new Token(TokenType.AtIdentifier, "asd", 0, 0));
TokenIterator tokenIterator = new TokenIterator(tokens);
try
{
tokenIterator.Next();
}
catch (ArgumentOutOfRangeException)
{
// First one is valid, it shouldn't fail here.
Assert.Fail();
}
tokenIterator.Next();
}
/* Function: TryToSkipLocaleSubstitutionIdentifier
* If the iterator is on a valid locale substitution identifier, advances it past it, returns it, and returns true.
* Otherwise the iterator will be left alone and it will return false.
*/
protected bool TryToSkipLocaleSubstitutionIdentifier(ref TokenIterator iterator, out string identifier, out string localeIdentifier)
{
if (iterator.Character != '$' && iterator.Character != '@')
{
identifier = null;
localeIdentifier = null;
return(false);
}
TokenIterator lookahead = iterator;
lookahead.Next();
if (lookahead.MatchesAcrossTokens("Locale{") == false)
{
identifier = null;
localeIdentifier = null;
return(false);
}
lookahead.NextByCharacters(7);
TokenIterator startOfLocaleIdentifier = lookahead;
while (lookahead.IsInBounds && lookahead.Character != '}')
{
lookahead.Next();
}
if (lookahead.Character != '}')
{
identifier = null;
localeIdentifier = null;
return(false);
}
localeIdentifier = startOfLocaleIdentifier.TextBetween(lookahead);
lookahead.Next();
identifier = iterator.TextBetween(lookahead);
iterator = lookahead;
return(true);
}
/* Function: TryToSkipSubstitutionDefinition
*
* If the iterator is on a valid substitution definition, advances it past it, determines its properties, and returns true. Otherwise the
* iterator will be left alone and it will return false.
*
* identifier - "$identifier" in "$identifier = value;"
* value - "value" in "$identifier = value;"
* declaration - "$identifier = value;" in "$identifier = value;"
*/
protected bool TryToSkipSubstitutionDefinition(ref TokenIterator iterator, out string identifier, out string value, out string declaration)
{
identifier = null;
value = null;
declaration = null;
TokenIterator lookahead = iterator;
if (TryToSkipSubstitutionIdentifier(ref lookahead, out identifier) == false)
{
return(false);
}
lookahead.NextPastWhitespace();
if (lookahead.Character != ':' && lookahead.Character != '=')
{
identifier = null;
return(false);
}
lookahead.Next();
lookahead.NextPastWhitespace();
TokenIterator startOfValue = lookahead;
while (lookahead.IsInBounds && lookahead.Character != ';' && lookahead.FundamentalType != FundamentalType.LineBreak)
{
GenericSkip(ref lookahead);
}
value = startOfValue.TextBetween(lookahead);
if (lookahead.Character == ';')
{
lookahead.Next();
}
declaration = iterator.TextBetween(lookahead);
iterator = lookahead;
return(true);
}
/* Function: TryToSkipClassParent
*
* Tries to move the iterator past a single class parent declaration.
*
* Supported Modes:
*
* - <ParseMode.IterateOnly>
* - <ParseMode.ParseClassPrototype>
* - Everything else is treated as <ParseMode.IterateOnly>.
*/
protected bool TryToSkipClassParent(ref TokenIterator iterator, ParseMode mode = ParseMode.IterateOnly)
{
TokenIterator lookahead = iterator;
if (lookahead.MatchesToken("metaclass"))
{
lookahead.Next();
TryToSkipWhitespace(ref lookahead);
if (lookahead.Character == '=')
{
if (mode == ParseMode.ParseClassPrototype)
{
iterator.ClassPrototypeParsingType = ClassPrototypeParsingType.Modifier;
}
lookahead.Next();
TryToSkipWhitespace(ref lookahead);
}
else
{
// Nevermind, reset
lookahead = iterator;
}
}
TokenIterator startOfIdentifier = lookahead;
if (TryToSkipIdentifier(ref lookahead) == false)
{
ResetTokensBetween(iterator, lookahead, mode);
return(false);
}
if (mode == ParseMode.ParseClassPrototype)
{
lookahead.Tokenizer.SetClassPrototypeParsingTypeBetween(startOfIdentifier, lookahead, ClassPrototypeParsingType.Name);
}
iterator = lookahead;
return(true);
}
void AppendLinkCandidates(TokenIterator start, TokenIterator end, StringBuilder output)
{
TokenIterator iterator = start;
TokenIterator linkableTypeStart = start;
int linkableTypes = 0;
for (;;)
{
while (iterator < end &&
iterator.PrototypeParsingType != PrototypeParsingType.TypeQualifier &&
iterator.PrototypeParsingType != PrototypeParsingType.Type)
{
iterator.Next();
}
if (iterator >= end)
{
break;
}
linkableTypeStart = iterator;
while (iterator < end &&
(iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier ||
iterator.PrototypeParsingType == PrototypeParsingType.Type))
{
iterator.Next();
}
if (linkableTypes > 0)
{
output.Append(", ");
}
linkableTypeStart.AppendTextBetweenTo(iterator, output);
linkableTypes++;
}
if (linkableTypes == 0)
{
output.Append("(none)");
}
}
/* Function: GetCompletePrototype
* Returns the bounds of the complete prototype, minus whitespace. This does NOT include pre-prototype lines.
*/
public void GetCompletePrototype(out TokenIterator start, out TokenIterator end)
{
Section beforeParameters = FindSection(SectionType.BeforeParameters);
start = tokenizer.FirstToken;
start.Next(beforeParameters.StartIndex);
Section afterParameters = FindSection(SectionType.AfterParameters);
end = start;
if (afterParameters == null)
{
end.Next(beforeParameters.EndIndex - beforeParameters.StartIndex);
}
else
{
end.Next(afterParameters.EndIndex - beforeParameters.StartIndex);
}
}
public void TestHasNext()
{
List <Token> tokens = new List <Token>();
tokens.Add(new Token(TokenType.DollarIdentifier, "a", 5, 2));
tokens.Add(new Token(TokenType.Word, "a", 5, 3));
tokens.Add(new Token(TokenType.AtIdentifier, "@b", 12, 0));
tokens.Add(new Token(TokenType.Word, "c", 14, 42));
TokenIterator tokenIterator = new TokenIterator(tokens);
Assert.IsTrue(tokenIterator.HasNextType(TokenType.DollarIdentifier));
Assert.IsTrue(tokenIterator.HasNextType(TokenType.DollarIdentifier, TokenType.Pipe));
Assert.IsTrue(tokenIterator.HasNextType(TokenType.Period, TokenType.DollarIdentifier, TokenType.Pipe));
Assert.IsFalse(tokenIterator.HasNextType(TokenType.Number));
tokenIterator.Next();
tokenIterator.Next();
tokenIterator.Next();
tokenIterator.Next();
Assert.IsFalse(tokenIterator.HasNextType(TokenType.Word));
}
/* Function: ExtractTypeLinks
* Goes through the prototype of the passed <Topic> and adds any type links it finds to <LinkSet>.
*/
protected void ExtractTypeLinks(Topic topic, LinkSet linkSet)
{
if (topic.Prototype == null)
{
return;
}
Language language = EngineInstance.Languages.FromID(topic.LanguageID);
TokenIterator symbolStart = topic.ParsedPrototype.Tokenizer.FirstToken;
TokenIterator symbolEnd;
while (symbolStart.IsInBounds)
{
if (symbolStart.PrototypeParsingType == PrototypeParsingType.Type ||
symbolStart.PrototypeParsingType == PrototypeParsingType.TypeQualifier)
{
symbolEnd = symbolStart;
do
{
symbolEnd.Next();
}while (symbolEnd.PrototypeParsingType == PrototypeParsingType.Type ||
symbolEnd.PrototypeParsingType == PrototypeParsingType.TypeQualifier);
if (language.IsBuiltInType(symbolStart, symbolEnd) == false)
{
Link link = new Link();
// ignore LinkID
link.Type = LinkType.Type;
link.Symbol = SymbolString.FromPlainText_NoParameters(symbolStart.Tokenizer.TextBetween(symbolStart, symbolEnd));
link.Context = topic.PrototypeContext;
// ignore contextID
link.FileID = topic.FileID;
link.ClassString = topic.ClassString;
// ignore classID
link.LanguageID = topic.LanguageID;
// ignore EndingSymbol
// ignore TargetTopicID
// ignore TargetScore
linkSet.Add(link);
}
symbolStart = symbolEnd;
}
else
{
symbolStart.Next();
}
}
}
/* Function: CountSymbols
*
* An internal function that detects whether the <TokenIterator> is on a stretch of symbols, and if so, returns
* true along with what the symbol is and how many there are. It will leave the start iterator equal to end or at
* the first token after the stretch.
*
* If the start iterator wasn't on a symbol it returns false, sets the symbol to null, the count to zero, and does
* not move the start iterator.
*/
private static bool CountSymbols(ref TokenIterator start, TokenIterator end, out char symbol, out int count)
{
if (start >= end || start.FundamentalType != FundamentalType.Symbol)
{
symbol = '\0';
count = 0;
return(false);
}
symbol = start.Character;
count = 1;
start.Next();
while (start < end && start.Character == symbol)
{
count++;
start.Next();
}
return(true);
}
/* Function: GetDefaultValue
* Returns the bounds of the default value as marked by <PrototypeParsingType.DefaultValueSeparator> and
* <PrototypeParsingType.DefaultValue, or false if it couldn't find it.
*/
virtual public bool GetDefaultValue(out TokenIterator defaultValueStart, out TokenIterator defaultValueEnd)
{
TokenIterator iterator = start;
while (iterator < end &&
iterator.PrototypeParsingType != PrototypeParsingType.DefaultValue)
{
iterator.Next();
}
defaultValueStart = iterator;
while (iterator.PrototypeParsingType == PrototypeParsingType.DefaultValue)
{
iterator.Next();
}
defaultValueEnd = iterator;
return(defaultValueStart != defaultValueEnd);
}
// Group: Protected Functions
// __________________________________________________________________________
/* Function: TryToSkipModifierBlock
* If the iterator is on a <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier>
* token, moves it past the entire block including any nested blocks.
*/
protected static bool TryToSkipModifierBlock(ref TokenIterator iterator)
{
if (iterator.PrototypeParsingType != PrototypeParsingType.OpeningTypeModifier &&
iterator.PrototypeParsingType != PrototypeParsingType.OpeningParamModifier)
{
return(false);
}
TokenIterator lookahead = iterator;
lookahead.Next();
int level = 1;
// We're going to cheat and assume all blocks are balanced and nested in a way that makes sense. This lets us handle
// both in a simple loop.
while (lookahead.IsInBounds)
{
if (lookahead.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier ||
lookahead.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier)
{
level++;
}
else if (lookahead.PrototypeParsingType == PrototypeParsingType.ClosingTypeModifier ||
lookahead.PrototypeParsingType == PrototypeParsingType.ClosingParamModifier)
{
level--;
if (level == 0)
{
lookahead.Next();
iterator = lookahead;
return(true);
}
}
lookahead.Next();
}
return(false);
}
/* Function: TryToSkipLineCommentSymbol
*
* If the iterator is on a line comment, moves it past it and returns true. Otherwise leaves the iterator alone and
* returns false. This handles the fact that block comments also start with -- and will not move past them.
*/
protected bool TryToSkipLineCommentSymbol(ref TokenIterator iterator)
{
if (iterator.MatchesAcrossTokens("--") &&
!IsOnOpeningBlockCommentSymbol(iterator))
{
iterator.Next(2);
return(true);
}
else
{
return(false);
}
}
/* Function: AddModifierBlock
* Adds a full modifier block marked with <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier>
* to the type builder.
*/
public void AddModifierBlock(TokenIterator openingToken, TokenIterator closingToken)
{
if ((openingToken.PrototypeParsingType != PrototypeParsingType.OpeningTypeModifier &&
openingToken.PrototypeParsingType != PrototypeParsingType.OpeningParamModifier) ||
(closingToken.PrototypeParsingType != PrototypeParsingType.ClosingTypeModifier &&
closingToken.PrototypeParsingType != PrototypeParsingType.ClosingParamModifier))
{
throw new InvalidOperationException();
}
TokenIterator iterator = openingToken;
TokenIterator end = closingToken;
end.Next();
while (iterator < end)
{
FundamentalType thisTokenType = (iterator.Character == '_' ? FundamentalType.Text : iterator.FundamentalType);
// Do we need to add a space?
if ((thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Text &&
(iterator.Tokenizer != lastTokenIterator.Tokenizer || iterator.TokenIndex != lastTokenIterator.TokenIndex + 1))
||
(thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Symbol &&
!dontAddSpaceAfterSymbol && (pastFirstText || lastSymbolWasBlock)))
{
rawText.Append(' ');
prototypeParsingTypes.Add(PrototypeParsingType.Null);
syntaxHighlightingTypes.Add(SyntaxHighlightingType.Null);
}
while (iterator < end)
{
iterator.AppendTokenTo(rawText);
prototypeParsingTypes.Add(iterator.PrototypeParsingType);
syntaxHighlightingTypes.Add(iterator.SyntaxHighlightingType);
iterator.Next();
}
lastTokenIterator = closingToken;
lastTokenType = thisTokenType;
lastSymbolWasBlock = true;
dontAddSpaceAfterSymbol = false;
if (thisTokenType == FundamentalType.Text)
{
pastFirstText = true;
}
}
}
/* Function: TryToSkipLineComment
* If the iterator is on the opening symbol of a line comment, skips over it and returns true. Otherwise leaves the iterator
* alone and returns false.
*/
protected bool TryToSkipLineComment(ref TokenIterator iterator, string symbol)
{
if (iterator.MatchesAcrossTokens(symbol))
{
iterator.NextByCharacters(symbol.Length);
while (iterator.IsInBounds && iterator.FundamentalType != FundamentalType.LineBreak)
{
iterator.Next();
}
if (iterator.FundamentalType == FundamentalType.LineBreak)
{
iterator.Next();
}
return(true);
}
else
{
return(false);
}
}
/* Function: GetBaseType
* Returns the bounds of the base type if one is marked by <PrototypeParsingType.Type> tokens, or false if it couldn't find it.
* It will also include type qualifiers ("Package.Class") but exclude modifiers (so "unsigned int*[]" would just be "int".)
*/
virtual public bool GetBaseType(out TokenIterator baseTypeStart, out TokenIterator baseTypeEnd)
{
TokenIterator iterator = start;
while (iterator < end &&
iterator.PrototypeParsingType != PrototypeParsingType.Type &&
iterator.PrototypeParsingType != PrototypeParsingType.TypeQualifier)
{
iterator.Next();
}
baseTypeStart = iterator;
while (iterator.PrototypeParsingType == PrototypeParsingType.Type ||
iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier)
{
iterator.Next();
}
baseTypeEnd = iterator;
return(baseTypeStart != baseTypeEnd);
}
/* Function: ToTokenizer
* Creates a <Tokenizer> from everything added to the type builder.
*/
public Tokenizer ToTokenizer()
{
Tokenizer tokenizer = new Tokenizer(rawText.ToString());
TokenIterator iterator = tokenizer.FirstToken;
while (iterator.IsInBounds)
{
iterator.PrototypeParsingType = prototypeParsingTypes[iterator.TokenIndex];
iterator.SyntaxHighlightingType = syntaxHighlightingTypes[iterator.TokenIndex];
iterator.Next();
}
return(tokenizer);
}
/* Function: GetAccessLevel
* Returns the <Languages.AccessLevel> if it can be determined. This should only be used with basic language support
* as it's not as reliable as the results from the dedicated language parsers.
*/
virtual public Languages.AccessLevel GetAccessLevel()
{
Languages.AccessLevel accessLevel = Languages.AccessLevel.Unknown;
TokenIterator iterator = start;
while (iterator < end)
{
if (iterator.FundamentalType == FundamentalType.Text &&
iterator.PrototypeParsingType == PrototypeParsingType.TypeModifier &&
IsStandaloneWord(iterator))
{
if (iterator.MatchesToken("public"))
{
accessLevel = Languages.AccessLevel.Public;
}
else if (iterator.MatchesToken("private"))
{
accessLevel = Languages.AccessLevel.Private;
}
else if (iterator.MatchesToken("protected"))
{
if (accessLevel == Languages.AccessLevel.Internal)
{
accessLevel = Languages.AccessLevel.ProtectedInternal;
}
else
{
accessLevel = Languages.AccessLevel.Protected;
}
}
else if (iterator.MatchesToken("internal"))
{
if (accessLevel == Languages.AccessLevel.Protected)
{
accessLevel = Languages.AccessLevel.ProtectedInternal;
}
else
{
accessLevel = Languages.AccessLevel.Internal;
}
}
}
iterator.Next();
}
return(accessLevel);
}
/* Function: RecalculateParameterStyle
* Determines whether the parameters are C-style ("int x") or Pascal-style ("x: int"). If it has no parameters or
* no types this will return C. Tokens must be marked with <PrototypeParsingType.Name>, <PrototypeParsingType.Type>,
* and <PrototypeParsingType.NameTypeSeparator> for this to work.
*/
public void RecalculateParameterStyle()
{
if (parameters == null)
{
parameterStyle = ParsedPrototype.ParameterStyle.C;
return;
}
foreach (var parameter in parameters)
{
bool foundName = false;
bool foundType = false;
TokenIterator iterator = parameter.Start;
while (iterator < parameter.End)
{
if (iterator.PrototypeParsingType == PrototypeParsingType.Name)
{
if (foundType)
{
parameterStyle = ParsedPrototype.ParameterStyle.C;
return;
}
else
{
foundName = true;
}
}
else if (iterator.PrototypeParsingType == PrototypeParsingType.Type)
{
if (foundName)
{
parameterStyle = ParsedPrototype.ParameterStyle.Pascal;
return;
}
else
{
foundType = true;
}
}
iterator.Next();
}
}
// If we hit the end without anything definitive, treat it as C
parameterStyle = ParsedPrototype.ParameterStyle.C;
}
/* Function: TryToSkipModifierBlock
* If the iterator is on a <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier>
* token, moves the iterator past the entire block including any nested blocks.
*/
protected bool TryToSkipModifierBlock(ref TokenIterator iterator)
{
TokenIterator closingModifier;
if (GetClosingModifier(iterator, out closingModifier))
{
iterator = closingModifier;
iterator.Next();
return(true);
}
else
{
return(false);
}
}
private void SelectItem()
{
var listBoxItem = (ListBoxItem <IObjectName>)ListBox.SelectedItem;
if (listBoxItem != null)
{
var selectedItem = listBoxItem.Item.UnquotedName;
var startIndex = _response.StartPosition;
var tokenIterator = new TokenIterator(_textBox.Text.Substring(startIndex));
var token = tokenIterator.Next();
int length;
if (token != null && token.StartPosition == 0)
{
length = token.EndPosition - token.StartPosition + 1;
}
else
{
length = 0;
}
var originalText = _textBox.Text.Substring(startIndex, length);
var originalItems = originalText.Split('.');
var newItems = selectedItem.Split('.');
var sb = new StringBuilder();
for (var i = 0; i < originalItems.Length - newItems.Length; i++)
{
if (sb.Length > 0)
{
sb.Append('.');
}
sb.Append(originalItems[i]);
}
for (var i = 0; i < newItems.Length; i++)
{
if (sb.Length > 0)
{
sb.Append('.');
}
sb.Append(newItems[i]);
}
var newText = sb.ToString();
// TODO
_completionForm.SelectItem(startIndex, length, listBoxItem.Item);
}
}
/* Function: TryToSkipDecorator
*
* Tries to move the iterator past a single decorator. Note that there may be more than one decorator in a row, so use <TryToSkipDecorators()>
* if you need to move past all of them.
*
* Supported Modes:
*
* - <ParseMode.IterateOnly>
* - <ParseMode.ParsePrototype>
* - Each decorator will create a new prototype section.
* - <ParseMode.ParseClassPrototype>
* - Will mark the first token with <ClassPrototypeParsingType.StartOfPrePrototypeLine> and the rest with <ClassPrototypeParsingType.PrePrototypeLine>.
* - Everything else is treated as <ParseMode.IterateOnly>.
*/
protected bool TryToSkipDecorator(ref TokenIterator iterator, ParseMode mode = ParseMode.IterateOnly)
{
if (iterator.Character != '@')
{
return(false);
}
TokenIterator lookahead = iterator;
lookahead.Next();
if (TryToSkipIdentifier(ref lookahead) == false)
{
return(false);
}
TokenIterator decoratorStart = iterator;
TokenIterator decoratorEnd = lookahead;
if (mode == ParseMode.SyntaxHighlight)
{
decoratorStart.SetSyntaxHighlightingTypeBetween(decoratorEnd, SyntaxHighlightingType.Metadata);
}
TryToSkipWhitespace(ref lookahead);
if (TryToSkipDecoratorParameters(ref lookahead, mode))
{
decoratorEnd = lookahead;
}
if (mode == ParseMode.ParsePrototype)
{
decoratorStart.PrototypeParsingType = PrototypeParsingType.StartOfPrototypeSection;
decoratorEnd.PrototypeParsingType = PrototypeParsingType.EndOfPrototypeSection;
}
else if (mode == ParseMode.ParseClassPrototype)
{
iterator.SetClassPrototypeParsingTypeBetween(lookahead, ClassPrototypeParsingType.PrePrototypeLine);
iterator.ClassPrototypeParsingType = ClassPrototypeParsingType.StartOfPrePrototypeLine;
}
iterator = decoratorEnd;
return(true);
}
/* Function: TryToSkipAnnotation
*
* Tries to move the iterator past a single annotation, like "@Preliminary" or "@Copynight("String")".
*
* Supported Modes:
*
* - <ParseMode.IterateOnly>
* - <ParseMode.SyntaxHighlight>
* - <ParseMode.ParsePrototype>
* - Each annotation will create a new prototype section.
* - Everything else is treated as <ParseMode.IterateOnly>.
*/
protected bool TryToSkipAnnotation(ref TokenIterator iterator, ParseMode mode = ParseMode.IterateOnly)
{
if (iterator.Character != '@')
{
return(false);
}
TokenIterator lookahead = iterator;
lookahead.Next();
// Whitespace is allowed between the @ and the identifier, though it's not recommended
TryToSkipWhitespace(ref lookahead, true, mode);
if (!TryToSkipIdentifier(ref lookahead, mode))
{
return(false);
}
// That's all we need to be successful. Try to find parameters though.
TokenIterator annotationStart = iterator;
TokenIterator annotationEnd = lookahead;
if (mode == ParseMode.SyntaxHighlight)
{
annotationStart.SetSyntaxHighlightingTypeBetween(annotationEnd, SyntaxHighlightingType.Metadata);
}
TryToSkipWhitespace(ref lookahead, true, mode);
if (TryToSkipAnnotationParameters(ref lookahead, mode))
{
annotationEnd = lookahead;
}
if (mode == ParseMode.ParsePrototype)
{
annotationStart.PrototypeParsingType = PrototypeParsingType.StartOfPrototypeSection;
annotationEnd.PrototypeParsingType = PrototypeParsingType.EndOfPrototypeSection;
}
iterator = annotationEnd;
return(true);
}
public static List <Token> Tokenize(string text)
{
var tokens = new List <Token>();
var iterator = new TokenIterator(text);
while (true)
{
var token = iterator.Next();
if (token == null)
{
break;
}
tokens.Add(token);
}
return(tokens);
}
public void TestTokenizeFileWithIterator()
{
Tokenizer tokenizer = Tokenizer.FromFile(RESOURCE_TEST_FILE);
TokenIterator it = tokenizer.GetTokenIterator();
Token token;
Token actualToken;
int index = 0;
Assert.IsTrue(FILE_EXPECTED_TOKEN_ELEMENTS.Length == it.Count * 4);
while (it.HasNext())
{
token = it.Next();
actualToken = new Token((TokenType)FILE_EXPECTED_TOKEN_ELEMENTS[index, 0],
(string)FILE_EXPECTED_TOKEN_ELEMENTS[index, 1],
(int)FILE_EXPECTED_TOKEN_ELEMENTS[index, 2],
(int)FILE_EXPECTED_TOKEN_ELEMENTS[index, 3]);
Assert.AreEqual(actualToken, token);
index++;
}
}
/* Function: GetClosingModifier
* If the iterator is on a <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier>
* token, returns a reference to the closing token. It will handle any nested blocks. If the iterator isn't on an appropriate
* token or it couldn't find the end of the block, returns false.
*/
protected bool GetClosingModifier(TokenIterator openingModifier, out TokenIterator closingModifier)
{
if (openingModifier.PrototypeParsingType != PrototypeParsingType.OpeningTypeModifier &&
openingModifier.PrototypeParsingType != PrototypeParsingType.OpeningParamModifier)
{
closingModifier = openingModifier;
return(false);
}
closingModifier = openingModifier;
closingModifier.Next();
int level = 1;
// We're going to cheat and assume all blocks are balanced and nested in a way that makes sense. This lets us handle
// both in a simple loop.
for (;;)
{
if (closingModifier >= end)
{
closingModifier = openingModifier;
return(false);
}
else if (closingModifier.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier ||
closingModifier.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier)
{
level++;
}
else if (closingModifier.PrototypeParsingType == PrototypeParsingType.ClosingTypeModifier ||
closingModifier.PrototypeParsingType == PrototypeParsingType.ClosingParamModifier)
{
level--;
if (level == 0)
{
return(true);
}
}
closingModifier.Next();
}
}
/* Function: GetSectionBounds
* Returns the bounds of the passed section and whether it exists. An index of zero represents the first section of that
* type, 1 represents the second, etc.
*/
protected bool GetSectionBounds(SectionType type, int index, out TokenIterator start, out TokenIterator end)
{
Section section = FindSection(type, index);
if (section == null)
{
start = tokenizer.LastToken;
end = start;
return(false);
}
else
{
start = tokenizer.FirstToken;
start.Next(section.StartIndex);
end = start;
end.Next(section.EndIndex - section.StartIndex);
return(true);
}
}
