public TokenizerResult(ITextSnapshot snapshot, ITokenizerCollection <TrackingToken> tokens, IList <TrackingToken> updatedTokens) { Snapshot = snapshot; Tokens = tokens; UpdatedTokens = updatedTokens; }
public override async Task <IParserResult> RunAsync(IDocument document, ITextSnapshot version, ITokenizerCollection <TrackingToken> trackingTokens, CancellationToken cancellation) { var tokens = trackingTokens .Where(t => t.Type != RadAsm2Lexer.WHITESPACE && t.Type != RadAsm2Lexer.LINE_COMMENT) .AsParallel() .AsOrdered() .WithCancellation(cancellation) .ToArray(); _referenceCandidates.Clear(); _definitionContainer.Clear(); var blocks = new List <IBlock>(); var errors = new List <IErrorToken>(); var currentBlock = new Block(); var parserState = ParserState.SearchInScope; var parenthCnt = 0; var searchInCondition = false; var preprocessBlock = false; blocks.Add(currentBlock); for (int i = 0; i < tokens.Length; i++) { cancellation.ThrowIfCancellationRequested(); var token = tokens[i]; if (token.Type == RadAsm2Lexer.PP_ELSE || token.Type == RadAsm2Lexer.PP_ELSIF || token.Type == RadAsm2Lexer.PP_ELIF) { preprocessBlock = true; } else if (token.Type == RadAsm2Lexer.PP_ENDIF) { preprocessBlock = false; } if (preprocessBlock) { if (token.Type == RadAsm2Lexer.IDENTIFIER) { TryAddInstruction(token.GetText(version), token, currentBlock, version, Instructions); } } else if (parserState == ParserState.SearchInScope) { if (token.Type == RadAsm2Lexer.BLOCK_COMMENT) { blocks.AppendBlock(new Block(currentBlock, BlockType.Comment, token, token, version)); } else if (token.Type == RadAsm2Lexer.EOL) { if (searchInCondition) { currentBlock.SetStart(tokens[i - 1].GetEnd(version)); searchInCondition = false; } if (tokens.Length - i > 3 && tokens[i + 1].Type == RadAsm2Lexer.IDENTIFIER && tokens[i + 2].Type == RadAsm2Lexer.COLON && tokens[i + 3].Type == RadAsm2Lexer.EOL) { var labelDefinition = new DefinitionToken(RadAsmTokenType.Label, tokens[i + 1], version); _definitionContainer.Add(currentBlock, labelDefinition); currentBlock.AddToken(labelDefinition); // lookbehind search references to label var labelReferences = _referenceCandidates .Where(t => t.text == labelDefinition.GetText()) .Reverse() .TakeWhile(t => currentBlock.Area.Contains(t.block.Area.Start)) .ToList(); foreach (var reference in labelReferences) { cancellation.ThrowIfCancellationRequested(); _referenceCandidates.Remove(reference); reference.block.AddToken(new ReferenceToken(RadAsmTokenType.LabelReference, reference.trackingToken, version, labelDefinition)); } i += 2; } } else if (token.Type == RadAsm2Lexer.FUNCTION || token.Type == RadAsm2Lexer.SHADER) { if (tokens.Length - i > 2 && tokens[i + 1].Type == RadAsm2Lexer.IDENTIFIER) { if (tokens[i + 2].Type == RadAsm2Lexer.EOL) { var funcDefinition = new FunctionToken(RadAsmTokenType.FunctionName, tokens[i + 1], version); _definitionContainer.Add(currentBlock, funcDefinition); currentBlock = blocks.AppendBlock(new FunctionBlock(currentBlock, BlockType.Function, token, version, funcDefinition)); currentBlock.SetStart(tokens[i + 1].GetEnd(version)); i += 1; } else if (tokens[i + 2].Type == RadAsm2Lexer.LPAREN) { var funcDefinition = new FunctionToken(RadAsmTokenType.FunctionName, tokens[i + 1], version); _definitionContainer.Add(currentBlock, funcDefinition); currentBlock = blocks.AppendBlock(new FunctionBlock(currentBlock, BlockType.Function, token, version, funcDefinition)); parserState = ParserState.SearchArguments; parenthCnt = 1; i += 2; } } } else if (token.Type == RadAsm2Lexer.IF) { currentBlock = blocks.AppendBlock(new Block(currentBlock, BlockType.Condition, token, version)); searchInCondition = true; } else if (token.Type == RadAsm2Lexer.ELSIF || token.Type == RadAsm2Lexer.ELSE) { if (tokens.Length > 2) { currentBlock.SetEnd(tokens[i - 1].Start.GetPosition(version), token, version); _definitionContainer.ClearScope(currentBlock); currentBlock = currentBlock.GetParent(); currentBlock = blocks.AppendBlock(new Block(currentBlock, BlockType.Condition, token, version)); searchInCondition = true; } } else if (token.Type == RadAsm2Lexer.FOR || token.Type == RadAsm2Lexer.WHILE) { currentBlock = blocks.AppendBlock(new Block(currentBlock, BlockType.Loop, token, version)); searchInCondition = true; } else if (token.Type == RadAsm2Lexer.END) { if (currentBlock.Type == BlockType.Function || currentBlock.Type == BlockType.Condition || currentBlock.Type == BlockType.Loop) { currentBlock.SetEnd(token.GetEnd(version), token, version); _definitionContainer.ClearScope(currentBlock); currentBlock = currentBlock.GetParent(); } } else if (token.Type == RadAsm2Lexer.REPEAT) { currentBlock = blocks.AppendBlock(new Block(currentBlock, BlockType.Repeat, token, version)); searchInCondition = true; } else if (token.Type == RadAsm2Lexer.UNTIL) { if (currentBlock.Type == BlockType.Repeat) { currentBlock.SetEnd(token.GetEnd(version), token, version); _definitionContainer.ClearScope(currentBlock); currentBlock = currentBlock.GetParent(); } } else if (token.Type == RadAsm2Lexer.VAR) { if (tokens.Length - i > 1 && tokens[i + 1].Type == RadAsm2Lexer.IDENTIFIER) { var variableDefinition = (tokens.Length - i > 3 && tokens[i + 2].Type == RadAsm2Lexer.EQ && tokens[i + 3].Type == RadAsm2Lexer.CONSTANT) ? new VariableToken(currentBlock.Type == BlockType.Root ? RadAsmTokenType.GlobalVariable : RadAsmTokenType.LocalVariable, tokens[i + 1], version, tokens[i + 3]) : new VariableToken(currentBlock.Type == BlockType.Root ? RadAsmTokenType.GlobalVariable : RadAsmTokenType.LocalVariable, tokens[i + 1], version); _definitionContainer.Add(currentBlock, variableDefinition); currentBlock.AddToken(variableDefinition); } } else if (token.Type == RadAsm2Lexer.IDENTIFIER) { var tokenText = token.GetText(version); if (!TryAddInstruction(tokenText, token, currentBlock, version, Instructions) && !TryAddReference(tokenText, token, currentBlock, version, cancellation)) { _referenceCandidates.AddLast((tokenText, token, currentBlock)); } } else if (token.Type == RadAsm2Lexer.PP_INCLUDE) { if (tokens.Length - i > 1 && tokens[i + 1].Type == RadAsm2Lexer.STRING_LITERAL) { await AddExternalDefinitionsAsync(document.Path, tokens[i + 1], version, currentBlock); i += 1; } } } else if (parserState == ParserState.SearchArguments) { if (token.Type == RadAsm2Lexer.LPAREN) { parenthCnt++; } else if (token.Type == RadAsm2Lexer.RPAREN) { if (--parenthCnt == 0) { currentBlock.SetStart(tokens[i].GetEnd(version)); parserState = ParserState.SearchInScope; } } else if (token.Type == RadAsm2Lexer.IDENTIFIER) { var parameterDefinition = new DefinitionToken(RadAsmTokenType.FunctionParameter, token, version); _definitionContainer.Add(currentBlock, parameterDefinition); currentBlock.AddToken(parameterDefinition); } } } foreach (var(text, trackingToken, block) in _referenceCandidates) { if (!TryAddReference(text, trackingToken, block, version, cancellation) && OtherInstructions.Contains(text)) { errors.Add(new ErrorToken(trackingToken, version, ErrorMessages.InvalidInstructionSetErrorMessage)); } } return(new ParserResult(blocks, errors)); }
public Task <IParserResult> RunAsync(IDocument document, ITextSnapshot version, ITokenizerCollection <TrackingToken> trackingTokens, CancellationToken cancellation) { var definitions = new Dictionary <string, DefinitionToken>(); IBlock rootBlock = new Block(); var blocks = new List <IBlock>() { rootBlock }; var tokens = trackingTokens .Where(t => t.Type != RadAsmDocLexer.WHITESPACE && t.Type != RadAsmDocLexer.BLOCK_COMMENT) .AsParallel() .AsOrdered() .WithCancellation(cancellation) .ToArray(); InstructionToken currentInstruction = null; for (int i = 0; i < tokens.Length; i++) { cancellation.ThrowIfCancellationRequested(); var token = tokens[i]; if (token.Type == RadAsmDocLexer.LET) { if (tokens.Length - i > 1 && tokens[i + 1].Type == RadAsmDocLexer.IDENTIFIER) { var definition = new VariableToken(RadAsmTokenType.GlobalVariable, tokens[i + 1], version); definitions.Add(definition.GetText(), definition); i += 1; } } else if (tokens.Length - i > 1 && token.Type == RadAsmDocLexer.EOL && tokens[i + 1].Type == RadAsmDocLexer.IDENTIFIER) { currentInstruction = new InstructionToken(RadAsmTokenType.Instruction, tokens[i + 1], version); rootBlock.AddToken(currentInstruction); i += 1; } else if (token.Type == RadAsmDocLexer.IDENTIFIER) { var text = token.GetText(version); if (definitions.TryGetValue(text, out var definition)) { var reference = new ReferenceToken(RadAsmTokenType.GlobalVariableReference, token, version, definition); rootBlock.AddToken(reference); currentInstruction?.Parameters.Add(reference); } else if (currentInstruction != null) { var parameter = new AnalysisToken(RadAsmTokenType.GlobalVariableReference, token, version); currentInstruction.Parameters.Add(parameter); } } else if (token.Type == RadAsmDocLexer.LCURVEBRACKET || token.Type == RadAsmDocLexer.EOL) { currentInstruction = null; } } var result = new ParserResult(blocks, new List <IErrorToken>()); return(Task.FromResult((IParserResult)result)); }