/// <summary> /// Include files /// </summary> /// <param name="pars"></param> public void Visit(ParsedIncludeFile pars) { var inc = new IncludeCodeItem { DisplayText = pars.Name, Flags = pars.Flags, SubText = null, DocumentOwner = pars.FilePath, GoToLine = pars.Line, GoToColumn = pars.Column }; if (pars.Flags.HasFlag(ParseFlag.NotFound)) { PushToCodeExplorer( GetExplorerListNode("Missing includes/variables", CodeExplorerIconType.MissingInclude), inc); } else { // To code explorer PushToCodeExplorer( GetExplorerListNode("Includes", CodeExplorerIconType.Include), inc); } }
/// <summary> /// Returns the list of tokens corresponding to the given include /// </summary> private List <Token> GetIncludeFileTokens(TokenString previousTokenString, ParsedIncludeFile parsedInclude) { // Parse the include file ? if (!string.IsNullOrEmpty(parsedInclude.IncludeFilePath)) { ProTokenizer proTokenizer; // did we already parsed this file in a previous parse session? if (SavedTokenizerInclude.ContainsKey(parsedInclude.IncludeFilePath)) { proTokenizer = SavedTokenizerInclude[parsedInclude.IncludeFilePath]; } else { // Parse it if (previousTokenString != null && !string.IsNullOrEmpty(previousTokenString.Value)) { proTokenizer = new ProTokenizer(Utils.ReadAllText(parsedInclude.IncludeFilePath), previousTokenString.Value[0] == '"', previousTokenString.Value[0] == '\''); } else { proTokenizer = new ProTokenizer(Utils.ReadAllText(parsedInclude.IncludeFilePath)); } if (!SavedTokenizerInclude.ContainsKey(parsedInclude.IncludeFilePath)) { SavedTokenizerInclude.Add(parsedInclude.IncludeFilePath, proTokenizer); } } _parsedIncludes.Add(parsedInclude); var includeNumber = (ushort)(_parsedIncludes.Count - 1); // add this include to the references and modify each token // Remove EOF List <Token> copiedTokens = new List <Token>(); for (int i = 0; i < proTokenizer.GetTokensList.Count - 1; i++) { var token = proTokenizer.GetTokensList[i]; var copiedToken = token.Copy(token.Line, token.Column, token.StartPosition, token.EndPosition); copiedToken.OwnerNumber = includeNumber; copiedTokens.Add(copiedToken); } return(copiedTokens); } parsedInclude.Flags |= ParseFlag.NotFound; return(null); }
public void Visit(ParsedIncludeFile pars) { AppendEverything(pars); }
/// <summary> /// Create the include reference to be seen in the explorer /// </summary> private ParsedIncludeFile CreateParsedIncludeFile(Token bracketToken, int startPos, int endPos) { // info we will extract from the current statement : ParsedIncludeFile newInclude = null; string fileName = ""; bool usesNamedArg = false;// true if the arguments used are with the format : &name="" bool expectingFirstArg = true; string argName = null; int argNumber = 1; var parameters = new Dictionary <string, string>(StringComparer.CurrentCultureIgnoreCase); var state = 0; for (int i = startPos + 1; i <= endPos - 1; i++) { var token = PeekAt(i); if (token is TokenComment) { continue; } switch (state) { case 0: // read the file name if (token is TokenWord || token is TokenString) { fileName += GetTokenStrippedValue(token); state++; } break; case 1: if (token is TokenSymbol && (token.Value.Equals("/") || token.Value.Equals("\\"))) { // it's a path, append it to the name of the run fileName += token.Value; state = 0; break; } // read the arguments if (expectingFirstArg) { // case of a {file.i &x="arg1" &x=arg2} if (token is TokenPreProcDirective) { argName = token.Value; usesNamedArg = true; expectingFirstArg = false; // case of a {file.i "arg1" arg2} } else if (!(token is TokenEol || token is TokenWhiteSpace)) { if (!parameters.ContainsKey(argNumber.ToString())) { parameters.Add(argNumber.ToString(), GetTokenStrippedValue(token)); } else { parameters[argNumber.ToString()] = GetTokenStrippedValue(token); } argNumber++; expectingFirstArg = false; } } else { if (usesNamedArg) { // still waiting to read the argument name if (argName == null) { if (token is TokenPreProcDirective) { argName = token.Value; } } else if (!(token is TokenEol || token is TokenWhiteSpace || token.Value == "=")) { if (!parameters.ContainsKey(argName)) { parameters.Add(argName, GetTokenStrippedValue(token)); } else { parameters[argName] = GetTokenStrippedValue(token); } argName = null; } } else if (!(token is TokenEol || token is TokenWhiteSpace)) { if (!parameters.ContainsKey(argNumber.ToString())) { parameters.Add(argNumber.ToString(), GetTokenStrippedValue(token)); } argNumber++; } } break; } } // we matched the include file name if (!string.IsNullOrEmpty(fileName)) { // try to find the file in the propath var fullFilePath = FindIncludeFullPath(fileName); newInclude = new ParsedIncludeFile(fileName, bracketToken, parameters, fullFilePath ?? fileName, _parsedIncludes[bracketToken.OwnerNumber]); AddParsedItem(newInclude, bracketToken.OwnerNumber); } return(newInclude); }
/// <summary> /// If it is an {&var} or {include.i}, replaces the token at "current position + posAhead" by a list of tokens. /// In case of {&var}, the list of tokens has been extracted when we found the GLOBAL/SCOPE-DEFINE. /// In case of {include.i}, we will read the file and tokenize it, then we will have the list of token we need to replace. /// </summary> private bool ReplaceIncludeAndPreprocVariablesAhead(int posAhead) { if (_context.InFalsePreProcIfBlock) { return(false); } /* * { include-file * [ argument ... | {&argument-name = "argument-value" } ... ] } * * This method should handle those cases : * { file.i &name=val &2="value" } -> {&name} and {&2} * {file.i val "value"} -> {1} {2} * * { &preprocessor-name } * { { n | &argument-name } } */ bool weReplacedSomething = false; // we check if the token + posAhead will be an include that needs to be replaced var toReplaceToken = PeekAt(posAhead); byte nbOfNestedReplacements = 0; while (toReplaceToken is TokenInclude || toReplaceToken is TokenPreProcVariable) { // a ~ is the escape character for an include or a pre-proc variable var previousToken = PeekAt(posAhead - 1); if (previousToken is TokenSymbol && previousToken.Value.Equals("~")) { break; } // replace the {include} present within this {include} var count = 1; while (true) { var curToken = PeekAt(posAhead + count); if (curToken is TokenEof) { return(false); // we didn't match the end of the include, better not do anything } if (curToken is TokenSymbol && curToken.Value == "}") { break; } if (!ReplaceIncludeAndPreprocVariablesAhead(posAhead + count)) { count++; } } count++; // number of tokens composing this include // get the characteristics of this include string replaceName = null; ParsedIncludeFile parsedInclude = null; string preprocValue = null; if (count >= 3) { if (toReplaceToken is TokenInclude) { parsedInclude = CreateParsedIncludeFile(toReplaceToken, posAhead, posAhead + count - 1); if (parsedInclude != null) { replaceName = !string.IsNullOrEmpty(parsedInclude.IncludeFilePath) ? parsedInclude.IncludeFilePath : parsedInclude.Name; } } else { replaceName = (toReplaceToken.Value == "{" ? "" : "&") + PeekAt(posAhead + 1).Value; preprocValue = CreateUsedPreprocVariable(toReplaceToken, replaceName); } } // remove the tokens composing this include RemoveTokens(posAhead, count); weReplacedSomething = true; // name not found, leave if (string.IsNullOrEmpty(replaceName)) { break; } // Handle the compiler message: Nested recursion limit of 255 exceeded in preprocessor expansion. (2936) // It happens when: // &GLOBAL-DEFINE name1 ~{&name2} // &GLOBAL-DEFINE name2 ~{&name1} // {&name1} if (++nbOfNestedReplacements >= 255) { break; } var prevToken = PeekAt(posAhead - 1); // get the list of tokens that will replace this include List <Token> valueTokens; if (toReplaceToken is TokenInclude) { // Handle the compiler message: Nested recursion limit of 255 exceeded in preprocessor expansion. (2936) // This will happen if you have an include that calls itself for instance var includeStackSize = 0; var parentInclude = parsedInclude.Parent; while (parentInclude != null) { includeStackSize++; parentInclude = parentInclude.Parent; } if (includeStackSize >= 255) { break; } valueTokens = GetIncludeFileTokens(prevToken as TokenString, parsedInclude); } else { valueTokens = GetPreProcVariableTokens(prevToken as TokenString, toReplaceToken, preprocValue); } // do we have a definition for the var/include? if (valueTokens != null) { // we have to "merge" the TokenWord at the beginning and end of what we are inserting, this allows to take care of // cases like : DEF VAR lc_{&val}_end AS CHAR NO-UNDO. if (MergeTokenAtPosition(posAhead - 1, valueTokens.FirstOrDefault() as TokenWord, true)) { valueTokens.RemoveAt(0); } if (MergeTokenAtPosition(posAhead, valueTokens.LastOrDefault() as TokenWord, false)) { valueTokens.RemoveAt(valueTokens.Count - 1); } // if we are in this case : MESSAGE "begin{include.i}end". we must try to merge this into one single string prevToken = PeekAt(posAhead - 1); var nextToken = PeekAt(posAhead); if (prevToken is TokenString && nextToken is TokenString) { while (MergeTokenAtPosition(posAhead - 1, valueTokens.FirstOrDefault(), true)) { var weMergedAString = valueTokens.FirstOrDefault() is TokenString; valueTokens.RemoveAt(0); if (weMergedAString) { // to handle the particular case where include.i contains something like : // word". MESSAGE "anothre mess break; } } while (MergeTokenAtPosition(posAhead, valueTokens.LastOrDefault(), false)) { var weMergedAString = valueTokens.LastOrDefault() is TokenString; valueTokens.RemoveAt(valueTokens.Count - 1); if (weMergedAString) { break; } } } } // if we have tokens insert, do it if (valueTokens != null && valueTokens.Count > 0) { InsertTokens(posAhead, valueTokens); } else { // make sure we don't have two TokenWord following each other, or we must merge them if (MergeTokenAtPosition(posAhead - 1, PeekAt(posAhead) as TokenWord, true) || MergeTokenAtPosition(posAhead - 1, PeekAt(posAhead) as TokenString, true)) { RemoveTokens(posAhead, 1); } } toReplaceToken = PeekAt(posAhead); } return(weReplacedSomething); }
/// <summary> /// If it is an {&var} or {include.i}, replaces the token at "current position + posAhead" by a list of tokens. /// In case of {&var}, the list of tokens has been extracted when we found the GLOBAL/SCOPE-DEFINE. /// In case of {include.i}, we will read the file and tokenize it, then we will have the list of token we need to replace. /// </summary> private bool ReplaceIncludeAndPreprocVariablesAhead(int posAhead) { /* * { include-file * [ argument ... | {&argument-name = "argument-value" } ... ] } * * This method should handle those cases : * { file.i &name=val &2="value" } -> {&name} and {&2} * {file.i val "value"} -> {1} {2} * * { &preprocessor-name } * { { n | &argument-name } } */ bool weReplacedSomething = false; // we check if the token + posAhead will be an include that needs to be replaced var toReplaceToken = PeekAt(posAhead); HashSet <string> replacedName = null; // keep track of replacement here while (toReplaceToken is TokenInclude || toReplaceToken is TokenPreProcVariable) { // replace the {include} present within this {include} var count = 1; while (true) { var curToken = PeekAt(posAhead + count); if (curToken is TokenEof) { return(false); // we didn't match the end of the include, better not do anything } if (curToken is TokenSymbol && curToken.Value == "}") { break; } if (!ReplaceIncludeAndPreprocVariablesAhead(posAhead + count)) { count++; } } count++; // number of tokens composing this include // get the caracteristics of this include string replaceName = null; ParsedIncludeFile parsedInclude = null; string preprocValue = null; if (count >= 3) { if (toReplaceToken is TokenInclude) { parsedInclude = CreateParsedIncludeFile(toReplaceToken, posAhead, posAhead + count - 1); if (parsedInclude != null) { replaceName = !string.IsNullOrEmpty(parsedInclude.FullFilePath) ? parsedInclude.FullFilePath : parsedInclude.Name; } } else { replaceName = (toReplaceToken.Value == "{" ? "" : "&") + PeekAt(posAhead + 1).Value; preprocValue = CreateUsedPreprocVariable(toReplaceToken, replaceName); } } // remove the tokens composing this include RemoveTokens(posAhead, count); weReplacedSomething = true; // name not found, leave if (string.IsNullOrEmpty(replaceName)) { break; } // make sure to not replace the same include in the same replacement loop, if we do that // this means we will go into an infinite loop case of a {&{&one}} with one=two and two=one if (replacedName == null) { replacedName = new HashSet <string>(StringComparer.CurrentCultureIgnoreCase) { _parsedIncludes[0].FullFilePath } } ; if (replacedName.Contains(replaceName)) { break; } replacedName.Add(replaceName); var prevToken = PeekAt(posAhead - 1); // get the list of tokens that will replace this include List <Token> valueTokens; if (toReplaceToken is TokenInclude) { valueTokens = GetIncludeFileTokens(prevToken as TokenString, parsedInclude); } else { valueTokens = GetPreProcVariableTokens(prevToken as TokenString, toReplaceToken, preprocValue); } // do we have a definition for the var/include? if (valueTokens != null) { // we have to "merge" the TokenWord at the beginning and end of what we are inserting, this allows to take care of // cases like : DEF VAR lc_{&val}_end AS CHAR NO-UNDO. if (MergeTokenAtPosition(posAhead - 1, valueTokens.FirstOrDefault() as TokenWord, true)) { valueTokens.RemoveAt(0); } if (MergeTokenAtPosition(posAhead, valueTokens.LastOrDefault() as TokenWord, false)) { valueTokens.RemoveAt(valueTokens.Count - 1); } // if we are in this case : MESSAGE "begin{include.i}end". we must try to merge this into one single string prevToken = PeekAt(posAhead - 1); var nextToken = PeekAt(posAhead); if (prevToken is TokenString && nextToken is TokenString) { while (MergeTokenAtPosition(posAhead - 1, valueTokens.FirstOrDefault(), true)) { var weMergedAString = valueTokens.FirstOrDefault() is TokenString; valueTokens.RemoveAt(0); if (weMergedAString) { // to handle the particular case where include.i contains something like : // word". MESSAGE "anothre mess break; } } while (MergeTokenAtPosition(posAhead, valueTokens.LastOrDefault(), false)) { var weMergedAString = valueTokens.LastOrDefault() is TokenString; valueTokens.RemoveAt(valueTokens.Count - 1); if (weMergedAString) { break; } } } } // if we have tokens insert, do it if (valueTokens != null && valueTokens.Count > 0) { InsertTokens(posAhead, valueTokens); } else { // make sure we don't have two TokenWord following each other, or we must merge them if (MergeTokenAtPosition(posAhead - 1, PeekAt(posAhead) as TokenWord, true) || MergeTokenAtPosition(posAhead - 1, PeekAt(posAhead) as TokenString, true)) { RemoveTokens(posAhead, 1); } } toReplaceToken = PeekAt(posAhead); } return(weReplacedSomething); }