private void WriteToStream(PreprocessorLine line) { foreach (Token token in line.TokenList) { _outputStream.Write(token.Value); } }
private void TraceSkippedLine(PreprocessorLine line) { if (_options.Trace) { Trace(string.Format("Skipped: {0}", line.DisplayLine)); } }
/// <summary> /// When two quoted strings appear directly next to each other then make them one /// quoted string /// </summary> /// <param name="line"></param> /// <remarks></remarks> private void CollapseAdjacentQuoteStrings(PreprocessorLine line) { List <Token> list = line.TokenList; // Loop for more Int32 index = 0; while (index < list.Count) { if (!list[index].IsQuotedString) { index += 1; continue; } // Found a quoted string, search for a partner Int32 nextIndex = index + 1; Token nextToken = null; bool done = false; while (nextIndex < list.Count && !done) { switch (list[nextIndex].TokenType) { case TokenType.WhiteSpace: case TokenType.NewLine: nextIndex += 1; break; case TokenType.QuotedStringAnsi: case TokenType.QuotedStringUnicode: nextToken = list[nextIndex]; done = true; break; default: done = true; break; } } if (nextToken != null) { // Create the new token string first = list[index].Value; string second = nextToken.Value; string str = "\"" + first.Substring(1, first.Length - 2) + second.Substring(1, second.Length - 2) + "\""; // Remove all of the tokens between these two and the second string list.RemoveRange(index, (nextIndex - index) + 1); list.Insert(index, new Token(TokenType.QuotedStringAnsi, str)); } else { index += 1; } } }
/// <summary> /// Chew through the current conditional branch. Stop when the next valid /// preprocessor branch is encountered. This will chew through any nestede branches /// with no regard for their content /// /// When this method is finished, a valid pre-processor line will be the next available /// line /// </summary> /// <remarks></remarks> private void ChewThroughConditionalBranch() { bool done = false; int nestedIfCount = 0; while (!done) { PreprocessorLine line = this.PeekNextLine(); if (line.FirstValidToken.TokenType == TokenType.EndOfStream) { return; } TokenType type = line.FirstValidToken.TokenType; if (nestedIfCount == 0) { // Not in a nested if, just look for the next valid preprocessor token switch (type) { case TokenType.PoundElse: case TokenType.PoundElseIf: case TokenType.PoundEndIf: done = true; break; case TokenType.PoundIf: case TokenType.PoundIfndef: nestedIfCount = +1; break; } } else { switch (type) { case TokenType.PoundIf: case TokenType.PoundIfndef: nestedIfCount += 1; break; case TokenType.PoundEndIf: nestedIfCount -= 1; break; } } // If we're not done yet then chew through the line if (!done) { TraceSkippedLine(line); this.GetNextLine(); } } }
/// <summary> /// Called when a define token is hit /// </summary> /// <remarks></remarks> private void ProcessPoundDefine(PreprocessorLine line) { // Get the non whitespace tokens List <Token> list = line.GetValidTokens(); ThrowIfFalse(list[0].TokenType == TokenType.PoundDefine); Macro macro = null; if (list.Count == 3 && list[1].TokenType == TokenType.Word) { string name = list[1].Value; macro = new Macro(name, list[2].Value); } else if (list.Count == 2 && list[1].TokenType == TokenType.Word) { string name = list[1].Value; macro = new Macro(name, string.Empty); } else if (list.Count == 1) { _scanner.AddWarning("Encountered an empty #define"); } else if (list.Count > 3 && list[1].TokenType == TokenType.Word && list[2].TokenType == TokenType.ParenOpen) { macro = ProcessPoundDefineMethod(line); } else { macro = ProcessPoundDefineComplexMacro(line); } if (macro != null) { Macro oldMacro = null; if (_macroMap.TryGetValue(macro.Name, out oldMacro) && oldMacro.IsPermanent) { TraceToStream("Kept: {0} -> {1} Attempted Value {2}", oldMacro.Name, oldMacro.Value, macro.Value); } else { _macroMap[macro.Name] = macro; if (macro.IsMethod) { MethodMacro method = (MethodMacro)macro; TraceToStream("Defined: {0} -> {1}", macro.Name, method.MethodSignature); } else { TraceToStream("Defined: {0} -> {1}", macro.Name, macro.Value); } } } }
/// <summary> /// Called when an #ifndef is encountered /// </summary> /// <param name="line"></param> /// <remarks></remarks> private void ProcessPoundIfndef(PreprocessorLine line) { bool isCondTrue = EvalauteConditional(line); TraceToStream("{0}: {1}", isCondTrue, line.DisplayLine); if (!isCondTrue) { // Start a processing loop this.ProcessLoop(); } else { ProcessConditionalRemainder(); } }
/// <summary> /// Peek at the next line /// </summary> /// <returns></returns> /// <remarks></remarks> private PreprocessorLine PeekNextLine() { ScannerMark mark = _scanner.Mark(); PreprocessorLine line = null; try { line = GetNextLine(); } finally { _scanner.Rollback(mark); } return(line); }
/// <summary> /// Chew completely through the remainder of the conditional. Basically consume /// the #endif line to match the #if/#elsif we've already processed /// </summary> /// <remarks></remarks> private void ChewThroughConditionalEnd() { bool done = false; while (!done) { ChewThroughConditionalBranch(); PreprocessorLine line = this.GetNextLine(); TraceSkippedLine(line); if (line.FirstValidToken.TokenType == TokenType.PoundEndIf) { done = true; } } }
/// <summary> /// Called when a #if is encountered. If the condition is true, /// it will eat the #if line and let parsing continue. Otherwise /// it will chew until it hits the branch that should be processed /// or it hits the #endif /// </summary> /// <remarks></remarks> private void ProcessPoundIf(PreprocessorLine line) { // The object here is to find the branch of the conditional that should // be processed bool isCondTrue = EvalauteConditional(line); TraceToStream("{0}: {1}", isCondTrue, line.DisplayLine); if (isCondTrue) { // Start another processing loop this.ProcessLoop(); } else { ProcessConditionalRemainder(); } }
/// <summary> /// Called for a #undef line /// </summary> /// <param name="line"></param> /// <remarks></remarks> private void ProcessPoundUndefine(PreprocessorLine line) { // Get the none whitespace tokens List <Token> list = line.GetValidTokens(); ThrowIfFalse(list[0].TokenType == TokenType.PoundUnDef); if (list.Count != 2 || list[1].TokenType != TokenType.Word) { _scanner.AddWarning("Error processing #undef"); } else { string name = list[1].Value; if (_macroMap.ContainsKey(name)) { _macroMap.Remove(name); TraceToStream("Undefined: {0}", name); } } }
private void CollapseExpandedPreprocessorLines(ref PreprocessorLine line) { if (line.FirstValidToken != null && line.FirstValidToken.TokenType == TokenType.Pound) { List <Token> list = line.GetValidTokens(); Token possibleToken = list[1]; Token poundToken = null; if (list.Count >= 2 && TokenHelper.TryConvertToPoundToken(possibleToken.Value, out poundToken)) { // Strip out everything # -> define List <Token> newList = new List <Token>(line.TokenList); bool done = false; while (!done) { if (newList.Count == 0) { Debug.Fail("Non-crititcal error reducing the preprocessor line"); return; } else if (object.ReferenceEquals(newList[0], possibleToken)) { newList.RemoveAt(0); newList.Insert(0, poundToken); done = true; } else { newList.RemoveAt(0); } } PreprocessorLine formattedLine = new PreprocessorLine(); formattedLine.FirstValidToken = poundToken; formattedLine.IsPreProcessorDirectiveLine = true; formattedLine.TokenList = newList; line = formattedLine; } } }
/// <summary> /// Called when the #if branch of a conditional is not true. Processes the branch /// </summary> /// <remarks></remarks> private void ProcessConditionalRemainder() { bool done = false; while (!done) { // It's possible to have unmatched #if blocks. If we hit the end of the stream this means // it is unbalanced so throw an exception if (_scanner.EndOfStream) { throw new PreProcessorException("Found unbalanced conditional preprocessor branch"); } // Look at the next branch ChewThroughConditionalBranch(); PreprocessorLine cur = this.GetNextLine(); switch (cur.FirstValidToken.TokenType) { case TokenType.PoundElse: // Start another processing loop this.ProcessLoop(); done = true; break; case TokenType.PoundElseIf: if (EvalauteConditional(cur)) { this.ProcessLoop(); done = true; } break; case TokenType.PoundEndIf: done = true; break; } } }
/// <summary> /// Evaluate a preprocessor conditional statement and return whether or not it is true /// </summary> /// <param name="line"></param> /// <returns></returns> /// <remarks></remarks> private bool EvalauteConditional(PreprocessorLine line) { List <Token> list = line.GetValidTokens(); // Remove the #pound token. We don't care what type of conditional this is, this just serves // to evaluate it and let the caller interpret the result list.RemoveAt(0); // Make sure that all "defined" expressions wrap the next value in () Int32 i = 0; while (i + 1 < list.Count) { Token cur = list[i]; if (cur.TokenType == TokenType.Word && 0 == string.CompareOrdinal("defined", cur.Value) && list[i + 1].TokenType == TokenType.Word) { list.Insert(i + 1, new Token(TokenType.ParenOpen, "(")); list.Insert(i + 3, new Token(TokenType.ParenClose, ")")); i += 3; } i += 1; } ExpressionValue value = null; if (!_eval.TryEvaluate(list, out value)) { _errorProvider.AddError("Could not evaluate expression {0}", line.ToString()); return(false); } return(value.ConvertToBool()); }
/// <summary> /// Core processing loop. Processes blocks of text. /// </summary> /// <remarks></remarks> private void ProcessLoop() { bool done = false; while (!done) { ScannerMark mark = _scanner.Mark(); try { PreprocessorLine line = this.GetNextLine(); ThrowIfFalse(line.TokenList.Count > 0); Token token = line.FirstValidToken; if (token == null) { WriteToStream(line); continue; } switch (token.TokenType) { case TokenType.PoundIf: ProcessPoundIf(line); break; case TokenType.PoundIfndef: ProcessPoundIfndef(line); break; case TokenType.PoundElse: case TokenType.PoundElseIf: // stop on a conditional branch end ChewThroughConditionalEnd(); done = true; break; case TokenType.EndOfStream: case TokenType.PoundEndIf: done = true; break; case TokenType.PoundPragma: ProcessPoundPragma(line); break; case TokenType.PoundDefine: ProcessPoundDefine(line); break; case TokenType.PoundUnDef: ProcessPoundUndefine(line); break; case TokenType.PoundInclude: ProcessPoundInclude(line); break; default: WriteToStream(line); break; } } catch (PreProcessorException ex) { if (ex.IsError) { _errorProvider.AddError(ex.Message); } else { _errorProvider.AddWarning(ex.Message); } _scanner.Rollback(mark); GetNextLine(); // Chew through the line } } }
/// <summary> /// Process a #pragma statement. /// </summary> /// <param name="line"></param> /// <remarks></remarks> private void ProcessPoundPragma(PreprocessorLine line) { // We don't support #pragma at this point // In particular #pragma push_macro and #pragma pop_macro }
/// <summary> /// Process a #include line. These take typically two forms /// #include "foo.h" /// #include >foo.h> /// /// </summary> /// <param name="line"></param> /// <remarks></remarks> private void ProcessPoundInclude(PreprocessorLine line) { if (!_options.FollowIncludes) { return; } // if the user did a <> include then there won't be any quotes around the string // so go ahead and redo the include to look like a "filename.h" include List <Token> list = new List <Token>(line.GetValidTokens()); // Get rid of the #include ThrowIfFalse(list[1].TokenType == TokenType.PoundInclude); list.RemoveAt(0); string name = null; if (list[1].TokenType == TokenType.OpLessThan) { name = string.Empty; list.RemoveAt(0); while (list[1].TokenType != TokenType.OpGreaterThan) { name += list[1].Value; list.RemoveAt(0); } list.RemoveAt(0); } else if (list[1].IsQuotedString) { name = TokenHelper.ConvertToString(list[1]); } else { name = null; } if (name == null) { _scanner.AddWarning("Invalid #include statement"); return; } // Now actually try and find the file. First check the custom list bool found = false; if (File.Exists(name)) { found = true; TraceToStream("include {0} followed -> {0}", name); TraceToStream("include {0} start", name); using (StreamReader reader = new StreamReader(name)) { ProcessCore(new TextReaderBag(name, reader)); } TraceToStream("include {0} end", name); } else if (_options.IncludePathList.Count > 0) { // Search through the path list found = false; foreach (string prefix in _options.IncludePathList) { string fullPath = Path.Combine(prefix, name); if (File.Exists(fullPath)) { found = true; TraceToStream("include {0} followed -> {1}", name, fullPath); TraceToStream("include {0} start", name); using (StreamReader reader = new StreamReader(fullPath)) { ProcessCore(new TextReaderBag(fullPath, reader)); } TraceToStream("include {0} end", name); break; } } } else { found = false; } if (!found) { _scanner.AddWarning("Could not locate include file {0}", name); TraceToStream("include {0} not followed", name); } }
/// <summary> /// Get the next line of tokens /// </summary> /// <returns></returns> /// <remarks></remarks> private PreprocessorLine GetNextLine() { PreprocessorLine line = new PreprocessorLine(); line.TokenList = new List <Token>(); Token lastValidToken = null; bool done = false; while (!done) { Token token = _scanner.GetNextToken(); line.TokenList.Add(token); bool isValid = false; if (token.TokenType == TokenType.NewLine) { // Check and see if this is a preprocessor directive token that ends with a // backslash. If so then remove the backslash from the stream and continue processing // the line if (lastValidToken != null && lastValidToken.TokenType == TokenType.BackSlash) { isValid = false; line.TokenList.Remove(lastValidToken); lastValidToken = null; } else { done = true; isValid = true; } } else if (token.TokenType == TokenType.EndOfStream) { done = true; isValid = true; // simulate a newline token line.TokenList.RemoveAt(line.TokenList.Count - 1); line.TokenList.Add(new Token(TokenType.NewLine, Environment.NewLine)); } else if (token.TokenType != TokenType.WhiteSpace) { isValid = true; } else { isValid = false; } if (isValid) { lastValidToken = token; if (line.FirstValidToken == null) { line.FirstValidToken = token; // See if this is a preprocessor line if (token.IsPreProcessorDirective) { line.IsPreProcessorDirectiveLine = true; } } } } // This should always have at least one valid token ThrowIfNull(line.FirstValidToken); // Check and see if the line looks like the following. If so convert it to a valid pre-processor line // # define foo CollapseExpandedPreprocessorLines(ref line); // If this is not a preprocessor directive line then we need to substitute all of the // #define'd tokens in the stream if (!line.IsPreProcessorDirectiveLine || (line.FirstValidToken != null && line.FirstValidToken.TokenType == TokenType.PoundInclude)) { ReplaceDefinedTokens(line); } // Collapse quoted strings that are adjacent to each other CollapseAdjacentQuoteStrings(line); return(line); }
private Macro ProcessPoundDefineComplexMacro(PreprocessorLine line) { // It's a complex macro. Go ahead and get the line information List <Token> list = new List <Token>(line.TokenList); int i = 0; // Strip the newlines while (i < list.Count) { if (list[i].TokenType == TokenType.NewLine) { list.RemoveAt(i); } else { i += 1; } } i = 0; // Get the #define token Token defineToken = null; while (i < list.Count) { if (list[i].TokenType == TokenType.PoundDefine) { defineToken = list[i]; break; } i += 1; } // Get the name token Token nameToken = null; while (i < list.Count) { if (list[i].TokenType == TokenType.Word) { nameToken = list[i]; break; } i += 1; } if (defineToken == null || nameToken == null) { _errorProvider.AddWarning("Error processing line: {0}", line.ToString()); return(new Macro(NativeSymbolBag.GenerateAnonymousName(), string.Empty)); } // i now points to the name token. Remove the range of tokens up until this point. Now remove the // whitespace on either end of the list list.RemoveRange(0, i + 1); while (list.Count > 0 && (list[0].TokenType == TokenType.WhiteSpace || list[0].TokenType == TokenType.NewLine)) { list.RemoveAt(0); } while (list.Count > 0 && (list[list.Count - 1].TokenType == TokenType.WhiteSpace || list[list.Count - 1].TokenType == TokenType.NewLine)) { list.RemoveAt(list.Count - 1); } // Create a string for all of the tokens var b = new StringBuilder(); foreach (Token cur in list) { b.Append(cur.Value); } return(new Macro(nameToken.Value, b.ToString())); }
/// <summary> /// Process a #define that is actually a function /// </summary> /// <param name="line"></param> /// <returns></returns> /// <remarks></remarks> private Macro ProcessPoundDefineMethod(PreprocessorLine line) { // First step is to parse out the name and parameters List <Token> list = line.GetValidTokens(); string name = list[1].Value; list.RemoveRange(0, 3); List <string> paramList = new List <string>(); while ((list[0].TokenType != TokenType.ParenClose)) { if (list[0].TokenType == TokenType.Word) { paramList.Add(list[0].Value); } else if (list[0].TokenType == TokenType.ParenOpen) { // ( is not legal inside a parameter list. This is a simple macro return(ProcessPoundDefineComplexMacro(line)); } list.RemoveAt(0); } // Now get the fullBody. We need the actual text for the fullBody so search through the true token list Int32 index = 0; while ((line.TokenList[index].TokenType != TokenType.ParenClose)) { index += 1; } index += 1; List <Token> fullBody = line.TokenList.GetRange(index, line.TokenList.Count - index); // Strip the trailing and ending whitespace on the fullBody while (fullBody.Count > 0 && (fullBody[0].TokenType == TokenType.WhiteSpace || fullBody[0].TokenType == TokenType.NewLine)) { fullBody.RemoveAt(0); } // Don't be fooled by a simple #define that simply wraps the entire fullBody inside a // set of (). if ((fullBody.Count == 0)) { return(ProcessPoundDefineComplexMacro(line)); } while (fullBody.Count > 0 && (fullBody[fullBody.Count - 1].TokenType == TokenType.WhiteSpace || fullBody[fullBody.Count - 1].TokenType == TokenType.NewLine)) { fullBody.RemoveAt(fullBody.Count - 1); } // Coy the body token list since we are about to change the data List <Token> body = new List <Token>(fullBody); // Collapse the whitespace around ## entries int i = 0; while (i + 1 < body.Count) { Token left = body[i]; Token right = body[i + 1]; if (left.TokenType == TokenType.Pound && right.TokenType == TokenType.Pound) { // First look at the right if (i + 2 < body.Count && body[i + 2].TokenType == TokenType.WhiteSpace) { body.RemoveAt(i + 2); } // Now look at the left if (i > 0 && body[i - 1].TokenType == TokenType.WhiteSpace) { body.RemoveAt(i - 1); } } i += 1; } index += 1; return(new MethodMacro(name, paramList, body, fullBody)); }
private void ReplaceDefinedTokens(PreprocessorLine line) { ThrowIfNull(line); int i = 0; List <Token> list = line.TokenList; while ((i < list.Count)) { Token token = list[i]; if (token.TokenType != TokenType.Word) { i += 1; continue; } Macro macro = null; if (_macroMap.TryGetValue(token.Value, out macro)) { // Remove the original token list.RemoveAt(i); List <Token> replaceList = null; if (macro.IsMethod) { MethodMacro method = (MethodMacro)macro; List <Token> args = ParseAndRemoveMacroMethodArguments(list, i); if (args == null) { // Parse did not succeed, move to the next token i += 1; } else { // Insert the tokens replaceList = ReplaceMethodMacro(method, args); } } else { // Use the scanner to create the replacement tokens replaceList = Scanner.TokenizeText(macro.Value, CreateScannerOptions()); } if (replaceList != null) { CollapseDoublePounds(replaceList); list.InsertRange(i, replaceList); } } else { i += 1; } } // Do one more pass to check and see if we need a recursive replace bool needAnotherPass = false; foreach (Token cur in line.TokenList) { if (cur.TokenType == TokenType.Word && _macroMap.ContainsKey(cur.Value)) { needAnotherPass = true; break; } } if (needAnotherPass) { ReplaceDefinedTokens(line); } }
/// <summary> /// Process a #pragma statement. /// </summary> /// <param name="line"></param> /// <remarks></remarks> private void ProcessPoundPragma(PreprocessorLine line) { // We don't support #pragma at this point }