internal override int scan(ref PreprocessorToken ppToken) { int token; do { token = pp.ReadToken(mac.body, ppToken); } while (token == ' '); // handle white space in macro // TODO : maybe fixed this original GL issue // TODO: preprocessor: properly handle whitespace (or lack of it) between tokens when expanding if (token == (int)CppEnums.IDENTIFIER) { int i; for (i = mac.argc - 1; i >= 0; i--) { if (mac.args[i] == ppToken.atom) { break; } } if (i >= 0) { pp.pushTokenStreamInput(args[i]); return(pp.scanToken(ref ppToken)); } } if (token == BasePreprocessorInput.END_OF_INPUT) { mac.busy = false; } return(token); }
internal override int scan(ref PreprocessorToken ppToken) { int token; do { token = pp.ReadToken(mac.body, ppToken); } while (token == ' '); // handle white space in macro // TODO : maybe fixed this original GL issue // TODO: preprocessor: properly handle whitespace (or lack of it) between tokens when expanding if (token == (int) CppEnums.IDENTIFIER) { int i; for (i = mac.argc - 1; i >= 0; i--) if (mac.args[i] == ppToken.atom) break; if (i >= 0) { pp.pushTokenStreamInput(args[i]); return pp.scanToken(ref ppToken); } } if (token == BasePreprocessorInput.END_OF_INPUT) mac.busy = false; return token; }
internal override int scan(ref PreprocessorToken ppToken) { if (done) return BasePreprocessorInput.END_OF_INPUT; done = true; return marker; }
internal override int scan(ref PreprocessorToken ppToken) { if (done) { return(BasePreprocessorInput.END_OF_INPUT); } done = true; return(marker); }
internal override int scan(ref PreprocessorToken ppToken) { if (done) return BasePreprocessorInput.END_OF_INPUT; int ret = token; ppToken = lval; done = true; return ret; }
// return a zero, for scanning a macro that was never defined internal override int scan(ref PreprocessorToken ppToken) { if (done) return BasePreprocessorInput.END_OF_INPUT; ppToken.name = "0"; ppToken.ival = 0; ppToken.space = false; done = true; return (int)CppEnums.INTCONSTANT; }
// return a zero, for scanning a macro that was never defined internal override int scan(ref PreprocessorToken ppToken) { if (done) { return(BasePreprocessorInput.END_OF_INPUT); } ppToken.name = "0"; ppToken.ival = 0; ppToken.space = false; done = true; return((int)CppEnums.INTCONSTANT); }
internal override int scan(ref PreprocessorToken ppToken) { if (done) { return(BasePreprocessorInput.END_OF_INPUT); } int ret = token; ppToken = lval; done = true; return(ret); }
/// <summary> /// Sets the program define as string. /// Too slow to use to preamble; bypass normal processing /// Does not support recursion on string; /// </summary> /// <param name="definedName">Defined name.</param> /// <param name="valueString">Value string.</param> public void DefineAs(string definedName, string valueString) { int atom = Atoms.LookUpAddString (definedName); Symbol result = null; if(!mSymbols.TryGetValue(atom, out result)) { result = this.Add (atom); } result.mac = new MacroSymbol (); result.mac.body = new TokenStream (); var packet = new PreprocessorToken (); packet.atom = Atoms.LookUpAddString (valueString); Atoms.RecordToken (result.mac.body, (int) CppEnums.STRCONSTANT, packet); }
/// <summary> /// Sets the program define as string. /// Too slow to use to preamble; bypass normal processing /// Does not support recursion on string; /// </summary> /// <param name="definedName">Defined name.</param> /// <param name="valueString">Value string.</param> public void AddPasteToken(string definedName, string valueString) { int atom = Atoms.LookUpAddString(definedName); Symbol result = null; if (!mSymbols.TryGetValue(atom, out result)) { result = this.Add(atom); } result.mac = new MacroSymbol(); result.mac.body = new TokenStream(); var packet = new PreprocessorToken(); packet.atom = Atoms.LookUpAddString(valueString); Atoms.RecordToken(result.mac.body, (int)CppEnums.TOKEN_PASTE, packet); }
/// <summary> /// Sets the program define as int. /// Too slow to use to preamble; bypass normal processing /// </summary> /// <param name="definedName">Defined name.</param> /// <param name="value">Value.</param> public void DefineAs(string definedName, int value) { int atom = Atoms.LookUpAddString(definedName); Symbol result = null; if (!mSymbols.TryGetValue(atom, out result)) { result = this.Add(atom); } result.mac = new MacroSymbol(); result.mac.body = new TokenStream(); var packet = new PreprocessorToken(); packet.name = value.ToString(); Atoms.RecordToken(result.mac.body, (int)CppEnums.INTCONSTANT, packet); }
/// <summary> /// Add a token to the end of a list for later playback. /// </summary> /// <param name="pTok">P tok.</param> /// <param name="token">Token.</param> /// <param name="ppToken">Pp token.</param> internal void RecordToken(TokenStream pTok, int token, PreprocessorToken ppToken) { if (token > 256) { pTok.lAddByte((UInt16)((token & 0x7f) + 0x80)); } else { pTok.lAddByte((UInt16)(token & 0x7f)); } switch (token) { case (int)CppEnums.IDENTIFIER: case (int)CppEnums.STRCONSTANT: string s = GetAtomString(ppToken.atom); foreach (var letter in s) { pTok.lAddByte((UInt16)letter); } pTok.lAddByte(0); break; case (int)CppEnums.INTCONSTANT: case (int)CppEnums.UINTCONSTANT: case (int)CppEnums.FLOATCONSTANT: case (int)CppEnums.DOUBLECONSTANT: string str = ppToken.name; foreach (var letter in str) { pTok.lAddByte((UInt16)letter); } pTok.lAddByte(0); break; default: break; } }
internal override int scan(ref PreprocessorToken ppToken) { return(pp.ReadToken(tokens, ppToken)); }
internal bool DoStuff(ParseContext parseContext, PreprocessorContext ppContext, InputScanner input, bool versionWillBeError) { //bool versionWillBeError = true; var unNeededSpaceTokens = new HashSet <char>(new char[] { ';', '(', ')', '[', ']' }); var noSpaceBeforeTokens = new HashSet <char>(new char[] { ',' }); var outputStream = new StringBuilder(); int lastLine = -1; // lastLine is the line number of the last token // processed. It is tracked in order for new-lines to be inserted when // a token appears on a new line. int lastToken = -1; parseContext.setScanner(input); ppContext.setInput(input, versionWillBeError); // Inserts newlines and incremnets lastLine until // lastLine >= line. Action <int> adjustLine = (line) => { int tokenLine = line - 1; while (lastLine < tokenLine) { if (lastLine >= 0) { outputStream.AppendLine(); } ++lastLine; } }; parseContext.ExtensionCallback = (int line, string extension, string behavior) => { adjustLine(line); outputStream.Append("#extension ").Append(extension).Append(" : ").Append(behavior); }; parseContext.LineCallback = (int line, bool hasSource, int sourceNum) => { // SourceNum is the number of the source-string that is being parsed. if (lastLine != -1) { outputStream.AppendLine(); } outputStream.Append("#line ").Append(line); if (hasSource) { outputStream.Append(" ").Append(sourceNum); } outputStream.AppendLine(); lastLine = System.Math.Max(line - 1, 1); }; parseContext.VersionCallback = (int line, int version, string str) => { adjustLine(line); outputStream.Append("#version ").Append(version); if (str != null) { outputStream.Append(" ").Append(str); } outputStream.AppendLine(); ++lastLine; }; parseContext.PragmaCallback = (int line, List <string> ops) => { adjustLine(line); outputStream.Append("#pragma "); foreach (var op in ops) { outputStream.Append(op); } }; parseContext.ErrorCallback = (int line, string errorMessage) => { adjustLine(line); outputStream.Append("#error ").Append(errorMessage); }; var token = new PreprocessorToken(); string tok = ppContext.tokenize(ref token); while (tok != null) { int tokenLine = token.loc.line - 1; // start at 0; bool newLine = false; while (lastLine < tokenLine) { if (lastLine > -1) { outputStream.AppendLine(); newLine = true; } ++lastLine; if (lastLine == tokenLine) { // Don't emit whitespace onto empty lines. // Copy any whitespace characters at the start of a line // from the input to the output. for (int i = 0; i < token.loc.column - 1; ++i) { outputStream.Append(" "); } } } // Output a space in between tokens, but not at the start of a line, // and also not around special tokens. This helps with readability // and consistency. if (!newLine && lastToken != -1 && (!unNeededSpaceTokens.Contains((char)token.token)) && (!unNeededSpaceTokens.Contains((char)lastToken)) && (!noSpaceBeforeTokens.Contains((char)token.token))) { outputStream.Append(" "); } lastToken = token.token; outputStream.Append(tok); tok = ppContext.tokenize(ref token); } outputStream.AppendLine(); Output = outputStream.ToString(); return(true); }
internal override int scan(ref PreprocessorToken ppToken) { return pp.ReadToken(tokens, ppToken); }
// Handle #pragma int CPPpragma(ref PreprocessorToken ppToken) { var tokens = new List<string>(); SourceLocation loc = ppToken.loc; // because we go to the next line before processing int token = scanToken(ref ppToken); while (token != '\n' && token != BasePreprocessorInput.EOF) { switch (token) { case (int) CppEnums.IDENTIFIER: tokens.Add(Symbols.Atoms.GetAtomString(ppToken.atom)); break; case (int) CppEnums.INTCONSTANT: case (int) CppEnums.UINTCONSTANT: case (int) CppEnums.FLOATCONSTANT: case (int) CppEnums.DOUBLECONSTANT: tokens.Add(ppToken.name); break; default: tokens.Add (Char.ConvertFromUtf32 (token).ToString ()); break; } token = scanToken(ref ppToken); } if (token == BasePreprocessorInput.EOF) parseContext.Error(loc, "directive must end with a newline", "#pragma", ""); else parseContext.handlePragma(loc, tokens); return token; }
// Handle #ifdef int CPPifdef(bool defined, ref PreprocessorToken ppToken) { int token = scanToken(ref ppToken); int name = ppToken.atom; if (++ifdepth > MAXIFNESTING) { parseContext.Error(ppToken.loc, "maximum nesting depth exceeded", "#ifdef", ""); return 0; } elsetracker++; if (token != (int) CppEnums.IDENTIFIER) { if (defined) parseContext.Error(ppToken.loc, "must be followed by macro name", "#ifdef", ""); else parseContext.Error(ppToken.loc, "must be followed by macro name", "#ifndef", ""); } else { Symbol s = Symbols.LookUp(name); token = scanToken(ref ppToken); if (token != '\n') { parseContext.Error(ppToken.loc, "unexpected tokens following #ifdef directive - expected a newline", "#ifdef", ""); while (token != '\n') token = scanToken(ref ppToken); } if (((s != null && !s.mac.undef)) != defined) token = CPPelse(true, ref ppToken); } return token; }
// Handle #extension int CPPextension(ref PreprocessorToken ppToken) { int line = ppToken.loc.line; int token = scanToken(ref ppToken); if (token=='\n') { parseContext.Error(ppToken.loc, "extension name not specified", "#extension", ""); return token; } if (token != (int) CppEnums.IDENTIFIER) parseContext.Error(ppToken.loc, "extension name expected", "#extension", ""); string extensionName = Symbols.Atoms.GetAtomString(ppToken.atom); token = scanToken(ref ppToken); if (token != ':') { parseContext.Error(ppToken.loc, "':' missing after extension name", "#extension", ""); return token; } token = scanToken(ref ppToken); if (token != (int) CppEnums.IDENTIFIER) { parseContext.Error(ppToken.loc, "behavior for extension not specified", "#extension", ""); return token; } parseContext.updateExtensionBehavior(line, extensionName, Symbols.Atoms.GetAtomString(ppToken.atom)); token = scanToken(ref ppToken); if (token == '\n') return token; else parseContext.Error(ppToken.loc, "extra tokens -- expected newline", "#extension",""); return token; }
/////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////// Floating point constants: ///////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////// /* */ /// <summary> /// Scan a single- or double-precision floating point constant. Assumes that the scanner /// has seen at least one digit, followed by either a decimal '.' or the /// letter 'e', or a precision ending (e.g., F or LF). /// </summary> /// <returns>The float const.</returns> /// <param name="len">Length.</param> /// <param name="ch">Ch.</param> /// <param name="ppToken">Pp token.</param> internal int lFloatConst(StringInputBuffer buffer, int len, int ch, PreprocessorToken ppToken) { bool HasDecimalOrExponent = false; int declen, exp, ExpSign; int str_len; bool isDouble = false; declen = 0; exp = 0; const int MAX_TOKEN_LENGTH = 1024; str_len=len; char[] str = buffer.name; if (ch == '.') { HasDecimalOrExponent = true; str[len++] = (char)ch; ch = getChar(); while (ch >= '0' && ch <= '9') { if (len < MAX_TOKEN_LENGTH) { declen++; if (len > 0 || ch != '0') { str[len] = (char)ch; len++; str_len++; } ch = getChar(); } else { parseContext.Error( ppToken.loc, "float literal too long", "", ""); len = 1; str_len = 1; } } } // Exponent: if (ch == 'e' || ch == 'E') { HasDecimalOrExponent = true; if (len >= MAX_TOKEN_LENGTH) { parseContext.Error( ppToken.loc, "float literal too long", "", ""); len = 1; str_len=1; } else { ExpSign = 1; str[len++] = (char)ch; ch = getChar(); if (ch == '+') { str[len++] = (char)ch; ch = getChar(); } else if (ch == '-') { ExpSign = -1; str[len++] = (char)ch; ch = getChar(); } if (ch >= '0' && ch <= '9') { while (ch >= '0' && ch <= '9') { if (len < MAX_TOKEN_LENGTH) { exp = exp*10 + ch - '0'; str[len++] = (char)ch; ch = getChar(); } else { parseContext.Error( ppToken.loc, "float literal too long", "", ""); len = 1; str_len=1; } } } else { parseContext.Error( ppToken.loc, "bad character in float exponent", "", ""); } exp *= ExpSign; } } if (len == 0) { ppToken.dval = 0.0; str = "0.0".ToCharArray(); } else { if (ch == 'l' || ch == 'L') { parseContext.doubleCheck( ppToken.loc, "double floating-point suffix"); if (! HasDecimalOrExponent) parseContext.Error( ppToken.loc, "float literal needs a decimal point or exponent", "", ""); int ch2 = getChar(); if (ch2 != 'f' && ch2 != 'F') { ungetChar(); ungetChar(); } else { if (len < MAX_TOKEN_LENGTH) { str[len++] = (char)ch; str[len++] = (char)ch2; isDouble = true; } else { parseContext.Error( ppToken.loc, "float literal too long", "", ""); len = 1; str_len=1; } } } else if (ch == 'f' || ch == 'F') { parseContext.ProfileRequires( ppToken.loc, Profile.EsProfile, 300, null, "floating-point suffix"); if ((parseContext.messages & MessageType.RelaxedErrors) == 0) parseContext.ProfileRequires (ppToken.loc, (Profile)~Profile.EsProfile, 120, null, "floating-point suffix"); if (! HasDecimalOrExponent) parseContext.Error( ppToken.loc, "float literal needs a decimal point or exponent", "", ""); if (len < MAX_TOKEN_LENGTH) str[len++] = (char)ch; else { parseContext.Error( ppToken.loc, "float literal too long", "", ""); len = 1; str_len=1; } } else ungetChar(); //str[len]='\0'; ppToken.dval = Double.Parse(new string(str)); } if (isDouble) return (int) CppEnums.DOUBLECONSTANT; else return (int) CppEnums.FLOATCONSTANT; }
private int MacroExpand(int atom, PreprocessorToken ppToken, bool expandUndef, bool newLineOkay) { Symbol sym = Symbols.LookUp(atom); int token; int depth = 0; ppToken.space = false; if (atom == __LINE__Atom) { ppToken.ival = parseContext.getCurrentLoc().line; ppToken.name = ppToken.ival.ToString(); UngetToken ((int)CppEnums.INTCONSTANT, ppToken); return 1; } if (atom == __FILE__Atom) { ppToken.ival = parseContext.getCurrentLoc().stringBias; ppToken.name = ppToken.ival.ToString (); UngetToken ((int)CppEnums.INTCONSTANT, ppToken); return 1; } if (atom == __VERSION__Atom) { ppToken.ival = parseContext.mVersion; ppToken.name = ppToken.ival.ToString (); UngetToken ((int)CppEnums.INTCONSTANT, ppToken); return 1; } // no recursive expansions if (sym != null && sym.mac.busy) return 0; // not expanding undefined macros if ((sym == null || sym.mac.undef) && ! expandUndef) return 0; // 0 is the value of an undefined macro if ((sym == null || sym.mac.undef) && expandUndef) { pushInput(new ZeroInput(this)); return -1; } MacroInput inp = new MacroInput(this); SourceLocation loc = ppToken.loc; // in case we go to the next line before discovering the error inp.mac = sym.mac; if (sym.mac.args != null && sym.mac.args.Length > 0) { token = scanToken(ref ppToken); if (newLineOkay) { while (token == '\n') token = scanToken(ref ppToken); } if (token != '(') { parseContext.Error(loc, "expected '(' following", "macro expansion", Symbols.Atoms.GetAtomString(atom)); UngetToken(token, ppToken); ppToken.atom = atom; inp = null; return 0; } inp.args.Clear (); for (int i = 0; i < inp.mac.argc; i++) inp.args.Add (new TokenStream ()); int arg = 0; bool tokenRecorded = false; do { depth = 0; while (true) { token = scanToken(ref ppToken); if (token == BasePreprocessorInput.EOF) { parseContext.Error(loc, "EOF in macro", "macro expansion", Symbols.Atoms.GetAtomString(atom)); inp = null; return 0; } if (token == '\n') { if (! newLineOkay) { parseContext.Error(loc, "end of line in macro substitution:", "macro expansion", Symbols.Atoms.GetAtomString(atom)); inp = null; return 0; } continue; } if (token == '#') { parseContext.Error(ppToken.loc, "unexpected '#'", "macro expansion", Symbols.Atoms.GetAtomString(atom)); inp = null; return 0; } if (inp.mac.argc == 0 && token != ')') break; if (depth == 0 && (token == ',' || token == ')')) break; if (token == '(') depth++; if (token == ')') depth--; Symbols.Atoms.RecordToken(inp.args[arg], token, ppToken); tokenRecorded = true; } if (token == ')') { if (inp.mac.argc == 1 && !tokenRecorded) break; arg++; break; } arg++; } while (arg < inp.mac.argc); if (arg < inp.mac.argc) parseContext.Error(loc, "Too few args in Macro", "macro expansion", Symbols.Atoms.GetAtomString(atom)); else if (token != ')') { depth=0; while (token != BasePreprocessorInput.EOF && (depth > 0 || token != ')')) { if (token == ')') depth--; token = scanToken(ref ppToken); if (token == '(') depth++; } if (token == BasePreprocessorInput.EOF) { parseContext.Error(loc, "EOF in macro", "macro expansion", Symbols.Atoms.GetAtomString(atom)); inp = null; return 0; } parseContext.Error(loc, "Too many args in macro", "macro expansion", Symbols.Atoms.GetAtomString(atom)); } for (int i = 0; i < inp.mac.argc; i++) { inp.args [i] = PrescanMacroArg (inp.args[i], ppToken, newLineOkay); } } pushInput(inp); sym.mac.busy = true; sym.mac.body.Rewind(); return 1; }
// Call when there should be no more tokens left on a line. int extraTokenCheck(int atom, ref PreprocessorToken ppToken, int token) { if (token != '\n') { const string message = "unexpected tokens following directive"; string label; if (atom == elseAtom) label = "#else"; else if (atom == elifAtom) label = "#elif"; else if (atom == endifAtom) label = "#endif"; else if (atom == ifAtom) label = "#if"; else if (atom == lineAtom) label = "#line"; else label = ""; if ((parseContext.messages & MessageType.RelaxedErrors) > 0) parseContext.Warn(ppToken.loc, message, label, ""); else parseContext.Error(ppToken.loc, message, label, ""); while (token != '\n') token = scanToken(ref ppToken); } return token; }
// Expand macros, skipping empty expansions, to get to the first real token in those expansions. int evalToToken(int token, bool shortCircuit, ref int res, ref bool err, ref PreprocessorToken ppToken) { bool escapedLoop = false; while (token == (int) CppEnums.IDENTIFIER && ppToken.atom != definedAtom) { int macroReturn = MacroExpand(ppToken.atom, ppToken, true, false); if (macroReturn == 0) { parseContext.Error(ppToken.loc, "can't evaluate expression", "preprocessor evaluation", ""); err = true; res = 0; token = scanToken(ref ppToken); break; } if (macroReturn == -1) { if (! shortCircuit && parseContext.mProfile == Profile.EsProfile) { const string message = "undefined macro in expression not allowed in es profile"; string name = Symbols.Atoms.GetAtomString(ppToken.atom); if ((parseContext.messages & MessageType.RelaxedErrors) > 0) parseContext.Warn(ppToken.loc, message, "preprocessor evaluation", name); else parseContext.Error(ppToken.loc, message, "preprocessor evaluation", name); } } token = scanToken(ref ppToken); } return token; }
int eval(int token, EvalPrecedence precedence, bool shortCircuit, ref int res, ref bool err, ref PreprocessorToken ppToken) { UnaryEvalOperation[] unop = { new UnaryEvalOperation{ token='+', op= op_pos }, new UnaryEvalOperation{ token='-', op= op_neg }, new UnaryEvalOperation{ token='~', op= op_cmpl}, new UnaryEvalOperation{ token='!', op= op_not }, }; SourceLocation loc = ppToken.loc; // because we sometimes read the newline before reporting the error if (token == (int) CppEnums.IDENTIFIER) { if (ppToken.atom == definedAtom) { bool needclose = false; token = scanToken(ref ppToken); if (token == '(') { needclose = true; token = scanToken(ref ppToken); } if (token != (int) CppEnums.IDENTIFIER) { parseContext.Error(loc, "incorrect directive, expected identifier", "preprocessor evaluation", ""); err = true; res = 0; return token; } Symbol s = Symbols.LookUp(ppToken.atom); // !s.mac.undef res = (s != null) ? (s.mac.undef ? 0 : 1) : 0; token = scanToken(ref ppToken); if (needclose) { if (token != ')') { parseContext.Error(loc, "expected ')'", "preprocessor evaluation", ""); err = true; res = 0; return token; } token = scanToken(ref ppToken); } } else { token = evalToToken(token, shortCircuit, ref res, ref err, ref ppToken); return eval(token, precedence, shortCircuit, ref res, ref err, ref ppToken); } } else if (token == (int) CppEnums.INTCONSTANT) { res = ppToken.ival; token = scanToken(ref ppToken); } else if (token == '(') { token = scanToken(ref ppToken); token = eval(token, EvalPrecedence.MIN_PRECEDENCE, shortCircuit, ref res, ref err, ref ppToken); if (! err) { if (token != ')') { parseContext.Error(loc, "expected ')'", "preprocessor evaluation", ""); err = true; res = 0; return token; } token = scanToken(ref ppToken); } } else { int op; for (op = unop.Length - 1; op >= 0; op--) { if (unop[op].token == token) break; } if (op >= 0) { token = scanToken(ref ppToken); token = eval(token, EvalPrecedence.UNARY, shortCircuit, ref res, ref err, ref ppToken); res = unop[op].op(res); } else { parseContext.Error(loc, "bad expression", "preprocessor evaluation", ""); err = true; res = 0; return token; } } token = evalToToken(token, shortCircuit, ref res, ref err, ref ppToken); // Perform evaluation of binary operation, if there is one, otherwise we are done. while (! err) { if (token == ')' || token == '\n') break; int op; for (op = binop.Count - 1; op >= 0; op--) { if (binop[op].token == token) break; } if (op < 0 || binop[op].precedence <= precedence) break; int leftSide = res; // Setup short-circuiting, needed for ES, unless already in a short circuit. // (Once in a short-circuit, can't turn off again, until that whole subexpression is done. if (! shortCircuit) { if ((token == (int) CppEnums.OR_OP && leftSide == 1) || (token == (int) CppEnums.AND_OP && leftSide == 0)) shortCircuit = true; } token = scanToken(ref ppToken); token = eval(token, binop[op].precedence, shortCircuit, ref res, ref err, ref ppToken); res = binop[op].op(leftSide, res); } return token; }
// #version: This is just for error checking: the version and profile are decided before preprocessing starts int CPPversion(ref PreprocessorToken ppToken) { int token = scanToken(ref ppToken); if (errorOnVersion || versionSeen) parseContext.Error(ppToken.loc, "must occur first in shader", "#version", ""); versionSeen = true; if (token == '\n') { parseContext.Error(ppToken.loc, "must be followed by version number", "#version", ""); return token; } if (token != (int) CppEnums.INTCONSTANT) parseContext.Error(ppToken.loc, "must be followed by version number", "#version", ""); ppToken.ival = int.Parse(ppToken.name); int versionNumber = ppToken.ival; int line = ppToken.loc.line; token = scanToken(ref ppToken); if (token == '\n') { parseContext.notifyVersion(line, versionNumber, null); return token; } else { if (ppToken.atom != coreAtom && ppToken.atom != compatibilityAtom && ppToken.atom != esAtom) parseContext.Error(ppToken.loc, "bad profile name; use es, core, or compatibility", "#version", ""); parseContext.notifyVersion(line, versionNumber, Symbols.Atoms.GetAtomString(ppToken.atom)); token = scanToken(ref ppToken); if (token == '\n') return token; else parseContext.Error(ppToken.loc, "bad tokens following profile -- expected newline", "#version", ""); } return token; }
// Handle #else //* Skip forward to appropriate spot. This is used both //** to skip to a #endif after seeing an #else, AND to skip to a #else, //** #elif, or #endif after a #if/#ifdef/#ifndef/#elif test was false. // int CPPelse(bool matchelse,ref PreprocessorToken ppToken) { int atom; int depth = 0; int token = scanToken(ref ppToken); while (token != BasePreprocessorInput.EOF) { if (token != '#') { while (token != '\n' && token != BasePreprocessorInput.EOF) token = scanToken(ref ppToken); if (token == BasePreprocessorInput.EOF) return BasePreprocessorInput.EOF; token = scanToken(ref ppToken); continue; } if ((token = scanToken(ref ppToken)) != (int) CppEnums.IDENTIFIER) continue; atom = ppToken.atom; if (atom == ifAtom || atom == ifdefAtom || atom == ifndefAtom) { depth++; ifdepth++; elsetracker++; } else if (atom == endifAtom) { token = extraTokenCheck(atom, ref ppToken, scanToken(ref ppToken)); elseSeen[elsetracker] = false; --elsetracker; if (depth == 0) { // found the #endif we are looking for // ORIGINALLY if (ifdepth) if (ifdepth > 0) --ifdepth; break; } --depth; --ifdepth; } else if (matchelse && depth == 0) { if (atom == elseAtom) { elseSeen[elsetracker] = true; token = extraTokenCheck(atom, ref ppToken, scanToken(ref ppToken)); // found the #else we are looking for break; } else if (atom == elifAtom) { if (elseSeen[elsetracker]) parseContext.Error(ppToken.loc, "#elif after #else", "#elif", ""); /* we decrement ifdepth here, because CPPif will increment * it and we really want to leave it alone */ //if (ifdepth) { if (ifdepth > 0) { --ifdepth; elseSeen[elsetracker] = false; --elsetracker; } return CPPif(ref ppToken); } } else if (atom == elseAtom) { if (elseSeen[elsetracker]) parseContext.Error(ppToken.loc, "#else after #else", "#else", ""); else elseSeen[elsetracker] = true; token = extraTokenCheck(atom, ref ppToken, scanToken(ref ppToken)); } else if (atom == elifAtom) { if (elseSeen[elsetracker]) parseContext.Error(ppToken.loc, "#elif after #else", "#elif", ""); } } return token; }
// Handle #error int CPPerror(ref PreprocessorToken ppToken) { int token = scanToken(ref ppToken); StringBuilder message = new StringBuilder(); SourceLocation loc = ppToken.loc; while (token != '\n') { if (token == (int) CppEnums.INTCONSTANT || token == (int) CppEnums.UINTCONSTANT || token == (int) CppEnums.FLOATCONSTANT || token == (int) CppEnums.DOUBLECONSTANT) { message.Append(ppToken.name); } else if (token == (int) CppEnums.IDENTIFIER || token == (int) CppEnums.STRCONSTANT) { message.Append(Symbols.Atoms.GetAtomString(ppToken.atom)); } else { message.Append(Symbols.Atoms.GetAtomString(token)); } message.Append(" "); token = scanToken(ref ppToken); } parseContext.notifyErrorDirective(loc.line, message.ToString()); //store this msg into the shader's information log..set the Compile Error flag!!!! parseContext.Error(loc, message.ToString(), "#error", ""); return '\n'; }
void UngetToken(int token, PreprocessorToken ppToken) { pushInput(new UngotTokenInput(this, token, ppToken)); }
// Handle #if int CPPif(ref PreprocessorToken ppToken) { int token = scanToken(ref ppToken); elsetracker++; //if (! ifdepth++) if (ifdepth++ <= 0) ifloc = ppToken.loc; if (ifdepth > MAXIFNESTING) { parseContext.Error(ppToken.loc, "maximum nesting depth exceeded", "#if", ""); return 0; } int res = 0; bool err = false; token = eval(token, (int) EvalPrecedence.MIN_PRECEDENCE, false, ref res, ref err, ref ppToken); token = extraTokenCheck(ifAtom, ref ppToken, token); if (res == 0 && !err) token = CPPelse(true, ref ppToken); return token; }
internal UngotTokenInput(PreprocessorContext pp, int t, PreprocessorToken p) : base(pp) { token = t; lval = p; }
// Handle #line int CPPline(ref PreprocessorToken ppToken) { // "#line must have, after macro substitution, one of the following forms: // "#line line // "#line line source-string-number" int token = scanToken(ref ppToken); if (token == '\n') { parseContext.Error(ppToken.loc, "must by followed by an integral literal", "#line", ""); return token; } int lineRes = 0; // Line number after macro expansion. int lineToken = 0; int fileRes = 0; // Source file number after macro expansion. bool hasFile = false; bool lineErr = false; bool fileErr = false; token = eval(token, EvalPrecedence.MIN_PRECEDENCE, false, ref lineRes, ref lineErr, ref ppToken); if (! lineErr) { lineToken = lineRes; if (token == '\n') ++lineRes; // Desktop, pre-version 3.30: "After processing this directive // (including its new-line), the implementation will behave as if it is compiling at line number line+1 and // source string number source-string-number." // // Desktop, version 3.30 and later, and ES: "After processing this directive // (including its new-line), the implementation will behave as if it is compiling at line number line and // source string number source-string-number. if (parseContext.mProfile == Profile.EsProfile || parseContext.mVersion >= 330) --lineRes; parseContext.setCurrentLine(lineRes); if (token != '\n') { token = eval(token, EvalPrecedence.MIN_PRECEDENCE, false, ref fileRes, ref fileErr, ref ppToken); if (! fileErr) parseContext.setCurrentString(fileRes); hasFile = true; } } if (!fileErr && !lineErr) { parseContext.notifyLineDirective(lineToken, hasFile, fileRes); } token = extraTokenCheck(lineAtom,ref ppToken, token); return token; }
internal int readCPPline(ref PreprocessorToken ppToken) { int token = scanToken(ref ppToken); bool isVersion = false; if (token == (int) CppEnums.IDENTIFIER) { if (ppToken.atom == defineAtom) { token = CPPdefine(ref ppToken); } else if (ppToken.atom == elseAtom) { // ORIGINALLY if (elsetracker[elseSeen]) if (elseSeen[elsetracker]) parseContext.Error(ppToken.loc, "#else after #else", "#else", ""); // ORIGINALLY elsetracker[elseSeen] = true; elseSeen[elsetracker] = true; //if (! ifdepth) if (ifdepth <= 0) parseContext.Error(ppToken.loc, "mismatched statements", "#else", ""); token = extraTokenCheck(elseAtom, ref ppToken, scanToken(ref ppToken)); token = CPPelse(false, ref ppToken); } else if (ppToken.atom == elifAtom) { // (! ifdepth) <==> (ifdepth <= 0) if (ifdepth <= 0) parseContext.Error(ppToken.loc, "mismatched statements", "#elif", ""); if (elseSeen[elsetracker]) parseContext.Error(ppToken.loc, "#elif after #else", "#elif", ""); // this token is really a dont care, but we still need to eat the tokens token = scanToken(ref ppToken); while (token != '\n') token = scanToken(ref ppToken); token = CPPelse(false, ref ppToken); } else if (ppToken.atom == endifAtom) { elseSeen[elsetracker] = false; --elsetracker; if (ifdepth <= 0) parseContext.Error(ppToken.loc, "mismatched statements", "#endif", ""); else --ifdepth; token = extraTokenCheck(endifAtom, ref ppToken, scanToken(ref ppToken)); } else if (ppToken.atom == ifAtom) { token = CPPif (ref ppToken); } else if (ppToken.atom == ifdefAtom) { token = CPPifdef(true, ref ppToken); } else if (ppToken.atom == ifndefAtom) { token = CPPifdef(false, ref ppToken); } else if (ppToken.atom == lineAtom) { token = CPPline(ref ppToken); } else if (ppToken.atom == pragmaAtom) { token = CPPpragma(ref ppToken); } else if (ppToken.atom == undefAtom) { token = CPPundef(ref ppToken); } else if (ppToken.atom == errorAtom) { token = CPPerror(ref ppToken); } else if (ppToken.atom == versionAtom) { token = CPPversion(ref ppToken); isVersion = true; } else if (ppToken.atom == extensionAtom) { token = CPPextension(ref ppToken); } else { parseContext.Error(ppToken.loc, "invalid directive:", "#", Symbols.Atoms.GetAtomString(ppToken.atom)); } } else if (token != '\n' && token != BasePreprocessorInput.EOF) parseContext.Error(ppToken.loc, "invalid directive", "#", ""); while (token != '\n' && token != 0 && token != BasePreprocessorInput.EOF) token = scanToken(ref ppToken); return token; }
// Handle #undef int CPPundef(ref PreprocessorToken ppToken) { int token = scanToken(ref ppToken); Symbol symb; if (token != (int) CppEnums.IDENTIFIER) { parseContext.Error(ppToken.loc, "must be followed by macro name", "#undef", ""); return token; } string name = Symbols.Atoms.GetAtomString(ppToken.atom); // TODO preprocessor simplification: the token text should have been built into the ppToken during scanToken() parseContext.ReservedPpErrorCheck(ppToken.loc, name, "#undef"); symb = Symbols.LookUp(ppToken.atom); if (symb != null) { symb.mac.undef = true; } token = scanToken(ref ppToken); if (token != '\n') parseContext.Error(ppToken.loc, "can only be followed by a single macro name", "#undef", ""); return token; }
///* //* Read the next token from a token stream (not the source stream, but stream used to hold a tokenized macro). //*/ internal int ReadToken(TokenStream pTok, PreprocessorToken ppToken) { char[] tokenText = buffer.tokenText; int ltoken = 0; int len = 0; int ch; ltoken = pTok.lReadByte(); ppToken.loc = parseContext.getCurrentLoc(); if (ltoken > 127) ltoken += 128; switch (ltoken) { case '#': if (pTok.lReadByte() == '#') { parseContext.requireProfile(ppToken.loc, ~Profile.EsProfile, "token pasting (##)"); parseContext.ProfileRequires(ppToken.loc, ~Profile.EsProfile, 130, null, "token pasting (##)"); parseContext.Error(ppToken.loc, "token pasting not implemented (internal error)", "##", ""); //return CPP_TOKEN_PASTE; return ReadToken(pTok, ppToken); } else pTok.lUnreadByte(); break; case (int) CppEnums.STRCONSTANT: case (int) CppEnums.IDENTIFIER: case (int) CppEnums.FLOATCONSTANT: case (int) CppEnums.DOUBLECONSTANT: case (int) CppEnums.INTCONSTANT: case (int) CppEnums.UINTCONSTANT: len = 0; ch = pTok.lReadByte (); while (ch != 0) { if (len < PreprocessorToken.maxTokenLength) { tokenText [len] = (char)ch; len++; ch = pTok.lReadByte(); } else { parseContext.Error (ppToken.loc, "token too long", "", ""); break; } } break; default: break; } // DY RESTRUCTURED CODE //tokenText[len] = 0; string text = new string (tokenText, 0, len); switch (ltoken) { case (int) CppEnums.IDENTIFIER: case (int) CppEnums.STRCONSTANT: ppToken.atom = Symbols.Atoms.LookUpAddString(text); break; case (int) CppEnums.FLOATCONSTANT: case (int) CppEnums.DOUBLECONSTANT: ppToken.name = text; ppToken.dval = float.Parse(ppToken.name); break; case (int) CppEnums.INTCONSTANT: case (int) CppEnums.UINTCONSTANT: ppToken.name = text; len = text.Length; if (len > 0 && tokenText[0] == '0') { if (len > 1 && (tokenText[1] == 'x' || tokenText[1] == 'X')) ppToken.ival = Convert.ToInt32(ppToken.name, 16); else ppToken.ival = Convert.ToInt32(ppToken.name, 8); } else ppToken.ival = int.Parse(ppToken.name); break; default: break; } return ltoken; }
// Get the next token from *stack* of input sources, popping input sources // that are out of tokens, down until an input sources is found that has a token. // Return EOF when there are no more tokens to be found by doing this. internal int scanToken(ref PreprocessorToken ppToken) { int token = BasePreprocessorInput.EOF; while (inputStack.Count > 0) { token = inputStack.Peek().scan(ref ppToken); if (token != BasePreprocessorInput.END_OF_INPUT) break; popInput(); } if (token == BasePreprocessorInput.END_OF_INPUT) return BasePreprocessorInput.EOF; return token; }
internal string tokenize(ref PreprocessorToken ppToken) { int token = '\n'; for(;;) { string tokenString = null; token = scanToken(ref ppToken); ppToken.token = token; if (token == BasePreprocessorInput.EOF) { missingEndifCheck(); return null; } if (token == '#') { if (previous_token == '\n') { token = readCPPline(ref ppToken); if (token == BasePreprocessorInput.EOF) { missingEndifCheck(); return null; } continue; } else { parseContext.Error(ppToken.loc, "preprocessor directive cannot be preceded by another token", "#", ""); return null; } } previous_token = token; if (token == '\n') continue; // expand macros if (token == (int) CppEnums.IDENTIFIER && MacroExpand(ppToken.atom, ppToken, false, true) != 0) continue; if (token == (int) CppEnums.IDENTIFIER) tokenString = Symbols.Atoms.GetAtomString(ppToken.atom); else if (token == (int) CppEnums.INTCONSTANT || token == (int) CppEnums.UINTCONSTANT || token == (int) CppEnums.FLOATCONSTANT || token == (int) CppEnums.DOUBLECONSTANT) tokenString = ppToken.name; else if (token == (int) CppEnums.STRCONSTANT) { parseContext.Error(ppToken.loc, "string literals not supported", "\"\"", ""); tokenString = null; } else if (token == '\'') { parseContext.Error(ppToken.loc, "character literals not supported", "\'", ""); tokenString = null; } else tokenString = Symbols.Atoms.GetAtomString(token); if (tokenString != null) { if (tokenString[0] != 0) parseContext.tokensBeforeEOF = true; return tokenString; } } }
// Handle #define int CPPdefine(ref PreprocessorToken ppToken) { MacroSymbol mac = new MacroSymbol (); Symbol symb; // get macro name int token = scanToken(ref ppToken); if (token != (int) CppEnums.IDENTIFIER) { parseContext.Error(ppToken.loc, "must be followed by macro name", "#define", ""); return token; } int atom = ppToken.atom; string definedName = Symbols.Atoms.GetAtomString(atom); if (ppToken.loc.stringBias >= 0) { // We are in user code; check for reserved name use: parseContext.ReservedPpErrorCheck(ppToken.loc, definedName, "#define"); } // gather parameters to the macro, between (...) token = scanToken(ref ppToken); if (token == '(' && ! ppToken.space) { int argc = 0; int[] args = new int[maxMacroArgs]; do { token = scanToken(ref ppToken); if (argc == 0 && token == ')') break; if (token != (int) CppEnums.IDENTIFIER) { parseContext.Error(ppToken.loc, "bad argument", "#define", ""); return token; } // check for duplication of parameter name bool duplicate = false; for (int a = 0; a < argc; ++a) { if (args[a] == ppToken.atom) { parseContext.Error(ppToken.loc, "duplicate macro parameter", "#define", ""); duplicate = true; break; } } if (! duplicate) { if (argc < maxMacroArgs) args[argc++] = ppToken.atom; else parseContext.Error(ppToken.loc, "too many macro parameters", "#define", ""); } token = scanToken(ref ppToken); } while (token == ','); if (token != ')') { parseContext.Error(ppToken.loc, "missing parenthesis", "#define", ""); return token; } mac.argc = argc; mac.args = args; token = scanToken(ref ppToken); } // record the definition of the macro SourceLocation defineLoc = ppToken.loc; // because ppToken is going to go to the next line before we report errors mac.body = new TokenStream(); while (token != '\n') { Symbols.Atoms.RecordToken(mac.body, token, ppToken); token = scanToken(ref ppToken); if (token != '\n' && ppToken.space) Symbols.Atoms.RecordToken(mac.body, ' ', ppToken); } // check for duplicate definition symb = Symbols.LookUp(atom); if (symb != null) { if (! symb.mac.undef) { // Already defined -- need to make sure they are identical: // "Two replacement lists are identical if and only if the preprocessing tokens in both have the same number, // ordering, spelling, and white-space separation, where all white-space separations are considered identical." if (symb.mac.argc != mac.argc) parseContext.Error(defineLoc, "Macro redefined; different number of arguments:", "#define", Symbols.Atoms.GetAtomString(atom)); else { for (int argc = 0; argc < mac.argc; argc++) { if (symb.mac.args[argc] != mac.args[argc]) parseContext.Error(defineLoc, "Macro redefined; different argument names:", "#define", Symbols.Atoms.GetAtomString(atom)); } symb.mac.body.Rewind(); mac.body.Rewind(); int newToken; do { int oldToken; PreprocessorToken oldPpToken = new PreprocessorToken(); PreprocessorToken newPpToken = new PreprocessorToken(); oldToken = ReadToken(symb.mac.body, oldPpToken); newToken = ReadToken(mac.body, newPpToken); if (oldToken != newToken || oldPpToken != newPpToken) { parseContext.Error(defineLoc, "Macro redefined; different substitutions:", "#define", Symbols.Atoms.GetAtomString(atom)); break; } } while (newToken > 0); } } } else symb = Symbols.Add(atom); symb.mac.body = null; symb.mac = mac; return '\n'; }
internal TokenStream PrescanMacroArg(TokenStream a, PreprocessorToken ppToken, bool newLineOkay) { int token; a.Rewind(); do { token = ReadToken(a, ppToken); if (token == (int) CppEnums.IDENTIFIER && Symbols.LookUp(ppToken.atom) != null) break; } while (token != BasePreprocessorInput.END_OF_INPUT); if (token == BasePreprocessorInput.END_OF_INPUT) return a; TokenStream n = new TokenStream (); pushInput(new MarkerInput(this)); pushTokenStreamInput(a); while ((token = scanToken(ref ppToken)) != MarkerInput.marker) { if (token == (int) CppEnums.IDENTIFIER && MacroExpand(ppToken.atom, ppToken, false, newLineOkay) != 0) continue; Symbols.Atoms.RecordToken(n, token, ppToken); } popInput(); return n; }
internal bool DoStuff(ParseContext parseContext, PreprocessorContext ppContext, InputScanner input, bool versionWillBeError) { //bool versionWillBeError = true; var unNeededSpaceTokens = new HashSet<char>(new char[]{';','(',')','[',']'}); var noSpaceBeforeTokens = new HashSet<char>(new char[]{','}); var outputStream = new StringBuilder(); int lastLine = -1; // lastLine is the line number of the last token // processed. It is tracked in order for new-lines to be inserted when // a token appears on a new line. int lastToken = -1; parseContext.setScanner(input); ppContext.setInput(input, versionWillBeError); // Inserts newlines and incremnets lastLine until // lastLine >= line. Action<int> adjustLine = (line) => { int tokenLine = line - 1; while(lastLine < tokenLine) { if (lastLine >= 0) { outputStream.AppendLine(); } ++lastLine; } }; parseContext.ExtensionCallback = (int line, string extension, string behavior) => { adjustLine(line); outputStream.Append("#extension ").Append(extension).Append(" : ").Append(behavior); }; parseContext.LineCallback = (int line, bool hasSource, int sourceNum) => { // SourceNum is the number of the source-string that is being parsed. if (lastLine != -1) { outputStream.AppendLine(); } outputStream.Append("#line ").Append(line); if (hasSource) { outputStream.Append(" ").Append(sourceNum); } outputStream.AppendLine(); lastLine = System.Math.Max(line - 1, 1); }; parseContext.VersionCallback = (int line, int version, string str) => { adjustLine(line); outputStream.Append("#version ").Append(version); if (str != null) { outputStream.Append(" ").Append(str); } outputStream.AppendLine(); ++lastLine; }; parseContext.PragmaCallback = (int line, List<string> ops) => { adjustLine(line); outputStream.Append("#pragma "); foreach(var op in ops) { outputStream.Append(op); } }; parseContext.ErrorCallback = (int line, string errorMessage) => { adjustLine(line); outputStream.Append("#error ").Append(errorMessage); }; var token = new PreprocessorToken(); string tok = ppContext.tokenize (ref token); while (tok != null) { int tokenLine = token.loc.line - 1; // start at 0; bool newLine = false; while (lastLine < tokenLine) { if (lastLine > -1) { outputStream.AppendLine(); newLine = true; } ++lastLine; if (lastLine == tokenLine) { // Don't emit whitespace onto empty lines. // Copy any whitespace characters at the start of a line // from the input to the output. for(int i = 0; i < token.loc.column - 1; ++i) { outputStream.Append(" "); } } } // Output a space in between tokens, but not at the start of a line, // and also not around special tokens. This helps with readability // and consistency. if (!newLine && lastToken != -1 && (!unNeededSpaceTokens.Contains ((char)token.token)) && (!unNeededSpaceTokens.Contains((char)lastToken)) && (!noSpaceBeforeTokens.Contains ((char)token.token))) { outputStream.Append(" "); } lastToken = token.token; outputStream.Append(tok); tok = ppContext.tokenize (ref token); } outputStream.AppendLine(); Output = outputStream.ToString(); return true; }