private static bool Else(CodeProcessorContext context, Stream data, TokenStream nodes)
{
UInt32 line = (UInt32)context.LineNumber;
UInt32 column = (UInt32)(data.Position - context.LinePosition);
while (!Preprocessor.LineBreak(data))
{
Preprocessor.Whitespace(context, data, nodes);
if (!Preprocessor.Comment(context, data))
{
break;
}
}
if (!Preprocessor.LineBreak(context, data, nodes))
{
return(Preprocessor.ThrowError(context, PreprocessorCodes.UnexpectedCharacter));
}
nodes.Add(TokenTypes.TokenSeparator);
nodes.Add(TokenTypes.Else);
nodes.Encode(line);
nodes.Encode(column);
nodes.Add(TokenTypes.TokenSeparator);
return(true);
}
private static void CreateToken(CodeProcessorContext context, Stream data, TokenStream nodes, TokenTypes type)
{
nodes.Add(type);
nodes.Encode((UInt32)context.LineNumber);
nodes.Encode((UInt32)(data.Position - context.LinePosition));
nodes.Add(TokenTypes.TokenSeparator);
Process(context, data, nodes);
nodes.Add(TokenTypes.TokenSeparator);
}
public static bool Any(CodeProcessorContext context, Stream data, TokenStream nodes)
{
nodes.Add(TokenTypes.Text);
nodes.Encode((UInt32)context.LineNumber);
nodes.Encode((UInt32)(data.Position - context.LinePosition));
nodes.Add(TokenTypes.TokenSeparator);
Process(context, data, nodes);
nodes.Add('\n');
nodes.Add(TokenTypes.TokenSeparator);
return(true);
}
public static bool Any(CodeProcessorContext context, Stream data, TokenStream nodes)
{
bool result = false;
if (nodes.CanWrite)
{
result = AnyProcessLineBreak(context, data, nodes, new Lazy <HashSet <UInt32> >());
if (result)
{
nodes.Add('\n');
}
}
else
{
while (!Preprocessor.LineBreak(context, data, nodes))
{
if (!Preprocessor.EscapedLineBreak(context, data, nodes) && !Preprocessor.Comment(context, data, nodes) && !Preprocessor.String(context, data, nodes))
{
data.Position++;
}
}
result = true;
}
return(result);
}
private static void CreateToken(long streamPos, UInt32 line, UInt32 column, long length, Stream data, TokenStream nodes, TokenTypes type)
{
long tmp = data.Position;
data.Position = streamPos;
nodes.Add(type);
nodes.Encode(line);
nodes.Encode(column);
nodes.Add(TokenTypes.TokenSeparator);
data.CopyRange(nodes, (int)length);
nodes.Add(TokenTypes.TokenSeparator);
data.Position = tmp;
}
private static bool ParameterReplacement(CodeProcessorContext context, Macro macro, TokenStream nodes, List <TokenStream> parameter)
{
IdentifierToken token; if (Identifier(macro.ReplacementList, out token))
{
if (token.Id == VariadicKeyword)
{
for (int i = macro.Parameter.Count; i < parameter.Count; i++)
{
parameter[i].Position = 0;
ReadLine(context, parameter[i], nodes);
if (i + 1 < parameter.Count)
{
nodes.Add(',');
}
}
macro.ReplacementList.Position += token.Length;
}
else if ((token.Id = (UInt32)macro.Parameter.IndexOf(token.Id)) != UInt32.MaxValue)
{
parameter[(int)token.Id].Position = 0;
ReadLine(context, parameter[(int)token.Id], nodes, nodes.Length);
macro.ReplacementList.Position += token.Length;
}
else
{
macro.ReplacementList.CopyRange(nodes, (int)token.Length);
}
return(true);
}
return(false);
}
public static bool Comment(CodeProcessorContext context, Stream data, TokenStream nodes)
{
if (data.Peek() != '/')
{
return(false);
}
long streamPos = data.Position;
data.Get();
if (SingleLineComment(context, data, nodes) ||
MultiLineComment(context, data, nodes))
{
long length = data.Position - streamPos;
data.Position -= length;
for (; length > 0; length--)
{
nodes.Add(data.Get());
}
return(true);
}
data.Position = streamPos;
return(false);
}
internal static void ProcessNoComments(CodeProcessorContext context, Stream data, TokenStream nodes, Lazy <HashSet <UInt32> > stack, bool definedConditions = false)
{
IdentifierToken token; if (Identifier(data, out token))
{
if (definedConditions && token.Id == DefinedKeyword)
{
Defined(context, data, nodes);
return;
}
object macro; if (context.Definitions.TryGetValue(token.Id, out macro))
{
long streamPos = data.Position;
data.Position += token.Length;
if (ExpandMacro(context, data, nodes, stack, (macro as Macro)))
{
return;
}
data.Position = streamPos;
}
data.CopyRange(nodes, (int)token.Length);
}
else if (!EscapedLineBreak(context, data) && !Comment(context, data) && !String(context, data, nodes))
{
nodes.Add(data.Get());
}
}
private static bool ExpandObjectMacro(CodeProcessorContext context, Stream data, TokenStream nodes, Lazy <HashSet <UInt32> > stack, Macro macro)
{
if (stack.Value.Contains(macro.Id))
{
return(false);
}
macro.ReplacementList.Position = 0;
TokenStream result = new TokenStream();
while (!macro.ReplacementList.Eof())
{
if (!TokenPasting(context, macro.ReplacementList, result, false))
{
result.Add(macro.ReplacementList.Get());
}
}
AnyPreventLineBreak(context, data, result, stack);
stack.Value.Add(macro.Id);
AnyPreventLineBreak(context, result, nodes, stack);
stack.Value.Remove(macro.Id);
return(true);
}
private static void ReadLine(CodeProcessorContext context, Stream data, TokenStream nodes, long offset = 0)
{
while (!LineBreak(context, data, nodes))
{
if (SimpleWhitespace(data))
{
if (nodes.Last != ' ' && (nodes.Length - offset) > 0 && !data.Eof())
{
nodes.Add(' ');
}
}
else if (!EscapedLineBreak(context, data) && !Comment(context, data) && !String(context, data, nodes))
{
nodes.Add(data.Get());
}
}
}
private static bool Defined(CodeProcessorContext context, Stream data, TokenStream nodes)
{
data.Position += 7;
Whitespace(context, data, nodes);
IdentifierToken token;
if (data.Peek() == '(')
{
data.Get();
Whitespace(context, data, nodes);
if (!Identifier(data, out token))
{
return(ThrowError(context, PreprocessorCodes.IdentifierExpected));
}
data.Position += token.Length;
Whitespace(context, data, nodes);
if (data.Peek() != ')')
{
return(ThrowError(context, PreprocessorCodes.MissingBracket));
}
data.Get();
}
else if (!Identifier(data, out token))
{
return(ThrowError(context, PreprocessorCodes.IdentifierExpected));
}
else
{
data.Position += token.Length;
}
if (context.Definitions.ContainsKey(token.Id))
{
nodes.Add('1');
}
else
{
nodes.Add('0');
}
return(true);
}
private static void Process(CodeProcessorContext context, Stream data, TokenStream nodes)
{
while (!Preprocessor.LineBreak(context, data, nodes))
{
if (!Preprocessor.EscapedLineBreak(context, data, nodes) && !Preprocessor.Comment(context, data, nodes) && !Preprocessor.String(context, data, nodes))
{
nodes.Add(data.Get());
}
}
}
private static bool Elif(CodeProcessorContext context, Stream data, TokenStream nodes)
{
nodes.Add(TokenTypes.TokenSeparator);
if (If(context, data, nodes, TokenTypes.ElseIf))
{
return(true);
}
return(false);
}
public static bool String(CodeProcessorContext context, Stream data, TokenStream nodes)
{
char c = data.Peek();
if (c != '\'' && c != '\"')
{
return(false);
}
long streamPos = data.Position;
data.Get();
while (!data.Eof())
{
if (data.Peek() == '\\')
{
data.Get();
if (data.Peek() == c)
{
data.Get();
}
else
{
data.Position--;
}
}
if (!EscapedLineBreak(context, data, nodes))
{
if (data.Get() == c)
{
long length = data.Position - streamPos;
data.Position -= length;
for (; length > 0; length--)
{
nodes.Add(data.Get());
}
return(true);
}
else if (LineBreak(context, data, nodes))
{
break;
}
}
}
data.Position = streamPos;
return(false);
}
private static bool Stringify(CodeProcessorContext context, Macro macro, TokenStream nodes, List <TokenStream> parameter)
{
IdentifierToken token; if (Stringifier(context, macro.ReplacementList, nodes, out token))
{
if (token.Id == VariadicKeyword)
{
nodes.Add('"');
for (int i = macro.Parameter.Count; i < parameter.Count; i++)
{
parameter[i].Position = 0;
ReadLine(context, parameter[i], nodes);
if (i + 1 < parameter.Count)
{
nodes.Add(',');
}
}
nodes.Add('"');
macro.ReplacementList.Position += token.Length;
}
else if ((token.Id = (UInt32)macro.Parameter.IndexOf(token.Id)) != UInt32.MaxValue)
{
nodes.Add('"');
parameter[(int)token.Id].Position = 0;
ReadLine(context, parameter[(int)token.Id], nodes, nodes.Length);
nodes.Add('"');
macro.ReplacementList.Position += token.Length;
}
else
{
return(ThrowError(context, PreprocessorCodes.ParameterExpected));
}
return(true);
}
return(false);
}
private static bool Endif(CodeProcessorContext context, Stream data, TokenStream nodes)
{
while (!Preprocessor.LineBreak(data))
{
Preprocessor.Whitespace(context, data, nodes);
if (!Preprocessor.Comment(context, data))
{
break;
}
}
if (!Preprocessor.LineBreak(context, data, nodes))
{
return(Preprocessor.ThrowError(context, PreprocessorCodes.UnexpectedCharacter));
}
nodes.Add(TokenTypes.TokenSeparator);
return(true);
}
public TokenStream Analize()
{
while (!EOF())
{
// skip all linebreaks and whitespace
while (true)
{
if (next == '\n')
{
readNext('\r');
lineNum++;
}
if (next == '\r')
{
lineNum++;
}
if (char.IsWhiteSpace(next))
{
readNext();
}
else
{
break;
}
}
bool comment = false;
var sb = new StringBuilder();
var count = tokens.Count();
// operators and separators
switch (next)
{
case '=':
if (readNext('='))
{
tokens.Add(Token.BOOL_EQ);
}
else
{
tokens.Add(Token.EQ);
}
break;
case '!':
if (readNext('='))
{
tokens.Add(Token.BOOL_NEQ);
}
else
{
tokens.Add(Token.BOOL_NOT);
}
break;
case '>':
if (readNext('='))
{
tokens.Add(Token.BOOL_GEQ);
}
else if (next == '>')
{
if (readNext('='))
{
tokens.Add(Token.R_SHIFT_SGN_EQ);
}
else if (next == '>')
{
if (readNext('='))
{
tokens.Add(Token.R_SHIFT_USGN_EQ);
}
else
{
tokens.Add(Token.R_SHIFT_USGN);
}
}
else
{
tokens.Add(Token.R_SHIFT_SGN);
}
}
else
{
tokens.Add(Token.BOOL_GRT);
}
break;
case '<':
if (readNext('='))
{
tokens.Add(Token.BOOL_LEQ);
}
else if (next == '<')
{
if (readNext('='))
{
tokens.Add(Token.L_SHIFT_SGN_EQ);
}
else if (next == '<')
{
if (readNext('='))
{
tokens.Add(Token.L_SHIFT_USGN_EQ);
}
else
{
tokens.Add(Token.L_SHIFT_USGN);
}
}
else
{
tokens.Add(Token.L_SHIFT_SGN);
}
}
else
{
tokens.Add(Token.BOOL_LST);
}
break;
case '-':
if (readNext('-'))
{
tokens.Add(Token.DECREMENT);
}
else if (next == '=')
{
readNext();
tokens.Add(Token.MINUS_EQ);
}
else if (next == '>')
{
readNext();
tokens.Add(Token.ARROW);
}
else
{
tokens.Add(Token.MINUS);
}
break;
case '+':
if (readNext('+'))
{
tokens.Add(Token.INCREMENT);
}
else if (next == '=')
{
readNext();
tokens.Add(Token.PLUS_EQ);
}
else
{
tokens.Add(Token.PLUS);
}
break;
case '&':
if (readNext('&'))
{
tokens.Add(Token.BOOL_AND);
}
else if (next == '=')
{
readNext();
tokens.Add(Token.BIT_AND_EQ);
}
else
{
tokens.Add(Token.BIT_AND);
}
break;
case '|':
if (readNext('|'))
{
tokens.Add(Token.BOOL_OR);
}
else if (next == '=')
{
readNext();
tokens.Add(Token.BIT_OR_EQ);
}
else
{
tokens.Add(Token.BIT_OR);
}
break;
case '*':
if (readNext('='))
{
tokens.Add(Token.MULT_EQ);
}
else
{
tokens.Add(Token.MULT);
}
break;
case '/':
if (readNext('/'))
{
comment = true;
while (!readNext('\n'))
{
;
}
}
else if (next == '*')
{
comment = true;
while (true)
{
while (!readNext('*'))
{
;
}
if (next == '/')
{
readNext(); break;
}
}
}
else if (next == '=')
{
tokens.Add(Token.DIV_EQ);
}
else
{
tokens.Add(Token.DIV);
}
break;
case '%':
if (readNext('='))
{
tokens.Add(Token.MOD_EQ);
}
else
{
tokens.Add(Token.MOD);
}
break;
case '^':
if (readNext('='))
{
tokens.Add(Token.XOR_EQ);
}
else
{
tokens.Add(Token.XOR);
}
break;
case ':':
if (readNext(':'))
{
tokens.Add(Token.DOUBLE_COLON);
}
else
{
tokens.Add(Token.TERNARY_ELSE);
}
break;
case '.':
if (readNext('.'))
{
if (readNext('.'))
{
tokens.Add(Token.ELIPSIS);
}
else
{
tokens.Add(Token.DOT);
tokens.Add(Token.DOT);
}
}
else
{
tokens.Add(Token.DOT);
}
break;
case '~':
readNext();
tokens.Add(Token.NOT);
break;
case '?':
readNext();
tokens.Add(Token.TERNARY_THEN);
break;
case '(':
readNext();
tokens.Add(Token.L_PAR);
break;
case ')':
readNext();
tokens.Add(Token.R_PAR);
break;
case '{':
readNext();
tokens.Add(Token.L_BRACE);
break;
case '}':
readNext();
tokens.Add(Token.R_BRACE);
break;
case '[':
readNext();
tokens.Add(Token.L_BRACKET);
break;
case ']':
readNext();
tokens.Add(Token.R_BRACKET);
break;
case ';':
readNext();
tokens.Add(Token.SEMICOLON);
break;
case ',':
readNext();
tokens.Add(Token.COMMA);
break;
case '@':
readNext();
tokens.Add(Token.AT);
break;
case '\'':
sb.Clear();
while (!readNext('\''))
{
if (next == '\\')
{
if (readNext('\''))
{
sb.Append("\'");
}
else
{
sb.Append("\\");
sb.Append(next);
}
}
else
{
sb.Append(next);
}
}
tokens.Add(new Token(Token.TokenType.CharLiteral, sb.ToString()));
break;
case '"':
sb.Clear();
while (!readNext('"'))
{
if (next == '\\')
{
if (readNext('"'))
{
sb.Append("\\\"");
}
else
{
sb.Append('\\');
sb.Append(next);
}
}
else
{
sb.Append(next);
}
}
tokens.Add(new Token(Token.TokenType.StringLiteral, sb.ToString()));
break;
}
if (tokens.Count() != count || comment)
{
continue;
}
// integers and floats
if (char.IsDigit(next))
{
var hexDigts = "abcdefABCDEF";
int numBase = 10;
if (next == '0')
{
if (readNext("xX"))
{
numBase = 16;
}
else if (next == 'b' || next == 'B')
{
readNext();
numBase = 2;
}
else
{
sb.Append(next);
numBase = 8;
}
}
sb.Clear();
sb.Append(next);
while (char.IsDigit(readNext()) || (numBase == 16 && hexDigts.Contains(next)) || next == '_')
{
if (next != '_')
{
sb.Append(next);
}
}
var numeral = sb.ToString();
if (next == 'l' || next == 'L')
{
tokens.Add(new Token(Token.TokenType.IntegerLiteral, Convert.ToInt64(numeral, numBase)));
}
else
{
string floatPart = "";
if ((numBase == 10 || numBase == 16) && next == '.')
{
sb.Clear();
sb.Append(next);
while (char.IsDigit(readNext()) || (numBase == 16 && hexDigts.Contains(next)) || next == '_')
{
if (next != '_')
{
sb.Append(next);
}
}
floatPart = sb.ToString();
}
string exponent = "";
if ((numBase == 16 && "pP".Contains(next)) || (numBase == 10 && "eE".Contains(next)))
{
sb.Clear();
if (readNext("+-") || char.IsDigit(next))
{
sb.Append(next);
}
while (char.IsDigit(readNext()) || next == '_')
{
if (next != '_')
{
sb.Append(next);
}
}
exponent = sb.ToString();
}
if (string.IsNullOrWhiteSpace(floatPart) && string.IsNullOrWhiteSpace(exponent) && !("fFdD".Contains(next)))
{
tokens.Add(new Token(Token.TokenType.IntegerLiteral, Convert.ToInt32(numeral, numBase)));
}
else
{
if (numBase == 10)
{
numeral += floatPart + exponent;
if (next == 'f' || next == 'F')
{
tokens.Add(new Token(Token.TokenType.FloatLiteral, float.Parse(numeral, System.Globalization.CultureInfo.InvariantCulture)));
}
else
{
tokens.Add(new Token(Token.TokenType.FloatLiteral, double.Parse(numeral, System.Globalization.CultureInfo.InvariantCulture)));
}
}
else
{
double num = int.Parse(numeral, System.Globalization.NumberStyles.HexNumber);
double b = 16;
foreach (var ch in floatPart)
{
if (ch != '.')
{
num += int.Parse(ch.ToString(), System.Globalization.NumberStyles.HexNumber) / b;
b *= 16;
}
}
num *= Math.Pow(2, Convert.ToDouble(exponent));
if (next == 'f' || next == 'F')
{
tokens.Add(new Token(Token.TokenType.FloatLiteral, Convert.ToSingle(num)));
}
else
{
tokens.Add(new Token(Token.TokenType.FloatLiteral, num));
}
}
}
}
continue;
}
// identifiers and keywords
if (char.IsLetter(next))
{
sb.Clear();
sb.Append(next.ToString());
while (char.IsLetterOrDigit(readNext()) || next == '_' || next == '$')
{
sb.Append(next);
}
var identifier = sb.ToString();
if (Token.Keywords.ContainsKey(identifier))
{
tokens.Add(Token.Keywords[identifier]);
}
else
{
tokens.Add(new Token(Token.TokenType.Identifier, identifier));
}
continue;
}
tokens.Add(new Token(Token.TokenType.Error, "Error: unknown symbol at line " + lineNum.ToString()));
}
tokens.Add(Token.LAMBDA);
return(tokens);
}
private static bool ExpandFunctionMacro(CodeProcessorContext context, Stream data, TokenStream nodes, Lazy <HashSet <UInt32> > stack, Macro macro)
{
if (stack.Value.Contains(macro.Id))
{
return(false);
}
Whitespace(context, data, nodes);
if (data.Peek() == '(')
{
long streamPos = data.Position;
List <TokenStream> parameter = new List <TokenStream>();
if (FunctionMacroParameter(context, data, parameter))
{
if ((macro.Parameter.Count != parameter.Count && !macro.Variadic))
{
return(ThrowError(context, PreprocessorCodes.ParameterMismatch));
}
bool hasVariableParameter = ((parameter.Count - macro.Parameter.Count) > 0);
macro.ReplacementList.Position = 0;
TokenStream preparsed = new TokenStream();
while (!macro.ReplacementList.Eof())
{
if
(
!Stringify(context, macro, preparsed, parameter) &&
!TokenPasting(context, macro, preparsed, parameter, hasVariableParameter) &&
!ParameterReplacement(context, macro, preparsed, parameter) &&
!String(context, macro.ReplacementList, preparsed) &&
!EscapedLineBreak(context, macro.ReplacementList) &&
!Comment(context, macro.ReplacementList)
)
{
preparsed.Add(macro.ReplacementList.Get());
}
}
TokenStream result = new TokenStream();
while (!preparsed.Eof())
{
IdentifierToken token; if (Identifier(preparsed, out token))
{
object objectMacro; if (context.Definitions.TryGetValue(token.Id, out objectMacro) && (objectMacro as Macro).Parameter.Count == 0 && !(objectMacro as Macro).Variadic)
{
long tmp = preparsed.Position;
preparsed.Position += token.Length;
if (ExpandObjectMacro(context, preparsed, result, stack, (objectMacro as Macro)))
{
continue;
}
preparsed.Position = tmp;
}
preparsed.CopyRange(result, (int)token.Length);
}
else if (!String(context, preparsed, result) && !Comment(context, macro.ReplacementList))
{
result.Add(preparsed.Get());
}
}
AnyPreventLineBreak(context, data, result, stack);
stack.Value.Add(macro.Id);
AnyPreventLineBreak(context, result, nodes, stack);
stack.Value.Remove(macro.Id);
return(true);
}
data.Position = streamPos;
}
return(false);
}
public void SetUp()
{
tokenStream = new TokenStream();
token = new Token("5", TokenType.Number);
tokenStream.Add(token);
}