Exemple #1
0
 private Token ReadIntegerToken(ref string content, ref int pos, TokenPosition position)
 {
     var literal = ReadLiteral(ref content, ref pos).ToLower();
     if (literal.StartsWith("0x")) {
         return ParsePrefixedHexInteger(literal, position);
     }
     if (literal.EndsWith("h")) {
         return ParsePostfixedHexInteger(literal, position);
     }
     return ReadDecimalInteger(ref literal, position);
 }
Exemple #2
0
 private Token ParsePrefixedHexInteger(string literal, TokenPosition position)
 {
     long val = 0;
     var pos = 2;
     if (pos >= literal.Length) throw new TokenException("Unexpeced end of hex constant", new Token { Position = position, StringValue = literal });
     while (pos < literal.Length) {
         var ch = literal[pos++];
         var hex = GetHexValue(ch);
         if (hex < 0) throw new TokenException("unexpected hex symbol '" + ch + "'", new Token { Position = position, StringValue = literal });
         val = val * 16 + hex;
     }
     return new Token {
         Type = TokenType.Integer,
         IntegerValue = val,
         StringValue = literal,
         Position = position
     };
 }
Exemple #3
0
 private static Token ReadDecimalInteger(ref string literal, TokenPosition position)
 {
     long val = 0;
     var pos = 0;
     while (pos < literal.Length) {
         var ch = literal[pos++];
         if (char.IsDigit(ch)) {
             val = val * 10 + (ch - '0');
         } else {
             throw new TokenException("unexpected decimal symbol '" + ch + "'", new Token { Position = position, StringValue = literal });
         }
     }
     return new Token {
         Type = TokenType.Integer,
         IntegerValue = val,
         StringValue = literal,
         Position = position
     };
 }
Exemple #4
0
 private static string ReadString(ref string content, ref int pos, TokenPosition position)
 {
     var quoteCh = content[pos++];
     var token = "";
     while (pos < content.Length) {
         var ch = content[pos++];
         if (ch == '\r' || ch == '\n') {
             throw new TokenException("missing end quote", new Token { Position = position });
         }
         if (ch == '\\') {
             if (pos >= content.Length) throw new TokenException("missing end quote", new Token { Position = position });
             ch = content[pos++];
             switch (ch) {
                 case 'n':
                     token += '\n';
                     break;
                 case 'r':
                     token += '\r';
                     break;
                 case 'a':
                     token += '\a';
                     break;
                 case 'b':
                     token += '\b';
                     break;
                 case 'f':
                     token += '\f';
                     break;
                 case 't':
                     token += '\t';
                     break;
                 case 'v':
                     token += '\v';
                     break;
                 case '\\':
                     token += '\\';
                     break;
                 case '\'':
                     token += '\'';
                     break;
                 case '\"':
                     token += '\"';
                     break;
                 case '0':
                     token += '\0';
                     break;
                 default:
                     throw new TokenException("invalid escape character " + ch, new Token { Position = position });
             }
         } else if (ch == quoteCh) {
             if (pos >= content.Length) return token;
             var next = content[pos++];
             if (next == quoteCh) {
                 token += ch;
             } else {
                 pos--;
                 return token;
             }
         } else {
             token += ch;
         }
     }
     throw new TokenException("missing end quote", new Token { Position = position });
 }
Exemple #5
0
        public IList<Token> Read(FileSource source)
        {
            var content = source.Content;
            var res = new List<Token>();
            var pos = 0;
            var lineNumber = 1;
            var lineStart = 0;
            while (pos < content.Length) {
                SkipWhitespace(ref content, ref pos);
                if (pos >= content.Length) break;
                var position = new TokenPosition { File = source, Line = lineNumber, LineStart = lineStart };

                var ch = content[pos++];
                var preview = (char)(0xffff);
                if (pos < content.Length) {
                    preview = content[pos];
                }
                if (ch == '>' && preview == '>') {
                    res.Add(new Token {
                        Type = TokenType.RightShift,
                        StringValue = ">",
                        Position = position
                    });
                    pos++;
                    continue;
                }
                if (ch == '<' && preview == '<') {
                    res.Add(new Token {
                        Type = TokenType.LeftShift,
                        StringValue = "<",
                        Position = position
                    });
                    pos++;
                    continue;
                }
                switch (ch) {
                    case '\r':
                    case '\n':
                        SkipNewLineChar(ref content, ref pos, ch);
                        lineNumber++;
                        lineStart = pos;
                        res.Add(new Token {
                            Type = TokenType.NewLine,
                            Position = position
                        });
                        break;
                    case ';':
                        SkipLine(ref content, ref pos);
                        lineNumber++;
                        lineStart = pos;
                        res.Add(new Token {
                            Type = TokenType.NewLine,
                            Position = position
                        });
                        break;
                    case '"':
                    case '\'':
                        pos--;
                        res.Add(new Token {
                            Type = TokenType.String,
                            StringValue = ReadString(ref content, ref pos, position),
                            Position = position
                        });
                        break;
                    case '\\':
                        SkipWhitespace(ref content, ref pos);
                        if (pos >= content.Length) throw new Exception("Unexpected end of file");
                        ch = content[pos++];
                        if (ch == '\r' || ch == '\n') {
                            SkipNewLineChar(ref content, ref pos, ch);
                            lineNumber++;
                            lineStart = pos;
                        } else {
                            throw new Exception("Unexpected character after backslash");
                        }
                        break;
                    default:
                        if (IsPunctuation(ch)) {
                            res.Add(new Token {
                                Type = GetPunctuationTokenType(ch),
                                StringValue = ch.ToString(),
                                Position = position
                            });
                        } else if (char.IsDigit(ch)) {
                            pos--;
                            res.Add(ReadIntegerToken(ref content, ref pos, position));
                        } else {
                            pos--;
                            res.Add(new Token {
                                Type = TokenType.Literal,
                                StringValue = ReadLiteral(ref content, ref pos),
                                Position = position
                            });
                        }
                        break;
                }
            }
            return res;
        }