示例#1
0
 private static void matchSymbol(Scanner s)
 {
     if (!s.isInitial())
         return;
     s.readChar();
     while (s.isSubsequent())
         s.readChar();
 }
示例#2
0
 // Remove the '"' delimiters surrounding the token that came
 // back from the lexer. Also 'unescape' any backslashes.
 private static string unescape(string s)
 {
     // Remove surrounding quotes
     s = s.Substring(1, s.Length - 2);
     // Regex.Unescape solves the problem of converting \n, \t etc
     // for us.
     return System.Text.RegularExpressions.Regex.Unescape(s);
 }
示例#3
0
 private static bool matchSymbol(Scanner s)
 {
     if (!s.isInitial())
         return false;
     s.readChar();
     while (s.isSubsequent())
         s.readChar();
     return true;
 }
示例#4
0
 public static IEnumerable<Datum> ReadDatums(Assembly assembly, string resourceFile)
 {
     var stream = assembly.GetManifestResourceStream(resourceFile);
     if (stream == null)
         throw new Exception(string.Format("Unable to find '{0}' embedded resource", resourceFile));
     var s = new Scanner(new StreamReader(stream)) { Filename = resourceFile };
     var p = new Parser(s);
     Datum d;
     while ((d = p.parse()) != null)
     {
         yield return d;
     }
 }
示例#5
0
 public Parser(Scanner s)
 {
     this.s = s;
     initTokens(s.Scan());
 }
示例#6
0
 // .Net method support.
 // Check if we received a "." or a ".symbol".
 // Given them different tokens so that the parser
 // can expand appropriately.
 private static TokenType? matchDot(Scanner s)
 {
     if (s.peek() != '.')
         return null;
     s.readChar();
     matchSymbol(s);
     return s.sb.Length > 1 ? TokenType.Symbol : TokenType.Dot;
 }
示例#7
0
 // TODO:
 // WE should implement actual reader macros for this part of things.
 private static TokenType? matchHash(Scanner s)
 {
     if (s.peek() != '#')
         return null;
     s.readChar();
     if(s.peek() == '(')
     {
         s.readChar();
         return TokenType.VectorOpen;
     }
     if(s.isOneOf("tfTF"))
     {
         s.readChar();
         return TokenType.Boolean;
     }
     throw s.fail("Unrecognized token");
 }
示例#8
0
        // This code is pretty hairy - the problem is
        // distinguishing between:
        // 1. a floating point number, which may be
        // in scientific notification, and might simply begin
        // with "."
        // 2. A symbol with a dot in it
        // 3. Just a lone "." used for separating lists

        // It's doubtful that it's entirely correct - but if you find a bug,
        // add it as a test to ScannerTest and amend this logic.
        private static TokenType? matchNumber(Scanner s)
        {
            if(s.peek() == '.')
                return leadingFloat(s);

            if(s.isOneOf("+-"))
            {
                s.readChar();
                if(s.peek() == '.')
                    return leadingFloat(s);
                var num = unsignedNumber(s);
                if (num != null)
                    return num;

                matchSymbol(s);
                return TokenType.Symbol;
            }
            return unsignedNumber(s);
        }
示例#9
0
 private static TokenType? leadingFloat(Scanner s)
 {
     var floatToken = remainingFloat(s);
     if (s.sb.Length == 1)
     {
         matchSymbol(s);
         return s.sb.Length == 1 ? TokenType.Dot : TokenType.Symbol;
     }
     return floatToken;
 }
示例#10
0
 private static TokenType? unsignedNumber(Scanner s)
 {
     if (!s.isDigit())
         return null;
     while (s.isDigit())
         s.readChar();
     if (s.isExponent())
         return readExponent(s);
     if (s.peek() != '.')
         return TokenType.Integer;
     return remainingFloat(s);
 }
示例#11
0
 private static TokenType? remainingFloat(Scanner s)
 {
     s.readChar(); // Skip the '.'
     while (s.isDigit() || s.isExponent())
     {
         if (s.isExponent())
             return readExponent(s);
         s.readChar();
     }
     return TokenType.Double;
 }
示例#12
0
 private static TokenType? readExponent(Scanner s)
 {
     s.readChar(); // Skip exponent market
     var read = false;
     if (s.isOneOf("+-"))
     {
         s.readChar();
         read = true;
     }
     while (s.isDigit())
     {
         read = true;
         s.readChar();
     }
     if(read)
         return TokenType.Double;
     // It's not a floating point number in exponent format.
     // Read it as a symbol instead.
     matchSymbol(s);
     return TokenType.Symbol;
 }
示例#13
0
 public Parser(Scanner s)
 {
     this.s = s;
     // Skip whitespace and comments
     tokens = s.Scan().Where(token => token.Type != TokenType.Space && token.Type != TokenType.Comment).GetEnumerator();
 }