Ejemplo n.º 1
0
            /// <summary>
            /// Gets a token sequence from the scanner stream
            /// </summary>
            /// <param name="scanner"></param>
            /// <returns></returns>
            protected IEnumerable <Token <ModelTokenType> > GetTokens(ITextStream scanner)
            {
                if (scanner == null)
                {
                    throw new ArgumentNullException("scanner");
                }

                // store for external access to Index/Line/Column
                this.Scanner = scanner;

                int             depth           = 0;
                NeedsValueDelim needsValueDelim = NeedsValueDelim.Forbidden;

                while (true)
                {
                    // skip comments and whitespace between tokens
                    JsonTokenizer.SkipCommentsAndWhitespace(scanner);

                    if (scanner.IsCompleted)
                    {
                        this.Scanner = StringStream.Null;
                        scanner.Dispose();
                        yield break;
                    }

                    bool hasUnaryOp = false;

                    char ch = scanner.Peek();
                    switch (ch)
                    {
                    case JsonGrammar.OperatorArrayBegin:
                    {
                        scanner.Pop();
                        if (needsValueDelim == NeedsValueDelim.Required)
                        {
                            throw new DeserializationException(JsonTokenizer.ErrorMissingValueDelim, scanner.Index, scanner.Line, scanner.Column);
                        }

                        yield return(ModelGrammar.TokenArrayBeginUnnamed);

                        depth++;
                        needsValueDelim = NeedsValueDelim.Forbidden;
                        continue;
                    }

                    case JsonGrammar.OperatorArrayEnd:
                    {
                        scanner.Pop();
                        if (needsValueDelim == NeedsValueDelim.CurrentIsDelim)
                        {
                            throw new DeserializationException(JsonTokenizer.ErrorExtraValueDelim, scanner.Index, scanner.Line, scanner.Column);
                        }

                        yield return(ModelGrammar.TokenArrayEnd);

                        // resetting at zero allows streaming mode
                        depth--;
                        needsValueDelim = (depth > 0) ? NeedsValueDelim.Required : NeedsValueDelim.Forbidden;
                        continue;
                    }

                    case JsonGrammar.OperatorObjectBegin:
                    {
                        scanner.Pop();
                        if (needsValueDelim == NeedsValueDelim.Required)
                        {
                            throw new DeserializationException(JsonTokenizer.ErrorMissingValueDelim, scanner.Index, scanner.Line, scanner.Column);
                        }

                        yield return(ModelGrammar.TokenObjectBeginUnnamed);

                        depth++;
                        needsValueDelim = NeedsValueDelim.Forbidden;
                        continue;
                    }

                    case JsonGrammar.OperatorObjectEnd:
                    {
                        scanner.Pop();
                        if (needsValueDelim == NeedsValueDelim.CurrentIsDelim)
                        {
                            throw new DeserializationException(JsonTokenizer.ErrorExtraValueDelim, scanner.Index, scanner.Line, scanner.Column);
                        }

                        yield return(ModelGrammar.TokenObjectEnd);

                        // resetting at zero allows streaming mode
                        depth--;
                        needsValueDelim = (depth > 0) ? NeedsValueDelim.Required : NeedsValueDelim.Forbidden;
                        continue;
                    }

                    case JsonGrammar.OperatorStringDelim:
                    case JsonGrammar.OperatorStringDelimAlt:
                    {
                        if (needsValueDelim == NeedsValueDelim.Required)
                        {
                            throw new DeserializationException(JsonTokenizer.ErrorMissingValueDelim, scanner.Index, scanner.Line, scanner.Column);
                        }

                        string value = JsonTokenizer.ScanString(scanner);

                        JsonTokenizer.SkipCommentsAndWhitespace(scanner);
                        if (scanner.Peek() == JsonGrammar.OperatorPairDelim)
                        {
                            scanner.Pop();
                            yield return(ModelGrammar.TokenProperty(new DataName(value)));

                            needsValueDelim = NeedsValueDelim.Forbidden;
                            continue;
                        }

                        yield return(ModelGrammar.TokenPrimitive(value));

                        needsValueDelim = NeedsValueDelim.Required;
                        continue;
                    }

                    case JsonGrammar.OperatorUnaryMinus:
                    case JsonGrammar.OperatorUnaryPlus:
                    {
                        hasUnaryOp = true;
                        break;
                    }

                    case JsonGrammar.OperatorValueDelim:
                    {
                        scanner.Pop();

                        if (needsValueDelim != NeedsValueDelim.Required)
                        {
                            throw new DeserializationException(JsonTokenizer.ErrorExtraValueDelim, scanner.Index, scanner.Line, scanner.Column);
                        }

                        needsValueDelim = NeedsValueDelim.CurrentIsDelim;
                        continue;
                    }

                    case JsonGrammar.OperatorPairDelim:
                    {
                        throw new DeserializationException(JsonTokenizer.ErrorUnrecognizedToken, scanner.Index + 1, scanner.Line, scanner.Column);
                    }
                    }

                    if (needsValueDelim == NeedsValueDelim.Required)
                    {
                        throw new DeserializationException(JsonTokenizer.ErrorMissingValueDelim, scanner.Index, scanner.Line, scanner.Column);
                    }

                    // scan for numbers
                    Token <ModelTokenType> token = JsonTokenizer.ScanNumber(scanner);
                    if (token != null)
                    {
                        yield return(token);

                        needsValueDelim = NeedsValueDelim.Required;
                        continue;
                    }

                    // hold for Infinity, clear for others
                    if (!hasUnaryOp)
                    {
                        ch = default(char);
                    }

                    // store for unterminated cases
                    long strPos  = scanner.Index + 1;
                    int  strLine = scanner.Line;
                    int  strCol  = scanner.Column;

                    // scan for identifiers, then check if they are keywords
                    string ident = JsonTokenizer.ScanIdentifier(scanner);
                    if (!String.IsNullOrEmpty(ident))
                    {
                        token = JsonTokenizer.ScanKeywords(scanner, ident, ch, out needsValueDelim);
                        if (token != null)
                        {
                            yield return(token);

                            continue;
                        }
                    }

                    throw new DeserializationException(JsonTokenizer.ErrorUnrecognizedToken, strPos, strLine, strCol);
                }
            }