/// <summary>
/// Gets a token sequence from the scanner stream
/// </summary>
/// <param name="scanner"></param>
/// <returns></returns>
protected IEnumerable <Token <ModelTokenType> > GetTokens(ITextStream scanner)
{
if (scanner == null)
{
throw new ArgumentNullException("scanner");
}
// store for external access to Index/Line/Column
this.Scanner = scanner;
int depth = 0;
NeedsValueDelim needsValueDelim = NeedsValueDelim.Forbidden;
while (true)
{
// skip comments and whitespace between tokens
JsonTokenizer.SkipCommentsAndWhitespace(scanner);
if (scanner.IsCompleted)
{
this.Scanner = StringStream.Null;
scanner.Dispose();
yield break;
}
bool hasUnaryOp = false;
char ch = scanner.Peek();
switch (ch)
{
case JsonGrammar.OperatorArrayBegin:
{
scanner.Pop();
if (needsValueDelim == NeedsValueDelim.Required)
{
throw new DeserializationException(JsonTokenizer.ErrorMissingValueDelim, scanner.Index, scanner.Line, scanner.Column);
}
yield return(ModelGrammar.TokenArrayBeginUnnamed);
depth++;
needsValueDelim = NeedsValueDelim.Forbidden;
continue;
}
case JsonGrammar.OperatorArrayEnd:
{
scanner.Pop();
if (needsValueDelim == NeedsValueDelim.CurrentIsDelim)
{
throw new DeserializationException(JsonTokenizer.ErrorExtraValueDelim, scanner.Index, scanner.Line, scanner.Column);
}
yield return(ModelGrammar.TokenArrayEnd);
// resetting at zero allows streaming mode
depth--;
needsValueDelim = (depth > 0) ? NeedsValueDelim.Required : NeedsValueDelim.Forbidden;
continue;
}
case JsonGrammar.OperatorObjectBegin:
{
scanner.Pop();
if (needsValueDelim == NeedsValueDelim.Required)
{
throw new DeserializationException(JsonTokenizer.ErrorMissingValueDelim, scanner.Index, scanner.Line, scanner.Column);
}
yield return(ModelGrammar.TokenObjectBeginUnnamed);
depth++;
needsValueDelim = NeedsValueDelim.Forbidden;
continue;
}
case JsonGrammar.OperatorObjectEnd:
{
scanner.Pop();
if (needsValueDelim == NeedsValueDelim.CurrentIsDelim)
{
throw new DeserializationException(JsonTokenizer.ErrorExtraValueDelim, scanner.Index, scanner.Line, scanner.Column);
}
yield return(ModelGrammar.TokenObjectEnd);
// resetting at zero allows streaming mode
depth--;
needsValueDelim = (depth > 0) ? NeedsValueDelim.Required : NeedsValueDelim.Forbidden;
continue;
}
case JsonGrammar.OperatorStringDelim:
case JsonGrammar.OperatorStringDelimAlt:
{
if (needsValueDelim == NeedsValueDelim.Required)
{
throw new DeserializationException(JsonTokenizer.ErrorMissingValueDelim, scanner.Index, scanner.Line, scanner.Column);
}
string value = JsonTokenizer.ScanString(scanner);
JsonTokenizer.SkipCommentsAndWhitespace(scanner);
if (scanner.Peek() == JsonGrammar.OperatorPairDelim)
{
scanner.Pop();
yield return(ModelGrammar.TokenProperty(new DataName(value)));
needsValueDelim = NeedsValueDelim.Forbidden;
continue;
}
yield return(ModelGrammar.TokenPrimitive(value));
needsValueDelim = NeedsValueDelim.Required;
continue;
}
case JsonGrammar.OperatorUnaryMinus:
case JsonGrammar.OperatorUnaryPlus:
{
hasUnaryOp = true;
break;
}
case JsonGrammar.OperatorValueDelim:
{
scanner.Pop();
if (needsValueDelim != NeedsValueDelim.Required)
{
throw new DeserializationException(JsonTokenizer.ErrorExtraValueDelim, scanner.Index, scanner.Line, scanner.Column);
}
needsValueDelim = NeedsValueDelim.CurrentIsDelim;
continue;
}
case JsonGrammar.OperatorPairDelim:
{
throw new DeserializationException(JsonTokenizer.ErrorUnrecognizedToken, scanner.Index + 1, scanner.Line, scanner.Column);
}
}
if (needsValueDelim == NeedsValueDelim.Required)
{
throw new DeserializationException(JsonTokenizer.ErrorMissingValueDelim, scanner.Index, scanner.Line, scanner.Column);
}
// scan for numbers
Token <ModelTokenType> token = JsonTokenizer.ScanNumber(scanner);
if (token != null)
{
yield return(token);
needsValueDelim = NeedsValueDelim.Required;
continue;
}
// hold for Infinity, clear for others
if (!hasUnaryOp)
{
ch = default(char);
}
// store for unterminated cases
long strPos = scanner.Index + 1;
int strLine = scanner.Line;
int strCol = scanner.Column;
// scan for identifiers, then check if they are keywords
string ident = JsonTokenizer.ScanIdentifier(scanner);
if (!String.IsNullOrEmpty(ident))
{
token = JsonTokenizer.ScanKeywords(scanner, ident, ch, out needsValueDelim);
if (token != null)
{
yield return(token);
continue;
}
}
throw new DeserializationException(JsonTokenizer.ErrorUnrecognizedToken, strPos, strLine, strCol);
}
}