public static IEnumerable <IToken> Rebuild(IEnumerable <IToken> tokens)
{
if (tokens == null)
{
throw new ArgumentNullException("tokens");
}
// Note: There are some limitations to what can be done here - for example "Test -1" can not be have the "-" and "1" recombined since,
// at this point, we don't know if "Test" is a value that 1 should be subtracted from or whether it's a function that is being
// called with "-1" as the argument. However, for some values of "Test", we CAN be sure that it's not a substraction; if the
// value is a keyword or a comparison operator (as in the "FOR", "TO" and "=" in "FOR a = -1 TO -10 STEP -1" then the minus
// sign must be part of a numeric value).
// At the beginning of a token set, if the first token is a minus sign then it is elligible to be part of a number. If it is not the
// first token then it may only incorporated into a number if preceded by another operator or an opening brace (which effectively
// would make the token the start of a new expression). The same principle applies to a decimal point. This is why the initial
// processor is a PeriodOrMinusSignOrNumberCouldIndicateStartOfNumber.
var processor = (IAmLookingForNumberContent)PeriodOrMinusSignOrNumberCouldIndicateStartOfNumber.Instance;
var numberContent = new PartialNumberContent();
var rebuiltTokens = new List <IToken>();
var tokenArray = tokens.ToArray();
if (tokenArray.Any(t => t == null))
{
throw new ArgumentException("Null reference encountered in tokens set");
}
for (var index = 0; index < tokenArray.Length; index++)
{
var result = processor.Process(tokenArray.Skip(index), numberContent);
if (result.ProcessedTokens.Any())
{
rebuiltTokens.AddRange(result.ProcessedTokens);
}
processor = result.NextProcessor;
numberContent = result.NumberContent;
}
if (numberContent.Tokens.Any())
{
var numbericValueToken = numberContent.TryToExpressNumericValueTokenFromCurrentTokens();
if (numbericValueToken == null)
{
rebuiltTokens.AddRange(numberContent.Tokens);
}
else
{
rebuiltTokens.Add(numbericValueToken);
}
}
return(rebuiltTokens.Select(t => (t is MemberAccessorOrDecimalPointToken) ? new MemberAccessorToken(t.LineIndex) : t));
}
public TokenProcessResult(PartialNumberContent numberContent, IEnumerable<IToken> processedTokens, IAmLookingForNumberContent nextProcessor)
{
if (numberContent == null)
throw new ArgumentNullException("numberContent");
if (processedTokens == null)
throw new ArgumentNullException("processedTokens");
if (nextProcessor == null)
throw new ArgumentNullException("nextProcessor");
NumberContent = numberContent;
ProcessedTokens = processedTokens.ToList().AsReadOnly();
if (ProcessedTokens.Any(t => t == null))
throw new ArgumentException("Null reference encountered in processedTokens set");
NextProcessor = nextProcessor;
}
