/// <summary>
/// Creates delimiter list from string,
/// can request lines if missing closing delimiter
/// </summary>
/// <param name="str">line to parse</param>
/// <param name="requestor">used to ask for additional lines if needed, may be null</param>
/// <param name="delims">used to ask questions about delimiters</param>
internal static DelimiterList Do(string str, IParseLineDelimiters delims, ILineRequestor requestor)
{
string[] strs = ParseChars.Do(str, delims);
int indent = Utility.CountIndent(str);
int end;
return Do(indent, str, strs, 0, ValueDelimiter.Line, delims, requestor, out end);
}
/// <summary>
/// Eval a list of DelimiterNodes and return a Value
/// </summary>
internal static Value Do(List<DelimiterNode> nodes, IScope scope, ILineRequestor requestor)
{
ListEval eval = new ListEval(nodes, scope);
while (eval.EvalNext(requestor))
;
return eval.GetValue();
}
private static DelimiterList Do(int indent, string original, string[] strs, int iStart, ValueDelimiter thisDelim,
IParseLineDelimiters delims, ILineRequestor requestor, out int iEnd)
{
List<DelimiterNode> nodes = new List<DelimiterNode>();
DelimiterType type = thisDelim.DelimiterType;
if (type == DelimiterType.AsComment)
{ // ignore everything up to the end delimiter
DelimiterList result = null;
if (ParseComment(strs, iStart, thisDelim, out iEnd, out result))
return result;
}
else if (type == DelimiterType.AsString)
{ // simply search for end and stuff everything in the middle into a single token
DelimiterList result = null;
if (ParseString(indent, strs, iStart, thisDelim, nodes, out iEnd, out result))
return result;
}
else // Value, Array && Raw
{ // handle as individual tokens and nested lists
DelimiterList result = null;
if (ParseMisc(indent, original, strs, iStart, thisDelim, delims, requestor, nodes, out iEnd, out result))
return result;
}
// didn't find closing delimiter, TODO request the next line
if (thisDelim != ValueDelimiter.Line && thisDelim.End != "")
throw new Loki3Exception().AddMissingEndDelimiter(thisDelim);
iEnd = strs.Length;
string trimmed = original.TrimStart(' ', '\t');
return new DelimiterList(thisDelim, nodes, indent, "", trimmed, null);
}
internal override Value Eval(DelimiterNode prev, DelimiterNode next, IScope paramScope, IScope bodyScope, INodeRequestor nodes, ILineRequestor requestor)
{
// if we need prev or next params but they're not there, simply return the function
// note: if we let m_function.Eval do this, we end up with CreateFunction.UserFunction
if ((m_function.ConsumesPrevious || m_function.ConsumesNext) && prev == null && next == null)
return this;
return m_function.Eval(prev, next, paramScope, m_scope, nodes, requestor);
}
/// <summary>
/// Evaluate line and body if needed.
/// 'requestor' is positioned on the next line to eval.
/// </summary>
internal static Value DoOne(ILineRequestor requestor, IScope scope)
{
DelimiterList list = requestor.GetCurrentLine(scope);
try
{
// eval using list's scope, falling back to passed in scope if not present
IScope theScope = (list.Scope != null ? list.Scope : scope);
Value value = EvalList.Do(list.Nodes, theScope, requestor);
// if only item on line was a func, see if it needs a body
ValueFunction func = value as ValueFunction;
if (func != null && func.RequiresBody())
// if line created a partial function that needs a body,
// eval all subsequent indented lines
value = EvalList.DoAddBody(func, scope, requestor);
// if only item was a map w/ one value, see if it needs a body
if (func == null && value is ValueMap)
{
Map map = value.AsMap;
if (map.Raw.Count == 1)
{
string key = "";
foreach (string k in map.Raw.Keys)
key = k;
func = map[key] as ValueFunction;
if (func != null && func.RequiresBody())
map[key] = EvalList.DoAddBody(func, scope, requestor);
}
}
// if only item was an array, see if last item needs a body
if (func == null && value is ValueArray)
{
List<Value> array = value.AsArray;
if (array.Count > 0)
{
func = array[array.Count - 1] as ValueFunction;
if (func != null && func.RequiresBody())
array[array.Count - 1] = EvalList.DoAddBody(func, scope, requestor);
}
}
requestor.Advance();
return value;
}
catch (Loki3Exception e)
{
e.AddLine(list.ToString());
throw e;
}
}
/// <summary>
/// If function requires a body & it follows current line, add on body.
/// 'requestor' will be advanced to the first line after the body.
/// </summary>
/// <returns>new function with body attached</returns>
internal static Value DoAddBody(ValueFunction function, IScope scope, ILineRequestor requestor)
{
List<DelimiterList> body = DoGetBody(scope, requestor);
// if no body to add to function, leave as-is
if (body.Count == 0)
return function;
// we've built the entire body - now pass it to function
Map map = new Map();
map[ValueFunction.keyBody] = new ValueLine(body, scope);
ValueFunctionPre functionPre = function as ValueFunctionPre;
return functionPre.Eval(new ValueMap(map), new ScopeChain(scope));
}
/// <summary>Evaluate lines till we run out</summary>
internal static Value Do(ILineRequestor requestor, IScope scope)
{
int lineNumber = 0;
try
{
Value value = null;
while (requestor.HasCurrent())
{
lineNumber = requestor.GetCurrentLineNumber();
value = EvalLines.DoOne(requestor, scope);
}
return value;
}
catch (Loki3Exception e)
{
e.AddLineNumber(lineNumber);
throw e;
}
}
/// <summary>
/// Get the body following the current line.
/// 'requestor' will be positioned on the last line of the body.
/// </summary>
internal static List<DelimiterList> DoGetBody(IScope scope, ILineRequestor requestor)
{
List<DelimiterList> body = new List<DelimiterList>();
// if we have a subset of all lines, we should simply use them as-is
if (requestor.IsSubset())
{
while (requestor.HasCurrent())
{
DelimiterList dline = requestor.GetCurrentLine(scope);
dline.Scope = scope; // use this scope when evaling later
body.Add(dline);
requestor.Advance();
}
return body;
}
// else just grab indented lines
DelimiterList pline = requestor.GetCurrentLine(scope);
int parentIndent = pline.Indent;
while (requestor.HasCurrent())
{
requestor.Advance();
DelimiterList childLine = requestor.GetCurrentLine(scope);
if (childLine == null || childLine.Indent <= parentIndent)
{
requestor.Rewind();
break; // now we have the body
}
childLine.Scope = scope; // use this scope when evaling later
// keep adding to the body
body.Add(childLine);
}
return body;
}
internal override Value Eval(DelimiterNode prev, DelimiterNode next, IScope paramScope, IScope bodyScope, INodeRequestor nodes, ILineRequestor requestor)
{
if (m_prev != null)
return m_nested.Eval(m_prev, next, paramScope, bodyScope, nodes, requestor);
return m_nested.Eval(prev, m_next, paramScope, bodyScope, nodes, requestor);
}
/// <summary>Examine overloads to figure out which function to eval</summary>
internal override Value Eval(DelimiterNode prev, DelimiterNode next, IScope paramScope, IScope bodyScope, INodeRequestor nodes, ILineRequestor requestor)
{
// if we've got a single function, simply use it
List<Value> functions = m_functions.AsArray;
if (m_functions.Count == 1)
{
ValueFunction single = functions[0] as ValueFunction;
return single.Eval(prev, next, paramScope, bodyScope, nodes, requestor);
}
if ((m_bConsumesPrevious && prev == null) || (m_bConsumesNext && next == null))
{
// if this is an infix with one side specified, create a partial
if (m_bConsumesPrevious && m_bConsumesNext && (prev != null || next != null))
return new PartialFunctionIn(this, prev, next);
// if we just needed a pre or post & it's not there, eval as ourselves
return this;
}
// todo: avoid evaling twice (or is it cached behind the scenes?)
// and pattern matching twice for the successfull function
Value value1 = (m_bConsumesPrevious ? EvalNode.Do(prev, paramScope, nodes, requestor) : null);
Value value2 = (m_bConsumesNext ? EvalNode.Do(next, paramScope, nodes, requestor) : null);
// eval the first function that's a full match,
// else eval the first function that was a match with leftover,
// else fail
ValueFunction best = null;
foreach (Value value in functions)
{
ValueFunction function = value as ValueFunction;
Value match1 = null, match2 = null;
Value leftover1 = null, leftover2 = null;
// check parameters
if (function.ConsumesPrevious)
if (!PatternChecker.Do(value1, function.Metadata[ValueFunction.keyPreviousPattern], false/*bShortPat*/, out match1, out leftover1))
continue;
if (function.ConsumesNext)
if (!PatternChecker.Do(value2, function.Metadata[ValueFunction.keyNextPattern], false/*bShortPat*/, out match2, out leftover2))
continue;
// if no leftover, we found our function
if (leftover1 == null && leftover2 == null)
{
best = function;
break;
}
// if this is the first function w/ leftover, we'll eval it if we don't find a later match
if (best == null)
best = function;
}
if (best == null)
// todo: NoMatches
throw new Loki3Exception();
// eval the best match we found
return best.Eval(prev, next, paramScope, bodyScope, nodes, requestor);
}
internal override Value Eval(DelimiterNode prev, DelimiterNode next, IScope paramScope, IScope bodyScope, INodeRequestor nodes, ILineRequestor requestor)
{
// if we need prev or next params but they're not there, simply return the function
if ((m_usePrevious || m_useNext) && prev == null && next == null)
return this;
// scope we'll add prev/next/body params to
ScopeChain localScope = new ScopeChain();
if (m_usePrevious)
{
if (prev == null)
{
if (m_useNext)
return new PartialFunctionIn(this, null, next);
throw new Loki3Exception().AddMissingValue(true/*bPrevious*/);
}
Value value1 = ComputeParam(Metadata[keyPreviousPattern], prev, paramScope, nodes, requestor);
Value match, leftover;
if (!PatternChecker.Do(value1, Metadata[keyPreviousPattern], false/*bShortPat*/, out match, out leftover))
throw new Loki3Exception().AddWrongPattern(Metadata[keyPreviousPattern], value1);
if (leftover != null)
{
if (m_useNext)
// currently can't do partials for infix
throw new Loki3Exception().AddWrongPattern(Metadata[keyPreviousPattern], value1);
// create a partial function that starts w/ match & still needs leftover
return new UserFunction(this, match, leftover, null);
}
Utility.AddToScope(m_pattern1, match, localScope);
}
if (m_useNext)
{
if (next == null)
{
if (m_usePrevious)
return new PartialFunctionIn(this, prev, null);
throw new Loki3Exception().AddMissingValue(false/*bPrevious*/);
}
Value value2 = ComputeParam(Metadata[keyNextPattern], next, paramScope, nodes, requestor);
Value match, leftover;
if (!PatternChecker.Do(value2, Metadata[keyNextPattern], false/*bShortPat*/, out match, out leftover))
throw new Loki3Exception().AddWrongPattern(Metadata[keyNextPattern], value2);
// if we created a function that needs a body, add it if present
ValueFunction matchFunc = match as ValueFunction;
if (matchFunc != null && matchFunc.RequiresBody() && requestor != null)
match = EvalList.DoAddBody(matchFunc, bodyScope, requestor);
if (leftover != null)
{
if (m_usePrevious)
// currently can't do partials for infix
throw new Loki3Exception().AddWrongPattern(Metadata[keyNextPattern], value2);
// create a partial function that starts w/ match & still needs leftover
return new UserFunction(this, match, null, leftover);
}
Utility.AddToScope(m_pattern2, match, localScope);
}
// tack on body if requested
if (Metadata.GetOptionalT<bool>("body?", false))
{
bool foundBody = false;
// if there's a following node, use it
DelimiterNode possibleBody = nodes.GetNext();
if (possibleBody != null)
{
localScope.SetValue("body", UseNodeAsBody(possibleBody));
foundBody = true;
}
// if no body, use the following lines
if (!foundBody && requestor != null)
{
List<DelimiterList> body = EvalList.DoGetBody(bodyScope, requestor);
if (body.Count != 0)
{
localScope.SetValue("body", new ValueLine(body, bodyScope));
foundBody = true;
}
}
if (!foundBody)
// create a partial function that only needs a body
return new UserFunction(this, localScope);
}
// create a new scope and add passed in arguments...
ScopeChain scope = (ShouldCreateScope ? new ScopeChain(bodyScope) : bodyScope as ScopeChain);
scope.Function = this;
if (m_fullPattern != null && m_passed != null)
Utility.AddToScope(m_fullPattern, m_passed, scope);
// ...and the prev/next/body params we just extracted
Utility.AddToScope(localScope, scope);
if (m_passedScope != null)
Utility.AddToScope(m_passedScope, scope);
// lazily parse
EnsureParsed(bodyScope);
// eval each line using current scope
try
{
Value retval = EvalBody.Do(m_parsedLines, scope);
scope.Exit();
return retval;
}
catch (PopStackException pop)
{ // if we're supposed to pop back to here then return, else keep throwing up the stack
if (pop.ScopeName == scope.Name)
return pop.Return;
throw pop;
}
catch (Loki3Exception e)
{ // if this scope catches exceptions, stop here
if (Loki3Exception.catchScopeName == scope.Name)
{
if (scope.Parent != null)
scope.Parent.SetValue(Loki3Exception.exceptionKey, new ValueMap(e.Errors));
return ValueNil.Nil;
}
throw e;
}
}
/// <summary>
/// If the pattern's metadata says it matches raw, return the unevaled value,
/// otherwise return the evaled value
/// </summary>
/// <param name="pattern">pattern (i.e. the param's pattern metadata)</param>
/// <param name="param">parameter node to either eval or wrap</param>
private Value ComputeParam(Value pattern, DelimiterNode param,
IScope paramScope, INodeRequestor nodes, ILineRequestor requestor)
{
Value keyType = null;
// does the pattern metadata ask for 'raw'?
if (pattern.Metadata != null && pattern.Metadata.TryGetValue(PatternData.keyType, out keyType) &&
keyType.Type == ValueType.String && keyType.AsString == ValueClasses.ClassOf(ValueType.Raw))
{
// if the param would eval to something raw, then we should eval it rather than try to wrap it
if (param.Token != null)
{
Value value = paramScope.GetValue(param.Token);
if (value != null && value.Type == ValueType.Raw)
return EvalNode.Do(param, paramScope, nodes, requestor);
}
// otherwise, is the param something that's not raw?
if (param.List == null || param.List.Delimiter.DelimiterType != DelimiterType.AsRaw)
{ // wrap unevaled value as raw
return new ValueRaw(param, paramScope);
}
}
return EvalNode.Do(param, paramScope, nodes, requestor);
}
internal abstract Value Eval(DelimiterNode prev, DelimiterNode next, IScope paramScope, IScope bodyScope, INodeRequestor nodes, ILineRequestor requestor);
internal override Value Eval(DelimiterNode prev, DelimiterNode next, IScope paramScope, IScope bodyScope, INodeRequestor nodes, ILineRequestor requestor)
{
System.Diagnostics.Debugger.Break();
return ValueNil.Nil;
}
internal override Value Eval(DelimiterNode prev, DelimiterNode next, IScope paramScope, IScope bodyScope, INodeRequestor nodes, ILineRequestor requestor)
{
if (next == null)
return this; // without parameters, it's still a function
Value post = EvalNode.Do(next, paramScope, nodes, requestor);
Value match, leftover;
if (!PatternChecker.Do(post, Metadata[keyNextPattern], false/*bShortPat*/, out match, out leftover))
throw new Loki3Exception().AddWrongPattern(Metadata[keyNextPattern], post);
if (leftover != null)
// create a partial function that starts w/ match & still needs leftover
return new PartialFunctionPre(this, match, leftover);
Value retval = Eval(match, bodyScope);
bodyScope.Exit();
return retval;
}
internal override Value Eval(DelimiterNode prev, DelimiterNode next, IScope paramScope, IScope bodyScope, INodeRequestor nodes, ILineRequestor requestor)
{
return new ValueInt(System.Environment.TickCount);
}
/// <summary>
/// Evaluate the right-most node with the highest precendence.
/// Return false when there are no more to eval.
/// </summary>
internal bool EvalNext(ILineRequestor requestor)
{
// find right-most node with highest precedence
m_evalIndex = -1;
int max = Int32.MaxValue;
int count = m_nodes.Count;
int empties = 0;
for (int i = 0; i < count; ++i)
{
NodeEval node = m_nodes[i];
if (!node.HasValue && !node.IsEmpty)
{ // node hasn't been evaled or consumed yet
int p = node.Precedence;
if (p < max)
{ // greater precedence than anything to the right
max = p;
m_evalIndex = i;
}
}
if (node.IsEmpty)
empties++;
}
// we're done if we didn't find anything
if (max == Int32.MaxValue)
return false;
// if the only thing left is a function, there's nothing further to do
if (empties == count - 1 && m_nodes[m_evalIndex].HasFunction && !m_nodes[m_evalIndex].ForceEval)
return false;
// if we found something we can't eval, it's an error
if (m_nodes[m_evalIndex].CantEval)
throw new Loki3Exception().AddBadToken(m_nodes[m_evalIndex].Node.Token);
// eval the one we found
m_nodes[m_evalIndex].Eval(m_scope, this, requestor);
return true;
}
/// <summary>
/// Evaluates a token, possibly requesting the previous and next values.
/// Returns a value.
/// </summary>
/// <param name="token">token representing a function or variable</param>
/// <param name="scope">used to request functions, variables, and delimiters if there's no scope attached to the node</param>
/// <param name="nodes">used to request previous and next nodes</param>
internal static Value Do(DelimiterNode node, IScope scope, INodeRequestor nodes, ILineRequestor requestor)
{
if (node.Value != null)
{ // node has already been evaluated
return node.Value;
}
else if (node.Token != null)
{ // function/variable or built-in
Token token = node.Token;
Value value = scope.GetValue(token);
if (value is ValueFunction && nodes != null)
{
ValueFunction function = value as ValueFunction;
// get previous & next nodes if needed
DelimiterNode previous = (function.ConsumesPrevious ? nodes.GetPrevious() : null);
DelimiterNode next = (function.ConsumesNext ? nodes.GetNext() : null);
scope.FunctionName = token.Value;
// evaluate
try
{
return function.Eval(previous, next, scope, scope, nodes, requestor);
}
catch (Loki3Exception e)
{ // this function is the correct context if there isn't already one there
if (!e.Errors.ContainsKey(Loki3Exception.keyFunction))
e.AddFunction(token.Value);
if (!e.Errors.ContainsKey(Loki3Exception.keyScope))
e.AddScope(scope);
throw e;
}
}
else if (value != null)
{
return value;
}
else
{
return EvalBuiltin.Do(token);
}
}
else if (node.List != null)
{ // delimited list of nodes
Value value = null;
DelimiterList list = node.List;
IScope listScope = (list.Scope != null ? list.Scope : scope);
DelimiterType type = list.Delimiter.DelimiterType;
// get contents as a Value
switch (type)
{
case DelimiterType.AsString:
value = new ValueString(list.Original);
break;
case DelimiterType.AsValue:
value = EvalList.Do(list.Nodes, listScope);
break;
case DelimiterType.AsArray:
List<Value> values = new List<Value>(list.Nodes.Count);
foreach (DelimiterNode subnode in list.Nodes)
{ // note: 'nodes' is null so functions don't get evaled
Value subvalue = Do(subnode, listScope, null, requestor);
values.Add(subvalue);
}
value = new ValueArray(values);
break;
case DelimiterType.AsEvaledArray:
value = EvalList.DoEvaledArray(list.Nodes, listScope);
break;
case DelimiterType.AsRaw:
value = new ValueRaw(list, listScope);
break;
case DelimiterType.AsArrayOfRaw:
List<Value> rawvalues = new List<Value>(list.Nodes.Count);
foreach (DelimiterNode subnode in list.Nodes)
rawvalues.Add(new ValueRaw(subnode, listScope));
value = new ValueArray(rawvalues);
break;
}
// run contents through a function if specified
ValueFunction function = list.Delimiter.Function;
if (function == null)
return value;
DelimiterNode next = new DelimiterNodeValue(value);
return function.Eval(null, next, scope, scope, nodes, requestor);
}
return new ValueNil();
}
/// <summary>Evaluate this node, possibly consuming adjacent nodes</summary>
internal void Eval(IScope scope, INodeRequestor nodes, ILineRequestor requestor)
{
if (m_state != NodeState.Node && m_state != NodeState.Function)
return;
// get new value
if (m_state == NodeState.Node)
{ // hasn't been evaled at all
m_value = EvalNode.Do(m_node, scope, nodes, requestor);
}
else if (m_state == NodeState.Function)
{ // previously resolved to a function
DelimiterNode previous = (m_func.ConsumesPrevious ? previous = nodes.GetPrevious() : null);
DelimiterNode next = (m_func.ConsumesNext ? next = nodes.GetNext() : null);
if ((m_func.ConsumesPrevious || m_func.ConsumesNext) && (previous == null && next == null))
{ // no prev/next parameters passed, perhaps it can use body?
if (m_func.RequiresBody() && requestor != null)
{ // tack on body if present
m_value = EvalList.DoAddBody(m_func, scope, requestor);
}
else
{ // function can't be evaled further
m_state = NodeState.Value;
return;
}
}
else
{
if (!m_func.ConsumesPrevious && !m_func.ConsumesNext && !m_func.RequiresBody() && !m_func.ForceEval)
{ // function can't be evaled further
m_state = NodeState.Value;
return;
}
m_value = m_func.Eval(previous, next, scope, scope, nodes, requestor);
}
}
if (m_value == null)
{ // e.g. because node was a comment
m_state = NodeState.Empty;
return;
}
// store new info about this node
m_node = new DelimiterNodeValue(m_value);
m_order = (int)m_value.Order;
if (m_value.Type == ValueType.Function)
{
m_state = NodeState.Function;
m_func = m_value as ValueFunction;
}
else
{
m_state = NodeState.Value;
}
}
internal override Value Eval(DelimiterNode prev, DelimiterNode next, IScope paramScope, IScope bodyScope, INodeRequestor nodes, ILineRequestor requestor)
{
Value value1 = EvalNode.Do(prev, paramScope, nodes, requestor);
Value value2 = EvalNode.Do(next, paramScope, nodes, requestor);
int sum = value1.AsInt + value2.AsInt;
return new ValueInt(sum);
}
// handle as individual tokens and nested lists
private static bool ParseMisc(int indent, string original, string[] strs, int iStart, ValueDelimiter thisDelim,
IParseLineDelimiters delims, ILineRequestor requestor, List<DelimiterNode> nodes,
out int iEnd, out DelimiterList result)
{
result = null;
iEnd = strs.Length;
for (int i = iStart; i < strs.Length; i++)
{
string s = strs[i];
// is this the end of current set of delimited tokens?
if (s == thisDelim.End)
{ // end delimiter
iEnd = i;
string subStr = GetSubStr(iStart, iEnd, strs);
result = new DelimiterList(thisDelim, nodes, indent, strs[iStart - 1], subStr, null);
return true;
}
// TODO: rework this so I don't need to check for : (e.g. for :}, when creating a delim)
if (s.Substring(s.Length - 1, 1) == thisDelim.End && (s[0] != ':' || s.Length > 2))
{ // end delimiter is part of final token
iEnd = i + 1;
string without = s.Substring(0, s.Length - 1);
Token token = new Token(without);
DelimiterNode node = new DelimiterNodeToken(token);
nodes.Add(node);
strs[i] = without;
string subStr = GetSubStr(iStart, iEnd, strs);
result = new DelimiterList(thisDelim, nodes, indent, strs[iStart - 1], subStr, null);
strs[i] = s;
--iEnd;
return true;
}
// is it a stand alone starting delimiter?
bool bAnyToken = false;
ValueDelimiter subDelim = (delims == null ? null : delims.GetDelim(s, out bAnyToken));
string[] strsToUse = strs;
bool bExtra = true;
if (subDelim == null && !bAnyToken)
{ // whole thing wasn't a delimiter, function, etc., how about the 1st char?
string s1 = s.Substring(0, 1);
subDelim = (delims == null ? null : delims.GetDelim(s1, out bAnyToken));
if (subDelim != null)
{ // copy across array, but break iStart into delim & remainder
bExtra = false;
strsToUse = new string[strs.Length + 1];
for (int j = 0; j < i; j++)
strsToUse[j] = strs[j];
strsToUse[i] = s1;
strsToUse[i + 1] = s.Substring(1);
for (int j = i + 1; j < strs.Length; j++)
strsToUse[j + 1] = strs[j];
}
}
if (subDelim != null)
{ // start delimiter
int end;
DelimiterList sublist = Do(0, original, strsToUse, i + 1, subDelim, delims, requestor, out end);
if (sublist != null)
{
DelimiterNodeList node = new DelimiterNodeList(sublist);
nodes.Add(node);
}
// skip past the sublist
i = (bExtra ? end : end - 1);
}
else
{ // stand alone token
Token token = new Token(s);
DelimiterNode node = new DelimiterNodeToken(token);
nodes.Add(node);
}
}
return false;
}