/*********
** Private methods
*********/
/// <summary>Parse a sequence of lexical character patterns into higher-level lexical tokens.</summary>
/// <param name="input">The lexical character patterns to parse.</param>
/// <param name="impliedBraces">Whether we're parsing a token context (so the outer '{{' and '}}' are implied); else parse as a tokenizable string which main contain a mix of literal and {{token}} values.</param>
/// <param name="trim">Whether the value should be trimmed.</param>
private IEnumerable <ILexToken> ParseBitQueue(Queue <LexBit> input, bool impliedBraces, bool trim)
{
// perform a raw parse
IEnumerable <ILexToken> RawParse()
{
// 'Implied braces' means we're parsing inside a token. This necessarily starts with a token name,
// optionally followed by input arguments.
if (impliedBraces)
{
while (input.Any())
{
// extract token
yield return(this.ExtractToken(input, impliedBraces: true));
if (!input.Any())
{
yield break;
}
// throw error if there's content after the token ends
var next = input.Peek();
throw new LexFormatException($"Unexpected {next.Type}, expected {LexBitType.Literal}");
}
yield break;
}
// Otherwise this is a tokenizable string which may contain a mix of literal and {{token}} values.
while (input.Any())
{
LexBit next = input.Peek();
switch (next.Type)
{
// start token
case LexBitType.StartToken:
yield return(this.ExtractToken(input, impliedBraces: false));
break;
// text/separator outside token
case LexBitType.Literal:
case LexBitType.PositionalInputArgSeparator:
case LexBitType.NamedInputArgSeparator:
input.Dequeue();
yield return(new LexTokenLiteral(next.Text));
break;
// anything else is invalid
default:
throw new LexFormatException($"Unexpected {next.Type}, expected {LexBitType.StartToken} or {LexBitType.Literal}");
}
}
}
string rawInput = string.Join("", input.Select(p => p.Text));
LinkedList <ILexToken> tokens;
try
{
tokens = new LinkedList <ILexToken>(RawParse());
}
catch (Exception ex)
{
throw new InvalidOperationException($"Error parsing '{rawInput}' as a tokenizable string", ex);
}
// normalize literal values
ISet <LinkedListNode <ILexToken> > removeQueue = new HashSet <LinkedListNode <ILexToken> >(new ObjectReferenceComparer <LinkedListNode <ILexToken> >());
for (LinkedListNode <ILexToken> node = tokens.First; node != null; node = node.Next)
{
// fetch info
if (!(node.Value is LexTokenLiteral current))
{
continue;
}
ILexToken previous = node.Previous?.Value;
ILexToken next = node.Next?.Value;
string newText = current.Text;
// collapse sequential literals
if (previous is LexTokenLiteral prevLiteral)
{
newText = prevLiteral.Text + newText;
removeQueue.Add(node.Previous);
}
// trim before/after separator
if (next?.Type == LexTokenType.TokenInput)
{
newText = newText.TrimEnd();
}
if (previous?.Type == LexTokenType.TokenInput)
{
newText = newText.TrimStart();
}
// trim whole result
if (trim && (previous == null || next == null))
{
if (previous == null)
{
newText = newText.TrimStart();
}
if (next == null)
{
newText = newText.TrimEnd();
}
if (newText == "")
{
removeQueue.Add(node);
}
}
// replace value if needed
if (current.Text != newText)
{
current.MigrateTo(newText);
}
}
foreach (LinkedListNode <ILexToken> entry in removeQueue)
{
tokens.Remove(entry);
}
// yield result
return(tokens);
}
private List <ILexToken> Next(ILexToken token, Match match, string line, int start, int end, List <ILexToken> ret)
{
Analize(line, start, match.Index, ret);
ret.Add(token);
return(Analize(line, match.Index + match.Length, end, ret));
}
/*********
** Public methods
*********/
/// <summary>Construct an instance.</summary>
/// <param name="lexToken">The underlying lex token.</param>
/// <param name="input">The parsed version of <see cref="Input"/>.</param>
public TokenStringPart(ILexToken lexToken, TokenString input)
{
this.LexToken = lexToken;
this.Input = input;
this.InputArgs = new InputArguments(input);
}
/*********
** Private methods
*********/
/// <summary>Parse a sequence of lexical character patterns into higher-level lexical tokens.</summary>
/// <param name="input">The lexical character patterns to parse.</param>
/// <param name="impliedBraces">Whether we're parsing a token context (so the outer '{{' and '}}' are implied); else parse as a tokenisable string which main contain a mix of literal and {{token}} values.</param>
/// <param name="trim">Whether the value should be trimmed.</param>
private IEnumerable <ILexToken> ParseBitQueue(Queue <LexBit> input, bool impliedBraces, bool trim)
{
// perform a raw parse
IEnumerable <ILexToken> RawParse()
{
// 'Implied braces' means we're parsing inside a token. This necessarily starts with a token name,
// optionally followed by an input argument and token pipes.
if (impliedBraces)
{
while (input.Any())
{
yield return(this.ExtractToken(input, impliedBraces: true));
if (!input.Any())
{
yield break;
}
var next = input.Peek();
switch (next.Type)
{
case LexBitType.TokenPipe:
yield return(new LexTokenPipe(input.Dequeue().Text));
break;
default:
throw new InvalidOperationException($"Unexpected {next.Type}, expected {LexBitType.Literal} or {LexBitType.TokenPipe}");
}
}
yield break;
}
// Otherwise this is a tokenisable string which may contain a mix of literal and {{token}} values.
while (input.Any())
{
LexBit next = input.Peek();
switch (next.Type)
{
// start token
case LexBitType.StartToken:
yield return(this.ExtractToken(input, impliedBraces: false));
break;
// pipe/separator outside token
case LexBitType.Literal:
case LexBitType.TokenPipe:
case LexBitType.InputArgSeparator:
input.Dequeue();
yield return(new LexTokenLiteral(next.Text));
break;
// anything else is invalid
default:
throw new InvalidOperationException($"Unexpected {next.Type}, expected {LexBitType.StartToken} or {LexBitType.Literal}");
}
}
}
// normalise literal values
LinkedList <ILexToken> tokens = new LinkedList <ILexToken>(RawParse());
IList <LinkedListNode <ILexToken> > removeQueue = new List <LinkedListNode <ILexToken> >();
for (LinkedListNode <ILexToken> node = tokens.First; node != null; node = node.Next)
{
if (node.Value.Type != LexTokenType.Literal)
{
continue;
}
// fetch info
ILexToken current = node.Value;
ILexToken previous = node.Previous?.Value;
ILexToken next = node.Next?.Value;
string newText = node.Value.Text;
// collapse sequential literals
if (previous?.Type == LexTokenType.Literal)
{
newText = previous.Text + newText;
removeQueue.Add(node.Previous);
}
// trim before/after separator
if (next?.Type == LexTokenType.TokenInput || next?.Type == LexTokenType.TokenPipe)
{
newText = newText.TrimEnd();
}
if (previous?.Type == LexTokenType.TokenInput || previous?.Type == LexTokenType.TokenPipe)
{
newText = newText.TrimStart();
}
// trim whole result
if (trim && (previous == null || next == null))
{
if (previous == null)
{
newText = newText.TrimStart();
}
if (next == null)
{
newText = newText.TrimEnd();
}
if (newText == "")
{
removeQueue.Add(node);
}
}
// replace value if needed
if (newText != current.Text)
{
node.Value = new LexTokenLiteral(newText);
}
}
foreach (LinkedListNode <ILexToken> entry in removeQueue)
{
tokens.Remove(entry);
}
// yield result
return(tokens);
}
