/// <summary>
/// Checks to see if a given token (representing a closing delimeter has properties. If it does then adds warning to that token
/// </summary>
/// <param name="closingToken">token representing delimeter</param>
/// <param name="tokenStream">remaining token stream</param>
private void WarnIfClosingTokenHasProperties(Token closingToken, IWaneTokenStream tokenStream)
{
//check for closing token properties
Token possiblePropertyToken = tokenStream.Peek();
if (possiblePropertyToken != null && IsPropertiesStartDelimeter(possiblePropertyToken))
{
//check if has full properties
List<Token> propertyTokens = GetRawTokensUntilAndIncludingEndingDelimeter(possiblePropertyToken, tokenStream);
if (propertyTokens.Count > 0)
closingToken.AddWarning("Can not put properties on closing delimeters.");
}
}
/// <summary>
/// Processes a stream or substream of tokens
/// </summary>
/// <param name="tokenStream">tokens to process</param>
/// <returns>list of tokens</returns>
private List<Token> ProcessTokenStream(IWaneTokenStream tokenStream)
{
var tokens = new List<Token>();
bool isEscaping = false;
while (!tokenStream.IsEndOfStream)
{
Token currentToken = tokenStream.ReadNextToken();
//handle escaped token
if (isEscaping)
{
currentToken.ChangeTokenTypeToText();
isEscaping = false;
}
//check if escape character and is escaping something
isEscaping = DelimeterCausesEscape(currentToken, tokenStream);
if (IsCustomDelimeter(currentToken) && !isEscaping)
{
List<Token> handledTokens = HandleCustomDelimeter(currentToken, tokenStream);
tokens.AddRange(handledTokens);
}
else if (currentToken.TokenType == TokenType.NewLine)
tokens.Add(currentToken);
else if(!isEscaping)
AppendTextToTokens(tokens, currentToken);
}
return tokens;
}
/// <summary>
/// Handles a custom delimeter from it's start until it's end.
/// Also handles tokens within those custom delimeters
/// </summary>
/// <param name="currentToken">current token (which is a delimeter)</param>
/// <param name="tokenStream"></param>
private List<Token> HandleCustomDelimeter(Token currentToken, IWaneTokenStream tokenStream)
{
if (!IsCustomDelimeter(currentToken))
throw new ArgumentException("Expected custome delimeter", "currentToken");
var tokens = new List<Token>();
currentToken.IsStartingDelimeter = true;
tokens.Add(currentToken);
//find ending delimeter
List<Token> tokensToInspect = GetRawTokensUntilAndIncludingEndingDelimeter(currentToken, tokenStream);
if (tokensToInspect.Count > 0)
{
//remove ending token else this will not be processed properly
Token closingToken = tokensToInspect.Last();
tokensToInspect.RemoveAt(tokensToInspect.Count - 1);
var internalStream = new WaneEnumerableTokenStream(tokensToInspect);
//get and remove properties
ExtractPropertiesFromTokenListAndApplyToCurrentToken(currentToken, internalStream);
if (!internalStream.IsEndOfStream)
{
List<Token> procesedInternalStreamTokens = ProcessTokenStream(internalStream);
tokens.AddRange(procesedInternalStreamTokens);
}
//add closing delimeter token
closingToken.IsEndingDelimeter = true;
tokens.Add(closingToken);
WarnIfClosingTokenHasProperties(closingToken, tokenStream);
}
else
{
//no closing delimeter
currentToken.ChangeTokenTypeToText();
currentToken.AddWarning("Delimeter start does not have a matching end delimeter, or the actual start delimeter has been escaped");
}
return tokens;
}
/// <summary>
/// Gets a sub stream of tokens to parse
/// </summary>
/// <param name="findEndingTokenFor">searches for a matching token in the stream, the match will be the end of the sub-stream (or then end of the text will be)</param>
/// <param name="tokenStream">token stream being read from</param>
/// <returns>sub stream of tokens, or empty list if no ending delimeter</returns>
private List<Token> GetRawTokensUntilAndIncludingEndingDelimeter(Token findEndingTokenFor, IWaneTokenStream tokenStream)
{
var subStream = new List<Token>();
bool hasFoundEndingToken = false;
string endingDelimeter = _delimeterSet.GetEndingDelimterForStartingDelimeter(findEndingTokenFor.Text);
Token endingToken = tokenStream.Find(endingDelimeter);
if (endingToken != null)
{
while (!tokenStream.IsEndOfStream && !hasFoundEndingToken)
{
Token token = tokenStream.ReadNextToken();
subStream.Add(token);
hasFoundEndingToken = token.Text == endingToken.Text;
}
}
return subStream;
}
/// <summary>
/// Extracts tokens which represent properties for a given token and apply put those properties onto the token
/// </summary>
/// <param name="currentToken">token that will have properties added to it</param>
/// <param name="tokenStream">token stream to inspect</param>
private void ExtractPropertiesFromTokenListAndApplyToCurrentToken(Token currentToken, IWaneTokenStream tokenStream)
{
if (IsPropertiesStartDelimeter(tokenStream.Peek()))
{
List<Token> rawPropertyTokens = GetRawTokensUntilAndIncludingEndingDelimeter(tokenStream.Peek(), tokenStream);
if (rawPropertyTokens.Count == 0)
currentToken.AddWarning("Properties start delimeter found, but could not find ending properties delimeter.");
else if(rawPropertyTokens.Count == 2)
currentToken.AddWarning("Properties declaration contains no key value pairs. Could not construct properties.");
else
ProcessProperties(currentToken, rawPropertyTokens);
}
}
/// <summary>
/// Determines if a token causes the next token to be escaped
/// </summary>
/// <param name="currentToken">the current token to check if it's an escape char, and if the escape char is escaping</param>
/// <param name="tokenStream">remaining token stream</param>
/// <returns>true if causes escape, otherwise false</returns>
private bool DelimeterCausesEscape(Token currentToken, IWaneTokenStream tokenStream)
{
if (IsEscapeDelimieter(currentToken))
{
Token nextToken = tokenStream.Peek();
if (nextToken.IsDelimeter)
return true;
else
currentToken.ChangeTokenTypeToText();
}
return false;
}
