/// <summary>
/// Tokenizes the source code and returns a list of tokens
/// </summary>
/// <param name="text">The CPS source code to tokenize (as an array of characters)</param>
/// <param name="version">Version of the tokenizer to use.</param>
/// <returns>A List of tokens</returns>
public static List <Token> Tokenize(char[] text, TokenizerVersion version)
{
var tokenizer = new PropertySheetTokenizer(text, version);
tokenizer.Tokenize();
return(tokenizer.Tokens);
}
/// <summary>
/// Tokenizes the source code and returns a list of tokens
/// </summary>
/// <param name="text">The CPS source code to tokenize (as an array of characters)</param>
/// <param name="version">Version of the tokenizer to use.</param>
/// <returns>A List of tokens</returns>
public static new List<Token> Tokenize(char[] text, TokenizerVersion version)
{
var tokenizer = new PropertySheetTokenizer(text, version);
tokenizer.Tokenize();
return tokenizer.Tokens;
}
protected PropertySheet Parse()
{
var tokenStream = PropertySheetTokenizer.Tokenize(_propertySheetText);
var state = ParseState.Global;
var enumerator = tokenStream.GetEnumerator();
var importFilename = string.Empty;
// var startFolder = System.Environment.CurrentDirectory;
var startFolder = Path.GetDirectoryName(_filename.GetFullPath());
// first, check for a .user file to auto-import
if (!string.IsNullOrEmpty(_filename))
{
var userSheetFilename = Path.Combine(startFolder, Path.GetFileName(_filename.GetFullPath()) + ".user");
if (File.Exists(userSheetFilename))
{
_propertySheet.UserOverride = ParseIncludedSheet(null, userSheetFilename, File.ReadAllText(userSheetFilename));
}
}
Token token;
Rule rule = null;
string ruleName = "*";
string ruleParameter = null;
string ruleClass = null;
string ruleId = null;
// PropertyRule property = null;
var sourceLocation = new SourceLocation {
Row = 0,
Column = 0,
SourceFile = null,
};
string propertyName = null;
string propertyLabelText = null;
string presentlyUnknownValue = null;
// string collectionName = null;
List <string> multidimensionalLambda = null;
enumerator.MoveNext();
do
{
token = enumerator.Current;
switch (token.Type)
{
// regardless where we are, skip over whitespace, etc.
case TokenType.WhiteSpace:
case TokenType.LineComment:
case TokenType.MultilineComment:
continue;
}
switch (state)
{
case ParseState.Global:
sourceLocation = new SourceLocation {
// will be the start of the next new rule.
Row = token.Row,
Column = token.Column,
SourceFile = _filename,
};
switch (token.Type)
{
case TokenType.Identifier: // look for identifier as the start of a selector
if (token.Data == "@import")
{
// special case to handle @import rules
state = ParseState.Import;
continue;
}
state = ParseState.Selector;
ruleName = token.Data;
continue;
case TokenType.Dot:
state = ParseState.SelectorDot;
ruleName = "*";
// take next identifier as the classname
continue;
case TokenType.Pound:
state = ParseState.SelectorPound;
ruleName = "*";
// take next identifier as the id
continue;
case TokenType.Semicolon: // tolerate extra semicolons.
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 100", "Expected one of '.' , '#' or identifier");
}
case ParseState.Import:
switch (token.Type)
{
case TokenType.StringLiteral:
case TokenType.Identifier:
state = ParseState.ImportFilename;
importFilename = token.Data;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 121", "Expected a string literal for filename (Missing semicolon?)");
}
case ParseState.ImportFilename:
switch (token.Type)
{
case TokenType.Semicolon:
if (!Import(importFilename, startFolder))
{
throw new EndUserParseException(token, _filename, "PSP 122", "Imported file '{0}' not found", importFilename);
}
state = ParseState.Global;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 121", "Expected a string literal for filename");
}
case ParseState.Selector:
switch (token.Type)
{
case TokenType.Dot:
state = ParseState.SelectorDot;
continue;
case TokenType.Pound:
state = ParseState.SelectorPound;
continue;
case TokenType.SelectorParameter:
ruleParameter = token.Data;
if (ruleParameter.IndexOfAny("\r\n".ToCharArray()) >= 0)
{
throw new EndUserParseException(token, _filename, "PSP 123", "Selector parameter may not contain CR/LFs (missing close bracket?): {0} ", Rule.CreateSelectorString(ruleName, ruleParameter, ruleClass, ruleId));
}
continue;
case TokenType.OpenBrace:
state = ParseState.InRule;
rule = _propertySheet[ruleName, ruleParameter, ruleClass, ruleId];
ruleName = null;
ruleParameter = null;
ruleClass = null;
ruleId = null;
rule.SourceLocation = sourceLocation;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 101", "Expected one of '.' , '#' , '[' or '{{' .");
}
case ParseState.SelectorDot:
switch (token.Type)
{
case TokenType.Identifier:
ruleClass = token.Data;
state = ParseState.Selector;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 102", "Expected identifier");
}
case ParseState.SelectorPound:
switch (token.Type)
{
case TokenType.Identifier:
ruleId = token.Data;
state = ParseState.Selector;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 103", "Expected identifier");
}
case ParseState.InRule:
switch (token.Type)
{
case TokenType.Semicolon: // extra semicolons are tolerated.
continue;
case TokenType.StringLiteral:
case TokenType.Identifier:
propertyName = token.Data;
state = ParseState.HavePropertyName;
sourceLocation = new SourceLocation {
Row = token.Row,
Column = token.Column,
SourceFile = _filename,
};
continue;
case TokenType.CloseBrace:
// this rule is DONE.
rule = null; // set this to null, so that we don't accidentally add new stuff to this rule.
state = ParseState.Global;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 104", "In rule, expected semi-colon ';', close-brace '}}' or value.");
}
case ParseState.HavePropertyName:
switch (token.Type)
{
case TokenType.Colon:
state = ParseState.HavePropertySeparator;
// property = rule.GetPropertyRule(propertyName);
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 105", "Found rule property name, expected colon ':'.");
}
case ParseState.HavePropertySeparator:
switch (token.Type)
{
case TokenType.StringLiteral:
case TokenType.NumericLiteral:
state = ParseState.HavePropertyLabel;
if ("@Literal" == token.RawData)
{
propertyLabelText = token.Data;
}
else
{
propertyLabelText = token.Data;
}
continue;
case TokenType.Identifier:
state = ParseState.HavePropertyLabel;
propertyLabelText = token.Data;
continue;
case TokenType.OpenBrace:
state = ParseState.InPropertyCollectionWithoutLabel;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 106", "After rule property name, expected value, open-brace '{{' or open-parenthesis '('.");
}
case ParseState.InPropertyCollectionWithoutLabel:
switch (token.Type)
{
case TokenType.StringLiteral:
case TokenType.NumericLiteral:
case TokenType.Identifier: {
// at this point it could be a collection, a label, or a value.
presentlyUnknownValue = token.Data;
state = ParseState.InPropertyCollectionWithoutLabelButHaveSomething;
// we're going to peek ahead and see if there is any characters we want to accept. (#.-+)
var peek = enumerator;
var cont = true;
var okToTakeIdentifierOrNumeric = false;
do
{
if (!peek.MoveNext())
{
break;
}
// if we find a character that we consider to be ok for labels here, we're going to add it in, and consume the token.
switch (peek.Current.Type)
{
case TokenType.Pound:
case TokenType.Dot:
case TokenType.Minus:
case TokenType.MinusMinus:
case TokenType.Plus:
case TokenType.PlusPlus:
enumerator.MoveNext();
presentlyUnknownValue = presentlyUnknownValue + enumerator.Current.Data;
okToTakeIdentifierOrNumeric = true;
break;
case TokenType.NumericLiteral:
case TokenType.Identifier:
if (!okToTakeIdentifierOrNumeric)
{
cont = false;
break;
}
enumerator.MoveNext();
presentlyUnknownValue = presentlyUnknownValue + enumerator.Current.Data;
okToTakeIdentifierOrNumeric = false;
break;
default:
cont = false;
break;
}
} while (cont);
}
continue;
case TokenType.CloseBrace:
state = ParseState.HavePropertyCompleted;
// this makes the semicolon optional.
//state = ParseState.InRule;
continue;
case TokenType.OpenParenthesis:
state = ParseState.OpenBraceExpectingMultidimesionalLamda;
multidimensionalLambda = new XList <string>();
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 107", "In property collection, expected value or close brace '}}'");
}
case ParseState.OpenBraceExpectingMultidimesionalLamda:
switch (token.Type)
{
case TokenType.StringLiteral:
case TokenType.NumericLiteral:
case TokenType.Identifier:
// looks like we have the name of a collection for a multidimensional lambda
multidimensionalLambda.Add(token.Data);
state = ParseState.HasMultidimensionalLambdaIdentifier;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 124", "In multidimensional lambda declaration, expected identifier, found '{0}'", token.Data);
}
case ParseState.HasMultidimensionalLambdaIdentifier:
switch (token.Type)
{
case TokenType.Comma:
state = ParseState.OpenBraceExpectingMultidimesionalLamda;
continue;
case TokenType.CloseParenthesis:
state = ParseState.NextTokenBetterBeLambda;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 125", "In multidimensional lambda declaration, expected close parenthesis or comma, found '{0}'", token.Data);
}
case ParseState.NextTokenBetterBeLambda:
switch (token.Type)
{
case TokenType.Lambda:
// we already knew that it was going to be this.
// the collection has all the appropriate values.
presentlyUnknownValue = null;
state = ParseState.HasLambda;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 125", "Expected lambda '=>' found '{0}'", token.Data);
}
case ParseState.InPropertyCollectionWithoutLabelButHaveSomething:
switch (token.Type)
{
case TokenType.Lambda:
multidimensionalLambda = new XList <string> {
presentlyUnknownValue
};
presentlyUnknownValue = null;
state = ParseState.HasLambda;
continue;
case TokenType.Equal:
// looks like it's gonna be a label = value type.
propertyLabelText = presentlyUnknownValue;
presentlyUnknownValue = null;
state = ParseState.HasEqualsInCollection;
continue;
case TokenType.Comma: {
// turns out its a simple collection item.
var pv = rule.GetPropertyRule(propertyName).GetPropertyValue(string.Empty);
pv.Add(presentlyUnknownValue);
pv.SourceLocation = sourceLocation;
presentlyUnknownValue = null;
state = ParseState.InPropertyCollectionWithoutLabel;
}
continue;
case TokenType.CloseBrace: {
// turns out its a simple collection item.
var pv = rule.GetPropertyRule(propertyName).GetPropertyValue(string.Empty);
pv.Add(presentlyUnknownValue);
pv.SourceLocation = sourceLocation;
presentlyUnknownValue = null;
// state = ParseState.HavePropertyCompleted;
// this makes the semicolon optional.
state = ParseState.InRule;
}
continue;
case TokenType.OpenBracket:
case TokenType.OpenBrace:
case TokenType.OpenParenthesis:
case TokenType.LessThan:
// starting a new script block
// presentlyUnknownValue is the script type
// the content goes until the matching close token.
continue;
case TokenType.Identifier:
case TokenType.NumericLiteral:
case TokenType.StringLiteral:
// starting a new script block
// presentlyUnknownValue is the script type
// the content of the string literal is the script content
rule.AddScriptedPropertyRule(propertyName, presentlyUnknownValue, token.Data, token.Data);
continue;
default:
string tokentext = token.RawData.ToString();
if (tokentext.Length == 1)
{
// starting a new script block
// presentlyUnknownValue is the script type
// the content goes until we see that same token again.
continue;
}
throw new EndUserParseException(token, _filename, "PSP 114", "after an value or identifier in a collection expected a '=>' or '=' or ',' .");
}
case ParseState.HasEqualsInCollection:
switch (token.Type)
{
case TokenType.StringLiteral:
case TokenType.NumericLiteral:
case TokenType.Identifier: {
var pv = rule.GetPropertyRule(propertyName).GetPropertyValue(propertyLabelText);
pv.Add(token.Data);
pv.SourceLocation = sourceLocation;
state = ParseState.InPropertyCollectionWithoutLabelWaitingForComma;
}
continue;
case TokenType.OpenBrace:
state = ParseState.InPropertyCollectionWithLabel;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 119", "after an equals '=' in a collection, expected a value or identifier.");
}
case ParseState.HasLambda:
switch (token.Type)
{
case TokenType.StringLiteral:
case TokenType.NumericLiteral:
case TokenType.Identifier:
propertyLabelText = token.Data;
state = ParseState.HasLambdaAndLabel;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 115", "After the '=>' in a collection, expected value or identifier.");
}
case ParseState.HasLambdaAndLabel:
switch (token.Type)
{
case TokenType.Equal:
state = ParseState.HasLambdaAndLabelAndEquals;
continue;
case TokenType.Semicolon:
case TokenType.Comma:
case TokenType.CloseBrace: {
// assumes "${DEFAULTLAMBDAVALUE}" for the lamda value
/* ORIG:
* var pv = property.GetPropertyValue(propertyLabelText, multidimensionalLambda);
* pv.Add("${DEFAULTLAMBDAVALUE}");
*/
var pv = rule.GetPropertyRule(propertyName).GetPropertyValue("", multidimensionalLambda);
pv.Add(propertyLabelText);
pv.SourceLocation = sourceLocation;
propertyLabelText = null;
multidimensionalLambda = null;
state = ParseState.InPropertyCollectionWithoutLabelWaitingForComma;
}
if (token.Type == TokenType.CloseBrace)
{
// and, we're closing out this property.
// state = ParseState.HavePropertyCompleted;
// this makes the semicolon optional.
state = ParseState.InRule;
}
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 116", "After the '{0} => {1}' in collection, expected '=' ", multidimensionalLambda.Aggregate("(", (current, each) => current + ", " + each) + ")", propertyLabelText);
}
case ParseState.HasLambdaAndLabelAndEquals:
switch (token.Type)
{
case TokenType.StringLiteral:
case TokenType.NumericLiteral:
case TokenType.Identifier: {
var pv = rule.GetPropertyRule(propertyName).GetPropertyValue(propertyLabelText, multidimensionalLambda);
pv.Add(token.Data);
pv.SourceLocation = sourceLocation;
propertyLabelText = null;
multidimensionalLambda = null;
state = ParseState.InPropertyCollectionWithoutLabelWaitingForComma;
}
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 117", "After the '{0} => {1} = ' in collection, expected a value or identifier", multidimensionalLambda.Aggregate("(", (current, each) => current + ", " + each) + ")",
propertyLabelText);
}
case ParseState.InPropertyCollectionWithoutLabelWaitingForComma:
switch (token.Type)
{
case TokenType.Comma:
case TokenType.Semicolon:
state = ParseState.InPropertyCollectionWithoutLabel;
continue;
case TokenType.CloseBrace:
// state = ParseState.HavePropertyCompleted;
// this makes the semicolon optional.
state = ParseState.InRule;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 118", "After complete expression or value in a collection, expected ',' or '}}'.");
}
case ParseState.InPropertyCollectionWithLabel:
switch (token.Type)
{
case TokenType.StringLiteral:
case TokenType.NumericLiteral:
case TokenType.Identifier:
presentlyUnknownValue = token.Data;
state = ParseState.HaveCollectionValue;
continue;
case TokenType.CloseBrace:
//state = ParseState.HavePropertyCompleted;
// this makes the semicolon optional.
state = ParseState.InPropertyCollectionWithoutLabelWaitingForComma;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 107", "In property collection, expected value or close brace '}}'");
}
case ParseState.HaveCollectionValue:
switch (token.Type)
{
case TokenType.Semicolon:
case TokenType.Comma: {
var pv = rule.GetPropertyRule(propertyName).GetPropertyValue(propertyLabelText);
pv.Add(presentlyUnknownValue);
pv.SourceLocation = sourceLocation;
// propertyLabelText = null;
state = ParseState.InPropertyCollectionWithLabel;
}
continue;
case TokenType.CloseBrace: {
var pv = rule.GetPropertyRule(propertyName).GetPropertyValue(propertyLabelText);
pv.Add(presentlyUnknownValue);
pv.SourceLocation = sourceLocation;
propertyLabelText = null;
state = ParseState.HavePropertyCompleted;
// this makes the semicolon optional.
// state = ParseState.InRule;
}
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 108", "With property collection value, expected comma ',' or close-brace '}}'.");
}
case ParseState.HavePropertyLabel:
switch (token.Type)
{
case TokenType.Equal:
state = ParseState.HavePropertyEquals;
continue;
case TokenType.Dot:
var t = SkipToNext(ref enumerator);
if (!t.HasValue)
{
throw new EndUserParseException(token, _filename, "PSP 109", "Unexpected end of Token stream [HavePropertyLabel]");
}
token = t.Value;
if (token.Type == TokenType.Identifier || token.Type == TokenType.NumericLiteral)
{
propertyLabelText += "." + token.Data;
}
else
{
throw new EndUserParseException(token, _filename, "PSP 110", "Expected identifier or numeric literal after Dot '.'.");
}
continue;
case TokenType.Semicolon: {
// it turns out that what we thought the label was, is really the property value,
// the label is an empty string
var pv = rule.GetPropertyRule(propertyName).GetPropertyValue(string.Empty);
pv.Add(propertyLabelText);
pv.SourceLocation = sourceLocation;
propertyName = propertyLabelText = null;
state = ParseState.InRule;
}
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 111", "After property value, expected semi-colon ';' or equals '='.");
}
case ParseState.HavePropertyEquals:
switch (token.Type)
{
case TokenType.Identifier:
case TokenType.StringLiteral:
case TokenType.NumericLiteral: {
// found our property-value. add it, and move along.
var pv = rule.GetPropertyRule(propertyName).GetPropertyValue(propertyLabelText);
pv.Add(token.Data);
pv.SourceLocation = sourceLocation;
propertyName = propertyLabelText = null;
state = ParseState.HavePropertyCompleted;
}
continue;
case TokenType.OpenBrace:
// we're starting a new collection (where we have the label already).
state = ParseState.InPropertyCollectionWithLabel;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 112", "After equals in property, expected value or close-brace '{B{'.");
}
case ParseState.HavePropertyCompleted:
switch (token.Type)
{
case TokenType.CloseBrace:
case TokenType.Semicolon:
state = ParseState.InRule;
continue;
default:
throw new EndUserParseException(token, _filename, "PSP 113", "After property completed, expected semi-colon ';'.");
}
default:
throw new EndUserParseException(token, _filename, "PSP 120", "CATS AND DOGS, LIVINGTOGETHER...");
}
} while (enumerator.MoveNext());
return(_propertySheet);
}
