/// <summary>
/// Tokenizes the source code and returns a list of tokens
/// </summary>
/// <param name="text">The CPS source code to tokenize (as an array of characters)</param>
/// <param name="version">Version of the tokenizer to use.</param>
/// <returns>A List of tokens</returns>
public static List <Token> Tokenize(char[] text, TokenizerVersion version)
{
var tokenizer = new PropertySheetTokenizer(text, version);
tokenizer.Tokenize();
return(tokenizer.Tokens);
}
public static List <Token> Tokenize(string text, TokenizerVersion version)
{
return(Tokenize(string.IsNullOrEmpty(text) ? new char[0] : text.ToCharArray(), version));
}
/// <summary>
/// Protected Constructor for tokenizing CPS code. Public access via the static Tokenize methods.
/// </summary>
/// <param name="text">the array of characters to tokenize.</param>
/// <param name="version">Which version of the parser to conform to.</param>
protected PropertySheetTokenizer(char[] text, TokenizerVersion version)
: base(text)
{
_version = version;
Keywords = CpsKeywords;
}
public static new List<Token> Tokenize(string text, TokenizerVersion version)
{
return Tokenize(string.IsNullOrEmpty(text) ? new char[0] : text.ToCharArray(), version);
}
/// <summary>
/// Protected Constructor for tokenizing CPS code. Public access via the static Tokenize methods.
/// </summary>
/// <param name="text">the array of characters to tokenize.</param>
/// <param name="version">Which version of the parser to conform to.</param>
protected PropertySheetTokenizer(char[] text, TokenizerVersion version)
: base(text)
{
_version = version;
Keywords = CpsKeywords;
}
/// <summary>
/// Tokenizes the source code and returns a list of tokens
/// </summary>
/// <param name="text">The CPS source code to tokenize (as an array of characters)</param>
/// <param name="version">Version of the tokenizer to use.</param>
/// <returns>A List of tokens</returns>
public static new List<Token> Tokenize(char[] text, TokenizerVersion version)
{
var tokenizer = new PropertySheetTokenizer(text, version);
tokenizer.Tokenize();
return tokenizer.Tokens;
}
