/// <summary> /// Tokenizes the source code and returns a list of tokens /// </summary> /// <param name="text">The CPS source code to tokenize (as an array of characters)</param> /// <param name="version">Version of the tokenizer to use.</param> /// <returns>A List of tokens</returns> public static List <Token> Tokenize(char[] text, TokenizerVersion version) { var tokenizer = new PropertySheetTokenizer(text, version); tokenizer.Tokenize(); return(tokenizer.Tokens); }
public static List <Token> Tokenize(string text, TokenizerVersion version) { return(Tokenize(string.IsNullOrEmpty(text) ? new char[0] : text.ToCharArray(), version)); }
/// <summary> /// Protected Constructor for tokenizing CPS code. Public access via the static Tokenize methods. /// </summary> /// <param name="text">the array of characters to tokenize.</param> /// <param name="version">Which version of the parser to conform to.</param> protected PropertySheetTokenizer(char[] text, TokenizerVersion version) : base(text) { _version = version; Keywords = CpsKeywords; }
public static new List<Token> Tokenize(string text, TokenizerVersion version) { return Tokenize(string.IsNullOrEmpty(text) ? new char[0] : text.ToCharArray(), version); }
/// <summary> /// Tokenizes the source code and returns a list of tokens /// </summary> /// <param name="text">The CPS source code to tokenize (as an array of characters)</param> /// <param name="version">Version of the tokenizer to use.</param> /// <returns>A List of tokens</returns> public static new List<Token> Tokenize(char[] text, TokenizerVersion version) { var tokenizer = new PropertySheetTokenizer(text, version); tokenizer.Tokenize(); return tokenizer.Tokens; }