/// <summary>
/// 实例化一个 Splitor
/// </summary>
/// <param name="text">源字符串</param>
/// <param name="seperators">依据该数组中的字符进行切割</param>
/// <param name="singles">需要把单个字符作为 token 的字符集</param>
public Tokenizer(string text, char[] seperators, char[] singles, SelfExistentToken whiteList)
{
this._text = text;
this._seperators = seperators;
this._singles = singles;
this._selftExistentToken = whiteList;
this.Tokens = MakeTokens();
}
/// <summary>
/// 根据传入的索引获取组成 token 的字符范围
/// </summary>
/// <param name="index">代表 token 首字符所在的位置</param>
/// <returns></returns>
private (int Start, int End) GetRange(ref int index, SelfExistentToken selfExistentToken)
{
int start = index;
while ((index < this._text.Length) &&
(!_seperators.Contains(_text[index])) &&
(!_singles.Contains(_text[index])) &&
(selfExistentToken.EndMark != _text[index]))
{
index++;
}
index -= 1;
int end = index;
return(start, end);
}
