/// <summary> /// 实例化一个 Splitor /// </summary> /// <param name="text">源字符串</param> /// <param name="seperators">依据该数组中的字符进行切割</param> /// <param name="singles">需要把单个字符作为 token 的字符集</param> public Tokenizer(string text, char[] seperators, char[] singles, SelfExistentToken whiteList) { this._text = text; this._seperators = seperators; this._singles = singles; this._selftExistentToken = whiteList; this.Tokens = MakeTokens(); }
/// <summary> /// 根据传入的索引获取组成 token 的字符范围 /// </summary> /// <param name="index">代表 token 首字符所在的位置</param> /// <returns></returns> private (int Start, int End) GetRange(ref int index, SelfExistentToken selfExistentToken) { int start = index; while ((index < this._text.Length) && (!_seperators.Contains(_text[index])) && (!_singles.Contains(_text[index])) && (selfExistentToken.EndMark != _text[index])) { index++; } index -= 1; int end = index; return(start, end); }