/* Function: CreateFromIterators * Creates a new tokenizer from the range between two <LineIterators>. The line the ending iterator is * on is not included in the range. The new tokenizer has a copy of the memory and is thus independent. This * is faster than creating a new tokenizer around a substring of the raw text because it doesn't need to be * tokenized all over again. It also carries over any defined token information like <CommentParsingTypes>. */ public Tokenizer CreateFromIterators(LineIterator start, LineIterator end) { Tokenizer result = CreateFromIterators(start.FirstToken(LineBoundsMode.Everything), end.FirstToken(LineBoundsMode.Everything)); result.lines = lines.GetRange(start.LineIndex, end.LineIndex - start.LineIndex); return(result); }
/* Function: CreateFromIterators * Creates a new tokenizer from the range between two <LineIterators>. The line the ending iterator is * on is not included in the range. The new tokenizer has a copy of the memory and is thus independent. This * is faster than creating a new tokenizer around a substring of the raw text because it doesn't need to be * tokenized all over again. It also carries over any defined token information like <CommentParsingTypes>. */ public static Tokenizer CreateFromIterators(LineIterator start, LineIterator end) { Tokenizer extracted = CreateFromIterators(start.FirstToken(LineBoundsMode.Everything), end.FirstToken(LineBoundsMode.Everything)); extracted.lines = start.Tokenizer.lines.GetRange(start.LineIndex, end.LineIndex - start.LineIndex); return(extracted); }