public void LineBreaksTest() { string sample = "Hello World"; foreach (var nl in new[] { "\r", "\r\n", "\n", "\u0085", "\u2028", "\u2029" }) { for (int breakscount = 0; breakscount < 512; breakscount += 17) { // construct sample text with {linecount} lines string text = string.Empty; for (var line = 0; line < breakscount; line++) { text += sample + nl; } text += sample; // test LineBreaks var linebreaks = LineBreaks.Create(text); Assert.AreEqual(linebreaks.LinesCount, breakscount + 1); for (int i = 0; i <= text.Length; i += 7) { Assert.AreEqual(linebreaks.GetLineFromPosition(i), (i / (sample.Length + nl.Length))); } } } }
public static PhpArray /*!*/ token_get_all(string source, int flags = 0) { var tokens = new PhpArray(); if (string.IsNullOrEmpty(source)) { return(tokens); } Tokens t; var lines = LineBreaks.Create(source); using (var tokenizer = new Lexer(new StringReader(source), Encoding.UTF8)) { while ((t = tokenizer.GetNextToken()) != Tokens.EOF) { if (tokenizer.TokenSpan.Length == 1 && (int)t == tokenizer.TokenText[0]) { // single char token tokens.Add(tokenizer.TokenText); } else { // other tokens.Add(new PhpArray(3) { (int)t, tokenizer.TokenText, lines.GetLineFromPosition(tokenizer.TokenSpan.Start) + 1, }); } // if (t == Tokens.T_ERROR) { break; } } } return(tokens); }
static PHPDocBlock NewPHPDoc(string phpdoc) { var linebreaks = LineBreaks.Create(phpdoc); return(new PHPDocBlock(phpdoc, new TextSpan(linebreaks, 0, phpdoc.Length))); }