public void LineBreaksTest()
{
string sample = "Hello World";
foreach (var nl in new[] { "\r", "\r\n", "\n", "\u0085", "\u2028", "\u2029" })
{
for (int breakscount = 0; breakscount < 512; breakscount += 17)
{
// construct sample text with {linecount} lines
string text = string.Empty;
for (var line = 0; line < breakscount; line++)
{
text += sample + nl;
}
text += sample;
// test LineBreaks
var linebreaks = LineBreaks.Create(text);
Assert.AreEqual(linebreaks.LinesCount, breakscount + 1);
for (int i = 0; i <= text.Length; i += 7)
{
Assert.AreEqual(linebreaks.GetLineFromPosition(i), (i / (sample.Length + nl.Length)));
}
}
}
}
public static PhpArray /*!*/ token_get_all(string source, int flags = 0)
{
var tokens = new PhpArray();
if (string.IsNullOrEmpty(source))
{
return(tokens);
}
Tokens t;
var lines = LineBreaks.Create(source);
using (var tokenizer = new Lexer(new StringReader(source), Encoding.UTF8))
{
while ((t = tokenizer.GetNextToken()) != Tokens.EOF)
{
if (tokenizer.TokenSpan.Length == 1 && (int)t == tokenizer.TokenText[0])
{
// single char token
tokens.Add(tokenizer.TokenText);
}
else
{
// other
tokens.Add(new PhpArray(3)
{
(int)t,
tokenizer.TokenText,
lines.GetLineFromPosition(tokenizer.TokenSpan.Start) + 1,
});
}
//
if (t == Tokens.T_ERROR)
{
break;
}
}
}
return(tokens);
}
static PHPDocBlock NewPHPDoc(string phpdoc)
{
var linebreaks = LineBreaks.Create(phpdoc);
return(new PHPDocBlock(phpdoc, new TextSpan(linebreaks, 0, phpdoc.Length)));
}