public static void TokenizeText(string text)
{
var temp = new StringReader(text);
PTBTokenizer ptbt = new PTBTokenizer(temp, new CoreLabelTokenFactory(), "");
while (ptbt.hasNext())
{
CoreLabel label = (CoreLabel)ptbt.next();
Console.WriteLine(String.Format("{0}\t| BEGIN_OFFSET: {1}\t| END_OFFSET: {2}", label.value(), label.beginPosition(), label.endPosition()));
}
}
