public virtual SharpEntropy.TrainingEvent ReadNextEvent()
{
SentenceDetectionEvent topEvent = mHead;
mHead = mHead.NextEvent;
if (null == mHead)
{
mTail = null;
}
return(topEvent);
}
public virtual TrainingEvent ReadNextEvent()
{
SentenceDetectionEvent topEvent = _head;
_head = _head.NextEvent;
if (_head == null)
{
_tail = null;
}
return(topEvent);
}
private void AddNewEvents(string token)
{
StringBuilder buffer = _buffer;
buffer.Append(token.Trim());
// only one sentence per token, so the end of sentence if at the very end of the token
int sentenceEndPosition = buffer.Length - 1;
//add following word to stringbuilder
if (_next != null && token.Length > 0)
{
int positionAfterFirstWordInNext = _next.IndexOf(" ");
if (positionAfterFirstWordInNext != -1)
{
// should maybe changes this so that it usually adds a space
// before the next sentence, but sometimes leaves no space.
buffer.Append(" ");
buffer.Append(_next.Substring(0, positionAfterFirstWordInNext));
}
}
foreach (var candidate in _scanner.GetPositions(buffer))
{
var pair = new Tuple <StringBuilder, int>(buffer, candidate);
string type = (candidate == sentenceEndPosition) ? "T" : "F";
var sentenceEvent = new SentenceDetectionEvent(type, _contextGenerator.GetContext(pair));
if (_tail != null)
{
_tail.NextEvent = sentenceEvent;
_tail = sentenceEvent;
}
else if (_head == null)
{
_head = sentenceEvent;
}
else if (_head.NextEvent == null)
{
_head.NextEvent = _tail = sentenceEvent;
}
}
buffer.Length = 0;
}
private void AddNewEvents(string token)
{
System.Text.StringBuilder buffer = mBuffer;
buffer.Append(token.Trim());
int sentenceEndPosition = buffer.Length - 1;
//add following word to stringbuilder
if (mNext != null && token.Length > 0)
{
int positionAfterFirstWordInNext = mNext.IndexOf(" ");
if (positionAfterFirstWordInNext != -1)
{
// should maybe changes this so that it usually adds a space
// before the next sentence, but sometimes leaves no space.
buffer.Append(" ");
buffer.Append(mNext.Substring(0, (positionAfterFirstWordInNext) - (0)));
}
}
for (System.Collections.IEnumerator iterator = mScanner.GetPositions(buffer).GetEnumerator(); iterator.MoveNext();)
{
int candidate = (int)iterator.Current;
Util.Pair <System.Text.StringBuilder, int> pair = new Util.Pair <System.Text.StringBuilder, int>(buffer, candidate);
string type = (candidate == sentenceEndPosition) ? "T" : "F";
SentenceDetectionEvent sentenceEvent = new SentenceDetectionEvent(type, mContextGenerator.GetContext(pair));
if (null != mTail)
{
mTail.NextEvent = sentenceEvent;
mTail = sentenceEvent;
}
else if (null == mHead)
{
mHead = sentenceEvent;
}
else if (null == mHead.NextEvent)
{
mHead.NextEvent = mTail = sentenceEvent;
}
}
buffer.Length = 0;
}
public virtual TrainingEvent ReadNextEvent()
{
SentenceDetectionEvent topEvent = _head;
_head = _head.NextEvent;
if (_head == null)
{
_tail = null;
}
return topEvent;
}
private void AddNewEvents(string token)
{
StringBuilder buffer = _buffer;
buffer.Append(token.Trim());
// only one sentence per token, so the end of sentence if at the very end of the token
int sentenceEndPosition = buffer.Length - 1;
//add following word to stringbuilder
if (_next != null && token.Length > 0)
{
int positionAfterFirstWordInNext = _next.IndexOf(" ");
if (positionAfterFirstWordInNext != - 1)
{
// should maybe changes this so that it usually adds a space
// before the next sentence, but sometimes leaves no space.
buffer.Append(" ");
buffer.Append(_next.Substring(0, positionAfterFirstWordInNext));
}
}
foreach (var candidate in _scanner.GetPositions(buffer))
{
var pair = new Tuple<StringBuilder, int>(buffer, candidate);
string type = (candidate == sentenceEndPosition) ? "T" : "F";
var sentenceEvent = new SentenceDetectionEvent(type, _contextGenerator.GetContext(pair));
if (_tail != null)
{
_tail.NextEvent = sentenceEvent;
_tail = sentenceEvent;
}
else if (_head == null)
{
_head = sentenceEvent;
}
else if (_head.NextEvent == null)
{
_head.NextEvent = _tail = sentenceEvent;
}
}
buffer.Length = 0;
}
private void AddNewEvents(string token)
{
System.Text.StringBuilder buffer = mBuffer;
buffer.Append(token.Trim());
int sentenceEndPosition = buffer.Length - 1;
//add following word to stringbuilder
if (mNext != null && token.Length > 0)
{
int positionAfterFirstWordInNext = mNext.IndexOf(" ");
if (positionAfterFirstWordInNext != - 1)
{
// should maybe changes this so that it usually adds a space
// before the next sentence, but sometimes leaves no space.
buffer.Append(" ");
buffer.Append(mNext.Substring(0, (positionAfterFirstWordInNext) - (0)));
}
}
for (System.Collections.IEnumerator iterator = mScanner.GetPositions(buffer).GetEnumerator(); iterator.MoveNext(); )
{
int candidate = (int) iterator.Current;
Util.Pair<System.Text.StringBuilder, int> pair = new Util.Pair<System.Text.StringBuilder, int>(buffer, candidate);
string type = (candidate == sentenceEndPosition) ? "T" : "F";
SentenceDetectionEvent sentenceEvent = new SentenceDetectionEvent(type, mContextGenerator.GetContext(pair));
if (null != mTail)
{
mTail.NextEvent = sentenceEvent;
mTail = sentenceEvent;
}
else if (null == mHead)
{
mHead = sentenceEvent;
}
else if (null == mHead.NextEvent)
{
mHead.NextEvent = mTail = sentenceEvent;
}
}
buffer.Length = 0;
}
public virtual SharpEntropy.TrainingEvent ReadNextEvent()
{
SentenceDetectionEvent topEvent = mHead;
mHead = mHead.NextEvent;
if (null == mHead)
{
mTail = null;
}
return topEvent;
}
