private void AddNewEvents()
{
List <string> tokenList = new List <string>();
List <string> tagList = new List <string>();
List <string> predicateList = new List <string>();
for (string line = mDataReader.NextToken(); line.Length > 0; line = mDataReader.NextToken())
{
string[] parts = line.Split(' ');
if (parts.Length != 3)
{
//skip this line; it is in error
}
else
{
tokenList.Add(parts[0]);
tagList.Add(parts[1]);
predicateList.Add(parts[2]);
}
}
mEvents = new SharpEntropy.TrainingEvent[tokenList.Count];
string[] tokens = tokenList.ToArray();
string[] tags = tagList.ToArray();
string[] predicates = predicateList.ToArray();
for (int eventIndex = 0, eventCount = mEvents.Length; eventIndex < eventCount; eventIndex++)
{
mEvents[eventIndex] = new SharpEntropy.TrainingEvent(predicates[eventIndex], mContextGenerator.GetContext(eventIndex, tokens, tags, predicates));
}
}
public SentenceDetectionEventReader(SharpEntropy.ITrainingDataReader <string> dataReader, IEndOfSentenceScanner scanner, SharpEntropy.IContextGenerator <Util.Pair <System.Text.StringBuilder, int> > contextGenerator)
{
mDataReader = dataReader;
mScanner = scanner;
mContextGenerator = contextGenerator;
if (mDataReader.HasNext())
{
string current = mDataReader.NextToken();
if (mDataReader.HasNext())
{
mNext = mDataReader.NextToken();
}
AddNewEvents(current);
}
}
/// <summary>
/// Creates a new event reader based on the specified data reader using the specified context generator.
/// </summary>
/// <param name="dataReader">
/// The data reader for this event reader.
/// </param>
/// <param name="contextGenerator">
/// The context generator which should be used in the creation of events for this event stream.
/// </param>
public NameFinderEventReader(SharpEntropy.ITrainingDataReader <string> dataReader, INameContextGenerator contextGenerator)
{
mDataReader = dataReader;
mContextGenerator = contextGenerator;
mEventIndex = 0;
mPreviousTags = new Dictionary <string, string>();
//prime events with first line of data stream.
if (mDataReader.HasNext())
{
mLine = mDataReader.NextToken();
if (mLine.Length == 0)
{
mPreviousTags.Clear();
}
else
{
AddEvents(mLine);
}
}
else
{
mEvents = new SharpEntropy.TrainingEvent[0];
}
}
public virtual bool HasNext()
{
if (mEventIndex < mEvents.Length)
{
return(true);
}
else if (mLine != null)
{ // previous result has not been consumed
return(true);
}
//find next non-blank line
while (mDataReader.HasNext())
{
mLine = mDataReader.NextToken();
if (mLine.Length == 0)
{
mPreviousTags.Clear();
}
else
{
return(true);
}
}
return(false);
}
public virtual bool HasNext()
{
if (null != mHead)
{
return(true);
}
while (null == mHead && (object)mNext != null)
{
string current = mNext;
if (mDataReader.HasNext())
{
mNext = (mDataReader.NextToken());
}
else
{
mNext = null;
}
AddNewEvents(current);
}
return(null != mHead);
}
private void AddNewEvents()
{
string parseString = mDataReader.NextToken();
//System.Console.WriteLine("ParserEventStream.AddNewEvents: " + parseString);
List <SharpEntropy.TrainingEvent> events = new List <SharpEntropy.TrainingEvent>();
Parse rootParse = Parse.FromParseString(parseString);
rootParse.UpdateHeads(mHeadRules);
Parse[] chunks = GetInitialChunks(rootParse);
if (mEventType == EventType.Tag)
{
AddTagEvents(events, chunks);
}
else if (mEventType == EventType.Chunk)
{
AddChunkEvents(events, chunks);
}
else
{
AddParseEvents(events, chunks);
}
mEvents = events.ToArray();
}
/// <summary>
/// Creates a new event reader based on the specified data reader using the specified context generator.
/// </summary>
/// <param name="dataReader">
/// The data reader for this event reader.
/// </param>
/// <param name="contextGenerator">
/// The context generator which should be used in the creation of events for this event stream.
/// </param>
public NameFinderEventReader(SharpEntropy.ITrainingDataReader<string> dataReader, INameContextGenerator contextGenerator)
{
mDataReader = dataReader;
mContextGenerator = contextGenerator;
mEventIndex = 0;
mPreviousTags = new Dictionary<string, string>();
//prime events with first line of data stream.
if (mDataReader.HasNext())
{
mLine = mDataReader.NextToken();
if (mLine.Length == 0)
{
mPreviousTags.Clear();
}
else
{
AddEvents(mLine);
}
}
else
{
mEvents = new SharpEntropy.TrainingEvent[0];
}
}
public SentenceDetectionEventReader(SharpEntropy.ITrainingDataReader<string> dataReader, IEndOfSentenceScanner scanner, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator)
{
mDataReader = dataReader;
mScanner = scanner;
mContextGenerator = contextGenerator;
if (mDataReader.HasNext())
{
string current = mDataReader.NextToken();
if (mDataReader.HasNext())
{
mNext = mDataReader.NextToken();
}
AddNewEvents(current);
}
}