public SentenceDetectionEventReader(ITrainingDataReader <string> dataReader, IEndOfSentenceScanner scanner, IContextGenerator <Tuple <StringBuilder, int> > contextGenerator) { _dataReader = dataReader; _scanner = scanner; _contextGenerator = contextGenerator; if (_dataReader.HasNext()) { string current = _dataReader.NextToken(); if (_dataReader.HasNext()) { _next = _dataReader.NextToken(); } AddNewEvents(current); } }
public SentenceDetectionEventReader(ITrainingDataReader<string> dataReader, IEndOfSentenceScanner scanner, IContextGenerator<Tuple<StringBuilder, int>> contextGenerator) { _dataReader = dataReader; _scanner = scanner; _contextGenerator = contextGenerator; if (_dataReader.HasNext()) { string current = _dataReader.NextToken(); if (_dataReader.HasNext()) { _next = _dataReader.NextToken(); } AddNewEvents(current); } }
/// <summary> /// Constructor sets up the training event reader based on a stream of training data. /// </summary> /// <param name="dataReader"> /// Stream of training data. /// </param> public BasicEventReader(ITrainingDataReader<string> dataReader) { mContext = new BasicContextGenerator(); mDataReader = dataReader; if (mDataReader.HasNext()) { mNextEvent = CreateEvent(mDataReader.NextToken()); } }
/// <summary> /// Returns the next Event object held in this EventReader. Each call to ReadNextEvent advances the EventReader. /// </summary> /// <returns> /// the Event object which is next in this EventReader /// </returns> public virtual TrainingEvent ReadNextEvent() { while (mNextEvent == null && mDataReader.HasNext()) { mNextEvent = CreateEvent(mDataReader.NextToken()); } TrainingEvent currentEvent = mNextEvent; if (mDataReader.HasNext()) { mNextEvent = CreateEvent(mDataReader.NextToken()); } else { mNextEvent = null; } return(currentEvent); }
/// <summary> /// Constructor sets up the training event reader based on a stream of training data. /// </summary> /// <param name="dataReader"> /// Stream of training data. /// </param> public BasicEventReader(ITrainingDataReader <string> dataReader) { mContext = new BasicContextGenerator(); mDataReader = dataReader; if (mDataReader.HasNext()) { mNextEvent = CreateEvent(mDataReader.NextToken()); } }
public virtual TrainingEvent ReadNextEvent() { var nextToken = _dataReader.NextToken(); // split on tab var parts = nextToken.Split('\t'); var isInvalid = parts.Last() == "1"; var type = isInvalid ? "INV" : "OK"; var email = parts.First(); var nextEvent = new TrainingEvent(type, _contextGenerator.GetContext(email)); return(nextEvent); }
public virtual bool HasNext() { if (_head != null) { return(true); } while (_head == null && _next != null) { string current = _next; _next = _dataReader.HasNext() ? _dataReader.NextToken() : null; AddNewEvents(current); } return(_head != null); }