示例#1
0
        private void AddNewEvents()
        {
            List <string> tokenList     = new List <string>();
            List <string> tagList       = new List <string>();
            List <string> predicateList = new List <string>();

            for (string line = mDataReader.NextToken(); line.Length > 0; line = mDataReader.NextToken())
            {
                string[] parts = line.Split(' ');
                if (parts.Length != 3)
                {
                    //skip this line; it is in error
                }
                else
                {
                    tokenList.Add(parts[0]);
                    tagList.Add(parts[1]);
                    predicateList.Add(parts[2]);
                }
            }
            mEvents = new SharpEntropy.TrainingEvent[tokenList.Count];
            string[] tokens     = tokenList.ToArray();
            string[] tags       = tagList.ToArray();
            string[] predicates = predicateList.ToArray();

            for (int eventIndex = 0, eventCount = mEvents.Length; eventIndex < eventCount; eventIndex++)
            {
                mEvents[eventIndex] = new SharpEntropy.TrainingEvent(predicates[eventIndex], mContextGenerator.GetContext(eventIndex, tokens, tags, predicates));
            }
        }
 public SentenceDetectionEventReader(SharpEntropy.ITrainingDataReader <string> dataReader, IEndOfSentenceScanner scanner, SharpEntropy.IContextGenerator <Util.Pair <System.Text.StringBuilder, int> > contextGenerator)
 {
     mDataReader       = dataReader;
     mScanner          = scanner;
     mContextGenerator = contextGenerator;
     if (mDataReader.HasNext())
     {
         string current = mDataReader.NextToken();
         if (mDataReader.HasNext())
         {
             mNext = mDataReader.NextToken();
         }
         AddNewEvents(current);
     }
 }
        /// <summary>
        /// Creates a new event reader based on the specified data reader using the specified context generator.
        /// </summary>
        /// <param name="dataReader">
        /// The data reader for this event reader.
        /// </param>
        /// <param name="contextGenerator">
        /// The context generator which should be used in the creation of events for this event stream.
        /// </param>
        public NameFinderEventReader(SharpEntropy.ITrainingDataReader <string> dataReader, INameContextGenerator contextGenerator)
        {
            mDataReader       = dataReader;
            mContextGenerator = contextGenerator;
            mEventIndex       = 0;
            mPreviousTags     = new Dictionary <string, string>();

            //prime events with first line of data stream.
            if (mDataReader.HasNext())
            {
                mLine = mDataReader.NextToken();
                if (mLine.Length == 0)
                {
                    mPreviousTags.Clear();
                }
                else
                {
                    AddEvents(mLine);
                }
            }
            else
            {
                mEvents = new SharpEntropy.TrainingEvent[0];
            }
        }
 public virtual bool HasNext()
 {
     if (mEventIndex < mEvents.Length)
     {
         return(true);
     }
     else if (mLine != null)
     {             // previous result has not been consumed
         return(true);
     }
     //find next non-blank line
     while (mDataReader.HasNext())
     {
         mLine = mDataReader.NextToken();
         if (mLine.Length == 0)
         {
             mPreviousTags.Clear();
         }
         else
         {
             return(true);
         }
     }
     return(false);
 }
        public virtual bool HasNext()
        {
            if (null != mHead)
            {
                return(true);
            }

            while (null == mHead && (object)mNext != null)
            {
                string current = mNext;
                if (mDataReader.HasNext())
                {
                    mNext = (mDataReader.NextToken());
                }
                else
                {
                    mNext = null;
                }
                AddNewEvents(current);
            }
            return(null != mHead);
        }
示例#6
0
        private void AddNewEvents()
        {
            string parseString = mDataReader.NextToken();
            //System.Console.WriteLine("ParserEventStream.AddNewEvents: " + parseString);
            List <SharpEntropy.TrainingEvent> events = new List <SharpEntropy.TrainingEvent>();
            Parse rootParse = Parse.FromParseString(parseString);

            rootParse.UpdateHeads(mHeadRules);
            Parse[] chunks = GetInitialChunks(rootParse);
            if (mEventType == EventType.Tag)
            {
                AddTagEvents(events, chunks);
            }
            else if (mEventType == EventType.Chunk)
            {
                AddChunkEvents(events, chunks);
            }
            else
            {
                AddParseEvents(events, chunks);
            }
            mEvents = events.ToArray();
        }
示例#7
0
		/// <summary>
		/// Creates a new event reader based on the specified data reader using the specified context generator.
		/// </summary>
		/// <param name="dataReader">
		/// The data reader for this event reader.
		/// </param>
		/// <param name="contextGenerator">
		/// The context generator which should be used in the creation of events for this event stream.
		/// </param>
		public NameFinderEventReader(SharpEntropy.ITrainingDataReader<string> dataReader, INameContextGenerator contextGenerator)
		{
			mDataReader = dataReader;
			mContextGenerator = contextGenerator;
			mEventIndex = 0;
            mPreviousTags = new Dictionary<string, string>();

			//prime events with first line of data stream.
			if (mDataReader.HasNext())
			{
				mLine = mDataReader.NextToken();
				if (mLine.Length == 0)
				{
					mPreviousTags.Clear();
				}
				else
				{
					AddEvents(mLine);
				}
			}
			else
			{
				mEvents = new SharpEntropy.TrainingEvent[0];
			}
		}
 public SentenceDetectionEventReader(SharpEntropy.ITrainingDataReader<string> dataReader, IEndOfSentenceScanner scanner, SharpEntropy.IContextGenerator<Util.Pair<System.Text.StringBuilder, int>> contextGenerator)
 {
     mDataReader = dataReader;
     mScanner = scanner;
     mContextGenerator = contextGenerator;
     if (mDataReader.HasNext())
     {
         string current = mDataReader.NextToken();
         if (mDataReader.HasNext())
         {
             mNext = mDataReader.NextToken();
         }
         AddNewEvents(current);
     }
 }