Пример #1
0
        private void  DoTestStopPositons(StopFilter stpf, bool enableIcrements)
        {
            Log("---> test with enable-increments-" + (enableIcrements?"enabled":"disabled"));
            stpf.EnablePositionIncrements = enableIcrements;
            ITermAttribute termAtt = stpf.GetAttribute <ITermAttribute>();
            IPositionIncrementAttribute posIncrAtt = stpf.GetAttribute <IPositionIncrementAttribute>();

            for (int i = 0; i < 20; i += 3)
            {
                Assert.IsTrue(stpf.IncrementToken());
                Log("Token " + i + ": " + stpf);
                System.String w = English.IntToEnglish(i).Trim();
                Assert.AreEqual(w, termAtt.Term, "expecting token " + i + " to be " + w);
                Assert.AreEqual(enableIcrements?(i == 0?1:3):1, posIncrAtt.PositionIncrement, "all but first token must have position increment of 3");
            }
            Assert.IsFalse(stpf.IncrementToken());
        }
Пример #2
0
		public virtual void  TestIgnoreCase()
		{
			System.IO.StringReader reader = new System.IO.StringReader("Now is The Time");
			System.String[] stopWords = new System.String[]{"is", "the", "Time"};
			TokenStream stream = new StopFilter(false, new WhitespaceTokenizer(reader), stopWords, true);
			TermAttribute termAtt = (TermAttribute) stream.GetAttribute(typeof(TermAttribute));
			Assert.IsTrue(stream.IncrementToken());
			Assert.AreEqual("Now", termAtt.Term());
			Assert.IsFalse(stream.IncrementToken());
		}
Пример #3
0
        public virtual void  TestIgnoreCase()
        {
            System.IO.StringReader reader    = new System.IO.StringReader("Now is The Time");
            System.String[]        stopWords = new System.String[] { "is", "the", "Time" };
            TokenStream            stream    = new StopFilter(false, new WhitespaceTokenizer(reader), stopWords, true);
            TermAttribute          termAtt   = (TermAttribute)stream.GetAttribute(typeof(TermAttribute));

            Assert.IsTrue(stream.IncrementToken());
            Assert.AreEqual("Now", termAtt.Term());
            Assert.IsFalse(stream.IncrementToken());
        }
Пример #4
0
		public virtual void  TestIgnoreCase()
		{
			System.IO.StringReader reader = new System.IO.StringReader("Now is The Time");
            var stopWords = Support.Compatibility.SetFactory.GetSet<string>();
            stopWords.UnionWith(new[] {"is", "the", "Time"});

			TokenStream stream = new StopFilter(false, new WhitespaceTokenizer(reader), stopWords, true);
            ITermAttribute termAtt = stream.GetAttribute<ITermAttribute>();
			Assert.IsTrue(stream.IncrementToken());
			Assert.AreEqual("Now", termAtt.Term);
			Assert.IsFalse(stream.IncrementToken());
		}
Пример #5
0
		public virtual void  TestStopFilt()
		{
			System.IO.StringReader reader = new System.IO.StringReader("Now is The Time");
			System.String[] stopWords = new System.String[]{"is", "the", "Time"};
			var stopSet = StopFilter.MakeStopSet(stopWords);
			TokenStream stream = new StopFilter(false, new WhitespaceTokenizer(reader), stopSet);
            ITermAttribute termAtt = stream.GetAttribute<ITermAttribute>();
			Assert.IsTrue(stream.IncrementToken());
			Assert.AreEqual("Now", termAtt.Term);
			Assert.IsTrue(stream.IncrementToken());
			Assert.AreEqual("The", termAtt.Term);
			Assert.IsFalse(stream.IncrementToken());
		}
Пример #6
0
        public virtual void  TestStopFilt()
        {
            System.IO.StringReader reader    = new System.IO.StringReader("Now is The Time");
            System.String[]        stopWords = new System.String[] { "is", "the", "Time" };
            var            stopSet           = StopFilter.MakeStopSet(stopWords);
            TokenStream    stream            = new StopFilter(false, new WhitespaceTokenizer(reader), stopSet);
            ITermAttribute termAtt           = stream.GetAttribute <ITermAttribute>();

            Assert.IsTrue(stream.IncrementToken());
            Assert.AreEqual("Now", termAtt.Term);
            Assert.IsTrue(stream.IncrementToken());
            Assert.AreEqual("The", termAtt.Term);
            Assert.IsFalse(stream.IncrementToken());
        }
Пример #7
0
        public virtual void  TestIgnoreCase()
        {
            System.IO.StringReader reader = new System.IO.StringReader("Now is The Time");
            var stopWords = Support.Compatibility.SetFactory.CreateHashSet <string>();

            stopWords.UnionWith(new[] { "is", "the", "Time" });

            TokenStream    stream  = new StopFilter(false, new WhitespaceTokenizer(reader), stopWords, true);
            ITermAttribute termAtt = stream.GetAttribute <ITermAttribute>();

            Assert.IsTrue(stream.IncrementToken());
            Assert.AreEqual("Now", termAtt.Term);
            Assert.IsFalse(stream.IncrementToken());
        }
Пример #8
0
		private void  DoTestStopPositons(StopFilter stpf, bool enableIcrements)
		{
			Log("---> test with enable-increments-" + (enableIcrements?"enabled":"disabled"));
			stpf.EnablePositionIncrements = enableIcrements;
            ITermAttribute termAtt = stpf.GetAttribute<ITermAttribute>();
            IPositionIncrementAttribute posIncrAtt = stpf.GetAttribute<IPositionIncrementAttribute>();
			for (int i = 0; i < 20; i += 3)
			{
				Assert.IsTrue(stpf.IncrementToken());
				Log("Token " + i + ": " + stpf);
				System.String w = English.IntToEnglish(i).Trim();
				Assert.AreEqual(w, termAtt.Term, "expecting token " + i + " to be " + w);
				Assert.AreEqual(enableIcrements?(i == 0?1:3):1, posIncrAtt.PositionIncrement, "all but first token must have position increment of 3");
			}
			Assert.IsFalse(stpf.IncrementToken());
		}