Exemple #1
0
 private Analyzer GetAnalyer(int type)
 {
     Analyzer analyzer;
     string path =System.Configuration.ConfigurationSettings.AppSettings["Data"].ToString();
     switch (type)
     {
         case 0:
             analyzer=new StockFooAnalyzer(path);
             break;
        case 1:
             analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT);
             break;
       case 2:
             analyzer = new SimpleAnalyzer();
             break;
       case 3:
             analyzer = new StopAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT);
             break;
      case 4:
             analyzer = new KeywordAnalyzer();
             break;
       case 5:
             analyzer = new WhitespaceAnalyzer();
             break;
      default:
             analyzer = new StockFooAnalyzer(path);
             break;
     }
     return analyzer;
 }
Exemple #2
0
		public static Query BuildQuery(string query, IndexQuery indexQuery, PerFieldAnalyzerWrapper analyzer)
		{
			var originalQuery = query;
			Analyzer keywordAnalyzer = new KeywordAnalyzer();
			try
			{
				var queryParser = new RangeQueryParser(Version.LUCENE_29, indexQuery.DefaultField ?? string.Empty, analyzer)
				{
					DefaultOperator = indexQuery.DefaultOperator == QueryOperator.Or
										? QueryParser.Operator.OR
										: QueryParser.Operator.AND,
					AllowLeadingWildcard = true
				};
				query = PreProcessUntokenizedTerms(query, queryParser);
				query = PreProcessSearchTerms(query);
				query = PreProcessDateTerms(query, queryParser);
				var generatedQuery = queryParser.Parse(query);
				generatedQuery = HandleMethods(generatedQuery);
				return generatedQuery;
			}
			catch (ParseException pe)
			{
				if (originalQuery == query)
					throw new ParseException("Could not parse: '" + query + "'", pe);
				throw new ParseException("Could not parse modified query: '" + query + "' original was: '" + originalQuery + "'", pe);

			}
			finally
			{
				keywordAnalyzer.Close();
			}
		}
        public virtual void  TestOffsets()
        {
            TokenStream      stream    = new KeywordAnalyzer().TokenStream("field", new System.IO.StringReader("abcd"));
            IOffsetAttribute offsetAtt = stream.AddAttribute <IOffsetAttribute>();

            Assert.IsTrue(stream.IncrementToken());
            Assert.AreEqual(0, offsetAtt.StartOffset);
            Assert.AreEqual(4, offsetAtt.EndOffset);
        }
        public IDictionary<string, string> Find(string id)
        {
            var keyworldAnalyzer = new KeywordAnalyzer();
            var parser = new QueryParser(Version.LUCENE_30, "id", keyworldAnalyzer);
            var query = parser.Parse(id);
            var scoreDocs = _searcher.Search(query, 1).ScoreDocs;

            if (scoreDocs != null && scoreDocs.Length > 0)
                return Docs.DocToDict(_searcher.Doc(scoreDocs[0].Doc));

            return new Dictionary<string, string>();
        }
Exemple #5
0
		public static Query BuildQuery(string query, PerFieldAnalyzerWrapper analyzer)
		{
			var keywordAnalyzer = new KeywordAnalyzer();
			try
		    {
		    	query = PreProcessUntokenizedTerms(analyzer, query, keywordAnalyzer);
		    	var queryParser = new RangeQueryParser(Version.LUCENE_29, "", analyzer);
				queryParser.SetAllowLeadingWildcard(true);
		    	return queryParser.Parse(query);;
			}
		    finally
		    {
				keywordAnalyzer.Close();
		    }
		}
Exemple #6
0
		public static Query BuildQuery(string query, PerFieldAnalyzerWrapper analyzer)
		{
			Analyzer keywordAnalyzer = new KeywordAnalyzer();
			try
			{
				var queryParser = new RangeQueryParser(Version.LUCENE_29, string.Empty, analyzer);
				query = PreProcessUntokenizedTerms(query, queryParser);
				query = PreProcessSearchTerms(query);
				queryParser.SetAllowLeadingWildcard(true); // not the recommended approach, should rather use ReverseFilter
				return queryParser.Parse(query);
			}
			finally
			{
				keywordAnalyzer.Close();
			}
		}
Exemple #7
0
		/// <summary>
		/// Detects untokenized fields and sets as NotAnalyzed in analyzer
		/// </summary>
		private static string PreProcessUntokenizedTerms(PerFieldAnalyzerWrapper analyzer, string query, ref Analyzer keywordAnalyzer)
		{
			var untokenizedMatches = untokenizedQuery.Matches(query);
			if (untokenizedMatches.Count < 1)
				return query;

			var sb = new StringBuilder(query);

			// Initialize a KeywordAnalyzer
			// KeywordAnalyzer will not tokenize the values
			keywordAnalyzer = new KeywordAnalyzer();

			// process in reverse order to leverage match string indexes
			for (var i = untokenizedMatches.Count; i > 0; i--)
			{
				var match = untokenizedMatches[i - 1];

				// specify that term for this field should not be tokenized
				analyzer.AddAnalyzer(match.Groups[1].Value, keywordAnalyzer);

				var term = match.Groups[2];

				// introduce " " around the term
				var startIndex = term.Index;
				var length = term.Length - 2;
				if (sb[startIndex + length - 1] != '"')
				{
					sb.Insert(startIndex + length, '"');
					length += 1;
				}
				if (sb[startIndex + 2] != '"')
				{
					sb.Insert(startIndex + 2, '"');
					length += 1;
				}
				// remove enclosing "[[" "]]" from term value (again in reverse order)
				sb.Remove(startIndex + length, 2);
				sb.Remove(startIndex, 2);
			}

			return sb.ToString();
		}
Exemple #8
0
		public RavenPerFieldAnalyzerWrapper CreateAnalyzer(Analyzer defaultAnalyzer, ICollection<Action> toDispose, bool forQuerying = false)
		{
			toDispose.Add(defaultAnalyzer.Close);

			string value;
			if (indexDefinition.Analyzers.TryGetValue(Constants.AllFields, out value))
			{
				defaultAnalyzer = IndexingExtensions.CreateAnalyzerInstance(Constants.AllFields, value);
				toDispose.Add(defaultAnalyzer.Close);
			}
			var perFieldAnalyzerWrapper = new RavenPerFieldAnalyzerWrapper(defaultAnalyzer);
			foreach (var analyzer in indexDefinition.Analyzers)
			{
				Analyzer analyzerInstance = IndexingExtensions.CreateAnalyzerInstance(analyzer.Key, analyzer.Value);
				toDispose.Add(analyzerInstance.Close);

				if (forQuerying)
				{
					var customAttributes = analyzerInstance.GetType().GetCustomAttributes(typeof(NotForQueryingAttribute), false);
					if (customAttributes.Length > 0)
						continue;
				}

				perFieldAnalyzerWrapper.AddAnalyzer(analyzer.Key, analyzerInstance);
			}
			StandardAnalyzer standardAnalyzer = null;
			KeywordAnalyzer keywordAnalyzer = null;
			foreach (var fieldIndexing in indexDefinition.Indexes)
			{
				switch (fieldIndexing.Value)
				{
					case FieldIndexing.NotAnalyzed:
						if (keywordAnalyzer == null)
						{
							keywordAnalyzer = new KeywordAnalyzer();
							toDispose.Add(keywordAnalyzer.Close);
						}
						perFieldAnalyzerWrapper.AddAnalyzer(fieldIndexing.Key, keywordAnalyzer);
						break;
					case FieldIndexing.Analyzed:
						if (indexDefinition.Analyzers.ContainsKey(fieldIndexing.Key))
							continue;
						if (standardAnalyzer == null)
						{
							standardAnalyzer = new StandardAnalyzer(Version.LUCENE_29);
							toDispose.Add(standardAnalyzer.Close);
						}
						perFieldAnalyzerWrapper.AddAnalyzer(fieldIndexing.Key, standardAnalyzer);
						break;
				}
			}
			return perFieldAnalyzerWrapper;
		}
		public virtual void  TestOffsets()
		{
			TokenStream stream = new KeywordAnalyzer().TokenStream("field", new System.IO.StringReader("abcd"));
			OffsetAttribute offsetAtt = (OffsetAttribute) stream.AddAttribute(typeof(OffsetAttribute));
			Assert.IsTrue(stream.IncrementToken());
			Assert.AreEqual(0, offsetAtt.StartOffset());
			Assert.AreEqual(4, offsetAtt.EndOffset());
		}
Exemple #10
0
		public PerFieldAnalyzerWrapper CreateAnalyzer(Analyzer defaultAnalyzer, ICollection<Action> toDispose)
		{
			toDispose.Add(defaultAnalyzer.Close);
			var perFieldAnalyzerWrapper = new PerFieldAnalyzerWrapper(defaultAnalyzer);
			foreach (var analyzer in indexDefinition.Analyzers)
			{
				Analyzer analyzerInstance = IndexingExtensions.CreateAnalyzerInstance(analyzer.Key, analyzer.Value);
				if (analyzerInstance == null)
					continue;
				toDispose.Add(analyzerInstance.Close);
				perFieldAnalyzerWrapper.AddAnalyzer(analyzer.Key, analyzerInstance);
			}
			StandardAnalyzer standardAnalyzer = null;
			KeywordAnalyzer keywordAnalyzer = null;
			foreach (var fieldIndexing in indexDefinition.Indexes)
			{
				switch (fieldIndexing.Value)
				{
					case FieldIndexing.NotAnalyzed:
						if (keywordAnalyzer == null)
						{
							keywordAnalyzer = new KeywordAnalyzer();
							toDispose.Add(keywordAnalyzer.Close);
						}
						perFieldAnalyzerWrapper.AddAnalyzer(fieldIndexing.Key, keywordAnalyzer);
						break;
					case FieldIndexing.Analyzed:
						if (indexDefinition.Analyzers.ContainsKey(fieldIndexing.Key))
							continue;
						if (standardAnalyzer == null)
						{
							standardAnalyzer = new StandardAnalyzer(Version.LUCENE_29);
							toDispose.Add(standardAnalyzer.Close);
						}
						perFieldAnalyzerWrapper.AddAnalyzer(fieldIndexing.Key, standardAnalyzer);
						break;
				}
			}
			return perFieldAnalyzerWrapper;
		}
		private static Query Parse(string q)
		{
			using (var defaultAnalyzer = new KeywordAnalyzer())
			using (var perFieldAnalyzerWrapper = new PerFieldAnalyzerWrapper(defaultAnalyzer))
				return QueryBuilder.BuildQuery(q, perFieldAnalyzerWrapper);
		}
        public static IndexWriter Create(AbstractConnection connection, Entity entity) {
            var dir = LuceneDirectoryFactory.Create(connection, entity);
            Analyzer defaultAnalyzer = new KeywordAnalyzer();

            var analyzer = new PerFieldAnalyzerWrapper(defaultAnalyzer);
            foreach (var field in GetFields(entity, connection.Version, connection.Logger)) {
                analyzer.AddAnalyzer(field.Key, field.Value);
            }
            return new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
        }
        public static IndexWriter Create(AbstractConnection connection, Process process, Entity entity) {
            using (var dir = LuceneDirectoryFactory.Create(connection, entity)) {
                Analyzer defaultAnalyzer = new KeywordAnalyzer();
                if (process.SearchTypes.ContainsKey("default")) {
                    defaultAnalyzer = LuceneAnalyzerFactory.Create(process.SearchTypes["default"].Analyzer, connection.Version);
                }

                var analyzer = new PerFieldAnalyzerWrapper(defaultAnalyzer);
                foreach (var field in GetFields(entity, connection.Version, connection.Logger)) {
                    analyzer.AddAnalyzer(field.Key, field.Value);
                }
                return new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
            }
        }
        /// <summary>
        /// Loads the data into the Lucene index
        /// </summary>
        /// <param name="directory">
        /// Directory where the index is located.
        /// </param>
        private void LoadLuceneIndex(SimpleFSDirectory directory)
        {
            Analyzer analyzer = new KeywordAnalyzer();

            // -----------
            // Store products into Lucene.
            // This will create a new index. Other requests will still be able to read the existing index.

            // Create writer that will overwrite the existing index
            using (var writer = new IndexWriter(directory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED))
            {
                IEnumerable<ProductSuggestion> productSuggestions = _productRepository.GetAllSuggestions();

                foreach (var productSuggestion in productSuggestions)
                {
                    // Storing all names in upper case, so we can do case insensitive search easily

                    var doc = new Document();
                    doc.Add(new Field("Object", productSuggestion.ToString(), Field.Store.YES, Field.Index.NO));
                    doc.Add(new Field("UcName", productSuggestion.ProductCode.ToUpper(), Field.Store.NO, Field.Index.NOT_ANALYZED));
                    doc.Add(new Field("SuggestionType", ProductSuggestionCode , Field.Store.YES, Field.Index.NO));
                    writer.AddDocument(doc);
                }

                IEnumerable<CategorySuggestion> categorySuggestions = _categoryRepository.GetAllSuggestions();

                foreach (var categorySuggestion in categorySuggestions)
                {
                    var doc = new Document();
                    doc.Add(new Field("Object", categorySuggestion.ToString(), Field.Store.YES, Field.Index.NO));
                    doc.Add(new Field("UcName", categorySuggestion.CategoryName.ToUpper(), Field.Store.NO, Field.Index.NOT_ANALYZED));
                    doc.Add(new Field("SuggestionType", CategorySuggestionCode, Field.Store.YES, Field.Index.NO));
                    writer.AddDocument(doc);
                }
            }
        }
Exemple #15
0
		private PerFieldAnalyzerWrapper CreateAnalyzer(ICollection<Action> toDispose)
    	{
    		var standardAnalyzer = new StandardAnalyzer(Version.LUCENE_29);
			toDispose.Add(standardAnalyzer.Close);
    		var perFieldAnalyzerWrapper = new PerFieldAnalyzerWrapper(standardAnalyzer);
    		foreach (var analyzer in indexDefinition.Analyzers)
    		{
    			var analyzerInstance = indexDefinition.CreateAnalyzerInstance(analyzer.Key, analyzer.Value);
				if(analyzerInstance == null)
					continue;
				toDispose.Add(analyzerInstance.Close);
    			perFieldAnalyzerWrapper.AddAnalyzer(analyzer.Key, analyzerInstance);
    		}
			KeywordAnalyzer keywordAnalyzer = null;
			foreach (var fieldIndexing in indexDefinition.Indexes)
			{
				switch (fieldIndexing.Value)
				{
					case FieldIndexing.NotAnalyzedNoNorms:
					case FieldIndexing.NotAnalyzed:
						if(keywordAnalyzer  == null)
						{
							keywordAnalyzer = new KeywordAnalyzer();
							toDispose.Add(keywordAnalyzer.Close);
						}
						perFieldAnalyzerWrapper.AddAnalyzer(fieldIndexing.Key, keywordAnalyzer);
						break;
				}
			}
    		return perFieldAnalyzerWrapper;
    	}