public static IEnumerable <BatchedDocument> ConvertJsonDocumentToLuceneDocument(IndexDefinition definition, IEnumerable <JsonDocument> jsonDocuments) { var converter = new JsonDocumentToLuceneDocumentConverter(definition); return(jsonDocuments.Select(jsonDocument => { var fields = converter.Index(jsonDocument, Field.Store.YES); // This might be No to save space.. var luceneDoc = new Document(); var documentIdField = new Field(Constants.DocumentIdFieldName, jsonDocument.DocumentId, Field.Store.YES, Field.Index.ANALYZED_NO_NORMS); luceneDoc.Add(documentIdField); var tempJsonDocument = new JsonDocumentDto { DocumentId = jsonDocument.DocumentId, MetadataAsJson = jsonDocument.Metadata.ToString(), DataAsJson = jsonDocument.DataAsJson.ToString() }; var jsonDocumentField = new Field(Constants.JsonDocumentFieldName, JsonConvert.SerializeObject(tempJsonDocument), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); luceneDoc.Add(jsonDocumentField); foreach (var field in fields) { luceneDoc.Add(field); } return new BatchedDocument { DocumentId = jsonDocument.DocumentId, Document = luceneDoc, SkipDeleteFromIndex = false, }; })); }
public void IndexJsonDocuments(BaristaIndexDefinition indexDefinition, IEnumerable <JsonDocumentDto> documents) { try { if (documents == null) { throw new ArgumentNullException("documents", @"A collection of documents must be specified."); } var jsonDocuments = documents as IList <JsonDocumentDto> ?? documents.ToList(); if (jsonDocuments.Any() == false) { throw new ArgumentNullException("documents", @"At least one document must be contained within the collection."); } var index = GetOrAddIndex(indexDefinition, true); try { //Add it to the index. var batch = new IndexingBatch(); //Update the indexDefinition for the index based on the options specified. foreach (var document in jsonDocuments) { UpdateIndexDefinitionFromFieldOptions(index.IndexDefinition, document.FieldOptions); } //Attempt to create a new Search.JsonDocument from the document var searchJsonDocuments = jsonDocuments.Select(document => new Search.JsonDocument { DocumentId = document.DocumentId, Metadata = document.MetadataAsJson.IsNullOrWhiteSpace() == false ? JObject.Parse(document.MetadataAsJson) : new JObject(), DataAsJson = JObject.Parse(document.DataAsJson) }); var luceneDocuments = JsonDocumentToLuceneDocumentConverter.ConvertJsonDocumentToLuceneDocument(index.IndexDefinition, searchJsonDocuments); foreach (var luceneDocument in luceneDocuments) { batch.Add(luceneDocument); } //TODO: Add the batch to a BlockingCollection<IndexingBatch> and run a thread that consumes the batches //See http://www.codethinked.com/blockingcollection-and-iproducerconsumercollection index.IndexDocuments(batch); } catch (OutOfMemoryException) { CloseIndexWriter(indexDefinition, false); } } catch (Exception ex) { throw new FaultException(ex.Message); } }