public static IEnumerable <BatchedDocument> ConvertJsonDocumentToLuceneDocument(IndexDefinition definition,
                                                                                        IEnumerable <JsonDocument> jsonDocuments)
        {
            var converter = new JsonDocumentToLuceneDocumentConverter(definition);

            return(jsonDocuments.Select(jsonDocument =>
            {
                var fields = converter.Index(jsonDocument, Field.Store.YES); // This might be No to save space..

                var luceneDoc = new Document();
                var documentIdField = new Field(Constants.DocumentIdFieldName, jsonDocument.DocumentId, Field.Store.YES,
                                                Field.Index.ANALYZED_NO_NORMS);
                luceneDoc.Add(documentIdField);

                var tempJsonDocument = new JsonDocumentDto
                {
                    DocumentId = jsonDocument.DocumentId,
                    MetadataAsJson = jsonDocument.Metadata.ToString(),
                    DataAsJson = jsonDocument.DataAsJson.ToString()
                };

                var jsonDocumentField = new Field(Constants.JsonDocumentFieldName, JsonConvert.SerializeObject(tempJsonDocument), Field.Store.YES,
                                                  Field.Index.NOT_ANALYZED_NO_NORMS);

                luceneDoc.Add(jsonDocumentField);

                foreach (var field in fields)
                {
                    luceneDoc.Add(field);
                }

                return new BatchedDocument
                {
                    DocumentId = jsonDocument.DocumentId,
                    Document = luceneDoc,
                    SkipDeleteFromIndex = false,
                };
            }));
        }
Exemple #2
0
        public void IndexJsonDocuments(BaristaIndexDefinition indexDefinition, IEnumerable <JsonDocumentDto> documents)
        {
            try
            {
                if (documents == null)
                {
                    throw new ArgumentNullException("documents", @"A collection of documents must be specified.");
                }

                var jsonDocuments = documents as IList <JsonDocumentDto> ?? documents.ToList();

                if (jsonDocuments.Any() == false)
                {
                    throw new ArgumentNullException("documents", @"At least one document must be contained within the collection.");
                }

                var index = GetOrAddIndex(indexDefinition, true);

                try
                {
                    //Add it to the index.
                    var batch = new IndexingBatch();

                    //Update the indexDefinition for the index based on the options specified.
                    foreach (var document in jsonDocuments)
                    {
                        UpdateIndexDefinitionFromFieldOptions(index.IndexDefinition, document.FieldOptions);
                    }

                    //Attempt to create a new Search.JsonDocument from the document
                    var searchJsonDocuments = jsonDocuments.Select(document => new Search.JsonDocument
                    {
                        DocumentId = document.DocumentId,
                        Metadata   = document.MetadataAsJson.IsNullOrWhiteSpace() == false
                                     ? JObject.Parse(document.MetadataAsJson)
                                     : new JObject(),
                        DataAsJson = JObject.Parse(document.DataAsJson)
                    });

                    var luceneDocuments =
                        JsonDocumentToLuceneDocumentConverter.ConvertJsonDocumentToLuceneDocument(index.IndexDefinition,
                                                                                                  searchJsonDocuments);

                    foreach (var luceneDocument in luceneDocuments)
                    {
                        batch.Add(luceneDocument);
                    }

                    //TODO: Add the batch to a BlockingCollection<IndexingBatch> and run a thread that consumes the batches
                    //See http://www.codethinked.com/blockingcollection-and-iproducerconsumercollection
                    index.IndexDocuments(batch);
                }
                catch (OutOfMemoryException)
                {
                    CloseIndexWriter(indexDefinition, false);
                }
            }
            catch (Exception ex)
            {
                throw new FaultException(ex.Message);
            }
        }