public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IOpenSearchResultCollection results) { OpenSearchFactory.RemoveLinksByRel(ref results, "self"); OpenSearchFactory.RemoveLinksByRel(ref results, "search"); IElasticCollection docs = type.FromOpenSearchResultCollection(results); BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = type.Index, Type = type.Type, Operations = new List<IBulkOperation>() }; RootObjectMapping currentMapping = null; try { var mappingResponse = client.GetMapping<IElasticType>(g => g.Index(type.Index.Name).Type(type.Type.Name)); currentMapping = mappingResponse.Mapping; } catch (Exception) { } var rootObjectMapping = type.GetRootMapping(); if (!rootObjectMapping.Equals(currentMapping)) { client.Map<IElasticType>(m => m.Index(type.Index.Name).Type(type.Type.Name)); } foreach (var doc in docs.ElasticItems) { var bulkIndexOperation = new BulkIndexOperation<IElasticItem>(doc); bulkIndexOperation.Id = ((IOpenSearchResultItem)doc).Identifier; bulkIndexOperation.Type = type.Type.Name; var bulkOp = bulkIndexOperation; bulkRequest.Operations.Add(bulkOp); } var response = client.Bulk(bulkRequest); BulkOperationsResponse ingestionResponse = new BulkOperationsResponse(); foreach (var item in response.Items) { if (!item.IsValid) { ingestionResponse.Errors++; continue; } if (item.Version == "1") ingestionResponse.Added++; else ingestionResponse.Updated++; } return ingestionResponse; }
internal IndexInformation CreateCatalogueIndex(Terradue.ElasticCas.Request.CreateIndexRequest createRequest, bool destroy = false) { if (client.IndexExists(i => i.Index(createRequest.IndexName)).Exists) { if (destroy) { client.DeleteIndex(d => d.Index(createRequest.IndexName)); } else { throw new InvalidOperationException(string.Format("'{0}' index already exists and cannot be overriden without data loss", createRequest.IndexName)); } } var response = client.CreateIndex(c => c.Index(createRequest.IndexName)); IndexInformation indexInformation = new IndexInformation(); var status = client.Status(s => s.Index(createRequest.IndexName)); indexInformation.Name = createRequest.IndexName; indexInformation.Shards = status.Shards; indexInformation.Mappings = new Dictionary <string, ICollection <PropertyNameMarker> >(); // Init mappings for each types declared if (createRequest.TypeNames == null || createRequest.TypeNames.Length == 0) { List <string> typeNames = new List <string>(); foreach (TypeExtensionNode node in AddinManager.GetExtensionNodes(typeof(IOpenSearchableElasticType))) { IOpenSearchableElasticType type = (IOpenSearchableElasticType)node.CreateInstance(); if (type is GenericJsonOpenSearchable) { continue; } IndexNameMarker indexName = new IndexNameMarker(); indexName.Name = createRequest.IndexName; PutMappingRequest putMappingRequest = new PutMappingRequest(indexName, type.Type); ((IPutMappingRequest)putMappingRequest).Mapping = type.GetRootMapping(); client.Map(putMappingRequest); indexInformation.Mappings.Add(type.Identifier, ((IPutMappingRequest)putMappingRequest).Mapping.Properties.Keys); typeNames.Add(type.Type.Name); } createRequest.TypeNames = typeNames.ToArray(); } else { foreach (TypeExtensionNode node in AddinManager.GetExtensionNodes(typeof(IElasticItem))) { IOpenSearchableElasticType type = (IOpenSearchableElasticType)node.CreateInstance(); foreach (string typeName in createRequest.TypeNames) { if (typeName == type.Type.Name) { PutMappingRequest putMappingRequest = new PutMappingRequest(type.Index, type.Type); ((IPutMappingRequest)putMappingRequest).Mapping = type.GetRootMapping(); client.Map(putMappingRequest); indexInformation.Mappings.Add(type.Identifier, ((IPutMappingRequest)putMappingRequest).Mapping.Properties.Keys); } } } } return(indexInformation); }
public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IOpenSearchResultCollection results) { OpenSearchFactory.RemoveLinksByRel(ref results, "self"); OpenSearchFactory.RemoveLinksByRel(ref results, "search"); IElasticCollection docs = type.FromOpenSearchResultCollection(results); BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = type.Index, Type = type.Type, Operations = new List <IBulkOperation>() }; RootObjectMapping currentMapping = null; try { var mappingResponse = client.GetMapping <IElasticType>(g => g.Index(type.Index.Name).Type(type.Type.Name)); currentMapping = mappingResponse.Mapping; } catch (Exception) { } var rootObjectMapping = type.GetRootMapping(); if (!rootObjectMapping.Equals(currentMapping)) { client.Map <IElasticType>(m => m.Index(type.Index.Name).Type(type.Type.Name)); } foreach (var doc in docs.ElasticItems) { var bulkIndexOperation = new BulkIndexOperation <IElasticItem>(doc); bulkIndexOperation.Id = ((IOpenSearchResultItem)doc).Identifier; bulkIndexOperation.Type = type.Type.Name; var bulkOp = bulkIndexOperation; bulkRequest.Operations.Add(bulkOp); } var response = client.Bulk(bulkRequest); BulkOperationsResponse ingestionResponse = new BulkOperationsResponse(); foreach (var item in response.Items) { if (!item.IsValid) { ingestionResponse.Errors++; continue; } if (item.Version == "1") { ingestionResponse.Added++; } else { ingestionResponse.Updated++; } } return(ingestionResponse); }