public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IOpenSearchResultCollection results)
{
OpenSearchFactory.RemoveLinksByRel(ref results, "self");
OpenSearchFactory.RemoveLinksByRel(ref results, "search");
IElasticCollection docs = type.FromOpenSearchResultCollection(results);
BulkRequest bulkRequest = new BulkRequest() {
Refresh = true,
Consistency = Consistency.One,
Index = type.Index,
Type = type.Type,
Operations = new List<IBulkOperation>()
};
RootObjectMapping currentMapping = null;
try {
var mappingResponse = client.GetMapping<IElasticType>(g => g.Index(type.Index.Name).Type(type.Type.Name));
currentMapping = mappingResponse.Mapping;
} catch (Exception) {
}
var rootObjectMapping = type.GetRootMapping();
if (!rootObjectMapping.Equals(currentMapping)) {
client.Map<IElasticType>(m => m.Index(type.Index.Name).Type(type.Type.Name));
}
foreach (var doc in docs.ElasticItems) {
var bulkIndexOperation = new BulkIndexOperation<IElasticItem>(doc);
bulkIndexOperation.Id = ((IOpenSearchResultItem)doc).Identifier;
bulkIndexOperation.Type = type.Type.Name;
var bulkOp = bulkIndexOperation;
bulkRequest.Operations.Add(bulkOp);
}
var response = client.Bulk(bulkRequest);
BulkOperationsResponse ingestionResponse = new BulkOperationsResponse();
foreach (var item in response.Items) {
if (!item.IsValid) {
ingestionResponse.Errors++;
continue;
}
if (item.Version == "1")
ingestionResponse.Added++;
else
ingestionResponse.Updated++;
}
return ingestionResponse;
}
internal IndexInformation CreateCatalogueIndex(Terradue.ElasticCas.Request.CreateIndexRequest createRequest, bool destroy = false)
{
if (client.IndexExists(i => i.Index(createRequest.IndexName)).Exists)
{
if (destroy)
{
client.DeleteIndex(d => d.Index(createRequest.IndexName));
}
else
{
throw new InvalidOperationException(string.Format("'{0}' index already exists and cannot be overriden without data loss", createRequest.IndexName));
}
}
var response = client.CreateIndex(c => c.Index(createRequest.IndexName));
IndexInformation indexInformation = new IndexInformation();
var status = client.Status(s => s.Index(createRequest.IndexName));
indexInformation.Name = createRequest.IndexName;
indexInformation.Shards = status.Shards;
indexInformation.Mappings = new Dictionary <string, ICollection <PropertyNameMarker> >();
// Init mappings for each types declared
if (createRequest.TypeNames == null || createRequest.TypeNames.Length == 0)
{
List <string> typeNames = new List <string>();
foreach (TypeExtensionNode node in AddinManager.GetExtensionNodes(typeof(IOpenSearchableElasticType)))
{
IOpenSearchableElasticType type = (IOpenSearchableElasticType)node.CreateInstance();
if (type is GenericJsonOpenSearchable)
{
continue;
}
IndexNameMarker indexName = new IndexNameMarker();
indexName.Name = createRequest.IndexName;
PutMappingRequest putMappingRequest = new PutMappingRequest(indexName, type.Type);
((IPutMappingRequest)putMappingRequest).Mapping = type.GetRootMapping();
client.Map(putMappingRequest);
indexInformation.Mappings.Add(type.Identifier, ((IPutMappingRequest)putMappingRequest).Mapping.Properties.Keys);
typeNames.Add(type.Type.Name);
}
createRequest.TypeNames = typeNames.ToArray();
}
else
{
foreach (TypeExtensionNode node in AddinManager.GetExtensionNodes(typeof(IElasticItem)))
{
IOpenSearchableElasticType type = (IOpenSearchableElasticType)node.CreateInstance();
foreach (string typeName in createRequest.TypeNames)
{
if (typeName == type.Type.Name)
{
PutMappingRequest putMappingRequest = new PutMappingRequest(type.Index, type.Type);
((IPutMappingRequest)putMappingRequest).Mapping = type.GetRootMapping();
client.Map(putMappingRequest);
indexInformation.Mappings.Add(type.Identifier, ((IPutMappingRequest)putMappingRequest).Mapping.Properties.Keys);
}
}
}
}
return(indexInformation);
}
public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IOpenSearchResultCollection results)
{
OpenSearchFactory.RemoveLinksByRel(ref results, "self");
OpenSearchFactory.RemoveLinksByRel(ref results, "search");
IElasticCollection docs = type.FromOpenSearchResultCollection(results);
BulkRequest bulkRequest = new BulkRequest()
{
Refresh = true,
Consistency = Consistency.One,
Index = type.Index,
Type = type.Type,
Operations = new List <IBulkOperation>()
};
RootObjectMapping currentMapping = null;
try {
var mappingResponse = client.GetMapping <IElasticType>(g => g.Index(type.Index.Name).Type(type.Type.Name));
currentMapping = mappingResponse.Mapping;
} catch (Exception) {
}
var rootObjectMapping = type.GetRootMapping();
if (!rootObjectMapping.Equals(currentMapping))
{
client.Map <IElasticType>(m => m.Index(type.Index.Name).Type(type.Type.Name));
}
foreach (var doc in docs.ElasticItems)
{
var bulkIndexOperation = new BulkIndexOperation <IElasticItem>(doc);
bulkIndexOperation.Id = ((IOpenSearchResultItem)doc).Identifier;
bulkIndexOperation.Type = type.Type.Name;
var bulkOp = bulkIndexOperation;
bulkRequest.Operations.Add(bulkOp);
}
var response = client.Bulk(bulkRequest);
BulkOperationsResponse ingestionResponse = new BulkOperationsResponse();
foreach (var item in response.Items)
{
if (!item.IsValid)
{
ingestionResponse.Errors++;
continue;
}
if (item.Version == "1")
{
ingestionResponse.Added++;
}
else
{
ingestionResponse.Updated++;
}
}
return(ingestionResponse);
}