public BulkOperationsResponse Delete(IOpenSearchableElasticType type, IOpenSearchResultCollection results) { BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = type.Index, Type = type.Type, Operations = new List<IBulkOperation>() }; foreach (var doc in results.Items) { var bulkDeleteOperation = new BulkDeleteOperation<IElasticItem>(doc.Identifier); bulkDeleteOperation.Type = type.Type.Name; var bulkOp = bulkDeleteOperation; bulkRequest.Operations.Add(bulkOp); } var response = client.Bulk(bulkRequest); BulkOperationsResponse ingestionResponse = new BulkOperationsResponse(); foreach (var item in response.Items) { if (!item.IsValid) { ingestionResponse.Errors++; } else { ingestionResponse.Deleted++; } } return ingestionResponse; }
public object Get(OpenSearchDescriptionGetRequest request) { IOpenSearchableElasticType type = ecf.GetOpenSearchableElasticTypeByNameOrDefault(request.IndexName, request.TypeName); OpenSearchDescription osd = type.GetProxyOpenSearchDescription(); return(new HttpResult(osd, "application/opensearchdescription+xml")); }
public HttpResult Post(IngestionRequest request) { IOpenSearchableElasticType type = ecf.GetOpenSearchableElasticTypeByNameOrDefault(request.IndexName, request.TypeName); var response = Ingest(type, Request); return(new HttpResult(response, "application/json")); }
internal ElasticOpenSearchRequest(ElasticClientWrapper client, string indexName, string typeName, System.Collections.Specialized.NameValueCollection parameters, IOpenSearchableElasticType type) : base(BuildOpenSearchUrl(indexName, typeName, parameters), "application/json") { this.parameters = parameters; this.type = type; this.indexName = indexName; this.typeName = typeName; this.client = client; }
public object Get(TypeGetRequest request) { IOpenSearchableElasticType type = ecf.GetOpenSearchableElasticTypeByNameOrDefault(request.IndexName, request.TypeName); NameValueCollection parameters = new NameValueCollection(); parameters.Set("uid", request.Id); return(OpenSearchService.Query(type, parameters)); }
public static HttpResult Query(IOpenSearchableElasticType document, NameValueCollection parameters, Type resultType = null) { // special case for description if (HttpContext.Current.Request.AcceptTypes != null && HttpContext.Current.Request.AcceptTypes[0] == "application/opensearchdescription+xml" || parameters["format"] == "description") { return new HttpResult(document.GetProxyOpenSearchDescription(), "application/opensearchdescription+xml"); } var result = QueryResult(document, parameters, resultType); return new HttpResult(result.SerializeToString(), result.ContentType); }
public object Delete(TypeDeleteRequest request) { IOpenSearchableElasticType type = ecf.GetOpenSearchableElasticTypeByNameOrDefault(request.IndexName, request.TypeName); NameValueCollection parameters = new NameValueCollection(); parameters.Set("uid", request.Id); var results = OpenSearchService.QueryResult(type, parameters); var response = Delete(type, results); return(new HttpResult(response, "application/json")); }
public static HttpResult Query(IOpenSearchableElasticType document, NameValueCollection parameters, Type resultType = null) { // special case for description if (HttpContext.Current.Request.AcceptTypes != null && HttpContext.Current.Request.AcceptTypes[0] == "application/opensearchdescription+xml" || parameters["format"] == "description") { return(new HttpResult(document.GetProxyOpenSearchDescription(), "application/opensearchdescription+xml")); } var result = QueryResult(document, parameters, resultType); return(new HttpResult(result.SerializeToString(), result.ContentType)); }
public static OpenSearchDescription GetDefaultOpenSearchDescription(IOpenSearchableElasticType type) { OpenSearchDescription osd = new OpenSearchDescription(); osd.ShortName = "Terradue Catalogue"; osd.Attribution = "Terradue"; osd.Contact = "*****@*****.**"; osd.Developer = "Terradue GeoSpatial Development Team"; osd.SyndicationRight = "open"; osd.AdultContent = "false"; osd.Language = "en-us"; osd.OutputEncoding = "UTF-8"; osd.InputEncoding = "UTF-8"; osd.Description = string.Format("This Search Service performs queries in the index {0}. There are several URL templates that return the results in different formats." + "This search service is in accordance with the OGC 10-032r3 specification.", type.Index.Name); OpenSearchEngine ose = type.GetOpenSearchEngine(new NameValueCollection()); var searchExtensions = ose.Extensions; List <OpenSearchDescriptionUrl> urls = new List <OpenSearchDescriptionUrl>(); NameValueCollection parameters = type.GetOpenSearchParameters(type.DefaultMimeType); UriBuilder searchUrl = new UriBuilder(string.Format("{0}/catalogue/{1}/{2}/search", Settings.BaseUrl, type.Index.Name, type.Type.Name)); NameValueCollection queryString = HttpUtility.ParseQueryString("?format=format"); parameters.AllKeys.FirstOrDefault(k => { queryString.Add(k, parameters[k]); return(false); }); foreach (int code in searchExtensions.Keys) { queryString.Set("format", searchExtensions[code].Identifier); string[] queryStrings = Array.ConvertAll(queryString.AllKeys, key => string.Format("{0}={1}", key, queryString[key])); searchUrl.Query = string.Join("&", queryStrings); urls.Add(new OpenSearchDescriptionUrl(searchExtensions[code].DiscoveryContentType, searchUrl.ToString(), "results")); } searchUrl = new UriBuilder(string.Format("{0}/catalogue/{1}/{2}/description", Settings.BaseUrl, type.Index.Name, type.Type.Name)); urls.Add(new OpenSearchDescriptionUrl("application/opensearchdescription+xml", searchUrl.ToString(), "self")); osd.Url = urls.ToArray(); return(osd); }
public object Get(OpenSearchQueryRequest request) { IOpenSearchableElasticType type = ecf.GetOpenSearchableElasticTypeByNameOrDefault(request.IndexName, request.TypeName); NameValueCollection parameters = new NameValueCollection(Request.QueryString); if (request.AdditionalParameters != null) { foreach (var key in request.AdditionalParameters.AllKeys) { parameters.Set(key, request.AdditionalParameters[key]); } } return(OpenSearchService.Query(type, parameters)); }
public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IHttpRequest request) { OpenSearchEngine ose = type.GetOpenSearchEngine(new NameValueCollection()); IOpenSearchEngineExtension osee = ose.GetExtensionByContentTypeAbility(request.ContentType); if (osee == null) { throw new NotImplementedException(string.Format("No OpenSearch extension found for reading {0}", Request.ContentType)); } MemoryOpenSearchResponse payload = new MemoryOpenSearchResponse(request.GetRawBody(), request.ContentType); IOpenSearchResultCollection results = osee.ReadNative(payload); return(Ingest(type, results)); }
public IOpenSearchableElasticType GetOpenSearchableElasticTypeByNameOrDefault(string indexName, string typeName, Dictionary <string, object> parameters = null) { var indexNameMarker = new IndexNameMarker(); indexNameMarker.Name = indexName; var typeNameMarker = new TypeNameMarker(); typeNameMarker.Name = typeName; IOpenSearchableElasticType type = GetOpenSearchableElasticTypeByName(indexName, typeName); if (type == null) { type = new GenericJsonOpenSearchable(indexName, typeName); } return(type); }
public static IOpenSearchResultCollection QueryResult(IOpenSearchableElasticType type, NameValueCollection parameters, Type resultType = null) { OpenSearchEngine ose = type.GetOpenSearchEngine(parameters); if ( resultType == null ) resultType = OpenSearchFactory.ResolveTypeFromRequest(HttpContext.Current.Request, ose); if ( resultType == typeof(ParametersResult) ){ return type.DescribeParameters(); } var result = ose.Query(type, parameters, resultType ); OpenSearchFactory.ReplaceSelfLinks(type, parameters, result, type.EntrySelfLinkTemplate, result.ContentType); OpenSearchFactory.ReplaceOpenSearchDescriptionLinks(type, result); result.Title = new TextSyndicationContent(string.Format("Result for OpenSearch query over type {0} in index {1}", type.Type.Name, type.Index.Name)); return result; }
public object Post(TypeImportRequest request) { OpenSearchEngine ose = new OpenSearchEngine(); ose.LoadPlugins(); ose.DefaultTimeOut = 60000; OpenSearchUrl url = new OpenSearchUrl(request.url); IOpenSearchable entity = new GenericOpenSearchable(url, ose); IOpenSearchableElasticType type = ecf.GetOpenSearchableElasticTypeByNameOrDefault(request.IndexName, request.TypeName); IOpenSearchResultCollection osres = ose.Query(entity, new NameValueCollection()); OpenSearchFactory.RemoveLinksByRel(ref osres, "alternate"); OpenSearchFactory.RemoveLinksByRel(ref osres, "via"); OpenSearchFactory.RemoveLinksByRel(ref osres, "self"); OpenSearchFactory.RemoveLinksByRel(ref osres, "search"); IElasticCollection documents = type.FromOpenSearchResultCollection(osres); BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = request.IndexName }; foreach (var doc in documents.Items) { bulkRequest.Operations.Add(new BulkIndexOperation <IElasticItem>((IElasticItem)doc) { Id = doc.Id }); } var response = client.Bulk(bulkRequest); return(response); }
public static IOpenSearchResultCollection QueryResult(IOpenSearchableElasticType type, NameValueCollection parameters, Type resultType = null) { OpenSearchEngine ose = type.GetOpenSearchEngine(parameters); if (resultType == null) { resultType = OpenSearchFactory.ResolveTypeFromRequest(HttpContext.Current.Request, ose); } if (resultType == typeof(ParametersResult)) { return(type.DescribeParameters()); } var result = ose.Query(type, parameters, resultType); OpenSearchFactory.ReplaceSelfLinks(type, parameters, result, type.EntrySelfLinkTemplate, result.ContentType); OpenSearchFactory.ReplaceOpenSearchDescriptionLinks(type, result); result.Title = new TextSyndicationContent(string.Format("Result for OpenSearch query over type {0} in index {1}", type.Type.Name, type.Index.Name)); return(result); }
public BulkOperationsResponse Delete(IOpenSearchableElasticType type, IOpenSearchResultCollection results) { BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = type.Index, Type = type.Type, Operations = new List <IBulkOperation>() }; foreach (var doc in results.Items) { var bulkDeleteOperation = new BulkDeleteOperation <IElasticItem>(doc.Identifier); bulkDeleteOperation.Type = type.Type.Name; var bulkOp = bulkDeleteOperation; bulkRequest.Operations.Add(bulkOp); } var response = client.Bulk(bulkRequest); BulkOperationsResponse ingestionResponse = new BulkOperationsResponse(); foreach (var item in response.Items) { if (!item.IsValid) { ingestionResponse.Errors++; } else { ingestionResponse.Deleted++; } } return(ingestionResponse); }
public IOpenSearchableElasticType GetOpenSearchableElasticTypeByName(string indexName, string typeName, Dictionary <string, object> parameters = null) { foreach (TypeExtensionNode node in AddinManager.GetExtensionNodes(typeof(IOpenSearchableElasticType))) { IOpenSearchableElasticType etype = (IOpenSearchableElasticType)node.CreateInstance(); if (etype.Identifier == typeName) { Type type = node.Type; var ctor = type.GetConstructor(new Type[2] { typeof(IndexNameMarker), typeof(TypeNameMarker) }); var indexNameMarker = new IndexNameMarker(); indexNameMarker.Name = indexName; var typeNameMarker = new TypeNameMarker(); typeNameMarker.Name = typeName; etype = (IOpenSearchableElasticType)ctor.Invoke(new object[2] { indexNameMarker, typeNameMarker }); etype.Parameters = parameters; return(etype); } } return(null); }
internal IndexInformation CreateCatalogueIndex(Terradue.ElasticCas.Request.CreateIndexRequest createRequest, bool destroy = false) { if (client.IndexExists(i => i.Index(createRequest.IndexName)).Exists) { if (destroy) { client.DeleteIndex(d => d.Index(createRequest.IndexName)); } else { throw new InvalidOperationException(string.Format("'{0}' index already exists and cannot be overriden without data loss", createRequest.IndexName)); } } var response = client.CreateIndex(c => c.Index(createRequest.IndexName)); IndexInformation indexInformation = new IndexInformation(); var status = client.Status(s => s.Index(createRequest.IndexName)); indexInformation.Name = createRequest.IndexName; indexInformation.Shards = status.Shards; indexInformation.Mappings = new Dictionary <string, ICollection <PropertyNameMarker> >(); // Init mappings for each types declared if (createRequest.TypeNames == null || createRequest.TypeNames.Length == 0) { List <string> typeNames = new List <string>(); foreach (TypeExtensionNode node in AddinManager.GetExtensionNodes(typeof(IOpenSearchableElasticType))) { IOpenSearchableElasticType type = (IOpenSearchableElasticType)node.CreateInstance(); if (type is GenericJsonOpenSearchable) { continue; } IndexNameMarker indexName = new IndexNameMarker(); indexName.Name = createRequest.IndexName; PutMappingRequest putMappingRequest = new PutMappingRequest(indexName, type.Type); ((IPutMappingRequest)putMappingRequest).Mapping = type.GetRootMapping(); client.Map(putMappingRequest); indexInformation.Mappings.Add(type.Identifier, ((IPutMappingRequest)putMappingRequest).Mapping.Properties.Keys); typeNames.Add(type.Type.Name); } createRequest.TypeNames = typeNames.ToArray(); } else { foreach (TypeExtensionNode node in AddinManager.GetExtensionNodes(typeof(IElasticItem))) { IOpenSearchableElasticType type = (IOpenSearchableElasticType)node.CreateInstance(); foreach (string typeName in createRequest.TypeNames) { if (typeName == type.Type.Name) { PutMappingRequest putMappingRequest = new PutMappingRequest(type.Index, type.Type); ((IPutMappingRequest)putMappingRequest).Mapping = type.GetRootMapping(); client.Map(putMappingRequest); indexInformation.Mappings.Add(type.Identifier, ((IPutMappingRequest)putMappingRequest).Mapping.Properties.Keys); } } } } return(indexInformation); }
public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IOpenSearchResultCollection results) { OpenSearchFactory.RemoveLinksByRel(ref results, "self"); OpenSearchFactory.RemoveLinksByRel(ref results, "search"); IElasticCollection docs = type.FromOpenSearchResultCollection(results); BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = type.Index, Type = type.Type, Operations = new List <IBulkOperation>() }; RootObjectMapping currentMapping = null; try { var mappingResponse = client.GetMapping <IElasticType>(g => g.Index(type.Index.Name).Type(type.Type.Name)); currentMapping = mappingResponse.Mapping; } catch (Exception) { } var rootObjectMapping = type.GetRootMapping(); if (!rootObjectMapping.Equals(currentMapping)) { client.Map <IElasticType>(m => m.Index(type.Index.Name).Type(type.Type.Name)); } foreach (var doc in docs.ElasticItems) { var bulkIndexOperation = new BulkIndexOperation <IElasticItem>(doc); bulkIndexOperation.Id = ((IOpenSearchResultItem)doc).Identifier; bulkIndexOperation.Type = type.Type.Name; var bulkOp = bulkIndexOperation; bulkRequest.Operations.Add(bulkOp); } var response = client.Bulk(bulkRequest); BulkOperationsResponse ingestionResponse = new BulkOperationsResponse(); foreach (var item in response.Items) { if (!item.IsValid) { ingestionResponse.Errors++; continue; } if (item.Version == "1") { ingestionResponse.Added++; } else { ingestionResponse.Updated++; } } return(ingestionResponse); }
public object Get(DynamicRouteRequest request) { // Match the route request to validate the route requested Match match = Regex.Match(Request.GetPhysicalPath(), @"\/api\/(?<indexName>\w*)\/(?<routeId>\w*)\/(?<route>.*)"); DynamicOpenSearchRoute routeDefinition = null; // Load the dynamic route according to the index and the route id if (match.Success) { routeDefinition = LoadRoute(match.Groups["indexName"].Value, match.Groups["routeId"].Value); } else { throw new InvalidOperationException(string.Format("incorrect API route: {0}", Request.GetPhysicalPath())); } // Loads the type of the dynamic route IOpenSearchableElasticType type = ecf.GetOpenSearchableElasticTypeByName(match.Groups["indexName"].Value, routeDefinition.TypeName); if (type == null) { throw new InvalidTypeModelException(routeDefinition.TypeName, string.Format("Type '{0}' is not found in the type extensions. Check that plugins are loaded", routeDefinition.TypeName)); } // split the route defintion string[] defintionRouteSplitted = routeDefinition.RouteFromIndex.Split('/'); // check that requested route elements are respected string[] requestRouteSplitted = match.Groups["route"].Value.Split('/'); if (requestRouteSplitted.Count() != defintionRouteSplitted.Count()) { throw new InvalidOperationException(string.Format("Incorrect API request: {0}. Must match defintion: {1}", match.Groups["route"].Value, routeDefinition.RouteFromIndex)); } // build a parameters table with params defintion RouteParameter[] parameters = new RouteParameter[defintionRouteSplitted.Count()]; for (int i = 0; i < defintionRouteSplitted.Count(); i++) { string routeElement = defintionRouteSplitted[i]; if (!routeElement.StartsWith("{")) { parameters[i] = null; continue; } string parameter = routeElement.Trim(new char[] { '{', '}' }); if (string.IsNullOrEmpty(parameter)) { throw new InvalidOperationException(string.Format("API route defintion error: {0}. Empty parameter.", routeDefinition.RouteFromIndex)); } if (!routeDefinition.RouteParameters.ContainsKey(parameter)) { throw new InvalidOperationException(string.Format("API route defintion error: {0}. Undefined parameter: {1}", routeDefinition.RouteFromIndex, routeElement)); } parameters[i] = routeDefinition.RouteParameters[parameter]; } // Now from requested route rebuild the opensearch request UriBuilder url = new UriBuilder(ecf.RootWebConfig.AppSettings.Settings["baseUrl"].Value); url.Path += string.Format("/catalogue/{0}/{1}/search", match.Groups["indexName"].Value, routeDefinition.TypeName); NameValueCollection osParameters = HttpUtility.ParseQueryString(""); for (int i = 0; i < requestRouteSplitted.Count(); i++) { if (parameters[i] == null) { continue; } osParameters.Add(parameters[i].OpenSearchParameterId, requestRouteSplitted[i]); } List <string> mimeTypes = Request.AcceptTypes.ToList(); if (osParameters.AllKeys.Contains("enctype")) { mimeTypes.Add(osParameters["enctype"]); } OpenSearchDescription osd = type.GetProxyOpenSearchDescription(); OpenSearchDescriptionUrl osdUrl = OpenSearchFactory.GetOpenSearchUrlByTypeAndMaxParam(osd, mimeTypes, osParameters); osParameters = OpenSearchFactory.ReplaceTemplateByIdentifier(osParameters, osdUrl); osParameters.Add(Request.QueryString); return(OpenSearchService.Query(type, osParameters)); }
public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IOpenSearchResultCollection results) { OpenSearchFactory.RemoveLinksByRel(ref results, "self"); OpenSearchFactory.RemoveLinksByRel(ref results, "search"); IElasticCollection docs = type.FromOpenSearchResultCollection(results); BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = type.Index, Type = type.Type, Operations = new List<IBulkOperation>() }; RootObjectMapping currentMapping = null; try { var mappingResponse = client.GetMapping<IElasticType>(g => g.Index(type.Index.Name).Type(type.Type.Name)); currentMapping = mappingResponse.Mapping; } catch (Exception) { } var rootObjectMapping = type.GetRootMapping(); if (!rootObjectMapping.Equals(currentMapping)) { client.Map<IElasticType>(m => m.Index(type.Index.Name).Type(type.Type.Name)); } foreach (var doc in docs.ElasticItems) { var bulkIndexOperation = new BulkIndexOperation<IElasticItem>(doc); bulkIndexOperation.Id = ((IOpenSearchResultItem)doc).Identifier; bulkIndexOperation.Type = type.Type.Name; var bulkOp = bulkIndexOperation; bulkRequest.Operations.Add(bulkOp); } var response = client.Bulk(bulkRequest); BulkOperationsResponse ingestionResponse = new BulkOperationsResponse(); foreach (var item in response.Items) { if (!item.IsValid) { ingestionResponse.Errors++; continue; } if (item.Version == "1") ingestionResponse.Added++; else ingestionResponse.Updated++; } return ingestionResponse; }
public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IHttpRequest request) { OpenSearchEngine ose = type.GetOpenSearchEngine(new NameValueCollection()); IOpenSearchEngineExtension osee = ose.GetExtensionByContentTypeAbility(request.ContentType); if (osee == null) throw new NotImplementedException(string.Format("No OpenSearch extension found for reading {0}", Request.ContentType)); MemoryOpenSearchResponse payload = new MemoryOpenSearchResponse(request.GetRawBody(), request.ContentType); IOpenSearchResultCollection results = osee.ReadNative(payload); return Ingest(type, results); }
public static ElasticOpenSearchRequest <T> Create(System.Collections.Specialized.NameValueCollection parameters, IOpenSearchableElasticType type) { ElasticCasFactory ecf = new ElasticCasFactory("ElasticOpenSearchRequest"); ElasticOpenSearchRequest <T> eosRequest = new ElasticOpenSearchRequest <T>(ecf.Client, type.Index.Name, type.Type.Name, parameters, type); return(eosRequest); }
public static OpenSearchDescription GetDefaultOpenSearchDescription(IOpenSearchableElasticType type) { OpenSearchDescription osd = new OpenSearchDescription(); osd.ShortName = "Terradue Catalogue"; osd.Attribution = "Terradue"; osd.Contact = "*****@*****.**"; osd.Developer = "Terradue GeoSpatial Development Team"; osd.SyndicationRight = "open"; osd.AdultContent = "false"; osd.Language = "en-us"; osd.OutputEncoding = "UTF-8"; osd.InputEncoding = "UTF-8"; osd.Description = string.Format("This Search Service performs queries in the index {0}. There are several URL templates that return the results in different formats." + "This search service is in accordance with the OGC 10-032r3 specification.", type.Index.Name); OpenSearchEngine ose = type.GetOpenSearchEngine(new NameValueCollection()); var searchExtensions = ose.Extensions; List<OpenSearchDescriptionUrl> urls = new List<OpenSearchDescriptionUrl>(); NameValueCollection parameters = type.GetOpenSearchParameters(type.DefaultMimeType); UriBuilder searchUrl = new UriBuilder(string.Format("{0}/catalogue/{1}/{2}/search", Settings.BaseUrl, type.Index.Name, type.Type.Name)); NameValueCollection queryString = HttpUtility.ParseQueryString("?format=format"); parameters.AllKeys.FirstOrDefault(k => { queryString.Add(k, parameters[k]); return false; }); foreach (int code in searchExtensions.Keys) { queryString.Set("format", searchExtensions[code].Identifier); string[] queryStrings = Array.ConvertAll(queryString.AllKeys, key => string.Format("{0}={1}", key, queryString[key])); searchUrl.Query = string.Join("&", queryStrings); urls.Add(new OpenSearchDescriptionUrl(searchExtensions[code].DiscoveryContentType, searchUrl.ToString(), "results")); } searchUrl = new UriBuilder(string.Format("{0}/catalogue/{1}/{2}/description", Settings.BaseUrl, type.Index.Name, type.Type.Name)); urls.Add(new OpenSearchDescriptionUrl("application/opensearchdescription+xml", searchUrl.ToString(), "self")); osd.Url = urls.ToArray(); return osd; }