/// <inheritdoc/> public override IEnumerable <Clue> BuildClues(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, IExternalSearchRequest request) { var organizationCode = this.GetOriginEntityCode(result.As <PermIdSocialResponse>()); var organizationClue = new Clue(organizationCode, context.Organization); this.PopulateMetadata(organizationClue.Data.EntityData, result.As <PermIdSocialResponse>()); yield return(organizationClue); if (result.As <PermIdSocialResponse>().Data.AdditionalInfo != null) { foreach (var person in result.As <PermIdSocialResponse>().Data.AdditionalInfo) { var personCode = this.GetPersonEntityCode(person); var personClue = new Clue(personCode, context.Organization); this.PopulatePersonMetadata(personClue.Data.EntityData, person); var personToOrganizationEdge = new EntityEdge(new EntityReference(personCode), new EntityReference(organizationCode), EntityEdgeType.WorksFor); personClue.Data.EntityData.OutgoingEdges.Add(personToOrganizationEdge); yield return(personClue); } } }
/// <inheritdoc/> public override IEnumerable <Clue> BuildClues(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, IExternalSearchRequest request) { var resultItem = result.As <Result>(); string id; if (request.QueryParameters.ContainsKey(Core.Data.Vocabularies.Vocabularies.CluedInOrganization.CodesCVR)) { id = request.QueryParameters.GetValue(Core.Data.Vocabularies.Vocabularies.CluedInOrganization.CodesCVR).FirstOrDefault(); } else { id = resultItem.Data.ActualAddress.AccessAddressId; } var code = new EntityCode(EntityType.Organization, CodeOrigin.CluedIn, id); var clue = new Clue(code, context.Organization); clue.Data.OriginProviderDefinitionId = Id; PopulateMetadata(clue.Data.EntityData, resultItem, request); yield return(clue); }
/// <inheritdoc/> public override IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query) { var apiKey = this.TokenProvider.ApiToken; foreach (var externalSearchQueryResult in InternalExecuteSearch(query, apiKey)) { yield return(externalSearchQueryResult); } }
public void LogResult(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result) { using (var systemContext = context.ApplicationContext.System.CreateExecutionContext()) { var dataStore = systemContext.Organization.DataStores.GetDataStore <ExternalSearchClearBitLogRecord>(); var resultItem = result.As <CompanyAutocompleteResult>(); var record = this.CreateRecord(context, query, result, resultItem.Data); dataStore.InsertOrUpdate(systemContext, record); } }
/// <summary>Builds the clues.</summary> /// <param name="context">The context.</param> /// <param name="query">The query.</param> /// <param name="result">The result.</param> /// <param name="request">The request.</param> /// <returns>The clues.</returns> public override IEnumerable <Clue> BuildClues(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, IExternalSearchRequest request) { if (result is IExternalSearchQueryResult <LibpostalResponse> libpostalResult) { var code = GetOriginEntityCode(libpostalResult, request); var clue = new Clue(code, context.Organization); clue.Data.EntityData.Codes.Add(request.EntityMetaData.Codes.First()); PopulateMetadata(clue.Data.EntityData, libpostalResult, request); return(new[] { clue }); } return(null); }
/// <inheritdoc/> public override IEnumerable <Clue> BuildClues(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, IExternalSearchRequest request) { var resultItem = result.As <GleifResponse>(); var code = this.GetOriginEntityCode(resultItem.Data.Data.First()?.Attributes.Lei); var clue = new Clue(code, context.Organization); this.PopulateMetadata(clue.Data.EntityData, resultItem, request); return(new[] { clue }); }
/// <summary>Builds the clues.</summary> /// <param name="context">The context.</param> /// <param name="query">The query.</param> /// <param name="result">The result.</param> /// <param name="request">The request.</param> /// <returns>The clues.</returns> public override IEnumerable <Clue> BuildClues(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, IExternalSearchRequest request) { var resultItem = result.As <CompanyNew>(); var code = new EntityCode(EntityType.Organization, GetCodeOrigin(), resultItem.Data.company_number); var clue = new Clue(code, context.Organization); clue.Data.OriginProviderDefinitionId = Id; PopulateMetadata(clue.Data.EntityData, resultItem.Data); yield return(clue); }
public override IEnumerable <Clue> BuildClues(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, IExternalSearchRequest request) { var resultItem = result.As <User>(); var code = GetOriginEntityCode(resultItem); var clue = new Clue(code, context.Organization); PopulateMetadata(clue.Data.EntityData, resultItem); // If necessary, you can create multiple clues and return them. return(new[] { clue }); }
/// <summary>Builds the clues.</summary> /// <param name="context">The context.</param> /// <param name="query">The query.</param> /// <param name="result">The result.</param> /// <param name="request">The request.</param> /// <returns>The clues.</returns> public override IEnumerable <Clue> BuildClues(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, IExternalSearchRequest request) { var resultItem = result.As <Person>(); var code = this.GetOriginEntityCode(resultItem); var clue = new Clue(code, context.Organization); clue.Data.OriginProviderDefinitionId = this.Id; this.PopulateMetadata(clue.Data.EntityData, resultItem); this.DownloadPreviewImage(context, resultItem.Data.avatar, clue); return(new[] { clue }); }
public void LogResult(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result) { using (var systemContext = context.ApplicationContext.System.CreateExecutionContext()) { var dataStore = systemContext.Organization.DataStores.GetDataStore <ExternalSearchGoogleKnowledgeGraphLogRecord>(); var resultItem = result.As <Result>(); if (resultItem.Data.id == null && resultItem.Data.name == null && resultItem.Data.description == null) { return; } var record = this.CreateRecord(context, query, result, resultItem.Data); dataStore.InsertOrUpdate(systemContext, record); } }
/// <summary>Builds the clues.</summary> /// <param name="context">The context.</param> /// <param name="query">The query.</param> /// <param name="result">The result.</param> /// <param name="request">The request.</param> /// <returns>The clues.</returns> public override IEnumerable <Clue> BuildClues(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, IExternalSearchRequest request) { if (context == null) { throw new ArgumentNullException(nameof(context)); } if (query == null) { throw new ArgumentNullException(nameof(query)); } if (result == null) { throw new ArgumentNullException(nameof(result)); } if (request == null) { throw new ArgumentNullException(nameof(request)); } using (context.Log.BeginScope("{0} {1}: query {2}, request {3}, result {4}", GetType().Name, "BuildClues", query, request, result)) { var resultItem = result.As <VatLayerResponse>(); var dirtyClue = request.CustomQueryInput.ToString(); var code = GetOriginEntityCode(resultItem); var clue = new Clue(code, context.Organization); if (!string.IsNullOrEmpty(dirtyClue)) { clue.Data.EntityData.Codes.Add(new EntityCode(EntityType.Organization, CodeOrigin.CluedIn.CreateSpecific("vatlayer"), dirtyClue)); } PopulateMetadata(clue.Data.EntityData, resultItem); context.Log.LogInformation("Clue produced, Id: '{Id}' OriginEntityCode: '{OriginEntityCode}' RawText: '{RawText}'", clue.Id, clue.OriginEntityCode, clue.RawText); return(new[] { clue }); } }
public override IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query) { var id = query.QueryParameters["id"].FirstOrDefault(); if (string.IsNullOrEmpty(id)) { yield break; } var user = _client.GetUser(id).Result; if (user != null) { yield return(new ExternalSearchQueryResult <User>(query, user)); } }
/// <summary>Executes the search.</summary> /// <param name="context">The context.</param> /// <param name="query">The query.</param> /// <returns>The results.</returns> public override IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query) { var name = query.QueryParameters.ContainsKey(ExternalSearchQueryParameter.Name) ? query.QueryParameters[ExternalSearchQueryParameter.Name].FirstOrDefault() : null; if (string.IsNullOrEmpty(name)) { yield break; } var client = new CompanyHouseClient(); var companies = client.GetCompanies(name); if (companies == null) { yield break; } foreach (var companyResult in companies) { var company = new CompanyNew(); company = client.GetCompany(companyResult.company_number); yield return(new ExternalSearchQueryResult <CompanyNew>(query, company)); } }
/// <inheritdoc/> public override IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query) { var address = query.QueryParameters[ExternalSearchQueryParameter.Name].FirstOrDefault(); var client = new RestClient("https://dawa.aws.dk/datavask/"); var request = new RestRequest("adresser", Method.GET); request.AddParameter("betegnelse", address); var response = client.Execute(request); var data = JsonConvert.DeserializeObject <AddressResponse>(response.Content); if (response.StatusCode == HttpStatusCode.OK) { var allowedCategories = new string[] { "A", "B" }; if (data != null && allowedCategories.Contains(data.Category)) { foreach (var result in data.Results) { yield return(new ExternalSearchQueryResult <Result>(query, result)); } } } else if (response.StatusCode == HttpStatusCode.NoContent || response.StatusCode == HttpStatusCode.NotFound) { yield break; } else if (response.ErrorException != null) { throw new AggregateException(response.ErrorException.Message, response.ErrorException); } else { throw new ApplicationException("Could not execute external search query - StatusCode:" + response.StatusCode); } }
/// <summary>Executes the search.</summary> /// <param name="context">The context.</param> /// <param name="query">The query.</param> /// <returns>The results.</returns> public override IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query) { var name = query.QueryParameters.GetValue <string, HashSet <string> >(ExternalSearchQueryParameter.Name.ToString(), new HashSet <string>()).FirstOrDefault(); var domain = query.QueryParameters.GetValue <string, HashSet <string> >(ExternalSearchQueryParameter.Domain.ToString(), new HashSet <string>()).FirstOrDefault(); if (string.IsNullOrEmpty(name) && string.IsNullOrEmpty(domain)) { yield break; } var client = new RestClient("https://autocomplete.clearbit.com"); var request = new RestRequest(string.Format("/v1/companies/suggest?query={0}", name ?? domain), Method.GET); var response = client.ExecuteTaskAsync <List <CompanyAutocompleteResult> >(request).Result; if (response.StatusCode == HttpStatusCode.OK) { foreach (var result in response.Data) { yield return(new ExternalSearchQueryResult <CompanyAutocompleteResult>(query, result)); } } else if (response.StatusCode == HttpStatusCode.NoContent || response.StatusCode == HttpStatusCode.NotFound) { yield break; } else if (response.ErrorException != null) { throw new AggregateException(response.ErrorException.Message, response.ErrorException); } else { throw new ApplicationException("Could not execute external search query - StatusCode:" + response.StatusCode); } }
private ExternalSearchGoogleKnowledgeGraphLogRecord CreateRecord(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, Result resultItem) { var record = new ExternalSearchGoogleKnowledgeGraphLogRecord { Id = ExternalSearchLogIdGenerator.GenerateId(query.ProviderId, query.EntityType, resultItem.id ?? string.Empty, resultItem.name ?? string.Empty, resultItem.description ?? string.Empty), ProviderId = query.ProviderId, EntityType = query.EntityType, ResultId = resultItem.id, Name = resultItem.name, ResultTypes = resultItem.type != null?string.Join(", ", resultItem.type) : null, Description = resultItem.description, Url = resultItem.url }; if (resultItem.detailedDescription != null) { record.DetailedDescriptionBody = resultItem.detailedDescription.articleBody; record.DetailedDescriptionUrl = resultItem.detailedDescription.url; record.DetailedDescriptionLicenseUrl = resultItem.detailedDescription.license; } return(record); }
public IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query, IDictionary <string, object> config, IProvider provider) { var jobData = new PermIdExternalSearchJobData(config); foreach (var externalSearchQueryResult in InternalExecuteSearch(query, jobData.ApiToken)) { yield return(externalSearchQueryResult); } }
/// <inheritdoc/> public override IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query) { var leiCode = query.QueryParameters[ExternalSearchQueryParameter.Identifier].FirstOrDefault(); if (string.IsNullOrEmpty(leiCode)) { yield break; } var client = new RestClient("https://api.gleif.org/api/v1/lei-records"); var request = new RestRequest("?page[size]=1&page[number]=1&filter[lei]=" + leiCode, Method.GET); var response = client.ExecuteAsync(request).Result; if (response.StatusCode == HttpStatusCode.OK) { // HACK: Removes the outer array from json string var responseData = response.Content; //.Substring(1, response.Content.Length - 2); var data = JsonConvert.DeserializeObject <GleifResponse>(responseData); if (data?.Data != null && data.Data.Any()) { yield return(new ExternalSearchQueryResult <GleifResponse>(query, data)); } } else if (response.StatusCode == HttpStatusCode.NoContent || response.StatusCode == HttpStatusCode.NotFound) { yield break; } else if (response.ErrorException != null) { throw new AggregateException(response.ErrorException.Message, response.ErrorException); } else { throw new ApplicationException("Could not execute external search query - StatusCode:" + response.StatusCode + "; Content: " + response.Content); } }
private static IEnumerable <IExternalSearchQueryResult> InternalExecuteSearch(IExternalSearchQuery query, string apiKey) { var name = query.QueryParameters[ExternalSearchQueryParameter.Name].FirstOrDefault(); var idList = new List <string>(); var apiToken = apiKey; if (string.IsNullOrEmpty(apiToken)) { throw new InvalidOperationException("PermId ApiToken has not been configured"); } if (string.IsNullOrEmpty(name)) { yield break; } var searchClient = new RestClient(" https://api-eit.refinitiv.com/permid"); var socialClient = new RestClient("https://permid.org/api/mdaas/getEntityById/"); if (!string.IsNullOrEmpty(name)) { var searchResult = RequestWrapper <PermIdSearchResponse>(searchClient, "search?q=" + name, apiKey); foreach (var res in searchResult.Result.Organizations.Entities) { idList.Add(res.Id.Split('-').Last()); } } foreach (var permId in idList) { var socialResult = RequestWrapper <PermIdSocialResponse>(socialClient, permId, apiKey); if (socialResult != null) { yield return(new ExternalSearchQueryResult <PermIdSocialResponse>(query, socialResult)); } } }
/// <summary>Executes the search.</summary> /// <param name="context">The context.</param> /// <param name="query">The query.</param> /// <returns>The results.</returns> public override IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query) { var url = ConfigurationManagerEx.AppSettings.GetValue("ExternalSearch.Libpostal.url", ""); if (url.IsNullOrEmpty()) { throw new Exception("Bad configuration"); } var client = new RestClient(url); var request = new RestRequest("parser", Method.POST); string address = null; request.AddHeader("Content-type", "application/json"); if (query.QueryParameters.ContainsKey("body")) { address = query.QueryParameters["body"].First(); } else { throw new Exception("Wrong query"); } request.AddJsonBody(new queryBody() { query = address }); var response = client.ExecuteTaskAsync <LibpostalResponse>(request).Result; if (response.StatusCode == HttpStatusCode.OK) { if (response.Content != null) { var data = new LibpostalResponse(); foreach (var item in JsonConvert.DeserializeObject <List <Items> >(response.Content)) { data.Items.Add(item); } yield return(new ExternalSearchQueryResult <LibpostalResponse>(query, data)); } } else if (response.StatusCode == HttpStatusCode.NoContent || response.StatusCode == HttpStatusCode.NotFound) { yield break; } else if (response.ErrorException != null) { throw new AggregateException(response.ErrorException.Message, response.ErrorException); } else { throw new ApplicationException("Could not execute external search query - StatusCode:" + response.StatusCode + "; Content: " + response.Content); } }
private IEnumerable <IExternalSearchQueryResult> InternalExecuteSearch(ExecutionContext context, IExternalSearchQuery query, string apiToken) { if (string.IsNullOrEmpty(apiToken)) { throw new InvalidOperationException("ApiToken for VatLayer must be provided."); } if (context == null) { throw new ArgumentNullException(nameof(context)); } if (query == null) { throw new ArgumentNullException(nameof(query)); } using (context.Log.BeginScope("{0} {1}: query {2}", GetType().Name, "ExecuteSearch", query)) { context.Log.LogTrace("Starting external search for Id: '{Id}' QueryKey: '{QueryKey}'", query.Id, query.QueryKey); var vat = query.QueryParameters[ExternalSearchQueryParameter.Identifier].FirstOrDefault(); if (string.IsNullOrEmpty(vat)) { context.Log.LogTrace("No parameter for '{Identifier}' in query, skipping execute search", ExternalSearchQueryParameter.Identifier); } else { // TODO missing try { } catch { } block ... vat = WebUtility.UrlEncode(vat); var client = new RestClient("http://www.apilayer.net/api"); var request = new RestRequest($"validate?access_key={apiToken}&vat_number={vat}&format=1", Method.GET); var response = client.ExecuteAsync <VatLayerResponse>(request).Result; if (response.StatusCode == HttpStatusCode.OK) { if (response.Data != null && response.Data.Valid) { var diagnostic = $"External search for Id: '{query.Id}' QueryKey: '{query.QueryKey}' produced results, CompanyName: '{response.Data.CompanyName}' VatNumber: '{response.Data.VatNumber}'"; context.Log.LogInformation(diagnostic); yield return(new ExternalSearchQueryResult <VatLayerResponse>(query, response.Data)); } else { var diagnostic = $"Failed external search for Id: '{query.Id}' QueryKey: '{query.QueryKey}' - StatusCode: '{response.StatusCode}' Content: '{response.Content}'"; context.Log.LogError(diagnostic); var content = JsonConvert.DeserializeObject <dynamic>(response.Content); if (content.error != null) { throw new InvalidOperationException( $"{content.error.info} - Type: {content.error.type} Code: {content.error.code}"); } // TODO else do what with content ? ... } } else if (response.StatusCode == HttpStatusCode.NoContent || response.StatusCode == HttpStatusCode.NotFound) { var diagnostic = $"External search for Id: '{query.Id}' QueryKey: '{query.QueryKey}' produced no results - StatusCode: '{response.StatusCode}' Content: '{response.Content}'"; context.Log.LogWarning(diagnostic); yield break; } else if (response.ErrorException != null) { var diagnostic = $"External search for Id: '{query.Id}' QueryKey: '{query.QueryKey}' produced no results - StatusCode: '{response.StatusCode}' Content: '{response.Content}'"; context.Log.LogError(diagnostic, response.ErrorException); throw new AggregateException(response.ErrorException.Message, response.ErrorException); } else { var diagnostic = $"Failed external search for Id: '{query.Id}' QueryKey: '{query.QueryKey}' - StatusCode: '{response.StatusCode}' Content: '{response.Content}'"; context.Log.LogError(diagnostic); throw new ApplicationException(diagnostic); } context.Log.LogTrace("Finished external search for Id: '{Id}' QueryKey: '{QueryKey}'", query.Id, query.QueryKey); } } }
public IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query, IDictionary <string, object> config, IProvider provider) { return(ExecuteSearch(context, query)); }
/// <summary>Executes the search.</summary> /// <param name="context">The context.</param> /// <param name="query">The query.</param> /// <returns>The results.</returns> public override IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query) { var identifier = query.QueryParameters[ExternalSearchQueryParameter.Identifier].FirstOrDefault(); if (string.IsNullOrEmpty(identifier)) { yield break; } var client = new RestClient("https://api.openfigi.com"); //TODO: Request var mappingEndpoint = "/v2/mapping"; var request = new RestRequest(mappingEndpoint, Method.POST); request.AddHeader("Content-Type", "application/json"); request.AddHeader("X-OPENFIGI-APIKEY", TokenProvider.ApiToken); request.AddParameter("idType", "TICKER"); request.AddParameter("idValue", identifier); var response = client.ExecuteTaskAsync <MappingResponse>(request).Result; if (response.StatusCode == HttpStatusCode.OK) { if (response.Data != null) { yield return(new ExternalSearchQueryResult <MappingResponse>(query, response.Data)); } } else if (response.StatusCode == HttpStatusCode.NoContent || response.StatusCode == HttpStatusCode.NotFound) { yield break; } else if (response.ErrorException != null) { throw new AggregateException(response.ErrorException.Message, response.ErrorException); } else { throw new ApplicationException("Could not execute external search query - StatusCode:" + response.StatusCode + "; Content: " + response.Content); } }
/// <summary>Executes the search.</summary> /// <param name="context">The context.</param> /// <param name="query">The query.</param> /// <returns>The results.</returns> public override IEnumerable <IExternalSearchQueryResult> ExecuteSearch(ExecutionContext context, IExternalSearchQuery query) { var name = query.QueryParameters[ExternalSearchQueryParameter.Name].FirstOrDefault(); if (string.IsNullOrEmpty(name)) { yield break; } var sharedApiToken = ConfigurationManagerEx.AppSettings["Providers.ExternalSearch.ClearBit.ApiToken"]; var client = new RestClient("https://prospector.clearbit.com"); var request = new RestRequest(string.Format("/v1/people/search"), Method.GET); request.AddParameter("domain", name); request.AddParameter("limit", 20); request.AddHeader("Authorization", "Bearer " + sharedApiToken); var response = client.ExecuteTaskAsync <ClearbitResponse>(request).Result; if (response.StatusCode == HttpStatusCode.OK) { yield return(new ExternalSearchQueryResult <Person>(query, response.Data.person)); } else if (response.StatusCode == HttpStatusCode.NoContent || response.StatusCode == HttpStatusCode.NotFound) { yield break; } else if (response.ErrorException != null) { throw new AggregateException(response.ErrorException.Message, response.ErrorException); } else { throw new ApplicationException("Could not execute external search query - StatusCode:" + response.StatusCode); } }
public IEnumerable <Clue> BuildClues(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, IExternalSearchRequest request, IDictionary <string, object> config, IProvider provider) { return(BuildClues(context, query, result, request)); }
private ExternalSearchClearBitLogRecord CreateRecord(ExecutionContext context, IExternalSearchQuery query, IExternalSearchQueryResult result, CompanyAutocompleteResult resultItem) { var record = new ExternalSearchClearBitLogRecord(); record.Id = ExternalSearchLogIdGenerator.GenerateId(query.ProviderId, query.EntityType, resultItem.Name, resultItem.Domain); record.ProviderId = query.ProviderId; record.EntityType = query.EntityType; record.Name = resultItem.Name; record.Domain = resultItem.Domain; record.Logo = resultItem.Logo; return(record); }