/// <summary>
/// Retrieving suggestions using the AWS CloudSearch Suggester. The AWS CloudSearch Suggester
/// returns results from the entire index and does not suport query filtering.
/// </summary>
/// <param name="parameters">Suggest request parameters</param>
/// <returns>SuggestResults</returns>
public SuggestResults RetieveSuggestions(NameValueCollection parameters)
{
SuggestResults suggestResults = new SuggestResults();
try
{
AmazonCloudSearchDomainClient client = GetCloudSearchClient();
SuggestRequest request = new SuggestRequest();
if (parameters["suggestername"] != null)
{
request.Suggester = parameters["suggestername"];
}
if (parameters["q"] != null)
{
request.Query = parameters["q"];
}
request.Size = parameters["size"] != null?Convert.ToInt32(parameters["size"]) : this.DefaultPageSize;
SuggestResponse results = client.Suggest(request);
suggestResults.Matches = results.Suggest.Suggestions.Select(c => c.Suggestion).ToList();
}
catch (Exception ex)
{
suggestResults.HasError = true;
suggestResults.ErrorDetail = ex.Message + " : " + ex.StackTrace;
}
return(suggestResults);
}
public UploadDocumentsResponse DeleteSingleConnectRecordInAwsCloudsearch(int participantId, int pinType)
{
var cloudSearch = new AmazonCloudSearchDomainClient(AwsAccessKeyId, AwsSecretAccessKey, AmazonSearchUrl);
var results = SearchConnectAwsCloudsearch($"(and participantid:{participantId} pintype:{pinType})", "_no_fields");
var deletelist = new List <AwsCloudsearchDto>();
foreach (var hit in results.Hits.Hit)
{
var deleterec = new AwsCloudsearchDto
{
id = hit.Id,
type = "delete"
};
deletelist.Add(deleterec);
}
// serialize
var json = JsonConvert.SerializeObject(deletelist, new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
});
var ms = new MemoryStream(Encoding.UTF8.GetBytes(json));
var upload = new UploadDocumentsRequest()
{
ContentType = ContentType.ApplicationJson,
Documents = ms
};
return(cloudSearch.UploadDocuments(upload));
}
/// <summary>
/// Run a query
/// </summary>
/// <param name="parameters">The query parameters</param>
/// <returns>matching results</returns>
public CloudSearchResults ExecuteQuery(NameValueCollection parameters)
{
CloudSearchResults result = new CloudSearchResults();
try
{
AmazonCloudSearchDomainClient client = GetCloudSearchClient();
SearchRequest request = BuildSearchRequest(parameters);
SearchResponse response = client.Search(request);
result.Items = response.Hits.Hit.Select(hit => CreateSearchResult(hit)).ToList();
result.Facets = (
from f in response.Facets
select new Facet
{
Name = f.Key,
Buckets = f.Value.Buckets.Select(b => new SI4T.Query.CloudSearch.Models.Bucket(b.Value, b.Count)).ToList()
}).ToList();
result.PageSize = Convert.ToInt32(request.Size);
result.Total = Convert.ToInt32(response.Hits.Found);
result.Start = Convert.ToInt32(request.Start);
result.QueryText = request.Query;
}
catch (Exception ex)
{
result.HasError = true;
result.ErrorDetail = ex.Message + " : " + ex.StackTrace;
}
return(result);
}
public void UploadNewPinToAws(PinDto pin)
{
var cloudSearch = new AmazonCloudSearchDomainClient(AwsAccessKeyId, AwsSecretAccessKey, AmazonSearchUrl);
var upload = GetObjectToUploadToAws(pin);
cloudSearch.UploadDocuments(upload);
}
public SearchResponse SearchConnectAwsCloudsearch(string querystring, string returnFields, int returnSize = 10000, GeoCoordinate originCoords = null, AwsBoundingBox boundingBox = null)
{
var cloudSearch = new AmazonCloudSearchDomainClient(AwsAccessKeyId, AwsSecretAccessKey, AmazonSearchUrl);
var searchRequest = new SearchRequest
{
Query = querystring,
QueryParser = QueryParser.Structured,
Size = returnSize,
Return = returnFields + ",_score"
};
if (boundingBox != null)
{
searchRequest.FilterQuery = $"latlong:['{boundingBox.UpperLeftCoordinates.Lat},{boundingBox.UpperLeftCoordinates.Lng}','{boundingBox.BottomRightCoordinates.Lat},{boundingBox.BottomRightCoordinates.Lng}']";
}
if (originCoords != null)
{
searchRequest.Expr = $"{{'distance':'haversin({originCoords.Latitude},{originCoords.Longitude},latlong.latitude,latlong.longitude)'}}"; // use to sort by proximity
searchRequest.Sort = "distance asc";
searchRequest.Return += ",distance";
}
var response = cloudSearch.Search(searchRequest);
return(response);
}
public void Init()
{
var apiKey = _configuration["AWS:apiKey"];
var apiSecret = _configuration["AWS:apiSecret"];
var searchUrl = _configuration["AWS:searchUrl"];
var docUrl = _configuration["AWS:docUrl"];
this.searchClient = this.searchClient ?? new AmazonCloudSearchDomainClient(apiKey, apiSecret, searchUrl);
this.documentClient = this.documentClient ?? new AmazonCloudSearchDomainClient(apiKey, apiSecret, docUrl);
}
public async Task <IEnumerable <FoundItem> > Search(string query, IImmutableList <string> markets, int maxResults)
{
Check.NotEmpty(query, nameof(query));
Check.NotNull(markets, nameof(markets));
Check.InRange(ref maxResults, nameof(maxResults), MinResults, MaxResults);
// parse and check query
var terms = ParseQuery(query);
if (terms.Length == 0)
{
throw new InvalidQueryException();
}
// build query
var request = new SearchRequest
{
Query = BuildQuery(terms),
FilterQuery = BuildFilterQuery(markets),
Size = maxResults,
QueryParser = QueryParser.Structured,
};
Logger.Info("Searching:");
Logger.Info($"Query: {request.Query}");
Logger.Info($"Filter: {request.FilterQuery}");
Logger.Info($"Size: {request.Size}");
try
{
using (var client = new AmazonCloudSearchDomainClient(_config.KeyId, _config.SecretKey, _config.ServiceUrl))
{
// execute search
var response = await client.SearchAsync(request);
Logger.Info($"Status: {response.HttpStatusCode}");
// build result
var result = new List <FoundItem>(response.Hits.Hit.Count);
foreach (var hit in response.Hits.Hit)
{
var documentItem = BuildDocumentItem(hit.Fields);
var foundItem = _mapper.Map <FoundItem>(documentItem);
(foundItem.Id, foundItem.ItemType) = DocumentId.Parse(hit.Id);
result.Add(foundItem);
}
return(result);
}
}
catch (Exception ex)
{
throw new SearchErrorException(ex);
}
}
protected IAmazonCloudSearchDomain CreateClient(string serviceUrl)
{
var config = new AmazonCloudSearchDomainConfig
{
ServiceURL = serviceUrl
};
Amazon.PowerShell.Utils.Common.PopulateConfig(this, config);
var client = new AmazonCloudSearchDomainClient(this._CurrentCredentials, config);
client.BeforeRequestEvent += RequestEventHandler;
client.AfterResponseEvent += ResponseEventHandler;
return(client);
}
public SearchResponse SearchByGroupId(string groupId)
{
var cloudSearch = new AmazonCloudSearchDomainClient(AwsAccessKeyId, AwsSecretAccessKey, AmazonSearchUrl);
var searchRequest = new SearchRequest
{
Query = "groupid: " + groupId,
QueryParser = QueryParser.Structured,
QueryOptions = "{'fields': ['groupid']}",
Size = 1,
Return = "_all_fields"
};
var response = cloudSearch.Search(searchRequest);
return(response);
}
public UploadDocumentsResponse UploadAllConnectRecordsToAwsCloudsearch()
{
var cloudSearch = new AmazonCloudSearchDomainClient(AwsAccessKeyId, AwsSecretAccessKey, AmazonSearchUrl);
var pinList = GetDataForCloudsearch();
//serialize
var json = JsonConvert.SerializeObject(pinList, new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
});
var ms = new MemoryStream(Encoding.UTF8.GetBytes(json));
var upload = new UploadDocumentsRequest()
{
ContentType = ContentType.ApplicationJson,
Documents = ms
};
return(cloudSearch.UploadDocuments(upload));
}
public IResultSet Search(ISearchQuery query)
{
var amazonSearchParameters = this.GetAmazonParams();
AmazonCloudSearchDomainConfig config = new AmazonCloudSearchDomainConfig();
config.ServiceURL = amazonSearchParameters[SearchEndPoint];
AmazonCloudSearchDomainClient domainClient = new AmazonCloudSearchDomainClient(amazonSearchParameters[AccessKey], amazonSearchParameters[SecretAccessKey], config);
List <string> suggestions = new List <string>();
StringBuilder highlights = new StringBuilder();
highlights.Append("{\'");
if (query == null)
{
throw new ArgumentNullException("query");
}
foreach (var field in query.HighlightedFields)
{
if (highlights.Length > 2)
{
highlights.Append(", \'");
}
highlights.Append(field.ToLowerInvariant());
highlights.Append("\':{} ");
SuggestRequest suggestRequest = new SuggestRequest();
suggestRequest.Suggester = this.GetSuggesterName(field);
suggestRequest.Size = query.Take;
suggestRequest.Query = query.Text;
SuggestResponse suggestion = domainClient.Suggest(suggestRequest);
foreach (var suggest in suggestion.Suggest.Suggestions)
{
suggestions.Add(suggest.Suggestion);
}
}
highlights.Append("}");
SearchRequest searchRequest = new SearchRequest();
if (query.Filter != null)
{
searchRequest.FilterQuery = this.BuildQueryFilter(query.Filter);
}
if (query.OrderBy != null)
{
searchRequest.Sort = string.Join(",", query.OrderBy);
}
if (query.Take > 0)
{
searchRequest.Size = query.Take;
}
if (query.Skip > 0)
{
searchRequest.Start = query.Skip;
}
searchRequest.Highlight = highlights.ToString();
searchRequest.Query = query.Text;
searchRequest.QueryParser = QueryParser.Simple;
var result = domainClient.Search(searchRequest).SearchResult;
return(new AmazonResultSet(result, suggestions));
}
/// <summary>
/// Retrieving suggestions using a custom implementation. This implementation assumes a 'literal'
/// field type to be used as source for suggestions. This field must have lowercase values.
/// Optionally an alternate display field can be provided which can be used populate the result.
/// This implementation returns similar results as the AWS CloudSearch Suggester when populate the
/// suggest field with the same value field as display field (but in lowercase).
/// </summary>
/// <param name="parameters">Suggest request parameters</param>
/// <param name="suggestFieldName">Name of the suggest field</param>
/// <param name="displayFieldName">Name of the display field</param>
/// <returns>SuggestResults</returns>
public SuggestResults RetieveFilteredSuggestions(NameValueCollection parameters, string suggestFieldName, string displayFieldName = null)
{
SuggestResults suggestResults = new SuggestResults();
try
{
AmazonCloudSearchDomainClient client = GetCloudSearchClient();
SearchRequest request = new SearchRequest();
if (parameters["q"] != null)
{
request.QueryParser = QueryParser.Structured;
request.Query = String.Format("(prefix field={0} '{1}')", suggestFieldName, parameters["q"].ToLower());
}
if (displayFieldName == null)
{
request.Facet = String.Format("{{'{0}':{{'sort':'bucket'}}}}", suggestFieldName);
request.Return = "_no_fields";
}
else
{
request.Return = displayFieldName;
request.Sort = String.Format("{0} asc", displayFieldName);
}
if (!String.IsNullOrEmpty(parameters["fq"]))
{
string filters = string.Empty;
foreach (string filterString in parameters["fq"].Split(','))
{
if (filterString.Contains(":"))
{
filters += (String.Format(" {0}:'{1}'", filterString.Split(':')[0], filterString.Split(':')[1]));
}
}
request.FilterQuery = String.Format("(and{0})", filters);
}
request.Size = parameters["size"] != null?Convert.ToInt32(parameters["size"]) : this.DefaultPageSize;
SearchResponse response = client.Search(request);
if (displayFieldName == null)
{
if (response.Facets.Count > 0)
{
suggestResults.Matches = response.Facets[suggestFieldName].Buckets.Select(b => b.Value).ToList();
}
}
else
{
if (response.Hits.Hit.Count > 0)
{
suggestResults.Matches = response.Hits.Hit.Select(h => h.Fields[displayFieldName].FirstOrDefault()).ToList();
}
}
}
catch (Exception ex)
{
suggestResults.HasError = true;
suggestResults.ErrorDetail = ex.Message + " : " + ex.StackTrace;
}
return(suggestResults);
}
public void Run()
{
var settings = new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
Formatting = Formatting.None,
};
using (var client = new AmazonCloudSearchDomainClient(_config.KeyId, _config.SecretKey, _config.ServiceUrl))
using (var batcher = new Batcher(_config.BatchSize, stream => UploadBatch(client, stream)))
{
foreach (var building in _reader.ReadBuildings(_config.BuildingsFilePath))
{
var addOperation = new AddOperation
{
Id = DocumentId.Build(building),
Fields = _mapper.Map <DocumentItem>(building),
};
var json = JsonConvert.SerializeObject(addOperation, settings);
batcher.Write(json);
}
foreach (var company in _reader.ReadCompanies(_config.CompaniesFilePath))
{
var addOperation = new AddOperation
{
Id = DocumentId.Build(company),
Fields = _mapper.Map <DocumentItem>(company),
};
var json = JsonConvert.SerializeObject(addOperation, settings);
batcher.Write(json);
}
}
void UploadBatch(IAmazonCloudSearchDomain client, Stream stream)
{
var request = new UploadDocumentsRequest();
request.ContentType = ContentType.ApplicationJson;
request.Documents = stream;
Logger.Info($"Uploading batch with {stream.Length} bytes");
var task = client.UploadDocumentsAsync(request);
if (!task.Wait(_config.Timeout))
{
Logger.Error("Operation timeout.");
return;
}
var response = task.Result;
Logger.Info($"Operation results: {response.Status}, " +
$"added {response.Adds} documents, " +
$"warnings: {response.Warnings.Count}");
foreach (var warning in response.Warnings)
{
Logger.Info($"Operation warning: {warning}");
}
}
}
