private ISearchIndexClient GetIndexForModel <T>()
{
var indexName = SearchableModelAttribute.GetIndexName(typeof(T));
if (_indexMapping.ContainsKey(indexName))
{
indexName = _indexMapping[indexName];
}
ISearchIndexClient indexClient = null;
if (_indexClients.ContainsKey(indexName))
{
indexClient = _indexClients[indexName];
}
else
{
indexClient = _searchClient.Indexes.GetClient(indexName);
_indexClients.Add(indexName, indexClient);
}
return(indexClient);
}
public bool Upload <T>(IEnumerable <T> records) where T : class
{
using (var operation = _telemetryClient.StartOperation <RequestTelemetry>("uploadSearchRecords"))
{
var indexName = SearchableModelAttribute.GetIndexName(typeof(T));
if (_indexMapping.ContainsKey(indexName))
{
indexName = _indexMapping[indexName];
}
ISearchIndexClient indexClient = null;
if (_indexClients.ContainsKey(indexName))
{
indexClient = _indexClients[indexName];
}
else
{
indexClient = _serviceClient.Indexes.GetClient(indexName);
_indexClients.Add(indexName, indexClient);
}
if (indexClient == null)
{
throw new Exception("Failed to get indexClient. Make sure index exists.");
}
// We cannot just throw all 200k+ records to azure search, that causes out of memory and http payload too large exceptions.
var recordsInOneStep = 200;
var totalRecords = records.Count();
var uploadedRecords = 0;
while (uploadedRecords < totalRecords)
{
var uploadSerializationStartTime = DateTime.UtcNow;
var recordsToUpload = records.Skip(uploadedRecords).Take(recordsInOneStep);
var actions = new List <IndexAction <Dictionary <string, object> > >();
foreach (var r in recordsToUpload)
{
Dictionary <string, object> recordSerialized = r.GetType().GetProperties()
.ToDictionary(x => x.Name, x => x.GetValue(r));
var indexAction = IndexAction.MergeOrUpload(recordSerialized);
actions.Add(indexAction);
}
var uploadSerializationTime = DateTime.UtcNow - uploadSerializationStartTime;
_telemetryClient.TrackMetric("azureSearchUploadChunkSerializationTime", uploadSerializationTime.TotalMilliseconds);
try
{
var uploadStartTime = DateTime.UtcNow;
indexClient.Documents.Index(IndexBatch.New(actions));
var uploadTime = DateTime.UtcNow - uploadStartTime;
_telemetryClient.TrackMetric("azureSearchChunkUploadTime", uploadTime.TotalMilliseconds);
}
catch (IndexBatchException e)
{
_telemetryClient.TrackException(e);
return(false);
}
actions.Clear();
uploadedRecords += recordsToUpload.Count();
//_logger.Info("Upload statistic:", new { UploadedRecords = uploadedRecords, TotalRecords = totalRecords, IndexName = indexName });
}
return(true);
}
}
public bool Delete <T>(IEnumerable <string> idList) where T : class
{
using (var operation = _telemetryClient.StartOperation <RequestTelemetry>("deleteSearchRecords"))
{
var indexName = SearchableModelAttribute.GetIndexName(typeof(T));
string keyName = SearchableModelAttribute.GetKeyPropertyName <T>();
if (_indexMapping.ContainsKey(indexName))
{
indexName = _indexMapping[indexName];
}
ISearchIndexClient indexClient = null;
if (_indexClients.ContainsKey(indexName))
{
indexClient = _indexClients[indexName];
}
else
{
indexClient = _serviceClient.Indexes.GetClient(indexName);
_indexClients.Add(indexName, indexClient);
}
if (indexClient == null)
{
throw new Exception("Failed to get indexClient. Make sure index exists.");
}
//Deleting 1000 records at a time using IndexBatch
var recordsInOneStep = 1000;
var totalRecords = idList.Count();
var deletedRecords = 0;
_telemetryClient.TrackMetric("azureSearchChunkDeleteItems", totalRecords);
while (deletedRecords < totalRecords)
{
var deleteStartTime = DateTime.UtcNow;
var recordsToDelete = idList.Skip(deletedRecords).Take(recordsInOneStep);
var indexBatchAction = IndexBatch.Delete(keyName, recordsToDelete);
try
{
indexClient.Documents.Index(indexBatchAction);
}
catch (IndexBatchException e)
{
_telemetryClient.TrackException(e);
return(false);
}
var deleteTime = DateTime.UtcNow - deleteStartTime;
_telemetryClient.TrackMetric("azureSearchChunkDeleteTime", deleteTime.TotalMilliseconds);
deletedRecords += recordsToDelete.Count();
}
return(true);
}
}
private string ConstructOneConditionFilter <T>(Filter filter)
{
var filterOperator = "";
var propertyName = GetPropertyName(filter.PropertyName);
if (string.IsNullOrEmpty(propertyName))
{
return(filterOperator);
}
switch (filter.Operator)
{
case FilterOperator.Equal:
filterOperator = "eq";
break;
case FilterOperator.Greater:
filterOperator = "gt";
break;
case FilterOperator.GreaterOrEqual:
filterOperator = "ge";
break;
case FilterOperator.Lower:
filterOperator = "lt";
break;
case FilterOperator.LowerOrEqual:
filterOperator = "le";
break;
case FilterOperator.NotEqual:
filterOperator = "ne";
break;
}
var filterValue = "";
switch (SearchableModelAttribute.GetPropertyTypeCode <T>(propertyName))
{
case TypeCode.Decimal:
case TypeCode.Double:
case TypeCode.Int16:
case TypeCode.Int32:
case TypeCode.Int64:
case TypeCode.Byte:
if (filter.Value == null)
{
filterValue += "null";
}
else
{
filterValue += $"{filter.Value}";
}
break;
case TypeCode.Char:
case TypeCode.String:
if (filter.Value == null)
{
filterValue += "''";
}
else
{
var v = filter.Value.ToString();
v = v.Replace("'", "''");
filterValue += $"'{v}'";
}
break;
case TypeCode.Boolean:
filterValue += $"{filter.Value.ToString().ToLower()}";
break;
case TypeCode.DateTime:
DateTime dt = (DateTime)filter.Value;
filterValue += $"{dt.ToString("yyyy-MM-ddTHH:mm:ss.fffZ")}";
break;
}
return($"{propertyName} {filterOperator} {filterValue}");
}
public async Task <SearchResult <ResultT> > Query <T, ResultT>(
SearchRequest searchRequest,
bool track = false
)
where T : class
where ResultT : class
{
var indexClient = GetIndexForModel <T>();
if (indexClient == null)
{
throw new Exception("Failed to get indexClient. Make sure index exists.");
}
var facetProperties = SearchableModelAttribute.GetFacetableProperties <T>();
List <string> facets = new List <string>();
if (searchRequest.FacetInfoToReturn != null && searchRequest.FacetInfoToReturn.Count > 0)
{
foreach (FacetInfoRequest f in searchRequest.FacetInfoToReturn)
{
var facetDefinition = GetPropertyName(f.FacetName);
if (f.Values != null && f.Values.Count() > 0)
{
facetDefinition += $",values:{string.Join("|", f.Values)}";
}
else
{
facetDefinition += $",count:{f.Count},sort:{f.Sort}";
}
facets.Add(facetDefinition);
}
}
Type resultType = typeof(ResultT);
MemberInfo[] props = resultType.GetMembers();
List <string> propertiesToSelect = props
.Where(p =>
(p.MemberType == MemberTypes.Field || p.MemberType == MemberTypes.Property) &&
p.GetCustomAttributes(typeof(IgnorePropertyAttribute), true).Length == 0
)
.Select(p => p.Name)
.ToList();
SearchParameters sp = new SearchParameters()
{
Select = propertiesToSelect,
SearchMode = searchRequest.SearchMode == "All" ? SearchMode.All : SearchMode.Any,
QueryType = QueryType.Full,
Top = searchRequest.Limit,
Skip = searchRequest.Offset,
// Add count
IncludeTotalResultCount = true,
// Add search highlights
HighlightFields = searchRequest.FieldsToHighlight.Select(f => GetPropertyName(f)).ToList(),
HighlightPreTag = HighlightTagStart,
HighlightPostTag = HighlightTagEnd,
// Add facets
Facets = facets,
};
if (!string.IsNullOrEmpty(searchRequest.ScoringProfile))
{
sp.ScoringProfile = searchRequest.ScoringProfile;
}
if (searchRequest.OrderBy.Count > 0)
{
var orderByFields = new List <string>();
foreach (var orderBy in searchRequest.OrderBy)
{
var orderByPropertyName = GetPropertyName(orderBy.Key);
orderByFields.Add($"{orderByPropertyName} {orderBy.Value}");
}
sp.OrderBy = orderByFields;
}
if (searchRequest.Filters != null && searchRequest.Filters.Count > 0)
{
List <string> filterStrings = new List <string>();
foreach (var f in searchRequest.Filters)
{
filterStrings.Add($"({ConstructConditionFilter<T>(f)})");
}
sp.Filter = string.Join(" and ", filterStrings);
}
string searchText = searchRequest.SearchText;
var headers = new Dictionary <string, List <string> >()
{
{ "x-ms-azs-return-searchid", new List <string>()
{
"true"
} }
};
var azureSearchResults = await indexClient.Documents.SearchWithHttpMessagesAsync(searchText, sp, customHeaders : headers);
IEnumerable <string> headerValues;
string searchId = string.Empty;
if (azureSearchResults.Response.Headers.TryGetValues("x-ms-azs-searchid", out headerValues))
{
searchId = headerValues.FirstOrDefault();
}
if (track)
{
var properties = new Dictionary <string, string> {
{ "SearchServiceName", _searchClient.SearchServiceName },
{ "SearchId", searchId },
{ "IndexName", indexClient.IndexName },
{ "QueryTerms", searchText },
{ "ResultCount", azureSearchResults.Body.Count.ToString() },
{ "ScoringProfile", sp.ScoringProfile }
};
_telemetryClient.TrackEvent("Search", properties);
}
SearchResult <ResultT> results = new SearchResult <ResultT>();
results.SearchId = searchId;
results.IndexName = indexClient.IndexName;
results.TotalRecordsFound = azureSearchResults.Body.Count;
if (azureSearchResults.Body.Facets != null)
{
foreach (var facetProp in facetProperties)
{
if (azureSearchResults.Body.Facets.ContainsKey(facetProp.Key.Name))
{
var facetValues = new List <FacetStats>();
foreach (var facetStat in azureSearchResults.Body.Facets[facetProp.Key.Name])
{
facetValues.Add(new FacetStats()
{
Value = facetStat.Value,
Count = facetStat.Count,
From = (double?)facetStat.From,
To = (double?)facetStat.To
});
}
results.FacetsStats.Add(facetProp.Key.Name, facetValues);
}
}
}
foreach (var record in azureSearchResults.Body.Results)
{
try
{
JObject obj = JObject.FromObject(record.Document);
var resultRecord = new SearchResultRecord <ResultT>();
resultRecord.Record = obj.ToObject <ResultT>();
resultRecord.Score = record.Score;
if (record.Highlights != null)
{
foreach (var highlight in record.Highlights)
{
var sourceFieldValue = obj.GetValue(highlight.Key).ToString();
foreach (var h in highlight.Value.ToList())
{
sourceFieldValue = sourceFieldValue.Replace(h.Replace(HighlightTagStart, String.Empty).Replace(HighlightTagEnd, String.Empty), h);
}
resultRecord.Highlights.Add(highlight.Key, new List <string>()
{
sourceFieldValue
});
}
}
results.Records.Add(resultRecord);
}
catch (Exception ex)
{
var a = ex;
}
}
return(results);
}
public async Task <bool> CreateIndex <T>() where T : class
{
try
{
var indexName = SearchableModelAttribute.GetIndexName(typeof(T));
if (string.IsNullOrEmpty(indexName))
{
throw new Exception("Model class should have SearchableModelAttribute with indexName specified.");
}
List <ScoringProfile> scoringProfiles = null;
string defaultScoringProfile = null;
string newIndexVersionSuffix = DateTime.Now.ToString("yyyyMMddHHmmss");
try
{
var existingIndexNames = await _serviceClient.Indexes.ListNamesAsync();
var foundIndexName = "";
long foundIndexVersion = 0;
var indexMatcher = new Regex(indexName + "-(?<timestamp>\\d{12})+");
// find last index
foreach (var existingIndexName in existingIndexNames)
{
var match = indexMatcher.Match(existingIndexName);
if (match.Success)
{
var timestampGroup = new List <Group>(match.Groups).FirstOrDefault(g => g.Name == "timestamp");
if (timestampGroup != null && timestampGroup.Success && timestampGroup.Value != null && timestampGroup.Value.Length > 0)
{
var version = long.Parse(timestampGroup.Value);
if (version > foundIndexVersion)
{
foundIndexName = existingIndexName;
foundIndexVersion = version;
}
}
}
}
if (string.IsNullOrEmpty(foundIndexName))
{
Console.WriteLine("Unable to find last index version for index: " + indexName + ". New index will be created.");
foundIndexName = indexName;
}
else
{
Console.WriteLine("Found last index version: " + foundIndexName);
}
var existingIndex = await _serviceClient.Indexes.GetAsync(foundIndexName);
scoringProfiles = (List <ScoringProfile>)existingIndex.ScoringProfiles;
Console.WriteLine("ScoringProfiles:");
Console.Write(JsonConvert.SerializeObject(scoringProfiles, Formatting.Indented));
string mydocpath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
var path = mydocpath + @"\AzureSearch_" + DateTime.Now.ToString("yyyyMMdd_HH_mm_ss") + "_scoringProfilesBackup.json";
using (StreamWriter outputFile = new StreamWriter(path))
{
outputFile.Write(JsonConvert.SerializeObject(scoringProfiles, Formatting.Indented));
}
defaultScoringProfile = existingIndex.DefaultScoringProfile;
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
List <Field> fields = new List <Field>();
List <Suggester> suggesters = new List <Suggester>();
PropertyInfo[] props = typeof(T).GetProperties();
foreach (PropertyInfo prop in props)
{
object[] attrs = prop.GetCustomAttributes(true);
foreach (object attr in attrs)
{
SearchablePropertyAttribute propertyAttribute = attr as SearchablePropertyAttribute;
if (propertyAttribute != null)
{
Type propertyType = prop.PropertyType;
if (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(Nullable <>))
{
propertyType = Nullable.GetUnderlyingType(propertyType);
}
var field = new Field();
field.Name = prop.Name;
switch (Type.GetTypeCode(propertyType))
{
case TypeCode.Int32:
field.Type = DataType.Int32;
break;
case TypeCode.Int64:
field.Type = DataType.Int64;
break;
case TypeCode.Double:
field.Type = DataType.Double;
break;
case TypeCode.Boolean:
field.Type = DataType.Boolean;
break;
case TypeCode.String:
field.Type = DataType.String;
if (propertyAttribute.IsSearchable && !propertyAttribute.UseForSuggestions &&
string.IsNullOrWhiteSpace(propertyAttribute.SearchAnalyzer) && string.IsNullOrWhiteSpace(propertyAttribute.IndexAnalyzer))
// Azure search doesn't support custom analyzer on fields enabled for suggestions
// If Search & IndexAnalyzers are specified, we cannot set Analyzer
{
field.Analyzer = "standardasciifolding.lucene";
}
break;
case TypeCode.Object:
var elementType = propertyType.GetElementType();
if (Type.GetTypeCode(elementType) != TypeCode.String)
{
throw new Exception("Unsupported array element type!");
}
field.Type = DataType.Collection(DataType.String);
if (propertyAttribute.IsSearchable && !propertyAttribute.UseForSuggestions) // Azure search doesn't support custom analyzer on fields enabled for suggestions
{
field.Analyzer = "standardasciifolding.lucene";
}
break;
case TypeCode.DateTime:
field.Type = DataType.DateTimeOffset;
break;
default:
throw new Exception($"Azure Search doesn't support {propertyType.Name} type.");
}
if (propertyAttribute.Analyzer != null && propertyAttribute.Analyzer != "")
{
field.Analyzer = propertyAttribute.Analyzer;
}
//SearchAnalyzer & IndexAnalyzer should be specified together
if (!string.IsNullOrWhiteSpace(propertyAttribute.SearchAnalyzer) && !string.IsNullOrWhiteSpace(propertyAttribute.IndexAnalyzer))
{
field.SearchAnalyzer = propertyAttribute.SearchAnalyzer;
field.IndexAnalyzer = propertyAttribute.IndexAnalyzer;
}
else if ((string.IsNullOrWhiteSpace(propertyAttribute.SearchAnalyzer) && !string.IsNullOrWhiteSpace(propertyAttribute.IndexAnalyzer)) ||
(!string.IsNullOrWhiteSpace(propertyAttribute.SearchAnalyzer) && string.IsNullOrWhiteSpace(propertyAttribute.IndexAnalyzer))
)
{
throw new Exception($"Both SearchAnalyzer & IndexAnalyzer are should be specified together.");
}
field.IsKey = propertyAttribute.IsKey;
field.IsFilterable = propertyAttribute.IsFilterable;
field.IsSortable = propertyAttribute.IsSortable;
field.IsSearchable = propertyAttribute.IsSearchable;
field.IsFacetable = propertyAttribute.IsFacetable;
if (propertyAttribute.SynonymMaps.Length > 0)
{
field.SynonymMaps = propertyAttribute.SynonymMaps;
}
fields.Add(field);
if (propertyAttribute.UseForSuggestions)
{
var suggester = new Suggester {
Name = field.Name, SourceFields = new[] { field.Name }
};
suggesters.Add(suggester);
}
}
}
}
string newIndexName = indexName + "-" + newIndexVersionSuffix;
var definition = new Index()
{
Name = newIndexName,
Fields = fields,
Suggesters = suggesters,
ScoringProfiles = scoringProfiles,
DefaultScoringProfile = defaultScoringProfile
};
await _serviceClient.Indexes.CreateOrUpdateAsync(definition);
Console.WriteLine($"Created index " + definition.Name);
//await _serviceClient.Indexes.DeleteAsync(indexName);
}
catch (Exception ex)
{
Console.WriteLine("Error creating index: {0}\r\n", ex.Message);
}
return(true);
}
