private ISearchIndexClient GetIndexForModel <T>()
{
var indexName = SearchableModelAttribute.GetIndexName(typeof(T));
if (_indexMapping.ContainsKey(indexName))
{
indexName = _indexMapping[indexName];
}
ISearchIndexClient indexClient = null;
if (_indexClients.ContainsKey(indexName))
{
indexClient = _indexClients[indexName];
}
else
{
indexClient = _searchClient.Indexes.GetClient(indexName);
_indexClients.Add(indexName, indexClient);
}
return(indexClient);
}
public bool Upload <T>(IEnumerable <T> records) where T : class
{
using (var operation = _telemetryClient.StartOperation <RequestTelemetry>("uploadSearchRecords"))
{
var indexName = SearchableModelAttribute.GetIndexName(typeof(T));
if (_indexMapping.ContainsKey(indexName))
{
indexName = _indexMapping[indexName];
}
ISearchIndexClient indexClient = null;
if (_indexClients.ContainsKey(indexName))
{
indexClient = _indexClients[indexName];
}
else
{
indexClient = _serviceClient.Indexes.GetClient(indexName);
_indexClients.Add(indexName, indexClient);
}
if (indexClient == null)
{
throw new Exception("Failed to get indexClient. Make sure index exists.");
}
// We cannot just throw all 200k+ records to azure search, that causes out of memory and http payload too large exceptions.
var recordsInOneStep = 200;
var totalRecords = records.Count();
var uploadedRecords = 0;
while (uploadedRecords < totalRecords)
{
var uploadSerializationStartTime = DateTime.UtcNow;
var recordsToUpload = records.Skip(uploadedRecords).Take(recordsInOneStep);
var actions = new List <IndexAction <Dictionary <string, object> > >();
foreach (var r in recordsToUpload)
{
Dictionary <string, object> recordSerialized = r.GetType().GetProperties()
.ToDictionary(x => x.Name, x => x.GetValue(r));
var indexAction = IndexAction.MergeOrUpload(recordSerialized);
actions.Add(indexAction);
}
var uploadSerializationTime = DateTime.UtcNow - uploadSerializationStartTime;
_telemetryClient.TrackMetric("azureSearchUploadChunkSerializationTime", uploadSerializationTime.TotalMilliseconds);
try
{
var uploadStartTime = DateTime.UtcNow;
indexClient.Documents.Index(IndexBatch.New(actions));
var uploadTime = DateTime.UtcNow - uploadStartTime;
_telemetryClient.TrackMetric("azureSearchChunkUploadTime", uploadTime.TotalMilliseconds);
}
catch (IndexBatchException e)
{
_telemetryClient.TrackException(e);
return(false);
}
actions.Clear();
uploadedRecords += recordsToUpload.Count();
//_logger.Info("Upload statistic:", new { UploadedRecords = uploadedRecords, TotalRecords = totalRecords, IndexName = indexName });
}
return(true);
}
}
public bool Delete <T>(IEnumerable <string> idList) where T : class
{
using (var operation = _telemetryClient.StartOperation <RequestTelemetry>("deleteSearchRecords"))
{
var indexName = SearchableModelAttribute.GetIndexName(typeof(T));
string keyName = SearchableModelAttribute.GetKeyPropertyName <T>();
if (_indexMapping.ContainsKey(indexName))
{
indexName = _indexMapping[indexName];
}
ISearchIndexClient indexClient = null;
if (_indexClients.ContainsKey(indexName))
{
indexClient = _indexClients[indexName];
}
else
{
indexClient = _serviceClient.Indexes.GetClient(indexName);
_indexClients.Add(indexName, indexClient);
}
if (indexClient == null)
{
throw new Exception("Failed to get indexClient. Make sure index exists.");
}
//Deleting 1000 records at a time using IndexBatch
var recordsInOneStep = 1000;
var totalRecords = idList.Count();
var deletedRecords = 0;
_telemetryClient.TrackMetric("azureSearchChunkDeleteItems", totalRecords);
while (deletedRecords < totalRecords)
{
var deleteStartTime = DateTime.UtcNow;
var recordsToDelete = idList.Skip(deletedRecords).Take(recordsInOneStep);
var indexBatchAction = IndexBatch.Delete(keyName, recordsToDelete);
try
{
indexClient.Documents.Index(indexBatchAction);
}
catch (IndexBatchException e)
{
_telemetryClient.TrackException(e);
return(false);
}
var deleteTime = DateTime.UtcNow - deleteStartTime;
_telemetryClient.TrackMetric("azureSearchChunkDeleteTime", deleteTime.TotalMilliseconds);
deletedRecords += recordsToDelete.Count();
}
return(true);
}
}
public async Task <bool> CreateIndex <T>() where T : class
{
try
{
var indexName = SearchableModelAttribute.GetIndexName(typeof(T));
if (string.IsNullOrEmpty(indexName))
{
throw new Exception("Model class should have SearchableModelAttribute with indexName specified.");
}
List <ScoringProfile> scoringProfiles = null;
string defaultScoringProfile = null;
string newIndexVersionSuffix = DateTime.Now.ToString("yyyyMMddHHmmss");
try
{
var existingIndexNames = await _serviceClient.Indexes.ListNamesAsync();
var foundIndexName = "";
long foundIndexVersion = 0;
var indexMatcher = new Regex(indexName + "-(?<timestamp>\\d{12})+");
// find last index
foreach (var existingIndexName in existingIndexNames)
{
var match = indexMatcher.Match(existingIndexName);
if (match.Success)
{
var timestampGroup = new List <Group>(match.Groups).FirstOrDefault(g => g.Name == "timestamp");
if (timestampGroup != null && timestampGroup.Success && timestampGroup.Value != null && timestampGroup.Value.Length > 0)
{
var version = long.Parse(timestampGroup.Value);
if (version > foundIndexVersion)
{
foundIndexName = existingIndexName;
foundIndexVersion = version;
}
}
}
}
if (string.IsNullOrEmpty(foundIndexName))
{
Console.WriteLine("Unable to find last index version for index: " + indexName + ". New index will be created.");
foundIndexName = indexName;
}
else
{
Console.WriteLine("Found last index version: " + foundIndexName);
}
var existingIndex = await _serviceClient.Indexes.GetAsync(foundIndexName);
scoringProfiles = (List <ScoringProfile>)existingIndex.ScoringProfiles;
Console.WriteLine("ScoringProfiles:");
Console.Write(JsonConvert.SerializeObject(scoringProfiles, Formatting.Indented));
string mydocpath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
var path = mydocpath + @"\AzureSearch_" + DateTime.Now.ToString("yyyyMMdd_HH_mm_ss") + "_scoringProfilesBackup.json";
using (StreamWriter outputFile = new StreamWriter(path))
{
outputFile.Write(JsonConvert.SerializeObject(scoringProfiles, Formatting.Indented));
}
defaultScoringProfile = existingIndex.DefaultScoringProfile;
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
List <Field> fields = new List <Field>();
List <Suggester> suggesters = new List <Suggester>();
PropertyInfo[] props = typeof(T).GetProperties();
foreach (PropertyInfo prop in props)
{
object[] attrs = prop.GetCustomAttributes(true);
foreach (object attr in attrs)
{
SearchablePropertyAttribute propertyAttribute = attr as SearchablePropertyAttribute;
if (propertyAttribute != null)
{
Type propertyType = prop.PropertyType;
if (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(Nullable <>))
{
propertyType = Nullable.GetUnderlyingType(propertyType);
}
var field = new Field();
field.Name = prop.Name;
switch (Type.GetTypeCode(propertyType))
{
case TypeCode.Int32:
field.Type = DataType.Int32;
break;
case TypeCode.Int64:
field.Type = DataType.Int64;
break;
case TypeCode.Double:
field.Type = DataType.Double;
break;
case TypeCode.Boolean:
field.Type = DataType.Boolean;
break;
case TypeCode.String:
field.Type = DataType.String;
if (propertyAttribute.IsSearchable && !propertyAttribute.UseForSuggestions &&
string.IsNullOrWhiteSpace(propertyAttribute.SearchAnalyzer) && string.IsNullOrWhiteSpace(propertyAttribute.IndexAnalyzer))
// Azure search doesn't support custom analyzer on fields enabled for suggestions
// If Search & IndexAnalyzers are specified, we cannot set Analyzer
{
field.Analyzer = "standardasciifolding.lucene";
}
break;
case TypeCode.Object:
var elementType = propertyType.GetElementType();
if (Type.GetTypeCode(elementType) != TypeCode.String)
{
throw new Exception("Unsupported array element type!");
}
field.Type = DataType.Collection(DataType.String);
if (propertyAttribute.IsSearchable && !propertyAttribute.UseForSuggestions) // Azure search doesn't support custom analyzer on fields enabled for suggestions
{
field.Analyzer = "standardasciifolding.lucene";
}
break;
case TypeCode.DateTime:
field.Type = DataType.DateTimeOffset;
break;
default:
throw new Exception($"Azure Search doesn't support {propertyType.Name} type.");
}
if (propertyAttribute.Analyzer != null && propertyAttribute.Analyzer != "")
{
field.Analyzer = propertyAttribute.Analyzer;
}
//SearchAnalyzer & IndexAnalyzer should be specified together
if (!string.IsNullOrWhiteSpace(propertyAttribute.SearchAnalyzer) && !string.IsNullOrWhiteSpace(propertyAttribute.IndexAnalyzer))
{
field.SearchAnalyzer = propertyAttribute.SearchAnalyzer;
field.IndexAnalyzer = propertyAttribute.IndexAnalyzer;
}
else if ((string.IsNullOrWhiteSpace(propertyAttribute.SearchAnalyzer) && !string.IsNullOrWhiteSpace(propertyAttribute.IndexAnalyzer)) ||
(!string.IsNullOrWhiteSpace(propertyAttribute.SearchAnalyzer) && string.IsNullOrWhiteSpace(propertyAttribute.IndexAnalyzer))
)
{
throw new Exception($"Both SearchAnalyzer & IndexAnalyzer are should be specified together.");
}
field.IsKey = propertyAttribute.IsKey;
field.IsFilterable = propertyAttribute.IsFilterable;
field.IsSortable = propertyAttribute.IsSortable;
field.IsSearchable = propertyAttribute.IsSearchable;
field.IsFacetable = propertyAttribute.IsFacetable;
if (propertyAttribute.SynonymMaps.Length > 0)
{
field.SynonymMaps = propertyAttribute.SynonymMaps;
}
fields.Add(field);
if (propertyAttribute.UseForSuggestions)
{
var suggester = new Suggester {
Name = field.Name, SourceFields = new[] { field.Name }
};
suggesters.Add(suggester);
}
}
}
}
string newIndexName = indexName + "-" + newIndexVersionSuffix;
var definition = new Index()
{
Name = newIndexName,
Fields = fields,
Suggesters = suggesters,
ScoringProfiles = scoringProfiles,
DefaultScoringProfile = defaultScoringProfile
};
await _serviceClient.Indexes.CreateOrUpdateAsync(definition);
Console.WriteLine($"Created index " + definition.Name);
//await _serviceClient.Indexes.DeleteAsync(indexName);
}
catch (Exception ex)
{
Console.WriteLine("Error creating index: {0}\r\n", ex.Message);
}
return(true);
}