private static async Task AddBatchToIndex(Listing[] listings)
{
//guard against more than 1000
if (listings.Length < 1000)
{
var managementClient = GetIndexManagementClient();
List <IndexOperation> operations = new List <IndexOperation>();
//rip through collection and make a batch of operations
foreach (Listing l in listings)
{
var flatOptions = string.Concat((IEnumerable <string>)l.Options.Select(o => o.Name));
var op = new IndexOperation(IndexOperationType.Upload, "Id", l.Id.ToString())
.WithProperty("Color", l.Color)
.WithProperty("Options", flatOptions)
.WithProperty("Package", l.Package)
.WithProperty("Type", l.Type)
.WithProperty("Image", l.Image);
operations.Add(op);
}
//TODO: should be able to batch up the operations
var result = await managementClient.PopulateAsync(Keys.ListingsServiceIndexName, operations.ToArray());
if (!result.IsSuccess)
{
Console.WriteLine("Adding records to the index failed!");
}
}
}
/// <summary>
/// Creates an insert operation for the supplied user
/// </summary>
/// <param name="document">the user to create an operation for</param>
/// <returns>insert operation</returns>
private IndexOperation CreateDocumentInsertOperation(Document document)
{
this.Log.LogInformation("creating user insert");
UserDocument user = null;
if (document is UserDocument)
{
user = (UserDocument)document;
}
else
{
this.Log.LogException("got document that is not a user document");
}
// check if important values are null
if (user == null || string.IsNullOrWhiteSpace(user.Key) || string.IsNullOrWhiteSpace(user.UserHandle) || string.IsNullOrWhiteSpace(user.AppHandle))
{
this.Log.LogException("got bad parameters");
}
IndexOperation operation = new IndexOperation(IndexOperationType.Upload, "key", user.Key)
.WithProperty("firstName", user.FirstName)
.WithProperty("lastName", user.LastName)
.WithProperty("appHandle", user.AppHandle)
.WithProperty("userHandle", user.UserHandle);
return(operation);
}
/// <summary>
/// Ensures that the revision status type exists in the index
/// </summary>
/// <param name="status"></param>
/// <returns></returns>
public void EnsureRevisionStatus(RevisionStatusType status)
{
using (new WriteLockDisposable(_locker))
{
var found = GetRevisionStatusType((Guid)status.Id.Value);
if (found == null)
{
//we need to create it
var op = new IndexOperation()
{
Item = new IndexItem
{
Id = status.Id.Value.ToString(),
Fields = new Dictionary <string, ItemField>
{
{ FixedRevisionIndexFields.RevisionStatusAlias, new ItemField(status.Alias) },
{ FixedRevisionIndexFields.RevisionStatusName, new ItemField(status.Name) },
{
FixedRevisionIndexFields.RevisionStatusIsSystem, new ItemField(Convert.ToInt32(status.IsSystem))
{
DataType = FieldDataType.Int
}
}
},
ItemCategory = "RevisionStatus"
},
Operation = IndexOperationType.Add
};
ExamineManager.PerformIndexing(op);
ClearCache(true, false);
}
}
}
/// <inheritdoc />
public virtual void Apply(AssociationType item, DbModel model)
{
Check.NotNull(item, "item");
if (item.Constraint == null)
{
return;
}
var consolidatedIndexes
= ConsolidatedIndex.BuildIndexes(
item.Name,
item.Constraint.ToProperties.Select(p => Tuple.Create(p.Name, p)));
var dependentColumnNames = item.Constraint.ToProperties.Select(p => p.Name);
if (!consolidatedIndexes.Any(c => c.Columns.SequenceEqual(dependentColumnNames)))
{
var name = IndexOperation.BuildDefaultName(dependentColumnNames);
var order = 0;
foreach (var dependentColumn in item.Constraint.ToProperties)
{
var newAnnotation = new IndexAnnotation(new IndexAttribute(name, order++));
var existingAnnotation = dependentColumn.Annotations.GetAnnotation(XmlConstants.IndexAnnotationWithPrefix);
if (existingAnnotation != null)
{
newAnnotation = (IndexAnnotation)((IndexAnnotation)existingAnnotation).MergeWith(newAnnotation);
}
dependentColumn.AddAnnotation(XmlConstants.IndexAnnotationWithPrefix, newAnnotation);
}
}
}
public CreateIndexOperation CreateCreateIndexOperation()
{
var columnNames = Columns.ToArray();
Debug.Assert(columnNames.Length > 0);
Debug.Assert(_index.Name != null || columnNames.Length == 1);
var operation = new CreateIndexOperation
{
Name = _index.Name ?? IndexOperation.BuildDefaultName(columnNames),
Table = _table
};
foreach (var columnName in columnNames)
{
operation.Columns.Add(columnName);
}
if (_index.IsClusteredConfigured)
{
operation.IsClustered = _index.IsClustered;
}
if (_index.IsUniqueConfigured)
{
operation.IsUnique = _index.IsUnique;
}
return(operation);
}
/// <summary>
/// Will batch add documents to the Azure Search Index. If any documents exist, they will be replaced.
/// </summary>
/// <param name="documents">list of documents, with a max limit of 1000</param>
/// <param name="createDocumentInsertOperation">method that creates the operation</param>
/// <returns>task to add documents</returns>
protected async Task AddDocuments(List <Document> documents, Func <Document, IndexOperation> createDocumentInsertOperation)
{
this.Log.LogInformation("adding documents to search index");
// skip if null or too large
if (documents == null || documents.Count == 0 || documents.Count > 1000)
{
this.Log.LogException("got bad parameters");
}
// generate operations
List <IndexOperation> operations = new List <IndexOperation>();
foreach (Document document in documents)
{
IndexOperation operation = createDocumentInsertOperation(document);
if (operation == null)
{
this.Log.LogException("got null operation");
}
operations.Add(operation);
}
// issue the insert
IApiResponse <IEnumerable <IndexOperationResult> > response = await this.managementClient.PopulateAsync(this.indexName, operations.ToArray());
if (response == null || !response.IsSuccess)
{
this.Log.LogException("got service failure: " + ((response == null || response.Error == null) ? string.Empty : response.Error.Message));
}
}
/// <inheritdoc />
public virtual void Apply(AssociationType item, DbModel model)
{
Check.NotNull <AssociationType>(item, nameof(item));
if (item.Constraint == null)
{
return;
}
IEnumerable <ConsolidatedIndex> source = ConsolidatedIndex.BuildIndexes(item.Name, item.Constraint.ToProperties.Select <EdmProperty, Tuple <string, EdmProperty> >((Func <EdmProperty, Tuple <string, EdmProperty> >)(p => Tuple.Create <string, EdmProperty>(p.Name, p))));
IEnumerable <string> dependentColumnNames = item.Constraint.ToProperties.Select <EdmProperty, string>((Func <EdmProperty, string>)(p => p.Name));
if (source.Any <ConsolidatedIndex>((Func <ConsolidatedIndex, bool>)(c => c.Columns.SequenceEqual <string>(dependentColumnNames))))
{
return;
}
string name = IndexOperation.BuildDefaultName(dependentColumnNames);
int num = 0;
foreach (EdmProperty toProperty in item.Constraint.ToProperties)
{
IndexAnnotation indexAnnotation = new IndexAnnotation(new IndexAttribute(name, num++));
object annotation = toProperty.Annotations.GetAnnotation("http://schemas.microsoft.com/ado/2013/11/edm/customannotation:Index");
if (annotation != null)
{
indexAnnotation = (IndexAnnotation)((IndexAnnotation)annotation).MergeWith((object)indexAnnotation);
}
toProperty.AddAnnotation("http://schemas.microsoft.com/ado/2013/11/edm/customannotation:Index", (object)indexAnnotation);
}
}
// main method for adding posts. Adds all the posts in the WordPressPosts collection.
public async Task AddPosts()
{
// if not previously connected, make a connection
if (!connected)
{
Connect();
}
// create the index if it hasn't already been created.
await CreateIndex();
// run index population in batches. The Reddog.Search client maxes out at 1000 operations or about 16 MB of data transfer, so we have set the maximum to 100 posts in a batch to be conservative.
int batchCount = 0;
List <IndexOperation> IndexOperationList = new List <IndexOperation>(maximumNumberOfDocumentsPerBatch);
foreach (WordPressPost post in WordPressPosts.Posts)
{
batchCount++;
// create an indexoperation with the appropriate metadata and supply it with the incoming WordPress post
IndexOperation indexOperation = new IndexOperation(IndexOperationType.MergeOrUpload, "Id", post.Id.ToString())
.WithProperty("Title", post.Title)
.WithProperty("Content", post.Content)
.WithProperty("Excerpt", post.Excerpt)
.WithProperty("CreateDate", post.CreateDate.ToUniversalTime())
.WithProperty("ModifiedDate", post.ModifiedDate.ToUniversalTime())
.WithProperty("CreateDateAsString", post.CreateDate.ToLongDateString())
.WithProperty("ModifiedDateAsString", post.ModifiedDate.ToLongDateString())
.WithProperty("Author", post.Author)
.WithProperty("Categories", post.Categories)
.WithProperty("Tags", post.Tags)
.WithProperty("Slug", post.Slug)
.WithProperty("CommentCount", post.CommentCount)
.WithProperty("CommentContent", post.CommentContent);
// add the index operation to the collection
IndexOperationList.Add(indexOperation);
// if we have added maximum number of documents per batch, add the collection of operations to the index and then reset the collection to add a new batch.
if (batchCount >= maximumNumberOfDocumentsPerBatch)
{
var result = await managementClient.PopulateAsync(Index, IndexOperationList.ToArray());
if (!result.IsSuccess)
{
Console.Out.WriteLine(result.Error.Message);
}
batchCount = 0;
IndexOperationList = new List <IndexOperation>(maximumNumberOfDocumentsPerBatch);
}
}
// look for any remaining items that have not yet been added to the index.
var remainingResult = await managementClient.PopulateAsync(Index, IndexOperationList.ToArray());
if (!remainingResult.IsSuccess)
{
Console.Out.WriteLine(remainingResult.Error.Message);
}
}
private void AddTableIndex(IndexOperation operation)
{
if (operation == null)
{
return;
}
AddTableIndex(operation.Table, operation.Columns, false, operation.Name);
}
internal async Task <bool> SendItemToIndexAsync(IndexOperation itemOperation)
{
var result = await ManagementClient.PopulateAsync(IndexName, itemOperation);
if (!result.IsSuccess)
{
Trace.Write("Error: " + result.Error.Message);
}
return(result.IsSuccess);
}
public static IndexOperation ToIndexOperation(this SearchQueueItem item)
{
var op = new IndexOperation(IndexOperationType.Upload, "id", item.Id.ToString(CultureInfo.InvariantCulture))
.WithProperty("title", item.Title)
.WithProperty("status", item.Status)
.WithProperty("lastupdatedate", item.LastUpdateDate)
.WithProperty("details", item.Details)
.WithProperty("tags", item.Tags)
.WithProperty("events", item.Events);
return(op);
}
private static void PersistToSearch(Guid id, RegisterRestoCommand command, Coordinates coordinates)
{
using (var ctx = new RestoContext())
{
var connection = ApiConnection.Create(CloudConfigurationManager.GetSetting("Azure.Search.ServiceName"),
CloudConfigurationManager.GetSetting("Azure.Search.ApiKey"));
var indexClient = new IndexManagementClient(connection);
var indexName = CloudConfigurationManager.GetSetting("Azure.Search.IndexName");
var restaurant = ctx.Restaurants
.Include(r => r.Accommodations.Select(a => a.Accommodation.Translations))
.FirstOrDefault(r => r.Id == id);
var operation = new IndexOperation(IndexOperationType.MergeOrUpload, "id", id.ToString())
.WithProperty("name", command.Name)
.WithProperty("locality", command.City)
.WithProperty("budget", command.Budget)
.WithProperty("rating", command.Rating)
.WithProperty("street", command.Street)
.WithProperty("accommodations", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("en"))
.WithProperty("accommodations_fr", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("fr"))
.WithProperty("accommodations_nl", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("nl"))
.WithProperty("cuisine", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("en"))
.WithProperty("cuisine_fr", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("fr"))
.WithProperty("cuisine_nl", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("nl"));
if (coordinates != null)
{
operation.WithGeographyPoint("location", coordinates.Longitude, coordinates.Latitude);
}
var response = indexClient.PopulateAsync(indexName, operation).Result;
// Error handling!
if (!response.IsSuccess)
{
Console.WriteLine("Error: " + response.StatusCode);
return;
}
else
{
var failed = response.Body.Where(r => !r.Status);
foreach (var item in failed)
{
Console.WriteLine("Failed: {0} ({1})", item.Key, item.ErrorMessage);
}
}
Console.WriteLine("Persisted to Search.");
}
}
public virtual void performIndexMaintance(Session paramSession, int paramInt, string paramString)
{
System.Collections.IList list = getTables(paramSession, paramInt, paramString);
foreach (string str in list)
{
System.Collections.IDictionary map = getIndexOperations(paramSession, paramInt, paramString, str);
foreach (DictionaryEntry entry in map.SetOfKeyValuePairs())
{
string str1 = (string)entry.Key;
IndexOperation indexOperation = (IndexOperation)entry.Value;
indexOperation.execute(paramSession, str, str1);
}
}
}
private async Task ExecuteAsync(IndexOperation action)
{
var indexer = await Factory.CreateAsync(schemaId.Id);
try
{
var sut = new TextIndexingProcess(indexer, State);
await action(sut);
}
finally
{
await Factory.CleanupAsync();
}
}
private void ConfigureIndexes(DbDatabaseMapping mapping, EntityType entityType)
{
DebugCheck.NotNull(mapping);
DebugCheck.NotNull(entityType);
var entityTypeMappings = mapping.GetEntityTypeMappings(entityType);
if (_keyConfiguration != null)
{
entityTypeMappings
.SelectMany(etm => etm.Fragments)
.Each(f => _keyConfiguration.Configure(f.Table));
}
foreach (var indexConfiguration in _indexConfigurations)
{
foreach (var entityTypeMapping in entityTypeMappings)
{
var propertyMappings = indexConfiguration.Key
.ToDictionary(
icp => icp,
icp => entityTypeMapping.GetPropertyMapping(
entityType.GetDeclaredPrimitiveProperty(icp)));
if (indexConfiguration.Key.Count > 1 && string.IsNullOrEmpty(indexConfiguration.Value.Name))
{
indexConfiguration.Value.Name = IndexOperation.BuildDefaultName(
indexConfiguration.Key.Select(icp => propertyMappings[icp].ColumnProperty.Name));
}
int sortOrder = 0;
foreach (var indexConfigurationProperty in indexConfiguration.Key)
{
var propertyMapping = propertyMappings[indexConfigurationProperty];
indexConfiguration.Value.Configure(
propertyMapping.ColumnProperty,
(indexConfiguration.Key.Count != 1 ?
sortOrder :
-1));
++sortOrder;
}
}
}
}
protected override void BeforeAdd(Dictionary <string, object> dic)
{
string browsFile = dic.GetValue("BrowsFile");
if (!string.IsNullOrEmpty(browsFile))
{
string fileStorePath = ConfigurationManager.AppSettings["FileStorePath"];
var fileNameArr = browsFile.Split(',');
foreach (var name in fileNameArr)
{
string tmpPath = fileStorePath + name;
string physicalPath = Server.MapPath("/" + tmpPath);
IndexOperation opera = IndexOperation.GetAddOpera(physicalPath);
opera.Id = dic.GetValue("Id");
opera.Title = dic.GetValue("Name");
SearchIndexManager.GetInstance().AddOpreation(opera);
}
}
}
public IActionResult Index()
{
bool result = false;
const int numberOfCycles = 10000;
var sw = Stopwatch.StartNew();
IndexOperation indexOperation = new IndexOperation(this.B2BElasticClient);
var indexName = nameof(Company).ToLower();//索引名称小写
var createSuccess = indexOperation.CreateIndex <Company>(indexName);
if (createSuccess)
{
int total = 0;
var list = this.CompanyDL.LoadPageItems(0, numberOfCycles, out total).ToList();
result = indexOperation.BulkAll(indexName, list);
}
sw.Stop();
return(this.Ok(result));
}
/// <summary>
/// Need to do a bit of cleanup now for Revision entries to ensure that we keep the records flagged as IsLatest to a minimum.
/// To do this we'll lookup all revisions in the index that are marked as IsLatest that have Ids of revisions that we've just committed
/// that have a UtcModified date of less than the ones we've just committed... thankfully, Lucene has a Range query that we can
/// use to do this cleanup
/// </summary>
/// <param name="revisionsCommitted">The revisions committed during this transaction</param>
private void CleanupIndexedRevisions(IEnumerable <Tuple <TypedEntity, RevisionData> > revisionsCommitted)
{
foreach (var r in revisionsCommitted)
{
//TODO: Figure out how Lucene is dealing with DateTime and UTC... currently were putting it back a day
// but if we can do an Hour that would be better, just need to figure out what it is storing.
var criteria = ExamineManager.CreateSearchCriteria()
.Must().EntityType <TypedEntity>()
.Must().HiveId(r.Item1.Id, FixedIndexedFields.EntityId)
.Must().Field(FixedRevisionIndexFields.IsLatest, "1".Escape())
.Must().Range(FixedIndexedFields.UtcModified, DateTime.MinValue, r.Item1.UtcModified.UtcDateTime.AddDays(-1));
foreach (var i in ExamineManager.Search(criteria.Compile()))
{
//convert the fields returned
var fields = i.Fields.ToDictionary(f => f.Key, f => new ItemField(f.Value));
//remove the flag
fields[FixedRevisionIndexFields.IsLatest] = new ItemField(0)
{
DataType = FieldDataType.Int
};
//now we need to update the index item to not be latest
var updateOp = new IndexOperation
{
Operation = IndexOperationType.Add,
Item = new IndexItem
{
Fields = fields,
Id = i.Id,
ItemCategory = i.Fields[LuceneIndexer.IndexCategoryFieldName]
}
};
//do the update
ExamineManager.PerformIndexing(updateOp);
}
}
}
public CreateIndexOperation CreateCreateIndexOperation()
{
string[] array = this.Columns.ToArray <string>();
CreateIndexOperation createIndexOperation1 = new CreateIndexOperation((object)null);
createIndexOperation1.Name = this._index.Name ?? IndexOperation.BuildDefaultName((IEnumerable <string>)array);
createIndexOperation1.Table = this._table;
CreateIndexOperation createIndexOperation2 = createIndexOperation1;
foreach (string str in array)
{
createIndexOperation2.Columns.Add(str);
}
if (this._index.IsClusteredConfigured)
{
createIndexOperation2.IsClustered = this._index.IsClustered;
}
if (this._index.IsUniqueConfigured)
{
createIndexOperation2.IsUnique = this._index.IsUnique;
}
return(createIndexOperation2);
}
protected virtual string IndexName(IndexOperation index, bool withSchema)
{
Contract.Requires(!string.IsNullOrWhiteSpace(index.Table));
Contract.Requires(!(!index.Columns.Any()));
var databaseName = index.Table.ToDatabaseName();
var name = new List <string>();
// check if I've to add the schema name before the index name
// needed during drop operation
if (withSchema)
{
name.Add(databaseName.Schema);
}
name.Add(string.Format(CultureInfo.InvariantCulture, "IX_{0}_{1}",
index.Table,
index.Columns.Join(separator: "_")).RestrictTo(128));
return(name.Join(Quote, "."));
}
/// <summary>
/// Creates an insert operation for the supplied topic
/// </summary>
/// <param name="document">the topic to create an operation for</param>
/// <returns>insert operation</returns>
private IndexOperation CreateDocumentInsertOperation(Document document)
{
this.Log.LogInformation("creating topic insert");
TopicDocument topic = null;
if (document is TopicDocument)
{
topic = (TopicDocument)document;
}
else
{
this.Log.LogException("got document that is not a topic document");
}
// check if important values are null
if (topic == null || string.IsNullOrWhiteSpace(topic.TopicHandle))
{
this.Log.LogException("got bad parameters");
}
if (topic.TopicTags == null)
{
topic.TopicTags = new List <string>();
}
IndexOperation operation = new IndexOperation(IndexOperationType.Upload, "topicHandle", topic.TopicHandle)
.WithProperty("topicTitle", topic.TopicTitle)
.WithProperty("topicText", topic.TopicText)
.WithProperty("topicTags", topic.TopicTags)
.WithProperty("appHandle", topic.AppHandle)
.WithProperty("userHandle", topic.UserHandle)
.WithProperty("searchWeight", topic.SearchWeight)
.WithProperty("topicLastModifiedTime", new DateTimeOffset(topic.TopicLastModifiedTime));
return(operation);
}
public async Task <HttpResponseMessage> Put(string indexName)
{
var request = Request;
if (!request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
// Get the current index.
var index = await _managementClient.GetIndexAsync(indexName);
if (!index.IsSuccess)
{
return(Request.CreateResponse(index.StatusCode, index));
}
var keyField = index.Body.Fields.FirstOrDefault(f => f.Key);
if (keyField == null)
{
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "Unable to find key field."));
}
// Read all files.
var root = System.Web.HttpContext.Current.Server.MapPath("~/App_Data/imports");
if (!Directory.Exists(root))
{
Directory.CreateDirectory(root);
}
var provider = new MultipartFormDataStreamProvider(root);
await request.Content.ReadAsMultipartAsync(provider);
// Operations.
var operations = new List <IndexOperation>();
// Process all files.
foreach (var file in provider.FileData)
{
using (var streamReader = new StreamReader(file.LocalFileName))
{
var parser = new CsvParser(streamReader);
parser.Configuration.Delimiter = CloudConfigurationManager.GetSetting("CsvDelimiter");
var header = parser.Read();
if (header == null)
{
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "The CSV file does not contain a header."));
}
var columns = header.ToList();
if (columns.IndexOf(keyField.Name) < 0)
{
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "The CSV file does not contain the key field."));
}
// Process all records.
while (true)
{
var row = parser.Read();
if (row == null)
{
break;
}
// Create a new operation.
var operation = new IndexOperation(IndexOperationType.Upload, keyField.Name, row[columns.IndexOf(keyField.Name)]);
for (int i = 0; i < row.Length; i++)
{
var columnName = columns[i];
if (columnName != keyField.Name)
{
var field = index.Body.Fields.FirstOrDefault(f => f.Name == columnName);
if (field == null)
{
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "Unknown field: " + field.Name));
}
if (field.Type == FieldType.StringCollection)
{
operation.Properties.Add(columnName, row[i].Contains("/") ? row[i].Split('/') : new[] { row[i] });
}
else if (field.Type == FieldType.Double)
{
double doubleValue = 0;
double.TryParse(row[i], out doubleValue);
operation.Properties.Add(columnName, doubleValue);
}
else if (field.Type == FieldType.Integer)
{
int intValue = 0;
int.TryParse(row[i], out intValue);
operation.Properties.Add(columnName, intValue);
}
else if (field.Type == FieldType.Boolean)
{
bool booleanValue = false;
bool.TryParse(row[i], out booleanValue);
operation.Properties.Add(columnName, booleanValue);
}
else if (field.Type == FieldType.DateTimeOffset)
{
DateTimeOffset dateValue = DateTimeOffset.MinValue;
DateTimeOffset.TryParse(row[i], out dateValue);
operation.Properties.Add(columnName, dateValue);
}
else if (field.Type == FieldType.GeographyPoint)
{
if (row[i].Contains('|'))
{
var coordinates = row[i].Split('|');
operation.Properties.Add(columnName, new { type = "Point", coordinates = new[]
{
double.Parse(coordinates[0], CultureInfo.InvariantCulture), // Latitude
double.Parse(coordinates[1], CultureInfo.InvariantCulture) // Longitude
} });
}
}
else
{
operation.Properties.Add(columnName, row[i]);
}
}
}
// Add operation to batch.
operations.Add(operation);
}
}
}
// Populate.
var result = await _managementClient.PopulateAsync(indexName, operations.ToArray());
if (!result.IsSuccess)
{
return(Request.CreateResponse(result.StatusCode, result));
}
return(Request.CreateResponse(HttpStatusCode.OK, result));
}
static string BuildIndexName(IndexOperation indexOperation)
{
return(!indexOperation.HasDefaultName
? indexOperation.Name
: IndexOperation.BuildDefaultName(new[] { ExtractName(indexOperation.Table) }.Concat(indexOperation.Columns)));
}
public void Apply(EntityType item, DbModel model)
{
// Build index info, consolidating indexes with the same name
var indexInfo = new List <IndexInfo>();
foreach (var p in item.Properties)
{
foreach (var mp in p.MetadataProperties)
{
var a = mp.Value as IndexAnnotation;
if (a == null)
{
continue;
}
foreach (var index in a.Indexes)
{
var info = index.Name != null?indexInfo.FirstOrDefault(e => e.Name == index.Name) : null;
if (info == null)
{
info = new IndexInfo {
Name = index.Name
};
indexInfo.Add(info);
}
else
{
var other = info.Entries[0].Index;
if (index.IsUnique != other.IsUnique || index.IsClustered != other.IsClustered)
{
throw new Exception("Invalid index configuration.");
}
}
info.Entries.Add(new IndexEntry {
Column = p, Annotaion = mp, Index = index
});
}
}
}
if (indexInfo.Count == 0)
{
return;
}
// Generate new name where needed
var entitySet = model.StoreModel.Container.EntitySets.First(es => es.ElementType == item);
foreach (var info in indexInfo)
{
var columns = info.Entries.OrderBy(e => e.Index.Order).Select(e => e.Column.Name);
if (info.Name == null || info.Name == IndexOperation.BuildDefaultName(columns))
{
bool unique = info.Entries[0].Index.IsUnique;
var name = string.Format("{0}_{1}_{2}", unique ? "UX" : "IX", entitySet.Table, string.Join("_", columns));
if (name.Length > 128)
{
name = name.Substring(0, 128);
}
if (info.Name == name)
{
continue;
}
foreach (var entry in info.Entries)
{
var index = new IndexAttribute(name);
if (entry.Index.Order >= 0)
{
index.Order = entry.Index.Order;
}
if (entry.Index.IsUniqueConfigured)
{
index.IsUnique = entry.Index.IsUnique;
}
if (entry.Index.IsClusteredConfigured)
{
index.IsClustered = entry.Index.IsClustered;
}
entry.Index = index;
entry.Modified = true;
}
}
}
// Apply the changes
foreach (var g in indexInfo.SelectMany(e => e.Entries).GroupBy(e => e.Annotaion))
{
if (g.Any(e => e.Modified))
{
g.Key.Value = new IndexAnnotation(g.Select(e => e.Index));
}
}
}
public IndexCommand OperationType(IndexOperation operation)
{
WithParameter("op_type", operation.AsString());
return(this);
}
/// <inheritdoc />
public void Apply(EntityType item, DbModel model)
{
var indexInfos = new List <IndexInfo>();
foreach (var property in item.Properties)
{
foreach (var metadataProperty in property.MetadataProperties)
{
if (!(metadataProperty.Value is IndexAnnotation annotation))
{
continue;
}
foreach (var index in annotation.Indexes)
{
var info = index.Name != null?indexInfos.FirstOrDefault(e => e.Name == index.Name) : null;
if (info == null)
{
info = new IndexInfo {
Name = index.Name
};
indexInfos.Add(info);
}
else
{
var other = info.Entries[0].Index;
if (index.IsUnique != other.IsUnique || index.IsClustered != other.IsClustered)
{
throw new Exception("Invalid index configuration.");
}
}
info.Entries.Add(new IndexEntry {
Column = property, Annotation = metadataProperty, Index = index
});
}
}
}
if (indexInfos.Count == 0)
{
return;
}
foreach (var indexInfo in indexInfos)
{
var columns = indexInfo.Entries.OrderBy(e => e.Index.Order).Select(e => e.Column.Name).ToList();
if (indexInfo.Name != null && indexInfo.Name != IndexOperation.BuildDefaultName(columns))
{
continue;
}
bool unique = indexInfo.Entries[0].Index.IsUnique;
string name = $"{(unique ? "UX" : "IX")}_{string.Join("_", columns.Select(column => column.Replace("_", string.Empty)))}";
if (name.Length > 128)
{
name = name.Substring(0, 128);
}
if (indexInfo.Name == name)
{
continue;
}
foreach (var entry in indexInfo.Entries)
{
var index = new IndexAttribute(name);
if (entry.Index.Order >= 0)
{
index.Order = entry.Index.Order;
}
if (entry.Index.IsUniqueConfigured)
{
index.IsUnique = entry.Index.IsUnique;
}
if (entry.Index.IsClusteredConfigured)
{
index.IsClustered = entry.Index.IsClustered;
}
entry.Index = index;
entry.Modified = true;
}
}
foreach (var indexInfo in indexInfos.SelectMany(ii => ii.Entries).GroupBy(e => e.Annotation))
{
if (indexInfo.Any(e => e.Modified))
{
indexInfo.Key.Value = new IndexAnnotation(indexInfo.Select(e => e.Index));
}
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="action">The operation to perform on the index</param>
/// <param name="parameters">The parameters to the operation</param>
public PendingIndexOperation(IndexOperation action, object[] parameters)
{
this.Action = action;
this.Parameters = parameters;
}
private static void Main(string[] args)
{
var indexName = CloudConfigurationManager.GetSetting("Azure.Search.IndexName");
Log("Starting full sync to: {0}", indexName);
var searchClient = GetSearchClient();
while (true)
{
using (var dbTimer = CallTimer.Start())
{
// Get batch of restaurants.
var restaurants = GetRestaurants(Skip, Take);
dbTimer.Stop();
// No results, EOF.
if (restaurants == null)
{
break;
}
// Flatten.
var operations = new List <IndexOperation>();
foreach (var restaurant in restaurants)
{
var indexOperation = new IndexOperation(IndexOperationType.MergeOrUpload, "id", restaurant.Id.ToString());
indexOperation
.WithProperty("internalName", restaurant.InternalName)
.WithProperty("name", restaurant.Name)
.WithProperty("postalCode", restaurant.PostalCode)
.WithProperty("locality", restaurant.Locality)
.WithProperty("street", restaurant.StreetAddress)
.WithProperty("website", restaurant.Website)
.WithProperty("budget", restaurant.Budget)
.WithProperty("rating", restaurant.Rating)
.WithProperty("fax", restaurant.Fax)
.WithProperty("mobile", restaurant.Mobile)
.WithProperty("phoneNumber", restaurant.PhoneNumber)
.WithProperty("email", restaurant.Email)
.WithProperty("hasImage", restaurant.HasImage)
// Translated content.
.WithProperty("region", restaurant.Region.TryGet("en"))
.WithProperty("region_nl", restaurant.Region.TryGet("nl"))
.WithProperty("region_fr", restaurant.Region.TryGet("fr"))
.WithProperty("description", restaurant.TryGet(r => r.Description, "en"))
.WithProperty("description_fr", restaurant.TryGet(r => r.Description, "fr"))
.WithProperty("description_nl", restaurant.TryGet(r => r.Description, "nl"))
.WithProperty("closing", restaurant.TryGet(r => r.Closing, "en"))
.WithProperty("closing_fr", restaurant.TryGet(r => r.Closing, "fr"))
.WithProperty("closing_nl", restaurant.TryGet(r => r.Closing, "nl"))
.WithProperty("setting", restaurant.TryGet(r => r.Setting, "en"))
.WithProperty("setting_fr", restaurant.TryGet(r => r.Setting, "fr"))
.WithProperty("setting_nl", restaurant.TryGet(r => r.Setting, "nl"))
// Translated tags.
.WithProperty("accommodations", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("en"))
.WithProperty("accommodations_fr", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("fr"))
.WithProperty("accommodations_nl", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("nl"))
.WithProperty("cuisine", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("en"))
.WithProperty("cuisine_fr", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("fr"))
.WithProperty("cuisine_nl", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("nl"))
.WithProperty("paymentFacilities", restaurant.PaymentFacilities.Select(a => a.PaymentFacility).TryGet <PaymentFacility, PaymentFacilityTranslation>("en"))
.WithProperty("paymentFacilities_fr", restaurant.PaymentFacilities.Select(a => a.PaymentFacility).TryGet <PaymentFacility, PaymentFacilityTranslation>("fr"))
.WithProperty("paymentFacilities_nl", restaurant.PaymentFacilities.Select(a => a.PaymentFacility).TryGet <PaymentFacility, PaymentFacilityTranslation>("nl"));
// Add geocoordinates if available.
if (restaurant.Longitude.HasValue && restaurant.Latitude.HasValue)
{
indexOperation.WithGeographyPoint("location", restaurant.Longitude.Value, restaurant.Latitude.Value);
}
// Add to batch.
operations.Add(indexOperation);
}
using (var searchTimer = CallTimer.Start())
{
var response = searchClient.PopulateAsync(indexName, operations.ToArray()).Result;
// Error handling!
if (!response.IsSuccess)
{
throw new Exception(response.StatusCode.ToString());
}
else
{
var failed = response.Body.Where(r => !r.Status);
foreach (var item in failed)
{
Log("Failed: {0} ({1})", item.Key, item.ErrorMessage);
}
}
// Move forward.
Skip += Take;
Processed += restaurants.Count();
// Done!
Log("Processed: {0} (Db: {1} s., Search: {2} s.)", Processed, dbTimer.TotalSeconds, searchTimer.TotalSeconds);
}
}
}
Log("Done!");
}
public static IndexOperation WithProperty(this IndexOperation operation, string name, object value)
{
operation.Properties.Add(name, value);
return(operation);
}