// main method for adding posts. Adds all the posts in the WordPressPosts collection.
public async Task AddPosts()
{
// if not previously connected, make a connection
if (!connected)
{
Connect();
}
// create the index if it hasn't already been created.
await CreateIndex();
// run index population in batches. The Reddog.Search client maxes out at 1000 operations or about 16 MB of data transfer, so we have set the maximum to 100 posts in a batch to be conservative.
int batchCount = 0;
List <IndexOperation> IndexOperationList = new List <IndexOperation>(maximumNumberOfDocumentsPerBatch);
foreach (WordPressPost post in WordPressPosts.Posts)
{
batchCount++;
// create an indexoperation with the appropriate metadata and supply it with the incoming WordPress post
IndexOperation indexOperation = new IndexOperation(IndexOperationType.MergeOrUpload, "Id", post.Id.ToString())
.WithProperty("Title", post.Title)
.WithProperty("Content", post.Content)
.WithProperty("Excerpt", post.Excerpt)
.WithProperty("CreateDate", post.CreateDate.ToUniversalTime())
.WithProperty("ModifiedDate", post.ModifiedDate.ToUniversalTime())
.WithProperty("CreateDateAsString", post.CreateDate.ToLongDateString())
.WithProperty("ModifiedDateAsString", post.ModifiedDate.ToLongDateString())
.WithProperty("Author", post.Author)
.WithProperty("Categories", post.Categories)
.WithProperty("Tags", post.Tags)
.WithProperty("Slug", post.Slug)
.WithProperty("CommentCount", post.CommentCount)
.WithProperty("CommentContent", post.CommentContent);
// add the index operation to the collection
IndexOperationList.Add(indexOperation);
// if we have added maximum number of documents per batch, add the collection of operations to the index and then reset the collection to add a new batch.
if (batchCount >= maximumNumberOfDocumentsPerBatch)
{
var result = await managementClient.PopulateAsync(Index, IndexOperationList.ToArray());
if (!result.IsSuccess)
{
Console.Out.WriteLine(result.Error.Message);
}
batchCount = 0;
IndexOperationList = new List <IndexOperation>(maximumNumberOfDocumentsPerBatch);
}
}
// look for any remaining items that have not yet been added to the index.
var remainingResult = await managementClient.PopulateAsync(Index, IndexOperationList.ToArray());
if (!remainingResult.IsSuccess)
{
Console.Out.WriteLine(remainingResult.Error.Message);
}
}
public async Task UpdatePostAsync(Spot spot)
{
var client = new IndexManagementClient(connection);
var result = await client.PopulateAsync("spot",
new IndexOperation(IndexOperationType.Merge, "id", spot.Uid)
.WithProperty("name", spot.Name)
.WithProperty("description", spot.Description)
.WithGeographyPoint("location", spot.Location.Longitude, spot.Location.Latitude)
);
}
private static void PersistToSearch(Guid id, RegisterRestoCommand command, Coordinates coordinates)
{
using (var ctx = new RestoContext())
{
var connection = ApiConnection.Create(CloudConfigurationManager.GetSetting("Azure.Search.ServiceName"),
CloudConfigurationManager.GetSetting("Azure.Search.ApiKey"));
var indexClient = new IndexManagementClient(connection);
var indexName = CloudConfigurationManager.GetSetting("Azure.Search.IndexName");
var restaurant = ctx.Restaurants
.Include(r => r.Accommodations.Select(a => a.Accommodation.Translations))
.FirstOrDefault(r => r.Id == id);
var operation = new IndexOperation(IndexOperationType.MergeOrUpload, "id", id.ToString())
.WithProperty("name", command.Name)
.WithProperty("locality", command.City)
.WithProperty("budget", command.Budget)
.WithProperty("rating", command.Rating)
.WithProperty("street", command.Street)
.WithProperty("accommodations", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("en"))
.WithProperty("accommodations_fr", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("fr"))
.WithProperty("accommodations_nl", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("nl"))
.WithProperty("cuisine", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("en"))
.WithProperty("cuisine_fr", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("fr"))
.WithProperty("cuisine_nl", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("nl"));
if (coordinates != null)
{
operation.WithGeographyPoint("location", coordinates.Longitude, coordinates.Latitude);
}
var response = indexClient.PopulateAsync(indexName, operation).Result;
// Error handling!
if (!response.IsSuccess)
{
Console.WriteLine("Error: " + response.StatusCode);
return;
}
else
{
var failed = response.Body.Where(r => !r.Status);
foreach (var item in failed)
{
Console.WriteLine("Failed: {0} ({1})", item.Key, item.ErrorMessage);
}
}
Console.WriteLine("Persisted to Search.");
}
}
public void AppendVersion(string schema, string version)
{
var client = new IndexManagementClient(_db);
var result = client.PopulateAsync("automaticmigration",
new IndexOperation(IndexOperationType.Upload, "Id", Guid.NewGuid().ToString())
.WithProperty("SchemaName", schema)
.WithProperty("Version", version)
.WithProperty("TimeOfUpdate", DateTimeOffset.Now)).Result;
if (!result.IsSuccess)
{
throw new ApplicationException("Could not store migration in AutomaticMigration index. Error: " + result.Error.Message);
}
}
public void UntrackSchemas(IEnumerable <string> schemas)
{
var queryClient = new IndexQueryClient(_db);
foreach (var schema in schemas)
{
var migrations = queryClient.SearchAsync("automaticmigration", new SearchQuery(schema)
.OrderBy("TimeOfUpdate")
.SearchField("SchemaName")).Result;
if (!migrations.IsSuccess)
{
throw new ApplicationException("Error untracking schema: could not find migrations in schema " + schema + ", error: " + migrations.Error.Message);
}
if (migrations.Body != null)
{
var indexClient = new IndexManagementClient(_db);
var result = indexClient.PopulateAsync("automaticmigration",
migrations.Body.Records.Select(record =>
new IndexOperation(IndexOperationType.Delete, "Id", record.Properties["Id"].ToString())).ToArray()).Result;
}
}
}
public async Task <HttpResponseMessage> Put(string indexName)
{
var request = Request;
if (!request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
// Get the current index.
var index = await _managementClient.GetIndexAsync(indexName);
if (!index.IsSuccess)
{
return(Request.CreateResponse(index.StatusCode, index));
}
var keyField = index.Body.Fields.FirstOrDefault(f => f.Key);
if (keyField == null)
{
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "Unable to find key field."));
}
// Read all files.
var root = System.Web.HttpContext.Current.Server.MapPath("~/App_Data/imports");
if (!Directory.Exists(root))
{
Directory.CreateDirectory(root);
}
var provider = new MultipartFormDataStreamProvider(root);
await request.Content.ReadAsMultipartAsync(provider);
// Operations.
var operations = new List <IndexOperation>();
// Process all files.
foreach (var file in provider.FileData)
{
using (var streamReader = new StreamReader(file.LocalFileName))
{
var parser = new CsvParser(streamReader);
parser.Configuration.Delimiter = CloudConfigurationManager.GetSetting("CsvDelimiter");
var header = parser.Read();
if (header == null)
{
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "The CSV file does not contain a header."));
}
var columns = header.ToList();
if (columns.IndexOf(keyField.Name) < 0)
{
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "The CSV file does not contain the key field."));
}
// Process all records.
while (true)
{
var row = parser.Read();
if (row == null)
{
break;
}
// Create a new operation.
var operation = new IndexOperation(IndexOperationType.Upload, keyField.Name, row[columns.IndexOf(keyField.Name)]);
for (int i = 0; i < row.Length; i++)
{
var columnName = columns[i];
if (columnName != keyField.Name)
{
var field = index.Body.Fields.FirstOrDefault(f => f.Name == columnName);
if (field == null)
{
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "Unknown field: " + field.Name));
}
if (field.Type == FieldType.StringCollection)
{
operation.Properties.Add(columnName, row[i].Contains("/") ? row[i].Split('/') : new[] { row[i] });
}
else if (field.Type == FieldType.Double)
{
double doubleValue = 0;
double.TryParse(row[i], out doubleValue);
operation.Properties.Add(columnName, doubleValue);
}
else if (field.Type == FieldType.Integer)
{
int intValue = 0;
int.TryParse(row[i], out intValue);
operation.Properties.Add(columnName, intValue);
}
else if (field.Type == FieldType.Boolean)
{
bool booleanValue = false;
bool.TryParse(row[i], out booleanValue);
operation.Properties.Add(columnName, booleanValue);
}
else if (field.Type == FieldType.DateTimeOffset)
{
DateTimeOffset dateValue = DateTimeOffset.MinValue;
DateTimeOffset.TryParse(row[i], out dateValue);
operation.Properties.Add(columnName, dateValue);
}
else if (field.Type == FieldType.GeographyPoint)
{
if (row[i].Contains('|'))
{
var coordinates = row[i].Split('|');
operation.Properties.Add(columnName, new { type = "Point", coordinates = new[]
{
double.Parse(coordinates[0], CultureInfo.InvariantCulture), // Latitude
double.Parse(coordinates[1], CultureInfo.InvariantCulture) // Longitude
} });
}
}
else
{
operation.Properties.Add(columnName, row[i]);
}
}
}
// Add operation to batch.
operations.Add(operation);
}
}
}
// Populate.
var result = await _managementClient.PopulateAsync(indexName, operations.ToArray());
if (!result.IsSuccess)
{
return(Request.CreateResponse(result.StatusCode, result));
}
return(Request.CreateResponse(HttpStatusCode.OK, result));
}