// main method for adding posts. Adds all the posts in the WordPressPosts collection. public async Task AddPosts() { // if not previously connected, make a connection if (!connected) { Connect(); } // create the index if it hasn't already been created. await CreateIndex(); // run index population in batches. The Reddog.Search client maxes out at 1000 operations or about 16 MB of data transfer, so we have set the maximum to 100 posts in a batch to be conservative. int batchCount = 0; List <IndexOperation> IndexOperationList = new List <IndexOperation>(maximumNumberOfDocumentsPerBatch); foreach (WordPressPost post in WordPressPosts.Posts) { batchCount++; // create an indexoperation with the appropriate metadata and supply it with the incoming WordPress post IndexOperation indexOperation = new IndexOperation(IndexOperationType.MergeOrUpload, "Id", post.Id.ToString()) .WithProperty("Title", post.Title) .WithProperty("Content", post.Content) .WithProperty("Excerpt", post.Excerpt) .WithProperty("CreateDate", post.CreateDate.ToUniversalTime()) .WithProperty("ModifiedDate", post.ModifiedDate.ToUniversalTime()) .WithProperty("CreateDateAsString", post.CreateDate.ToLongDateString()) .WithProperty("ModifiedDateAsString", post.ModifiedDate.ToLongDateString()) .WithProperty("Author", post.Author) .WithProperty("Categories", post.Categories) .WithProperty("Tags", post.Tags) .WithProperty("Slug", post.Slug) .WithProperty("CommentCount", post.CommentCount) .WithProperty("CommentContent", post.CommentContent); // add the index operation to the collection IndexOperationList.Add(indexOperation); // if we have added maximum number of documents per batch, add the collection of operations to the index and then reset the collection to add a new batch. if (batchCount >= maximumNumberOfDocumentsPerBatch) { var result = await managementClient.PopulateAsync(Index, IndexOperationList.ToArray()); if (!result.IsSuccess) { Console.Out.WriteLine(result.Error.Message); } batchCount = 0; IndexOperationList = new List <IndexOperation>(maximumNumberOfDocumentsPerBatch); } } // look for any remaining items that have not yet been added to the index. var remainingResult = await managementClient.PopulateAsync(Index, IndexOperationList.ToArray()); if (!remainingResult.IsSuccess) { Console.Out.WriteLine(remainingResult.Error.Message); } }
public async Task UpdatePostAsync(Spot spot) { var client = new IndexManagementClient(connection); var result = await client.PopulateAsync("spot", new IndexOperation(IndexOperationType.Merge, "id", spot.Uid) .WithProperty("name", spot.Name) .WithProperty("description", spot.Description) .WithGeographyPoint("location", spot.Location.Longitude, spot.Location.Latitude) ); }
private static void PersistToSearch(Guid id, RegisterRestoCommand command, Coordinates coordinates) { using (var ctx = new RestoContext()) { var connection = ApiConnection.Create(CloudConfigurationManager.GetSetting("Azure.Search.ServiceName"), CloudConfigurationManager.GetSetting("Azure.Search.ApiKey")); var indexClient = new IndexManagementClient(connection); var indexName = CloudConfigurationManager.GetSetting("Azure.Search.IndexName"); var restaurant = ctx.Restaurants .Include(r => r.Accommodations.Select(a => a.Accommodation.Translations)) .FirstOrDefault(r => r.Id == id); var operation = new IndexOperation(IndexOperationType.MergeOrUpload, "id", id.ToString()) .WithProperty("name", command.Name) .WithProperty("locality", command.City) .WithProperty("budget", command.Budget) .WithProperty("rating", command.Rating) .WithProperty("street", command.Street) .WithProperty("accommodations", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("en")) .WithProperty("accommodations_fr", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("fr")) .WithProperty("accommodations_nl", restaurant.Accommodations.Select(a => a.Accommodation).TryGet <Accommodation, AccommodationTranslation>("nl")) .WithProperty("cuisine", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("en")) .WithProperty("cuisine_fr", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("fr")) .WithProperty("cuisine_nl", restaurant.Cuisines.Select(a => a.Cuisine).TryGet <Cuisine, CuisineTranslation>("nl")); if (coordinates != null) { operation.WithGeographyPoint("location", coordinates.Longitude, coordinates.Latitude); } var response = indexClient.PopulateAsync(indexName, operation).Result; // Error handling! if (!response.IsSuccess) { Console.WriteLine("Error: " + response.StatusCode); return; } else { var failed = response.Body.Where(r => !r.Status); foreach (var item in failed) { Console.WriteLine("Failed: {0} ({1})", item.Key, item.ErrorMessage); } } Console.WriteLine("Persisted to Search."); } }
public void AppendVersion(string schema, string version) { var client = new IndexManagementClient(_db); var result = client.PopulateAsync("automaticmigration", new IndexOperation(IndexOperationType.Upload, "Id", Guid.NewGuid().ToString()) .WithProperty("SchemaName", schema) .WithProperty("Version", version) .WithProperty("TimeOfUpdate", DateTimeOffset.Now)).Result; if (!result.IsSuccess) { throw new ApplicationException("Could not store migration in AutomaticMigration index. Error: " + result.Error.Message); } }
public void UntrackSchemas(IEnumerable <string> schemas) { var queryClient = new IndexQueryClient(_db); foreach (var schema in schemas) { var migrations = queryClient.SearchAsync("automaticmigration", new SearchQuery(schema) .OrderBy("TimeOfUpdate") .SearchField("SchemaName")).Result; if (!migrations.IsSuccess) { throw new ApplicationException("Error untracking schema: could not find migrations in schema " + schema + ", error: " + migrations.Error.Message); } if (migrations.Body != null) { var indexClient = new IndexManagementClient(_db); var result = indexClient.PopulateAsync("automaticmigration", migrations.Body.Records.Select(record => new IndexOperation(IndexOperationType.Delete, "Id", record.Properties["Id"].ToString())).ToArray()).Result; } } }
public async Task <HttpResponseMessage> Put(string indexName) { var request = Request; if (!request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } // Get the current index. var index = await _managementClient.GetIndexAsync(indexName); if (!index.IsSuccess) { return(Request.CreateResponse(index.StatusCode, index)); } var keyField = index.Body.Fields.FirstOrDefault(f => f.Key); if (keyField == null) { return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "Unable to find key field.")); } // Read all files. var root = System.Web.HttpContext.Current.Server.MapPath("~/App_Data/imports"); if (!Directory.Exists(root)) { Directory.CreateDirectory(root); } var provider = new MultipartFormDataStreamProvider(root); await request.Content.ReadAsMultipartAsync(provider); // Operations. var operations = new List <IndexOperation>(); // Process all files. foreach (var file in provider.FileData) { using (var streamReader = new StreamReader(file.LocalFileName)) { var parser = new CsvParser(streamReader); parser.Configuration.Delimiter = CloudConfigurationManager.GetSetting("CsvDelimiter"); var header = parser.Read(); if (header == null) { return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "The CSV file does not contain a header.")); } var columns = header.ToList(); if (columns.IndexOf(keyField.Name) < 0) { return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "The CSV file does not contain the key field.")); } // Process all records. while (true) { var row = parser.Read(); if (row == null) { break; } // Create a new operation. var operation = new IndexOperation(IndexOperationType.Upload, keyField.Name, row[columns.IndexOf(keyField.Name)]); for (int i = 0; i < row.Length; i++) { var columnName = columns[i]; if (columnName != keyField.Name) { var field = index.Body.Fields.FirstOrDefault(f => f.Name == columnName); if (field == null) { return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "Unknown field: " + field.Name)); } if (field.Type == FieldType.StringCollection) { operation.Properties.Add(columnName, row[i].Contains("/") ? row[i].Split('/') : new[] { row[i] }); } else if (field.Type == FieldType.Double) { double doubleValue = 0; double.TryParse(row[i], out doubleValue); operation.Properties.Add(columnName, doubleValue); } else if (field.Type == FieldType.Integer) { int intValue = 0; int.TryParse(row[i], out intValue); operation.Properties.Add(columnName, intValue); } else if (field.Type == FieldType.Boolean) { bool booleanValue = false; bool.TryParse(row[i], out booleanValue); operation.Properties.Add(columnName, booleanValue); } else if (field.Type == FieldType.DateTimeOffset) { DateTimeOffset dateValue = DateTimeOffset.MinValue; DateTimeOffset.TryParse(row[i], out dateValue); operation.Properties.Add(columnName, dateValue); } else if (field.Type == FieldType.GeographyPoint) { if (row[i].Contains('|')) { var coordinates = row[i].Split('|'); operation.Properties.Add(columnName, new { type = "Point", coordinates = new[] { double.Parse(coordinates[0], CultureInfo.InvariantCulture), // Latitude double.Parse(coordinates[1], CultureInfo.InvariantCulture) // Longitude } }); } } else { operation.Properties.Add(columnName, row[i]); } } } // Add operation to batch. operations.Add(operation); } } } // Populate. var result = await _managementClient.PopulateAsync(indexName, operations.ToArray()); if (!result.IsSuccess) { return(Request.CreateResponse(result.StatusCode, result)); } return(Request.CreateResponse(HttpStatusCode.OK, result)); }