public async Task <bool> UpdateIndex(IEnumerable <SearchEngineItem> items) { using (var memoryStream = new MemoryStream()) { var jsonSerializer = new DataContractJsonSerializer(typeof(BatchDocumentItem[])); var batch = items.Select(x => new BatchDocumentItem(x)).ToArray(); jsonSerializer.WriteObject(memoryStream, batch); memoryStream.Position = 0; // reset cursor to able pass stream to AWS var upload = new UploadDocumentsRequest { ContentType = ContentType.ApplicationJson, Documents = memoryStream }; var result = await documentClient.UploadDocumentsAsync(upload); return(true); // todo handle response message from AWS. } }
public void Run() { var settings = new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore, Formatting = Formatting.None, }; using (var client = new AmazonCloudSearchDomainClient(_config.KeyId, _config.SecretKey, _config.ServiceUrl)) using (var batcher = new Batcher(_config.BatchSize, stream => UploadBatch(client, stream))) { foreach (var building in _reader.ReadBuildings(_config.BuildingsFilePath)) { var addOperation = new AddOperation { Id = DocumentId.Build(building), Fields = _mapper.Map <DocumentItem>(building), }; var json = JsonConvert.SerializeObject(addOperation, settings); batcher.Write(json); } foreach (var company in _reader.ReadCompanies(_config.CompaniesFilePath)) { var addOperation = new AddOperation { Id = DocumentId.Build(company), Fields = _mapper.Map <DocumentItem>(company), }; var json = JsonConvert.SerializeObject(addOperation, settings); batcher.Write(json); } } void UploadBatch(IAmazonCloudSearchDomain client, Stream stream) { var request = new UploadDocumentsRequest(); request.ContentType = ContentType.ApplicationJson; request.Documents = stream; Logger.Info($"Uploading batch with {stream.Length} bytes"); var task = client.UploadDocumentsAsync(request); if (!task.Wait(_config.Timeout)) { Logger.Error("Operation timeout."); return; } var response = task.Result; Logger.Info($"Operation results: {response.Status}, " + $"added {response.Adds} documents, " + $"warnings: {response.Warnings.Count}"); foreach (var warning in response.Warnings) { Logger.Info($"Operation warning: {warning}"); } } }