public async Task <bool> UpdateIndex(IEnumerable <SearchEngineItem> items)
{
using (var memoryStream = new MemoryStream())
{
var jsonSerializer = new DataContractJsonSerializer(typeof(BatchDocumentItem[]));
var batch = items.Select(x => new BatchDocumentItem(x)).ToArray();
jsonSerializer.WriteObject(memoryStream, batch);
memoryStream.Position = 0; // reset cursor to able pass stream to AWS
var upload = new UploadDocumentsRequest
{
ContentType = ContentType.ApplicationJson,
Documents = memoryStream
};
var result = await documentClient.UploadDocumentsAsync(upload);
return(true); // todo handle response message from AWS.
}
}
public void Run()
{
var settings = new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
Formatting = Formatting.None,
};
using (var client = new AmazonCloudSearchDomainClient(_config.KeyId, _config.SecretKey, _config.ServiceUrl))
using (var batcher = new Batcher(_config.BatchSize, stream => UploadBatch(client, stream)))
{
foreach (var building in _reader.ReadBuildings(_config.BuildingsFilePath))
{
var addOperation = new AddOperation
{
Id = DocumentId.Build(building),
Fields = _mapper.Map <DocumentItem>(building),
};
var json = JsonConvert.SerializeObject(addOperation, settings);
batcher.Write(json);
}
foreach (var company in _reader.ReadCompanies(_config.CompaniesFilePath))
{
var addOperation = new AddOperation
{
Id = DocumentId.Build(company),
Fields = _mapper.Map <DocumentItem>(company),
};
var json = JsonConvert.SerializeObject(addOperation, settings);
batcher.Write(json);
}
}
void UploadBatch(IAmazonCloudSearchDomain client, Stream stream)
{
var request = new UploadDocumentsRequest();
request.ContentType = ContentType.ApplicationJson;
request.Documents = stream;
Logger.Info($"Uploading batch with {stream.Length} bytes");
var task = client.UploadDocumentsAsync(request);
if (!task.Wait(_config.Timeout))
{
Logger.Error("Operation timeout.");
return;
}
var response = task.Result;
Logger.Info($"Operation results: {response.Status}, " +
$"added {response.Adds} documents, " +
$"warnings: {response.Warnings.Count}");
foreach (var warning in response.Warnings)
{
Logger.Info($"Operation warning: {warning}");
}
}
}