/// <summary>
/// Initiates the asynchronous execution of the UploadDocuments operation.
/// <seealso cref="Amazon.CloudSearchDomain.IAmazonCloudSearchDomain"/>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the UploadDocuments operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task <UploadDocumentsResponse> UploadDocumentsAsync(UploadDocumentsRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new UploadDocumentsRequestMarshaller();
var unmarshaller = UploadDocumentsResponseUnmarshaller.Instance;
return(Invoke <IRequest, UploadDocumentsRequest, UploadDocumentsResponse>(request, marshaller, unmarshaller, signer, cancellationToken));
}
IAsyncResult invokeUploadDocuments(UploadDocumentsRequest request, AsyncCallback callback, object state, bool synchronized)
{
var marshaller = new UploadDocumentsRequestMarshaller();
var unmarshaller = UploadDocumentsResponseUnmarshaller.Instance;
return(Invoke(request, callback, state, synchronized, marshaller, unmarshaller, signer));
}
/// <summary>
/// Posts a batch of documents to a search domain for indexing. A document batch is a
/// collection of add and delete operations that represent the documents you want to add,
/// update, or delete from your domain. Batches can be described in either JSON or XML.
/// Each item that you want Amazon CloudSearch to return as a search result (such as a
/// product) is represented as a document. Every document has a unique ID and one or more
/// fields that contain the data that you want to search and return in results. Individual
/// documents cannot contain more than 1 MB of data. The entire batch cannot exceed 5
/// MB. To get the best possible upload performance, group add and delete operations in
/// batches that are close the 5 MB limit. Submitting a large volume of single-document
/// batches can overload a domain's document service.
///
///
/// <para>
/// The endpoint for submitting <code>UploadDocuments</code> requests is domain-specific.
/// To get the document endpoint for your domain, use the Amazon CloudSearch configuration
/// service <code>DescribeDomains</code> action. A domain's endpoints are also displayed
/// on the domain dashboard in the Amazon CloudSearch console.
/// </para>
///
/// <para>
/// For more information about formatting your data for Amazon CloudSearch, see <a href="http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html">Preparing
/// Your Data</a> in the <i>Amazon CloudSearch Developer Guide</i>. For more information
/// about uploading data for indexing, see <a href="http://docs.aws.amazon.com/cloudsearch/latest/developerguide/uploading-data.html">Uploading
/// Data</a> in the <i>Amazon CloudSearch Developer Guide</i>.
/// </para>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the UploadDocuments service method.</param>
///
/// <returns>The response from the UploadDocuments service method, as returned by CloudSearchDomain.</returns>
/// <exception cref="Amazon.CloudSearchDomain.Model.DocumentServiceException">
/// Information about any problems encountered while processing an upload request.
/// </exception>
public UploadDocumentsResponse UploadDocuments(UploadDocumentsRequest request)
{
var marshaller = new UploadDocumentsRequestMarshaller();
var unmarshaller = UploadDocumentsResponseUnmarshaller.Instance;
return(Invoke <UploadDocumentsRequest, UploadDocumentsResponse>(request, marshaller, unmarshaller));
}
internal virtual UploadDocumentsResponse UploadDocuments(UploadDocumentsRequest request)
{
var marshaller = UploadDocumentsRequestMarshaller.Instance;
var unmarshaller = UploadDocumentsResponseUnmarshaller.Instance;
return(Invoke <UploadDocumentsRequest, UploadDocumentsResponse>(request, marshaller, unmarshaller));
}
public UploadDocumentsResponse DeleteSingleConnectRecordInAwsCloudsearch(int participantId, int pinType)
{
var cloudSearch = new AmazonCloudSearchDomainClient(AwsAccessKeyId, AwsSecretAccessKey, AmazonSearchUrl);
var results = SearchConnectAwsCloudsearch($"(and participantid:{participantId} pintype:{pinType})", "_no_fields");
var deletelist = new List <AwsCloudsearchDto>();
foreach (var hit in results.Hits.Hit)
{
var deleterec = new AwsCloudsearchDto
{
id = hit.Id,
type = "delete"
};
deletelist.Add(deleterec);
}
// serialize
var json = JsonConvert.SerializeObject(deletelist, new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
});
var ms = new MemoryStream(Encoding.UTF8.GetBytes(json));
var upload = new UploadDocumentsRequest()
{
ContentType = ContentType.ApplicationJson,
Documents = ms
};
return(cloudSearch.UploadDocuments(upload));
}
/// <summary>
/// This method uploads a set of documents onto your cloudsearch search domain.
/// Your objects must already have been converted to the cloudsearch upload format
/// for document batches (
/// </summary>
public bool UploadDocuments(List <AWSCloudSearchDocumentDocumentOperation> documents)
{
ClearErrorInfo();
try
{
UploadDocumentsRequest uploadRequest = new UploadDocumentsRequest();
uploadRequest.ContentType = "application/json";
uploadRequest.Documents = new MemoryStream(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(documents) ?? ""));
UploadDocumentsResponse uploadResponse = CloudSearchClient.UploadDocuments(uploadRequest);
// Check response for errors
if (uploadResponse.HttpStatusCode != HttpStatusCode.OK)
{
ErrorCode = Convert.ToInt32(uploadResponse.HttpStatusCode);
ErrorMessage = "Http Error [" + uploadResponse.HttpStatusCode.ToString() + "]";
}
}
catch (Exception ex)
{
ErrorCode = -1;
ErrorMessage = ex.Message + "::" + ex.InnerException;
}
return(ErrorCode == 0);
}
/// <summary>
/// Initiates the asynchronous execution of the UploadDocuments operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the UploadDocuments operation on AmazonCloudSearchDomainClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndUploadDocuments
/// operation.</returns>
public virtual IAsyncResult BeginUploadDocuments(UploadDocumentsRequest request, AsyncCallback callback, object state)
{
var marshaller = new UploadDocumentsRequestMarshaller();
var unmarshaller = UploadDocumentsResponseUnmarshaller.Instance;
return(BeginInvoke <UploadDocumentsRequest>(request, marshaller, unmarshaller,
callback, state));
}
/// <summary>
/// Initiates the asynchronous execution of the UploadDocuments operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the UploadDocuments operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public virtual Task <UploadDocumentsResponse> UploadDocumentsAsync(UploadDocumentsRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = UploadDocumentsRequestMarshaller.Instance;
var unmarshaller = UploadDocumentsResponseUnmarshaller.Instance;
return(InvokeAsync <UploadDocumentsRequest, UploadDocumentsResponse>(request, marshaller,
unmarshaller, cancellationToken));
}
/// <summary>
/// Posts a batch of documents to a search domain for indexing. A document batch is a
/// collection of add and delete operations that represent the documents you want to add,
/// update, or delete from your domain. Batches can be described in either JSON or XML.
/// Each item that you want Amazon CloudSearch to return as a search result (such as a
/// product) is represented as a document. Every document has a unique ID and one or more
/// fields that contain the data that you want to search and return in results. Individual
/// documents cannot contain more than 1 MB of data. The entire batch cannot exceed 5
/// MB. To get the best possible upload performance, group add and delete operations in
/// batches that are close the 5 MB limit. Submitting a large volume of single-document
/// batches can overload a domain's document service.
///
///
/// <para>
/// The endpoint for submitting <code>UploadDocuments</code> requests is domain-specific.
/// To get the document endpoint for your domain, use the Amazon CloudSearch configuration
/// service <code>DescribeDomains</code> action. A domain's endpoints are also displayed
/// on the domain dashboard in the Amazon CloudSearch console.
/// </para>
///
/// <para>
/// For more information about formatting your data for Amazon CloudSearch, see <a href="http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html">Preparing
/// Your Data</a> in the <i>Amazon CloudSearch Developer Guide</i>. For more information
/// about uploading data for indexing, see <a href="http://docs.aws.amazon.com/cloudsearch/latest/developerguide/uploading-data.html">Uploading
/// Data</a> in the <i>Amazon CloudSearch Developer Guide</i>.
/// </para>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the UploadDocuments service method.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
///
/// <returns>The response from the UploadDocuments service method, as returned by CloudSearchDomain.</returns>
/// <exception cref="Amazon.CloudSearchDomain.Model.DocumentServiceException">
/// Information about any problems encountered while processing an upload request.
/// </exception>
public virtual Task <UploadDocumentsResponse> UploadDocumentsAsync(UploadDocumentsRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var options = new InvokeOptions();
options.RequestMarshaller = UploadDocumentsRequestMarshaller.Instance;
options.ResponseUnmarshaller = UploadDocumentsResponseUnmarshaller.Instance;
return(InvokeAsync <UploadDocumentsResponse>(request, options, cancellationToken));
}
/// <summary>
/// Initiates the asynchronous execution of the UploadDocuments operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the UploadDocuments operation on AmazonCloudSearchDomainClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndUploadDocuments
/// operation.</returns>
public virtual IAsyncResult BeginUploadDocuments(UploadDocumentsRequest request, AsyncCallback callback, object state)
{
var options = new InvokeOptions();
options.RequestMarshaller = UploadDocumentsRequestMarshaller.Instance;
options.ResponseUnmarshaller = UploadDocumentsResponseUnmarshaller.Instance;
return(BeginInvoke(request, options, callback, state));
}
/// <summary>
/// Posts a batch of documents to a search domain for indexing. A document batch is a
/// collection of add and delete operations that represent the documents you want to add,
/// update, or delete from your domain. Batches can be described in either JSON or XML.
/// Each item that you want Amazon CloudSearch to return as a search result (such as a
/// product) is represented as a document. Every document has a unique ID and one or more
/// fields that contain the data that you want to search and return in results. Individual
/// documents cannot contain more than 1 MB of data. The entire batch cannot exceed 5
/// MB. To get the best possible upload performance, group add and delete operations in
/// batches that are close the 5 MB limit. Submitting a large volume of single-document
/// batches can overload a domain's document service.
///
///
/// <para>
/// The endpoint for submitting <code>UploadDocuments</code> requests is domain-specific.
/// To get the document endpoint for your domain, use the Amazon CloudSearch configuration
/// service <code>DescribeDomains</code> action. A domain's endpoints are also displayed
/// on the domain dashboard in the Amazon CloudSearch console.
/// </para>
///
/// <para>
/// For more information about formatting your data for Amazon CloudSearch, see <a href="http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html">Preparing
/// Your Data</a> in the <i>Amazon CloudSearch Developer Guide</i>. For more information
/// about uploading data for indexing, see <a href="http://docs.aws.amazon.com/cloudsearch/latest/developerguide/uploading-data.html">Uploading
/// Data</a> in the <i>Amazon CloudSearch Developer Guide</i>.
/// </para>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the UploadDocuments service method.</param>
///
/// <returns>The response from the UploadDocuments service method, as returned by CloudSearchDomain.</returns>
/// <exception cref="Amazon.CloudSearchDomain.Model.DocumentServiceException">
/// Information about any problems encountered while processing an upload request.
/// </exception>
public virtual UploadDocumentsResponse UploadDocuments(UploadDocumentsRequest request)
{
var options = new InvokeOptions();
options.RequestMarshaller = UploadDocumentsRequestMarshaller.Instance;
options.ResponseUnmarshaller = UploadDocumentsResponseUnmarshaller.Instance;
return(Invoke <UploadDocumentsResponse>(request, options));
}
internal UploadDocumentsResponse UploadDocuments(UploadDocumentsRequest request)
{
var task = UploadDocumentsAsync(request);
try
{
return(task.Result);
}
catch (AggregateException e)
{
ExceptionDispatchInfo.Capture(e.InnerException).Throw();
return(null);
}
}
public async Task <bool> UpdateIndex(IEnumerable <SearchEngineItem> items)
{
using (var memoryStream = new MemoryStream())
{
var jsonSerializer = new DataContractJsonSerializer(typeof(BatchDocumentItem[]));
var batch = items.Select(x => new BatchDocumentItem(x)).ToArray();
jsonSerializer.WriteObject(memoryStream, batch);
memoryStream.Position = 0; // reset cursor to able pass stream to AWS
var upload = new UploadDocumentsRequest
{
ContentType = ContentType.ApplicationJson,
Documents = memoryStream
};
var result = await documentClient.UploadDocumentsAsync(upload);
return(true); // todo handle response message from AWS.
}
}
public object Execute(ExecutorContext context)
{
var cmdletContext = context as CmdletContext;
// create request
var request = new UploadDocumentsRequest();
if (cmdletContext.ContentType != null)
{
request.ContentType = cmdletContext.ContentType;
}
if (cmdletContext.Documents != null)
{
request.Documents = cmdletContext.Documents;
}
if (cmdletContext.FilePath != null)
{
request.FilePath = cmdletContext.FilePath;
}
CmdletOutput output;
// issue call
using (var client = CreateClient(cmdletContext.ServiceUrl))
{
try
{
var response = CallAWSServiceOperation(client, request);
object pipelineOutput = cmdletContext.Select(response, this);
output = new CmdletOutput
{
PipelineOutput = pipelineOutput,
ServiceResponse = response
};
}
catch (Exception e)
{
output = new CmdletOutput {
ErrorResponse = e
};
}
return(output);
}
}
public UploadDocumentsResponse UploadAllConnectRecordsToAwsCloudsearch()
{
var cloudSearch = new AmazonCloudSearchDomainClient(AwsAccessKeyId, AwsSecretAccessKey, AmazonSearchUrl);
var pinList = GetDataForCloudsearch();
//serialize
var json = JsonConvert.SerializeObject(pinList, new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
});
var ms = new MemoryStream(Encoding.UTF8.GetBytes(json));
var upload = new UploadDocumentsRequest()
{
ContentType = ContentType.ApplicationJson,
Documents = ms
};
return(cloudSearch.UploadDocuments(upload));
}
public UploadDocumentsRequest GetObjectToUploadToAws(PinDto pin)
{
AwsConnectDto awsPinObject = Mapper.Map <AwsConnectDto>(pin);
if (pin.PinType == PinType.GATHERING)
{
awsPinObject.AddressId = pin.Gathering.Address.AddressID;
awsPinObject.Latitude = pin.Gathering.Address.Latitude;
awsPinObject.Longitude = pin.Gathering.Address.Longitude;
awsPinObject.City = pin.Gathering.Address.City;
awsPinObject.LatLong = (pin.Gathering.Address.Latitude == null || pin.Gathering.Address.Longitude == null)
? "0 , 0" : $"{pin.Gathering.Address.Latitude} , {pin.Gathering.Address.Longitude}";
awsPinObject.State = pin.Gathering.Address.State;
awsPinObject.Zip = pin.Gathering.Address.PostalCode;
awsPinObject.GroupStartDate = pin.Gathering.StartDate;
awsPinObject.GroupId = pin.Gathering.GroupId;
awsPinObject.GroupTypeId = pin.Gathering.GroupTypeId;
awsPinObject.GroupDescription = pin.Gathering.GroupDescription;
awsPinObject.GroupName = pin.Gathering.GroupName;
}
AwsCloudsearchDto awsPostPinObject = new AwsCloudsearchDto("add", GenerateAwsPinId(pin), awsPinObject);
var pinlist = new List <AwsCloudsearchDto> {
awsPostPinObject
};
string jsonAwsObject = JsonConvert.SerializeObject(pinlist, new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
});
MemoryStream jsonAwsPinDtoStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonAwsObject));
UploadDocumentsRequest upload = new UploadDocumentsRequest()
{
ContentType = ContentType.ApplicationJson,
Documents = jsonAwsPinDtoStream
};
return(upload);
}
public void Run()
{
var settings = new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
Formatting = Formatting.None,
};
using (var client = new AmazonCloudSearchDomainClient(_config.KeyId, _config.SecretKey, _config.ServiceUrl))
using (var batcher = new Batcher(_config.BatchSize, stream => UploadBatch(client, stream)))
{
foreach (var building in _reader.ReadBuildings(_config.BuildingsFilePath))
{
var addOperation = new AddOperation
{
Id = DocumentId.Build(building),
Fields = _mapper.Map <DocumentItem>(building),
};
var json = JsonConvert.SerializeObject(addOperation, settings);
batcher.Write(json);
}
foreach (var company in _reader.ReadCompanies(_config.CompaniesFilePath))
{
var addOperation = new AddOperation
{
Id = DocumentId.Build(company),
Fields = _mapper.Map <DocumentItem>(company),
};
var json = JsonConvert.SerializeObject(addOperation, settings);
batcher.Write(json);
}
}
void UploadBatch(IAmazonCloudSearchDomain client, Stream stream)
{
var request = new UploadDocumentsRequest();
request.ContentType = ContentType.ApplicationJson;
request.Documents = stream;
Logger.Info($"Uploading batch with {stream.Length} bytes");
var task = client.UploadDocumentsAsync(request);
if (!task.Wait(_config.Timeout))
{
Logger.Error("Operation timeout.");
return;
}
var response = task.Result;
Logger.Info($"Operation results: {response.Status}, " +
$"added {response.Adds} documents, " +
$"warnings: {response.Warnings.Count}");
foreach (var warning in response.Warnings)
{
Logger.Info($"Operation warning: {warning}");
}
}
}
/// <summary>
/// Posts a batch of documents to a search domain for indexing. A document batch is a
/// collection of add and delete operations that represent the documents you want to add,
/// update, or delete from your domain. Batches can be described in either JSON or XML.
/// Each item that you want Amazon CloudSearch to return as a search result (such as a
/// product) is represented as a document. Every document has a unique ID and one or more
/// fields that contain the data that you want to search and return in results. Individual
/// documents cannot contain more than 1 MB of data. The entire batch cannot exceed 5
/// MB. To get the best possible upload performance, group add and delete operations in
/// batches that are close the 5 MB limit. Submitting a large volume of single-document
/// batches can overload a domain's document service.
///
///
/// <para>
/// The endpoint for submitting <code>UploadDocuments</code> requests is domain-specific.
/// To get the document endpoint for your domain, use the Amazon CloudSearch configuration
/// service <code>DescribeDomains</code> action. A domain's endpoints are also displayed
/// on the domain dashboard in the Amazon CloudSearch console.
/// </para>
///
/// <para>
/// For more information about formatting your data for Amazon CloudSearch, see <a href="http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html">Preparing
/// Your Data</a> in the <i>Amazon CloudSearch Developer Guide</i>. For more information
/// about uploading data for indexing, see <a href="http://docs.aws.amazon.com/cloudsearch/latest/developerguide/uploading-data.html">Uploading
/// Data</a> in the <i>Amazon CloudSearch Developer Guide</i>.
/// </para>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the UploadDocuments service method.</param>
///
/// <returns>The response from the UploadDocuments service method, as returned by CloudSearchDomain.</returns>
/// <exception cref="DocumentServiceException">
/// Information about any problems encountered while processing an upload request.
/// </exception>
public UploadDocumentsResponse UploadDocuments(UploadDocumentsRequest request)
{
IAsyncResult asyncResult = invokeUploadDocuments(request, null, null, true);
return(EndUploadDocuments(asyncResult));
}
/// <summary>
/// Initiates the asynchronous execution of the UploadDocuments operation.
/// <seealso cref="Amazon.CloudSearchDomain.IAmazonCloudSearchDomain"/>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the UploadDocuments operation on AmazonCloudSearchDomainClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndUploadDocuments
/// operation.</returns>
public IAsyncResult BeginUploadDocuments(UploadDocumentsRequest request, AsyncCallback callback, object state)
{
return(invokeUploadDocuments(request, callback, state, false));
}
