static async Task UpsertSimpleExample(AmazonSQSExtendedClient client)
{
// index a simple document
var simpleMessage = new
{
verb = "upsert",
index = "test",
document = new
{
id = "123456789", // ID managed by Umbraco
site = "website", // as opposed to datahub|sac|mhc
title = "An example searchable document",
content = "This is a searchable document made purely for example purposes.",
url = "http://example.com/pages/123456789", // the URL of the page, for clicking through
keywords = new []
{
new { vocab = "http://vocab.jncc.gov.uk/jncc-web", value = "Example" }
},
published_date = "2019-01-14",
}
};
var basicResponse = await client.SendMessageAsync(Env.Var.SqsEndpoint,
JsonConvert.SerializeObject(simpleMessage, Formatting.None)
);
Console.WriteLine(basicResponse.MessageId);
}
static async Task UpsertPdfExample(AmazonSQSExtendedClient client)
{
// index a PDF
var pdf = File.ReadAllBytes(@"./OffshoreBrighton_SACO_V1.0.pdf");
var pdfEncoded = Convert.ToBase64String(pdf);
var pdfMessage = new
{
verb = "upsert",
index = "test",
document = new
{
id = "987654321",
site = "website",
title = "An example PDF document",
// content = "The content field will be set with the contents of the parsed file",
url = "http://example.com/downloads/987654321.pdf",
keywords = new []
{
new { vocab = "http://vocab.jncc.gov.uk/jncc-web", value = "Example" }
},
published_date = "2019-01-14",
file_base64 = pdfEncoded, // base-64 encoded file
file_extension = "pdf", // when this is a downloadable
file_bytes = "1048576", // file such as a PDF, etc.
}
};
var pdfResponse = await client.SendMessageAsync(Env.Var.SqsEndpoint,
JsonConvert.SerializeObject(pdfMessage, Formatting.None)
);
Console.WriteLine(pdfResponse.MessageId);
}
public async Task Long_Message_Async_It_Is_Not_Stored_In_S3_If_IsLargePayloadSupportEnabled_Is_False()
{
var extendedClient = new AmazonSQSExtendedClient(sqsMock.Object, new ExtendedClientConfiguration().WithLargePayloadSupportDisabled());
var body = GenerateLongString(SQSExtendedClientConstants.DEFAULT_MESSAGE_SIZE_THRESHOLD + 1);
var messageRequest = new SendMessageRequest(SQS_QUEUE_NAME, body);
await extendedClient.SendMessageAsync(messageRequest);
s3Mock.Verify(s => s.PutObjectAsync(It.IsAny <PutObjectRequest>(), It.IsAny <CancellationToken>()), Times.Never);
}
public async Task Short_Message_Async_It_Is_Stored_In_S3_If_AlwaysThroughS3_Configured()
{
var extendedClient = new AmazonSQSExtendedClient(
sqsMock.Object,
new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(s3Mock.Object, S3_BUCKET_NAME).WithAlwaysThroughS3(true));
var body = GenerateLongString(SQSExtendedClientConstants.DEFAULT_MESSAGE_SIZE_THRESHOLD + 1);
var messageRequest = new SendMessageRequest(SQS_QUEUE_NAME, body);
await extendedClient.SendMessageAsync(messageRequest);
s3Mock.Verify(s => s.PutObjectAsync(It.IsAny <PutObjectRequest>(), It.IsAny <CancellationToken>()), Times.Once);
sqsMock.Verify(s => s.SendMessageAsync(It.Is <SendMessageRequest>(r => MessagePointerIsCorrect(r.MessageBody) && LargePayloadAttributeIsAdded(r.MessageAttributes)), default(CancellationToken)));
}
public async Task Short_Message_Async_It_Is_Stored_In_S3_If_Exceeds_Threshold()
{
var extendedClient = new AmazonSQSExtendedClient(
sqsMock.Object,
new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(s3Mock.Object, S3_BUCKET_NAME).WithMessageSizeThreshold(100));
var body = GenerateLongString(101);
var messageRequest = new SendMessageRequest(SQS_QUEUE_NAME, body);
await extendedClient.SendMessageAsync(messageRequest);
s3Mock.Verify(s => s.PutObjectAsync(It.IsAny <PutObjectRequest>(), It.IsAny <CancellationToken>()), Times.Once);
sqsMock.Verify(s => s.SendMessageAsync(It.Is <SendMessageRequest>(r => MessagePointerIsCorrect(r.MessageBody) && LargePayloadAttributeIsAdded(r.MessageAttributes)), default(CancellationToken)), Times.Once);
}
private async Task QueueRequestAsync(SearchIndexQueueRequestModel request)
{
var message = JsonConvert.SerializeObject(request, Formatting.None, _jsonSettings);
var response = await _sqsExtendedClient.SendMessageAsync(_searchConfiguration.AWSSQSEndpoint, message);
if (response.HttpStatusCode != System.Net.HttpStatusCode.OK)
{
LogHelper.Warn <SearchIndexingQueueService>("[Failure] Document Request (ID: {0}, Title: {1}) has not been pushed up to SQS. Response HTTP Status Code: {2}. MD5 of message attributes: {3}. MD5 of message body: {4}.", () => request.Document.NodeId, () => request.Document.Title, () => response.HttpStatusCode, () => response.MD5OfMessageAttributes, () => response.MD5OfMessageBody);
}
else
{
LogHelper.Info <SearchIndexingQueueService>("[Success] Document Request (ID: {0}, Title: {1}) has been pushed up to SQS.", () => request.Document.NodeId, () => request.Document.Title);
}
}
static async Task UpsertDatahubAssetExample(AmazonSQSExtendedClient client)
{
// index an asset with several large PDF resources
var pdf = File.ReadAllBytes(@"../Datahub.Web/data/OffshoreBrighton_SACO_V1.0.pdf");
var pdfEncoded = Convert.ToBase64String(pdf);
var resources = from i in Enumerable.Range(1, 50)
select new
{
title = "An example searchable PDF resource #" + i,
url = "http://example.com/pages/123456789", // the URL of the page, for clicking through
keywords = new []
{
new { vocab = "http://vocab.jncc.gov.uk/jncc-web", value = "Example" }
},
file_base64 = pdfEncoded, // base-64 encoded file
file_extension = "pdf", // when this is a downloadable
file_bytes = "1048576", // file such as a PDF, etc.
published_date = "2019-02-15",
};
var bigMessage = new
{
verb = "upsert",
index = "dev",
document = new
{
id = "123456789",
site = "datahub", // as opposed to website|sac|mhc
title = "An example searchable document with resources :-)",
content = "This is a searchable document made purely for example purposes.",
url = "http://example.com/pages/123456789", // the URL of the page, for clicking through
keywords = new []
{
new { vocab = "http://vocab.jncc.gov.uk/jncc-web", value = "Example" }
},
published_date = "2019-01-14",
},
resources = resources.ToArray(),
};
var basicResponse = await client.SendMessageAsync(Env.Var.SqsEndpoint,
JsonConvert.SerializeObject(bigMessage, Formatting.None)
);
Console.WriteLine(basicResponse.MessageId);
}
public async Task Long_Message_Async_S3KeyProvider_Is_Used_If_Configured()
{
var mockS3Provider = new Mock <IS3KeyProvider>();
mockS3Provider.Setup(m => m.GenerateName()).Returns("CustomPrefix" + Guid.NewGuid().ToString("N"));
var extendedClient = new AmazonSQSExtendedClient(
sqsMock.Object,
new ExtendedClientConfiguration()
.WithLargePayloadSupportEnabled(s3Mock.Object, S3_BUCKET_NAME)
.WithS3KeyProvider(mockS3Provider.Object));
var body = GenerateLongString(SQSExtendedClientConstants.DEFAULT_MESSAGE_SIZE_THRESHOLD + 1);
var messageRequest = new SendMessageRequest(SQS_QUEUE_NAME, body);
await extendedClient.SendMessageAsync(messageRequest);
mockS3Provider.Verify(s => s.GenerateName(), Times.Once);
s3Mock.Verify(s => s.PutObjectAsync(It.Is <PutObjectRequest>(r => r.Key.StartsWith("CustomPrefix")), It.IsAny <CancellationToken>()), Times.Once);
sqsMock.Verify(s => s.SendMessageAsync(It.Is <SendMessageRequest>(r => MessagePointerIsCorrect(r.MessageBody, "CustomPrefix") && LargePayloadAttributeIsAdded(r.MessageAttributes)), It.IsAny <CancellationToken>()), Times.Once);
}
static async Task DeleteExample(AmazonSQSExtendedClient client)
{
var deleteMessage = new
{
verb = "delete",
index = "test",
document = new
{
id = "123456789",
site = "website",
}
};
var deleteResponse = await client.SendMessageAsync(Env.Var.SqsEndpoint,
JsonConvert.SerializeObject(deleteMessage, Formatting.None)
);
Console.WriteLine(deleteResponse.MessageId);
}
static void ProcessAssetLists()
{
Console.WriteLine("sqs endpoint: {0}", Env.Var.SqsEndpoint);
Console.WriteLine("sqs s3 bucket", Env.Var.SqsPayloadBucket);
int errors = 0;
using (var s3 = GetS3Client())
using (var sqs = GetSQSClient())
using (var sqsExtendedClient = new AmazonSQSExtendedClient(sqs,
new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(s3, Env.Var.SqsPayloadBucket)
))
{
foreach (var assetList in Env.Var.AssetLists)
{
Console.WriteLine("Processing {0}", assetList.Url);
List <Asset> assets;
try
{
assets = GetAssetList(assetList.Url);
}
catch (Exception e)
{
Console.WriteLine("Unable to get asset list from {0}. Error: {1}", assetList.Url, e.Message);
errors++;
continue;
}
for (int i = 0; i < assets.Count; i++)
{
var asset = assets[i];
var assetId = String.Format("{0}-{1}", assetList.IdPrefix, asset.Id);
Console.WriteLine("Processing Asset {0}, {1} {2}", i + 1, assetId, asset.Title);
if (AssetValidator.IsValid(asset, errors))
{
var assetFileUrl = GetFileUrl(assetList.Url, asset.FileName);
AssetFile file;
try
{
file = GetAssetFile(assetFileUrl, asset);
}
catch (Exception e)
{
Console.WriteLine("Unable to get asset file from {0}. Error: {1}", assetFileUrl, e.Message);
errors++;
continue;
}
var culture = CultureInfo.CreateSpecificCulture("en-GB");
var style = DateTimeStyles.None;
var message = new
{
verb = "upsert",
index = Env.Var.EsIndex,
document = new
{
id = assetId,
site = Env.Var.EsSite,
title = asset.Title,
url = file.Url,
published_date = DateTime.Parse(asset.PublicationDate, culture, style).ToString("yyyy-MM-dd"),
file_base64 = file.EncodedFile, // base-64 encoded file
file_extension = file.Extension, // when this is a downloadable
file_bytes = file.Bytes.ToString(), // file such as a PDF, etc.
}
};
var response = sqsExtendedClient.SendMessageAsync(Env.Var.SqsEndpoint,
JsonConvert.SerializeObject(message, Formatting.None)
).GetAwaiter().GetResult();
Console.WriteLine("Created Message Id {0}", response.MessageId);
if (Env.Var.AssetQueryDelay > 0)
{
Console.WriteLine("Waiting {0}ms before getting next asset", Env.Var.AssetQueryDelay);
Thread.Sleep(Env.Var.AssetQueryDelay);
}
}
}
}
;
}
if (errors > 0)
{
Console.WriteLine("{0} errors occured during processing", errors);
}
else
{
Console.WriteLine("Indexing Complete, no errors");
}
}
public static async Task <SendMessageResponse> SendMessage(AmazonSQSExtendedClient client, string endpoint, string message)
{
return(await client.SendMessageAsync(endpoint, message));
}