static async Task UpsertSimpleExample(AmazonSQSExtendedClient client) { // index a simple document var simpleMessage = new { verb = "upsert", index = "test", document = new { id = "123456789", // ID managed by Umbraco site = "website", // as opposed to datahub|sac|mhc title = "An example searchable document", content = "This is a searchable document made purely for example purposes.", url = "http://example.com/pages/123456789", // the URL of the page, for clicking through keywords = new [] { new { vocab = "http://vocab.jncc.gov.uk/jncc-web", value = "Example" } }, published_date = "2019-01-14", } }; var basicResponse = await client.SendMessageAsync(Env.Var.SqsEndpoint, JsonConvert.SerializeObject(simpleMessage, Formatting.None) ); Console.WriteLine(basicResponse.MessageId); }
static async Task UpsertPdfExample(AmazonSQSExtendedClient client) { // index a PDF var pdf = File.ReadAllBytes(@"./OffshoreBrighton_SACO_V1.0.pdf"); var pdfEncoded = Convert.ToBase64String(pdf); var pdfMessage = new { verb = "upsert", index = "test", document = new { id = "987654321", site = "website", title = "An example PDF document", // content = "The content field will be set with the contents of the parsed file", url = "http://example.com/downloads/987654321.pdf", keywords = new [] { new { vocab = "http://vocab.jncc.gov.uk/jncc-web", value = "Example" } }, published_date = "2019-01-14", file_base64 = pdfEncoded, // base-64 encoded file file_extension = "pdf", // when this is a downloadable file_bytes = "1048576", // file such as a PDF, etc. } }; var pdfResponse = await client.SendMessageAsync(Env.Var.SqsEndpoint, JsonConvert.SerializeObject(pdfMessage, Formatting.None) ); Console.WriteLine(pdfResponse.MessageId); }
public async Task Long_Message_Async_It_Is_Not_Stored_In_S3_If_IsLargePayloadSupportEnabled_Is_False() { var extendedClient = new AmazonSQSExtendedClient(sqsMock.Object, new ExtendedClientConfiguration().WithLargePayloadSupportDisabled()); var body = GenerateLongString(SQSExtendedClientConstants.DEFAULT_MESSAGE_SIZE_THRESHOLD + 1); var messageRequest = new SendMessageRequest(SQS_QUEUE_NAME, body); await extendedClient.SendMessageAsync(messageRequest); s3Mock.Verify(s => s.PutObjectAsync(It.IsAny <PutObjectRequest>(), It.IsAny <CancellationToken>()), Times.Never); }
public async Task Short_Message_Async_It_Is_Stored_In_S3_If_AlwaysThroughS3_Configured() { var extendedClient = new AmazonSQSExtendedClient( sqsMock.Object, new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(s3Mock.Object, S3_BUCKET_NAME).WithAlwaysThroughS3(true)); var body = GenerateLongString(SQSExtendedClientConstants.DEFAULT_MESSAGE_SIZE_THRESHOLD + 1); var messageRequest = new SendMessageRequest(SQS_QUEUE_NAME, body); await extendedClient.SendMessageAsync(messageRequest); s3Mock.Verify(s => s.PutObjectAsync(It.IsAny <PutObjectRequest>(), It.IsAny <CancellationToken>()), Times.Once); sqsMock.Verify(s => s.SendMessageAsync(It.Is <SendMessageRequest>(r => MessagePointerIsCorrect(r.MessageBody) && LargePayloadAttributeIsAdded(r.MessageAttributes)), default(CancellationToken))); }
public async Task Short_Message_Async_It_Is_Stored_In_S3_If_Exceeds_Threshold() { var extendedClient = new AmazonSQSExtendedClient( sqsMock.Object, new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(s3Mock.Object, S3_BUCKET_NAME).WithMessageSizeThreshold(100)); var body = GenerateLongString(101); var messageRequest = new SendMessageRequest(SQS_QUEUE_NAME, body); await extendedClient.SendMessageAsync(messageRequest); s3Mock.Verify(s => s.PutObjectAsync(It.IsAny <PutObjectRequest>(), It.IsAny <CancellationToken>()), Times.Once); sqsMock.Verify(s => s.SendMessageAsync(It.Is <SendMessageRequest>(r => MessagePointerIsCorrect(r.MessageBody) && LargePayloadAttributeIsAdded(r.MessageAttributes)), default(CancellationToken)), Times.Once); }
private async Task QueueRequestAsync(SearchIndexQueueRequestModel request) { var message = JsonConvert.SerializeObject(request, Formatting.None, _jsonSettings); var response = await _sqsExtendedClient.SendMessageAsync(_searchConfiguration.AWSSQSEndpoint, message); if (response.HttpStatusCode != System.Net.HttpStatusCode.OK) { LogHelper.Warn <SearchIndexingQueueService>("[Failure] Document Request (ID: {0}, Title: {1}) has not been pushed up to SQS. Response HTTP Status Code: {2}. MD5 of message attributes: {3}. MD5 of message body: {4}.", () => request.Document.NodeId, () => request.Document.Title, () => response.HttpStatusCode, () => response.MD5OfMessageAttributes, () => response.MD5OfMessageBody); } else { LogHelper.Info <SearchIndexingQueueService>("[Success] Document Request (ID: {0}, Title: {1}) has been pushed up to SQS.", () => request.Document.NodeId, () => request.Document.Title); } }
static async Task UpsertDatahubAssetExample(AmazonSQSExtendedClient client) { // index an asset with several large PDF resources var pdf = File.ReadAllBytes(@"../Datahub.Web/data/OffshoreBrighton_SACO_V1.0.pdf"); var pdfEncoded = Convert.ToBase64String(pdf); var resources = from i in Enumerable.Range(1, 50) select new { title = "An example searchable PDF resource #" + i, url = "http://example.com/pages/123456789", // the URL of the page, for clicking through keywords = new [] { new { vocab = "http://vocab.jncc.gov.uk/jncc-web", value = "Example" } }, file_base64 = pdfEncoded, // base-64 encoded file file_extension = "pdf", // when this is a downloadable file_bytes = "1048576", // file such as a PDF, etc. published_date = "2019-02-15", }; var bigMessage = new { verb = "upsert", index = "dev", document = new { id = "123456789", site = "datahub", // as opposed to website|sac|mhc title = "An example searchable document with resources :-)", content = "This is a searchable document made purely for example purposes.", url = "http://example.com/pages/123456789", // the URL of the page, for clicking through keywords = new [] { new { vocab = "http://vocab.jncc.gov.uk/jncc-web", value = "Example" } }, published_date = "2019-01-14", }, resources = resources.ToArray(), }; var basicResponse = await client.SendMessageAsync(Env.Var.SqsEndpoint, JsonConvert.SerializeObject(bigMessage, Formatting.None) ); Console.WriteLine(basicResponse.MessageId); }
public async Task Long_Message_Async_S3KeyProvider_Is_Used_If_Configured() { var mockS3Provider = new Mock <IS3KeyProvider>(); mockS3Provider.Setup(m => m.GenerateName()).Returns("CustomPrefix" + Guid.NewGuid().ToString("N")); var extendedClient = new AmazonSQSExtendedClient( sqsMock.Object, new ExtendedClientConfiguration() .WithLargePayloadSupportEnabled(s3Mock.Object, S3_BUCKET_NAME) .WithS3KeyProvider(mockS3Provider.Object)); var body = GenerateLongString(SQSExtendedClientConstants.DEFAULT_MESSAGE_SIZE_THRESHOLD + 1); var messageRequest = new SendMessageRequest(SQS_QUEUE_NAME, body); await extendedClient.SendMessageAsync(messageRequest); mockS3Provider.Verify(s => s.GenerateName(), Times.Once); s3Mock.Verify(s => s.PutObjectAsync(It.Is <PutObjectRequest>(r => r.Key.StartsWith("CustomPrefix")), It.IsAny <CancellationToken>()), Times.Once); sqsMock.Verify(s => s.SendMessageAsync(It.Is <SendMessageRequest>(r => MessagePointerIsCorrect(r.MessageBody, "CustomPrefix") && LargePayloadAttributeIsAdded(r.MessageAttributes)), It.IsAny <CancellationToken>()), Times.Once); }
static async Task DeleteExample(AmazonSQSExtendedClient client) { var deleteMessage = new { verb = "delete", index = "test", document = new { id = "123456789", site = "website", } }; var deleteResponse = await client.SendMessageAsync(Env.Var.SqsEndpoint, JsonConvert.SerializeObject(deleteMessage, Formatting.None) ); Console.WriteLine(deleteResponse.MessageId); }
static void ProcessAssetLists() { Console.WriteLine("sqs endpoint: {0}", Env.Var.SqsEndpoint); Console.WriteLine("sqs s3 bucket", Env.Var.SqsPayloadBucket); int errors = 0; using (var s3 = GetS3Client()) using (var sqs = GetSQSClient()) using (var sqsExtendedClient = new AmazonSQSExtendedClient(sqs, new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(s3, Env.Var.SqsPayloadBucket) )) { foreach (var assetList in Env.Var.AssetLists) { Console.WriteLine("Processing {0}", assetList.Url); List <Asset> assets; try { assets = GetAssetList(assetList.Url); } catch (Exception e) { Console.WriteLine("Unable to get asset list from {0}. Error: {1}", assetList.Url, e.Message); errors++; continue; } for (int i = 0; i < assets.Count; i++) { var asset = assets[i]; var assetId = String.Format("{0}-{1}", assetList.IdPrefix, asset.Id); Console.WriteLine("Processing Asset {0}, {1} {2}", i + 1, assetId, asset.Title); if (AssetValidator.IsValid(asset, errors)) { var assetFileUrl = GetFileUrl(assetList.Url, asset.FileName); AssetFile file; try { file = GetAssetFile(assetFileUrl, asset); } catch (Exception e) { Console.WriteLine("Unable to get asset file from {0}. Error: {1}", assetFileUrl, e.Message); errors++; continue; } var culture = CultureInfo.CreateSpecificCulture("en-GB"); var style = DateTimeStyles.None; var message = new { verb = "upsert", index = Env.Var.EsIndex, document = new { id = assetId, site = Env.Var.EsSite, title = asset.Title, url = file.Url, published_date = DateTime.Parse(asset.PublicationDate, culture, style).ToString("yyyy-MM-dd"), file_base64 = file.EncodedFile, // base-64 encoded file file_extension = file.Extension, // when this is a downloadable file_bytes = file.Bytes.ToString(), // file such as a PDF, etc. } }; var response = sqsExtendedClient.SendMessageAsync(Env.Var.SqsEndpoint, JsonConvert.SerializeObject(message, Formatting.None) ).GetAwaiter().GetResult(); Console.WriteLine("Created Message Id {0}", response.MessageId); if (Env.Var.AssetQueryDelay > 0) { Console.WriteLine("Waiting {0}ms before getting next asset", Env.Var.AssetQueryDelay); Thread.Sleep(Env.Var.AssetQueryDelay); } } } } ; } if (errors > 0) { Console.WriteLine("{0} errors occured during processing", errors); } else { Console.WriteLine("Indexing Complete, no errors"); } }
public static async Task <SendMessageResponse> SendMessage(AmazonSQSExtendedClient client, string endpoint, string message) { return(await client.SendMessageAsync(endpoint, message)); }