private async Task <List <Product> > ReadS3Message(S3EventNotification.S3EventNotificationRecord notification) { var result = new List <Product>(); try { var client = new AmazonS3Client(RegionEndpoint.USEast1); var request = new GetObjectRequest { BucketName = notification.S3.Bucket.Name, Key = notification.S3.Object.Key }; using (GetObjectResponse response = await client.GetObjectAsync(request)) using (var responseStream = response.ResponseStream) using (var reader = new StreamReader(responseStream)) { var responseBody = reader.ReadToEnd(); result = ParseCSV(responseBody); } } catch (AmazonS3Exception e) { Console.WriteLine("Error encountered ***. Message:'{0}' when writing an object", e.Message); } catch (Exception e) { Console.WriteLine("Unknown encountered on server. Message:'{0}' when writing an object", e.Message); } return(result); }
private GetObjectResponse GetObjectFromS3(S3EventNotification.S3EventNotificationRecord s3Record) { var request = new GetObjectRequest { BucketName = s3Record.S3.Bucket.Name, Key = s3Record.S3.Object.Key }; return(_s3.GetObjectAsync(request).Result); }
private async Task PutResizedObjectAsync(S3EventNotification.S3EventNotificationRecord record, Stream imageStream) => await S3Client.PutObjectAsync(new PutObjectRequest { BucketName = record.S3.Bucket.Name, Key = $"{Path.GetFileNameWithoutExtension(record.S3.Object.Key)}_thumbnail{Path.GetExtension(record.S3.Object.Key)}", InputStream = imageStream, TagSet = new List <Tag> { new Tag { Key = "Compressed", Value = "true" } } });
public void CanLoadACsvIntoTheDatabase() { var mockDatabaseActions = new Mock <IDatabaseActions>(); var handler = new Handler(mockDatabaseActions.Object); var tableName = "test"; Environment.SetEnvironmentVariable("DB_TABLE_NAME", tableName); CreateTable("test"); var bucketData = new S3EventNotification.S3Entity { Bucket = new S3EventNotification.S3BucketEntity { Name = "testBucket" }, Object = new S3EventNotification.S3ObjectEntity { Key = "test/key.csv" } }; //S3 record mock var testRecord = new S3EventNotification.S3EventNotificationRecord(); testRecord.AwsRegion = "eu-west-2"; testRecord.S3 = bucketData; var s3EventMock = new S3EventNotification(); s3EventMock.Records = new List <S3EventNotification.S3EventNotificationRecord> { testRecord }; var contextMock = new Mock <ILambdaContext>(); //set up Database actions mockDatabaseActions.Setup(x => x.CopyDataToDatabase(tableName, contextMock.Object, testRecord.AwsRegion, bucketData.Bucket.Name, bucketData.Object.Key)); mockDatabaseActions.Setup(x => x.AddExtension(contextMock.Object)); mockDatabaseActions.Setup(x => x.CreateTable(contextMock.Object, It.IsAny <string>())); mockDatabaseActions.Setup(x => x.TruncateTable(contextMock.Object, It.IsAny <string>())); mockDatabaseActions.Setup(x => x.SetupDatabase(contextMock.Object)).Returns(() => new NpgsqlConnection()); Assert.DoesNotThrow(() => handler.LoadCsv(s3EventMock, contextMock.Object)); mockDatabaseActions.Verify(y => y.SetupDatabase(contextMock.Object), Times.Once); mockDatabaseActions.Verify(y => y.AddExtension(contextMock.Object), Times.Once); mockDatabaseActions.Verify(y => y.TruncateTable(contextMock.Object, It.IsAny <string>()), Times.Once); mockDatabaseActions.Verify(y => y.CreateTable(contextMock.Object, It.IsAny <string>()), Times.Once); mockDatabaseActions.Verify(y => y.CopyDataToDatabase(tableName, contextMock.Object, testRecord.AwsRegion, bucketData.Bucket.Name, bucketData.Object.Key), Times.Once); }
private async Task <Stream> ResizeImageAsync(S3EventNotification.S3EventNotificationRecord record) { var imageSize = int.Parse(Environment.GetEnvironmentVariable("THUMBNAIL_SIZE")); Stream imageStream = new MemoryStream(); using (var objectResponse = await S3Client.GetObjectAsync(record.S3.Bucket.Name, record.S3.Object.Key)) using (Stream responseStream = objectResponse.ResponseStream) { using (Image <Rgba32> image = Image.Load(responseStream)) { image.Mutate(ctx => ctx.Resize(imageSize, imageSize)); image.Save(imageStream, new JpegEncoder()); imageStream.Seek(0, SeekOrigin.Begin); } } return(imageStream); }
private async Task <List <Product> > ReadS3MessageLineByLine(S3EventNotification.S3EventNotificationRecord notification) { var result = new List <Product>(); try { var client = new AmazonS3Client(RegionEndpoint.USEast1); var request = new GetObjectRequest { BucketName = notification.S3.Bucket.Name, Key = notification.S3.Object.Key }; using (GetObjectResponse response = await client.GetObjectAsync(request)) using (CsvReader reader = new CsvReader(ProcessRowFromFile)) { reader.DelimitedFileProcessed += (sender, args) => { Console.WriteLine($"Processing file complete"); }; await reader.ProcessDelimted(response.ResponseStream, 5000); } } catch (AmazonS3Exception e) { Console.WriteLine("Error encountered ***. Message:'{0}' when writing an object", e.Message); } catch (Exception e) { Console.WriteLine("Unknown encountered on server. Message:'{0}' when writing an object", e.Message); } return(result); }
protected abstract Task ProcessMessage(S3EventNotification.S3EventNotificationRecord message);
private async Task <EmailMessageInfo> CreateEmailMessageAsync(string requestId, string messageId, S3EventNotification.S3EventNotificationRecord record) { Stopwatch stopwatch = Stopwatch.StartNew(); GetObjectResponse response = await _s3Client.GetObjectAsync(record.S3.Bucket.Name, record.S3.Object.Key) .TimeoutAfter(_config.TimeoutS3) .ConfigureAwait(false); _log.Debug($"Retrieving aggregate report from bucket took {stopwatch.Elapsed}"); stopwatch.Stop(); string originalUri = $"{record.S3.Bucket.Name}/{record.S3.Object.Key}"; return(new EmailMessageInfo(new EmailMetadata(requestId, messageId, originalUri, record.S3.Object.Key, record.S3.Object.Size / 1024), response.ResponseStream)); }
public async Task ProcessRecord(S3EventNotification.S3EventNotificationRecord record, ILambdaContext context) { var cameraKey = record.S3.Object.Key.Split('/')[1]; var s3GetResult = await s3.GetObjectAsync(record.S3.Bucket.Name, record.S3.Object.Key); var classNamesParameterName = $"/Cameras/{cameraKey}/ClassNames"; var sceneCodeParameterName = $"/Cameras/{cameraKey}/SceneCode"; var observationBoundingBoxParameterName = $"/Cameras/{cameraKey}/ObservationBoundingBox"; if (!cameraParameters.ContainsKey(observationBoundingBoxParameterName)) { try { var getResult = await ssm.GetParameterAsync(new GetParameterRequest { Name = observationBoundingBoxParameterName }); cameraParameters.Add(observationBoundingBoxParameterName, getResult.Parameter.Value); context.Logger.LogLine( $"Set {observationBoundingBoxParameterName} = {cameraParameters[observationBoundingBoxParameterName]}"); } catch (Exception exception) { context.Logger.LogLine($"Didn't add parameter. {observationBoundingBoxParameterName}"); context.Logger.LogLine(exception.Message); } } if (!cameraParameters.ContainsKey(classNamesParameterName)) { var getResult = await ssm.GetParameterAsync(new GetParameterRequest { Name = classNamesParameterName }); cameraParameters.Add(classNamesParameterName, getResult.Parameter.Value); context.Logger.LogLine($"Set {classNamesParameterName} = {cameraParameters[classNamesParameterName]}"); } if (!cameraParameters.ContainsKey(sceneCodeParameterName)) { var getResult = await ssm.GetParameterAsync(new GetParameterRequest { Name = sceneCodeParameterName }); cameraParameters.Add(sceneCodeParameterName, getResult.Parameter.Value); context.Logger.LogLine($"Set {sceneCodeParameterName} = {cameraParameters[sceneCodeParameterName]}"); } var memoryStream = new MemoryStream(); await s3GetResult.ResponseStream.CopyToAsync(memoryStream); // Crop the area of interest. // TODO: Get the x, y, w, h from parmeter store. var croppedMemoryStream = new MemoryStream(); var cropImage = cameraParameters.ContainsKey(observationBoundingBoxParameterName); if (cropImage) { var parts = cameraParameters[observationBoundingBoxParameterName].Split(','); memoryStream.Position = 0; var sourceImage = Image.Load(memoryStream); var x = int.Parse(parts[0]); var y = int.Parse(parts[1]); var lowerX = int.Parse(parts[2]); var lowerY = int.Parse(parts[3]); var w = lowerX - x; var h = lowerY - y; sourceImage.Mutate(i => i.Crop(new Rectangle(x, y, w, h))); context.Logger.LogLine("Trying to save croped image."); sourceImage.Save(croppedMemoryStream, new JpegEncoder()); croppedMemoryStream.Position = 0; } var labelsResult = await rekognition.DetectLabelsAsync(new DetectLabelsRequest { Image = new Amazon.Rekognition.Model.Image { Bytes = cropImage ? croppedMemoryStream : memoryStream }, MaxLabels = 100, MinConfidence = 70 }); if (cropImage) { croppedMemoryStream.Position = 0; } else { memoryStream.Position = 0; } var metricData = new List <MetricDatum>(); var personMetric = new MetricDatum { MetricName = "Confidence", StorageResolution = 1, TimestampUtc = DateTime.UtcNow, Unit = StandardUnit.Percent, Dimensions = new List <Dimension> { new Dimension { Name = "CameraKey", Value = cameraKey }, new Dimension { Name = "Source", Value = "Rekognition" }, new Dimension { Name = "Label", Value = "Person" } } }; if (labelsResult.Labels.Any(label => label.Name == "Person")) { var confidence = Convert.ToDouble(labelsResult.Labels.Single(l => l.Name == "Person").Confidence); personMetric.StatisticValues = new StatisticSet { Minimum = confidence, Maximum = confidence, SampleCount = 1, Sum = 1 }; } else { personMetric.StatisticValues = new StatisticSet { Minimum = 0, Maximum = 0, SampleCount = 1, Sum = 1 }; } metricData.Add(personMetric); var objectDetectionResult = await sageMakerRuntime.InvokeEndpointAsync(new InvokeEndpointRequest { Accept = "application/jsonlines", ContentType = "application/x-image", EndpointName = cameraParameters[sceneCodeParameterName], Body = cropImage ? croppedMemoryStream : memoryStream }); if (cropImage) { croppedMemoryStream.Close(); } else { memoryStream.Close(); } using (var streamReader = new StreamReader(objectDetectionResult.Body)) { var json = streamReader.ReadToEnd(); context.Logger.Log($"SageMaker Endpoint Result: {json}"); var predictionResult = JsonConvert.DeserializeObject <dynamic>(json).prediction; var classNames = cameraParameters[classNamesParameterName].Split(','); var predictions = new List <Prediction>(); foreach (var pr in predictionResult) { predictions.Add(new Prediction { ClassName = classNames[Convert.ToInt32(pr[0].Value)], Confidence = Convert.ToDouble(pr[1].Value) * 100 }); } foreach (var classNotPredicted in classNames.Where(cn => predictions.All(p => p.ClassName != cn))) { predictions.Add(new Prediction { ClassName = classNotPredicted, Confidence = 0 }); } foreach (var classGroup in predictions.GroupBy(p => p.ClassName)) { metricData.Add(new MetricDatum { MetricName = "Confidence", StorageResolution = 1, TimestampUtc = DateTime.UtcNow, Unit = StandardUnit.Percent, Dimensions = new List <Dimension> { new Dimension { Name = "CameraKey", Value = cameraKey }, new Dimension { Name = "Source", Value = cameraParameters[sceneCodeParameterName] }, new Dimension { Name = "Label", Value = classGroup.Key } }, StatisticValues = new StatisticSet { Minimum = classGroup.Min(c => c.Confidence), Maximum = classGroup.Max(c => c.Confidence), Sum = classGroup.Count(), SampleCount = classGroup.Count() } }); } } await cloudWatch.PutMetricDataAsync(new PutMetricDataRequest { Namespace = "Cameras", MetricData = metricData }); }
private async Task <bool> CheckCompressedTagAsync(S3EventNotification.S3EventNotificationRecord record) => (await S3Client.GetObjectTaggingAsync(new GetObjectTaggingRequest { BucketName = record.S3.Bucket.Name, Key = record.S3.Object.Key })).Tagging.Any(tag => tag.Key == "Compressed" && tag.Value == "true");