private async Task <List <Product> > ReadS3Message(S3EventNotification.S3EventNotificationRecord notification)
{
var result = new List <Product>();
try
{
var client = new AmazonS3Client(RegionEndpoint.USEast1);
var request = new GetObjectRequest
{
BucketName = notification.S3.Bucket.Name,
Key = notification.S3.Object.Key
};
using (GetObjectResponse response = await client.GetObjectAsync(request))
using (var responseStream = response.ResponseStream)
using (var reader = new StreamReader(responseStream))
{
var responseBody = reader.ReadToEnd();
result = ParseCSV(responseBody);
}
}
catch (AmazonS3Exception e)
{
Console.WriteLine("Error encountered ***. Message:'{0}' when writing an object", e.Message);
}
catch (Exception e)
{
Console.WriteLine("Unknown encountered on server. Message:'{0}' when writing an object", e.Message);
}
return(result);
}
private GetObjectResponse GetObjectFromS3(S3EventNotification.S3EventNotificationRecord s3Record)
{
var request = new GetObjectRequest
{
BucketName = s3Record.S3.Bucket.Name,
Key = s3Record.S3.Object.Key
};
return(_s3.GetObjectAsync(request).Result);
}
private async Task PutResizedObjectAsync(S3EventNotification.S3EventNotificationRecord record, Stream imageStream) =>
await S3Client.PutObjectAsync(new PutObjectRequest
{
BucketName = record.S3.Bucket.Name,
Key = $"{Path.GetFileNameWithoutExtension(record.S3.Object.Key)}_thumbnail{Path.GetExtension(record.S3.Object.Key)}",
InputStream = imageStream,
TagSet = new List <Tag>
{
new Tag
{
Key = "Compressed",
Value = "true"
}
}
});
public void CanLoadACsvIntoTheDatabase()
{
var mockDatabaseActions = new Mock <IDatabaseActions>();
var handler = new Handler(mockDatabaseActions.Object);
var tableName = "test";
Environment.SetEnvironmentVariable("DB_TABLE_NAME", tableName);
CreateTable("test");
var bucketData = new S3EventNotification.S3Entity
{
Bucket = new S3EventNotification.S3BucketEntity {
Name = "testBucket"
},
Object = new S3EventNotification.S3ObjectEntity {
Key = "test/key.csv"
}
};
//S3 record mock
var testRecord = new S3EventNotification.S3EventNotificationRecord();
testRecord.AwsRegion = "eu-west-2";
testRecord.S3 = bucketData;
var s3EventMock = new S3EventNotification();
s3EventMock.Records = new List <S3EventNotification.S3EventNotificationRecord> {
testRecord
};
var contextMock = new Mock <ILambdaContext>();
//set up Database actions
mockDatabaseActions.Setup(x => x.CopyDataToDatabase(tableName, contextMock.Object, testRecord.AwsRegion, bucketData.Bucket.Name, bucketData.Object.Key));
mockDatabaseActions.Setup(x => x.AddExtension(contextMock.Object));
mockDatabaseActions.Setup(x => x.CreateTable(contextMock.Object, It.IsAny <string>()));
mockDatabaseActions.Setup(x => x.TruncateTable(contextMock.Object, It.IsAny <string>()));
mockDatabaseActions.Setup(x => x.SetupDatabase(contextMock.Object)).Returns(() => new NpgsqlConnection());
Assert.DoesNotThrow(() => handler.LoadCsv(s3EventMock, contextMock.Object));
mockDatabaseActions.Verify(y => y.SetupDatabase(contextMock.Object), Times.Once);
mockDatabaseActions.Verify(y => y.AddExtension(contextMock.Object), Times.Once);
mockDatabaseActions.Verify(y => y.TruncateTable(contextMock.Object, It.IsAny <string>()), Times.Once);
mockDatabaseActions.Verify(y => y.CreateTable(contextMock.Object, It.IsAny <string>()), Times.Once);
mockDatabaseActions.Verify(y => y.CopyDataToDatabase(tableName, contextMock.Object, testRecord.AwsRegion, bucketData.Bucket.Name, bucketData.Object.Key), Times.Once);
}
private async Task <Stream> ResizeImageAsync(S3EventNotification.S3EventNotificationRecord record)
{
var imageSize = int.Parse(Environment.GetEnvironmentVariable("THUMBNAIL_SIZE"));
Stream imageStream = new MemoryStream();
using (var objectResponse = await S3Client.GetObjectAsync(record.S3.Bucket.Name, record.S3.Object.Key))
using (Stream responseStream = objectResponse.ResponseStream)
{
using (Image <Rgba32> image = Image.Load(responseStream))
{
image.Mutate(ctx => ctx.Resize(imageSize, imageSize));
image.Save(imageStream, new JpegEncoder());
imageStream.Seek(0, SeekOrigin.Begin);
}
}
return(imageStream);
}
private async Task <List <Product> > ReadS3MessageLineByLine(S3EventNotification.S3EventNotificationRecord notification)
{
var result = new List <Product>();
try
{
var client = new AmazonS3Client(RegionEndpoint.USEast1);
var request = new GetObjectRequest
{
BucketName = notification.S3.Bucket.Name,
Key = notification.S3.Object.Key
};
using (GetObjectResponse response = await client.GetObjectAsync(request))
using (CsvReader reader = new CsvReader(ProcessRowFromFile))
{
reader.DelimitedFileProcessed += (sender, args) =>
{
Console.WriteLine($"Processing file complete");
};
await reader.ProcessDelimted(response.ResponseStream, 5000);
}
}
catch (AmazonS3Exception e)
{
Console.WriteLine("Error encountered ***. Message:'{0}' when writing an object", e.Message);
}
catch (Exception e)
{
Console.WriteLine("Unknown encountered on server. Message:'{0}' when writing an object", e.Message);
}
return(result);
}
protected abstract Task ProcessMessage(S3EventNotification.S3EventNotificationRecord message);
private async Task <EmailMessageInfo> CreateEmailMessageAsync(string requestId, string messageId, S3EventNotification.S3EventNotificationRecord record)
{
Stopwatch stopwatch = Stopwatch.StartNew();
GetObjectResponse response = await _s3Client.GetObjectAsync(record.S3.Bucket.Name, record.S3.Object.Key)
.TimeoutAfter(_config.TimeoutS3)
.ConfigureAwait(false);
_log.Debug($"Retrieving aggregate report from bucket took {stopwatch.Elapsed}");
stopwatch.Stop();
string originalUri = $"{record.S3.Bucket.Name}/{record.S3.Object.Key}";
return(new EmailMessageInfo(new EmailMetadata(requestId, messageId, originalUri, record.S3.Object.Key, record.S3.Object.Size / 1024), response.ResponseStream));
}
public async Task ProcessRecord(S3EventNotification.S3EventNotificationRecord record, ILambdaContext context)
{
var cameraKey = record.S3.Object.Key.Split('/')[1];
var s3GetResult = await s3.GetObjectAsync(record.S3.Bucket.Name, record.S3.Object.Key);
var classNamesParameterName = $"/Cameras/{cameraKey}/ClassNames";
var sceneCodeParameterName = $"/Cameras/{cameraKey}/SceneCode";
var observationBoundingBoxParameterName = $"/Cameras/{cameraKey}/ObservationBoundingBox";
if (!cameraParameters.ContainsKey(observationBoundingBoxParameterName))
{
try
{
var getResult = await ssm.GetParameterAsync(new GetParameterRequest
{
Name = observationBoundingBoxParameterName
});
cameraParameters.Add(observationBoundingBoxParameterName, getResult.Parameter.Value);
context.Logger.LogLine(
$"Set {observationBoundingBoxParameterName} = {cameraParameters[observationBoundingBoxParameterName]}");
}
catch (Exception exception)
{
context.Logger.LogLine($"Didn't add parameter. {observationBoundingBoxParameterName}");
context.Logger.LogLine(exception.Message);
}
}
if (!cameraParameters.ContainsKey(classNamesParameterName))
{
var getResult = await ssm.GetParameterAsync(new GetParameterRequest
{
Name = classNamesParameterName
});
cameraParameters.Add(classNamesParameterName, getResult.Parameter.Value);
context.Logger.LogLine($"Set {classNamesParameterName} = {cameraParameters[classNamesParameterName]}");
}
if (!cameraParameters.ContainsKey(sceneCodeParameterName))
{
var getResult = await ssm.GetParameterAsync(new GetParameterRequest
{
Name = sceneCodeParameterName
});
cameraParameters.Add(sceneCodeParameterName, getResult.Parameter.Value);
context.Logger.LogLine($"Set {sceneCodeParameterName} = {cameraParameters[sceneCodeParameterName]}");
}
var memoryStream = new MemoryStream();
await s3GetResult.ResponseStream.CopyToAsync(memoryStream);
// Crop the area of interest.
// TODO: Get the x, y, w, h from parmeter store.
var croppedMemoryStream = new MemoryStream();
var cropImage = cameraParameters.ContainsKey(observationBoundingBoxParameterName);
if (cropImage)
{
var parts = cameraParameters[observationBoundingBoxParameterName].Split(',');
memoryStream.Position = 0;
var sourceImage = Image.Load(memoryStream);
var x = int.Parse(parts[0]);
var y = int.Parse(parts[1]);
var lowerX = int.Parse(parts[2]);
var lowerY = int.Parse(parts[3]);
var w = lowerX - x;
var h = lowerY - y;
sourceImage.Mutate(i => i.Crop(new Rectangle(x, y, w, h)));
context.Logger.LogLine("Trying to save croped image.");
sourceImage.Save(croppedMemoryStream, new JpegEncoder());
croppedMemoryStream.Position = 0;
}
var labelsResult = await rekognition.DetectLabelsAsync(new DetectLabelsRequest
{
Image = new Amazon.Rekognition.Model.Image
{
Bytes = cropImage ? croppedMemoryStream : memoryStream
},
MaxLabels = 100,
MinConfidence = 70
});
if (cropImage)
{
croppedMemoryStream.Position = 0;
}
else
{
memoryStream.Position = 0;
}
var metricData = new List <MetricDatum>();
var personMetric = new MetricDatum
{
MetricName = "Confidence",
StorageResolution = 1,
TimestampUtc = DateTime.UtcNow,
Unit = StandardUnit.Percent,
Dimensions = new List <Dimension>
{
new Dimension {
Name = "CameraKey", Value = cameraKey
},
new Dimension {
Name = "Source", Value = "Rekognition"
},
new Dimension {
Name = "Label", Value = "Person"
}
}
};
if (labelsResult.Labels.Any(label => label.Name == "Person"))
{
var confidence = Convert.ToDouble(labelsResult.Labels.Single(l => l.Name == "Person").Confidence);
personMetric.StatisticValues = new StatisticSet
{
Minimum = confidence,
Maximum = confidence,
SampleCount = 1,
Sum = 1
};
}
else
{
personMetric.StatisticValues = new StatisticSet
{
Minimum = 0,
Maximum = 0,
SampleCount = 1,
Sum = 1
};
}
metricData.Add(personMetric);
var objectDetectionResult = await sageMakerRuntime.InvokeEndpointAsync(new InvokeEndpointRequest
{
Accept = "application/jsonlines",
ContentType = "application/x-image",
EndpointName = cameraParameters[sceneCodeParameterName],
Body = cropImage ? croppedMemoryStream : memoryStream
});
if (cropImage)
{
croppedMemoryStream.Close();
}
else
{
memoryStream.Close();
}
using (var streamReader = new StreamReader(objectDetectionResult.Body))
{
var json = streamReader.ReadToEnd();
context.Logger.Log($"SageMaker Endpoint Result: {json}");
var predictionResult = JsonConvert.DeserializeObject <dynamic>(json).prediction;
var classNames = cameraParameters[classNamesParameterName].Split(',');
var predictions = new List <Prediction>();
foreach (var pr in predictionResult)
{
predictions.Add(new Prediction
{
ClassName = classNames[Convert.ToInt32(pr[0].Value)],
Confidence = Convert.ToDouble(pr[1].Value) * 100
});
}
foreach (var classNotPredicted in classNames.Where(cn => predictions.All(p => p.ClassName != cn)))
{
predictions.Add(new Prediction
{
ClassName = classNotPredicted,
Confidence = 0
});
}
foreach (var classGroup in predictions.GroupBy(p => p.ClassName))
{
metricData.Add(new MetricDatum
{
MetricName = "Confidence",
StorageResolution = 1,
TimestampUtc = DateTime.UtcNow,
Unit = StandardUnit.Percent,
Dimensions = new List <Dimension>
{
new Dimension {
Name = "CameraKey", Value = cameraKey
},
new Dimension {
Name = "Source", Value = cameraParameters[sceneCodeParameterName]
},
new Dimension {
Name = "Label", Value = classGroup.Key
}
},
StatisticValues = new StatisticSet
{
Minimum = classGroup.Min(c => c.Confidence),
Maximum = classGroup.Max(c => c.Confidence),
Sum = classGroup.Count(),
SampleCount = classGroup.Count()
}
});
}
}
await cloudWatch.PutMetricDataAsync(new PutMetricDataRequest
{
Namespace = "Cameras",
MetricData = metricData
});
}
private async Task <bool> CheckCompressedTagAsync(S3EventNotification.S3EventNotificationRecord record) =>
(await S3Client.GetObjectTaggingAsync(new GetObjectTaggingRequest
{
BucketName = record.S3.Bucket.Name,
Key = record.S3.Object.Key
})).Tagging.Any(tag => tag.Key == "Compressed" && tag.Value == "true");
