private async Task InstallProductEurexAsync(Stream stream, CancellationToken cancellationToken)
{
var enumerable = csvConverter.Convert <EurexProductSpecification>(stream, CumulusConstants.EurexProductCsvSeperator);
var totalProductsImported = 0;
using (var bae = new BufferedAsyncEnumerable <EurexProductSpecification>(enumerable, CumulusConstants.DynamoDbBatchWriteSize))
{
while (await bae.MoveNextAsync())
{
var productCollection = bae.Current.Select(sourceProduct => new Product
{
ProductName = sourceProduct.ProductName,
ProductISIN = sourceProduct.ProductISIN,
StockMarket = (int)StockMarket.Eurex,
NumberOfTradesAll = 0,
NumberOfTradesLastDay = 0,
NumberOfTradesLastWeek = 0,
NumberOfTradesMonthToDate = 0,
NumberOfTradesYearToDate = 0,
})
.ToList();
await cumulusDbContext.SaveProductCollectionAsync(productCollection, cancellationToken);
totalProductsImported += productCollection.Count;
}
}
this.lambdaContext.Logger.LogLine($"{totalProductsImported} products imported from eurex resource");
}
public async Task ImportObjectInternalAsync(StockMarket sourceBucketType, string objectKey, char csvSeperator, CancellationToken cancellationToken)
{
var objectID = this.encryptionService.GetBucketObjectHash(sourceBucketType, objectKey);
var bucketName = this.GetBucketName(sourceBucketType);
var bucketObject = await this.repository.GetItemAsync <BucketObject>(objectID, cancellationToken);
if (bucketObject == null)
{
var objectMetaData = await amazonS3.GetObjectMetadataAsync(bucketName, objectKey, cancellationToken);
bucketObject = new BucketObject
{
ObjectID = objectID,
ObjectKey = objectKey,
SourceBucketType = (int)sourceBucketType,
Status = (int)BucketObjectStatus.Detected,
ObjectSize = objectMetaData.Headers.ContentLength,
CurrentStatusErrorCount = 0,
CurrentStatusLastError = null,
VersionNumber = null
};
}
if (bucketObject.Status > (int)BucketObjectStatus.Queued)
{
return;
}
bucketObject.Status = (int)BucketObjectStatus.Processing;
await this.repository.SaveItemAsync(bucketObject, cancellationToken);
try
{
var objectStream = await amazonS3.GetObjectStreamAsync(bucketName, objectKey, null, cancellationToken);
var enumerable = csvConverter.Convert <TradeActivity>(objectStream, csvSeperator);
using (var bae = new BufferedAsyncEnumerable <TradeActivity>(enumerable, CumulusConstants.DynamoDbBatchWriteSize))
{
while (await bae.MoveNextAsync())
{
await productService.SaveTraceActivityCollectionAsync(bae.Current, cancellationToken);
}
}
bucketObject.Status = (int)BucketObjectStatus.Processed;
}
catch (Exception exception)
{
bucketObject.CurrentStatusLastError = exception.Message;
bucketObject.CurrentStatusErrorCount++;
throw;
}
finally
{
await this.repository.SaveItemAsync(bucketObject, cancellationToken);
}
}
