private async Task InstallProductEurexAsync(Stream stream, CancellationToken cancellationToken)
{
var enumerable = csvConverter.Convert <EurexProductSpecification>(stream, CumulusConstants.EurexProductCsvSeperator);
var totalProductsImported = 0;
using (var bae = new BufferedAsyncEnumerable <EurexProductSpecification>(enumerable, CumulusConstants.DynamoDbBatchWriteSize))
{
while (await bae.MoveNextAsync())
{
var productCollection = bae.Current.Select(sourceProduct => new Product
{
ProductName = sourceProduct.ProductName,
ProductISIN = sourceProduct.ProductISIN,
StockMarket = (int)StockMarket.Eurex,
NumberOfTradesAll = 0,
NumberOfTradesLastDay = 0,
NumberOfTradesLastWeek = 0,
NumberOfTradesMonthToDate = 0,
NumberOfTradesYearToDate = 0,
})
.ToList();
await cumulusDbContext.SaveProductCollectionAsync(productCollection, cancellationToken);
totalProductsImported += productCollection.Count;
}
}
this.lambdaContext.Logger.LogLine($"{totalProductsImported} products imported from eurex resource");
}
public void ShouldConvert()
{
//Arrange
var csvString = "Title; Author; Year; Price; In Stock; Binding; Description \nHitchhiker's guide to the galaxy;Douglas Adams;1978;42;no;Hardcover;The Hitchhiker's Guide to the Galaxy is a fictional electronic guide book in the multimedia scifi/comedy series of the same name by Douglas Adams";
var seperator = ';';
//Act
var books = _csvConverter.Convert <Book>(csvString, seperator).ToList();
//Assert
Assert.Equal("Douglas Adams", books.First().Author);
}
public List <T> ReadCsvFile <T>(StreamReader reader, ICsvConverter <T> converter) where T : class
{
var dataList = new List <T>();
var lineCount = 0;
while (!reader.EndOfStream)
{
var csvData = reader.ReadLine();
lineCount++;
if (lineCount > 10 && dataList.Count == 0)
{
throw new Exception("Too many errors occured trying to read file.");
}
if (lineCount == 1)
{
//Assume header line and ignore
continue;
}
try
{
var data = converter.Convert(lineCount, csvData);
dataList.Add(data);
}
catch (Exception)
{
_logger.LogWarning($"Invalid data on line {lineCount}");
}
}
return(dataList);
}
public RecordsDataSet GetPredefinedSet(int key)
{
foreach (var file in _fileStorage.GetFiles())
{
if (file.Key == key && CheckIfCsv(file.Path))
{
return(_csvConverter.Convert(_csvParser.ParseDynamic(file.Path)));
}
}
return(null);
}
private void AttachCsv(string path)
{
try
{
var text = File.ReadAllText(path);
var books = _csvConverter.Convert <Book>(text, Separator).ToList();
var bookModels = _dataService.BuildBookModelsFromBooks(books);
Bindings = new ObservableCollection <string>(_dataService.GetBindings(books));
BookModels = new ObservableCollection <BookModel>(bookModels);
IsCSVAttached = true;
}
catch (Exception)
{
MessageBox.Show("Something went wrong, make sure you are using right CSV or try to change separator from \"Settings\" tab");
}
}
public async Task ImportObjectInternalAsync(StockMarket sourceBucketType, string objectKey, char csvSeperator, CancellationToken cancellationToken)
{
var objectID = this.encryptionService.GetBucketObjectHash(sourceBucketType, objectKey);
var bucketName = this.GetBucketName(sourceBucketType);
var bucketObject = await this.repository.GetItemAsync <BucketObject>(objectID, cancellationToken);
if (bucketObject == null)
{
var objectMetaData = await amazonS3.GetObjectMetadataAsync(bucketName, objectKey, cancellationToken);
bucketObject = new BucketObject
{
ObjectID = objectID,
ObjectKey = objectKey,
SourceBucketType = (int)sourceBucketType,
Status = (int)BucketObjectStatus.Detected,
ObjectSize = objectMetaData.Headers.ContentLength,
CurrentStatusErrorCount = 0,
CurrentStatusLastError = null,
VersionNumber = null
};
}
if (bucketObject.Status > (int)BucketObjectStatus.Queued)
{
return;
}
bucketObject.Status = (int)BucketObjectStatus.Processing;
await this.repository.SaveItemAsync(bucketObject, cancellationToken);
try
{
var objectStream = await amazonS3.GetObjectStreamAsync(bucketName, objectKey, null, cancellationToken);
var enumerable = csvConverter.Convert <TradeActivity>(objectStream, csvSeperator);
using (var bae = new BufferedAsyncEnumerable <TradeActivity>(enumerable, CumulusConstants.DynamoDbBatchWriteSize))
{
while (await bae.MoveNextAsync())
{
await productService.SaveTraceActivityCollectionAsync(bae.Current, cancellationToken);
}
}
bucketObject.Status = (int)BucketObjectStatus.Processed;
}
catch (Exception exception)
{
bucketObject.CurrentStatusLastError = exception.Message;
bucketObject.CurrentStatusErrorCount++;
throw;
}
finally
{
await this.repository.SaveItemAsync(bucketObject, cancellationToken);
}
}
