private async Task InstallProductEurexAsync(Stream stream, CancellationToken cancellationToken) { var enumerable = csvConverter.Convert <EurexProductSpecification>(stream, CumulusConstants.EurexProductCsvSeperator); var totalProductsImported = 0; using (var bae = new BufferedAsyncEnumerable <EurexProductSpecification>(enumerable, CumulusConstants.DynamoDbBatchWriteSize)) { while (await bae.MoveNextAsync()) { var productCollection = bae.Current.Select(sourceProduct => new Product { ProductName = sourceProduct.ProductName, ProductISIN = sourceProduct.ProductISIN, StockMarket = (int)StockMarket.Eurex, NumberOfTradesAll = 0, NumberOfTradesLastDay = 0, NumberOfTradesLastWeek = 0, NumberOfTradesMonthToDate = 0, NumberOfTradesYearToDate = 0, }) .ToList(); await cumulusDbContext.SaveProductCollectionAsync(productCollection, cancellationToken); totalProductsImported += productCollection.Count; } } this.lambdaContext.Logger.LogLine($"{totalProductsImported} products imported from eurex resource"); }
public void ShouldConvert() { //Arrange var csvString = "Title; Author; Year; Price; In Stock; Binding; Description \nHitchhiker's guide to the galaxy;Douglas Adams;1978;42;no;Hardcover;The Hitchhiker's Guide to the Galaxy is a fictional electronic guide book in the multimedia scifi/comedy series of the same name by Douglas Adams"; var seperator = ';'; //Act var books = _csvConverter.Convert <Book>(csvString, seperator).ToList(); //Assert Assert.Equal("Douglas Adams", books.First().Author); }
public List <T> ReadCsvFile <T>(StreamReader reader, ICsvConverter <T> converter) where T : class { var dataList = new List <T>(); var lineCount = 0; while (!reader.EndOfStream) { var csvData = reader.ReadLine(); lineCount++; if (lineCount > 10 && dataList.Count == 0) { throw new Exception("Too many errors occured trying to read file."); } if (lineCount == 1) { //Assume header line and ignore continue; } try { var data = converter.Convert(lineCount, csvData); dataList.Add(data); } catch (Exception) { _logger.LogWarning($"Invalid data on line {lineCount}"); } } return(dataList); }
public RecordsDataSet GetPredefinedSet(int key) { foreach (var file in _fileStorage.GetFiles()) { if (file.Key == key && CheckIfCsv(file.Path)) { return(_csvConverter.Convert(_csvParser.ParseDynamic(file.Path))); } } return(null); }
private void AttachCsv(string path) { try { var text = File.ReadAllText(path); var books = _csvConverter.Convert <Book>(text, Separator).ToList(); var bookModels = _dataService.BuildBookModelsFromBooks(books); Bindings = new ObservableCollection <string>(_dataService.GetBindings(books)); BookModels = new ObservableCollection <BookModel>(bookModels); IsCSVAttached = true; } catch (Exception) { MessageBox.Show("Something went wrong, make sure you are using right CSV or try to change separator from \"Settings\" tab"); } }
public async Task ImportObjectInternalAsync(StockMarket sourceBucketType, string objectKey, char csvSeperator, CancellationToken cancellationToken) { var objectID = this.encryptionService.GetBucketObjectHash(sourceBucketType, objectKey); var bucketName = this.GetBucketName(sourceBucketType); var bucketObject = await this.repository.GetItemAsync <BucketObject>(objectID, cancellationToken); if (bucketObject == null) { var objectMetaData = await amazonS3.GetObjectMetadataAsync(bucketName, objectKey, cancellationToken); bucketObject = new BucketObject { ObjectID = objectID, ObjectKey = objectKey, SourceBucketType = (int)sourceBucketType, Status = (int)BucketObjectStatus.Detected, ObjectSize = objectMetaData.Headers.ContentLength, CurrentStatusErrorCount = 0, CurrentStatusLastError = null, VersionNumber = null }; } if (bucketObject.Status > (int)BucketObjectStatus.Queued) { return; } bucketObject.Status = (int)BucketObjectStatus.Processing; await this.repository.SaveItemAsync(bucketObject, cancellationToken); try { var objectStream = await amazonS3.GetObjectStreamAsync(bucketName, objectKey, null, cancellationToken); var enumerable = csvConverter.Convert <TradeActivity>(objectStream, csvSeperator); using (var bae = new BufferedAsyncEnumerable <TradeActivity>(enumerable, CumulusConstants.DynamoDbBatchWriteSize)) { while (await bae.MoveNextAsync()) { await productService.SaveTraceActivityCollectionAsync(bae.Current, cancellationToken); } } bucketObject.Status = (int)BucketObjectStatus.Processed; } catch (Exception exception) { bucketObject.CurrentStatusLastError = exception.Message; bucketObject.CurrentStatusErrorCount++; throw; } finally { await this.repository.SaveItemAsync(bucketObject, cancellationToken); } }