public Benchmark() { var config = new CsvImportConfiguration() { Culture = "en-US", Delimiter = ";", DateFormat = "yyyyMMdd HHmmss", HasHeaderRecord = false }; var importer = new CsvImporter("Data/EURUSD.csv", config); _data = importer.ImportAsync("EURUSD").Result.ToArray(); }
public IHttpActionResult DoImport(CsvImportConfiguration importConfiguration) { var notification = new ImportNotification(CurrentPrincipal.GetCurrentUserName()) { Title = "Import catalog from CSV", Description = "starting import...." }; _notifier.Upsert(notification); var importJob = new CsvCatalogImportJob(); BackgroundJob.Enqueue(() => importJob.DoImport(importConfiguration, notification)); return(Ok(notification)); }
public void ImportFromCsvWithTime() { var config = new CsvImportConfiguration() { Culture = "en-US", Delimiter = ";", DateFormat = "yyyyMMdd HHmmss", HasHeaderRecord = false }; var importer = new CsvImporter("EURUSD.csv", config); var candles = importer.ImportAsync("EURUSD").Result; Assert.AreEqual(744, candles.Count); var firstIOhlcvData = candles.First(); Assert.AreEqual(new DateTime(2000, 5, 30, 17, 27, 00), firstIOhlcvData.DateTime); }
public Benchmark() { var config = new CsvImportConfiguration() { Culture = "en-US", Delimiter = ";", DateFormat = "yyyyMMdd HHmmss", HasHeaderRecord = false }; var importer = new CsvImporter("Data/EURUSD.csv", config); _data = importer.ImportAsync("EURUSD").Result.ToArray(); _tradeData = new ITickTrade[_n]; var d = DateTimeOffset.Now; for (int i = 0; i < _n; i++) { _tradeData[i] = new Trade(d.AddSeconds(i), 1, 1); } }
public IHttpActionResult GetMappingConfiguration([FromUri] string fileUrl, [FromUri] string delimiter = ";") { var retVal = new CsvImportConfiguration { Delimiter = delimiter, FileUrl = fileUrl }; var mappingItems = new List <CsvImportMappingItem>(); mappingItems.AddRange(ReflectionUtility.GetPropertyNames <coreModel.CatalogProduct>(x => x.Name, x => x.Category).Select(x => new CsvImportMappingItem { EntityColumnName = x, IsRequired = true })); mappingItems.AddRange(new string[] { "Sku", "ParentSku", "Review", "PrimaryImage", "AltImage", "SeoUrl", "SeoDescription", "SeoTitle", "PriceId", "Price", "SalePrice", "Currency", "AllowBackorder", "Quantity", "FulfilmentCenterId" } .Select(x => new CsvImportMappingItem { EntityColumnName = x, IsRequired = false })); mappingItems.AddRange(ReflectionUtility.GetPropertyNames <coreModel.CatalogProduct>(x => x.Id, x => x.MainProductId, x => x.CategoryId, x => x.IsActive, x => x.IsBuyable, x => x.TrackInventory, x => x.ManufacturerPartNumber, x => x.Gtin, x => x.MeasureUnit, x => x.WeightUnit, x => x.Weight, x => x.Height, x => x.Length, x => x.Width, x => x.TaxType, x => x.ProductType, x => x.ShippingType, x => x.Vendor, x => x.DownloadType, x => x.DownloadExpiration, x => x.HasUserAgreement).Select(x => new CsvImportMappingItem { EntityColumnName = x, IsRequired = false })); retVal.MappingItems = mappingItems.ToArray(); //Read csv headers and try to auto map fields by name using (var reader = new CsvReader(new StreamReader(_blobStorageProvider.OpenReadOnly(fileUrl)))) { reader.Configuration.Delimiter = delimiter; while (reader.Read()) { var csvColumns = reader.FieldHeaders; retVal.CsvColumns = csvColumns; //default columns mapping if (csvColumns.Any()) { foreach (var mappingItem in retVal.MappingItems) { var entityColumnName = mappingItem.EntityColumnName; var betterMatchCsvColumn = csvColumns.Select(x => new { csvColumn = x, distance = x.ComputeLevenshteinDistance(entityColumnName) }) .Where(x => x.distance < 2) .OrderBy(x => x.distance) .Select(x => x.csvColumn) .FirstOrDefault(); if (betterMatchCsvColumn != null) { mappingItem.CsvColumnName = betterMatchCsvColumn; mappingItem.CustomValue = null; } } } } } //All not mapped properties may be a product property retVal.PropertyCsvColumns = retVal.CsvColumns.Except(retVal.MappingItems.Where(x => x.CsvColumnName != null).Select(x => x.CsvColumnName)).ToArray(); //Generate ETag for identifying csv format retVal.ETag = string.Join(";", retVal.CsvColumns).GetMD5Hash(); return(Ok(retVal)); }
public CsvParser(string fileName, CsvImportConfiguration importConfiguration) { this.fileName = fileName; this.importConfiguration = importConfiguration; BuildRecognition(); }
public CsvDictionarySource(string fileName, CsvImportConfiguration importConfiguration) { parser = new CsvParser(fileName, importConfiguration); }
public CsvKeyValueSource(string fileName, CsvImportConfiguration importConfiguration) { innerSource = new CsvParser(fileName, importConfiguration); }
public CsvImporter(string path, CsvImportConfiguration configuration) : this(path, configuration.CultureInfo) { _format = configuration.DateFormat; _delimiter = configuration.Delimiter; _hasHeader = configuration.HasHeaderRecord; }