public Benchmark()
{
var config = new CsvImportConfiguration()
{
Culture = "en-US",
Delimiter = ";",
DateFormat = "yyyyMMdd HHmmss",
HasHeaderRecord = false
};
var importer = new CsvImporter("Data/EURUSD.csv", config);
_data = importer.ImportAsync("EURUSD").Result.ToArray();
}
public IHttpActionResult DoImport(CsvImportConfiguration importConfiguration)
{
var notification = new ImportNotification(CurrentPrincipal.GetCurrentUserName())
{
Title = "Import catalog from CSV",
Description = "starting import...."
};
_notifier.Upsert(notification);
var importJob = new CsvCatalogImportJob();
BackgroundJob.Enqueue(() => importJob.DoImport(importConfiguration, notification));
return(Ok(notification));
}
public void ImportFromCsvWithTime()
{
var config = new CsvImportConfiguration()
{
Culture = "en-US",
Delimiter = ";",
DateFormat = "yyyyMMdd HHmmss",
HasHeaderRecord = false
};
var importer = new CsvImporter("EURUSD.csv", config);
var candles = importer.ImportAsync("EURUSD").Result;
Assert.AreEqual(744, candles.Count);
var firstIOhlcvData = candles.First();
Assert.AreEqual(new DateTime(2000, 5, 30, 17, 27, 00), firstIOhlcvData.DateTime);
}
public Benchmark()
{
var config = new CsvImportConfiguration()
{
Culture = "en-US",
Delimiter = ";",
DateFormat = "yyyyMMdd HHmmss",
HasHeaderRecord = false
};
var importer = new CsvImporter("Data/EURUSD.csv", config);
_data = importer.ImportAsync("EURUSD").Result.ToArray();
_tradeData = new ITickTrade[_n];
var d = DateTimeOffset.Now;
for (int i = 0; i < _n; i++)
{
_tradeData[i] = new Trade(d.AddSeconds(i), 1, 1);
}
}
public IHttpActionResult GetMappingConfiguration([FromUri] string fileUrl, [FromUri] string delimiter = ";")
{
var retVal = new CsvImportConfiguration
{
Delimiter = delimiter,
FileUrl = fileUrl
};
var mappingItems = new List <CsvImportMappingItem>();
mappingItems.AddRange(ReflectionUtility.GetPropertyNames <coreModel.CatalogProduct>(x => x.Name, x => x.Category).Select(x => new CsvImportMappingItem {
EntityColumnName = x, IsRequired = true
}));
mappingItems.AddRange(new string[] { "Sku", "ParentSku", "Review", "PrimaryImage", "AltImage", "SeoUrl", "SeoDescription", "SeoTitle",
"PriceId", "Price", "SalePrice", "Currency", "AllowBackorder", "Quantity", "FulfilmentCenterId" }
.Select(x => new CsvImportMappingItem {
EntityColumnName = x, IsRequired = false
}));
mappingItems.AddRange(ReflectionUtility.GetPropertyNames <coreModel.CatalogProduct>(x => x.Id, x => x.MainProductId, x => x.CategoryId, x => x.IsActive, x => x.IsBuyable, x => x.TrackInventory,
x => x.ManufacturerPartNumber, x => x.Gtin, x => x.MeasureUnit, x => x.WeightUnit, x => x.Weight,
x => x.Height, x => x.Length, x => x.Width, x => x.TaxType, x => x.ProductType, x => x.ShippingType,
x => x.Vendor, x => x.DownloadType, x => x.DownloadExpiration, x => x.HasUserAgreement).Select(x => new CsvImportMappingItem {
EntityColumnName = x, IsRequired = false
}));
retVal.MappingItems = mappingItems.ToArray();
//Read csv headers and try to auto map fields by name
using (var reader = new CsvReader(new StreamReader(_blobStorageProvider.OpenReadOnly(fileUrl))))
{
reader.Configuration.Delimiter = delimiter;
while (reader.Read())
{
var csvColumns = reader.FieldHeaders;
retVal.CsvColumns = csvColumns;
//default columns mapping
if (csvColumns.Any())
{
foreach (var mappingItem in retVal.MappingItems)
{
var entityColumnName = mappingItem.EntityColumnName;
var betterMatchCsvColumn = csvColumns.Select(x => new { csvColumn = x, distance = x.ComputeLevenshteinDistance(entityColumnName) })
.Where(x => x.distance < 2)
.OrderBy(x => x.distance)
.Select(x => x.csvColumn)
.FirstOrDefault();
if (betterMatchCsvColumn != null)
{
mappingItem.CsvColumnName = betterMatchCsvColumn;
mappingItem.CustomValue = null;
}
}
}
}
}
//All not mapped properties may be a product property
retVal.PropertyCsvColumns = retVal.CsvColumns.Except(retVal.MappingItems.Where(x => x.CsvColumnName != null).Select(x => x.CsvColumnName)).ToArray();
//Generate ETag for identifying csv format
retVal.ETag = string.Join(";", retVal.CsvColumns).GetMD5Hash();
return(Ok(retVal));
}
public CsvParser(string fileName, CsvImportConfiguration importConfiguration)
{
this.fileName = fileName;
this.importConfiguration = importConfiguration;
BuildRecognition();
}
public CsvDictionarySource(string fileName, CsvImportConfiguration importConfiguration)
{
parser = new CsvParser(fileName, importConfiguration);
}
public CsvKeyValueSource(string fileName, CsvImportConfiguration importConfiguration)
{
innerSource = new CsvParser(fileName, importConfiguration);
}
public CsvImporter(string path, CsvImportConfiguration configuration) : this(path, configuration.CultureInfo)
{
_format = configuration.DateFormat;
_delimiter = configuration.Delimiter;
_hasHeader = configuration.HasHeaderRecord;
}
