public static IDataTable FromFile(
string fileName,
Stream stream,
long contentLength,
CsvConfiguration configuration,
int skip = 0,
int take = int.MaxValue)
{
Guard.NotEmpty(fileName, nameof(fileName));
Guard.NotNull(stream, nameof(stream));
Guard.NotNull(configuration, nameof(configuration));
if (contentLength == 0)
{
throw Error.Argument("fileName", "The posted file '{0}' does not contain any data.".FormatInvariant(fileName));
}
IDataReader dataReader = null;
try
{
var fileExt = System.IO.Path.GetExtension(fileName).ToLowerInvariant();
switch (fileExt)
{
case ".xlsx":
dataReader = new ExcelDataReader(stream, true); // TODO: let the user specify if excel file has headers
break;
default:
dataReader = new CsvDataReader(new StreamReader(stream), configuration);
break;
}
var table = LightweightDataTable.FromDataReader(dataReader, skip, take);
if (table.Columns.Count == 0 || table.Rows.Count == 0)
{
throw Error.InvalidOperation("The posted file '{0}' does not contain any columns or data rows.".FormatInvariant(fileName));
}
return(table);
}
catch (Exception ex)
{
throw ex;
}
finally
{
if (dataReader != null)
{
if (!dataReader.IsClosed)
{
dataReader.Dispose();
}
dataReader = null;
}
}
}
private void ImportCoreInner(DataImporterContext ctx, ImportFile file)
{
var context = ctx.ExecuteContext;
var profile = ctx.Request.Profile;
if (context.Abort == DataExchangeAbortion.Hard)
{
return;
}
if (!File.Exists(file.Path))
{
throw new SmartException($"File does not exist {file.Path}.");
}
using (var stream = new FileStream(file.Path, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var csvConfiguration = file.IsCsv
? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration)
: CsvConfiguration.ExcelFriendlyConfiguration;
context.DataTable = LightweightDataTable.FromFile(
file.Name,
stream,
stream.Length,
csvConfiguration,
profile.Skip,
profile.Take > 0 ? profile.Take : int.MaxValue);
context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap;
context.File = file;
try
{
ctx.Importer.Execute(context);
}
catch (Exception ex)
{
context.Abort = DataExchangeAbortion.Hard;
context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}.");
}
finally
{
context.Result.EndDateUtc = DateTime.UtcNow;
if (context.IsMaxFailures)
{
context.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
}
if (ctx.CancellationToken.IsCancellationRequested)
{
context.Result.AddWarning("Import aborted. A cancellation has been requested.");
}
}
}
}
public static IDataTable FromDataReader(
IDataReader reader,
int skip = 0,
int take = int.MaxValue)
{
Guard.NotNull(reader, nameof(reader));
if (reader.IsClosed)
{
throw new ArgumentException("This operation is invalid when the reader is closed.", "reader");
}
var columns = new List <IDataColumn>(reader.FieldCount);
var data = new List <object[]>();
var schema = reader.GetSchemaTable();
var nameCol = schema.Columns[SchemaTableColumn.ColumnName];
var typeCol = schema.Columns[SchemaTableColumn.DataType];
foreach (DataRow schemaRow in schema.Rows)
{
var column = new LightweightDataColumn((string)schemaRow[nameCol], (Type)schemaRow[typeCol]);
columns.Add(column);
}
var fieldCount = reader.FieldCount;
take = Math.Min(take, int.MaxValue - skip);
int i = -1;
while (reader.Read())
{
i++;
if (skip > i)
{
continue;
}
if (i >= skip + take)
{
break;
}
var values = new object[fieldCount];
reader.GetValues(values);
data.Add(values);
}
var table = new LightweightDataTable(columns, data);
return(table);
}
public void CanResolveColumnIndexNames()
{
var columns = new List<IDataColumn>
{
new LightweightDataColumn("Attr[Color]", typeof(string)),
new LightweightDataColumn("Attr[Size]", typeof(string)),
new LightweightDataColumn("material", typeof(string)),
new LightweightDataColumn("Name[en]", typeof(string)),
new LightweightDataColumn("Name[de]", typeof(string)),
new LightweightDataColumn("Name[fr]", typeof(string)),
new LightweightDataColumn("name_it", typeof(string)),
};
var table = new LightweightDataTable(columns, new List<object[]>());
var segmenter = new ImportDataSegmenter(table, new ColumnMap());
segmenter.ColumnMap.AddMapping("Attr[Material]", "material");
segmenter.ColumnMap.AddMapping("Name[it]", "name_it");
var attrs = segmenter.GetColumnIndexes("Attr");
attrs.ShouldSequenceEqual(new string[] {"Material", "Color", "Size"});
var langs = segmenter.GetColumnIndexes("Name");
langs.ShouldSequenceEqual(new string[] { "it", "en", "de", "fr" });
}
private void ImportCoreInner(DataImporterContext ctx, string filePath)
{
if (ctx.ExecuteContext.Abort == DataExchangeAbortion.Hard)
{
return;
}
{
var logHead = new StringBuilder();
logHead.AppendLine();
logHead.AppendLine(new string('-', 40));
logHead.AppendLine("SmartStore.NET:\t\tv." + SmartStoreVersion.CurrentFullVersion);
logHead.Append("Import profile:\t\t" + ctx.Request.Profile.Name);
logHead.AppendLine(ctx.Request.Profile.Id == 0 ? " (volatile)" : " (Id {0})".FormatInvariant(ctx.Request.Profile.Id));
logHead.AppendLine("Entity:\t\t\t" + ctx.Request.Profile.EntityType.ToString());
logHead.AppendLine("File:\t\t\t" + Path.GetFileName(filePath));
var customer = _services.WorkContext.CurrentCustomer;
logHead.Append("Executed by:\t\t" + (customer.Email.HasValue() ? customer.Email : customer.SystemName));
ctx.Log.Information(logHead.ToString());
}
if (!File.Exists(filePath))
{
throw new SmartException("File does not exist {0}.".FormatInvariant(filePath));
}
CsvConfiguration csvConfiguration = null;
var extension = Path.GetExtension(filePath);
if ((new string[] { ".csv", ".txt", ".tab" }).Contains(extension, StringComparer.OrdinalIgnoreCase))
{
var converter = new CsvConfigurationConverter();
csvConfiguration = converter.ConvertFrom <CsvConfiguration>(ctx.Request.Profile.FileTypeConfiguration);
}
if (csvConfiguration == null)
{
csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
}
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
ctx.ExecuteContext.DataTable = LightweightDataTable.FromFile(
Path.GetFileName(filePath),
stream,
stream.Length,
csvConfiguration,
ctx.Request.Profile.Skip,
ctx.Request.Profile.Take > 0 ? ctx.Request.Profile.Take : int.MaxValue
);
try
{
ctx.Importer.Execute(ctx.ExecuteContext);
}
catch (Exception exception)
{
ctx.ExecuteContext.Abort = DataExchangeAbortion.Hard;
ctx.ExecuteContext.Result.AddError(exception, "The importer failed: {0}.".FormatInvariant(exception.ToAllMessages()));
}
if (ctx.ExecuteContext.IsMaxFailures)
{
ctx.ExecuteContext.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
}
if (ctx.CancellationToken.IsCancellationRequested)
{
ctx.ExecuteContext.Result.AddWarning("Import aborted. A cancellation has been requested.");
}
}
}
public static IDataTable FromDataReader(
IDataReader reader,
int skip = 0,
int take = int.MaxValue)
{
Guard.ArgumentNotNull(() => reader);
if (reader.IsClosed)
throw new ArgumentException("This operation is invalid when the reader is closed.", "reader");
var columns = new List<IDataColumn>(reader.FieldCount);
var data = new List<object[]>();
var schema = reader.GetSchemaTable();
var nameCol = schema.Columns[SchemaTableColumn.ColumnName];
var typeCol = schema.Columns[SchemaTableColumn.DataType];
foreach (DataRow schemaRow in schema.Rows)
{
var column = new LightweightDataColumn((string)schemaRow[nameCol], (Type)schemaRow[typeCol]);
columns.Add(column);
}
var fieldCount = reader.FieldCount;
take = Math.Min(take, int.MaxValue - skip);
int i = -1;
while (reader.Read())
{
i++;
if (skip > i)
continue;
if (i >= skip + take)
break;
var values = new object[fieldCount];
reader.GetValues(values);
data.Add(values);
}
var table = new LightweightDataTable(columns, data);
return table;
}
