public static IDataTable FromFile( string fileName, Stream stream, long contentLength, CsvConfiguration configuration, int skip = 0, int take = int.MaxValue) { Guard.NotEmpty(fileName, nameof(fileName)); Guard.NotNull(stream, nameof(stream)); Guard.NotNull(configuration, nameof(configuration)); if (contentLength == 0) { throw Error.Argument("fileName", "The posted file '{0}' does not contain any data.".FormatInvariant(fileName)); } IDataReader dataReader = null; try { var fileExt = System.IO.Path.GetExtension(fileName).ToLowerInvariant(); switch (fileExt) { case ".xlsx": dataReader = new ExcelDataReader(stream, true); // TODO: let the user specify if excel file has headers break; default: dataReader = new CsvDataReader(new StreamReader(stream), configuration); break; } var table = LightweightDataTable.FromDataReader(dataReader, skip, take); if (table.Columns.Count == 0 || table.Rows.Count == 0) { throw Error.InvalidOperation("The posted file '{0}' does not contain any columns or data rows.".FormatInvariant(fileName)); } return(table); } catch (Exception ex) { throw ex; } finally { if (dataReader != null) { if (!dataReader.IsClosed) { dataReader.Dispose(); } dataReader = null; } } }
private void ImportCoreInner(DataImporterContext ctx, ImportFile file) { var context = ctx.ExecuteContext; var profile = ctx.Request.Profile; if (context.Abort == DataExchangeAbortion.Hard) { return; } if (!File.Exists(file.Path)) { throw new SmartException($"File does not exist {file.Path}."); } using (var stream = new FileStream(file.Path, FileMode.Open, FileAccess.Read, FileShare.Read)) { var csvConfiguration = file.IsCsv ? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration) : CsvConfiguration.ExcelFriendlyConfiguration; context.DataTable = LightweightDataTable.FromFile( file.Name, stream, stream.Length, csvConfiguration, profile.Skip, profile.Take > 0 ? profile.Take : int.MaxValue); context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap; context.File = file; try { ctx.Importer.Execute(context); } catch (Exception ex) { context.Abort = DataExchangeAbortion.Hard; context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}."); } finally { context.Result.EndDateUtc = DateTime.UtcNow; if (context.IsMaxFailures) { context.Result.AddWarning("Import aborted. The maximum number of failures has been reached."); } if (ctx.CancellationToken.IsCancellationRequested) { context.Result.AddWarning("Import aborted. A cancellation has been requested."); } } } }
public static IDataTable FromDataReader( IDataReader reader, int skip = 0, int take = int.MaxValue) { Guard.NotNull(reader, nameof(reader)); if (reader.IsClosed) { throw new ArgumentException("This operation is invalid when the reader is closed.", "reader"); } var columns = new List <IDataColumn>(reader.FieldCount); var data = new List <object[]>(); var schema = reader.GetSchemaTable(); var nameCol = schema.Columns[SchemaTableColumn.ColumnName]; var typeCol = schema.Columns[SchemaTableColumn.DataType]; foreach (DataRow schemaRow in schema.Rows) { var column = new LightweightDataColumn((string)schemaRow[nameCol], (Type)schemaRow[typeCol]); columns.Add(column); } var fieldCount = reader.FieldCount; take = Math.Min(take, int.MaxValue - skip); int i = -1; while (reader.Read()) { i++; if (skip > i) { continue; } if (i >= skip + take) { break; } var values = new object[fieldCount]; reader.GetValues(values); data.Add(values); } var table = new LightweightDataTable(columns, data); return(table); }
public void CanResolveColumnIndexNames() { var columns = new List<IDataColumn> { new LightweightDataColumn("Attr[Color]", typeof(string)), new LightweightDataColumn("Attr[Size]", typeof(string)), new LightweightDataColumn("material", typeof(string)), new LightweightDataColumn("Name[en]", typeof(string)), new LightweightDataColumn("Name[de]", typeof(string)), new LightweightDataColumn("Name[fr]", typeof(string)), new LightweightDataColumn("name_it", typeof(string)), }; var table = new LightweightDataTable(columns, new List<object[]>()); var segmenter = new ImportDataSegmenter(table, new ColumnMap()); segmenter.ColumnMap.AddMapping("Attr[Material]", "material"); segmenter.ColumnMap.AddMapping("Name[it]", "name_it"); var attrs = segmenter.GetColumnIndexes("Attr"); attrs.ShouldSequenceEqual(new string[] {"Material", "Color", "Size"}); var langs = segmenter.GetColumnIndexes("Name"); langs.ShouldSequenceEqual(new string[] { "it", "en", "de", "fr" }); }
private void ImportCoreInner(DataImporterContext ctx, string filePath) { if (ctx.ExecuteContext.Abort == DataExchangeAbortion.Hard) { return; } { var logHead = new StringBuilder(); logHead.AppendLine(); logHead.AppendLine(new string('-', 40)); logHead.AppendLine("SmartStore.NET:\t\tv." + SmartStoreVersion.CurrentFullVersion); logHead.Append("Import profile:\t\t" + ctx.Request.Profile.Name); logHead.AppendLine(ctx.Request.Profile.Id == 0 ? " (volatile)" : " (Id {0})".FormatInvariant(ctx.Request.Profile.Id)); logHead.AppendLine("Entity:\t\t\t" + ctx.Request.Profile.EntityType.ToString()); logHead.AppendLine("File:\t\t\t" + Path.GetFileName(filePath)); var customer = _services.WorkContext.CurrentCustomer; logHead.Append("Executed by:\t\t" + (customer.Email.HasValue() ? customer.Email : customer.SystemName)); ctx.Log.Information(logHead.ToString()); } if (!File.Exists(filePath)) { throw new SmartException("File does not exist {0}.".FormatInvariant(filePath)); } CsvConfiguration csvConfiguration = null; var extension = Path.GetExtension(filePath); if ((new string[] { ".csv", ".txt", ".tab" }).Contains(extension, StringComparer.OrdinalIgnoreCase)) { var converter = new CsvConfigurationConverter(); csvConfiguration = converter.ConvertFrom <CsvConfiguration>(ctx.Request.Profile.FileTypeConfiguration); } if (csvConfiguration == null) { csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration; } using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { ctx.ExecuteContext.DataTable = LightweightDataTable.FromFile( Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, ctx.Request.Profile.Skip, ctx.Request.Profile.Take > 0 ? ctx.Request.Profile.Take : int.MaxValue ); try { ctx.Importer.Execute(ctx.ExecuteContext); } catch (Exception exception) { ctx.ExecuteContext.Abort = DataExchangeAbortion.Hard; ctx.ExecuteContext.Result.AddError(exception, "The importer failed: {0}.".FormatInvariant(exception.ToAllMessages())); } if (ctx.ExecuteContext.IsMaxFailures) { ctx.ExecuteContext.Result.AddWarning("Import aborted. The maximum number of failures has been reached."); } if (ctx.CancellationToken.IsCancellationRequested) { ctx.ExecuteContext.Result.AddWarning("Import aborted. A cancellation has been requested."); } } }
public static IDataTable FromDataReader( IDataReader reader, int skip = 0, int take = int.MaxValue) { Guard.ArgumentNotNull(() => reader); if (reader.IsClosed) throw new ArgumentException("This operation is invalid when the reader is closed.", "reader"); var columns = new List<IDataColumn>(reader.FieldCount); var data = new List<object[]>(); var schema = reader.GetSchemaTable(); var nameCol = schema.Columns[SchemaTableColumn.ColumnName]; var typeCol = schema.Columns[SchemaTableColumn.DataType]; foreach (DataRow schemaRow in schema.Rows) { var column = new LightweightDataColumn((string)schemaRow[nameCol], (Type)schemaRow[typeCol]); columns.Add(column); } var fieldCount = reader.FieldCount; take = Math.Min(take, int.MaxValue - skip); int i = -1; while (reader.Read()) { i++; if (skip > i) continue; if (i >= skip + take) break; var values = new object[fieldCount]; reader.GetValues(values); data.Add(values); } var table = new LightweightDataTable(columns, data); return table; }