Beispiel #1
0
        public static IDataTable FromFile(
            string fileName,
            Stream stream,
            long contentLength,
            CsvConfiguration configuration,
            int skip = 0,
            int take = int.MaxValue)
        {
            Guard.NotEmpty(fileName, nameof(fileName));
            Guard.NotNull(stream, nameof(stream));
            Guard.NotNull(configuration, nameof(configuration));

            if (contentLength == 0)
            {
                throw Error.Argument("fileName", "The posted file '{0}' does not contain any data.".FormatInvariant(fileName));
            }

            IDataReader dataReader = null;

            try
            {
                var fileExt = System.IO.Path.GetExtension(fileName).ToLowerInvariant();

                switch (fileExt)
                {
                case ".xlsx":
                    dataReader = new ExcelDataReader(stream, true);     // TODO: let the user specify if excel file has headers
                    break;

                default:
                    dataReader = new CsvDataReader(new StreamReader(stream), configuration);
                    break;
                }

                var table = LightweightDataTable.FromDataReader(dataReader, skip, take);

                if (table.Columns.Count == 0 || table.Rows.Count == 0)
                {
                    throw Error.InvalidOperation("The posted file '{0}' does not contain any columns or data rows.".FormatInvariant(fileName));
                }

                return(table);
            }
            catch (Exception ex)
            {
                throw ex;
            }
            finally
            {
                if (dataReader != null)
                {
                    if (!dataReader.IsClosed)
                    {
                        dataReader.Dispose();
                    }
                    dataReader = null;
                }
            }
        }
Beispiel #2
0
        private void ImportCoreInner(DataImporterContext ctx, ImportFile file)
        {
            var context = ctx.ExecuteContext;
            var profile = ctx.Request.Profile;

            if (context.Abort == DataExchangeAbortion.Hard)
            {
                return;
            }

            if (!File.Exists(file.Path))
            {
                throw new SmartException($"File does not exist {file.Path}.");
            }

            using (var stream = new FileStream(file.Path, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                var csvConfiguration = file.IsCsv
                    ? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration)
                    : CsvConfiguration.ExcelFriendlyConfiguration;

                context.DataTable = LightweightDataTable.FromFile(
                    file.Name,
                    stream,
                    stream.Length,
                    csvConfiguration,
                    profile.Skip,
                    profile.Take > 0 ? profile.Take : int.MaxValue);

                context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap;
                context.File      = file;

                try
                {
                    ctx.Importer.Execute(context);
                }
                catch (Exception ex)
                {
                    context.Abort = DataExchangeAbortion.Hard;
                    context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}.");
                }
                finally
                {
                    context.Result.EndDateUtc = DateTime.UtcNow;

                    if (context.IsMaxFailures)
                    {
                        context.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
                    }
                    if (ctx.CancellationToken.IsCancellationRequested)
                    {
                        context.Result.AddWarning("Import aborted. A cancellation has been requested.");
                    }
                }
            }
        }
Beispiel #3
0
        public static IDataTable FromDataReader(
            IDataReader reader,
            int skip = 0,
            int take = int.MaxValue)
        {
            Guard.NotNull(reader, nameof(reader));

            if (reader.IsClosed)
            {
                throw new ArgumentException("This operation is invalid when the reader is closed.", "reader");
            }

            var columns = new List <IDataColumn>(reader.FieldCount);
            var data    = new List <object[]>();

            var schema = reader.GetSchemaTable();

            var nameCol = schema.Columns[SchemaTableColumn.ColumnName];
            var typeCol = schema.Columns[SchemaTableColumn.DataType];

            foreach (DataRow schemaRow in schema.Rows)
            {
                var column = new LightweightDataColumn((string)schemaRow[nameCol], (Type)schemaRow[typeCol]);
                columns.Add(column);
            }

            var fieldCount = reader.FieldCount;

            take = Math.Min(take, int.MaxValue - skip);

            int i = -1;

            while (reader.Read())
            {
                i++;

                if (skip > i)
                {
                    continue;
                }

                if (i >= skip + take)
                {
                    break;
                }

                var values = new object[fieldCount];
                reader.GetValues(values);
                data.Add(values);
            }

            var table = new LightweightDataTable(columns, data);

            return(table);
        }
        public void CanResolveColumnIndexNames()
        {
            var columns = new List<IDataColumn>
            {
                new LightweightDataColumn("Attr[Color]", typeof(string)),
                new LightweightDataColumn("Attr[Size]", typeof(string)),
                new LightweightDataColumn("material", typeof(string)),
                new LightweightDataColumn("Name[en]", typeof(string)),
                new LightweightDataColumn("Name[de]", typeof(string)),
                new LightweightDataColumn("Name[fr]", typeof(string)),
                new LightweightDataColumn("name_it", typeof(string)),
            };

            var table = new LightweightDataTable(columns, new List<object[]>());
            var segmenter = new ImportDataSegmenter(table, new ColumnMap());
            segmenter.ColumnMap.AddMapping("Attr[Material]", "material");
            segmenter.ColumnMap.AddMapping("Name[it]", "name_it");

            var attrs = segmenter.GetColumnIndexes("Attr");
            attrs.ShouldSequenceEqual(new string[] {"Material", "Color", "Size"});

            var langs = segmenter.GetColumnIndexes("Name");
            langs.ShouldSequenceEqual(new string[] { "it", "en", "de", "fr" });
        }
Beispiel #5
0
        private void ImportCoreInner(DataImporterContext ctx, string filePath)
        {
            if (ctx.ExecuteContext.Abort == DataExchangeAbortion.Hard)
            {
                return;
            }

            {
                var logHead = new StringBuilder();
                logHead.AppendLine();
                logHead.AppendLine(new string('-', 40));
                logHead.AppendLine("SmartStore.NET:\t\tv." + SmartStoreVersion.CurrentFullVersion);
                logHead.Append("Import profile:\t\t" + ctx.Request.Profile.Name);
                logHead.AppendLine(ctx.Request.Profile.Id == 0 ? " (volatile)" : " (Id {0})".FormatInvariant(ctx.Request.Profile.Id));

                logHead.AppendLine("Entity:\t\t\t" + ctx.Request.Profile.EntityType.ToString());
                logHead.AppendLine("File:\t\t\t" + Path.GetFileName(filePath));

                var customer = _services.WorkContext.CurrentCustomer;
                logHead.Append("Executed by:\t\t" + (customer.Email.HasValue() ? customer.Email : customer.SystemName));

                ctx.Log.Information(logHead.ToString());
            }

            if (!File.Exists(filePath))
            {
                throw new SmartException("File does not exist {0}.".FormatInvariant(filePath));
            }

            CsvConfiguration csvConfiguration = null;
            var extension = Path.GetExtension(filePath);

            if ((new string[] { ".csv", ".txt", ".tab" }).Contains(extension, StringComparer.OrdinalIgnoreCase))
            {
                var converter = new CsvConfigurationConverter();
                csvConfiguration = converter.ConvertFrom <CsvConfiguration>(ctx.Request.Profile.FileTypeConfiguration);
            }

            if (csvConfiguration == null)
            {
                csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
            }

            using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                ctx.ExecuteContext.DataTable = LightweightDataTable.FromFile(
                    Path.GetFileName(filePath),
                    stream,
                    stream.Length,
                    csvConfiguration,
                    ctx.Request.Profile.Skip,
                    ctx.Request.Profile.Take > 0 ? ctx.Request.Profile.Take : int.MaxValue
                    );

                try
                {
                    ctx.Importer.Execute(ctx.ExecuteContext);
                }
                catch (Exception exception)
                {
                    ctx.ExecuteContext.Abort = DataExchangeAbortion.Hard;
                    ctx.ExecuteContext.Result.AddError(exception, "The importer failed: {0}.".FormatInvariant(exception.ToAllMessages()));
                }

                if (ctx.ExecuteContext.IsMaxFailures)
                {
                    ctx.ExecuteContext.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
                }

                if (ctx.CancellationToken.IsCancellationRequested)
                {
                    ctx.ExecuteContext.Result.AddWarning("Import aborted. A cancellation has been requested.");
                }
            }
        }
        public static IDataTable FromDataReader(
			IDataReader reader, 
			int skip = 0, 
			int take = int.MaxValue)
        {
            Guard.ArgumentNotNull(() => reader);

            if (reader.IsClosed)
                throw new ArgumentException("This operation is invalid when the reader is closed.", "reader");

            var columns = new List<IDataColumn>(reader.FieldCount);
            var data = new List<object[]>();

            var schema = reader.GetSchemaTable();

            var nameCol = schema.Columns[SchemaTableColumn.ColumnName];
            var typeCol = schema.Columns[SchemaTableColumn.DataType];

            foreach (DataRow schemaRow in schema.Rows)
            {
                var column = new LightweightDataColumn((string)schemaRow[nameCol], (Type)schemaRow[typeCol]);
                columns.Add(column);
            }

            var fieldCount = reader.FieldCount;

            take = Math.Min(take, int.MaxValue - skip);

            int i = -1;
            while (reader.Read())
            {
                i++;

                if (skip > i)
                    continue;

                if (i >= skip + take)
                    break;

                var values = new object[fieldCount];
                reader.GetValues(values);
                data.Add(values);
            }

            var table = new LightweightDataTable(columns, data);

            return table;
        }