public FileInformation Read(FileInfo fileInfo) { var fileInformation = new FileInformation(fileInfo); var names = new List<string>(); var stream = System.IO.File.Open(fileInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); var isXml = fileInfo.Extension.Equals(".xlsx", StringComparison.OrdinalIgnoreCase); var excelReader = isXml ? ExcelReaderFactory.CreateOpenXmlReader(stream) : ExcelReaderFactory.CreateBinaryReader(stream); excelReader.Read(); for (var i = 0; i < excelReader.FieldCount; i++) { var name = excelReader.GetString(i); if (name != null) names.Add(name); } excelReader.Close(); foreach (var name in names) { var field = new Field("string", _request.DefaultLength, FieldType.NonKey, true, string.Empty) { Name = name }; fileInformation.Fields.Add(field); } return fileInformation; }
private static TflProcess BuildProcess( FileInformation fileInformation, FileInspectionRequest request, TflConnection output, ILogger logger) { var process = new TflProcess{ Name = request.EntityName, Star = request.ProcessName, StarEnabled = false, ViewEnabled = false, PipelineThreading = "MultiThreaded", Connections = new List<TflConnection> { new TflConnection{ Name = "input", Provider = "file", File = fileInformation.FileInfo.FullName, Delimiter = fileInformation.Delimiter == default(char) ? "|" : fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture), Start = fileInformation.FirstRowIsHeader ? 2 : 1 }.WithDefaults(), output }, Entities = new List<TflEntity> { new TflEntity{ Name = request.EntityName, Connection = "input", PrependProcessNameToOutputName = false, DetectChanges = false, Fields = GetFields(new FieldInspector(logger).Inspect(fileInformation, request), logger, request.EntityName) }.WithDefaults() } }.WithDefaults(); return process; }
private static TflProcess BuildProcess( FileInformation fileInformation, FileInspectionRequest request, TflConnection output, ILogger logger) { var process = new TflRoot().GetDefaultOf<TflProcess>(p => { p.Name = request.EntityName; p.Star = request.ProcessName; p.StarEnabled = false; p.ViewEnabled = false; p.PipelineThreading = "MultiThreaded"; p.Connections = new List<TflConnection> { p.GetDefaultOf<TflConnection>(c => { c.Name = "input"; c.Provider = "file"; c.File = fileInformation.FileInfo.FullName; c.Delimiter = fileInformation.Delimiter == default(char) ? "|" : fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture); c.Start = fileInformation.FirstRowIsHeader ? 2 : 1; }), output }; p.Entities = new List<TflEntity> { p.GetDefaultOf<TflEntity>(e => { e.Name = request.EntityName; e.Connection = "input"; e.PrependProcessNameToOutputName = false; e.DetectChanges = false; e.Fields = GetFields(p, new FieldInspector(logger).Inspect(fileInformation, request), logger, request.EntityName); }) }; }); return process; }
public Fields Inspect(FileInformation fileInformation, FileInspectionRequest request) { var process = new TflProcess{ Name = request.ProcessName, StarEnabled = false, ViewEnabled = false, PipelineThreading = "MultiThreaded" }.WithDefaults(); process.Connections = new List<TflConnection> { new TflConnection { Name = "input", Provider = "file", File = fileInformation.FileInfo.FullName, Delimiter = fileInformation.Delimiter == default(char) ? "|" : fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture), Start = fileInformation.FirstRowIsHeader ? 2 : 1 }.WithDefaults(), new TflConnection { Name = "output", Provider = "internal" }.WithDefaults() }; process.Entities.Add(new TflEntity { Name = request.EntityName, PrependProcessNameToOutputName = false, DetectChanges = false, Sample = System.Convert.ToInt32(request.Sample) }.WithDefaults()); foreach (var fd in fileInformation.Fields) { var field = fd; process.Entities[0].Fields.Add(new TflField{ Name = field.Name, Length = field.Length, Type = field.Type, QuotedWith = field.QuotedWith }.WithDefaults()); } for (var i = 0; i < request.DataTypes.Count; i++) { var dataType = request.DataTypes[i]; foreach (var field in fileInformation.Fields) { var result = IsDataTypeField(field.Name, dataType); process.Entities[0].CalculatedFields.Add( new TflField{ Name = result, Type = "bool", Input = false, Transforms = new List<TflTransform> { new TflTransform { Method = "typeconversion", Type = dataType, Parameter = field.Name, IgnoreEmpty = request.IgnoreEmpty }.WithDefaults() } }.WithDefaults() ); } } foreach (var field in fileInformation.Fields) { var result = LengthField(field.Name); process.Entities[0].CalculatedFields.Add( new TflField { Name = result, Type = "int32", Transforms = new List<TflTransform> { new TflTransform { Method = "length", Parameter = field.Name }.WithDefaults() } }.WithDefaults() ); } var runner = ProcessFactory.CreateSingle(new TflRoot(process).Processes[0], _logger); var results = runner.Execute().ToList(); if (results.Count <= 0) { _logger.Warn("Nothing imported from in {0}!", fileInformation.FileInfo.Name); return fileInformation.Fields; } foreach (var field in fileInformation.Fields) { if (!results.All(row => row[field.Name].Equals(string.Empty))) { foreach (var dataType in request.DataTypes) { var result = IsDataTypeField(field.Name, dataType); if (!results.All(row => row[result].Equals(true))) continue; field.Type = dataType; field.Length = request.MinLength.ToString(CultureInfo.InvariantCulture); break; } } if (!field.Type.Equals("string")) continue; var length = results.Max(row => (int)row[LengthField(field.Name)]) + 1; if (request.MaxLength > 0 && length > request.MaxLength) { length = request.MaxLength; } if (request.MinLength > 0 && length < request.MinLength) { length = request.MinLength; } field.Length = length.ToString(CultureInfo.InvariantCulture); } return fileInformation.Fields; }
public Fields Inspect(FileInformation fileInformation) { return Inspect(fileInformation, new FileInspectionRequest(fileInformation.FileInfo.Name)); }