public FileInformation Read(FileInfo fileInfo) {
var fileInformation = new FileInformation(fileInfo);
var names = new List<string>();
var stream = System.IO.File.Open(fileInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
var isXml = fileInfo.Extension.Equals(".xlsx", StringComparison.OrdinalIgnoreCase);
var excelReader = isXml ? ExcelReaderFactory.CreateOpenXmlReader(stream) : ExcelReaderFactory.CreateBinaryReader(stream);
excelReader.Read();
for (var i = 0; i < excelReader.FieldCount; i++) {
var name = excelReader.GetString(i);
if (name != null)
names.Add(name);
}
excelReader.Close();
foreach (var name in names) {
var field = new Field("string", _request.DefaultLength, FieldType.NonKey, true, string.Empty) {
Name = name
};
fileInformation.Fields.Add(field);
}
return fileInformation;
}
private static TflProcess BuildProcess(
FileInformation fileInformation,
FileInspectionRequest request,
TflConnection output,
ILogger logger) {
var process = new TflProcess{
Name = request.EntityName,
Star = request.ProcessName,
StarEnabled = false,
ViewEnabled = false,
PipelineThreading = "MultiThreaded",
Connections = new List<TflConnection> {
new TflConnection{
Name = "input",
Provider = "file",
File = fileInformation.FileInfo.FullName,
Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture),
Start = fileInformation.FirstRowIsHeader ? 2 : 1
}.WithDefaults(),
output
},
Entities = new List<TflEntity> {
new TflEntity{
Name = request.EntityName,
Connection = "input",
PrependProcessNameToOutputName = false,
DetectChanges = false,
Fields = GetFields(new FieldInspector(logger).Inspect(fileInformation, request), logger, request.EntityName)
}.WithDefaults()
}
}.WithDefaults();
return process;
}
private static TflProcess BuildProcess(
FileInformation fileInformation,
FileInspectionRequest request,
TflConnection output,
ILogger logger) {
var process = new TflRoot().GetDefaultOf<TflProcess>(p => {
p.Name = request.EntityName;
p.Star = request.ProcessName;
p.StarEnabled = false;
p.ViewEnabled = false;
p.PipelineThreading = "MultiThreaded";
p.Connections = new List<TflConnection> {
p.GetDefaultOf<TflConnection>(c => {
c.Name = "input";
c.Provider = "file";
c.File = fileInformation.FileInfo.FullName;
c.Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture);
c.Start = fileInformation.FirstRowIsHeader ? 2 : 1;
}),
output
};
p.Entities = new List<TflEntity> {
p.GetDefaultOf<TflEntity>(e => {
e.Name = request.EntityName;
e.Connection = "input";
e.PrependProcessNameToOutputName = false;
e.DetectChanges = false;
e.Fields = GetFields(p, new FieldInspector(logger).Inspect(fileInformation, request), logger, request.EntityName);
})
};
});
return process;
}
public Fields Inspect(FileInformation fileInformation, FileInspectionRequest request) {
var process = new TflProcess{
Name = request.ProcessName,
StarEnabled = false,
ViewEnabled = false,
PipelineThreading = "MultiThreaded"
}.WithDefaults();
process.Connections = new List<TflConnection> {
new TflConnection {
Name = "input",
Provider = "file",
File = fileInformation.FileInfo.FullName,
Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture),
Start = fileInformation.FirstRowIsHeader ? 2 : 1
}.WithDefaults(),
new TflConnection {
Name = "output",
Provider = "internal"
}.WithDefaults()
};
process.Entities.Add(new TflEntity {
Name = request.EntityName,
PrependProcessNameToOutputName = false,
DetectChanges = false,
Sample = System.Convert.ToInt32(request.Sample)
}.WithDefaults());
foreach (var fd in fileInformation.Fields) {
var field = fd;
process.Entities[0].Fields.Add(new TflField{
Name = field.Name,
Length = field.Length,
Type = field.Type,
QuotedWith = field.QuotedWith
}.WithDefaults());
}
for (var i = 0; i < request.DataTypes.Count; i++) {
var dataType = request.DataTypes[i];
foreach (var field in fileInformation.Fields) {
var result = IsDataTypeField(field.Name, dataType);
process.Entities[0].CalculatedFields.Add(
new TflField{
Name = result,
Type = "bool",
Input = false,
Transforms = new List<TflTransform> {
new TflTransform {
Method = "typeconversion",
Type = dataType,
Parameter = field.Name,
IgnoreEmpty = request.IgnoreEmpty
}.WithDefaults()
}
}.WithDefaults()
);
}
}
foreach (var field in fileInformation.Fields) {
var result = LengthField(field.Name);
process.Entities[0].CalculatedFields.Add(
new TflField {
Name = result,
Type = "int32",
Transforms = new List<TflTransform> {
new TflTransform {
Method = "length",
Parameter = field.Name
}.WithDefaults()
}
}.WithDefaults()
);
}
var runner = ProcessFactory.CreateSingle(new TflRoot(process).Processes[0], _logger);
var results = runner.Execute().ToList();
if (results.Count <= 0) {
_logger.Warn("Nothing imported from in {0}!", fileInformation.FileInfo.Name);
return fileInformation.Fields;
}
foreach (var field in fileInformation.Fields) {
if (!results.All(row => row[field.Name].Equals(string.Empty))) {
foreach (var dataType in request.DataTypes) {
var result = IsDataTypeField(field.Name, dataType);
if (!results.All(row => row[result].Equals(true)))
continue;
field.Type = dataType;
field.Length = request.MinLength.ToString(CultureInfo.InvariantCulture);
break;
}
}
if (!field.Type.Equals("string"))
continue;
var length = results.Max(row => (int)row[LengthField(field.Name)]) + 1;
if (request.MaxLength > 0 && length > request.MaxLength) {
length = request.MaxLength;
}
if (request.MinLength > 0 && length < request.MinLength) {
length = request.MinLength;
}
field.Length = length.ToString(CultureInfo.InvariantCulture);
}
return fileInformation.Fields;
}
public Fields Inspect(FileInformation fileInformation) {
return Inspect(fileInformation, new FileInspectionRequest(fileInformation.FileInfo.Name));
}