public SeparatedFileSchema BuildSchema(RawFileSchema rawSchema)
{
var separatedFileSchema = new SeparatedFileSchema();
var rowDefinitionList = rawSchema.Schema.RowDefinitions;
separatedFileSchema.Delimeter = rawSchema.Delimeter;
foreach (var rowDefinition in rowDefinitionList)
{
var columnDefinitions = rowDefinition.ColumnDefinitions;
var rowSchema = new SeparatedValueSchema();
foreach (var columnDefinition in columnDefinitions)
{
var definition = GetColumnDefinition(columnDefinition);
definition.NullHandler = ConstantNullHandler.For("NULL");
rowSchema.AddColumn(definition);
}
separatedFileSchema.SeparatedRecordSchemas.Add(new SeparatedRecordSchema
{
RecordIdentifier = rowDefinition.Identifier,
SeparatedValueSchema = rowSchema
});
}
return(separatedFileSchema);
}
public FixedLengthFileSchema BuildSchema(RawFileSchema rawSchema)
{
var fixedLengthFileSchema = new FixedLengthFileSchema();
var rowDefinitionList = rawSchema.Schema.RowDefinitions;
foreach (var rowDefinition in rowDefinitionList)
{
var columnDefinitions = rowDefinition.ColumnDefinitions;
var rowSchema = new FixedLengthSchema();
foreach (var columnDefinition in columnDefinitions)
{
var definition = GetColumnDefinition(columnDefinition);
definition.NullHandler = ConstantNullHandler.For("NULL");
rowSchema.AddColumn(
definition, BuildWindow(columnDefinition)
);
}
fixedLengthFileSchema.FixedLengthRecordSchemas.Add(new FixedLengthRecordSchema
{
RecordIdentifier = rowDefinition.Identifier,
FixedLengthSchema = rowSchema
});
}
return(fixedLengthFileSchema);
}
public void Generate(RawFileSchema rawFileSchema, DataSource dataSource)
{
var validationErrors = _schemaValidator.ValidateSchema(rawFileSchema);
if (validationErrors.Any())
{
throw new Exception("Errors");
}
var fileFormat = rawFileSchema.FileFormat.ToEnum <FileSchemaType>();
var rootDirectory = Path.Combine(_rootFolder, rawFileSchema.RootOutputDir);
if (!Directory.Exists(rootDirectory))
{
Directory.CreateDirectory(rootDirectory);
}
var generatedFilePath = Path.Combine(rootDirectory, rawFileSchema.OutputFileName);
if (!string.IsNullOrWhiteSpace(rawFileSchema.OutputFileExtension))
{
generatedFilePath += rawFileSchema.OutputFileExtension;
}
switch (fileFormat)
{
case FileSchemaType.Fixed:
var fixedLengthFileSchemaBuilder = new FixedLengthSchemaBuilder();
var fixedFileSchema = fixedLengthFileSchemaBuilder.BuildSchema(rawFileSchema);
_fileWriter.Write(generatedFilePath, dataSource, fixedFileSchema);
break;
case FileSchemaType.Separated:
var separatedFileSchemaBuilder = new SeparatedSchemaBuilder();
var separatedFileSchema = separatedFileSchemaBuilder.BuildSchema(rawFileSchema);
_fileWriter.Write(generatedFilePath, dataSource, separatedFileSchema);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public IEnumerable <ValidationResult> ValidateSchema(RawFileSchema fileSchema)
{
var validationFailureResults = new List <ValidationResult>();
var validationResult = _fluentSchemaValidator.Validate(fileSchema);
if (!validationResult.IsValid)
{
foreach (var validationError in validationResult.Errors)
{
validationFailureResults.Add(new ValidationResult
{
ColumnName = validationError.PropertyName,
ErrorMessage = validationError.ErrorMessage,
AttemptedValue = validationError.AttemptedValue.ToString()
});
}
}
return(validationFailureResults);
}
public void ParseFile(RawFileSchema rawFileSchema)
{
var validationErrors = _schemaValidator.ValidateSchema(rawFileSchema);
if (validationErrors.Any())
{
throw new Exception("Errors");
}
var fileFormat = rawFileSchema.FileFormat.ToEnum <FileSchemaType>();
var generatedFilePath = Path.Combine(rawFileSchema.RootOutputDir, rawFileSchema.OutputFileName);
if (!string.IsNullOrWhiteSpace(rawFileSchema.OutputFileExtension))
{
generatedFilePath += rawFileSchema.OutputFileExtension;
}
switch (fileFormat)
{
case FileSchemaType.Fixed:
var fixedLengthFileSchemaBuilder = new FixedLengthSchemaBuilder();
var fixedFileSchema = fixedLengthFileSchemaBuilder.BuildSchema(rawFileSchema);
using (var sr = new StreamReader(File.OpenRead(generatedFilePath)))
{
//var fixedLengthSchemaReader = new FlatFiles.FixedLengthReader(sr, );
}
break;
case FileSchemaType.Separated:
var separatedFileSchemaBuilder = new SeparatedSchemaBuilder();
var separatedFileSchema = separatedFileSchemaBuilder.BuildSchema(rawFileSchema);
//_fileWriter.Write(generatedFilePath, dataSource, separatedFileSchema);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public Core.ResultDefinition ValidateFile()
{
var schemaData = string.Empty;
var schemaParser = new SchemaParser();
var validationResults = new List <ValidationResult>();
var validator = new Validators.FixedLengthValidator();
if (this.RawSchema == null)
{
try
{
using (var sr = new StreamReader(File.OpenRead(this.SchemaFilePath)))
{
schemaData = sr.ReadToEnd();
}
}
catch (Exception ex)
{
var errorMsg = String.Format(null, SharedResources.SchemaError, ex.Message);
Console.WriteLine($"{errorMsg}");
}
this.RawSchema = schemaParser.ParseSchema(schemaData);
}
var separatedFileSchemaBuilder = new SeparatedSchemaBuilder();
var separatedFileSchema = separatedFileSchemaBuilder.BuildSchema(this.RawSchema);
var schemas = separatedFileSchema.SeparatedRecordSchemas;
var count = 0;
var options = new SeparatedValueOptions
{
Separator = separatedFileSchema.Delimeter,
PreserveWhiteSpace = true
};
foreach (var inputLine in File.ReadLines(this.FilePath))
{
Console.WriteLine($"Started parsing line...{count}");
if (inputLine != null)
{
SeparatedValueSchema actualSchema;
TextReader stringReader = new StringReader(inputLine);
var schema = schemas.FirstOrDefault(x => inputLine.StartsWith(x.RecordIdentifier));
count++;
var innerResults = new List <ValidationResult>();
var line = inputLine;
if (schema != null)
{
var parser = new SeparatedValueReader(stringReader, schema.SeparatedValueSchema, options);
if (!inputLine.Contains(options.Separator))
{
var error = new ValidationResult
{
Record = inputLine,
ErrorMessages = new List <string>
{
$"Column Separator Error: Record could not be parsed as separator {options.Separator} defined in schema is not found.",
},
HasErrors = true,
RowNumber = count,
};
validationResults.Add(error);
continue;
}
try
{
actualSchema = parser.GetSchema();
parser.Read();
var values = parser.GetValues();
for (int i = 0; i < values.Length; i++)
{
var validationResult = new ValidationResult
{
Record = inputLine,
ColumnName = actualSchema.ColumnDefinitions[i].ColumnName,
ColumnType = actualSchema.ColumnDefinitions[i].ColumnType.Name,
ActualRowLength = inputLine.Length,
MaxColumns = actualSchema.ColumnDefinitions.HandledCount,
ParsedValueLength = (values[i].ToString() ?? string.Empty).Length,
RawValueLength = (values[i].ToString() ?? string.Empty).Length,
ParsedValue = values[i].ToString() ?? string.Empty,
RawValue = values[i].ToString() ?? string.Empty,
RowNumber = count,
ParsedValuesCount = values.Count()
};
}
}
catch (Exception ex)
{
var errorMsg = string.Empty;
if (ex.InnerException != null && ex.InnerException.Message != null)
{
errorMsg = ex.InnerException.Message;
Console.WriteLine(ex.InnerException.Message);
}
else
{
errorMsg = ex.Message;
Console.WriteLine(ex.Message);
}
var error = new ValidationResult
{
RowNumber = count,
Record = inputLine,
ErrorMessages = new List <string>
{
$"Exception: {errorMsg}"
},
HasErrors = true
};
validationResults.Add(error);
}
}
else
{
var error = new ValidationResult
{
RowNumber = count,
Record = inputLine,
ErrorMessages = new List <string>
{
$"Exception: The row does not contain a record identifier that match the schema."
},
HasErrors = true
};
validationResults.Add(error);
}
}
}
var results = new Core.ResultDefinition
{
Results = validationResults,
TotalLinesProcessed = count,
FileFormat = this.RawSchema.FileFormat
};
return(results);
}
public SeparatedFileValidator(string filePath, string schemaFilePath, RawFileSchema rawFileSchema)
{
this.FilePath = filePath;
this.SchemaFilePath = schemaFilePath;
this.RawSchema = rawFileSchema;
}
public Core.ResultDefinition ValidateFile()
{
var schemaData = string.Empty;
var schemaParser = new SchemaParser();
var validationResults = new List <ValidationResult>();
var validator = new Validators.FixedLengthValidator();
if (this.RawSchema == null)
{
try
{
using (var sr = new StreamReader(File.OpenRead(this.SchemaFilePath)))
{
schemaData = sr.ReadToEnd();
}
}
catch (Exception ex)
{
var errorMsg = String.Format(null, SharedResources.SchemaError, ex.Message);
Console.WriteLine($"{errorMsg}");
}
this.RawSchema = schemaParser.ParseSchema(schemaData);
}
var fixedLengthFileSchemaBuilder = new FixedLengthSchemaBuilder();
var fixedFileSchema = fixedLengthFileSchemaBuilder.BuildSchema(this.RawSchema);
var schemas = fixedFileSchema.FixedLengthRecordSchemas;
var count = 0;
foreach (var inputLine in File.ReadLines(this.FilePath))
{
Console.WriteLine($"Started parsing line...{count}");
if (inputLine != null)
{
FixedLengthSchema actualSchema;
TextReader stringReader = new StringReader(inputLine);
var schema = schemas.FirstOrDefault(x => inputLine.StartsWith(x.RecordIdentifier));
count++;
var innerResults = new List <ValidationResult>();
var line = inputLine;
if (schema != null)
{
var parser = new FixedLengthReader(stringReader, schema.FixedLengthSchema);
try
{
actualSchema = parser.GetSchema();
parser.Read();
var rawValues = GetRawValues(inputLine, schema.FixedLengthSchema);
var values = parser.GetValues();
for (int i = 0; i < values.Length; i++)
{
var validationResult = new ValidationResult
{
Record = inputLine,
ColumnName = actualSchema.ColumnDefinitions[i].ColumnName,
ColumnType = actualSchema.ColumnDefinitions[i].ColumnType.Name,
ActualRowLength = inputLine.Length,
MaxRowLength = actualSchema.TotalWidth,
ParsedValueLength = (values[i].ToString() ?? string.Empty).Length,
RawValueLength = rawValues[i].ToString().Length,
ColumnLength = actualSchema.Windows[i].Width,
FillCharacter = actualSchema.Windows[i].FillCharacter != null ? actualSchema.Windows[i].FillCharacter.ToString() : "Not specified",
TextAlignment = actualSchema.Windows[i].Alignment != null ? actualSchema.Windows[i].Alignment.ToString() : "Not specified",
ParsedValue = values[i].ToString() ?? string.Empty,
RawValue = rawValues[i].ToString() ?? string.Empty,
RowNumber = count
};
var failures = validator.Validate(validationResult);
if (failures.Errors != null && failures.Errors.Any() && !failures.IsValid)
{
var errors = new List <string>();
foreach (var error in failures.Errors)
{
errors.Add(error.ErrorMessage);
}
validationResult.ErrorMessages = errors;
validationResult.HasErrors = true;
line = line.Replace(validationResult.RawValue, $"<strong style='color:{HtmlColors[i%HtmlColors.Length]}'>{validationResult.RawValue}</strong>");
validationResult.Record = line;
validationResults.Add(validationResult);
}
}
}
catch (Exception ex)
{
var errorMsg = string.Empty;
if (ex.InnerException != null && ex.InnerException.Message != null)
{
errorMsg = ex.InnerException.Message;
Console.WriteLine(ex.InnerException.Message);
}
else
{
errorMsg = ex.Message;
Console.WriteLine(ex.Message);
}
var error = new ValidationResult
{
RowNumber = count,
Record = inputLine,
ErrorMessages = new List <string>
{
$"Exception: {errorMsg}"
},
HasErrors = true
};
validationResults.Add(error);
}
}
else
{
var error = new ValidationResult
{
RowNumber = count,
Record = inputLine,
ErrorMessages = new List <string>
{
$"Exception: The row does not contain a record identifier that match the schema."
},
HasErrors = true
};
validationResults.Add(error);
}
}
}
var results = new Core.ResultDefinition
{
Results = validationResults,
TotalLinesProcessed = count,
FileFormat = this.RawSchema.FileFormat,
FilePath = this.FilePath,
SchemaPath = this.SchemaFilePath
};
return(results);
}
public FixedLengthFileValidator(string filePath, string schemaFilePath, RawFileSchema rawFileSchema)
{
this.FilePath = filePath;
this.SchemaFilePath = schemaFilePath;
this.RawSchema = rawFileSchema;
}