public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type)
{
if (!(dataSourceLocation is XMLDataSourceLocation))
{
return new ParsingResult(ResultLevel.FATAL, dataSourceLocation.GetType().ToString() + " is not supported by XML Node Parser", null, dataSourceLocation);
}
if (!(dataToImport is XMLDataToImport))
{
return new ParsingResult(ResultLevel.FATAL, dataToImport.GetType().ToString() + " is not supported by XML Node Parser", null, dataSourceLocation);
}
var castedDataToImport = dataToImport as XMLDataToImport;
var castedDataSourceLocation = dataSourceLocation as XMLDataSourceLocation;
try
{
var rawData = GetRawDataValue(castedDataSourceLocation, castedDataToImport);
var parsedValue = ParseRawValue(type, rawData);
return new ParsingResult(ResultLevel.INFO, "Parsing value successfully", parsedValue, dataSourceLocation);
}
catch (Exception e)
{
return new ParsingResult(ResultLevel.FATAL, "Exception caught: " + e, null, dataSourceLocation);
}
}
public IExtractedDataset <T> Extract <T>(IDataToImport dataToImport) where T : new()
{
var extractedDataset = new ExtractedDataset <T>(_thresholdLevel);
var validatedDataToImportResult = ValidateDataToImport(dataToImport);
extractedDataset.AddParsingResults(validatedDataToImportResult.Item2);
if (!validatedDataToImportResult.Item1)
{
//format not valid, return
return(extractedDataset);
}
var csvDataSource = dataToImport as XMLDataToImport;
var rawData = csvDataSource.Data as XDocument;
XNamespace rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
var parentNodes = rawData.Descendants().Where(x => x.Name.LocalName == _rootNode);
foreach (XElement element in parentNodes)
{
var extractResultsForNode = ExtractDataForSingleNode <T>(_extractConfigurations, new XMLDataToImport("", new XDocument(element)));
extractedDataset.AddParsingResults(extractResultsForNode);
}
return(extractedDataset);
}
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type)
{
if (!(dataSourceLocation is XMLDataSourceLocation))
{
return(new ParsingResult(ResultLevel.FATAL, dataSourceLocation.GetType().ToString() + " is not supported by XML Node Parser", null, dataSourceLocation));
}
if (!(dataToImport is XMLDataToImport))
{
return(new ParsingResult(ResultLevel.FATAL, dataToImport.GetType().ToString() + " is not supported by XML Node Parser", null, dataSourceLocation));
}
var castedDataToImport = dataToImport as XMLDataToImport;
var castedDataSourceLocation = dataSourceLocation as XMLDataSourceLocation;
try
{
var rawData = GetRawDataValue(castedDataSourceLocation, castedDataToImport);
var parsedValue = ParseRawValue(type, rawData);
return(new ParsingResult(ResultLevel.INFO, "Parsing value successfully", parsedValue, dataSourceLocation));
}
catch (Exception e)
{
return(new ParsingResult(ResultLevel.FATAL, "Exception caught: " + e, null, dataSourceLocation));
}
}
public IExtractedDataset <T> Extract <T>(IDataToImport dataToImport) where T : new()
{
var extractedDataset = new ExtractedDataset <T>(_thresholdLevel);
var validatedDataToImportResult = ValidateDataToImport(dataToImport);
extractedDataset.AddParsingResults(validatedDataToImportResult.Item2);
if (!validatedDataToImportResult.Item1)
{
//format not valid, return
return(extractedDataset);
}
var results = new List <IResult>();
var model = new T();
IDataSourceLocation currentLocation = null;
foreach (var configuration in _extractConfigurations)
{
if (configuration is SimpleXMLExtractConfiguration)
{
currentLocation = new XMLDataSourceLocation(((SimpleXMLExtractConfiguration)configuration).ElementName, ((SimpleXMLExtractConfiguration)configuration).AttributeName);
}
results.AddRange(((SimpleXMLExtractConfiguration)configuration).ExtractData(model, dataToImport, currentLocation));
}
var parsingResult = new ParsingResult(ResultLevel.DEBUG, "Extract data from single row success", model, null);
results.Add(parsingResult);
extractedDataset.AddParsingResults(results);
return(extractedDataset);
}
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type)
{
var castedDataToImport = dataToImport as JSONDataToImport;
var castedJObject = castedDataToImport.Data as JObject;
var castedDataSourceLocation = dataSourceLocation as JSONDataSourceLocation;
var locationPath = castedDataSourceLocation.Path;
if (castedDataSourceLocation.IsArray)
{
var rawValues = castedJObject.SelectTokens(locationPath).Values<string>();
var valueParser = _parserFactory.GetValueParser(type);
var parsedValue = from value in rawValues
select valueParser.Parse(value);
return new ParsingResult(ResultLevel.INFO, "Parse value from " + dataToImport.ToString() + " successfully, the value is " + parsedValue.ToString(), parsedValue.ToList(), castedDataSourceLocation);
}
else
{
var rawValue = (castedJObject.SelectTokens(locationPath).ElementAt(castedDataSourceLocation.Index.Value) as JValue).Value;
var valueParser = _parserFactory.GetValueParser(type);
var parsedValue = valueParser.Parse(rawValue);
return new ParsingResult(ResultLevel.INFO, "Parse value from " + dataToImport.ToString() + " successfully " + parsedValue, parsedValue, castedDataSourceLocation);
}
}
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type)
{
if (!(dataSourceLocation is CSVDataSourceLocation))
{
return new ParsingResult(ResultLevel.FATAL, dataSourceLocation.GetType().ToString() + " is not supported by CSV Cell Parser", null, null);
}
if (!(dataToImport is CSVDataToImport))
{
return new ParsingResult(ResultLevel.FATAL, dataToImport.GetType().ToString() + " is not supported by CSV Cell Parser", null, null);
}
var castedDataToImport = dataToImport as CSVDataToImport;
var castedDataSourceLocation = dataSourceLocation as CSVDataSourceLocation;
try
{
var rawData = GetRawDataValue(castedDataSourceLocation, castedDataToImport);
var parsedValue = ParseRawValue(type, rawData);
return new ParsingResult(ResultLevel.INFO, "Parsing value successfully", parsedValue, castedDataSourceLocation);
}
catch (IndexOutOfRangeException)
{
return new ParsingResult(ResultLevel.FATAL, "Index is out of range", null, castedDataSourceLocation);
}
}
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type)
{
var castedDataToImport = dataToImport as JSONDataToImport;
var castedJObject = castedDataToImport.Data as JObject;
var castedDataSourceLocation = dataSourceLocation as JSONDataSourceLocation;
var locationPath = castedDataSourceLocation.Path;
if (castedDataSourceLocation.IsArray)
{
var rawValues = castedJObject.SelectTokens(locationPath).Values <string>();
var valueParser = _parserFactory.GetValueParser(type);
var parsedValue = from value in rawValues
select valueParser.Parse(value);
return(new ParsingResult(ResultLevel.INFO, "Parse value from " + dataToImport.ToString() + " successfully, the value is " + parsedValue.ToString(), parsedValue.ToList(), castedDataSourceLocation));
}
else
{
var rawValue = (castedJObject.SelectTokens(locationPath).ElementAt(castedDataSourceLocation.Index.Value) as JValue).Value;
var valueParser = _parserFactory.GetValueParser(type);
var parsedValue = valueParser.Parse(rawValue);
return(new ParsingResult(ResultLevel.INFO, "Parse value from " + dataToImport.ToString() + " successfully " + parsedValue, parsedValue, castedDataSourceLocation));
}
}
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type)
{
if (!(dataSourceLocation is CSVDataSourceLocation))
{
return(new ParsingResult(ResultLevel.FATAL, dataSourceLocation.GetType().ToString() + " is not supported by CSV Cell Parser", null, null));
}
if (!(dataToImport is CSVDataToImport))
{
return(new ParsingResult(ResultLevel.FATAL, dataToImport.GetType().ToString() + " is not supported by CSV Cell Parser", null, null));
}
var castedDataToImport = dataToImport as CSVDataToImport;
var castedDataSourceLocation = dataSourceLocation as CSVDataSourceLocation;
try
{
var rawData = GetRawDataValue(castedDataSourceLocation, castedDataToImport);
var parsedValue = ParseRawValue(type, rawData);
return(new ParsingResult(ResultLevel.INFO, "Parsing value successfully", parsedValue, castedDataSourceLocation));
}
catch (IndexOutOfRangeException)
{
return(new ParsingResult(ResultLevel.FATAL, "Index is out of range", null, castedDataSourceLocation));
}
}
public override IEnumerable<IResult> ExtractData(object model, IDataToImport dataToImport)
{
var extractResult = _dataImporter.Extract<SampleFileData>(dataToImport);
_valueAssigner.AssignValue(model, _propertyPath, extractResult.ExtractedEntities, typeof(SampleFileData));
return extractResult.AllParsingResults.Where(x => x is BaseResult);
}
public override IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport)
{
var extractResult = _dataImporter.Extract <ChemistryFileData>(dataToImport);
_valueAssigner.AssignValue(model, _propertyPath, extractResult.ExtractedEntities, typeof(ChemistryFileData));
return(extractResult.AllParsingResults.Where(x => x is BaseResult));
}
private Tuple <bool, IEnumerable <IResult> > ValidateHeaderFileDataToImport(IDataToImport dataToImport)
{
var allValidationResults = from rule in _validationRules
select rule.Validate(dataToImport);
var isValid = !allValidationResults.Any(x => ResultLevelHelper.LevelIsHigherThanOrEqualToThreshold(_thresholdLevel, x.Level));
return(new Tuple <bool, IEnumerable <IResult> >(isValid, allValidationResults));
}
public override IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport, IDataSourceLocation currentLocation)
{
if (!(dataToImport is JSONDataToImport))
{
throw new ArgumentException("The SimpleJSONExtractConfiguration only accept JSONDataToImport to extract data");
}
if (!(currentLocation is JSONDataSourceLocation))
{
throw new ArgumentException("The SimpleJSONExtractConfiguration only accept JSONDataSourceLocation to extract data");
}
throw new NotImplementedException();
}
public IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport, IDataSourceLocation locationToParse)
{
var results = new List <IResult>();
try
{
var parsingResult = _parser.Parse(dataToImport, locationToParse, _propertyType) as IParsingResult;
_valueAssigner.AssignValue(model, _propertyPath, parsingResult.Value, _propertyType);
results.Add(new BaseResult(ResultLevel.DEBUG, string.Format("Extract data from xml file and assign to model {0}", locationToParse.ToString())));
return(results);
}
catch (Exception ex)
{
results.Add(new BaseResult(ResultLevel.ERROR, string.Format("Extract data from xml file and assign to model fail. {0}", locationToParse.ToString())));
}
return(results);
}
public IResult Validate(IDataToImport dataToImport)
{
var csvDataToImport = (CSVDataToImport)dataToImport;
if (csvDataToImport != null)
{
if(string.IsNullOrEmpty(csvDataToImport.FileName))
{
return new BaseResult(ResultLevel.ERROR, "System fail to compare file extension because data file name is empty/null");
}
else
{
var isValid = _caseSensitive ?
csvDataToImport.FileName.EndsWith(_acceptedFileExtension, StringComparison.Ordinal) :
csvDataToImport.FileName.EndsWith(_acceptedFileExtension, StringComparison.OrdinalIgnoreCase);
if (isValid)
{
return new BaseResult(ResultLevel.INFO,
string.Format("File name {0} matches the extension {1}",
csvDataToImport.FileName,
_acceptedFileExtension));
}
else
{
return new BaseResult(ResultLevel.ERROR,
string.Format("File name {0} does not match the extension {1}",
csvDataToImport.FileName,
_acceptedFileExtension));
}
}
}
else
{
return new BaseResult(ResultLevel.ERROR, "Data to import is not in CSV format");
}
}
public IResult Validate(IDataToImport dataToImport)
{
try
{
var valueParser = _parserFactory.GetElementParser(_elementValueType);
var parsedValue = valueParser.Parse(dataToImport, _location, _elementValueType) as IParsingResult;
var isCriteriaMet = _criteria.Meet(parsedValue.Value);
if (isCriteriaMet)
{
return(new BaseResult(ResultLevel.INFO, "Criteria matches"));
}
else
{
return(new BaseResult(ResultLevel.ERROR, "Criteria not matches"));
}
}
catch (Exception)
{
return(new BaseResult(ResultLevel.ERROR, "System not able to compare to the criteria since parsing value from data fail. Failing at " + _location.ToString()));
}
}
public override IEnumerable<IResult> ExtractData(object model, IDataToImport dataToImport, IDataSourceLocation currentLocation)
{
var results = new List<IResult>();
try
{
var parsingResult = _parser.Parse(dataToImport, currentLocation, _propertyType) as IParsingResult;
_valueAssigner.AssignValue(model, _propertyPath, parsingResult.Value, _propertyType);
results.Add(new BaseResult(ResultLevel.DEBUG, string.Format("Extract data from csv file and assign to model {0}", currentLocation.ToString())));
return results;
}
catch(Exception ex)
{
results.Add(new BaseResult(ResultLevel.ERROR, string.Format("Extract data from csv file and assign to model fail. {0}", currentLocation.ToString())));
}
return results;
}
public void CSVParseTestSuccess()
{
var dataSource = new CSVDataSource(_csvDataFilePath);
var dataSourceLocation = new CSVDataSourceLocation(6, 3);
var dataDateTimeSourceLocation = new CSVDataSourceLocation(6, 0);
dataToImport = dataSource.FetchData();
var parser = new CellParser(new DefaultParserFactory());
var parseDoubleResult = parser.Parse <double>(dataToImport, dataSourceLocation);
Assert.NotNull(parseDoubleResult);
Assert.AreEqual(parseDoubleResult.Level, ResultLevel.INFO);
Assert.AreEqual(parseDoubleResult.Message, "Parsing value successfully");
Assert.AreEqual(0.3, ((IParsingResult)parseDoubleResult).Value);
var parseDateTimeResult = parser.Parse <DateTime>(dataToImport, dataDateTimeSourceLocation);
Assert.NotNull(parseDateTimeResult);
Assert.AreEqual(parseDateTimeResult.Level, ResultLevel.INFO);
Assert.AreEqual(parseDateTimeResult.Message, "Parsing value successfully");
Assert.AreEqual(new DateTime(2013, 12, 12, 14, 0, 0), ((IParsingResult)parseDateTimeResult).Value);
}
public IResult Validate(IDataToImport dataToImport)
{
var csvDataToImport = (CSVDataToImport)dataToImport;
if (csvDataToImport != null)
{
if (string.IsNullOrEmpty(csvDataToImport.FileName))
{
return(new BaseResult(ResultLevel.ERROR, "System fail to compare file extension because data file name is empty/null"));
}
else
{
var isValid = _caseSensitive ?
csvDataToImport.FileName.EndsWith(_acceptedFileExtension, StringComparison.Ordinal) :
csvDataToImport.FileName.EndsWith(_acceptedFileExtension, StringComparison.OrdinalIgnoreCase);
if (isValid)
{
return(new BaseResult(ResultLevel.INFO,
string.Format("File name {0} matches the extension {1}",
csvDataToImport.FileName,
_acceptedFileExtension)));
}
else
{
return(new BaseResult(ResultLevel.ERROR,
string.Format("File name {0} does not match the extension {1}",
csvDataToImport.FileName,
_acceptedFileExtension)));
}
}
}
else
{
return(new BaseResult(ResultLevel.ERROR, "Data to import is not in CSV format"));
}
}
public IResult Validate(IDataToImport dataToImport)
{
try
{
var valueParser = _parserFactory.GetCellParser(_cellValueType);
var parsedValue = valueParser.Parse(dataToImport, _location, _cellValueType) as IParsingResult;
var isCriteriaMet = _criteria.Meet(parsedValue.Value);
if (isCriteriaMet)
{
return new BaseResult(ResultLevel.INFO, "Criteria matches");
}
else
{
return new BaseResult(ResultLevel.ERROR, "Criteria not matches");
}
}
catch(Exception)
{
return new BaseResult(ResultLevel.ERROR, "System not able to compare to the criteria since parsing value from data fail.");
}
}
public override IExtractedDataset <T> Extract <T>(IDataToImport dataToImport)
{
var extractedDataset = new ExtractedDataset <T>(_thresholdLevel);
var validatedDataToImportResult = ValidateDataToImport(dataToImport);
extractedDataset.AddParsingResults(validatedDataToImportResult.Item2);
if (!validatedDataToImportResult.Item1)
{
//format not valid, return
return(extractedDataset);
}
var csvDataSource = dataToImport as CSVDataToImport;
var rawData = csvDataSource.Data as string[][];
for (var i = _startRow; i < rawData.Length; i++)
{
var extractResultsForRow = ExtractDataForSingleRow <T>(_extractConfigurations, dataToImport, i);
extractedDataset.AddParsingResults(extractResultsForRow);
}
return(extractedDataset);
}
protected IEnumerable <IResult> ExtractDataForSingleNode <T>(IList <IExtractConfiguration> extractConfigurations, IDataToImport dataToImport) where T : new()
{
var resultsForSingleNode = new List <IResult>();
var model = new T();
IDataSourceLocation currentLocation = null;
foreach (var configuration in _extractConfigurations)
{
if (configuration is SimpleXMLExtractConfiguration)
{
currentLocation = new XMLDataSourceLocation(((SimpleXMLExtractConfiguration)configuration).ElementName, ((SimpleXMLExtractConfiguration)configuration).AttributeName);
}
resultsForSingleNode.AddRange(((SimpleXMLExtractConfiguration)configuration).ExtractData(model, dataToImport, currentLocation));
}
var parsingResult = new ParsingResult(ResultLevel.DEBUG, "Extract data from single row success", model, new XMLDataSourceLocation("", ""));
resultsForSingleNode.Add(parsingResult);
return(resultsForSingleNode);
}
public abstract IEnumerable<IResult> ExtractData(object model, IDataToImport dataToImport,
IDataSourceLocation currentLocation);
private IEnumerable <IResult> ExtractHeaderFile(object model, IList <IExtractConfiguration> extractConfiguration, IDataToImport xmlDataToImport)
{
var results = new List <IResult>();
var validatedDataToImportResult = ValidateHeaderFileDataToImport(xmlDataToImport);
results.AddRange(validatedDataToImportResult.Item2);
IDataSourceLocation currentLocation = null;
foreach (var configuration in extractConfiguration.Where(x => x is ISimpleExtractConfiguration))
{
if (configuration is SimpleXMLExtractConfiguration)
{
currentLocation = new XMLDataSourceLocation(((SimpleXMLExtractConfiguration)configuration).ElementName, ((SimpleXMLExtractConfiguration)configuration).AttributeName);
}
results.AddRange(((SimpleXMLExtractConfiguration)configuration).ExtractData(model, xmlDataToImport, currentLocation));
}
return(results);
}
public override IExtractedDataset <T> Extract <T>(IDataToImport dataToImport)
{
var extractedDataset = new ExtractedDataset <T>(_thresholdLevel);
ChemistryFileChildObjectExtractConfiguration chemistryDataExtractConfiguration = null;
SampleFileChildObjectExtractConfiguration sampleDataExtractConfiguration = null;
var castedDataToImport = dataToImport as ESDATDataToImport;
if (castedDataToImport == null)
{
extractedDataset.AddParsingResult(new BaseResult(ResultLevel.FATAL, "Data to Import needs to be ESDATDataToImport"));
}
try
{
chemistryDataExtractConfiguration = _extractConfigurations.Where(x => x is ChemistryFileChildObjectExtractConfiguration)
.Cast <ChemistryFileChildObjectExtractConfiguration>()
.SingleOrDefault();
}
catch (Exception ex)
{
chemistryDataExtractConfiguration = null;
extractedDataset.AddParsingResult(new BaseResult(ResultLevel.FATAL, "ESDAT data importer needs to have one and only one Chemistry file extract configuration"));
}
try
{
sampleDataExtractConfiguration = _extractConfigurations.Where(x => x is SampleFileChildObjectExtractConfiguration)
.Cast <SampleFileChildObjectExtractConfiguration>()
.SingleOrDefault();
}
catch (Exception ex)
{
sampleDataExtractConfiguration = null;
extractedDataset.AddParsingResult(new BaseResult(ResultLevel.FATAL, "ESDAT data importer needs to have one and only one Sample file extract configuration"));
}
var model = new T();
if (castedDataToImport.HeaderFileToImport == null)
{
var castedModel = model as ESDATModel;
castedModel.LabName = _wqDefaultValueProvider.OrganizationNameSampleCollection;
extractedDataset.AddParsingResults(new List <IResult> {
new BaseResult(ResultLevel.WARN, "Header file is null, use the default organization name in the default value provider")
});
}
else
{
var headerFileExtractResults = ExtractHeaderFile(model, _extractConfigurations, castedDataToImport.HeaderFileToImport);
extractedDataset.AddParsingResults(headerFileExtractResults);
}
if (chemistryDataExtractConfiguration != null && sampleDataExtractConfiguration != null)
{
var chemistryFileExtractResults = ExtractChemistryFileData(model, chemistryDataExtractConfiguration, castedDataToImport.ChemistryFileToImport);
extractedDataset.AddParsingResults(chemistryFileExtractResults);
var sampleFileExtractResults = ExtractSampleFileData(model, sampleDataExtractConfiguration, castedDataToImport.SampleFileToImport);
extractedDataset.AddParsingResults(sampleFileExtractResults);
}
extractedDataset.AddParsingResults(new List <IResult> {
new ParsingResult(ResultLevel.DEBUG, "Extract data into ESDAT model", model, null)
});
return(extractedDataset);
}
public override bool IsDataSupported(IDataToImport dataToImport)
{
return(dataToImport is ESDATDataToImport);
}
private IEnumerable <IResult> ExtractSampleFileData(object model, SampleFileChildObjectExtractConfiguration configuration, IDataToImport dataToImport)
{
var extractResults = configuration.ExtractData(model, dataToImport);
return(extractResults);
}
public abstract IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport,
IDataSourceLocation currentLocation);
public abstract IEnumerable<IResult> ExtractData(object model, IDataToImport dataToImport);
public abstract IExtractedDataset <T> Extract <T>(IDataToImport dataToImport) where T : new();
protected IEnumerable <IResult> ExtractDataForSingleRow <T>(IList <IExtractConfiguration> extractConfigurations, IDataToImport dataSource, int currentRow) where T : new()
{
var resultsForSingleRow = new List <IResult>();
var model = new T();
IDataSourceLocation currentLocation = null;
foreach (var configuration in extractConfigurations.Where(x => x is ISimpleExtractConfiguration).Cast <ISimpleExtractConfiguration>())
{
if (configuration is SimpleCSVExtractConfiguration)
{
currentLocation = new CSVDataSourceLocation(currentRow, ((SimpleCSVExtractConfiguration)configuration).ColumnIndex);
}
resultsForSingleRow.AddRange(configuration.ExtractData(model, dataSource, currentLocation));
}
var parsingResult = new ParsingResult(ResultLevel.DEBUG, "Extract data from single row success", model, new CSVDataSourceLocation(currentRow, 1));
resultsForSingleRow.Add(parsingResult);
return(resultsForSingleRow);
}
public override bool IsDataSupported(IDataToImport dataToImport)
{
return(ValidateDataToImport(dataToImport).Item1);
}
public abstract IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport);
public abstract bool IsDataSupported(IDataToImport dataToImport);