public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type) { if (!(dataSourceLocation is XMLDataSourceLocation)) { return new ParsingResult(ResultLevel.FATAL, dataSourceLocation.GetType().ToString() + " is not supported by XML Node Parser", null, dataSourceLocation); } if (!(dataToImport is XMLDataToImport)) { return new ParsingResult(ResultLevel.FATAL, dataToImport.GetType().ToString() + " is not supported by XML Node Parser", null, dataSourceLocation); } var castedDataToImport = dataToImport as XMLDataToImport; var castedDataSourceLocation = dataSourceLocation as XMLDataSourceLocation; try { var rawData = GetRawDataValue(castedDataSourceLocation, castedDataToImport); var parsedValue = ParseRawValue(type, rawData); return new ParsingResult(ResultLevel.INFO, "Parsing value successfully", parsedValue, dataSourceLocation); } catch (Exception e) { return new ParsingResult(ResultLevel.FATAL, "Exception caught: " + e, null, dataSourceLocation); } }
public IExtractedDataset <T> Extract <T>(IDataToImport dataToImport) where T : new() { var extractedDataset = new ExtractedDataset <T>(_thresholdLevel); var validatedDataToImportResult = ValidateDataToImport(dataToImport); extractedDataset.AddParsingResults(validatedDataToImportResult.Item2); if (!validatedDataToImportResult.Item1) { //format not valid, return return(extractedDataset); } var csvDataSource = dataToImport as XMLDataToImport; var rawData = csvDataSource.Data as XDocument; XNamespace rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; var parentNodes = rawData.Descendants().Where(x => x.Name.LocalName == _rootNode); foreach (XElement element in parentNodes) { var extractResultsForNode = ExtractDataForSingleNode <T>(_extractConfigurations, new XMLDataToImport("", new XDocument(element))); extractedDataset.AddParsingResults(extractResultsForNode); } return(extractedDataset); }
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type) { if (!(dataSourceLocation is XMLDataSourceLocation)) { return(new ParsingResult(ResultLevel.FATAL, dataSourceLocation.GetType().ToString() + " is not supported by XML Node Parser", null, dataSourceLocation)); } if (!(dataToImport is XMLDataToImport)) { return(new ParsingResult(ResultLevel.FATAL, dataToImport.GetType().ToString() + " is not supported by XML Node Parser", null, dataSourceLocation)); } var castedDataToImport = dataToImport as XMLDataToImport; var castedDataSourceLocation = dataSourceLocation as XMLDataSourceLocation; try { var rawData = GetRawDataValue(castedDataSourceLocation, castedDataToImport); var parsedValue = ParseRawValue(type, rawData); return(new ParsingResult(ResultLevel.INFO, "Parsing value successfully", parsedValue, dataSourceLocation)); } catch (Exception e) { return(new ParsingResult(ResultLevel.FATAL, "Exception caught: " + e, null, dataSourceLocation)); } }
public IExtractedDataset <T> Extract <T>(IDataToImport dataToImport) where T : new() { var extractedDataset = new ExtractedDataset <T>(_thresholdLevel); var validatedDataToImportResult = ValidateDataToImport(dataToImport); extractedDataset.AddParsingResults(validatedDataToImportResult.Item2); if (!validatedDataToImportResult.Item1) { //format not valid, return return(extractedDataset); } var results = new List <IResult>(); var model = new T(); IDataSourceLocation currentLocation = null; foreach (var configuration in _extractConfigurations) { if (configuration is SimpleXMLExtractConfiguration) { currentLocation = new XMLDataSourceLocation(((SimpleXMLExtractConfiguration)configuration).ElementName, ((SimpleXMLExtractConfiguration)configuration).AttributeName); } results.AddRange(((SimpleXMLExtractConfiguration)configuration).ExtractData(model, dataToImport, currentLocation)); } var parsingResult = new ParsingResult(ResultLevel.DEBUG, "Extract data from single row success", model, null); results.Add(parsingResult); extractedDataset.AddParsingResults(results); return(extractedDataset); }
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type) { var castedDataToImport = dataToImport as JSONDataToImport; var castedJObject = castedDataToImport.Data as JObject; var castedDataSourceLocation = dataSourceLocation as JSONDataSourceLocation; var locationPath = castedDataSourceLocation.Path; if (castedDataSourceLocation.IsArray) { var rawValues = castedJObject.SelectTokens(locationPath).Values<string>(); var valueParser = _parserFactory.GetValueParser(type); var parsedValue = from value in rawValues select valueParser.Parse(value); return new ParsingResult(ResultLevel.INFO, "Parse value from " + dataToImport.ToString() + " successfully, the value is " + parsedValue.ToString(), parsedValue.ToList(), castedDataSourceLocation); } else { var rawValue = (castedJObject.SelectTokens(locationPath).ElementAt(castedDataSourceLocation.Index.Value) as JValue).Value; var valueParser = _parserFactory.GetValueParser(type); var parsedValue = valueParser.Parse(rawValue); return new ParsingResult(ResultLevel.INFO, "Parse value from " + dataToImport.ToString() + " successfully " + parsedValue, parsedValue, castedDataSourceLocation); } }
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type) { if (!(dataSourceLocation is CSVDataSourceLocation)) { return new ParsingResult(ResultLevel.FATAL, dataSourceLocation.GetType().ToString() + " is not supported by CSV Cell Parser", null, null); } if (!(dataToImport is CSVDataToImport)) { return new ParsingResult(ResultLevel.FATAL, dataToImport.GetType().ToString() + " is not supported by CSV Cell Parser", null, null); } var castedDataToImport = dataToImport as CSVDataToImport; var castedDataSourceLocation = dataSourceLocation as CSVDataSourceLocation; try { var rawData = GetRawDataValue(castedDataSourceLocation, castedDataToImport); var parsedValue = ParseRawValue(type, rawData); return new ParsingResult(ResultLevel.INFO, "Parsing value successfully", parsedValue, castedDataSourceLocation); } catch (IndexOutOfRangeException) { return new ParsingResult(ResultLevel.FATAL, "Index is out of range", null, castedDataSourceLocation); } }
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type) { var castedDataToImport = dataToImport as JSONDataToImport; var castedJObject = castedDataToImport.Data as JObject; var castedDataSourceLocation = dataSourceLocation as JSONDataSourceLocation; var locationPath = castedDataSourceLocation.Path; if (castedDataSourceLocation.IsArray) { var rawValues = castedJObject.SelectTokens(locationPath).Values <string>(); var valueParser = _parserFactory.GetValueParser(type); var parsedValue = from value in rawValues select valueParser.Parse(value); return(new ParsingResult(ResultLevel.INFO, "Parse value from " + dataToImport.ToString() + " successfully, the value is " + parsedValue.ToString(), parsedValue.ToList(), castedDataSourceLocation)); } else { var rawValue = (castedJObject.SelectTokens(locationPath).ElementAt(castedDataSourceLocation.Index.Value) as JValue).Value; var valueParser = _parserFactory.GetValueParser(type); var parsedValue = valueParser.Parse(rawValue); return(new ParsingResult(ResultLevel.INFO, "Parse value from " + dataToImport.ToString() + " successfully " + parsedValue, parsedValue, castedDataSourceLocation)); } }
public IResult Parse(IDataToImport dataToImport, IDataSourceLocation dataSourceLocation, Type type) { if (!(dataSourceLocation is CSVDataSourceLocation)) { return(new ParsingResult(ResultLevel.FATAL, dataSourceLocation.GetType().ToString() + " is not supported by CSV Cell Parser", null, null)); } if (!(dataToImport is CSVDataToImport)) { return(new ParsingResult(ResultLevel.FATAL, dataToImport.GetType().ToString() + " is not supported by CSV Cell Parser", null, null)); } var castedDataToImport = dataToImport as CSVDataToImport; var castedDataSourceLocation = dataSourceLocation as CSVDataSourceLocation; try { var rawData = GetRawDataValue(castedDataSourceLocation, castedDataToImport); var parsedValue = ParseRawValue(type, rawData); return(new ParsingResult(ResultLevel.INFO, "Parsing value successfully", parsedValue, castedDataSourceLocation)); } catch (IndexOutOfRangeException) { return(new ParsingResult(ResultLevel.FATAL, "Index is out of range", null, castedDataSourceLocation)); } }
public override IEnumerable<IResult> ExtractData(object model, IDataToImport dataToImport) { var extractResult = _dataImporter.Extract<SampleFileData>(dataToImport); _valueAssigner.AssignValue(model, _propertyPath, extractResult.ExtractedEntities, typeof(SampleFileData)); return extractResult.AllParsingResults.Where(x => x is BaseResult); }
public override IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport) { var extractResult = _dataImporter.Extract <ChemistryFileData>(dataToImport); _valueAssigner.AssignValue(model, _propertyPath, extractResult.ExtractedEntities, typeof(ChemistryFileData)); return(extractResult.AllParsingResults.Where(x => x is BaseResult)); }
private Tuple <bool, IEnumerable <IResult> > ValidateHeaderFileDataToImport(IDataToImport dataToImport) { var allValidationResults = from rule in _validationRules select rule.Validate(dataToImport); var isValid = !allValidationResults.Any(x => ResultLevelHelper.LevelIsHigherThanOrEqualToThreshold(_thresholdLevel, x.Level)); return(new Tuple <bool, IEnumerable <IResult> >(isValid, allValidationResults)); }
public override IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport, IDataSourceLocation currentLocation) { if (!(dataToImport is JSONDataToImport)) { throw new ArgumentException("The SimpleJSONExtractConfiguration only accept JSONDataToImport to extract data"); } if (!(currentLocation is JSONDataSourceLocation)) { throw new ArgumentException("The SimpleJSONExtractConfiguration only accept JSONDataSourceLocation to extract data"); } throw new NotImplementedException(); }
public IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport, IDataSourceLocation locationToParse) { var results = new List <IResult>(); try { var parsingResult = _parser.Parse(dataToImport, locationToParse, _propertyType) as IParsingResult; _valueAssigner.AssignValue(model, _propertyPath, parsingResult.Value, _propertyType); results.Add(new BaseResult(ResultLevel.DEBUG, string.Format("Extract data from xml file and assign to model {0}", locationToParse.ToString()))); return(results); } catch (Exception ex) { results.Add(new BaseResult(ResultLevel.ERROR, string.Format("Extract data from xml file and assign to model fail. {0}", locationToParse.ToString()))); } return(results); }
public IResult Validate(IDataToImport dataToImport) { var csvDataToImport = (CSVDataToImport)dataToImport; if (csvDataToImport != null) { if(string.IsNullOrEmpty(csvDataToImport.FileName)) { return new BaseResult(ResultLevel.ERROR, "System fail to compare file extension because data file name is empty/null"); } else { var isValid = _caseSensitive ? csvDataToImport.FileName.EndsWith(_acceptedFileExtension, StringComparison.Ordinal) : csvDataToImport.FileName.EndsWith(_acceptedFileExtension, StringComparison.OrdinalIgnoreCase); if (isValid) { return new BaseResult(ResultLevel.INFO, string.Format("File name {0} matches the extension {1}", csvDataToImport.FileName, _acceptedFileExtension)); } else { return new BaseResult(ResultLevel.ERROR, string.Format("File name {0} does not match the extension {1}", csvDataToImport.FileName, _acceptedFileExtension)); } } } else { return new BaseResult(ResultLevel.ERROR, "Data to import is not in CSV format"); } }
public IResult Validate(IDataToImport dataToImport) { try { var valueParser = _parserFactory.GetElementParser(_elementValueType); var parsedValue = valueParser.Parse(dataToImport, _location, _elementValueType) as IParsingResult; var isCriteriaMet = _criteria.Meet(parsedValue.Value); if (isCriteriaMet) { return(new BaseResult(ResultLevel.INFO, "Criteria matches")); } else { return(new BaseResult(ResultLevel.ERROR, "Criteria not matches")); } } catch (Exception) { return(new BaseResult(ResultLevel.ERROR, "System not able to compare to the criteria since parsing value from data fail. Failing at " + _location.ToString())); } }
public override IEnumerable<IResult> ExtractData(object model, IDataToImport dataToImport, IDataSourceLocation currentLocation) { var results = new List<IResult>(); try { var parsingResult = _parser.Parse(dataToImport, currentLocation, _propertyType) as IParsingResult; _valueAssigner.AssignValue(model, _propertyPath, parsingResult.Value, _propertyType); results.Add(new BaseResult(ResultLevel.DEBUG, string.Format("Extract data from csv file and assign to model {0}", currentLocation.ToString()))); return results; } catch(Exception ex) { results.Add(new BaseResult(ResultLevel.ERROR, string.Format("Extract data from csv file and assign to model fail. {0}", currentLocation.ToString()))); } return results; }
public void CSVParseTestSuccess() { var dataSource = new CSVDataSource(_csvDataFilePath); var dataSourceLocation = new CSVDataSourceLocation(6, 3); var dataDateTimeSourceLocation = new CSVDataSourceLocation(6, 0); dataToImport = dataSource.FetchData(); var parser = new CellParser(new DefaultParserFactory()); var parseDoubleResult = parser.Parse <double>(dataToImport, dataSourceLocation); Assert.NotNull(parseDoubleResult); Assert.AreEqual(parseDoubleResult.Level, ResultLevel.INFO); Assert.AreEqual(parseDoubleResult.Message, "Parsing value successfully"); Assert.AreEqual(0.3, ((IParsingResult)parseDoubleResult).Value); var parseDateTimeResult = parser.Parse <DateTime>(dataToImport, dataDateTimeSourceLocation); Assert.NotNull(parseDateTimeResult); Assert.AreEqual(parseDateTimeResult.Level, ResultLevel.INFO); Assert.AreEqual(parseDateTimeResult.Message, "Parsing value successfully"); Assert.AreEqual(new DateTime(2013, 12, 12, 14, 0, 0), ((IParsingResult)parseDateTimeResult).Value); }
public IResult Validate(IDataToImport dataToImport) { var csvDataToImport = (CSVDataToImport)dataToImport; if (csvDataToImport != null) { if (string.IsNullOrEmpty(csvDataToImport.FileName)) { return(new BaseResult(ResultLevel.ERROR, "System fail to compare file extension because data file name is empty/null")); } else { var isValid = _caseSensitive ? csvDataToImport.FileName.EndsWith(_acceptedFileExtension, StringComparison.Ordinal) : csvDataToImport.FileName.EndsWith(_acceptedFileExtension, StringComparison.OrdinalIgnoreCase); if (isValid) { return(new BaseResult(ResultLevel.INFO, string.Format("File name {0} matches the extension {1}", csvDataToImport.FileName, _acceptedFileExtension))); } else { return(new BaseResult(ResultLevel.ERROR, string.Format("File name {0} does not match the extension {1}", csvDataToImport.FileName, _acceptedFileExtension))); } } } else { return(new BaseResult(ResultLevel.ERROR, "Data to import is not in CSV format")); } }
public IResult Validate(IDataToImport dataToImport) { try { var valueParser = _parserFactory.GetCellParser(_cellValueType); var parsedValue = valueParser.Parse(dataToImport, _location, _cellValueType) as IParsingResult; var isCriteriaMet = _criteria.Meet(parsedValue.Value); if (isCriteriaMet) { return new BaseResult(ResultLevel.INFO, "Criteria matches"); } else { return new BaseResult(ResultLevel.ERROR, "Criteria not matches"); } } catch(Exception) { return new BaseResult(ResultLevel.ERROR, "System not able to compare to the criteria since parsing value from data fail."); } }
public override IExtractedDataset <T> Extract <T>(IDataToImport dataToImport) { var extractedDataset = new ExtractedDataset <T>(_thresholdLevel); var validatedDataToImportResult = ValidateDataToImport(dataToImport); extractedDataset.AddParsingResults(validatedDataToImportResult.Item2); if (!validatedDataToImportResult.Item1) { //format not valid, return return(extractedDataset); } var csvDataSource = dataToImport as CSVDataToImport; var rawData = csvDataSource.Data as string[][]; for (var i = _startRow; i < rawData.Length; i++) { var extractResultsForRow = ExtractDataForSingleRow <T>(_extractConfigurations, dataToImport, i); extractedDataset.AddParsingResults(extractResultsForRow); } return(extractedDataset); }
protected IEnumerable <IResult> ExtractDataForSingleNode <T>(IList <IExtractConfiguration> extractConfigurations, IDataToImport dataToImport) where T : new() { var resultsForSingleNode = new List <IResult>(); var model = new T(); IDataSourceLocation currentLocation = null; foreach (var configuration in _extractConfigurations) { if (configuration is SimpleXMLExtractConfiguration) { currentLocation = new XMLDataSourceLocation(((SimpleXMLExtractConfiguration)configuration).ElementName, ((SimpleXMLExtractConfiguration)configuration).AttributeName); } resultsForSingleNode.AddRange(((SimpleXMLExtractConfiguration)configuration).ExtractData(model, dataToImport, currentLocation)); } var parsingResult = new ParsingResult(ResultLevel.DEBUG, "Extract data from single row success", model, new XMLDataSourceLocation("", "")); resultsForSingleNode.Add(parsingResult); return(resultsForSingleNode); }
public abstract IEnumerable<IResult> ExtractData(object model, IDataToImport dataToImport, IDataSourceLocation currentLocation);
private IEnumerable <IResult> ExtractHeaderFile(object model, IList <IExtractConfiguration> extractConfiguration, IDataToImport xmlDataToImport) { var results = new List <IResult>(); var validatedDataToImportResult = ValidateHeaderFileDataToImport(xmlDataToImport); results.AddRange(validatedDataToImportResult.Item2); IDataSourceLocation currentLocation = null; foreach (var configuration in extractConfiguration.Where(x => x is ISimpleExtractConfiguration)) { if (configuration is SimpleXMLExtractConfiguration) { currentLocation = new XMLDataSourceLocation(((SimpleXMLExtractConfiguration)configuration).ElementName, ((SimpleXMLExtractConfiguration)configuration).AttributeName); } results.AddRange(((SimpleXMLExtractConfiguration)configuration).ExtractData(model, xmlDataToImport, currentLocation)); } return(results); }
public override IExtractedDataset <T> Extract <T>(IDataToImport dataToImport) { var extractedDataset = new ExtractedDataset <T>(_thresholdLevel); ChemistryFileChildObjectExtractConfiguration chemistryDataExtractConfiguration = null; SampleFileChildObjectExtractConfiguration sampleDataExtractConfiguration = null; var castedDataToImport = dataToImport as ESDATDataToImport; if (castedDataToImport == null) { extractedDataset.AddParsingResult(new BaseResult(ResultLevel.FATAL, "Data to Import needs to be ESDATDataToImport")); } try { chemistryDataExtractConfiguration = _extractConfigurations.Where(x => x is ChemistryFileChildObjectExtractConfiguration) .Cast <ChemistryFileChildObjectExtractConfiguration>() .SingleOrDefault(); } catch (Exception ex) { chemistryDataExtractConfiguration = null; extractedDataset.AddParsingResult(new BaseResult(ResultLevel.FATAL, "ESDAT data importer needs to have one and only one Chemistry file extract configuration")); } try { sampleDataExtractConfiguration = _extractConfigurations.Where(x => x is SampleFileChildObjectExtractConfiguration) .Cast <SampleFileChildObjectExtractConfiguration>() .SingleOrDefault(); } catch (Exception ex) { sampleDataExtractConfiguration = null; extractedDataset.AddParsingResult(new BaseResult(ResultLevel.FATAL, "ESDAT data importer needs to have one and only one Sample file extract configuration")); } var model = new T(); if (castedDataToImport.HeaderFileToImport == null) { var castedModel = model as ESDATModel; castedModel.LabName = _wqDefaultValueProvider.OrganizationNameSampleCollection; extractedDataset.AddParsingResults(new List <IResult> { new BaseResult(ResultLevel.WARN, "Header file is null, use the default organization name in the default value provider") }); } else { var headerFileExtractResults = ExtractHeaderFile(model, _extractConfigurations, castedDataToImport.HeaderFileToImport); extractedDataset.AddParsingResults(headerFileExtractResults); } if (chemistryDataExtractConfiguration != null && sampleDataExtractConfiguration != null) { var chemistryFileExtractResults = ExtractChemistryFileData(model, chemistryDataExtractConfiguration, castedDataToImport.ChemistryFileToImport); extractedDataset.AddParsingResults(chemistryFileExtractResults); var sampleFileExtractResults = ExtractSampleFileData(model, sampleDataExtractConfiguration, castedDataToImport.SampleFileToImport); extractedDataset.AddParsingResults(sampleFileExtractResults); } extractedDataset.AddParsingResults(new List <IResult> { new ParsingResult(ResultLevel.DEBUG, "Extract data into ESDAT model", model, null) }); return(extractedDataset); }
public override bool IsDataSupported(IDataToImport dataToImport) { return(dataToImport is ESDATDataToImport); }
private IEnumerable <IResult> ExtractSampleFileData(object model, SampleFileChildObjectExtractConfiguration configuration, IDataToImport dataToImport) { var extractResults = configuration.ExtractData(model, dataToImport); return(extractResults); }
public abstract IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport, IDataSourceLocation currentLocation);
public abstract IEnumerable<IResult> ExtractData(object model, IDataToImport dataToImport);
public abstract IExtractedDataset <T> Extract <T>(IDataToImport dataToImport) where T : new();
protected IEnumerable <IResult> ExtractDataForSingleRow <T>(IList <IExtractConfiguration> extractConfigurations, IDataToImport dataSource, int currentRow) where T : new() { var resultsForSingleRow = new List <IResult>(); var model = new T(); IDataSourceLocation currentLocation = null; foreach (var configuration in extractConfigurations.Where(x => x is ISimpleExtractConfiguration).Cast <ISimpleExtractConfiguration>()) { if (configuration is SimpleCSVExtractConfiguration) { currentLocation = new CSVDataSourceLocation(currentRow, ((SimpleCSVExtractConfiguration)configuration).ColumnIndex); } resultsForSingleRow.AddRange(configuration.ExtractData(model, dataSource, currentLocation)); } var parsingResult = new ParsingResult(ResultLevel.DEBUG, "Extract data from single row success", model, new CSVDataSourceLocation(currentRow, 1)); resultsForSingleRow.Add(parsingResult); return(resultsForSingleRow); }
public override bool IsDataSupported(IDataToImport dataToImport) { return(ValidateDataToImport(dataToImport).Item1); }
public abstract IEnumerable <IResult> ExtractData(object model, IDataToImport dataToImport);
public abstract bool IsDataSupported(IDataToImport dataToImport);