public DataFileView Create(DataFile dataFile) { var cacheKey = dataFile.Id; DataFileView cache = null; ; if (container.ContainsKey(cacheKey)) cache = container[cacheKey]; else { cache = container[cacheKey] = new DataFileView(dataFile); } return cache; }
public static long Create(DataFile model) { long id = 0; using (var con = EstablishConnection()) { con.Open(); con.Execute(@" INSERT INTO DataFiles(name, rawTableName, createdDate, relativePath) VALUES(@Name,@RawTableName, @CreatedDate, @RelativePath); ", model); id = con.Query<long>("SELECT last_insert_rowid();").First(); model.Id = id; con.Close(); } return id; }
public static void Activate(DataFile dataFile) { using(var con = EstablishConnection()) { con.Open(); var transaction = con.BeginTransaction(); con.Execute(@" Update DataFiles SET IsActivated = 0; Update DataFiles SET IsActivated = 1 WHERE id = @Id; ", dataFile); dataFile.IsActivated = true; transaction.Commit(); con.Close(); } }
public bool CheckStringRuleInValidationStatus(DataFile dataFile, AttributeDefinition AttributeDefinition, List<IStringRule> rules) { IEnumerable<string> charValues = DataRepository.GetDistinctValues(AttributeDefinition, dataFile.RawTableName); if (rules.Count == 0) return true; foreach (var charValue in charValues) { var isApplied = false; foreach (var rule in rules) { var result = rule.Apply(charValue); if (result.HasValue) { isApplied = true; break; } } if (!isApplied) return false; } return true; }
public bool CheckNumericRuleInValidationStatus(DataFile dataFile, AttributeDefinition AttributeDefinition, List<INumericRule> rules) { IEnumerable<string> charValues = DataRepository.GetDistinctValues(AttributeDefinition, dataFile.RawTableName); if (rules.Count == 0) return true; foreach (var charValue in charValues) { var isApplied = false; try { foreach (var rule in rules) { var result = rule.Apply(double.Parse(charValue)); } isApplied = true; }catch(Exception e){ isApplied = false; } if (!isApplied) return false; } return true; }
public DataFileView(DataFile dataFile) { this.DataFile = dataFile; }
public void CreateMapTable(DataFile dataFile, List<AttributeDefinition> AttributeDefinitions) { StringRuleFactory stringRuleFactory = new StringRuleFactory(); NumericRuleFactory numericRuleFactory = new NumericRuleFactory(); var rows = DataRepository.GetDataForTable(dataFile.RawTableName); var records = new List<string[]>(); SortedDictionary<string, SortedDictionary<string,int>> sDictionary = new SortedDictionary<string, SortedDictionary<string,int>>(); SortedDictionary<string,int> maxKeysDictionary = new SortedDictionary<string, int>(); foreach (var row in rows) { var rowSqls = new List<string>(); rowSqls.Add(row.Id.ToString()); foreach (var AttributeDefinition in AttributeDefinitions) { string sValue = ((IDictionary<string, object>)row)[AttributeDefinition.Name].ToString(); if (AttributeDefinition.AttributeDataType == "String") { if (AttributeDefinition.IsAutoEncoding) { if (!sDictionary.ContainsKey(AttributeDefinition.Name)) { sDictionary[AttributeDefinition.Name] = new SortedDictionary<string,int>(); } if (!sDictionary[AttributeDefinition.Name].ContainsKey(sValue)) { sDictionary[AttributeDefinition.Name][sValue] = sDictionary[AttributeDefinition.Name].Count + 1; } var val = sDictionary[AttributeDefinition.Name][sValue]; rowSqls.Add(val.ToString()); } else if (AttributeDefinition.ValidationStatus == "Valid" && AttributeDefinition.MapRules.Count !=0) { var iValue = ApplyStringRule(AttributeDefinition, sValue, stringRuleFactory); rowSqls.Add(iValue.Value.ToString()); } else { rowSqls.Add(sValue); } } else { rowSqls.Add(ApplyNumericRule(AttributeDefinition, sValue, numericRuleFactory).ToString()); } } records.Add(rowSqls.ToArray()); } DataRepository.GenerateMapTable(dataFile, AttributeDefinitions, records); var decisionTableId = DecisionTableRepository.Create(new DecisionTable { TableName = dataFile.MapTableName, Content = string.Format("Create map table {0} from file {0}", dataFile.Name), CreatedDate = DateTime.Now.ToNormalDateTimeString() }); DecisionTableHistoryRepository.Create(new DecisionTableHistory { Action = EDTHistoryAction.CreateMapTable, DecisionTableId = decisionTableId, Decription = "", CreatedDate = DateTime.Now.ToNormalDateTimeString(), ParentId = 0 }); }
public IEnumerable<AppliedRuleValue> GetValuesWithNoRuleAppliedInStringRule(DataFile dataFile, AttributeDefinition AttributeDefinition, List<IStringRule> rules) { IEnumerable<string> charValues = DataRepository.GetDistinctValues(AttributeDefinition, dataFile.RawTableName); List<AppliedRuleValue> appliedRuleValues = new List<AppliedRuleValue>(); Dictionary<string, int> dict = new Dictionary<string, int>(); foreach (var charValue in charValues) { if (rules.Count == 0) { appliedRuleValues.Add(new AppliedRuleValue { IsApplied = false, Value = charValue }); } else { var appliedRuleValue = new AppliedRuleValue { Value = charValue, IsApplied = false }; foreach (var rule in rules) { var result = rule.Apply(charValue); if (result.HasValue) { appliedRuleValue.IsApplied = true; break; } } appliedRuleValues.Add(appliedRuleValue); } } List<int> removedIndices = new List<int>(); // remove duplicate applied rule values for (int i = appliedRuleValues.Count - 1; i >= 0 ; i--) { var appliedRuleValue = appliedRuleValues[i]; if (dict.ContainsKey(appliedRuleValue.Value)) { removedIndices.Add(i); } else { dict[appliedRuleValue.Value] = i; } } foreach (var removedIndex in removedIndices) { appliedRuleValues.RemoveAt(removedIndex); } return appliedRuleValues; }
public IEnumerable<AppliedRuleValue> GetValuesWithNoRuleAppliedInNumericRule(DataFile dataFile, AttributeDefinition AttributeDefinition, List<INumericRule> rules) { IEnumerable<string> charValues = DataRepository.GetDistinctValues(AttributeDefinition, dataFile.RawTableName); List<AppliedRuleValue> appliedRuleValues = new List<AppliedRuleValue>(); bool isError = false; foreach (var charValue in charValues) { if (rules.Count == 0 || isError) { appliedRuleValues.Add(new AppliedRuleValue {IsApplied = false, Value = charValue }); } else { try { foreach (var rule in rules) { var result = rule.Apply(float.Parse(charValue)); } appliedRuleValues.Add(new AppliedRuleValue { IsApplied = true, Value = charValue }); } catch (FormatException e) { appliedRuleValues.Add(new AppliedRuleValue { IsApplied = false, Value = charValue }); isError = true; } } } return appliedRuleValues; }
public void Active(DataFile dataFile) { DataFileRepository.Activate(dataFile); }
public static void UploadData(string fileName, string absoluteFilePath, string relativeFilePath) { // create data file var dataFile = new DataFile { RawTableName = string.Format("RawTable{0}",DateTime.Now.ToTimeStamps()), Name = fileName, CreatedDate = DateTime.Now.ToNormalDateTimeString(), RelativePath = relativeFilePath }; DataFileRepository.Create(dataFile); using (var con = EstablishConnection()) { con.Open(); var transaction = con.BeginTransaction(); var csvReader = new CsvReader(new StreamReader(absoluteFilePath), new CsvHelper.Configuration.CsvConfiguration { //Delimiter = "\t", TrimFields = true, TrimHeaders = true }); csvReader.Read(); // create column definitions var fieldHeaders = csvReader.FieldHeaders; var AttributeDefinitions = new List<AttributeDefinition>(); for (int i = 0; i < fieldHeaders.Length; i++) { var fieldHeader = fieldHeaders[i]; var AttributeDefinition = new AttributeDefinition { RawName = fieldHeader, Name = string.Format("Column_{0}", i + 1), AttributeIndex = i, DataFileId = dataFile.Id }; AttributeDefinitions.Add(AttributeDefinition); } AttributeDefinitionRepository.BulkInsert(con, AttributeDefinitions); // create raw table DataRepository.CreateRawTable(con, dataFile.RawTableName, AttributeDefinitions); // fill data into raw table var records = new List<string[]>(); while (csvReader.Read()) records.Add(csvReader.CurrentRecord); DataRepository.BulkInsertRawTable(con, dataFile.RawTableName, AttributeDefinitions, records); transaction.Commit(); con.Close(); } }
public static void GenerateMapTable(DataFile dataFile, List<AttributeDefinition> AttributeDefinitions, List<string[]> records) { // create data file var mapTableName = dataFile.MapTableName = string.Format("MapTable{0}", DateTime.Now.ToTimeStamps()); DataFileRepository.UpdateMapTable(dataFile); using (var con = EstablishConnection()) { con.Open(); var transaction = con.BeginTransaction(); CreateMapTable(con, mapTableName, AttributeDefinitions); BulkInsertMapTable(con, mapTableName, AttributeDefinitions, records); transaction.Commit(); con.Close(); } }
public static void UpdateMapTable(DataFile dataFile) { using (var con = EstablishConnection()) { con.Open(); con.Execute(@"Update DataFiles SET MapTableName = @MapTableName, IsMapped = 1 WHERE Id = @Id", dataFile); con.Close(); } }