private void FillData(string[] aDatas, FlatFile aFlatFile) { this.m_Name = aDatas[0]; this.m_DataTye = (DataType)Enum.Parse(typeof(DataType), aDatas[1], true); this.m_Length = Convert.ToInt32(aDatas[2]); int thePos = (int)aFlatFile + 3; if (String.IsNullOrEmpty(aDatas[thePos]) == false) { this.m_Index = Convert.ToInt32(aDatas[thePos]); } }
private static IEnumerable<PersonFlat> ReadFile() { //using the flattmapper as the documentation using (var fileStream = File.OpenRead(DataFile)) { var flatFile = new FlatFile<PersonFlat>(_layout, fileStream); return flatFile.Read() //we need a ToList here since flatmapper does iteractive reading, //closing the stream before enumerating the collection will throw an error. .ToList(); } }
public void can_write_read_stream() { using (var memory = new MemoryStream()) { var flatFile = new FlatFile<TestObject>(layout, memory, HandleEntryReadError); flatFile.Write(objects); memory.Seek(0, SeekOrigin.Begin); var objectsAfterRead = flatFile.Read().ToList(); Assert.True(objects.SequenceEqual(objectsAfterRead)); } }
static void Main(string[] args) { // create temporary list of objects List<FileRecord> records = new List<FileRecord>(); records.Add(new FileRecord { ID = 1, Account = "12345", Balance = 12.21, Date = "20111211" }); records.Add(new FileRecord { ID = 2, Account = "23456", Balance = 100.00, Date = "20111211" }); records.Add(new FileRecord { ID = 3, Account = "98765", Balance = 1000.00, Date = "20111211" }); records.Add(new FileRecord { ID = 4, Account = "90786", Balance = 99.99, Date = "20111211" }); records.Add(new FileRecord { ID = 5, Account = "34567", Balance = 123.45, Date = "20111211" }); FlatFile<FileRecord> flatFile = new FlatFile<FileRecord>(); flatFile.writeFile(records, "FlatFile.txt"); }
private static int Main(string[] args) { if (!File.Exists("config.json")) { File.WriteAllText("config.json", JsonConvert.SerializeObject(new Config { TwitterInfo = new TwitterInfo() }, Formatting.Indented)); Log.Error("config.json not found, generated file."); return(1); } Config = JsonConvert.DeserializeObject <Config>(File.ReadAllText("config.json")); Twitter.TwitterInfo = Config.TwitterInfo; Twitter.Login(); _lastUpdate = DateTime.UtcNow.AddSeconds(-601); FlatFile.Initialize(); MobileExport.Initialize(); ParserDictionary.Add(Platform.Pc, new WorldStateParser(Platform.Pc)); ParserDictionary.Add(Platform.PS4, new WorldStateParser(Platform.PS4)); ParserDictionary.Add(Platform.Xbox, new WorldStateParser(Platform.Xbox)); //ParserDictionary.Add(Platform.PcChina, new WorldStateParser(Platform.PcChina)); //FlatFile.SerializeAlerts(Platform.Pc); //FlatFile.SerializeAlerts(Platform.PS4); //FlatFile.SerializeInvasion(Platform.Pc); //FlatFile.SerializeInvasion(Platform.PS4); (new Thread(Update)).Start(args.Length == 0); while (true) { try { var word = Console.ReadLine(); if (word == null) { continue; } var parts = word.Split(new[] { ' ' }, 2); var command = parts[0]; switch (command) { case "reload": Config = JsonConvert.DeserializeObject <Config>(File.ReadAllText("config.json")); break; case "names": { /*FlatFile.PlanetNames = * JsonConvert.DeserializeObject<Dictionary<string, string>>( * File.ReadAllText("planetnames.json")); * Log.Info("Loaded " + FlatFile.PlanetNames.Count + " planet names from file.");*/ FlatFile.PlanetRegionNames = JsonConvert.DeserializeObject <Dictionary <string, string> >( File.ReadAllText("planetnamesregion.json")); Log.Info("Loaded " + FlatFile.PlanetRegionNames.Count + " planet names with region from file."); FlatFile.ItemNames = JsonConvert.DeserializeObject <Dictionary <string, string> >( File.ReadAllText("names.json")); Log.Info("Loaded " + FlatFile.ItemNames.Count + " item names from file."); FlatFile.LanguageStrings = JsonConvert.DeserializeObject <Dictionary <string, string> >( File.ReadAllText("strings.json")); Log.Info("Loaded " + FlatFile.LanguageStrings.Count + " language strings from file."); break; } case "currenttick": { Log.InfoFormat("Current tick: {0}", _updateCount); break; } case "settick": { long newcount = 0; if (long.TryParse(parts[1], out newcount)) { _updateCount = newcount; } else { Log.Warn("Could not parse the param into an int."); } break; } case "exit": case "quit": Log.InfoFormat("Current count: {0}", _updateCount); _exiting = true; _scheduler.Shutdown(); return(0); case "forcequit": case "forceexit": Log.InfoFormat("Current count: {0}", _updateCount); Environment.Exit(0); return(0); } } catch (Exception e) { Log.Error("Exception thrown when executing command."); Log.Error(e.ToString()); } } }
public DataDictionary(string[] aDatas, FlatFile aFlatFile) { this.FillData(aDatas, aFlatFile); }
protected override void DoRun(FlatFile flatFile) { _numberOfChars = 0; _currentFlatFile = flatFile; }
public string ProcessMsd1Records(string filename, FlatFile fileInfo) { _fileInfo = fileInfo; string record = string.Empty; bodyErrorMsg.Clear(); var lines = File.ReadAllLines(filename); var noMsd1Data = lines.Length < 2 ? "There are no records to process in this file. Please correct it or contact the helpdesk for advice." : string.Empty; recordErrors.AppendLine(noMsd1Data).AppendLine(); for (int i = 1; i < lines.Length; i++) { record = lines[i]; int recordMinLength = 59; int startIndex = 8; try { if (!string.IsNullOrEmpty(record.Trim())) { int endIndex = Math.Min(record.Length - startIndex, recordMinLength); int endIndexPrevious = lines[i - 1].Length - startIndex; int endIndexCurrent = record.Length - startIndex; string currentLineLength = endIndexCurrent > startIndex && record.Length >= recordMinLength?record.Substring(startIndex, endIndex) : string.Empty; if (string.IsNullOrEmpty(currentLineLength)) { recordErrors.AppendLine().AppendFormat('"' + record + '"' + " is not a valid record because it is missing some data"); } else { string previousLineLength = endIndexPrevious > startIndex && lines[i - 1].Length >= recordMinLength ? lines[i - 1].Substring(startIndex, Math.Min(endIndexPrevious, endIndex)) : string.Empty; if (i == 1 || string.IsNullOrEmpty(lines[i - 1].Trim()) || !currentLineLength.Equals(previousLineLength)) { _msd1.Msd1Id = _helperService.GetUniqueKey(); } ValidateAndPopulateMsd1DataFromFlatFile(record, filename); if (string.IsNullOrEmpty(recordErrors.ToString().Trim()) && _msd1.Msd1Id != null) { AddCargoSummaryToMsd1Data(record); int endIndexNext = i == (lines.Length - 1) ? 0 : lines[i + 1].Length; string nextLineLength = endIndexNext > startIndex && lines[i + 1].Length >= recordMinLength ? lines[i + 1].Substring(startIndex, endIndex) : string.Empty; if (i == (lines.Length - 1) || !record.Substring(startIndex, endIndex).Equals(nextLineLength)) { AddMsd1DataToDatabase(); } } } if (!string.IsNullOrEmpty(recordErrors.ToString().Trim())) { bodyErrorMsg.AppendLine(" #Line from your file").AppendLine().AppendLine(record) .AppendLine(" #What is wrong and how to fix it").AppendLine().Append(recordErrors) .AppendLine().Append(" ___").AppendLine(); recordErrors.Clear(); } } } catch (Exception err) { _logger.LogError("Ascii Record error:", err.Message, err.InnerException != null ? err.InnerException.Message : string.Empty, err.StackTrace); if (_fileInfo != null) { recordErrors.AppendLine(err.Message).Append(" Please correct it or contact the helpdesk for advice."); bodyErrorMsg.AppendLine(" #Line from your file").AppendLine().AppendLine(record) .AppendLine(" #What is wrong and how to fix it").AppendLine().Append(recordErrors) .AppendLine().Append(" ___").AppendLine(); recordErrors.Clear(); } } } return(bodyErrorMsg.ToString()); }
static int Main(string[] args) { //1. Logger initialization Logger logger = LogManager.GetCurrentClassLogger(); int TimePart = 0; decimal PercentagePart = 0M; bool? RunSettingsPart = null; bool? HDDCheckSettingsPart = null; if (args.Length > 0) { if (args.Where(x => x.ToLower() == "-help").ToList().Count > 0) { logger.Info("-----------------------------------------------------------------------------"); logger.Info("-t for Time setting in mins"); logger.Info("-p for Percentage setting"); logger.Info("-r for Run setting (True/False)"); logger.Info("-h for HDD Check (True/False)"); logger.Info("Example : >N3PS.File.Compare.exe -hFalse"); logger.Info("Meaning program will be running 30 mins, 100% random record picked up and New table to be created for processing or not."); logger.Info("-----------------------------------------------------------------------------"); return(0); } try { var timePart = args.Where(x => x.ToLower().Contains("-t")).ToList(); if (timePart.Count > 0) { TimePart = Convert.ToInt32(timePart[0].ToLower().Replace("-t", "")); logger.Info($"TimePart Argument Value : {TimePart}"); } var percentagePart = args.Where(x => x.ToLower().Contains("-p")).ToList(); if (percentagePart.Count > 0) { PercentagePart = Convert.ToDecimal(percentagePart[0].ToLower().Replace("-p", "")); logger.Info($"PercentagePart Argument Value : {PercentagePart}"); } var runSettingsPart = args.Where(x => x.ToLower().Contains("-r")).ToList(); if (runSettingsPart.Count > 0) { RunSettingsPart = Convert.ToBoolean(runSettingsPart[0].ToLower().Replace("-r", "")); logger.Info($"RunSettingsPart Argument Value : {RunSettingsPart}"); } var runHddCheckPart = args.Where(x => x.ToLower().Contains("-h")).ToList(); if (runHddCheckPart.Count > 0) { HDDCheckSettingsPart = Convert.ToBoolean(runHddCheckPart[0].ToLower().Replace("-h", "")); logger.Info($"HDDCheckSettingsPart Argument Value : {HDDCheckSettingsPart}"); } } catch (Exception excp) { logger.Error("Error in processing the command level arguments. " + excp.ToString() + " --- " + excp.StackTrace); } } //2. XML File Initialization string FlatFileXmlName = @".\XMLFiles\FileFormat.xml"; string SettingsXmlName = @".\XMLFiles\Settings.xml"; //string ValidationRuleXmlName = @".\XMLFiles\ValidationRule.xml"; //3. Convert FlatFile to C# objects FlatFile flatFile = new FlatFile(); FlatFile fetchedFlatFileObj = flatFile.GetInstance(FlatFileXmlName, logger); if (fetchedFlatFileObj == null) { logger.Error($"Error while loading the Flat File XML : {FlatFileXmlName}"); return(0); } //4.Convert Settings File to C# objects SettingsFile settingsFile = new SettingsFile(); SettingsFile fetchedSettingsObj = settingsFile.GetInstance(SettingsXmlName, logger); if (fetchedSettingsObj == null) { logger.Error($"Error while loading the Settings File XML : {SettingsXmlName}"); return(0); } if (TimePart != 0) { logger.Info($"Overidden Time Part from Settings.xml: {TimePart}"); fetchedSettingsObj.Time = TimePart; } if (PercentagePart != 0M) { logger.Info($"Overidden Percentage from Settings.xml: {PercentagePart}"); fetchedSettingsObj.Percentage = PercentagePart; } if (RunSettingsPart != null) { logger.Info($"Overidden Run Settings from Settings.xml: {RunSettingsPart}"); fetchedSettingsObj.NewRun = RunSettingsPart.Value; } logger.Info("Settings : Start ----------------------"); logger.Info($"Time : {fetchedSettingsObj.Time}"); logger.Info($"Percentage : {fetchedSettingsObj.Percentage}"); logger.Info($"NewRun : {fetchedSettingsObj.NewRun}"); logger.Info("Settings : END ----------------------"); ////5. Convert ValidationRule to C# objects //ValidationRuleFile validationRuleFile = new ValidationRuleFile(); //ValidationRuleFile fetchedValidationRuleObj = validationRuleFile.GetInstance(ValidationRuleXmlName, logger); //if (fetchedValidationRuleObj == null) //{ // logger.Error($"Error while loading the Validation Rule File XML : {ValidationRuleXmlName}"); // return 0; //} //var dllsDetails = fetchedValidationRuleObj.ValidationRules.Where(x => x.DLLInfo != null && !string.IsNullOrEmpty(x.DLLInfo.DLLName) ).ToList(); //Hashtable assemblyDetails = new Hashtable(); //if (dllsDetails.Count() > 0) //{ // foreach (ValidationsRule rule in dllsDetails) // { // FileInfo f = new FileInfo(@".\ExternalDLLs\" + rule.DLLInfo.DLLName); // logger.Info($"Full File Name : {f.FullName}"); // if (!System.IO.File.Exists(f.FullName)) // { // logger.Error($"External DLL is not exist {rule.DLLInfo.DLLName} in ExternalDLLs folder."); // return 0; // } // else // { // Assembly assembly = Assembly.LoadFile(f.FullName); // assemblyDetails.Add(rule.ColumnNumber, assembly); // } // } //} //6. HDD Size Check if (HDDCheckSettingsPart == null) { HDDCheckSettingsPart = true; } //Check for free space if (HDDCheckSettingsPart.Value) { HDDCheck check = new HDDCheck(); bool isFreeSpaceAvailable = check.IsEnoughSpaceAvailable(fetchedFlatFileObj.FlatFilePath1, logger); if (!isFreeSpaceAvailable) { return(0); } } else { logger.Info("HDD Check is Skipped."); } //logger.Info($"Flat file Path : {fetchedFlatFileObj.FlatFilePath}"); string DBName = "ICA"; SQLiteHelper sqlManipulation = new SQLiteHelper(); bool isDBExist = sqlManipulation.IsDBExist(DBName); if (!isDBExist) { sqlManipulation.CreateDB(DBName, logger); } //SQLiteHelper sqlLite = new SQLiteHelper(); System.Data.SQLite.SQLiteConnection m_dbConnection = sqlManipulation.OpenDBConnection1(DBName); string tableName1 = "ICATable1"; string tableName2 = "ICATable2"; string tableNameDeleted1 = "ICADeletedTable1"; string tableNameInserted2 = "ICAInsertedTable2"; string CreateDeletedTableSQLQuery1 = flatFile.GetFlatFileDeletedTableScript(tableNameDeleted1, fetchedFlatFileObj); string CreateInsertedTableSQLQuery2 = flatFile.GetFlatFileInsertTableScript(tableNameInserted2, fetchedFlatFileObj); string processedtableName1 = "ProcessedICATable1"; string processedtableName2 = "ProcessedICATable2"; FileHelper helper = new FileHelper(); if (fetchedSettingsObj.NewRun) { string CreateTableSQLQuery1 = flatFile.GetFlatFileTableScript(tableName1, fetchedFlatFileObj); string CreateTableSQLQuery2 = flatFile.GetFlatFileTableScript(tableName2, fetchedFlatFileObj); string CreateProcessedTableSQLQuery1 = flatFile.CreateFlatFileTableScript(processedtableName1); string CreateProcessedTableSQLQuery2 = flatFile.CreateFlatFileTableScript(processedtableName2); bool isExist = sqlManipulation.CheckTableExists(m_dbConnection, tableName1, logger); if (isExist) { sqlManipulation.DeleteTable(m_dbConnection, DBName, tableName1, logger); } sqlManipulation.CreateTable(m_dbConnection, DBName, CreateTableSQLQuery1, logger); isExist = sqlManipulation.CheckTableExists(m_dbConnection, tableName2, logger); if (isExist) { sqlManipulation.DeleteTable(m_dbConnection, DBName, tableName2, logger); } sqlManipulation.CreateTable(m_dbConnection, DBName, CreateTableSQLQuery2, logger); //sqlManipulation.DeleteTable(DBName, processedtableName1, logger); //sqlManipulation.DeleteTable(DBName, processedtableName2, logger); //sqlManipulation.CreateTable(DBName, CreateTableSQLQuery1, logger); //sqlManipulation.CreateTable(DBName, CreateTableSQLQuery2, logger); isExist = sqlManipulation.CheckTableExists(m_dbConnection, processedtableName1, logger); if (isExist) { sqlManipulation.DeleteTable(m_dbConnection, DBName, processedtableName1, logger); } sqlManipulation.CreateTable(m_dbConnection, DBName, CreateProcessedTableSQLQuery1, logger); isExist = sqlManipulation.CheckTableExists(m_dbConnection, processedtableName2, logger); if (isExist) { sqlManipulation.DeleteTable(m_dbConnection, DBName, processedtableName2, logger); } sqlManipulation.CreateTable(m_dbConnection, DBName, CreateProcessedTableSQLQuery2, logger); isExist = sqlManipulation.CheckTableExists(m_dbConnection, tableNameDeleted1, logger); if (isExist) { sqlManipulation.DeleteTable(m_dbConnection, DBName, tableNameDeleted1, logger); } sqlManipulation.CreateTable(m_dbConnection, DBName, CreateDeletedTableSQLQuery1, logger); isExist = sqlManipulation.CheckTableExists(m_dbConnection, tableNameInserted2, logger); if (isExist) { sqlManipulation.DeleteTable(m_dbConnection, DBName, tableNameInserted2, logger); } sqlManipulation.CreateTable(m_dbConnection, DBName, CreateInsertedTableSQLQuery2, logger); //sqlManipulation.CreateTable(DBName, CreateProcessedTableSQLQuery1, logger); //sqlManipulation.CreateTable(DBName, CreateProcessedTableSQLQuery2, logger); //sqlManipulation.CloseDBConnection(m_dbConnection); //var conn = new SQLiteConnection( "foofoo"); List <DynamicClass1> objList = new List <DynamicClass1>(); //SQLiteConnection m_dbConnection1 = sqlManipulation.OpenDBConnection(DBName); // m_dbConnection1.CreateTable<DynamicClass1>(); helper.InsertInto(sqlManipulation, fetchedFlatFileObj, fetchedSettingsObj, m_dbConnection, DBName, tableName1, processedtableName1, fetchedFlatFileObj.FlatFilePath1, logger, objList, true); List <DynamicClass2> objList1 = new List <DynamicClass2>(); helper.InsertInto(sqlManipulation, fetchedFlatFileObj, fetchedSettingsObj, m_dbConnection, DBName, tableName2, processedtableName2, fetchedFlatFileObj.FlatFilePath2, logger, objList1, false); } int table1Records = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection, tableName1, logger); //set logger.Info("Comaprison is started."); int table2Records = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection, tableName2, logger); //List< ProcessedDetails1> list1 = m_dbConnection.Query<ProcessedDetails1>($"SELECT IsError, Count(1) AS TotalRecords FROM {processedtableName1} GROUP BY IsError"); m_dbConnection.Close(); Thread.Sleep(1000); m_dbConnection = null; GC.Collect(); GC.WaitForPendingFinalizers(); System.Data.SQLite.SQLiteConnection m_dbConnection1 = sqlManipulation.OpenDBConnection1(DBName); //m_dbConnection1.Open(); var list1 = sqlManipulation.GetTotalRecordsInTable(m_dbConnection1, $"SELECT IsError, Count(1) AS TotalRecords FROM {processedtableName1} GROUP BY IsError", logger); //m_dbConnection1.Open(); helper.Compare(fetchedFlatFileObj, fetchedSettingsObj, sqlManipulation, m_dbConnection1, DBName, tableName1, tableName2, processedtableName1, processedtableName2, tableNameDeleted1, tableNameInserted2, table1Records, list1, logger); int tab1 = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, tableName1, logger); int tab2 = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, tableName2, logger); int insTab = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, tableNameInserted2, logger); int delTab = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, tableNameDeleted1, logger); int fetchedTab1 = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, processedtableName1, logger); int fetchedTab2 = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, processedtableName2, logger); logger.Info("-----------------------Report--------------------------------"); logger.Info($"{tableName1} : {tab1}"); logger.Info($"{tableName2} : {tab2}"); logger.Info($"{tableNameInserted2} : {insTab}"); logger.Info($"{tableNameDeleted1} : {delTab}"); logger.Info($"{processedtableName1} : {fetchedTab1}"); logger.Info($"{processedtableName2} : {fetchedTab2}"); logger.Info("-------------------------------------------------------"); helper.WriteToFile(sqlManipulation, fetchedFlatFileObj, m_dbConnection1, DBName, tableNameInserted2, logger, insTab); helper.WriteToFile(sqlManipulation, fetchedFlatFileObj, m_dbConnection1, DBName, tableNameDeleted1, logger, delTab); sqlManipulation.CloseDBConnection(m_dbConnection1); /*DataSet dsTotalRecords = sqlManipulation.GetTotalRecords(m_dbConnection, tableName, logger); * FileHelper helper = new FileHelper(); * ProcessedDetails processedDetails = helper.ValidateFile(fetchedFlatFileObj, fetchedSettingsObj, fetchedValidationRuleObj, sqlManipulation, m_dbConnection, DBName, tableName, assemblyDetails, dsTotalRecords, logger); * * DataSet dsTotalRecords1 = sqlManipulation.GetTotalRecords(m_dbConnection, tableName, logger); * int totalRecords = 0; * int totalError = 0; * int totalSuccessProcessed = 0; * if (dsTotalRecords1 != null) * { * DataTable dt = dsTotalRecords1.Tables[0]; * foreach(DataRow dr in dt.Rows) * { * int tr = 0; * if(dr["TotalRecords"] != DBNull.Value) * { * tr = Convert.ToInt32(dr["TotalRecords"].ToString()); * if (dr["IsError"] != DBNull.Value && Convert.ToBoolean(dr["IsError"].ToString())) * { * totalError = totalError + tr; * } * else * { * * totalSuccessProcessed = totalSuccessProcessed + tr; * * } * * totalRecords = totalRecords + tr; * } * } * } * logger.Info("------------------------------------------------------------"); * logger.Info($"Total Records: " + processedDetails.TotalRecords); * logger.Info($"Total Records Processed: " + totalRecords);//(processedDetails.TotalErrorRecords + processedDetails.TotalSeccessfullyProcessedRecords)); * logger.Info($"Total Error Records: " + totalError);// processedDetails.TotalErrorRecords); * logger.Info($"Total Successfully Processed Records: " + totalSuccessProcessed);// processedDetails.TotalSeccessfullyProcessedRecords); * logger.Info("------------------------------------------------------------"); * sqlManipulation.CloseDBConnection(m_dbConnection); * * //sqlLite.CreateTable(DBName, CreateTableSQLQuery, logger); * //sqlLite.InsertRecord(DBName, tableName, 1, "Nothing", logger); * //DataSet dt = sqlLite.RetrieveRecord(DBName, tableName, 1, logger); * //FileHelper f = new FileHelper(); * //f.ValidateFile(@"C:\Users\vishal.chilka\Desktop\ZSB120OM.OUT"); * * return 0;*/ // sqlManipulation.CloseDBConnection(m_dbConnection); return(0); }
private static void CreateTestFile() { var testData = new[] { new PersonFlat { Age = 34, Gender = "Male", Name = "Joao", Number = "21", City = "Lisbon", Street = "Main Street" }, new PersonFlat { Age = 33, Gender = "Male", Name = "Joao", Number = "20", City = "Lisbon", Street = "Main Street" }, new PersonFlat { Age = 32, Gender = "Male", Name = "Joao", Number = "19", City = "Lisbon", Street = "Main Street" } }; using (var fileStream = File.OpenWrite(DataFile)) { var flatFile = new FlatFile<PersonFlat>(_layout, fileStream); flatFile.Write(testData); } }
public async Task <bool> SaveIncomingFile(FlatFile flatFile, string fileName, Stream fileStream) { return(await SaveFileStream(flatFile, EFlatFilePath.Incoming, fileName, fileStream)); }
/// <summary> /// Adds a guid to the file name and moves it to the Incoming directory. /// </summary> /// <param name="flatFile"></param> /// <param name="fileName"></param> /// <param name="fromDirectory"></param> /// <param name="toDirectory"></param> /// <returns></returns> public async Task <bool> MoveFile(FlatFile flatFile, string fileName, EFlatFilePath fromDirectory, EFlatFilePath toDirectory) { return(await MoveFile(flatFile, fromDirectory, toDirectory, fileName)); }
public static string GetTacticalAlertDescription(dynamic alert) { return("Tactical Alert - " + FlatFile.GetString((string)alert.Desc) + " - Conclave: " + alert.MaxConclave); }
public static ConfigurationFile Load(string filePath) { return(FlatFile.ReadFile(filePath).XmlDeserialize <ConfigurationFile>()); }
protected abstract void DoRun(FlatFile flatFile);
public void InitiateAsciiFileProcess(string filename) { try { _fileInfo = new FlatFile(); _logFileData = new LogFileData(); var senderId = ProcessFileHeader(filename); if (!string.IsNullOrEmpty(senderId) && string.IsNullOrEmpty(headerErrors.ToString().Trim())) { if (_fileInfo != null && _fileInfo.IsAmendment.HasValue && _fileInfo.IsAmendment.Value == 1 && previousFileRefId > -1) { switch (_fileInfo.TableRef.ToUpper()) { case "TABLE-1-UO-Q": case "TABLE-1-UQ-Q": _msd1DataService.DeleteAllPreviousMsd1Data(previousFileRefId); break; case "TABLE-2-FT-A": _msd2DataService.DeleteAllPreviousMsd2Data(previousFileRefId); break; case "TABLE-3-LA-A": _msd3DataService.DeleteAllPreviousMsd3Data(previousFileRefId); break; } } if ((!isDuplicatedFile && _fileInfo.IsAmendment.Value == 0) || (_fileInfo.IsAmendment.Value == 1 && previousFileRefId > -1)) { switch (_fileInfo.TableRef.ToUpper()) { case "TABLE-1-UO-Q": case "TABLE-1-UQ-Q": var errorMassage = _msd1FileProcess.ProcessMsd1Records(filename, _fileInfo); bodyErrorMsg.Append(errorMassage); break; case "TABLE-2-FT-A": errorMassage = _msd2FileProcess.ProcessMsd2Records(filename, _fileInfo); bodyErrorMsg.Append(errorMassage); break; case "TABLE-3-LA-A": errorMassage = _msd3FileProcess.ProcessMsd3Records(filename, _fileInfo); bodyErrorMsg.Append(errorMassage); break; } } else { var errmsg = (_fileInfo.IsAmendment.Value == 1 && previousFileRefId == -1) ? string.Format("{0} is an amendment file but there is no file with the same submission reference to amend - please contact the helpdesk for advice.", _fileInfo.FileName.Substring(0, _fileInfo.FileName.IndexOf('.') + 4)) : string.Format("{0} has the same sender reference as a file you have already sent, so it has not been processed." + " If it is not a duplicate sent by mistake, please use a different sender reference in the header and resubmit the file.", _fileInfo.FileName.Substring(0, _fileInfo.FileName.IndexOf('.') + 4)); bodyErrorMsg.AppendFormat(errmsg); } if (!string.IsNullOrEmpty(bodyErrorMsg.ToString().Trim())) { AddErrorLogToDbAndNotifyUser(filename, bodyErrorMsg.ToString()); } bodyErrorMsg.Clear(); } headerErrors.Clear(); } catch (Exception err) { _logger.LogError("Ascii File error:", err.Message, err.InnerException != null ? err.InnerException.Message : string.Empty, err.StackTrace); if (_fileInfo != null) { bodyErrorMsg.AppendLine(filename + Environment.NewLine + err.Message); AddErrorLogToDbAndNotifyUser(filename, bodyErrorMsg.ToString()); } } finally { ArchiveProcessedFilesFromBucket(filename, false); bodyErrorMsg.Clear(); headerErrors.Clear(); _fileInfo = null; } }
public async Task <List <DexihFileProperties> > GetFiles(FlatFile flatFile, EFlatFilePath path) { return(await GetFileList(flatFile, path)); }
public ProcessedDetails Compare(FlatFile fetchedFlatFileObj, SettingsFile fetchedSettingsObj, SQLiteHelper sqlManipulation, System.Data.SQLite.SQLiteConnection connection, string DBName, string tableName1, string tableName2, string processedTable1, string processedTable2, string deletedTable, string insertedTable, int totalRecords, DataSet dsTotalRecords, Logger logger) { //string[] allLines = System.IO.File.ReadAllLines(fetchedFlatFileObj.FlatFilePath1); if (fetchedSettingsObj.Percentage == 0) { fetchedSettingsObj.Percentage = 100; } decimal totalCountDec = Convert.ToDecimal((fetchedSettingsObj.Percentage / 100) * totalRecords); logger.Info($"Total Count as per Percentage : {totalCountDec.ToString("#.00")}"); int totalCount = Convert.ToInt32(totalCountDec); if ((totalCount * 1.0M) < totalCountDec) { totalCount++; logger.Info($"Total Count as per Percentage after converting to Integer : {totalCount}"); } ProcessedDetails processedDetails = new ProcessedDetails(); int TotalRecords = 0; int TotalErrorRecords = 0; int TotalSeccessfullyProcessedRecords = 0; if (dsTotalRecords.Tables[0].Rows.Count > 0) { foreach (DataRow dr in dsTotalRecords.Tables[0].Rows) { int records = Convert.ToInt32(dr["TotalRecords"].ToString()); if (Convert.ToBoolean(dr[0].ToString())) { TotalErrorRecords = TotalErrorRecords + records; } else { TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords + records; } TotalRecords = records + TotalRecords; } if (TotalRecords >= totalRecords) { logger.Info("All records are processed, in case of any error please check."); processedDetails.TotalRecords = TotalRecords; processedDetails.TotalErrorRecords = TotalErrorRecords; processedDetails.TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords; return(processedDetails); } } TotalRecords = totalRecords; logger.Info($"Total count = {totalCount}, ({fetchedSettingsObj.Percentage} / 100 )* {totalRecords}"); // SQLiteConnection connection = sqlManipulation.OpenDBConnection(DBName); DateTime startTime = DateTime.Now; if (fetchedSettingsObj.Time == 0) { fetchedSettingsObj.Time = 24 * 60 * 365; } DateTime endTime = startTime.AddMinutes(fetchedSettingsObj.Time); logger.Info($"Start Time : {startTime}"); logger.Info($"End Time : {endTime}"); Random rmd = new Random(); ParallelOptions ops = new ParallelOptions(); ops.MaxDegreeOfParallelism = 10; object obj = new object(); for (int i = 0; i < GetFactors(totalCount); i++) { int startCount = (i * 100); int lastCount = ((i + 1) * 100) > totalCount ? totalCount : ((i + 1) * 100); //for (int i = 0; i < totalCount; i++) List <int> bags = GetBags(totalRecords, lastCount - startCount, sqlManipulation, connection, processedTable1, logger); logger.Info($"startCount : {startCount}, lastCount : {lastCount}"); Parallel.ForEach(bags, ops, (randomLineNumber, loopState) => { try { bool isError = false; // int randomLineNumber = -1; //DataSet ds = new DataSet(); //lock (obj) //{ // randomLineNumber = rmd.Next(totalRecords); // //logger.Info($"Generated Line Number : {randomLineNumber + 1}"); // ds = sqlManipulation.RetrieveRecord(connection, processedTable1, randomLineNumber + 1, logger); // int loopIteration = 1; // bool allCompleted = false; // while (ds.Tables[0].Rows.Count > 0) // { // randomLineNumber = randomLineNumber + 1; // if (randomLineNumber > totalRecords - 1) // { // randomLineNumber = 0; // } // ds = sqlManipulation.RetrieveRecord(connection, processedTable1, randomLineNumber + 1, logger); // //logger.Info($"Total Records with Flat File Line Number : {randomLineNumber + 1} and Total Returned Count : {ds.Tables[0].Rows.Count}"); // if (loopIteration > TotalRecords) // { // allCompleted = true; // break; // } // loopIteration++; // } // if (allCompleted) // loopState.Stop(); // //break; // //loopState.Stop(); // logger.Info($"Random Line Number : {randomLineNumber + 1}"); //} //logger.Info($"Random Line Number : {randomLineNumber + 1}"); string query1 = $"SELECT * FROM {tableName1} WHERE FlatFileRowNumber={randomLineNumber + 1};"; //sqlManipulation.GetTotalRecordsInTable(connection, query1, logger); //Sqlite.TableMapping map = new Sqlite.TableMapping(Type.GetType(tableName1)); ////string query = "select * from " + TableName; //object[] objLists = new object[] { }; //List <object> list = connection.Query(map, query1, objLists).ToList(); DataSet ds1 = sqlManipulation.GetTotalRecordsInTable(connection, query1, logger); //List<DynamicClass1> list = null;//connection.Query<DynamicClass1>(query1, fieldLists.Cast<object>().ToArray()); DataRow randomLineContentFromTable1 = null; string fetchedNRIC = null; if (ds1.Tables[0].Rows.Count > 0) { randomLineContentFromTable1 = ds1.Tables[0].Rows[0]; fetchedNRIC = randomLineContentFromTable1[(fetchedFlatFileObj.PrimaryKeyColumnNumber)].ToString(); } //logger.Info($"fetchedNRIC : {fetchedNRIC}"); string query2 = $"SELECT * FROM {tableName2} WHERE NRICNumber='{fetchedNRIC}';"; DataSet ds2 = sqlManipulation.GetTotalRecordsInTable(connection, query2, logger); //DataSet ds2 = sqlManipulation.RetrieveRecord(connection, query2, logger); //logger.Info(query2); if (ds2.Tables[0].Rows.Count <= 0) { string query3 = $"INSERT INTO {deletedTable} SELECT * FROM {tableName1} WHERE FlatFileRowNumber={randomLineNumber + 1};"; //logger.Info($"Query 3 : {query3}"); sqlManipulation.InsertRecord(connection, query3, logger); } else { //string query3 = $"INSERT INTO {insertedTable} SELECT * FROM {tableName1} WHERE FlatFileRowNumber={randomLineNumber + 1};"; //string hashKey1 = string.Empty; DataRow randomLineContentFromTable2 = ds2.Tables[0].Rows[0]; //namicClass1 class1 = (list.FirstOrDefault() as DynamicClass1); //DataRow randomLineContentFromTable2 = ds2.Tables[0].Rows[0]; string hashKey1 = randomLineContentFromTable1["HashingKey"].ToString(); string hashKey2 = randomLineContentFromTable2["HashingKey"].ToString(); //logger.Info($"HashKey : {hashKey1} : {hashKey2}"); if (hashKey1 != hashKey2) { bool isDifference = false; foreach (Fields f1 in fetchedFlatFileObj.fields) { //DynamicClass2 class2 = (list2.FirstOrDefault() as DynamicClass2); var propSource = randomLineContentFromTable1[f1.FieldName].ToString(); //DynamicClass2 class2 = (list2.FirstOrDefault() as DynamicClass2); var propDest = randomLineContentFromTable2[f1.FieldName].ToString(); if (propSource != propDest) { isDifference = true; } } if (isDifference) { string query4 = $"INSERT INTO {insertedTable} SELECT * FROM {tableName1} WHERE FlatFileRowNumber={randomLineNumber + 1};"; //connection.CreateTable<ICADeletedTable1>(); //connection.Insert(class1); //logger.Info($"Query 4 : {query4}"); sqlManipulation.InsertRecord(connection, query4, logger); } } else { //logger.Info($"Record is matching : {randomLineNumber + 1}"); } } //Check Validations //if (randomLineContent.Length > fetchedFlatFileObj.RecordSize) //{ // logger.Error($"Flat file line number {randomLineNumber + 1} exceeded specified Record Size."); // isError = true; //} //if (randomLineContent.Length < fetchedFlatFileObj.RecordSize) //{ // logger.Error($"Flat file line number {randomLineNumber + 1} less than specified Record Size."); // isError = true; //} //if (!isError) //{ // TotalSeccessfullyProcessedRecords++; //} //else //{ // TotalErrorRecords++; //} //DataSet dsMain = sqlManipulation.RetrieveRecord(connection, processedTable1, randomLineNumber + 1, logger); //if (dsMain.Tables[0].Rows.Count <= 0) //{ // sqlManipulation.InsertRecord(connection, processedTable1, randomLineNumber + 1, isError, logger); //} if (endTime < DateTime.Now) { logger.Info($"As per settings ended the program at End Time {endTime}"); //break; loopState.Stop(); } // } } catch (Exception excp) { logger.Error($"Error in Comapring record : {randomLineNumber} " + excp.ToString() + " ---- " + excp.StackTrace); } }); } processedDetails.TotalRecords = TotalRecords; processedDetails.TotalErrorRecords = TotalErrorRecords; processedDetails.TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords; return(processedDetails); }
public abstract Task <bool> CreateDirectory(FlatFile file, EFlatFilePath path);
public string ProcessMsd3Records(GesmesHelpers gesmesHelpers, string filename, FlatFile fileInfo) { _fileInfo = fileInfo; string record = string.Empty; var allLines = File.ReadAllLines(filename); var lines = allLines.Where(c => c.Contains("ARR")).ToList(); for (int i = 0; i < lines.Count; i++) { string currentRecord = string.Empty; try { if (!string.IsNullOrEmpty(lines[i]) && lines[i].Substring(0, 3) == "ARR") { string[] msd3Elements = gesmesHelpers.Split(":", lines[i].Replace("'", "")); currentRecord = string.Join('|', msd3Elements, 0, 7); string lastRecord = (i == 0) ? string.Empty : string.Join('|', gesmesHelpers.Split(":", lines[i - 1]), 0, 7); string nextRecord = (i == (lines.Count - 1)) ? string.Empty : string.Join('|', gesmesHelpers.Split(":", lines[i + 1]), 0, 7); if (i == 0 || !currentRecord.Equals(lastRecord)) { _msd3.Id = _helperService.GetUniqueKey(); ValidateAndPopulateMsd3DataFromFlatFile(msd3Elements); } if ((string.IsNullOrEmpty(recordErrors.ToString().Trim()))) { AddShipplingLineAndAgents(msd3Elements); if (!(currentRecord.Equals(nextRecord))) { AddMsd3DataToDatabase(); _msd3.Msd3agents.Clear(); } } if (!(string.IsNullOrEmpty(recordErrors.ToString().Trim()))) { bodyErrorMsg.AppendLine(" #Line from your file").AppendLine().AppendLine(lines[i]) .AppendLine(" #What is wrong and how to fix it").AppendLine().Append(recordErrors) .AppendLine().Append(" ___").AppendLine(); recordErrors.Clear(); } } } catch (Exception err) { _logger.LogError("Gesmes Record error:", err.Message, err.InnerException != null ? err.InnerException.Message : string.Empty, err.StackTrace); if (_fileInfo != null) { recordErrors.AppendLine(err.Message).Append(" Please correct it or contact the helpdesk for advice."); bodyErrorMsg.AppendLine(" #Line from your file").AppendLine().AppendLine(lines[i]) .AppendLine(" #What is wrong and how to fix it").AppendLine().Append(recordErrors) .AppendLine().Append(" ___").AppendLine(); recordErrors.Clear(); } } } return(bodyErrorMsg.ToString()); }
public abstract Task <bool> MoveFile(FlatFile file, EFlatFilePath fromPath, EFlatFilePath toPath, string fileName);
public ActionResult UploadOnlyFile(HttpPostedFileBase file) { try { if (file != null && file.ContentLength > 0) { bool saveToDB = ConfigurationManager.AppSettings["saveToDB"].Equals("1"); bool useRealFileName = ConfigurationManager.AppSettings["useRealFileName"].Equals("1"); bool useExtension = ConfigurationManager.AppSettings["useExtension"].Equals("1"); string _Path = ConfigurationManager.AppSettings["rootPath"]; string _fileExtenstion = useExtension ? Path.GetExtension(file.FileName) : ""; string _fileName = useRealFileName ? Path.GetFileName(file.FileName) : string.Format("{0}{1}", Guid.NewGuid().ToString(), _fileExtenstion); if (saveToDB) { // Save this model to Database var newFile = new FlatFile { FileName = _fileName, ContentType = file.ContentType }; using (var reader = new BinaryReader(file.InputStream)) { // Save to DB newFile.Content = reader.ReadBytes(file.ContentLength); using (var context = new DBContext()) { context.FlatFiles.Add(newFile); context.SaveChanges(); } // END Save to DB } } else { using (var reader = new BinaryReader(file.InputStream)) { // Save to path string _savePath = Path.Combine(Server.MapPath(_Path), _fileName); file.SaveAs(_savePath); // END Save to path } } TempData["prevUploadStatus"] = true; return(RedirectToAction("UploadOnlyFile")); } else { // Do something if no file uploaded return(RedirectToAction("UploadOnlyFile")); } } catch (Exception ex) { TempData["prevUploadStatus"] = false; return(RedirectToAction("UploadOnlyFile")); } }
public abstract Task <bool> DeleteFile(FlatFile file, EFlatFilePath path, string fileName);
protected override void DoRun(FlatFile flatFile) { _numberOfOcurrencesForCurrentFile = 0; _currentFlatFile = flatFile; }
public override async Task <Table> GetSourceTableInfo(Table originalTable, CancellationToken cancellationToken) { try { var flatFile = (FlatFile)originalTable; if (flatFile.FileConfiguration == null || flatFile.FileSample == null) { throw new ConnectionException($"The properties have not been set to import the flat files structure. Required properties are (FileFormat)FileFormat and (Stream)FileSample."); } var stream = new MemoryStream(); var writer = new StreamWriter(stream); writer.Write(flatFile.FileSample); writer.Flush(); stream.Position = 0; FileHandlerBase fileHandler = null; switch (flatFile.FormatType) { case ETypeCode.Json: fileHandler = new FileHandlerJson(flatFile, flatFile.RowPath); break; case ETypeCode.Text: fileHandler = new FileHandlerText(flatFile, flatFile.FileConfiguration); break; case ETypeCode.Xml: fileHandler = new FileHandlerXml(flatFile, flatFile.RowPath); break; default: throw new ConnectionException($"The source type {flatFile.FormatType} is not currently supported."); } var columns = await fileHandler.GetSourceColumns(stream); //The new datatable that will contain the table schema var newFlatFile = new FlatFile(); flatFile.Name = originalTable.Name; newFlatFile.Columns.Clear(); newFlatFile.LogicalName = newFlatFile.Name; newFlatFile.Description = ""; newFlatFile.FileConfiguration = flatFile.FileConfiguration; foreach (var column in columns) { newFlatFile.Columns.Add(column); } var col = new TableColumn() { //add the basic properties Name = "FileName", LogicalName = "FileName", IsInput = false, DataType = ETypeCode.String, DeltaType = TableColumn.EDeltaType.FileName, Description = "The name of the file the record was loaded from.", AllowDbNull = false, IsUnique = false }; newFlatFile.Columns.Add(col); return(newFlatFile); } catch (Exception ex) { throw new ConnectionException($"Failed to import the file structure. {ex.Message}", ex); } }
public DataDictionary(string aDataLine, FlatFile aFlatFile) { string[] theDatas = aDataLine.Split(','); this.FillData(theDatas, aFlatFile); }
public abstract Task <DexihFiles> GetFileEnumerator(FlatFile file, EFlatFilePath path, string searchPattern);
protected override void DoRun(FlatFile flatFile) { _numberRecords = 0; _currentFlatFile = flatFile; }
public abstract Task <List <DexihFileProperties> > GetFileList(FlatFile file, EFlatFilePath path);
public void Run(FlatFile flatFile) { _stopwatch.Start(); DoRun(flatFile); _stopwatch.Stop(); }
public abstract Task <Stream> GetWriteFileStream(FlatFile file, EFlatFilePath path, string fileName);
public FlatFile IsInFlatFile(string nip, string bankAccount) { if (!(flatFileData?.skrotypodatnikowczynnych?.Count >= 0)) { throw new System.Exception("Json file is not loaded. Use first LoadFlatFileAsync()"); } if (!nip.IsValidNIP()) { return(FlatFile.InvalidNip); } if (string.IsNullOrEmpty(bankAccount) || string.IsNullOrWhiteSpace(bankAccount) || bankAccount?.Length < 26) { return(FlatFile.InvalidBankAccount); } switch (CheckInBody(bankAccount)) { case FlatFile.FoundInActiveVatPayer: return(FlatFile.FoundInActiveVatPayer); case FlatFile.FoundInExemptVatPayer: return(FlatFile.FoundInExemptVatPayer); case FlatFile.InvalidNip: return(FlatFile.InvalidNip); case FlatFile.InvalidBankAccount: return(FlatFile.InvalidBankAccount); case FlatFile.NotFound: break; default: break; } string bankBranchNumber = bankAccount.Substring(2, 8); string maskToCompare = string.Empty; foreach (var item in flatFileData.maski) { if (bankBranchNumber.Equals(item.Substring(2, 8), StringComparison.OrdinalIgnoreCase)) { int IndexFrom = item.IndexOf("Y"); int range = item.Count(p => p.Equals('Y')); string VirtualAccount = Regex.Replace(item, "Y\\w*Y", bankAccount.Substring(IndexFrom, range)); FlatFile checkResult = CheckInBody(VirtualAccount); if (checkResult == FlatFile.NotFound) { continue; } else { return(checkResult); } } } return(FlatFile.NotFound); FlatFile CheckInBody(string account) { string hash = (flatFileData.naglowek.datagenerowaniadanych + nip + account).SHA512(Convert.ToInt32(flatFileData.naglowek.liczbatransformacji)); foreach (var item in flatFileData.skrotypodatnikowczynnych) { if (item.Equals(hash, StringComparison.OrdinalIgnoreCase)) { return(FlatFile.FoundInActiveVatPayer); } } foreach (var item in flatFileData.skrotypodatnikowzwolnionych) { if (item.Equals(hash, StringComparison.OrdinalIgnoreCase)) { return(FlatFile.FoundInExemptVatPayer); } } return(FlatFile.NotFound); } }
public abstract Task <bool> SaveFileStream(FlatFile file, EFlatFilePath path, string fileName, Stream fileStream);
public abstract string GetFullPath(FlatFile file, EFlatFilePath path);
public Table GetTable(DexihHub hub, Connection connection, InputColumn[] inputColumns, TransformSettings transformSettings, string referenceTableAlias = null) { Table table; if (connection == null) { table = new Table(); } else { EConnectionCategory category; if (connection is ConnectionFlatFile) { category = EConnectionCategory.File; } else { var connectionReference = Connections.GetConnection(connection.GetType()); category = connectionReference.ConnectionCategory; } switch (category) { case EConnectionCategory.File: table = new FlatFile(); if (FileFormatKey == null) { ((FlatFile)table).FileConfiguration = new FileConfiguration(); } else { var fileFormat = hub.DexihFileFormats.SingleOrDefault(f => f.IsValid && f.Key == FileFormatKey); if (fileFormat == null) { throw new RepositoryException( $"The file format for the table {Name} with key {FileFormatKey} could not be found."); } ((FlatFile)table).FileConfiguration = fileFormat?.GetFileFormat(); } break; case EConnectionCategory.WebService: table = new WebService(); break; default: table = new Table(); break; } } // create a temporary copy and exclude the fileFormat (which is different between DexihTable & Table) this.CopyProperties(table, false); switch (table) { case FlatFile flatFile: if (transformSettings.HasVariables()) { flatFile.FileIncomingPath = transformSettings.InsertHubVariables(flatFile.FileIncomingPath); flatFile.FileMatchPattern = transformSettings.InsertHubVariables(flatFile.FileMatchPattern); flatFile.FileOutgoingPath = transformSettings.InsertHubVariables(flatFile.FileOutgoingPath); flatFile.FileProcessedPath = transformSettings.InsertHubVariables(flatFile.FileProcessedPath); flatFile.FileRejectedPath = transformSettings.InsertHubVariables(flatFile.FileRejectedPath); flatFile.FileRootPath = transformSettings.InsertHubVariables(flatFile.FileRootPath); } break; case WebService restFunction: if (transformSettings.HasVariables()) { restFunction.RestfulUri = transformSettings.InsertHubVariables(restFunction.RestfulUri); restFunction.RowPath = transformSettings.InsertHubVariables(restFunction.RowPath); } break; } foreach (var dbColumn in DexihTableColumns.Where(c => c.IsValid && c.DeltaType != EDeltaType.IgnoreField).OrderBy(c => c.Position)) { table.Columns.Add(dbColumn.GetTableColumn(inputColumns, referenceTableAlias)); } foreach (var dbIndex in DexihTableIndexes.Where(c => c.IsValid)) { table.Indexes.Add(new TableIndex() { Name = dbIndex.Name, Columns = dbIndex.Columns.Select(ti => { var column = DexihTableColumns.SingleOrDefault(c => c.Key == ti.ColumnKey); if (column != null) { return(new TableIndexColumn(column.Name, ti.Direction)); } return(null); }).Where(c => c != null).ToList() }); } return(table); }
protected override void DoRun(FlatFile flatFile) { }
public void InsertInto(SQLiteHelper sqlManipulation, FlatFile fetchedFlatFileObj, SettingsFile fetchedSettingsObj, System.Data.SQLite.SQLiteConnection connection, string dBName, string tableName, string processedTableName, string filePath, Logger logger, dynamic objList, bool isFirst) { string[] allLines = System.IO.File.ReadAllLines(filePath); /*if (fetchedSettingsObj.Percentage == 0) * { * fetchedSettingsObj.Percentage = 100; * } * decimal totalCountDec = Convert.ToDecimal((fetchedSettingsObj.Percentage / 100) * allLines.Length); * logger.Info($"Total Count as per Percentage : {totalCountDec.ToString("#.00")}"); * int totalCount = Convert.ToInt32(totalCountDec); * if ((totalCount * 1.0M) < totalCountDec) * { * * totalCount++; * logger.Info($"Total Count as per Percentage after converting to Integer : {totalCount}"); * } * * * * ProcessedDetails processedDetails = new ProcessedDetails(); * int TotalRecords = 0; * int TotalErrorRecords = 0; * int TotalSeccessfullyProcessedRecords = 0; * DataSet dsTotalRecords = sqlManipulation.GetTotalRecords(connection, processedTableName, logger); * if (dsTotalRecords.Tables[0].Rows.Count > 0) * { * foreach (DataRow dr in dsTotalRecords.Tables[0].Rows) * { * int records = Convert.ToInt32(dr[1].ToString()); * * if (Convert.ToBoolean(dr[0].ToString())) * { * TotalErrorRecords = TotalErrorRecords + records; * } * else * { * TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords + records; * } * TotalRecords = records + TotalRecords; * * } * * if (TotalRecords >= allLines.Length) * { * logger.Info("All records are processed, in case of any error please check."); * * processedDetails.TotalRecords = TotalRecords; * processedDetails.TotalErrorRecords = TotalErrorRecords; * processedDetails.TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords; * return processedDetails; * } * } * * TotalRecords = allLines.Length; * logger.Info($"Total count = {totalCount}, ({fetchedSettingsObj.Percentage} / 100 )* {allLines.Length}"); * * * // SQLiteConnection connection = sqlManipulation.OpenDBConnection(DBName); * * DateTime startTime = DateTime.Now; * if (fetchedSettingsObj.Time == 0) * { * fetchedSettingsObj.Time = 24 * 60 * 365; * } * * * DateTime endTime = startTime.AddMinutes(fetchedSettingsObj.Time); * logger.Info($"Start Time : {startTime}"); * logger.Info($"End Time : {endTime}"); * * Random rmd = new Random();*/ ParallelOptions ops = new ParallelOptions(); ops.MaxDegreeOfParallelism = 1; //for (int i = 0; i < totalCount; i++) ThreadLocal <KeyedHashAlgorithm> sha1 = new ThreadLocal <KeyedHashAlgorithm>(() => { var kha = KeyedHashAlgorithm.Create("HMACSHA256"); kha.Key = Encoding.UTF8.GetBytes("N3PS"); return(kha); }); logger.Info($"allLines.Length : {allLines.Length}"); var fieldLists = new List <Field>(); foreach (Fields f in fetchedFlatFileObj.fields) { Field f1 = new Field(f.FieldName, typeof(string)); fieldLists.Add(f1); } fieldLists.Add(new Field("HashingKey", typeof(string))); //dynamic obj = new DynamicClass1(fieldLists); //List<DynamicClass1> objList = new List<DynamicClass1>(); for (int i = 0; i < GetFactors(allLines.Length); i++) { int lastCount = ((i + 1) * 100) > allLines.Length ? allLines.Length: ((i + 1) * 100); StringBuilder query = new StringBuilder(); query.Append($"INSERT INTO {tableName} Values"); Parallel.For((i * 100), lastCount, ops, (rand, loopState) => { try { //bool isError = false; int randomLineNumber = -1; DataSet ds = new DataSet(); randomLineNumber = rand; // rmd.Next(allLines.Length - 1); //logger.Info($"Generated Line Number : {randomLineNumber + 1}"); //ds = sqlManipulation.RetrieveRecord(connection, processedTableName, randomLineNumber + 1, logger); //int loopIteration = 1; //bool allCompleted = false; //while (ds.Tables[0].Rows.Count > 0) //{ // randomLineNumber = randomLineNumber + 1; // if (randomLineNumber > allLines.Length - 1) // { // randomLineNumber = 0; // } // ds = sqlManipulation.RetrieveRecord(connection, processedTableName, randomLineNumber + 1, logger); // //logger.Info($"Total Records with Flat File Line Number : {randomLineNumber + 1} and Total Returned Count : {ds.Tables[0].Rows.Count}"); // if (loopIteration > TotalRecords) // { // allCompleted = true; // break; // } // loopIteration++; //} //if (allCompleted) // break; //loopState.Stop(); //logger.Info($"Random Line Number : {randomLineNumber + 1}"); string randomLineContent = allLines[randomLineNumber]; //Check Validations if (randomLineContent.Length > fetchedFlatFileObj.RecordSize) { logger.Error($"Flat file line number {randomLineNumber + 1} exceeded specified Record Size."); //isError = true; } if (randomLineContent.Length < fetchedFlatFileObj.RecordSize) { logger.Error($"Flat file line number {randomLineNumber + 1} less than specified Record Size."); //isError = true; } //StringBuilder cols = new StringBuilder(); //StringBuilder values = new StringBuilder(); //StringBuilder combinedValues = new StringBuilder(); ////cols.AppendLine("("); //cols.Append("FlatFileRowNumber,"); ////values.AppendLine("("); //values.Append($"{randomLineNumber+1},"); dynamic obj = null; if (isFirst) { obj = new DynamicClass1(fieldLists); } else { obj = new DynamicClass2(fieldLists); } query.Append($"({(randomLineNumber+1)},"); StringBuilder vals = new StringBuilder(); vals.Append((randomLineNumber + 1)); for (int ik = 0; ik < fetchedFlatFileObj.fields.Count; ik++) //foreach (KeyValuePair<Type, object> kvp in ((IDictionary<Type, object>)obj)) { string content = randomLineContent.Substring(fetchedFlatFileObj.fields[ik].StartPosition - 1, fetchedFlatFileObj.fields[ik].Length).Trim(); //cols.Append(fields.FieldName + ","); //values.Append($"'{content.Replace("'","''")}',"); //combinedValues.Append(content); //PropertyInfo info = obj.GetType().GetProperty(fields.FieldName); //KeyValuePair<Type, object> p = obj._fields[fetchedFlatFileObj.fields[ik].FieldName]; query.Append($"'{content.Replace("'","''")}',"); vals.Append($"{content}"); //obj._fields[fetchedFlatFileObj.fields[ik].FieldName] = content; // p[ = content; //obj.TrySetMember(fields.FieldName, content); //info.SetValue(obj, content); //obj[fields.ColumnNumber] = content; } var buffer = sha1.Value.ComputeHash(Encoding.Unicode.GetBytes(vals.ToString())); //query[query.ToString().Length - 1] = ' '; if (Encoding.Unicode.GetString(buffer, 0, buffer.Length).Contains("'")) { } query.Append($"'{Encoding.Unicode.GetString(buffer, 0, buffer.Length).Replace("'", "''").Replace("\0","+")}'"); query.Append($"),"); query.AppendLine(Environment.NewLine); //objList.Add(obj); //connection.Insert(obj); //cols[cols.ToString().Length - 1] = ' '; //values[values.ToString().Length - 1] = ' '; //tl.Value.ComputeHash(Encoding.UTF8.GetBytes("message")); //string insertQuery = $"INSERT INTO {tableName} ({cols.ToString()},HashingKey) VALUES ({values.ToString()},'{Encoding.Unicode.GetString(buffer, 0, buffer.Length).Replace("'","+")}')"; //logger.Error($"SQL Query : {insertQuery}"); //if(!sqlManipulation.InsertRecord(connection, insertQuery, logger)) //{ // logger.Error($"Error in iserting record : {randomLineNumber} -- {insertQuery}"); //} //if (!isError) //{ // TotalSeccessfullyProcessedRecords++; //} //else //{ // TotalErrorRecords++; //} //DataSet ds1 = sqlManipulation.RetrieveRecord(connection, processedTableName, randomLineNumber + 1, logger); //if (ds1.Tables[0].Rows.Count <= 0) //{ // sqlManipulation.InsertRecord(connection, processedTableName, randomLineNumber + 1, isError, logger); //} //if (endTime < DateTime.Now) //{ // logger.Info($"As per settings ended the program at End Time {endTime}"); // break; // //loopState.Stop(); //} } catch (Exception excp) { logger.Error($"Error in inserting record : {i} : " + excp.ToString() + " ---- " + excp.StackTrace); } }); query[query.ToString().Length - 5] = ' '; //SQLiteConnection connection = new SQLiteConnection(new SQLite.Net.Platform.Win32.SQLitePlatformWin32(), $"{DBName}.sqlite"); // connection = sqlManipulation.OpenDBConnection(dBName); //connection.CreateTable<DynamicClass1>(CreateFlags.None); sqlManipulation.InsertRecord(connection, query.ToString(), logger); Thread.Sleep(100); //connection.InsertAll(objList); //processedDetails.TotalRecords = TotalRecords; //processedDetails.TotalErrorRecords = TotalErrorRecords; //processedDetails.TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords; //return processedDetails; } }