private void FillData(string[] aDatas, FlatFile aFlatFile)
{
this.m_Name = aDatas[0];
this.m_DataTye = (DataType)Enum.Parse(typeof(DataType), aDatas[1], true);
this.m_Length = Convert.ToInt32(aDatas[2]);
int thePos = (int)aFlatFile + 3;
if (String.IsNullOrEmpty(aDatas[thePos]) == false)
{
this.m_Index = Convert.ToInt32(aDatas[thePos]);
}
}
private static IEnumerable<PersonFlat> ReadFile()
{
//using the flattmapper as the documentation
using (var fileStream = File.OpenRead(DataFile))
{
var flatFile = new FlatFile<PersonFlat>(_layout, fileStream);
return flatFile.Read()
//we need a ToList here since flatmapper does iteractive reading,
//closing the stream before enumerating the collection will throw an error.
.ToList();
}
}
public void can_write_read_stream()
{
using (var memory = new MemoryStream())
{
var flatFile = new FlatFile<TestObject>(layout, memory, HandleEntryReadError);
flatFile.Write(objects);
memory.Seek(0, SeekOrigin.Begin);
var objectsAfterRead = flatFile.Read().ToList();
Assert.True(objects.SequenceEqual(objectsAfterRead));
}
}
static void Main(string[] args)
{
// create temporary list of objects
List<FileRecord> records = new List<FileRecord>();
records.Add(new FileRecord { ID = 1, Account = "12345", Balance = 12.21, Date = "20111211" });
records.Add(new FileRecord { ID = 2, Account = "23456", Balance = 100.00, Date = "20111211" });
records.Add(new FileRecord { ID = 3, Account = "98765", Balance = 1000.00, Date = "20111211" });
records.Add(new FileRecord { ID = 4, Account = "90786", Balance = 99.99, Date = "20111211" });
records.Add(new FileRecord { ID = 5, Account = "34567", Balance = 123.45, Date = "20111211" });
FlatFile<FileRecord> flatFile = new FlatFile<FileRecord>();
flatFile.writeFile(records, "FlatFile.txt");
}
private static int Main(string[] args)
{
if (!File.Exists("config.json"))
{
File.WriteAllText("config.json", JsonConvert.SerializeObject(new Config
{
TwitterInfo = new TwitterInfo()
},
Formatting.Indented));
Log.Error("config.json not found, generated file.");
return(1);
}
Config = JsonConvert.DeserializeObject <Config>(File.ReadAllText("config.json"));
Twitter.TwitterInfo = Config.TwitterInfo;
Twitter.Login();
_lastUpdate = DateTime.UtcNow.AddSeconds(-601);
FlatFile.Initialize();
MobileExport.Initialize();
ParserDictionary.Add(Platform.Pc, new WorldStateParser(Platform.Pc));
ParserDictionary.Add(Platform.PS4, new WorldStateParser(Platform.PS4));
ParserDictionary.Add(Platform.Xbox, new WorldStateParser(Platform.Xbox));
//ParserDictionary.Add(Platform.PcChina, new WorldStateParser(Platform.PcChina));
//FlatFile.SerializeAlerts(Platform.Pc);
//FlatFile.SerializeAlerts(Platform.PS4);
//FlatFile.SerializeInvasion(Platform.Pc);
//FlatFile.SerializeInvasion(Platform.PS4);
(new Thread(Update)).Start(args.Length == 0);
while (true)
{
try
{
var word = Console.ReadLine();
if (word == null)
{
continue;
}
var parts = word.Split(new[] { ' ' }, 2);
var command = parts[0];
switch (command)
{
case "reload":
Config = JsonConvert.DeserializeObject <Config>(File.ReadAllText("config.json"));
break;
case "names":
{
/*FlatFile.PlanetNames =
* JsonConvert.DeserializeObject<Dictionary<string, string>>(
* File.ReadAllText("planetnames.json"));
* Log.Info("Loaded " + FlatFile.PlanetNames.Count + " planet names from file.");*/
FlatFile.PlanetRegionNames =
JsonConvert.DeserializeObject <Dictionary <string, string> >(
File.ReadAllText("planetnamesregion.json"));
Log.Info("Loaded " + FlatFile.PlanetRegionNames.Count +
" planet names with region from file.");
FlatFile.ItemNames =
JsonConvert.DeserializeObject <Dictionary <string, string> >(
File.ReadAllText("names.json"));
Log.Info("Loaded " + FlatFile.ItemNames.Count + " item names from file.");
FlatFile.LanguageStrings =
JsonConvert.DeserializeObject <Dictionary <string, string> >(
File.ReadAllText("strings.json"));
Log.Info("Loaded " + FlatFile.LanguageStrings.Count + " language strings from file.");
break;
}
case "currenttick":
{
Log.InfoFormat("Current tick: {0}", _updateCount);
break;
}
case "settick":
{
long newcount = 0;
if (long.TryParse(parts[1], out newcount))
{
_updateCount = newcount;
}
else
{
Log.Warn("Could not parse the param into an int.");
}
break;
}
case "exit":
case "quit":
Log.InfoFormat("Current count: {0}", _updateCount);
_exiting = true;
_scheduler.Shutdown();
return(0);
case "forcequit":
case "forceexit":
Log.InfoFormat("Current count: {0}", _updateCount);
Environment.Exit(0);
return(0);
}
}
catch (Exception e)
{
Log.Error("Exception thrown when executing command.");
Log.Error(e.ToString());
}
}
}
public DataDictionary(string[] aDatas, FlatFile aFlatFile)
{
this.FillData(aDatas, aFlatFile);
}
protected override void DoRun(FlatFile flatFile)
{
_numberOfChars = 0;
_currentFlatFile = flatFile;
}
public string ProcessMsd1Records(string filename, FlatFile fileInfo)
{
_fileInfo = fileInfo;
string record = string.Empty;
bodyErrorMsg.Clear();
var lines = File.ReadAllLines(filename);
var noMsd1Data = lines.Length < 2 ? "There are no records to process in this file. Please correct it or contact the helpdesk for advice." : string.Empty;
recordErrors.AppendLine(noMsd1Data).AppendLine();
for (int i = 1; i < lines.Length; i++)
{
record = lines[i];
int recordMinLength = 59;
int startIndex = 8;
try
{
if (!string.IsNullOrEmpty(record.Trim()))
{
int endIndex = Math.Min(record.Length - startIndex, recordMinLength);
int endIndexPrevious = lines[i - 1].Length - startIndex;
int endIndexCurrent = record.Length - startIndex;
string currentLineLength = endIndexCurrent > startIndex && record.Length >= recordMinLength?record.Substring(startIndex, endIndex) : string.Empty;
if (string.IsNullOrEmpty(currentLineLength))
{
recordErrors.AppendLine().AppendFormat('"' + record + '"' + " is not a valid record because it is missing some data");
}
else
{
string previousLineLength = endIndexPrevious > startIndex && lines[i - 1].Length >= recordMinLength ? lines[i - 1].Substring(startIndex, Math.Min(endIndexPrevious, endIndex)) : string.Empty;
if (i == 1 || string.IsNullOrEmpty(lines[i - 1].Trim()) || !currentLineLength.Equals(previousLineLength))
{
_msd1.Msd1Id = _helperService.GetUniqueKey();
}
ValidateAndPopulateMsd1DataFromFlatFile(record, filename);
if (string.IsNullOrEmpty(recordErrors.ToString().Trim()) && _msd1.Msd1Id != null)
{
AddCargoSummaryToMsd1Data(record);
int endIndexNext = i == (lines.Length - 1) ? 0 : lines[i + 1].Length;
string nextLineLength = endIndexNext > startIndex && lines[i + 1].Length >= recordMinLength ? lines[i + 1].Substring(startIndex, endIndex) : string.Empty;
if (i == (lines.Length - 1) || !record.Substring(startIndex, endIndex).Equals(nextLineLength))
{
AddMsd1DataToDatabase();
}
}
}
if (!string.IsNullOrEmpty(recordErrors.ToString().Trim()))
{
bodyErrorMsg.AppendLine(" #Line from your file").AppendLine().AppendLine(record)
.AppendLine(" #What is wrong and how to fix it").AppendLine().Append(recordErrors)
.AppendLine().Append(" ___").AppendLine();
recordErrors.Clear();
}
}
}
catch (Exception err)
{
_logger.LogError("Ascii Record error:", err.Message, err.InnerException != null ? err.InnerException.Message : string.Empty, err.StackTrace);
if (_fileInfo != null)
{
recordErrors.AppendLine(err.Message).Append(" Please correct it or contact the helpdesk for advice.");
bodyErrorMsg.AppendLine(" #Line from your file").AppendLine().AppendLine(record)
.AppendLine(" #What is wrong and how to fix it").AppendLine().Append(recordErrors)
.AppendLine().Append(" ___").AppendLine();
recordErrors.Clear();
}
}
}
return(bodyErrorMsg.ToString());
}
static int Main(string[] args)
{
//1. Logger initialization
Logger logger = LogManager.GetCurrentClassLogger();
int TimePart = 0;
decimal PercentagePart = 0M;
bool? RunSettingsPart = null;
bool? HDDCheckSettingsPart = null;
if (args.Length > 0)
{
if (args.Where(x => x.ToLower() == "-help").ToList().Count > 0)
{
logger.Info("-----------------------------------------------------------------------------");
logger.Info("-t for Time setting in mins");
logger.Info("-p for Percentage setting");
logger.Info("-r for Run setting (True/False)");
logger.Info("-h for HDD Check (True/False)");
logger.Info("Example : >N3PS.File.Compare.exe -hFalse");
logger.Info("Meaning program will be running 30 mins, 100% random record picked up and New table to be created for processing or not.");
logger.Info("-----------------------------------------------------------------------------");
return(0);
}
try
{
var timePart = args.Where(x => x.ToLower().Contains("-t")).ToList();
if (timePart.Count > 0)
{
TimePart = Convert.ToInt32(timePart[0].ToLower().Replace("-t", ""));
logger.Info($"TimePart Argument Value : {TimePart}");
}
var percentagePart = args.Where(x => x.ToLower().Contains("-p")).ToList();
if (percentagePart.Count > 0)
{
PercentagePart = Convert.ToDecimal(percentagePart[0].ToLower().Replace("-p", ""));
logger.Info($"PercentagePart Argument Value : {PercentagePart}");
}
var runSettingsPart = args.Where(x => x.ToLower().Contains("-r")).ToList();
if (runSettingsPart.Count > 0)
{
RunSettingsPart = Convert.ToBoolean(runSettingsPart[0].ToLower().Replace("-r", ""));
logger.Info($"RunSettingsPart Argument Value : {RunSettingsPart}");
}
var runHddCheckPart = args.Where(x => x.ToLower().Contains("-h")).ToList();
if (runHddCheckPart.Count > 0)
{
HDDCheckSettingsPart = Convert.ToBoolean(runHddCheckPart[0].ToLower().Replace("-h", ""));
logger.Info($"HDDCheckSettingsPart Argument Value : {HDDCheckSettingsPart}");
}
}
catch (Exception excp)
{
logger.Error("Error in processing the command level arguments. " + excp.ToString() + " --- " + excp.StackTrace);
}
}
//2. XML File Initialization
string FlatFileXmlName = @".\XMLFiles\FileFormat.xml";
string SettingsXmlName = @".\XMLFiles\Settings.xml";
//string ValidationRuleXmlName = @".\XMLFiles\ValidationRule.xml";
//3. Convert FlatFile to C# objects
FlatFile flatFile = new FlatFile();
FlatFile fetchedFlatFileObj = flatFile.GetInstance(FlatFileXmlName, logger);
if (fetchedFlatFileObj == null)
{
logger.Error($"Error while loading the Flat File XML : {FlatFileXmlName}");
return(0);
}
//4.Convert Settings File to C# objects
SettingsFile settingsFile = new SettingsFile();
SettingsFile fetchedSettingsObj = settingsFile.GetInstance(SettingsXmlName, logger);
if (fetchedSettingsObj == null)
{
logger.Error($"Error while loading the Settings File XML : {SettingsXmlName}");
return(0);
}
if (TimePart != 0)
{
logger.Info($"Overidden Time Part from Settings.xml: {TimePart}");
fetchedSettingsObj.Time = TimePart;
}
if (PercentagePart != 0M)
{
logger.Info($"Overidden Percentage from Settings.xml: {PercentagePart}");
fetchedSettingsObj.Percentage = PercentagePart;
}
if (RunSettingsPart != null)
{
logger.Info($"Overidden Run Settings from Settings.xml: {RunSettingsPart}");
fetchedSettingsObj.NewRun = RunSettingsPart.Value;
}
logger.Info("Settings : Start ----------------------");
logger.Info($"Time : {fetchedSettingsObj.Time}");
logger.Info($"Percentage : {fetchedSettingsObj.Percentage}");
logger.Info($"NewRun : {fetchedSettingsObj.NewRun}");
logger.Info("Settings : END ----------------------");
////5. Convert ValidationRule to C# objects
//ValidationRuleFile validationRuleFile = new ValidationRuleFile();
//ValidationRuleFile fetchedValidationRuleObj = validationRuleFile.GetInstance(ValidationRuleXmlName, logger);
//if (fetchedValidationRuleObj == null)
//{
// logger.Error($"Error while loading the Validation Rule File XML : {ValidationRuleXmlName}");
// return 0;
//}
//var dllsDetails = fetchedValidationRuleObj.ValidationRules.Where(x => x.DLLInfo != null && !string.IsNullOrEmpty(x.DLLInfo.DLLName) ).ToList();
//Hashtable assemblyDetails = new Hashtable();
//if (dllsDetails.Count() > 0)
//{
// foreach (ValidationsRule rule in dllsDetails)
// {
// FileInfo f = new FileInfo(@".\ExternalDLLs\" + rule.DLLInfo.DLLName);
// logger.Info($"Full File Name : {f.FullName}");
// if (!System.IO.File.Exists(f.FullName))
// {
// logger.Error($"External DLL is not exist {rule.DLLInfo.DLLName} in ExternalDLLs folder.");
// return 0;
// }
// else
// {
// Assembly assembly = Assembly.LoadFile(f.FullName);
// assemblyDetails.Add(rule.ColumnNumber, assembly);
// }
// }
//}
//6. HDD Size Check
if (HDDCheckSettingsPart == null)
{
HDDCheckSettingsPart = true;
}
//Check for free space
if (HDDCheckSettingsPart.Value)
{
HDDCheck check = new HDDCheck();
bool isFreeSpaceAvailable = check.IsEnoughSpaceAvailable(fetchedFlatFileObj.FlatFilePath1, logger);
if (!isFreeSpaceAvailable)
{
return(0);
}
}
else
{
logger.Info("HDD Check is Skipped.");
}
//logger.Info($"Flat file Path : {fetchedFlatFileObj.FlatFilePath}");
string DBName = "ICA";
SQLiteHelper sqlManipulation = new SQLiteHelper();
bool isDBExist = sqlManipulation.IsDBExist(DBName);
if (!isDBExist)
{
sqlManipulation.CreateDB(DBName, logger);
}
//SQLiteHelper sqlLite = new SQLiteHelper();
System.Data.SQLite.SQLiteConnection m_dbConnection = sqlManipulation.OpenDBConnection1(DBName);
string tableName1 = "ICATable1";
string tableName2 = "ICATable2";
string tableNameDeleted1 = "ICADeletedTable1";
string tableNameInserted2 = "ICAInsertedTable2";
string CreateDeletedTableSQLQuery1 = flatFile.GetFlatFileDeletedTableScript(tableNameDeleted1, fetchedFlatFileObj);
string CreateInsertedTableSQLQuery2 = flatFile.GetFlatFileInsertTableScript(tableNameInserted2, fetchedFlatFileObj);
string processedtableName1 = "ProcessedICATable1";
string processedtableName2 = "ProcessedICATable2";
FileHelper helper = new FileHelper();
if (fetchedSettingsObj.NewRun)
{
string CreateTableSQLQuery1 = flatFile.GetFlatFileTableScript(tableName1, fetchedFlatFileObj);
string CreateTableSQLQuery2 = flatFile.GetFlatFileTableScript(tableName2, fetchedFlatFileObj);
string CreateProcessedTableSQLQuery1 = flatFile.CreateFlatFileTableScript(processedtableName1);
string CreateProcessedTableSQLQuery2 = flatFile.CreateFlatFileTableScript(processedtableName2);
bool isExist = sqlManipulation.CheckTableExists(m_dbConnection, tableName1, logger);
if (isExist)
{
sqlManipulation.DeleteTable(m_dbConnection, DBName, tableName1, logger);
}
sqlManipulation.CreateTable(m_dbConnection, DBName, CreateTableSQLQuery1, logger);
isExist = sqlManipulation.CheckTableExists(m_dbConnection, tableName2, logger);
if (isExist)
{
sqlManipulation.DeleteTable(m_dbConnection, DBName, tableName2, logger);
}
sqlManipulation.CreateTable(m_dbConnection, DBName, CreateTableSQLQuery2, logger);
//sqlManipulation.DeleteTable(DBName, processedtableName1, logger);
//sqlManipulation.DeleteTable(DBName, processedtableName2, logger);
//sqlManipulation.CreateTable(DBName, CreateTableSQLQuery1, logger);
//sqlManipulation.CreateTable(DBName, CreateTableSQLQuery2, logger);
isExist = sqlManipulation.CheckTableExists(m_dbConnection, processedtableName1, logger);
if (isExist)
{
sqlManipulation.DeleteTable(m_dbConnection, DBName, processedtableName1, logger);
}
sqlManipulation.CreateTable(m_dbConnection, DBName, CreateProcessedTableSQLQuery1, logger);
isExist = sqlManipulation.CheckTableExists(m_dbConnection, processedtableName2, logger);
if (isExist)
{
sqlManipulation.DeleteTable(m_dbConnection, DBName, processedtableName2, logger);
}
sqlManipulation.CreateTable(m_dbConnection, DBName, CreateProcessedTableSQLQuery2, logger);
isExist = sqlManipulation.CheckTableExists(m_dbConnection, tableNameDeleted1, logger);
if (isExist)
{
sqlManipulation.DeleteTable(m_dbConnection, DBName, tableNameDeleted1, logger);
}
sqlManipulation.CreateTable(m_dbConnection, DBName, CreateDeletedTableSQLQuery1, logger);
isExist = sqlManipulation.CheckTableExists(m_dbConnection, tableNameInserted2, logger);
if (isExist)
{
sqlManipulation.DeleteTable(m_dbConnection, DBName, tableNameInserted2, logger);
}
sqlManipulation.CreateTable(m_dbConnection, DBName, CreateInsertedTableSQLQuery2, logger);
//sqlManipulation.CreateTable(DBName, CreateProcessedTableSQLQuery1, logger);
//sqlManipulation.CreateTable(DBName, CreateProcessedTableSQLQuery2, logger);
//sqlManipulation.CloseDBConnection(m_dbConnection);
//var conn = new SQLiteConnection( "foofoo");
List <DynamicClass1> objList = new List <DynamicClass1>();
//SQLiteConnection m_dbConnection1 = sqlManipulation.OpenDBConnection(DBName);
// m_dbConnection1.CreateTable<DynamicClass1>();
helper.InsertInto(sqlManipulation, fetchedFlatFileObj, fetchedSettingsObj, m_dbConnection, DBName, tableName1, processedtableName1, fetchedFlatFileObj.FlatFilePath1, logger, objList, true);
List <DynamicClass2> objList1 = new List <DynamicClass2>();
helper.InsertInto(sqlManipulation, fetchedFlatFileObj, fetchedSettingsObj, m_dbConnection, DBName, tableName2, processedtableName2, fetchedFlatFileObj.FlatFilePath2, logger, objList1, false);
}
int table1Records = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection, tableName1, logger);
//set
logger.Info("Comaprison is started.");
int table2Records = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection, tableName2, logger);
//List< ProcessedDetails1> list1 = m_dbConnection.Query<ProcessedDetails1>($"SELECT IsError, Count(1) AS TotalRecords FROM {processedtableName1} GROUP BY IsError");
m_dbConnection.Close();
Thread.Sleep(1000);
m_dbConnection = null;
GC.Collect();
GC.WaitForPendingFinalizers();
System.Data.SQLite.SQLiteConnection m_dbConnection1 = sqlManipulation.OpenDBConnection1(DBName);
//m_dbConnection1.Open();
var list1 = sqlManipulation.GetTotalRecordsInTable(m_dbConnection1, $"SELECT IsError, Count(1) AS TotalRecords FROM {processedtableName1} GROUP BY IsError", logger);
//m_dbConnection1.Open();
helper.Compare(fetchedFlatFileObj, fetchedSettingsObj, sqlManipulation, m_dbConnection1, DBName, tableName1, tableName2, processedtableName1, processedtableName2, tableNameDeleted1, tableNameInserted2, table1Records, list1, logger);
int tab1 = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, tableName1, logger);
int tab2 = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, tableName2, logger);
int insTab = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, tableNameInserted2, logger);
int delTab = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, tableNameDeleted1, logger);
int fetchedTab1 = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, processedtableName1, logger);
int fetchedTab2 = sqlManipulation.GetTotalRecordsInTable1(m_dbConnection1, processedtableName2, logger);
logger.Info("-----------------------Report--------------------------------");
logger.Info($"{tableName1} : {tab1}");
logger.Info($"{tableName2} : {tab2}");
logger.Info($"{tableNameInserted2} : {insTab}");
logger.Info($"{tableNameDeleted1} : {delTab}");
logger.Info($"{processedtableName1} : {fetchedTab1}");
logger.Info($"{processedtableName2} : {fetchedTab2}");
logger.Info("-------------------------------------------------------");
helper.WriteToFile(sqlManipulation, fetchedFlatFileObj, m_dbConnection1, DBName, tableNameInserted2, logger, insTab);
helper.WriteToFile(sqlManipulation, fetchedFlatFileObj, m_dbConnection1, DBName, tableNameDeleted1, logger, delTab);
sqlManipulation.CloseDBConnection(m_dbConnection1);
/*DataSet dsTotalRecords = sqlManipulation.GetTotalRecords(m_dbConnection, tableName, logger);
* FileHelper helper = new FileHelper();
* ProcessedDetails processedDetails = helper.ValidateFile(fetchedFlatFileObj, fetchedSettingsObj, fetchedValidationRuleObj, sqlManipulation, m_dbConnection, DBName, tableName, assemblyDetails, dsTotalRecords, logger);
*
* DataSet dsTotalRecords1 = sqlManipulation.GetTotalRecords(m_dbConnection, tableName, logger);
* int totalRecords = 0;
* int totalError = 0;
* int totalSuccessProcessed = 0;
* if (dsTotalRecords1 != null)
* {
* DataTable dt = dsTotalRecords1.Tables[0];
* foreach(DataRow dr in dt.Rows)
* {
* int tr = 0;
* if(dr["TotalRecords"] != DBNull.Value)
* {
* tr = Convert.ToInt32(dr["TotalRecords"].ToString());
* if (dr["IsError"] != DBNull.Value && Convert.ToBoolean(dr["IsError"].ToString()))
* {
* totalError = totalError + tr;
* }
* else
* {
*
* totalSuccessProcessed = totalSuccessProcessed + tr;
*
* }
*
* totalRecords = totalRecords + tr;
* }
* }
* }
* logger.Info("------------------------------------------------------------");
* logger.Info($"Total Records: " + processedDetails.TotalRecords);
* logger.Info($"Total Records Processed: " + totalRecords);//(processedDetails.TotalErrorRecords + processedDetails.TotalSeccessfullyProcessedRecords));
* logger.Info($"Total Error Records: " + totalError);// processedDetails.TotalErrorRecords);
* logger.Info($"Total Successfully Processed Records: " + totalSuccessProcessed);// processedDetails.TotalSeccessfullyProcessedRecords);
* logger.Info("------------------------------------------------------------");
* sqlManipulation.CloseDBConnection(m_dbConnection);
*
* //sqlLite.CreateTable(DBName, CreateTableSQLQuery, logger);
* //sqlLite.InsertRecord(DBName, tableName, 1, "Nothing", logger);
* //DataSet dt = sqlLite.RetrieveRecord(DBName, tableName, 1, logger);
* //FileHelper f = new FileHelper();
* //f.ValidateFile(@"C:\Users\vishal.chilka\Desktop\ZSB120OM.OUT");
*
* return 0;*/
// sqlManipulation.CloseDBConnection(m_dbConnection);
return(0);
}
private static void CreateTestFile()
{
var testData = new[]
{
new PersonFlat
{
Age = 34,
Gender = "Male",
Name = "Joao",
Number = "21",
City = "Lisbon",
Street = "Main Street"
},
new PersonFlat
{
Age = 33,
Gender = "Male",
Name = "Joao",
Number = "20",
City = "Lisbon",
Street = "Main Street"
},
new PersonFlat
{
Age = 32,
Gender = "Male",
Name = "Joao",
Number = "19",
City = "Lisbon",
Street = "Main Street"
}
};
using (var fileStream = File.OpenWrite(DataFile))
{
var flatFile = new FlatFile<PersonFlat>(_layout, fileStream);
flatFile.Write(testData);
}
}
public async Task <bool> SaveIncomingFile(FlatFile flatFile, string fileName, Stream fileStream)
{
return(await SaveFileStream(flatFile, EFlatFilePath.Incoming, fileName, fileStream));
}
/// <summary>
/// Adds a guid to the file name and moves it to the Incoming directory.
/// </summary>
/// <param name="flatFile"></param>
/// <param name="fileName"></param>
/// <param name="fromDirectory"></param>
/// <param name="toDirectory"></param>
/// <returns></returns>
public async Task <bool> MoveFile(FlatFile flatFile, string fileName, EFlatFilePath fromDirectory, EFlatFilePath toDirectory)
{
return(await MoveFile(flatFile, fromDirectory, toDirectory, fileName));
}
public static string GetTacticalAlertDescription(dynamic alert)
{
return("Tactical Alert - " + FlatFile.GetString((string)alert.Desc) + " - Conclave: " + alert.MaxConclave);
}
public static ConfigurationFile Load(string filePath)
{
return(FlatFile.ReadFile(filePath).XmlDeserialize <ConfigurationFile>());
}
protected abstract void DoRun(FlatFile flatFile);
public void InitiateAsciiFileProcess(string filename)
{
try
{
_fileInfo = new FlatFile();
_logFileData = new LogFileData();
var senderId = ProcessFileHeader(filename);
if (!string.IsNullOrEmpty(senderId) && string.IsNullOrEmpty(headerErrors.ToString().Trim()))
{
if (_fileInfo != null &&
_fileInfo.IsAmendment.HasValue &&
_fileInfo.IsAmendment.Value == 1 &&
previousFileRefId > -1)
{
switch (_fileInfo.TableRef.ToUpper())
{
case "TABLE-1-UO-Q":
case "TABLE-1-UQ-Q":
_msd1DataService.DeleteAllPreviousMsd1Data(previousFileRefId);
break;
case "TABLE-2-FT-A":
_msd2DataService.DeleteAllPreviousMsd2Data(previousFileRefId);
break;
case "TABLE-3-LA-A":
_msd3DataService.DeleteAllPreviousMsd3Data(previousFileRefId);
break;
}
}
if ((!isDuplicatedFile && _fileInfo.IsAmendment.Value == 0) ||
(_fileInfo.IsAmendment.Value == 1 && previousFileRefId > -1))
{
switch (_fileInfo.TableRef.ToUpper())
{
case "TABLE-1-UO-Q":
case "TABLE-1-UQ-Q":
var errorMassage = _msd1FileProcess.ProcessMsd1Records(filename, _fileInfo);
bodyErrorMsg.Append(errorMassage);
break;
case "TABLE-2-FT-A":
errorMassage = _msd2FileProcess.ProcessMsd2Records(filename, _fileInfo);
bodyErrorMsg.Append(errorMassage);
break;
case "TABLE-3-LA-A":
errorMassage = _msd3FileProcess.ProcessMsd3Records(filename, _fileInfo);
bodyErrorMsg.Append(errorMassage);
break;
}
}
else
{
var errmsg = (_fileInfo.IsAmendment.Value == 1 && previousFileRefId == -1) ?
string.Format("{0} is an amendment file but there is no file with the same submission reference to amend - please contact the helpdesk for advice.", _fileInfo.FileName.Substring(0, _fileInfo.FileName.IndexOf('.') + 4)) :
string.Format("{0} has the same sender reference as a file you have already sent, so it has not been processed." +
" If it is not a duplicate sent by mistake, please use a different sender reference in the header and resubmit the file.", _fileInfo.FileName.Substring(0, _fileInfo.FileName.IndexOf('.') + 4));
bodyErrorMsg.AppendFormat(errmsg);
}
if (!string.IsNullOrEmpty(bodyErrorMsg.ToString().Trim()))
{
AddErrorLogToDbAndNotifyUser(filename, bodyErrorMsg.ToString());
}
bodyErrorMsg.Clear();
}
headerErrors.Clear();
}
catch (Exception err)
{
_logger.LogError("Ascii File error:", err.Message, err.InnerException != null ? err.InnerException.Message : string.Empty, err.StackTrace);
if (_fileInfo != null)
{
bodyErrorMsg.AppendLine(filename + Environment.NewLine + err.Message);
AddErrorLogToDbAndNotifyUser(filename, bodyErrorMsg.ToString());
}
}
finally
{
ArchiveProcessedFilesFromBucket(filename, false);
bodyErrorMsg.Clear();
headerErrors.Clear();
_fileInfo = null;
}
}
public async Task <List <DexihFileProperties> > GetFiles(FlatFile flatFile, EFlatFilePath path)
{
return(await GetFileList(flatFile, path));
}
public ProcessedDetails Compare(FlatFile fetchedFlatFileObj, SettingsFile fetchedSettingsObj, SQLiteHelper sqlManipulation, System.Data.SQLite.SQLiteConnection connection, string DBName, string tableName1, string tableName2, string processedTable1, string processedTable2, string deletedTable, string insertedTable, int totalRecords, DataSet dsTotalRecords, Logger logger)
{
//string[] allLines = System.IO.File.ReadAllLines(fetchedFlatFileObj.FlatFilePath1);
if (fetchedSettingsObj.Percentage == 0)
{
fetchedSettingsObj.Percentage = 100;
}
decimal totalCountDec = Convert.ToDecimal((fetchedSettingsObj.Percentage / 100) * totalRecords);
logger.Info($"Total Count as per Percentage : {totalCountDec.ToString("#.00")}");
int totalCount = Convert.ToInt32(totalCountDec);
if ((totalCount * 1.0M) < totalCountDec)
{
totalCount++;
logger.Info($"Total Count as per Percentage after converting to Integer : {totalCount}");
}
ProcessedDetails processedDetails = new ProcessedDetails();
int TotalRecords = 0;
int TotalErrorRecords = 0;
int TotalSeccessfullyProcessedRecords = 0;
if (dsTotalRecords.Tables[0].Rows.Count > 0)
{
foreach (DataRow dr in dsTotalRecords.Tables[0].Rows)
{
int records = Convert.ToInt32(dr["TotalRecords"].ToString());
if (Convert.ToBoolean(dr[0].ToString()))
{
TotalErrorRecords = TotalErrorRecords + records;
}
else
{
TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords + records;
}
TotalRecords = records + TotalRecords;
}
if (TotalRecords >= totalRecords)
{
logger.Info("All records are processed, in case of any error please check.");
processedDetails.TotalRecords = TotalRecords;
processedDetails.TotalErrorRecords = TotalErrorRecords;
processedDetails.TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords;
return(processedDetails);
}
}
TotalRecords = totalRecords;
logger.Info($"Total count = {totalCount}, ({fetchedSettingsObj.Percentage} / 100 )* {totalRecords}");
// SQLiteConnection connection = sqlManipulation.OpenDBConnection(DBName);
DateTime startTime = DateTime.Now;
if (fetchedSettingsObj.Time == 0)
{
fetchedSettingsObj.Time = 24 * 60 * 365;
}
DateTime endTime = startTime.AddMinutes(fetchedSettingsObj.Time);
logger.Info($"Start Time : {startTime}");
logger.Info($"End Time : {endTime}");
Random rmd = new Random();
ParallelOptions ops = new ParallelOptions();
ops.MaxDegreeOfParallelism = 10;
object obj = new object();
for (int i = 0; i < GetFactors(totalCount); i++)
{
int startCount = (i * 100);
int lastCount = ((i + 1) * 100) > totalCount ? totalCount : ((i + 1) * 100);
//for (int i = 0; i < totalCount; i++)
List <int> bags = GetBags(totalRecords, lastCount - startCount, sqlManipulation, connection, processedTable1, logger);
logger.Info($"startCount : {startCount}, lastCount : {lastCount}");
Parallel.ForEach(bags, ops, (randomLineNumber, loopState) =>
{
try
{
bool isError = false;
// int randomLineNumber = -1;
//DataSet ds = new DataSet();
//lock (obj)
//{
// randomLineNumber = rmd.Next(totalRecords);
// //logger.Info($"Generated Line Number : {randomLineNumber + 1}");
// ds = sqlManipulation.RetrieveRecord(connection, processedTable1, randomLineNumber + 1, logger);
// int loopIteration = 1;
// bool allCompleted = false;
// while (ds.Tables[0].Rows.Count > 0)
// {
// randomLineNumber = randomLineNumber + 1;
// if (randomLineNumber > totalRecords - 1)
// {
// randomLineNumber = 0;
// }
// ds = sqlManipulation.RetrieveRecord(connection, processedTable1, randomLineNumber + 1, logger);
// //logger.Info($"Total Records with Flat File Line Number : {randomLineNumber + 1} and Total Returned Count : {ds.Tables[0].Rows.Count}");
// if (loopIteration > TotalRecords)
// {
// allCompleted = true;
// break;
// }
// loopIteration++;
// }
// if (allCompleted)
// loopState.Stop();
// //break;
// //loopState.Stop();
// logger.Info($"Random Line Number : {randomLineNumber + 1}");
//}
//logger.Info($"Random Line Number : {randomLineNumber + 1}");
string query1 = $"SELECT * FROM {tableName1} WHERE FlatFileRowNumber={randomLineNumber + 1};";
//sqlManipulation.GetTotalRecordsInTable(connection, query1, logger);
//Sqlite.TableMapping map = new Sqlite.TableMapping(Type.GetType(tableName1));
////string query = "select * from " + TableName;
//object[] objLists = new object[] { };
//List <object> list = connection.Query(map, query1, objLists).ToList();
DataSet ds1 = sqlManipulation.GetTotalRecordsInTable(connection, query1, logger);
//List<DynamicClass1> list = null;//connection.Query<DynamicClass1>(query1, fieldLists.Cast<object>().ToArray());
DataRow randomLineContentFromTable1 = null;
string fetchedNRIC = null;
if (ds1.Tables[0].Rows.Count > 0)
{
randomLineContentFromTable1 = ds1.Tables[0].Rows[0];
fetchedNRIC = randomLineContentFromTable1[(fetchedFlatFileObj.PrimaryKeyColumnNumber)].ToString();
}
//logger.Info($"fetchedNRIC : {fetchedNRIC}");
string query2 = $"SELECT * FROM {tableName2} WHERE NRICNumber='{fetchedNRIC}';";
DataSet ds2 = sqlManipulation.GetTotalRecordsInTable(connection, query2, logger);
//DataSet ds2 = sqlManipulation.RetrieveRecord(connection, query2, logger);
//logger.Info(query2);
if (ds2.Tables[0].Rows.Count <= 0)
{
string query3 = $"INSERT INTO {deletedTable} SELECT * FROM {tableName1} WHERE FlatFileRowNumber={randomLineNumber + 1};";
//logger.Info($"Query 3 : {query3}");
sqlManipulation.InsertRecord(connection, query3, logger);
}
else
{
//string query3 = $"INSERT INTO {insertedTable} SELECT * FROM {tableName1} WHERE FlatFileRowNumber={randomLineNumber + 1};";
//string hashKey1 = string.Empty;
DataRow randomLineContentFromTable2 = ds2.Tables[0].Rows[0];
//namicClass1 class1 = (list.FirstOrDefault() as DynamicClass1);
//DataRow randomLineContentFromTable2 = ds2.Tables[0].Rows[0];
string hashKey1 = randomLineContentFromTable1["HashingKey"].ToString();
string hashKey2 = randomLineContentFromTable2["HashingKey"].ToString();
//logger.Info($"HashKey : {hashKey1} : {hashKey2}");
if (hashKey1 != hashKey2)
{
bool isDifference = false;
foreach (Fields f1 in fetchedFlatFileObj.fields)
{
//DynamicClass2 class2 = (list2.FirstOrDefault() as DynamicClass2);
var propSource = randomLineContentFromTable1[f1.FieldName].ToString();
//DynamicClass2 class2 = (list2.FirstOrDefault() as DynamicClass2);
var propDest = randomLineContentFromTable2[f1.FieldName].ToString();
if (propSource != propDest)
{
isDifference = true;
}
}
if (isDifference)
{
string query4 = $"INSERT INTO {insertedTable} SELECT * FROM {tableName1} WHERE FlatFileRowNumber={randomLineNumber + 1};";
//connection.CreateTable<ICADeletedTable1>();
//connection.Insert(class1);
//logger.Info($"Query 4 : {query4}");
sqlManipulation.InsertRecord(connection, query4, logger);
}
}
else
{
//logger.Info($"Record is matching : {randomLineNumber + 1}");
}
}
//Check Validations
//if (randomLineContent.Length > fetchedFlatFileObj.RecordSize)
//{
// logger.Error($"Flat file line number {randomLineNumber + 1} exceeded specified Record Size.");
// isError = true;
//}
//if (randomLineContent.Length < fetchedFlatFileObj.RecordSize)
//{
// logger.Error($"Flat file line number {randomLineNumber + 1} less than specified Record Size.");
// isError = true;
//}
//if (!isError)
//{
// TotalSeccessfullyProcessedRecords++;
//}
//else
//{
// TotalErrorRecords++;
//}
//DataSet dsMain = sqlManipulation.RetrieveRecord(connection, processedTable1, randomLineNumber + 1, logger);
//if (dsMain.Tables[0].Rows.Count <= 0)
//{
// sqlManipulation.InsertRecord(connection, processedTable1, randomLineNumber + 1, isError, logger);
//}
if (endTime < DateTime.Now)
{
logger.Info($"As per settings ended the program at End Time {endTime}");
//break;
loopState.Stop();
}
// }
}
catch (Exception excp)
{
logger.Error($"Error in Comapring record : {randomLineNumber} " + excp.ToString() + " ---- " + excp.StackTrace);
}
});
}
processedDetails.TotalRecords = TotalRecords;
processedDetails.TotalErrorRecords = TotalErrorRecords;
processedDetails.TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords;
return(processedDetails);
}
public abstract Task <bool> CreateDirectory(FlatFile file, EFlatFilePath path);
public string ProcessMsd3Records(GesmesHelpers gesmesHelpers, string filename, FlatFile fileInfo)
{
_fileInfo = fileInfo;
string record = string.Empty;
var allLines = File.ReadAllLines(filename);
var lines = allLines.Where(c => c.Contains("ARR")).ToList();
for (int i = 0; i < lines.Count; i++)
{
string currentRecord = string.Empty;
try
{
if (!string.IsNullOrEmpty(lines[i]) && lines[i].Substring(0, 3) == "ARR")
{
string[] msd3Elements = gesmesHelpers.Split(":", lines[i].Replace("'", ""));
currentRecord = string.Join('|', msd3Elements, 0, 7);
string lastRecord = (i == 0) ? string.Empty : string.Join('|', gesmesHelpers.Split(":", lines[i - 1]), 0, 7);
string nextRecord = (i == (lines.Count - 1)) ? string.Empty : string.Join('|', gesmesHelpers.Split(":", lines[i + 1]), 0, 7);
if (i == 0 || !currentRecord.Equals(lastRecord))
{
_msd3.Id = _helperService.GetUniqueKey();
ValidateAndPopulateMsd3DataFromFlatFile(msd3Elements);
}
if ((string.IsNullOrEmpty(recordErrors.ToString().Trim())))
{
AddShipplingLineAndAgents(msd3Elements);
if (!(currentRecord.Equals(nextRecord)))
{
AddMsd3DataToDatabase();
_msd3.Msd3agents.Clear();
}
}
if (!(string.IsNullOrEmpty(recordErrors.ToString().Trim())))
{
bodyErrorMsg.AppendLine(" #Line from your file").AppendLine().AppendLine(lines[i])
.AppendLine(" #What is wrong and how to fix it").AppendLine().Append(recordErrors)
.AppendLine().Append(" ___").AppendLine();
recordErrors.Clear();
}
}
}
catch (Exception err)
{
_logger.LogError("Gesmes Record error:", err.Message, err.InnerException != null ? err.InnerException.Message : string.Empty, err.StackTrace);
if (_fileInfo != null)
{
recordErrors.AppendLine(err.Message).Append(" Please correct it or contact the helpdesk for advice.");
bodyErrorMsg.AppendLine(" #Line from your file").AppendLine().AppendLine(lines[i])
.AppendLine(" #What is wrong and how to fix it").AppendLine().Append(recordErrors)
.AppendLine().Append(" ___").AppendLine();
recordErrors.Clear();
}
}
}
return(bodyErrorMsg.ToString());
}
public abstract Task <bool> MoveFile(FlatFile file, EFlatFilePath fromPath, EFlatFilePath toPath, string fileName);
public ActionResult UploadOnlyFile(HttpPostedFileBase file)
{
try
{
if (file != null && file.ContentLength > 0)
{
bool saveToDB = ConfigurationManager.AppSettings["saveToDB"].Equals("1");
bool useRealFileName = ConfigurationManager.AppSettings["useRealFileName"].Equals("1");
bool useExtension = ConfigurationManager.AppSettings["useExtension"].Equals("1");
string _Path = ConfigurationManager.AppSettings["rootPath"];
string _fileExtenstion = useExtension ? Path.GetExtension(file.FileName) : "";
string _fileName = useRealFileName ? Path.GetFileName(file.FileName) : string.Format("{0}{1}", Guid.NewGuid().ToString(), _fileExtenstion);
if (saveToDB)
{
// Save this model to Database
var newFile = new FlatFile
{
FileName = _fileName,
ContentType = file.ContentType
};
using (var reader = new BinaryReader(file.InputStream))
{
// Save to DB
newFile.Content = reader.ReadBytes(file.ContentLength);
using (var context = new DBContext())
{
context.FlatFiles.Add(newFile);
context.SaveChanges();
}
// END Save to DB
}
}
else
{
using (var reader = new BinaryReader(file.InputStream))
{
// Save to path
string _savePath = Path.Combine(Server.MapPath(_Path), _fileName);
file.SaveAs(_savePath);
// END Save to path
}
}
TempData["prevUploadStatus"] = true;
return(RedirectToAction("UploadOnlyFile"));
}
else
{
// Do something if no file uploaded
return(RedirectToAction("UploadOnlyFile"));
}
}
catch (Exception ex)
{
TempData["prevUploadStatus"] = false;
return(RedirectToAction("UploadOnlyFile"));
}
}
public abstract Task <bool> DeleteFile(FlatFile file, EFlatFilePath path, string fileName);
protected override void DoRun(FlatFile flatFile)
{
_numberOfOcurrencesForCurrentFile = 0;
_currentFlatFile = flatFile;
}
public override async Task <Table> GetSourceTableInfo(Table originalTable, CancellationToken cancellationToken)
{
try
{
var flatFile = (FlatFile)originalTable;
if (flatFile.FileConfiguration == null || flatFile.FileSample == null)
{
throw new ConnectionException($"The properties have not been set to import the flat files structure. Required properties are (FileFormat)FileFormat and (Stream)FileSample.");
}
var stream = new MemoryStream();
var writer = new StreamWriter(stream);
writer.Write(flatFile.FileSample);
writer.Flush();
stream.Position = 0;
FileHandlerBase fileHandler = null;
switch (flatFile.FormatType)
{
case ETypeCode.Json:
fileHandler = new FileHandlerJson(flatFile, flatFile.RowPath);
break;
case ETypeCode.Text:
fileHandler = new FileHandlerText(flatFile, flatFile.FileConfiguration);
break;
case ETypeCode.Xml:
fileHandler = new FileHandlerXml(flatFile, flatFile.RowPath);
break;
default:
throw new ConnectionException($"The source type {flatFile.FormatType} is not currently supported.");
}
var columns = await fileHandler.GetSourceColumns(stream);
//The new datatable that will contain the table schema
var newFlatFile = new FlatFile();
flatFile.Name = originalTable.Name;
newFlatFile.Columns.Clear();
newFlatFile.LogicalName = newFlatFile.Name;
newFlatFile.Description = "";
newFlatFile.FileConfiguration = flatFile.FileConfiguration;
foreach (var column in columns)
{
newFlatFile.Columns.Add(column);
}
var col = new TableColumn()
{
//add the basic properties
Name = "FileName",
LogicalName = "FileName",
IsInput = false,
DataType = ETypeCode.String,
DeltaType = TableColumn.EDeltaType.FileName,
Description = "The name of the file the record was loaded from.",
AllowDbNull = false,
IsUnique = false
};
newFlatFile.Columns.Add(col);
return(newFlatFile);
}
catch (Exception ex)
{
throw new ConnectionException($"Failed to import the file structure. {ex.Message}", ex);
}
}
public DataDictionary(string aDataLine, FlatFile aFlatFile)
{
string[] theDatas = aDataLine.Split(',');
this.FillData(theDatas, aFlatFile);
}
public abstract Task <DexihFiles> GetFileEnumerator(FlatFile file, EFlatFilePath path, string searchPattern);
protected override void DoRun(FlatFile flatFile)
{
_numberRecords = 0;
_currentFlatFile = flatFile;
}
public abstract Task <List <DexihFileProperties> > GetFileList(FlatFile file, EFlatFilePath path);
public void Run(FlatFile flatFile)
{
_stopwatch.Start();
DoRun(flatFile);
_stopwatch.Stop();
}
public abstract Task <Stream> GetWriteFileStream(FlatFile file, EFlatFilePath path, string fileName);
public FlatFile IsInFlatFile(string nip, string bankAccount)
{
if (!(flatFileData?.skrotypodatnikowczynnych?.Count >= 0))
{
throw new System.Exception("Json file is not loaded. Use first LoadFlatFileAsync()");
}
if (!nip.IsValidNIP())
{
return(FlatFile.InvalidNip);
}
if (string.IsNullOrEmpty(bankAccount) || string.IsNullOrWhiteSpace(bankAccount) || bankAccount?.Length < 26)
{
return(FlatFile.InvalidBankAccount);
}
switch (CheckInBody(bankAccount))
{
case FlatFile.FoundInActiveVatPayer:
return(FlatFile.FoundInActiveVatPayer);
case FlatFile.FoundInExemptVatPayer:
return(FlatFile.FoundInExemptVatPayer);
case FlatFile.InvalidNip:
return(FlatFile.InvalidNip);
case FlatFile.InvalidBankAccount:
return(FlatFile.InvalidBankAccount);
case FlatFile.NotFound:
break;
default:
break;
}
string bankBranchNumber = bankAccount.Substring(2, 8);
string maskToCompare = string.Empty;
foreach (var item in flatFileData.maski)
{
if (bankBranchNumber.Equals(item.Substring(2, 8), StringComparison.OrdinalIgnoreCase))
{
int IndexFrom = item.IndexOf("Y");
int range = item.Count(p => p.Equals('Y'));
string VirtualAccount = Regex.Replace(item, "Y\\w*Y", bankAccount.Substring(IndexFrom, range));
FlatFile checkResult = CheckInBody(VirtualAccount);
if (checkResult == FlatFile.NotFound)
{
continue;
}
else
{
return(checkResult);
}
}
}
return(FlatFile.NotFound);
FlatFile CheckInBody(string account)
{
string hash = (flatFileData.naglowek.datagenerowaniadanych + nip + account).SHA512(Convert.ToInt32(flatFileData.naglowek.liczbatransformacji));
foreach (var item in flatFileData.skrotypodatnikowczynnych)
{
if (item.Equals(hash, StringComparison.OrdinalIgnoreCase))
{
return(FlatFile.FoundInActiveVatPayer);
}
}
foreach (var item in flatFileData.skrotypodatnikowzwolnionych)
{
if (item.Equals(hash, StringComparison.OrdinalIgnoreCase))
{
return(FlatFile.FoundInExemptVatPayer);
}
}
return(FlatFile.NotFound);
}
}
public abstract Task <bool> SaveFileStream(FlatFile file, EFlatFilePath path, string fileName, Stream fileStream);
public abstract string GetFullPath(FlatFile file, EFlatFilePath path);
public Table GetTable(DexihHub hub, Connection connection, InputColumn[] inputColumns, TransformSettings transformSettings, string referenceTableAlias = null)
{
Table table;
if (connection == null)
{
table = new Table();
}
else
{
EConnectionCategory category;
if (connection is ConnectionFlatFile)
{
category = EConnectionCategory.File;
}
else
{
var connectionReference = Connections.GetConnection(connection.GetType());
category = connectionReference.ConnectionCategory;
}
switch (category)
{
case EConnectionCategory.File:
table = new FlatFile();
if (FileFormatKey == null)
{
((FlatFile)table).FileConfiguration = new FileConfiguration();
}
else
{
var fileFormat =
hub.DexihFileFormats.SingleOrDefault(f => f.IsValid && f.Key == FileFormatKey);
if (fileFormat == null)
{
throw new RepositoryException(
$"The file format for the table {Name} with key {FileFormatKey} could not be found.");
}
((FlatFile)table).FileConfiguration = fileFormat?.GetFileFormat();
}
break;
case EConnectionCategory.WebService:
table = new WebService();
break;
default:
table = new Table();
break;
}
}
// create a temporary copy and exclude the fileFormat (which is different between DexihTable & Table)
this.CopyProperties(table, false);
switch (table)
{
case FlatFile flatFile:
if (transformSettings.HasVariables())
{
flatFile.FileIncomingPath = transformSettings.InsertHubVariables(flatFile.FileIncomingPath);
flatFile.FileMatchPattern = transformSettings.InsertHubVariables(flatFile.FileMatchPattern);
flatFile.FileOutgoingPath = transformSettings.InsertHubVariables(flatFile.FileOutgoingPath);
flatFile.FileProcessedPath = transformSettings.InsertHubVariables(flatFile.FileProcessedPath);
flatFile.FileRejectedPath = transformSettings.InsertHubVariables(flatFile.FileRejectedPath);
flatFile.FileRootPath = transformSettings.InsertHubVariables(flatFile.FileRootPath);
}
break;
case WebService restFunction:
if (transformSettings.HasVariables())
{
restFunction.RestfulUri = transformSettings.InsertHubVariables(restFunction.RestfulUri);
restFunction.RowPath = transformSettings.InsertHubVariables(restFunction.RowPath);
}
break;
}
foreach (var dbColumn in DexihTableColumns.Where(c => c.IsValid && c.DeltaType != EDeltaType.IgnoreField).OrderBy(c => c.Position))
{
table.Columns.Add(dbColumn.GetTableColumn(inputColumns, referenceTableAlias));
}
foreach (var dbIndex in DexihTableIndexes.Where(c => c.IsValid))
{
table.Indexes.Add(new TableIndex()
{
Name = dbIndex.Name,
Columns = dbIndex.Columns.Select(ti =>
{
var column = DexihTableColumns.SingleOrDefault(c => c.Key == ti.ColumnKey);
if (column != null)
{
return(new TableIndexColumn(column.Name, ti.Direction));
}
return(null);
}).Where(c => c != null).ToList()
});
}
return(table);
}
protected override void DoRun(FlatFile flatFile)
{
}
public void InsertInto(SQLiteHelper sqlManipulation, FlatFile fetchedFlatFileObj, SettingsFile fetchedSettingsObj, System.Data.SQLite.SQLiteConnection connection, string dBName, string tableName, string processedTableName, string filePath, Logger logger, dynamic objList, bool isFirst)
{
string[] allLines = System.IO.File.ReadAllLines(filePath);
/*if (fetchedSettingsObj.Percentage == 0)
* {
* fetchedSettingsObj.Percentage = 100;
* }
* decimal totalCountDec = Convert.ToDecimal((fetchedSettingsObj.Percentage / 100) * allLines.Length);
* logger.Info($"Total Count as per Percentage : {totalCountDec.ToString("#.00")}");
* int totalCount = Convert.ToInt32(totalCountDec);
* if ((totalCount * 1.0M) < totalCountDec)
* {
*
* totalCount++;
* logger.Info($"Total Count as per Percentage after converting to Integer : {totalCount}");
* }
*
*
*
* ProcessedDetails processedDetails = new ProcessedDetails();
* int TotalRecords = 0;
* int TotalErrorRecords = 0;
* int TotalSeccessfullyProcessedRecords = 0;
* DataSet dsTotalRecords = sqlManipulation.GetTotalRecords(connection, processedTableName, logger);
* if (dsTotalRecords.Tables[0].Rows.Count > 0)
* {
* foreach (DataRow dr in dsTotalRecords.Tables[0].Rows)
* {
* int records = Convert.ToInt32(dr[1].ToString());
*
* if (Convert.ToBoolean(dr[0].ToString()))
* {
* TotalErrorRecords = TotalErrorRecords + records;
* }
* else
* {
* TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords + records;
* }
* TotalRecords = records + TotalRecords;
*
* }
*
* if (TotalRecords >= allLines.Length)
* {
* logger.Info("All records are processed, in case of any error please check.");
*
* processedDetails.TotalRecords = TotalRecords;
* processedDetails.TotalErrorRecords = TotalErrorRecords;
* processedDetails.TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords;
* return processedDetails;
* }
* }
*
* TotalRecords = allLines.Length;
* logger.Info($"Total count = {totalCount}, ({fetchedSettingsObj.Percentage} / 100 )* {allLines.Length}");
*
*
* // SQLiteConnection connection = sqlManipulation.OpenDBConnection(DBName);
*
* DateTime startTime = DateTime.Now;
* if (fetchedSettingsObj.Time == 0)
* {
* fetchedSettingsObj.Time = 24 * 60 * 365;
* }
*
*
* DateTime endTime = startTime.AddMinutes(fetchedSettingsObj.Time);
* logger.Info($"Start Time : {startTime}");
* logger.Info($"End Time : {endTime}");
*
* Random rmd = new Random();*/
ParallelOptions ops = new ParallelOptions();
ops.MaxDegreeOfParallelism = 1;
//for (int i = 0; i < totalCount; i++)
ThreadLocal <KeyedHashAlgorithm> sha1 = new ThreadLocal <KeyedHashAlgorithm>(() =>
{
var kha = KeyedHashAlgorithm.Create("HMACSHA256");
kha.Key = Encoding.UTF8.GetBytes("N3PS");
return(kha);
});
logger.Info($"allLines.Length : {allLines.Length}");
var fieldLists = new List <Field>();
foreach (Fields f in fetchedFlatFileObj.fields)
{
Field f1 = new Field(f.FieldName, typeof(string));
fieldLists.Add(f1);
}
fieldLists.Add(new Field("HashingKey", typeof(string)));
//dynamic obj = new DynamicClass1(fieldLists);
//List<DynamicClass1> objList = new List<DynamicClass1>();
for (int i = 0; i < GetFactors(allLines.Length); i++)
{
int lastCount = ((i + 1) * 100) > allLines.Length ? allLines.Length: ((i + 1) * 100);
StringBuilder query = new StringBuilder();
query.Append($"INSERT INTO {tableName} Values");
Parallel.For((i * 100), lastCount, ops, (rand, loopState) =>
{
try
{
//bool isError = false;
int randomLineNumber = -1;
DataSet ds = new DataSet();
randomLineNumber = rand; // rmd.Next(allLines.Length - 1);
//logger.Info($"Generated Line Number : {randomLineNumber + 1}");
//ds = sqlManipulation.RetrieveRecord(connection, processedTableName, randomLineNumber + 1, logger);
//int loopIteration = 1;
//bool allCompleted = false;
//while (ds.Tables[0].Rows.Count > 0)
//{
// randomLineNumber = randomLineNumber + 1;
// if (randomLineNumber > allLines.Length - 1)
// {
// randomLineNumber = 0;
// }
// ds = sqlManipulation.RetrieveRecord(connection, processedTableName, randomLineNumber + 1, logger);
// //logger.Info($"Total Records with Flat File Line Number : {randomLineNumber + 1} and Total Returned Count : {ds.Tables[0].Rows.Count}");
// if (loopIteration > TotalRecords)
// {
// allCompleted = true;
// break;
// }
// loopIteration++;
//}
//if (allCompleted)
// break;
//loopState.Stop();
//logger.Info($"Random Line Number : {randomLineNumber + 1}");
string randomLineContent = allLines[randomLineNumber];
//Check Validations
if (randomLineContent.Length > fetchedFlatFileObj.RecordSize)
{
logger.Error($"Flat file line number {randomLineNumber + 1} exceeded specified Record Size.");
//isError = true;
}
if (randomLineContent.Length < fetchedFlatFileObj.RecordSize)
{
logger.Error($"Flat file line number {randomLineNumber + 1} less than specified Record Size.");
//isError = true;
}
//StringBuilder cols = new StringBuilder();
//StringBuilder values = new StringBuilder();
//StringBuilder combinedValues = new StringBuilder();
////cols.AppendLine("(");
//cols.Append("FlatFileRowNumber,");
////values.AppendLine("(");
//values.Append($"{randomLineNumber+1},");
dynamic obj = null;
if (isFirst)
{
obj = new DynamicClass1(fieldLists);
}
else
{
obj = new DynamicClass2(fieldLists);
}
query.Append($"({(randomLineNumber+1)},");
StringBuilder vals = new StringBuilder();
vals.Append((randomLineNumber + 1));
for (int ik = 0; ik < fetchedFlatFileObj.fields.Count; ik++)
//foreach (KeyValuePair<Type, object> kvp in ((IDictionary<Type, object>)obj))
{
string content = randomLineContent.Substring(fetchedFlatFileObj.fields[ik].StartPosition - 1, fetchedFlatFileObj.fields[ik].Length).Trim();
//cols.Append(fields.FieldName + ",");
//values.Append($"'{content.Replace("'","''")}',");
//combinedValues.Append(content);
//PropertyInfo info = obj.GetType().GetProperty(fields.FieldName);
//KeyValuePair<Type, object> p = obj._fields[fetchedFlatFileObj.fields[ik].FieldName];
query.Append($"'{content.Replace("'","''")}',");
vals.Append($"{content}");
//obj._fields[fetchedFlatFileObj.fields[ik].FieldName] = content;
// p[ = content;
//obj.TrySetMember(fields.FieldName, content);
//info.SetValue(obj, content);
//obj[fields.ColumnNumber] = content;
}
var buffer = sha1.Value.ComputeHash(Encoding.Unicode.GetBytes(vals.ToString()));
//query[query.ToString().Length - 1] = ' ';
if (Encoding.Unicode.GetString(buffer, 0, buffer.Length).Contains("'"))
{
}
query.Append($"'{Encoding.Unicode.GetString(buffer, 0, buffer.Length).Replace("'", "''").Replace("\0","+")}'");
query.Append($"),");
query.AppendLine(Environment.NewLine);
//objList.Add(obj);
//connection.Insert(obj);
//cols[cols.ToString().Length - 1] = ' ';
//values[values.ToString().Length - 1] = ' ';
//tl.Value.ComputeHash(Encoding.UTF8.GetBytes("message"));
//string insertQuery = $"INSERT INTO {tableName} ({cols.ToString()},HashingKey) VALUES ({values.ToString()},'{Encoding.Unicode.GetString(buffer, 0, buffer.Length).Replace("'","+")}')";
//logger.Error($"SQL Query : {insertQuery}");
//if(!sqlManipulation.InsertRecord(connection, insertQuery, logger))
//{
// logger.Error($"Error in iserting record : {randomLineNumber} -- {insertQuery}");
//}
//if (!isError)
//{
// TotalSeccessfullyProcessedRecords++;
//}
//else
//{
// TotalErrorRecords++;
//}
//DataSet ds1 = sqlManipulation.RetrieveRecord(connection, processedTableName, randomLineNumber + 1, logger);
//if (ds1.Tables[0].Rows.Count <= 0)
//{
// sqlManipulation.InsertRecord(connection, processedTableName, randomLineNumber + 1, isError, logger);
//}
//if (endTime < DateTime.Now)
//{
// logger.Info($"As per settings ended the program at End Time {endTime}");
// break;
// //loopState.Stop();
//}
}
catch (Exception excp)
{
logger.Error($"Error in inserting record : {i} : " + excp.ToString() + " ---- " + excp.StackTrace);
}
});
query[query.ToString().Length - 5] = ' ';
//SQLiteConnection connection = new SQLiteConnection(new SQLite.Net.Platform.Win32.SQLitePlatformWin32(), $"{DBName}.sqlite");
// connection = sqlManipulation.OpenDBConnection(dBName);
//connection.CreateTable<DynamicClass1>(CreateFlags.None);
sqlManipulation.InsertRecord(connection, query.ToString(), logger);
Thread.Sleep(100);
//connection.InsertAll(objList);
//processedDetails.TotalRecords = TotalRecords;
//processedDetails.TotalErrorRecords = TotalErrorRecords;
//processedDetails.TotalSeccessfullyProcessedRecords = TotalSeccessfullyProcessedRecords;
//return processedDetails;
}
}
