private void bwForLoadCsvToDb_DoWork(object sender, DoWorkEventArgs e)
{
WriteLog("Begin");
try
{
var pluginCodeText = (string)((Dictionary <string, object>)e.Argument)["plugin_code"];
//PreprocessPluginCodeCombo.Text;
//var pluginCodeText = (string)PreprocessPluginCodeCombo.SelectedItem;
plugin = PluginInterface.buildPluginByCode(pluginCodeText, this);
WriteLog("Loaded Plugin: " + plugin);
Dictionary <string, string> fieldFilterConfigDict = new Dictionary <string, string>();
/*
* foreach (var item in fieldFilterConfigList)
* {
* fieldFilterConfigDict.Add(item.ColumnName, item.FilterCode);
* WriteLog("FILTER ITEM: " + item.ColumnName + " => " + item.FilterCode, "DEBUG");
* }
*/
for (int i = 0; i < ColumnFilterDataGridView.Rows.Count; i++)
{
if (ColumnFilterDataGridView.Rows[i].Cells[0].Value == null || ColumnFilterDataGridView.Rows[i].Cells[1].Value == null)
{
continue;
}
WriteLog("CHECK FILTER ITEM: " + i, "DEBUG");
fieldFilterConfigDict.Add((string)ColumnFilterDataGridView.Rows[i].Cells[0].Value, (string)ColumnFilterDataGridView.Rows[i].Cells[1].Value);
}
foreach (var item in fieldFilterConfigDict)
{
WriteLog("FILTER ITEM: " + item.Key + " => " + item.Value, "DEBUG");
}
string[] files = CSVParser.ReadFilesInDir(SourceDirTextBox.Text, ".csv");
if (files.Length <= 0)
{
WriteLog("No CSV File Found", "WARNING");
return;
}
var parser = new CSVParser(
Convert.ToInt32(CPSColumnsLimitationNumber.Value),
CPSHasHeaderCheckerBox.Checked,
Convert.ToInt32(CPSIgnoreHeaderRowsNumber.Value),
Convert.ToInt32(CPSIgnoreTailRowsNumber.Value),
CPSIgnoreQuotesCheckerBox.Checked,
fieldFilterConfigDict,
plugin.ExtraFields()
);
CurrentTableName = "excel";// "table_" + (new Random()).Next();
TheSqliteAgent = new SqliteAgent(CurrentTableName);
Regex regexForRemoveNumbers = new Regex("[0-9]+");
Regex regexForRemoveTailSigns = new Regex("[{}\\[\\]\\- ［]+$");
bool isFirstFile = true;
for (int i = 0; i < files.Length; i++)
{
var file = files[i];
WriteLog("Opening file [" + (i + 1) + "/" + files.Length + "]: " + file);
parser.OpenFileAndLoadFields(file);
List <string> headers = parser.GetCurrentHeaders();
List <string> fieldNames = parser.GetAutoFieldNames();
/*
* Column Title -> Field_X
*/
headerFieldNameMap = parser.GetHeaderFieldNameDictionary();
/*
* for (int j = 0; j < fieldNames.Count; j++)
* {
* WriteLog("PARSED HEADER: " + headers[j] + " => " + fieldNames[j],"DEBUG");
* }
*/
if (isFirstFile)
{
// create SQLite Table
if (headerFieldNameMap.Count > 0)
{
TheSqliteAgent.EnsureTableWithFields(new List <string>(headerFieldNameMap.Values));
}
else
{
TheSqliteAgent.EnsureTableWithFields(fieldNames);
}
// then
isFirstFile = false;
}
Dictionary <string, string> rowDict;
while ((rowDict = parser.SafeReadOneRow()) != null)
{
Dictionary <string, string> dataRow = new Dictionary <string, string>();
foreach (var entry in rowDict)
{
//WriteLog("Entry: " + entry.Key + " => " + entry.Value,"DEBUG");
if (!headerFieldNameMap.ContainsKey(entry.Key))
{
continue;
}
String process_value = String.Copy(entry.Value);
if (fieldFilterConfigDict.Count > 0)
{
string filterCode = "ORIGINAL";
if (fieldFilterConfigDict.ContainsKey(entry.Key))
{
filterCode = fieldFilterConfigDict[entry.Key];
}
switch (filterCode)
{
case "REMOVE_NUMBERS":
process_value = regexForRemoveNumbers.Replace(process_value, "");
process_value = regexForRemoveTailSigns.Replace(process_value, "");
break;
case "ORIGINAL":
default:
// do nothing
break;
}
}
dataRow.Add(headerFieldNameMap[entry.Key], process_value);
}
// todo: preprocess
if (plugin != null)
{
plugin.PreprocessRawRow(dataRow, headerFieldNameMap);
}
TheSqliteAgent.InsertRow(dataRow);
}
parser.CloseReader();
// finally
bwForLoadCsvToDb.ReportProgress(100 * (i + 1) / files.Length);
}
WriteLog("Done");
}catch (Exception exception)
{
WriteLog("Exception: " + exception.Message, "ERROR");
WriteLog("Stack: " + exception.StackTrace, "ERROR");
}
}
