public static string ProcessImportFile(string importFolder, string archiveFolder, string errorFolder,
string importCSVFileSearchPattern, string importFileSearchToDBTableNamePattern,
string cleanupTablesStoredProcName, string endProcessingStoredProcName,
SpringBaseDao baseDao, string tempFolderPath,
ICompressionHelper compressionHelper, out bool success, out bool foundFile)
{
StringBuilder statusString = new StringBuilder();
string tempOutputDir = null;
success = false;
foundFile = false;
string importZipFile = null;
try
{
ValidateFolder(importFolder, false);
ValidateFolder(archiveFolder, true);
ValidateFolder(errorFolder, true);
importZipFile = FileUtils.GetOldestFile(importFolder, "*.zip", false);
if (string.IsNullOrEmpty(importZipFile))
{
OutputStatus(statusString, "Did not find any files to process within the import folder \"{0}\", stopping thread execution ...",
importFolder);
success = true;
return(statusString.ToString());
}
foundFile = true;
OutputStatus(statusString, "Extracting import files from the zip file \"{0}\"", importZipFile);
IList <string> importCSVFiles = GetImportFiles(importZipFile, importCSVFileSearchPattern, tempFolderPath,
compressionHelper);
OutputStatus(statusString, "Extracted {0} import files", importCSVFiles.Count.ToString());
tempOutputDir = Path.GetDirectoryName(importCSVFiles[0]);
CallStoredProc(cleanupTablesStoredProcName, baseDao, statusString);
baseDao.TransactionTemplate.Execute(delegate
{
foreach (string importFilePath in importCSVFiles)
{
string fileName = Path.GetFileName(importFilePath);
OutputStatus(statusString, "Processing import file \"{0}\"", fileName);
string dbTableName = GetDBTableName(importFilePath, importCSVFileSearchPattern, importFileSearchToDBTableNamePattern);
OutputStatus(statusString, "Determined DB table name of \"{0}\" for input file \"{1}\"",
dbTableName, fileName);
using (TextFieldParser textFieldParser = new TextFieldParser(importFilePath))
{
textFieldParser.TextFieldType = FieldType.Delimited;
textFieldParser.Delimiters = new string[] { COMMA };
string[] columnNames = textFieldParser.ReadFields();
if (CollectionUtils.IsNullOrEmpty(columnNames))
{
throw new FileNotFoundException(string.Format("No column names were found in the CSV file \"{0}\"",
fileName));
}
string semicolonSeparatedColumnNames = string.Join(SEMICOLON, columnNames);
OutputStatus(statusString, "Determined {0} insert column names as \"{1}\" for DB table \"{2}\"",
columnNames.Length.ToString(), semicolonSeparatedColumnNames,
dbTableName);
object[] insertValues = new object[columnNames.Length];
int totalRows = 0;
baseDao.DoBulkInsert(dbTableName, semicolonSeparatedColumnNames,
delegate(int currentInsertIndex)
{
if (textFieldParser.EndOfData)
{
return(null);
}
string[] columnValues;
try
{
columnValues = textFieldParser.ReadFields();
if (columnValues.Length != insertValues.Length)
{
throw new InvalidDataException(string.Format("The CSV file \"{0}\" does not contain a valid number of column values at row {1}: {2} values expected and {3} values found",
fileName, (currentInsertIndex + 1).ToString(),
insertValues.Length.ToString(), columnValues.Length.ToString()));
}
}
catch (MalformedLineException)
{
// This may be caused by single quotes inside the value instead of double quotes.
// Try to fix up here:
columnValues = textFieldParser.ErrorLine.Split(',');
if (CollectionUtils.IsNullOrEmpty(columnValues) ||
(columnValues.Length != insertValues.Length))
{
throw;
}
for (int j = 0; j < columnValues.Length; ++j)
{
string curValue = columnValues[j];
if ((curValue.Length > 1) && (curValue[0] == '\"') &&
(curValue[curValue.Length - 1] == '\"'))
{
columnValues[j] = curValue.Substring(1, curValue.Length - 2);
}
}
}
for (int i = 0; i < insertValues.Length; ++i)
{
insertValues[i] = columnValues[i];
}
totalRows++;
return(insertValues);
});
OutputStatus(statusString, "Inserted {0} rows into DB table \"{1}\"",
totalRows.ToString(), dbTableName);
}
}
return(null);
});
CallStoredProc(endProcessingStoredProcName, baseDao, statusString);
// Finally, move the import zip file to the archive folder
if (!string.IsNullOrEmpty(archiveFolder))
{
string newFilePath = FileUtils.MoveFileToFolderAppendDateTimeTicksToName(importZipFile, archiveFolder);
OutputStatus(statusString, "Moved import file \"{0}\" to \"{1}\"",
importZipFile, newFilePath);
}
success = true;
OutputStatus(statusString, "Successfully finished processing {0} import files",
importCSVFiles.Count);
}
catch (Exception ex)
{
OutputStatus(statusString, "ERROR ********************* An error occurred processing the import files: {0}",
ExceptionUtils.ToLongString(ex));
}
finally
{
if (tempOutputDir != null)
{
FileUtils.SafeDeleteDirectory(tempOutputDir);
}
// Finally, move the import zip file to the archive folder
if (!success && (importZipFile != null) && !string.IsNullOrEmpty(errorFolder))
{
try
{
string newFilePath = FileUtils.MoveFileToFolderAppendDateTimeTicksToName(importZipFile, errorFolder);
OutputStatus(statusString, "Moved import file \"{0}\" to \"{1}\"",
importZipFile, newFilePath);
}
catch (Exception ex3)
{
OutputStatus(statusString, "Failed to move import file \"{0}\" to folder \"{1}\": {2}",
importZipFile, errorFolder, ExceptionUtils.ToLongString(ex3));
}
}
}
return(statusString.ToString());
}
private void DoBulkInsert(IDbCommand dbCommand, string tableName, string semicolonSeparatedColumnNames,
Windsor.Commons.Spring.SpringBaseDao.GetInsertValuesDelegate del)
{
_dao.DoBulkInsert(tableName, semicolonSeparatedColumnNames, del);
}
