public static string ProcessImportFile(string importFolder, string archiveFolder, string errorFolder, string importCSVFileSearchPattern, string importFileSearchToDBTableNamePattern, string cleanupTablesStoredProcName, string endProcessingStoredProcName, SpringBaseDao baseDao, string tempFolderPath, ICompressionHelper compressionHelper, out bool success, out bool foundFile) { StringBuilder statusString = new StringBuilder(); string tempOutputDir = null; success = false; foundFile = false; string importZipFile = null; try { ValidateFolder(importFolder, false); ValidateFolder(archiveFolder, true); ValidateFolder(errorFolder, true); importZipFile = FileUtils.GetOldestFile(importFolder, "*.zip", false); if (string.IsNullOrEmpty(importZipFile)) { OutputStatus(statusString, "Did not find any files to process within the import folder \"{0}\", stopping thread execution ...", importFolder); success = true; return(statusString.ToString()); } foundFile = true; OutputStatus(statusString, "Extracting import files from the zip file \"{0}\"", importZipFile); IList <string> importCSVFiles = GetImportFiles(importZipFile, importCSVFileSearchPattern, tempFolderPath, compressionHelper); OutputStatus(statusString, "Extracted {0} import files", importCSVFiles.Count.ToString()); tempOutputDir = Path.GetDirectoryName(importCSVFiles[0]); CallStoredProc(cleanupTablesStoredProcName, baseDao, statusString); baseDao.TransactionTemplate.Execute(delegate { foreach (string importFilePath in importCSVFiles) { string fileName = Path.GetFileName(importFilePath); OutputStatus(statusString, "Processing import file \"{0}\"", fileName); string dbTableName = GetDBTableName(importFilePath, importCSVFileSearchPattern, importFileSearchToDBTableNamePattern); OutputStatus(statusString, "Determined DB table name of \"{0}\" for input file \"{1}\"", dbTableName, fileName); using (TextFieldParser textFieldParser = new TextFieldParser(importFilePath)) { textFieldParser.TextFieldType = FieldType.Delimited; textFieldParser.Delimiters = new string[] { COMMA }; string[] columnNames = textFieldParser.ReadFields(); if (CollectionUtils.IsNullOrEmpty(columnNames)) { throw new FileNotFoundException(string.Format("No column names were found in the CSV file \"{0}\"", fileName)); } string semicolonSeparatedColumnNames = string.Join(SEMICOLON, columnNames); OutputStatus(statusString, "Determined {0} insert column names as \"{1}\" for DB table \"{2}\"", columnNames.Length.ToString(), semicolonSeparatedColumnNames, dbTableName); object[] insertValues = new object[columnNames.Length]; int totalRows = 0; baseDao.DoBulkInsert(dbTableName, semicolonSeparatedColumnNames, delegate(int currentInsertIndex) { if (textFieldParser.EndOfData) { return(null); } string[] columnValues; try { columnValues = textFieldParser.ReadFields(); if (columnValues.Length != insertValues.Length) { throw new InvalidDataException(string.Format("The CSV file \"{0}\" does not contain a valid number of column values at row {1}: {2} values expected and {3} values found", fileName, (currentInsertIndex + 1).ToString(), insertValues.Length.ToString(), columnValues.Length.ToString())); } } catch (MalformedLineException) { // This may be caused by single quotes inside the value instead of double quotes. // Try to fix up here: columnValues = textFieldParser.ErrorLine.Split(','); if (CollectionUtils.IsNullOrEmpty(columnValues) || (columnValues.Length != insertValues.Length)) { throw; } for (int j = 0; j < columnValues.Length; ++j) { string curValue = columnValues[j]; if ((curValue.Length > 1) && (curValue[0] == '\"') && (curValue[curValue.Length - 1] == '\"')) { columnValues[j] = curValue.Substring(1, curValue.Length - 2); } } } for (int i = 0; i < insertValues.Length; ++i) { insertValues[i] = columnValues[i]; } totalRows++; return(insertValues); }); OutputStatus(statusString, "Inserted {0} rows into DB table \"{1}\"", totalRows.ToString(), dbTableName); } } return(null); }); CallStoredProc(endProcessingStoredProcName, baseDao, statusString); // Finally, move the import zip file to the archive folder if (!string.IsNullOrEmpty(archiveFolder)) { string newFilePath = FileUtils.MoveFileToFolderAppendDateTimeTicksToName(importZipFile, archiveFolder); OutputStatus(statusString, "Moved import file \"{0}\" to \"{1}\"", importZipFile, newFilePath); } success = true; OutputStatus(statusString, "Successfully finished processing {0} import files", importCSVFiles.Count); } catch (Exception ex) { OutputStatus(statusString, "ERROR ********************* An error occurred processing the import files: {0}", ExceptionUtils.ToLongString(ex)); } finally { if (tempOutputDir != null) { FileUtils.SafeDeleteDirectory(tempOutputDir); } // Finally, move the import zip file to the archive folder if (!success && (importZipFile != null) && !string.IsNullOrEmpty(errorFolder)) { try { string newFilePath = FileUtils.MoveFileToFolderAppendDateTimeTicksToName(importZipFile, errorFolder); OutputStatus(statusString, "Moved import file \"{0}\" to \"{1}\"", importZipFile, newFilePath); } catch (Exception ex3) { OutputStatus(statusString, "Failed to move import file \"{0}\" to folder \"{1}\": {2}", importZipFile, errorFolder, ExceptionUtils.ToLongString(ex3)); } } } return(statusString.ToString()); }
private void DoBulkInsert(IDbCommand dbCommand, string tableName, string semicolonSeparatedColumnNames, Windsor.Commons.Spring.SpringBaseDao.GetInsertValuesDelegate del) { _dao.DoBulkInsert(tableName, semicolonSeparatedColumnNames, del); }