public void ExtractFileToDataBaseWtihoutId(DCEngineCommonTxtParam dcEngineCommonTxtParam, string businessDate, string extractFilePath, string tableName, ref string result) { try { DataTable dt = TableGetSchema(tableName); FileImportToDataBaseWithoutId(dcEngineCommonTxtParam, businessDate, extractFilePath, ref dt, tableName); } catch (Exception ex) { result = ex.ToString(); } }
public override string FileExtract(string extractFilePath, FileInfoType fileInfoType, string businessDate, string tableName, string fileImportFormatDirectory) { string result = string.Empty; DCEngineCommonTxtParam dcEngineCommonTxtParam = new DCEngineCommonTxtParam(); dcEngineCommonTxtParam.GetContentPattern = @"(?<key>[^\n]+(?=\n?))"; dcEngineCommonTxtParam.FileEncoding = "gb2312"; dcEngineCommonTxtParam.ContentSplit = "~"; dcEngineCommonTxtParam.IsAddEnter = false; dcEngineCommonTxtParam.RowExcept = string.Empty; dcEngineCommonTxtParam.ByteSplit = string.Empty; dcEngineCommonTxtParam.ColumnSplitType = (byte)ColumnSplitType.按分隔符; dcEngineCommonTxtParam.StringSplitOptions = StringSplitOptions.None; dcEngineCommonTxtParam.FileInfoType = fileInfoType; DCCommonTxtProcess dcCommonTxtProcess = new DCCommonTxtProcess(); dcCommonTxtProcess.ExtractFileToDataBase(dcEngineCommonTxtParam, businessDate, extractFilePath, tableName, ref result); return result; }
private void FileContentToDataTable(string fileContent, DCEngineCommonTxtParam dcEngineCommonTxtParam, string businessDate, string extractFilePath, string[] columnSplitContents, int fixedColumnCount, ref DataTable dtFileContent) { dtFileContent.Clear(); MatchCollection matchCollection = Regex.Matches(fileContent, dcEngineCommonTxtParam.GetContentPattern, RegexOptions.IgnoreCase); if (dcEngineCommonTxtParam.ColumnSplitType == (byte)ColumnSplitType.按字节截取) { foreach (Match match in matchCollection) //遍历文件每一行 { string currentRow = match.Groups["key"].Value; //排除指定关键字的行 if (!string.IsNullOrEmpty(dcEngineCommonTxtParam.RowExcept) && currentRow.Contains(dcEngineCommonTxtParam.RowExcept)) { continue; } int indexColumn = 0; DataRow newRow = dtFileContent.NewRow(); newRow[indexColumn++] = Guid.NewGuid(); if (dcEngineCommonTxtParam.FileInfoType != FileInfoType.BillBase && dcEngineCommonTxtParam.FileInfoType != FileInfoType.NoMasterBase) { newRow[indexColumn++] = businessDate; } byte[] byteCurrentRow = Encoding.GetEncoding(dcEngineCommonTxtParam.FileEncoding).GetBytes(currentRow); foreach (string columnSplit in columnSplitContents) { try { string[] param = columnSplit.Replace(",", ",").Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries); int index = int.Parse(param[0]); int count = int.Parse(param[1]); if (index + count > byteCurrentRow.Length) { count = byteCurrentRow.Length - index; } string column = Encoding.GetEncoding(dcEngineCommonTxtParam.FileEncoding).GetString(byteCurrentRow, index, count); newRow[indexColumn++] = column; } catch (Exception) { string errorInfo = string.Format("文件:{0},当前行按字节分列时出错:{1}", extractFilePath, currentRow); throw new Exception(errorInfo); } } dtFileContent.Rows.Add(newRow); } //end foreach } else //按字符分割 { foreach (Match match in matchCollection) //遍历文件每一行 { string currentRow = match.Groups["key"].Value; if (string.IsNullOrEmpty(currentRow.Trim())) { continue; } //排除指定关键字的行 if (!string.IsNullOrEmpty(dcEngineCommonTxtParam.RowExcept) && currentRow.Contains(dcEngineCommonTxtParam.RowExcept)) { continue; } string[] columns = null; try { columns = currentRow.Split(new string[] { dcEngineCommonTxtParam.ContentSplit }, dcEngineCommonTxtParam.StringSplitOptions); } catch (Exception) { string errorInfo = string.Format("文件:{0},当前行按字符分列时出错:{1}", extractFilePath, currentRow); throw new Exception(errorInfo); } //因为数据库最左边多了二列,存放Id及业务发生日 if (columns.Length + fixedColumnCount != dtFileContent.Columns.Count) { string errorInfo = string.Format("\r\n分割列数与数据库表列数不一致\r\n文件:{0}\r\n行:{1}", extractFilePath, match.Groups["key"].Value); throw new Exception(errorInfo); } int indexColumn = 0; DataRow newRow = dtFileContent.NewRow(); newRow[indexColumn++] = Guid.NewGuid(); if (dcEngineCommonTxtParam.FileInfoType != FileInfoType.BillBase && dcEngineCommonTxtParam.FileInfoType != FileInfoType.NoMasterBase) { newRow[indexColumn++] = businessDate; } foreach (string column in columns) { newRow[indexColumn++] = column; } dtFileContent.Rows.Add(newRow); } // end foreach } //end //按字符分割 }
private void FileImportToDataBaseWithoutId(DCEngineCommonTxtParam dcEngineCommonTxtParam, string businessDate, string extractFilePath, ref DataTable dtFileContent, string tableName) { string[] columnSplitContents = null; int fixedColumnCount = 1; if (dcEngineCommonTxtParam.ColumnSplitType == (byte)ColumnSplitType.按字节截取) { columnSplitContents = dcEngineCommonTxtParam.ByteSplit.Split(new string[] { "|" }, StringSplitOptions.RemoveEmptyEntries); //因为数据库最左边多了二列,存放Id及业务发生日 if (columnSplitContents.Length + fixedColumnCount != dtFileContent.Columns.Count) { string errorInfo = string.Format("源文件{0}按字节分割的列数:{1}与数据库列数:{2}不一致", extractFilePath, (columnSplitContents.Length + 1).ToString(), dtFileContent.Columns.Count.ToString()); throw new Exception(errorInfo); } } StringBuilder sbFileContent = new StringBuilder(); string fileContent; int submitRowCount = 0; using (StreamReader sr = new StreamReader(extractFilePath, Encoding.GetEncoding(dcEngineCommonTxtParam.FileEncoding))) { while (sr.Peek() > 0) { sbFileContent.AppendLine(sr.ReadLine()); submitRowCount++; if (submitRowCount == NMBConfig.FileBatchSize) { fileContent = sbFileContent.ToString(); sbFileContent.Clear(); submitRowCount = 0; FileContentToDataTableWithoutId(fileContent, dcEngineCommonTxtParam, businessDate, extractFilePath, columnSplitContents, fixedColumnCount, ref dtFileContent); DataTableSaveToDataBase(ref dtFileContent, tableName); } } if (sbFileContent.Length > 0) { fileContent = sbFileContent.ToString(); FileContentToDataTableWithoutId(fileContent, dcEngineCommonTxtParam, businessDate, extractFilePath, columnSplitContents, fixedColumnCount, ref dtFileContent); DataTableSaveToDataBase(ref dtFileContent, tableName); } sbFileContent.Clear(); fileContent = null; } }