private List <String> loadFileFromList(clsFileList verifyFile) { if (!File.Exists(verifyFile.FullFilePath)) { Utils.WriteToLog(Utils.LogSeverity.Error, processName, String.Format("{0} does not exist.", verifyFile.FullFilePath)); Console.WriteLine("{0} does not exist.", verifyFile.FullFilePath); return(null); } StreamReader sr = File.OpenText(verifyFile.FullFilePath); List <String> contents = new List <String>(); String inputLine; while ((inputLine = sr.ReadLine()) != null) { // Remove any weird characters in the line contents.Add(inputLine.Replace('', ' ')); } sr.Close(); return(contents); /* * foreach (string s in contents) * { * Console.WriteLine(s); * } */ }
private void parkFile(clsFileList vf, ProcessFile pf) { //DocLibHelper dlh = new DocLibHelper(); // move file into Parked folder string destFilename = rootLocation + @"PipelineData\5.Parked" + @"\" + vf.Filename; //Console.WriteLine("Parked location: " + destFilename); Console.ReadLine(); Utils.CopyLocalFileFromTo(vf.FullFilePath, destFilename, true); //Console.WriteLine("copied file to parked"); Console.ReadLine(); // set db status pf.FileStatus = (int)Utils.FileStatus.Parked; pf.Name = destFilename; //Console.WriteLine("attempting to write parked status to db"); Console.ReadLine(); windARTDataContext.SubmitChanges(); // set Sharepoint status //dlh.UpdateFileStatus(pf.ProcessFileId, Utils.FileStatus.Parked, 0, 0); Utils.WriteToLog(Utils.LogSeverity.Warning, processName, vf.FullFilePath + " parked - unknown formart"); }
private void unParkFile(clsFileList vf) { //once we have decided that the format, check to see if there is lingering file in parked //if there is, delete it. string targetFilename = ConfigurationSettings.AppSettings["FolderParked"] + @"\" + vf.Filename; if (File.Exists(targetFilename)) { File.Delete(targetFilename); Utils.WriteToLog(Utils.LogSeverity.Info, processName, vf.Filename + " removed from parked files"); } }
private void unParkFile(clsFileList vf) { //once we have decided that the format, check to see if there is lingering file in parked //if there is, delete it. string targetFilename = rootLocation + @"PipelineData\5.Parked" + @"\" + vf.Filename; if (File.Exists(targetFilename)) { File.Delete(targetFilename); Utils.WriteToLog(Utils.LogSeverity.Info, processName, vf.Filename + " removed from parked files"); } }
public void ProcessEncryptedFilesTest() { Dictionary <string, clsFileList> encryptedFiles = new Dictionary <string, clsFileList>(); clsFileList cfl = new clsFileList(); DecryptFile df = new DecryptFile(); //cfl.FullFilePath = @"C:\PipelineData\999.TestFiles\090120070402046.RWD"; cfl.FullFilePath = @"C:\PipelineData\999.TestFiles\090120040516215.RWD"; cfl.Encrypted = true; encryptedFiles.Add(cfl.FullFilePath, cfl); df.processEncryptedFiles(encryptedFiles); }
private bool checkHeaderMatchs(List <string> importedFile, clsFileList fl) { string dbHeaderRowContents = string.Empty; // only check file formats after the last one we checked var FileFormatQuery = from fileFormats in windARTDataContext.FileFormats where fileFormats.FileLocationId == fl.FileLocationId && fileFormats.FileFormatId > fl.FileFormatId orderby fileFormats.FileFormatId select fileFormats; foreach (FileFormat ff in FileFormatQuery) { int hr = (int)ff.HeaderRowNumber - 1; // Catch negative index values before they cause any harm if ((hr > importedFile.Count - 1) || (hr < 0)) { Utils.WriteToLog(Utils.LogSeverity.Warning, processName, String.Format("Found a HeaderRowNumber of {0} in a file with {1} lines. File format {2}", hr, importedFile.Count, ff.FileFormatId)); } else { // Fix any abnormal characters, that Excel will have magic-ed away from us // during the DefineUFL phase // Treat raw file contnets and db contents in the same manner for consistency dbHeaderRowContents = ff.HeaderRowContents; dbHeaderRowContents = dbHeaderRowContents.Replace("\"", ""); importedFile[hr] = importedFile[hr].Replace("\"", ""); // Ensure our array is at least as big as the line we are trying to pull out if (importedFile[hr] == dbHeaderRowContents) { fl.FileFormatId = ff.FileFormatId; //fl.SiteId = (int)ff.siteId; return(true); } } } return(false); }
public void getFileList(Dictionary <string, clsFileList> fileList, string folder, string fileMask, bool recursive) { DirectoryInfo di = new DirectoryInfo(folder); FileInfo[] rgFiles; rgFiles = di.GetFiles(fileMask, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); foreach (FileInfo fi in rgFiles) { clsFileList fl = new clsFileList(); fl.Filename = fi.Name; fl.Directory = fi.Directory.FullName; fl.FileLocationId = 0; fileList.Add(fi.FullName, fl); Utils.WriteToLog(Utils.LogSeverity.Info, processName, "Found file " + fi.FullName); } }
private void parkFile(clsFileList vf, ProcessFile pf) { DocLibHelper dlh = new DocLibHelper(); // move file into Parked folder string destFilename = ConfigurationSettings.AppSettings["FolderParked"] + @"\" + vf.Filename; Utils.CopyLocalFileFromTo(vf.FullFilePath, destFilename, true); // set db status pf.FileStatus = (int)Utils.FileStatus.Parked; pf.Name = destFilename; windARTDataContext.SubmitChanges(); // set Sharepoint status dlh.UpdateFileStatus(pf.ProcessFileId, Utils.FileStatus.Parked, 0, 0); Utils.WriteToLog(Utils.LogSeverity.Warning, processName, vf.FullFilePath + " parked - unknown formart"); }
private bool checkMultiLineHeaderMatchs(List <string> importedFile, clsFileList fl) { var MultiLineHeaderRowsQuery = from multiLineHeaderRows in windARTDataContext.MultiLineHeaderRows where multiLineHeaderRows.FileFormatId == fl.FileFormatId select multiLineHeaderRows; foreach (MultiLineHeaderRow ml in MultiLineHeaderRowsQuery) { int hr = (int)ml.HeaderRowNumber - 1; string compareLine = importedFile[hr]; compareLine = compareLine.Replace("\"", ""); // Sometimes we see trailing tabs which confuses the match // Maybe we will see other trailing delims that will cause trouble? while (compareLine.EndsWith("\t")) { compareLine = compareLine.Remove(compareLine.Length - 1); } // MS 17 Oct 2008 // Sometimes Excel strips away blank columns for us, // and this causes problems when comparing it to the raw file // e.g. we compare: XXX with XXX,,,,,,,,, // So we strip away the trailing commas // This may need to be tweaked later? while (compareLine.EndsWith(",")) { compareLine = compareLine.Remove(compareLine.Length - 1); } // Do they match? if (compareLine != ml.HeaderRowContents) { return(false); } } return(true); }
public void getFileList(Dictionary <string, clsFileList> fileList) { var FileLocationsQuery = from fileLocations in windARTDataContext.FileLocations /* where fileLocations.Enabled == true */ select fileLocations; foreach (var v in FileLocationsQuery) { DirectoryInfo di = new DirectoryInfo(v.Folder); try { FileInfo[] rgFiles = di.GetFiles(v.FileMask, v.Recurse == true ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); foreach (FileInfo fi in rgFiles) { // Ensure we haven't already matched this file from a previous loop if (!fileList.ContainsKey(fi.FullName)) { clsFileList fl = new clsFileList(); fl.Filename = fi.Name; fl.Directory = fi.Directory.FullName; fl.OriginalFullFilepath = fl.FullFilePath; fl.FileLocationId = (int)v.FileLocationId; fl.Encrypted = (bool)v.Encrypted; fileList.Add(fi.FullName, fl); } // Creates too much logging //Utils.WriteToLog(Utils.LogSeverity.Info, processName, "Found file " + fi.FullName); } } catch (Exception e) { Utils.WriteToLog(Utils.LogSeverity.Error, processName, "Error reading files from: " + v.Folder + "\n" + e.Message); } } }
private bool checkHeaderMatchs(List <string> importedFile, clsFileList fl) { string dbHeaderRowContents = string.Empty; System.Linq.IOrderedQueryable <FileFormat> FileFormatQuery; // only check file formats after the last one we checked if (fl.FileFormatId == 0) {//get everything on the first iteration FileFormatQuery = from fileFormats in windARTDataContext.FileFormats where fileFormats.FileLocationId == fl.FileLocationId where fileFormats.FileFormatId > fl.FileFormatId orderby fileFormats.FileFormatId descending select fileFormats; } else {//only choose formats less than ones we've already processed FileFormatQuery = from fileFormats in windARTDataContext.FileFormats where fileFormats.FileLocationId == fl.FileLocationId where fileFormats.FileFormatId < fl.FileFormatId orderby fileFormats.FileFormatId descending select fileFormats; } foreach (FileFormat ff in FileFormatQuery) { int hr = (int)ff.HeaderRowNumber - 1; // Catch negative index values before they cause any harm if ((hr > importedFile.Count - 1) || (hr < 0)) { Utils.WriteToLog(Utils.LogSeverity.Warning, processName, String.Format("Found a HeaderRowNumber of {0} in a file with {1} lines. File format {2}", hr, importedFile.Count, ff.FileFormatId)); } else { // Fix any abnormal characters, that Excel will have magic-ed away from us // during the DefineUFL phase // Treat raw file contnets and db contents in the same manner for consistency dbHeaderRowContents = ff.HeaderRowContents; dbHeaderRowContents = dbHeaderRowContents.Replace("\"", ""); string fileHeaderRow = importedFile[hr].Replace("\"", "").Replace('"', ' '); // Ensure our array is at least as big as the line we are trying to pull out //comment out for production ****** //if (ff.FileFormatId == 446) //{ // Debug.WriteLine("found id"); //} //************************* if (fileHeaderRow == dbHeaderRowContents) { fl.FileFormatId = ff.FileFormatId; return(true); } } } return(false); }
private void deleteColumns(List <string> importedFileContent, int fileFormatID, clsFileList vf) { string delimiter = string.Empty; string delim = windARTDataContext.UFLDetails.Where(c => c.UFLDetailId == fileFormatID).Select(c => c.Delimitor).Single().ToString(); if (delim == "TAB") { delimiter = "\t"; } else { delimiter = ","; } List <int> GarbageCols = new List <int>(); //get header row number from db int headerRow = (int)windARTDataContext.FileFormats.Where(c => c.FileFormatId == fileFormatID).Select(c => c.HeaderRowNumber).First(); List <string> row = new List <string>(); string UpdatedRow = string.Empty; string header = windARTDataContext.UFLDetails.Where(c => c.UFLDetailId == fileFormatID).Select(c => c.PiTagList).Single(); List <string> headerCols = header.Split('|').ToList(); if (importedFileContent == null) { return; } //find columns marked as N/A and change values to a missing numeric value //this is done because many of the N/A columns have text in them and cause the bulk insert to fail int count = 0; foreach (string s in headerCols) { if (s == "N/A") { GarbageCols.Add(count); } count++; } if (GarbageCols.Count > 0) { for (int i = headerRow; i < importedFileContent.Count(); i++) { row = importedFileContent[i].Split(delimiter.ToCharArray()).ToList(); foreach (int j in GarbageCols) { row[j] = "-9999.99"; } //save updated row back to the comma delim data string seperator = string.Empty; foreach (string s in row) { seperator = delimiter; UpdatedRow += s + seperator; } //remove trailing comma UpdatedRow = UpdatedRow.Remove(UpdatedRow.Length - 1); importedFileContent[i] = UpdatedRow; UpdatedRow = string.Empty; } } }