private List <String> loadFileFromList(clsFileList verifyFile)
{
if (!File.Exists(verifyFile.FullFilePath))
{
Utils.WriteToLog(Utils.LogSeverity.Error, processName, String.Format("{0} does not exist.", verifyFile.FullFilePath));
Console.WriteLine("{0} does not exist.", verifyFile.FullFilePath);
return(null);
}
StreamReader sr = File.OpenText(verifyFile.FullFilePath);
List <String> contents = new List <String>();
String inputLine;
while ((inputLine = sr.ReadLine()) != null)
{
// Remove any weird characters in the line
contents.Add(inputLine.Replace('', ' '));
}
sr.Close();
return(contents);
/*
* foreach (string s in contents)
* {
* Console.WriteLine(s);
* }
*/
}
private void parkFile(clsFileList vf, ProcessFile pf)
{
//DocLibHelper dlh = new DocLibHelper();
// move file into Parked folder
string destFilename = rootLocation + @"PipelineData\5.Parked" + @"\" + vf.Filename;
//Console.WriteLine("Parked location: " + destFilename);
Console.ReadLine();
Utils.CopyLocalFileFromTo(vf.FullFilePath, destFilename, true);
//Console.WriteLine("copied file to parked");
Console.ReadLine();
// set db status
pf.FileStatus = (int)Utils.FileStatus.Parked;
pf.Name = destFilename;
//Console.WriteLine("attempting to write parked status to db");
Console.ReadLine();
windARTDataContext.SubmitChanges();
// set Sharepoint status
//dlh.UpdateFileStatus(pf.ProcessFileId, Utils.FileStatus.Parked, 0, 0);
Utils.WriteToLog(Utils.LogSeverity.Warning, processName, vf.FullFilePath + " parked - unknown formart");
}
private void unParkFile(clsFileList vf)
{
//once we have decided that the format, check to see if there is lingering file in parked
//if there is, delete it.
string targetFilename = ConfigurationSettings.AppSettings["FolderParked"] + @"\" + vf.Filename;
if (File.Exists(targetFilename))
{
File.Delete(targetFilename);
Utils.WriteToLog(Utils.LogSeverity.Info, processName, vf.Filename + " removed from parked files");
}
}
private void unParkFile(clsFileList vf)
{
//once we have decided that the format, check to see if there is lingering file in parked
//if there is, delete it.
string targetFilename = rootLocation + @"PipelineData\5.Parked" + @"\" + vf.Filename;
if (File.Exists(targetFilename))
{
File.Delete(targetFilename);
Utils.WriteToLog(Utils.LogSeverity.Info, processName, vf.Filename + " removed from parked files");
}
}
public void ProcessEncryptedFilesTest()
{
Dictionary <string, clsFileList> encryptedFiles = new Dictionary <string, clsFileList>();
clsFileList cfl = new clsFileList();
DecryptFile df = new DecryptFile();
//cfl.FullFilePath = @"C:\PipelineData\999.TestFiles\090120070402046.RWD";
cfl.FullFilePath = @"C:\PipelineData\999.TestFiles\090120040516215.RWD";
cfl.Encrypted = true;
encryptedFiles.Add(cfl.FullFilePath, cfl);
df.processEncryptedFiles(encryptedFiles);
}
private bool checkHeaderMatchs(List <string> importedFile, clsFileList fl)
{
string dbHeaderRowContents = string.Empty;
// only check file formats after the last one we checked
var FileFormatQuery = from fileFormats in windARTDataContext.FileFormats
where fileFormats.FileLocationId == fl.FileLocationId &&
fileFormats.FileFormatId > fl.FileFormatId
orderby fileFormats.FileFormatId
select fileFormats;
foreach (FileFormat ff in FileFormatQuery)
{
int hr = (int)ff.HeaderRowNumber - 1;
// Catch negative index values before they cause any harm
if ((hr > importedFile.Count - 1) || (hr < 0))
{
Utils.WriteToLog(Utils.LogSeverity.Warning, processName, String.Format("Found a HeaderRowNumber of {0} in a file with {1} lines. File format {2}", hr, importedFile.Count, ff.FileFormatId));
}
else
{
// Fix any abnormal characters, that Excel will have magic-ed away from us
// during the DefineUFL phase
// Treat raw file contnets and db contents in the same manner for consistency
dbHeaderRowContents = ff.HeaderRowContents;
dbHeaderRowContents = dbHeaderRowContents.Replace("\"", "");
importedFile[hr] = importedFile[hr].Replace("\"", "");
// Ensure our array is at least as big as the line we are trying to pull out
if (importedFile[hr] == dbHeaderRowContents)
{
fl.FileFormatId = ff.FileFormatId;
//fl.SiteId = (int)ff.siteId;
return(true);
}
}
}
return(false);
}
public void getFileList(Dictionary <string, clsFileList> fileList, string folder, string fileMask, bool recursive)
{
DirectoryInfo di = new DirectoryInfo(folder);
FileInfo[] rgFiles;
rgFiles = di.GetFiles(fileMask, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
foreach (FileInfo fi in rgFiles)
{
clsFileList fl = new clsFileList();
fl.Filename = fi.Name;
fl.Directory = fi.Directory.FullName;
fl.FileLocationId = 0;
fileList.Add(fi.FullName, fl);
Utils.WriteToLog(Utils.LogSeverity.Info, processName, "Found file " + fi.FullName);
}
}
private void parkFile(clsFileList vf, ProcessFile pf)
{
DocLibHelper dlh = new DocLibHelper();
// move file into Parked folder
string destFilename = ConfigurationSettings.AppSettings["FolderParked"] + @"\" + vf.Filename;
Utils.CopyLocalFileFromTo(vf.FullFilePath, destFilename, true);
// set db status
pf.FileStatus = (int)Utils.FileStatus.Parked;
pf.Name = destFilename;
windARTDataContext.SubmitChanges();
// set Sharepoint status
dlh.UpdateFileStatus(pf.ProcessFileId, Utils.FileStatus.Parked, 0, 0);
Utils.WriteToLog(Utils.LogSeverity.Warning, processName, vf.FullFilePath + " parked - unknown formart");
}
private bool checkMultiLineHeaderMatchs(List <string> importedFile, clsFileList fl)
{
var MultiLineHeaderRowsQuery = from multiLineHeaderRows in windARTDataContext.MultiLineHeaderRows
where multiLineHeaderRows.FileFormatId == fl.FileFormatId
select multiLineHeaderRows;
foreach (MultiLineHeaderRow ml in MultiLineHeaderRowsQuery)
{
int hr = (int)ml.HeaderRowNumber - 1;
string compareLine = importedFile[hr];
compareLine = compareLine.Replace("\"", "");
// Sometimes we see trailing tabs which confuses the match
// Maybe we will see other trailing delims that will cause trouble?
while (compareLine.EndsWith("\t"))
{
compareLine = compareLine.Remove(compareLine.Length - 1);
}
// MS 17 Oct 2008
// Sometimes Excel strips away blank columns for us,
// and this causes problems when comparing it to the raw file
// e.g. we compare: XXX with XXX,,,,,,,,,
// So we strip away the trailing commas
// This may need to be tweaked later?
while (compareLine.EndsWith(","))
{
compareLine = compareLine.Remove(compareLine.Length - 1);
}
// Do they match?
if (compareLine != ml.HeaderRowContents)
{
return(false);
}
}
return(true);
}
public void getFileList(Dictionary <string, clsFileList> fileList)
{
var FileLocationsQuery = from fileLocations in windARTDataContext.FileLocations
/* where fileLocations.Enabled == true */
select fileLocations;
foreach (var v in FileLocationsQuery)
{
DirectoryInfo di = new DirectoryInfo(v.Folder);
try
{
FileInfo[] rgFiles = di.GetFiles(v.FileMask, v.Recurse == true ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
foreach (FileInfo fi in rgFiles)
{
// Ensure we haven't already matched this file from a previous loop
if (!fileList.ContainsKey(fi.FullName))
{
clsFileList fl = new clsFileList();
fl.Filename = fi.Name;
fl.Directory = fi.Directory.FullName;
fl.OriginalFullFilepath = fl.FullFilePath;
fl.FileLocationId = (int)v.FileLocationId;
fl.Encrypted = (bool)v.Encrypted;
fileList.Add(fi.FullName, fl);
}
// Creates too much logging
//Utils.WriteToLog(Utils.LogSeverity.Info, processName, "Found file " + fi.FullName);
}
}
catch (Exception e)
{
Utils.WriteToLog(Utils.LogSeverity.Error, processName, "Error reading files from: " + v.Folder + "\n" + e.Message);
}
}
}
private bool checkHeaderMatchs(List <string> importedFile, clsFileList fl)
{
string dbHeaderRowContents = string.Empty;
System.Linq.IOrderedQueryable <FileFormat> FileFormatQuery;
// only check file formats after the last one we checked
if (fl.FileFormatId == 0)
{//get everything on the first iteration
FileFormatQuery = from fileFormats in windARTDataContext.FileFormats
where fileFormats.FileLocationId == fl.FileLocationId
where fileFormats.FileFormatId > fl.FileFormatId
orderby fileFormats.FileFormatId descending
select fileFormats;
}
else
{//only choose formats less than ones we've already processed
FileFormatQuery = from fileFormats in windARTDataContext.FileFormats
where fileFormats.FileLocationId == fl.FileLocationId
where fileFormats.FileFormatId < fl.FileFormatId
orderby fileFormats.FileFormatId descending
select fileFormats;
}
foreach (FileFormat ff in FileFormatQuery)
{
int hr = (int)ff.HeaderRowNumber - 1;
// Catch negative index values before they cause any harm
if ((hr > importedFile.Count - 1) || (hr < 0))
{
Utils.WriteToLog(Utils.LogSeverity.Warning, processName,
String.Format("Found a HeaderRowNumber of {0} in a file with {1} lines. File format {2}", hr, importedFile.Count, ff.FileFormatId));
}
else
{
// Fix any abnormal characters, that Excel will have magic-ed away from us
// during the DefineUFL phase
// Treat raw file contnets and db contents in the same manner for consistency
dbHeaderRowContents = ff.HeaderRowContents;
dbHeaderRowContents = dbHeaderRowContents.Replace("\"", "");
string fileHeaderRow = importedFile[hr].Replace("\"", "").Replace('"', ' ');
// Ensure our array is at least as big as the line we are trying to pull out
//comment out for production ******
//if (ff.FileFormatId == 446)
//{
// Debug.WriteLine("found id");
//}
//*************************
if (fileHeaderRow == dbHeaderRowContents)
{
fl.FileFormatId = ff.FileFormatId;
return(true);
}
}
}
return(false);
}
private void deleteColumns(List <string> importedFileContent, int fileFormatID, clsFileList vf)
{
string delimiter = string.Empty;
string delim = windARTDataContext.UFLDetails.Where(c => c.UFLDetailId == fileFormatID).Select(c => c.Delimitor).Single().ToString();
if (delim == "TAB")
{
delimiter = "\t";
}
else
{
delimiter = ",";
}
List <int> GarbageCols = new List <int>();
//get header row number from db
int headerRow = (int)windARTDataContext.FileFormats.Where(c => c.FileFormatId == fileFormatID).Select(c => c.HeaderRowNumber).First();
List <string> row = new List <string>();
string UpdatedRow = string.Empty;
string header = windARTDataContext.UFLDetails.Where(c => c.UFLDetailId == fileFormatID).Select(c => c.PiTagList).Single();
List <string> headerCols = header.Split('|').ToList();
if (importedFileContent == null)
{
return;
}
//find columns marked as N/A and change values to a missing numeric value
//this is done because many of the N/A columns have text in them and cause the bulk insert to fail
int count = 0;
foreach (string s in headerCols)
{
if (s == "N/A")
{
GarbageCols.Add(count);
}
count++;
}
if (GarbageCols.Count > 0)
{
for (int i = headerRow; i < importedFileContent.Count(); i++)
{
row = importedFileContent[i].Split(delimiter.ToCharArray()).ToList();
foreach (int j in GarbageCols)
{
row[j] = "-9999.99";
}
//save updated row back to the comma delim data
string seperator = string.Empty;
foreach (string s in row)
{
seperator = delimiter;
UpdatedRow += s + seperator;
}
//remove trailing comma
UpdatedRow = UpdatedRow.Remove(UpdatedRow.Length - 1);
importedFileContent[i] = UpdatedRow;
UpdatedRow = string.Empty;
}
}
}
