예제 #1
0
        private List <String> loadFileFromList(clsFileList verifyFile)
        {
            if (!File.Exists(verifyFile.FullFilePath))
            {
                Utils.WriteToLog(Utils.LogSeverity.Error, processName, String.Format("{0} does not exist.", verifyFile.FullFilePath));
                Console.WriteLine("{0} does not exist.", verifyFile.FullFilePath);
                return(null);
            }

            StreamReader  sr       = File.OpenText(verifyFile.FullFilePath);
            List <String> contents = new List <String>();
            String        inputLine;

            while ((inputLine = sr.ReadLine()) != null)
            {
                // Remove any weird characters in the line
                contents.Add(inputLine.Replace('', ' '));
            }
            sr.Close();

            return(contents);

            /*
             * foreach (string s in contents)
             * {
             *  Console.WriteLine(s);
             * }
             */
        }
예제 #2
0
        private void parkFile(clsFileList vf, ProcessFile pf)
        {
            //DocLibHelper dlh = new DocLibHelper();

            // move file into Parked folder
            string destFilename = rootLocation + @"PipelineData\5.Parked" + @"\" + vf.Filename;

            //Console.WriteLine("Parked location: " + destFilename);
            Console.ReadLine();

            Utils.CopyLocalFileFromTo(vf.FullFilePath, destFilename, true);

            //Console.WriteLine("copied file to parked");
            Console.ReadLine();

            // set db status
            pf.FileStatus = (int)Utils.FileStatus.Parked;
            pf.Name       = destFilename;

            //Console.WriteLine("attempting to write parked status to db");
            Console.ReadLine();

            windARTDataContext.SubmitChanges();

            // set Sharepoint status
            //dlh.UpdateFileStatus(pf.ProcessFileId, Utils.FileStatus.Parked, 0, 0);

            Utils.WriteToLog(Utils.LogSeverity.Warning, processName, vf.FullFilePath + " parked - unknown formart");
        }
예제 #3
0
        private void unParkFile(clsFileList vf)
        {
            //once we have decided that the format, check to see if there is lingering file in parked
            //if there is, delete it.
            string targetFilename = ConfigurationSettings.AppSettings["FolderParked"] + @"\" + vf.Filename;

            if (File.Exists(targetFilename))
            {
                File.Delete(targetFilename);
                Utils.WriteToLog(Utils.LogSeverity.Info, processName, vf.Filename + " removed from parked files");
            }
        }
예제 #4
0
        private void unParkFile(clsFileList vf)
        {
            //once we have decided that the format, check to see if there is lingering file in parked
            //if there is, delete it.
            string targetFilename = rootLocation + @"PipelineData\5.Parked" + @"\" + vf.Filename;

            if (File.Exists(targetFilename))
            {
                File.Delete(targetFilename);
                Utils.WriteToLog(Utils.LogSeverity.Info, processName, vf.Filename + " removed from parked files");
            }
        }
예제 #5
0
        public void ProcessEncryptedFilesTest()
        {
            Dictionary <string, clsFileList> encryptedFiles = new Dictionary <string, clsFileList>();
            clsFileList cfl = new clsFileList();
            DecryptFile df  = new DecryptFile();

            //cfl.FullFilePath = @"C:\PipelineData\999.TestFiles\090120070402046.RWD";
            cfl.FullFilePath = @"C:\PipelineData\999.TestFiles\090120040516215.RWD";
            cfl.Encrypted    = true;


            encryptedFiles.Add(cfl.FullFilePath, cfl);

            df.processEncryptedFiles(encryptedFiles);
        }
예제 #6
0
        private bool checkHeaderMatchs(List <string> importedFile, clsFileList fl)
        {
            string dbHeaderRowContents = string.Empty;

            // only check file formats after the last one we checked
            var FileFormatQuery = from fileFormats in windARTDataContext.FileFormats
                                  where fileFormats.FileLocationId == fl.FileLocationId &&
                                  fileFormats.FileFormatId > fl.FileFormatId
                                  orderby fileFormats.FileFormatId
                                  select fileFormats;

            foreach (FileFormat ff in FileFormatQuery)
            {
                int hr = (int)ff.HeaderRowNumber - 1;


                // Catch negative index values before they cause any harm
                if ((hr > importedFile.Count - 1) || (hr < 0))
                {
                    Utils.WriteToLog(Utils.LogSeverity.Warning, processName, String.Format("Found a HeaderRowNumber of {0} in a file with {1} lines.  File format {2}", hr, importedFile.Count, ff.FileFormatId));
                }
                else
                {
                    // Fix any abnormal characters, that Excel will have magic-ed away from us
                    // during the DefineUFL phase
                    // Treat raw file contnets and db contents in the same manner for consistency
                    dbHeaderRowContents = ff.HeaderRowContents;
                    dbHeaderRowContents = dbHeaderRowContents.Replace("\"", "");

                    importedFile[hr] = importedFile[hr].Replace("\"", "");

                    // Ensure our array is at least as big as the line we are trying to pull out
                    if (importedFile[hr] == dbHeaderRowContents)
                    {
                        fl.FileFormatId = ff.FileFormatId;
                        //fl.SiteId = (int)ff.siteId;
                        return(true);
                    }
                }
            }

            return(false);
        }
예제 #7
0
        public void getFileList(Dictionary <string, clsFileList> fileList, string folder, string fileMask, bool recursive)
        {
            DirectoryInfo di = new DirectoryInfo(folder);

            FileInfo[] rgFiles;

            rgFiles = di.GetFiles(fileMask, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);

            foreach (FileInfo fi in rgFiles)
            {
                clsFileList fl = new clsFileList();
                fl.Filename       = fi.Name;
                fl.Directory      = fi.Directory.FullName;
                fl.FileLocationId = 0;
                fileList.Add(fi.FullName, fl);

                Utils.WriteToLog(Utils.LogSeverity.Info, processName, "Found file " + fi.FullName);
            }
        }
예제 #8
0
        private void parkFile(clsFileList vf, ProcessFile pf)
        {
            DocLibHelper dlh = new DocLibHelper();

            // move file into Parked folder
            string destFilename = ConfigurationSettings.AppSettings["FolderParked"] + @"\" + vf.Filename;

            Utils.CopyLocalFileFromTo(vf.FullFilePath, destFilename, true);

            // set db status
            pf.FileStatus = (int)Utils.FileStatus.Parked;
            pf.Name       = destFilename;

            windARTDataContext.SubmitChanges();

            // set Sharepoint status
            dlh.UpdateFileStatus(pf.ProcessFileId, Utils.FileStatus.Parked, 0, 0);

            Utils.WriteToLog(Utils.LogSeverity.Warning, processName, vf.FullFilePath + " parked - unknown formart");
        }
예제 #9
0
        private bool checkMultiLineHeaderMatchs(List <string> importedFile, clsFileList fl)
        {
            var MultiLineHeaderRowsQuery = from multiLineHeaderRows in windARTDataContext.MultiLineHeaderRows
                                           where multiLineHeaderRows.FileFormatId == fl.FileFormatId
                                           select multiLineHeaderRows;

            foreach (MultiLineHeaderRow ml in MultiLineHeaderRowsQuery)
            {
                int hr = (int)ml.HeaderRowNumber - 1;

                string compareLine = importedFile[hr];
                compareLine = compareLine.Replace("\"", "");

                // Sometimes we see trailing tabs which confuses the match
                // Maybe we will see other trailing delims that will cause trouble?
                while (compareLine.EndsWith("\t"))
                {
                    compareLine = compareLine.Remove(compareLine.Length - 1);
                }

                // MS 17 Oct 2008
                // Sometimes Excel strips away blank columns for us,
                // and this causes problems when comparing it to the raw file
                // e.g. we compare: XXX with XXX,,,,,,,,,
                // So we strip away the trailing commas
                // This may need to be tweaked later?
                while (compareLine.EndsWith(","))
                {
                    compareLine = compareLine.Remove(compareLine.Length - 1);
                }

                // Do they match?
                if (compareLine != ml.HeaderRowContents)
                {
                    return(false);
                }
            }

            return(true);
        }
예제 #10
0
        public void getFileList(Dictionary <string, clsFileList> fileList)
        {
            var FileLocationsQuery = from fileLocations in windARTDataContext.FileLocations
                                     /* where fileLocations.Enabled == true */
                                     select fileLocations;

            foreach (var v in FileLocationsQuery)
            {
                DirectoryInfo di = new DirectoryInfo(v.Folder);
                try
                {
                    FileInfo[] rgFiles = di.GetFiles(v.FileMask, v.Recurse == true ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);

                    foreach (FileInfo fi in rgFiles)
                    {
                        // Ensure we haven't already matched this file from a previous loop
                        if (!fileList.ContainsKey(fi.FullName))
                        {
                            clsFileList fl = new clsFileList();
                            fl.Filename             = fi.Name;
                            fl.Directory            = fi.Directory.FullName;
                            fl.OriginalFullFilepath = fl.FullFilePath;
                            fl.FileLocationId       = (int)v.FileLocationId;
                            fl.Encrypted            = (bool)v.Encrypted;
                            fileList.Add(fi.FullName, fl);
                        }
                        // Creates too much logging
                        //Utils.WriteToLog(Utils.LogSeverity.Info, processName, "Found file " + fi.FullName);
                    }
                }
                catch (Exception e)
                {
                    Utils.WriteToLog(Utils.LogSeverity.Error, processName, "Error reading files from: " + v.Folder + "\n" + e.Message);
                }
            }
        }
예제 #11
0
        private bool checkHeaderMatchs(List <string> importedFile, clsFileList fl)
        {
            string dbHeaderRowContents = string.Empty;

            System.Linq.IOrderedQueryable <FileFormat> FileFormatQuery;
            // only check file formats after the last one we checked
            if (fl.FileFormatId == 0)
            {//get everything on the first iteration
                FileFormatQuery = from fileFormats in windARTDataContext.FileFormats
                                  where fileFormats.FileLocationId == fl.FileLocationId
                                  where fileFormats.FileFormatId > fl.FileFormatId
                                  orderby fileFormats.FileFormatId descending
                                  select fileFormats;
            }
            else
            {//only choose formats less than ones we've already processed
                FileFormatQuery = from fileFormats in windARTDataContext.FileFormats
                                  where fileFormats.FileLocationId == fl.FileLocationId
                                  where fileFormats.FileFormatId < fl.FileFormatId
                                  orderby fileFormats.FileFormatId descending
                                  select fileFormats;
            }

            foreach (FileFormat ff in FileFormatQuery)
            {
                int hr = (int)ff.HeaderRowNumber - 1;


                // Catch negative index values before they cause any harm
                if ((hr > importedFile.Count - 1) || (hr < 0))
                {
                    Utils.WriteToLog(Utils.LogSeverity.Warning, processName,
                                     String.Format("Found a HeaderRowNumber of {0} in a file with {1} lines.  File format {2}", hr, importedFile.Count, ff.FileFormatId));
                }
                else
                {
                    // Fix any abnormal characters, that Excel will have magic-ed away from us
                    // during the DefineUFL phase
                    // Treat raw file contnets and db contents in the same manner for consistency
                    dbHeaderRowContents = ff.HeaderRowContents;
                    dbHeaderRowContents = dbHeaderRowContents.Replace("\"", "");

                    string fileHeaderRow = importedFile[hr].Replace("\"", "").Replace('"', ' ');

                    // Ensure our array is at least as big as the line we are trying to pull out

                    //comment out for production ******
                    //if (ff.FileFormatId == 446)
                    //{
                    //    Debug.WriteLine("found id");
                    //}

                    //*************************

                    if (fileHeaderRow == dbHeaderRowContents)
                    {
                        fl.FileFormatId = ff.FileFormatId;

                        return(true);
                    }
                }
            }

            return(false);
        }
예제 #12
0
        private void deleteColumns(List <string> importedFileContent, int fileFormatID, clsFileList vf)
        {
            string delimiter = string.Empty;
            string delim     = windARTDataContext.UFLDetails.Where(c => c.UFLDetailId == fileFormatID).Select(c => c.Delimitor).Single().ToString();

            if (delim == "TAB")
            {
                delimiter = "\t";
            }
            else
            {
                delimiter = ",";
            }

            List <int> GarbageCols = new List <int>();

            //get header row number from db
            int headerRow = (int)windARTDataContext.FileFormats.Where(c => c.FileFormatId == fileFormatID).Select(c => c.HeaderRowNumber).First();

            List <string> row        = new List <string>();
            string        UpdatedRow = string.Empty;

            string header = windARTDataContext.UFLDetails.Where(c => c.UFLDetailId == fileFormatID).Select(c => c.PiTagList).Single();


            List <string> headerCols = header.Split('|').ToList();

            if (importedFileContent == null)
            {
                return;
            }

            //find columns marked as N/A and change values to a missing numeric value
            //this is done because many of the N/A columns have text in them and cause the bulk insert to fail
            int count = 0;

            foreach (string s in headerCols)
            {
                if (s == "N/A")
                {
                    GarbageCols.Add(count);
                }

                count++;
            }
            if (GarbageCols.Count > 0)
            {
                for (int i = headerRow; i < importedFileContent.Count(); i++)
                {
                    row = importedFileContent[i].Split(delimiter.ToCharArray()).ToList();

                    foreach (int j in GarbageCols)
                    {
                        row[j] = "-9999.99";
                    }


                    //save updated row back to the comma delim data
                    string seperator = string.Empty;
                    foreach (string s in row)
                    {
                        seperator = delimiter;


                        UpdatedRow += s + seperator;
                    }

                    //remove trailing comma
                    UpdatedRow = UpdatedRow.Remove(UpdatedRow.Length - 1);

                    importedFileContent[i] = UpdatedRow;
                    UpdatedRow             = string.Empty;
                }
            }
        }