/// <summary>
        /// Take a CSV file and chop it into multiple chunks of a specified maximum size.
        /// </summary>
        /// <param name="filename">The input filename to chop</param>
        /// <param name="out_folder">The folder where the chopped CSV will be saved</param>
        /// <param name="maxLinesPerFile">The maximum number of lines to put into each file</param>
        /// <param name="settings">The CSV settings to use when chopping this file into chunks (Default: CSV)</param>
        /// <returns>Number of files chopped</returns>
        public static int ChopFile(string filename, string out_folder, int maxLinesPerFile, CSVSettings settings = null)
        {
            // Default settings
            if (settings == null)
            {
                settings = CSVSettings.CSV;
            }

            // Let's begin parsing
            int          file_id     = 1;
            int          line_count  = 0;
            string       file_prefix = Path.GetFileNameWithoutExtension(filename);
            string       ext         = Path.GetExtension(filename);
            CSVWriter    cw          = null;
            StreamWriter sw          = null;

            // Read in lines from the file
            using (var sr = new StreamReader(filename))
            {
                using (CSVReader cr = new CSVReader(sr, settings))
                {
                    // Okay, let's do the real work
                    foreach (string[] line in cr.Lines())
                    {
                        // Do we need to create a file for writing?
                        if (cw == null)
                        {
                            string fn = Path.Combine(out_folder, file_prefix + file_id.ToString() + ext);
                            sw = new StreamWriter(fn);
                            cw = new CSVWriter(sw, settings);
                            if (settings.HeaderRowIncluded)
                            {
                                cw.WriteLine(cr.Headers);
                            }
                        }

                        // Write one line
                        cw.WriteLine(line);

                        // Count lines - close the file if done
                        line_count++;
                        if (line_count >= maxLinesPerFile)
                        {
                            cw.Dispose();
                            cw = null;
                            file_id++;
                            line_count = 0;
                        }
                    }
                }
            }

            // Ensore the final CSVWriter is closed properly
            if (cw != null)
            {
                cw.Dispose();
                cw = null;
            }
            return(file_id);
        }
Beispiel #2
0
        /// <summary>
        /// Take a CSV file and chop it into multiple chunks of a specified maximum size.
        /// </summary>
        /// <param name="filename"></param>
        /// <param name="out_folder"></param>
        /// <param name="first_row_are_headers"></param>
        /// <param name="max_lines_per_file"></param>
        /// <returns>Number of files chopped</returns>
        public static int ChopFile(string filename, string out_folder, bool first_row_are_headers, int max_lines_per_file, char delim = CSV.DEFAULT_DELIMITER, char qual = CSV.DEFAULT_QUALIFIER)
        {
            int       file_id     = 1;
            int       line_count  = 0;
            string    file_prefix = Path.GetFileNameWithoutExtension(filename);
            string    ext         = Path.GetExtension(filename);
            CSVWriter cw          = null;

            // Read in lines from the file
            using (CSVReader cr = new CSVReader(filename, delim, qual, first_row_are_headers)) {
                // Okay, let's do the real work
                foreach (string[] line in cr.Lines())
                {
                    // Do we need to create a file for writing?
                    if (cw == null)
                    {
                        string fn = Path.Combine(out_folder, file_prefix + file_id.ToString() + ext);
                        cw = new CSVWriter(fn, delim, qual);
                        if (first_row_are_headers)
                        {
                            cw.WriteLine(cr.Headers);
                        }
                    }

                    // Write one line
                    cw.WriteLine(line);

                    // Count lines - close the file if done
                    line_count++;
                    if (line_count >= max_lines_per_file)
                    {
                        cw.Dispose();
                        cw = null;
                        file_id++;
                        line_count = 0;
                    }
                }
            }

            // Ensore the final CSVWriter is closed properly
            if (cw != null)
            {
                cw.Dispose();
                cw = null;
            }
            return(file_id);
        }