/// <summary> /// Take a CSV file and chop it into multiple chunks of a specified maximum size. /// </summary> /// <param name="filename">The input filename to chop</param> /// <param name="out_folder">The folder where the chopped CSV will be saved</param> /// <param name="maxLinesPerFile">The maximum number of lines to put into each file</param> /// <param name="settings">The CSV settings to use when chopping this file into chunks (Default: CSV)</param> /// <returns>Number of files chopped</returns> public static int ChopFile(string filename, string out_folder, int maxLinesPerFile, CSVSettings settings = null) { // Default settings if (settings == null) { settings = CSVSettings.CSV; } // Let's begin parsing int file_id = 1; int line_count = 0; string file_prefix = Path.GetFileNameWithoutExtension(filename); string ext = Path.GetExtension(filename); CSVWriter cw = null; StreamWriter sw = null; // Read in lines from the file using (var sr = new StreamReader(filename)) { using (CSVReader cr = new CSVReader(sr, settings)) { // Okay, let's do the real work foreach (string[] line in cr.Lines()) { // Do we need to create a file for writing? if (cw == null) { string fn = Path.Combine(out_folder, file_prefix + file_id.ToString() + ext); sw = new StreamWriter(fn); cw = new CSVWriter(sw, settings); if (settings.HeaderRowIncluded) { cw.WriteLine(cr.Headers); } } // Write one line cw.WriteLine(line); // Count lines - close the file if done line_count++; if (line_count >= maxLinesPerFile) { cw.Dispose(); cw = null; file_id++; line_count = 0; } } } } // Ensore the final CSVWriter is closed properly if (cw != null) { cw.Dispose(); cw = null; } return(file_id); }
/// <summary> /// Take a CSV file and chop it into multiple chunks of a specified maximum size. /// </summary> /// <param name="filename"></param> /// <param name="out_folder"></param> /// <param name="first_row_are_headers"></param> /// <param name="max_lines_per_file"></param> /// <returns>Number of files chopped</returns> public static int ChopFile(string filename, string out_folder, bool first_row_are_headers, int max_lines_per_file, char delim = CSV.DEFAULT_DELIMITER, char qual = CSV.DEFAULT_QUALIFIER) { int file_id = 1; int line_count = 0; string file_prefix = Path.GetFileNameWithoutExtension(filename); string ext = Path.GetExtension(filename); CSVWriter cw = null; // Read in lines from the file using (CSVReader cr = new CSVReader(filename, delim, qual, first_row_are_headers)) { // Okay, let's do the real work foreach (string[] line in cr.Lines()) { // Do we need to create a file for writing? if (cw == null) { string fn = Path.Combine(out_folder, file_prefix + file_id.ToString() + ext); cw = new CSVWriter(fn, delim, qual); if (first_row_are_headers) { cw.WriteLine(cr.Headers); } } // Write one line cw.WriteLine(line); // Count lines - close the file if done line_count++; if (line_count >= max_lines_per_file) { cw.Dispose(); cw = null; file_id++; line_count = 0; } } } // Ensore the final CSVWriter is closed properly if (cw != null) { cw.Dispose(); cw = null; } return(file_id); }