public ExportFileInfoList(string stageUri, int totalRows
, ExportQueryBase exportQueryBase
, ExportFileOptions exportFileOptions)
{
int intKey = 1;
// should we append date format
bool AppendDateFlag = exportFileOptions.AppendDateFormat;
int batchSize = exportFileOptions.MaxRowsPerFile;
string filePrefix = exportQueryBase.OutputFilePrefix;
if (totalRows > 0)
{
for (int intIdx = 1; intIdx <= totalRows; intIdx = intIdx + batchSize)
{
// get file name
string strDate = AppendDateFlag ? FileHelper.GetFormattedDate(exportFileOptions.FileDateFormat.ToString()) : string.Empty;
string strFileName = $"{filePrefix}{strDate}.{intKey.ToString()}{exportFileOptions.ExportToFileTypeToString}";
strFileName = System.IO.Path.Combine(stageUri, strFileName);
// get end row
int intEndRow = intIdx + batchSize - 1;
// if end row spills more than total rows then reset it to total rows.
intEndRow = (totalRows < intEndRow) ? totalRows : intEndRow;
// add export file object
this.Add(intKey, new ExportFileInfo(strFileName, intIdx, intEndRow, exportFileOptions.CompressionType));
intKey++;
}
}
}
public ExportDataResults ExportQueryOutput(ExportQueryList exportQueryList, ExportFileOptions exportFileOptions)
{
ExportDataResults exportResults = new ExportDataResults(exportQueryList.Count);
ExportDataResult exportOutcome = null;
foreach (var e in exportQueryList)
{
exportOutcome = new ExportDataResult(e, null);
try
{
exportOutcome = ExportSingleQueryOutput(e, exportFileOptions);
}
catch (Exception ex)
{
exportOutcome.ExportError = ex;
}
exportResults.Add(exportOutcome);
}
return(exportResults);
}
public ExportDataResult ExportSingleTable(ExportTable export, ExportFileOptions exportFileOptions)
{
ExportDataResult exportResult = new ExportDataResult(export, null);
// Log that validation has passed
ExportDataTable _ExportDataTable = new ExportDataTable();
_ExportDataTable.Table = SQLHelper.ExecuteDataTable(this.SQLConnectionString, export.SQLCommandText, out Exception outEx);
exportResult.ExportError = outEx;
// throw exception if we got one from SQL layer
if (outEx == null)
{
// create a file and export it
exportResult.ExportedFiles = CreateFiles(_ExportDataTable, export, exportFileOptions);
}
//ExportFileInfoList exportFileInfoList = new ExportFileInfoList(this.StagingUri, totalRows, export, exportFileOptions);
return(exportResult);
}
public MemoryStream RowsToStream(int startRow, int maxRowsInMemory, ExportFileOptions exportFileOptions)
{
string ColDelimiter = exportFileOptions.ColDelimiterString;
bool IncludeTextQualifier = exportFileOptions.IncludeTextQualifiers;
int StartRow = startRow;
int NoOfRows = maxRowsInMemory;
Encoding FileEncoding = exportFileOptions.FileEncoding;
bool IncludeHeaders = exportFileOptions.IncludeHeaders;
string Rowdelimiter = exportFileOptions.RowDelimiterString;
DataTable miniDT = null;
MemoryStream memStream = null;
StringBuilder sbData = null;
// get row count in data table
int intTotalRowCount = ((this.Table != null) && (this.Table.Rows != null)) ? this.Table.Rows.Count : 0;
// Get Rows from Data Table only if it is for a chunck of records.
if (intTotalRowCount <= NoOfRows)
{
miniDT = this.Table;
}
else
{
miniDT = GetDataTableRange(StartRow, NoOfRows);
}
if (miniDT != null)
{
memStream = new MemoryStream(1000);
StreamWriter streamWriter = new StreamWriter(memStream, FileEncoding);
try
{
// write header if needed
if (IncludeHeaders)
{
streamWriter.Write(this.Header(ColDelimiter, IncludeTextQualifier));
}
// Convert column data into CSV/Txt rows.
foreach (DataRow dr in miniDT.Rows)
{
sbData = new StringBuilder(100);
foreach (var column in dr.ItemArray)
{
sbData.Append(ColumnToString(ColDelimiter, IncludeTextQualifier, column));
}
if ((ColDelimiter.Length > 0) && (Rowdelimiter.Length > 0))
{
sbData.Replace(ColDelimiter, Rowdelimiter, sbData.Length - 1, 1);
}
// write to the stream
streamWriter.Write(sbData.ToString());
}
// flush
streamWriter.Flush();
}
catch (Exception ex)
{
// dispose memory stream
memStream.Dispose();
// dispose stream
streamWriter.Dispose();
throw ex;
}
}
// return
return(memStream);
}
private ExportFileInfoList CreateFiles(ExportDataTable exportDataTable, ExportQueryBase exportQueryBase, ExportFileOptions exportFileOptions)
{
ExportFileInfoList exportFileList = null;
if ((exportDataTable == null) || (exportDataTable?.Table == null))
{
throw new ArgumentNullException("Data Table is missing", $"Data Table to export data is null{exportQueryBase.ToString()}");
}
int intTotalRows = exportDataTable.Table.Rows.Count;
// figure out batch size
int maxRowsPerFile = exportFileOptions.SplitFiles ? exportFileOptions.MaxRowsPerFile : intTotalRows;
// Files split by Max Rows per file.
exportFileList = new ExportFileInfoList(this.StagingUri, intTotalRows, exportQueryBase, exportFileOptions);
if (exportFileList == null)
{
throw new InvalidDataException($"Export File List is null for:{exportQueryBase.ToString()}");
}
// FOR EACH FILE
foreach (var e in exportFileList)
{
// Add Encoding information if needed
// Create a new Stream
Encoding encoding = exportFileOptions.FileEncoding;
e.Value.Created = false;
e.Value.Compressed = false;
e.Value.Exported = false;
// Split the file writing into chunks in case it is too big.
int maxRowsInMemory = exportFileOptions.MaxRowsInMemory;
if (e.Value.NoOfRows > maxRowsInMemory)
{
for (int intChunkStart = 1; intChunkStart <= e.Value.NoOfRows;)
{
// Rows to be added for each file.
using (MemoryStream memStream = exportDataTable.RowsToStream(intChunkStart, maxRowsInMemory, exportFileOptions))
{
// write to file.
FileHelper.CreateFile(e.Value.FileName, memStream, exportFileOptions.CompressionType.ToString());
}
// next set
intChunkStart += maxRowsInMemory;
}
}
else
{
// Add Header for each file if needed
// Rows to be added for each file.
using (MemoryStream memStream = exportDataTable.RowsToStream(e.Value.StartRow, e.Value.NoOfRows, exportFileOptions))
{
// write to file.
FileHelper.CreateFile(e.Value.FileName, memStream, exportFileOptions.CompressionType.ToString());
}
}
if (FileHelper.CheckFileExists(e.Value.OutputFileName))
{
e.Value.Created = true;
e.Value.Compressed = (exportFileOptions.CompressionType != FileCompressionTypeEnum.None);
}
else
{
e.Value.FileExportError = new FileNotFoundException($"File didnt get created! {e.Value.FileName}");
}
bool CopyResultFlag = false;
e.Value.ExportedFileName = string.Empty;
// if file exists then export it
if ((e.Value.Created) && (this.DestinationUri.Length > 0))
{
try
{
CopyResultFlag = FileHelper.CopyFileTo(e.Value.OutputFileName, this.DestinationUri);
}
catch (Exception ex)
{
e.Value.FileExportError = ex;
CopyResultFlag = false;
}
if (CopyResultFlag)
{
string strUri = (string.IsNullOrEmpty(this.StagingUri)) ? "" : this.StagingUri;
e.Value.ExportedFileName = $"{this.DestinationUri}{e.Value.OutputFileName.Replace(strUri, "")}";
e.Value.Exported = CopyResultFlag;
}
}
}
// return file list and status
return(exportFileList);
}