/// <summary>
/// Opens an file for writing
/// </summary>
/// <param name="path">The path.</param>
/// <remarks>
/// There could be one steps a) in case its to be uploaded a temp file will be created
/// </remarks>
public static ImprovedStream OpenWrite(string path, IProcessDisplay processDisplay = null, string recipient = null)
{
var retVal = new ImprovedStream()
{
WritePath = path.RemovePrefix(),
ProcessDisplay = processDisplay,
Recipient = recipient
};
if (path.AssumePgp() || path.AssumeGZip())
{
Log.Debug("Creating temporary file");
retVal.TempFile = Path.GetTempFileName();
// download the file to a temp file
retVal.BaseStream = File.Create(retVal.TempFile);
}
else
{
FileSystemUtils.FileDelete(path.LongPathPrefix());
retVal.BaseStream = File.Create(path.LongPathPrefix());
}
retVal.Stream = retVal.BaseStream;
return(retVal);
}
/// <summary>
/// Adds the tree data node with child's.
/// </summary>
/// <param name="root">The root.</param>
/// <param name="rootNode">The root node.</param>
/// <param name="process">Progress display</param>
private void AddTreeDataNodeWithChild([NotNull] TreeData root, [CanBeNull] TreeNode rootNode,
[NotNull] IProcessDisplay process)
{
if (process == null)
{
throw new ArgumentNullException(nameof(process));
}
root.Visited = true;
var treeNode = new TreeNode(root.NodeTitle)
{
Tag = root
};
if (rootNode == null)
{
m_TreeView.Nodes.Add(treeNode);
}
else
{
rootNode.Nodes.Add(treeNode);
}
if (root.Children.Count > 0)
{
treeNode.Nodes.AddRange(BuildSubNodes(root, process));
}
}
public CsvFileWriter([NotNull] string id, [NotNull] string fullPath, bool hasFieldHeader, [CanBeNull] IValueFormat valueFormat = null, [CanBeNull] IFileFormat fileFormat = null,
int codePageId = 65001, bool byteOrderMark = true, [CanBeNull] string fileSettingDisplay = null, [CanBeNull] IEnumerable <IColumn> columnDefinition = null, [CanBeNull] string recipient = null,
bool unencyrpted = false, [CanBeNull] string identifierInContainer = null, [CanBeNull] string header = null, [CanBeNull] string footer = null,
[CanBeNull] IProcessDisplay processDisplay = null)
: base(id, fullPath, hasFieldHeader, valueFormat, fileFormat, recipient, unencyrpted, identifierInContainer, footer, header, columnDefinition, fileSettingDisplay, processDisplay)
{
m_CodePageId = codePageId;
m_ByteOrderMark = byteOrderMark;
m_FieldQualifier = fileFormat.FieldQualifierChar.ToStringHandle0();
m_FieldDelimiter = fileFormat.FieldDelimiterChar.ToStringHandle0();
if (fileFormat.EscapeChar != '\0')
{
m_QualifyCharArray = new[] { (char)0x0a, (char)0x0d };
m_FieldQualifierEscaped = fileFormat.EscapeChar + m_FieldQualifier;
m_FieldDelimiterEscaped = fileFormat.EscapeChar + m_FieldDelimiter;
}
else
{
// Delimiters are not escaped
m_FieldDelimiterEscaped = m_FieldDelimiter;
// but require quoting
m_QualifyCharArray = new[] { (char)0x0a, (char)0x0d, fileFormat.FieldDelimiterChar };
// the Qualifier is repeated to so it can be recognized as not to be end the quoting
m_FieldQualifierEscaped = m_FieldQualifier + m_FieldQualifier;
}
}
/// <summary>
/// Builds the sub nodes.
/// </summary>
/// <param name="parent">The parent ID.</param>
/// <param name="process">Progress display</param>
/// <returns></returns>
private TreeNode[] BuildSubNodes([NotNull] TreeData parent, IProcessDisplay process)
{
if (process == null)
{
throw new ArgumentNullException(nameof(process));
}
var treeNodes = new List <TreeNode>();
foreach (var child in parent.Children)
{
process.CancellationToken.ThrowIfCancellationRequested();
Extensions.ProcessUIElements();
if (child.Visited)
{
var treeNode = new TreeNode("Cycle -> " + child.Title)
{
Tag = child
};
treeNodes.Add(treeNode);
}
else
{
child.Visited = true;
var treeNode = new TreeNode(child.NodeTitle, BuildSubNodes(child, process))
{
Tag = child
};
treeNodes.Add(treeNode);
}
}
return(treeNodes.ToArray());
}
private static IFileWriter DefaultFileWriter([NotNull] IFileSettingPhysicalFile physicalFile,
[CanBeNull] IProcessDisplay processDisplay)
{
IFileWriter writer = null;
switch (physicalFile)
{
case ICsvFile csv when !csv.JsonFormat:
writer = new CsvFileWriter(csv, processDisplay);
break;
case StructuredFile structuredFile:
writer = new StructuredFileWriter(structuredFile, processDisplay);
break;
}
if (writer == null)
{
throw new NotImplementedException($"Writer for {physicalFile} not found");
}
writer.WriteFinished += (sender, args) =>
{
physicalFile.ProcessTimeUtc = DateTime.UtcNow;
if (physicalFile.SetLatestSourceTimeForWrite)
{
new FileSystemUtils.FileInfo(physicalFile.FullPath).LastWriteTimeUtc = physicalFile.LatestSourceTimeUtc;
}
};
return(writer);
}
private async Task GetBatchByTimeSpan(TimeSpan maxDuration, bool restoreError,
[NotNull] IProcessDisplay processDisplay, Action <DataTable> action)
{
if (m_DataReaderWrapper == null)
{
return;
}
if (processDisplay == null)
{
throw new ArgumentNullException(nameof(processDisplay));
}
try
{
processDisplay.SetMaximum(100);
var dt = await m_DataReaderWrapper.LoadDataTable(maxDuration, restoreError,
(l, i) => processDisplay.SetProcess($"Reading data...\nRecord: {l:N0}", i, false),
processDisplay.CancellationToken).ConfigureAwait(false);
action.Invoke(dt);
await m_RefreshDisplayAsync(FilterType.All, processDisplay.CancellationToken)
.ConfigureAwait(false);
if (m_DataReaderWrapper.EndOfFile)
{
Dispose();
}
}
catch (Exception e)
{
Logger.Error(e, nameof(TwoStepDataTableLoader) + "." + nameof(GetBatchByTimeSpan));
}
}
public static void SetMaximum([CanBeNull] this IProcessDisplay processDisplay, long maximum)
{
if (processDisplay is IProcessDisplayTime processDisplayTime)
{
processDisplayTime.Maximum = maximum;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="StructuredFileWriter" /> class.
/// </summary>
/// <param name="file">The file.</param>
/// <param name="processDisplay">The process display.</param>
public StructuredFileWriter([NotNull] StructuredFile file,
[CanBeNull] IProcessDisplay processDisplay)
: base(file, processDisplay)
{
m_Row = file.Row;
m_XMLEncode = file.XMLEncode;
m_JSONEncode = file.JSONEncode;
}
private static IFileReader DefaultFileReader([NotNull] IFileSetting setting, [CanBeNull] string timeZone,
[CanBeNull] IProcessDisplay processDisplay)
{
switch (setting)
{
case ICsvFile csv when csv.JsonFormat:
return(new JsonFileReader(csv, processDisplay));
case ICsvFile csv:
return(new CsvFileReader(csv, processDisplay));
default:
throw new NotImplementedException($"Reader for {setting} not found");
}
}
/// <summary>
/// Copy a file locally and provide progress
/// </summary>
/// <param name="sourceFile">The file to be copied from</param>
/// <param name="destFile">The file to be created / overwritten</param>
/// <param name="onlyChanged">
/// Checks if the source file is newer or has a different length, if not file will not be copied,
/// </param>
/// <param name="processDisplay">A process display</param>
public static async Task FileCopy([NotNull] string sourceFile, [NotNull] string destFile, bool onlyChanged,
[NotNull] IProcessDisplay processDisplay)
{
if (onlyChanged)
{
var fiSource = new FileInfo(sourceFile);
var fiDestInfo = new FileInfo(destFile);
if (fiDestInfo.Exists && fiSource.LastWriteTimeUtc <= fiDestInfo.LastWriteTimeUtc &&
fiSource.Length == fiDestInfo.Length)
{
return;
}
}
if (FileExists(sourceFile))
{
FileDelete(destFile);
}
using (var fromStream = OpenRead(sourceFile))
using (var toStream = OpenWrite(destFile))
{
var bytes = new byte[81920];
int bytesRead;
long totalReads = 0;
long oldMax = 0;
if (processDisplay is IProcessDisplayTime processDisplayTime)
{
oldMax = processDisplayTime.Maximum;
processDisplayTime.Maximum = fromStream.Length;
}
var intervalAction = new IntervalAction();
while ((bytesRead = await fromStream.ReadAsync(bytes, 0, bytes.Length, processDisplay.CancellationToken)
.ConfigureAwait(false)) > 0)
{
processDisplay.CancellationToken.ThrowIfCancellationRequested();
totalReads += bytesRead;
await toStream.WriteAsync(bytes, 0, bytesRead, processDisplay.CancellationToken).ConfigureAwait(false);
intervalAction.Invoke(pos => processDisplay.SetProcess("Copy file", pos, false), totalReads);
}
processDisplay.SetMaximum(oldMax);
}
}
/// <summary>
/// Refreshes the settings assuming the file has changed, checks CodePage, Delimiter, Start Row and Header
/// </summary>
/// <param name="file">The file.</param>
/// <param name="display">The display.</param>
public static void RefreshCsvFile(this ICsvFile file, IProcessDisplay display)
{
Contract.Requires(file != null);
Contract.Requires(display != null);
var root = ApplicationSetting.ToolSetting.RootFolder;
file.FileName.GetAbsolutePath(root);
display.SetProcess("Checking delimited file");
GuessCodePage(file);
if (display.CancellationToken.IsCancellationRequested)
{
return;
}
display.SetProcess("Code Page: " +
EncodingHelper.GetEncodingName(file.CurrentEncoding.CodePage, true, file.ByteOrderMark));
file.FileFormat.FieldDelimiter = GuessDelimiter(file);
if (display.CancellationToken.IsCancellationRequested)
{
return;
}
display.SetProcess("Delimiter: " + file.FileFormat.FieldDelimiter);
file.SkipRows = GuessStartRow(file);
if (display.CancellationToken.IsCancellationRequested)
{
return;
}
if (file.SkipRows > 0)
{
display.SetProcess("Start Row: " + file.SkipRows.ToString(CultureInfo.InvariantCulture));
}
file.HasFieldHeader = GuessHasHeader(file, display.CancellationToken);
display.SetProcess("Column Header: " + file.HasFieldHeader);
}
public async Task StartAsync([NotNull] IFileSetting fileSetting, bool includeError, TimeSpan durationInitial,
[NotNull] IProcessDisplay processDisplay, [CanBeNull] EventHandler <WarningEventArgs> addWarning)
{
m_FileReader = FunctionalDI.GetFileReader(fileSetting, TimeZoneInfo.Local.Id, processDisplay);
if (m_FileReader == null)
{
throw new FileReaderException($"Could not get reader for {fileSetting}");
}
RowErrorCollection warningList = null;
if (addWarning != null)
{
warningList = new RowErrorCollection(m_FileReader);
m_FileReader.Warning += addWarning;
m_FileReader.Warning -= warningList.Add;
}
Logger.Information("Reading data for display");
await m_FileReader.OpenAsync(processDisplay.CancellationToken).ConfigureAwait(false);
if (addWarning != null)
{
warningList.HandleIgnoredColumns(m_FileReader);
warningList.PassWarning += addWarning;
}
m_DataReaderWrapper = new DataReaderWrapper(m_FileReader, fileSetting.RecordLimit, includeError,
fileSetting.DisplayStartLineNo, fileSetting.DisplayRecordNo, fileSetting.DisplayEndLineNo);
m_ActionBegin?.Invoke();
await GetBatchByTimeSpan(durationInitial, includeError, processDisplay, m_SetDataTable).ConfigureAwait(false);
m_SetLoadNextBatchAsync?.Invoke(process => GetBatchByTimeSpan(TimeSpan.MaxValue, includeError, process, (dt) => m_GetDataTable().Merge(dt)));
m_ActionFinished?.Invoke(m_DataReaderWrapper);
}
/// <summary>
/// Gets the column header of a file
/// </summary>
/// <param name="fileSetting">The file setting.</param>
/// <param name="processDisplay">The get process display.</param>
/// <returns>
/// An array of string with the column headers where the column is empty
/// </returns>
public static string[] GetEmptyColumnHeader(IFileSetting fileSetting, IProcessDisplay processDisplay)
{
Contract.Requires(fileSetting != null);
Contract.Requires(processDisplay != null);
Contract.Ensures(Contract.Result <string[]>() != null);
var emptyColumns = new List <string>();
if (!fileSetting.HasFieldHeader)
{
return(emptyColumns.ToArray());
}
using (var fileReader = fileSetting.GetFileReader())
{
Contract.Assume(fileReader != null);
fileReader.ProcessDisplay = processDisplay;
fileReader.Open(true, processDisplay.CancellationToken);
if (fileSetting is CsvFile)
{
for (var column = 0; column < fileReader.FieldCount; column++)
{
var col = fileReader.GetColumn(column);
if (col.Size == 0)
{
emptyColumns.Add(col.Name);
}
}
}
else
{
var columnHasData = new HashSet <int>();
for (var row = 0; row < 2000 && fileReader.Read(); row++)
{
for (var column = 0; column < fileReader.FieldCount; column++)
{
if (columnHasData.Contains(column))
{
continue;
}
if (fileReader[column].ToString().Length > 0)
{
columnHasData.Add(column);
}
}
if (columnHasData.Count == fileReader.FieldCount)
{
break;
}
}
for (var column = 0; column < fileReader.FieldCount; column++)
{
if (!columnHasData.Contains(column))
{
emptyColumns.Add(fileReader.GetName(column));
}
}
}
}
return(emptyColumns.ToArray());
}
/// <summary>
/// Fills the Column Format for reader fileSettings
/// </summary>
/// <param name="fileSetting">The file setting to check, and fill</param>
/// <param name="addTextColumns">if set to <c>true</c> event string columns are added.</param>
/// <param name="processDisplay">The process display.</param>
public static IList <string> FillGuessColumnFormatReader(this IFileSetting fileSetting, bool addTextColumns,
IProcessDisplay processDisplay)
{
if (processDisplay == null)
{
throw new ArgumentNullException(nameof(processDisplay));
}
Contract.Requires(fileSetting != null);
var result = new List <string>();
// if we should not detect, we can finish
if (!ApplicationSetting.FillGuessSettings.DetectBoolean && !ApplicationSetting.FillGuessSettings.DetectGUID &&
!ApplicationSetting.FillGuessSettings.DectectNumbers &&
!ApplicationSetting.FillGuessSettings.DetectDateTime &&
!ApplicationSetting.FillGuessSettings.DectectPercentage &&
!ApplicationSetting.FillGuessSettings.SerialDateTime)
{
return(result);
}
var resetSkipRows = false;
try
{
// Make sure that if we do have a CSV file without header that we will skip the first row that
// might contain headers, but its simply set as without headers.
if (fileSetting is CsvFile && !fileSetting.HasFieldHeader && fileSetting.SkipRows == 0)
{
fileSetting.SkipRows = 1;
resetSkipRows = true;
}
var othersValueFormatDate = CommonDateFormat(fileSetting.Column.Select(x => x.ValueFormat));
using (var fileReader = fileSetting.GetFileReader())
{
Contract.Assume(fileReader != null);
// fileReader.ProcessDisplay = processDisplay;
fileReader.Open(false, processDisplay.CancellationToken);
if (fileReader.FieldCount == 0 || fileReader.EndOfFile)
{
return(result);
}
processDisplay.SetProcess("Getting column headers");
processDisplay.Maximum = fileReader.FieldCount;
var columnNamesInFile = new List <string>();
for (var colindex = 0; colindex < fileReader.FieldCount; colindex++)
{
var newColumn = fileReader.GetColumn(colindex);
Contract.Assume(newColumn != null);
columnNamesInFile.Add(newColumn.Name);
var oldColumn = fileSetting.GetColumn(newColumn.Name);
processDisplay.SetProcess(newColumn.Name + " – Getting values", colindex);
var samples = GetSampleValues(fileReader, ApplicationSetting.FillGuessSettings.CheckedRecords,
colindex, ApplicationSetting.FillGuessSettings.SampleValues, fileSetting.TreatTextAsNull,
processDisplay.CancellationToken);
if (samples.IsEmpty())
{
processDisplay.SetProcess(newColumn.Name + " – No values found", colindex);
if (!addTextColumns)
{
continue;
}
result.Add($"{newColumn.Name} – No values found – Format : {newColumn.GetTypeAndFormatDescription()}");
fileSetting.ColumnAdd(newColumn);
}
else
{
var detect = !(ApplicationSetting.FillGuessSettings.IgnoreIdColums &&
StringUtils.AssumeIDColumn(newColumn.Name) > 0);
if (samples.Count < 10)
{
processDisplay.SetProcess($"{newColumn.Name} – Only {samples.Count} values found in {ApplicationSetting.FillGuessSettings.CheckedRecords} rows", colindex);
}
else
{
processDisplay.SetProcess($"{newColumn.Name} – {samples.Count} values found – Examining format", colindex);
}
var checkResult = GuessValueFormat(samples, ApplicationSetting.FillGuessSettings.MinSamplesForIntDate,
ApplicationSetting.FillGuessSettings.TrueValue,
ApplicationSetting.FillGuessSettings.FalseValue,
ApplicationSetting.FillGuessSettings.DetectBoolean && detect,
ApplicationSetting.FillGuessSettings.DetectGUID && detect,
ApplicationSetting.FillGuessSettings.DectectNumbers && detect,
ApplicationSetting.FillGuessSettings.DetectDateTime && detect,
ApplicationSetting.FillGuessSettings.DectectPercentage && detect,
ApplicationSetting.FillGuessSettings.SerialDateTime && detect,
ApplicationSetting.FillGuessSettings.CheckNamedDates && detect,
othersValueFormatDate,
processDisplay.CancellationToken);
if (checkResult == null)
{
if (addTextColumns)
{
checkResult = new CheckResult {
FoundValueFormat = new ValueFormat()
}
}
;
else
{
continue;
}
}
// if we have a mapping to a template that expects a integer and we only have integers but not enough...
if (oldColumn != null)
{
var oldValueFormat = oldColumn.GetTypeAndFormatDescription();
// if we have a date value format already store this
if (othersValueFormatDate == null && checkResult.FoundValueFormat.DataType == DataType.DateTime && checkResult.PossibleMatch)
{
othersValueFormatDate = checkResult.FoundValueFormat;
}
if (checkResult.FoundValueFormat.Equals(oldColumn.ValueFormat))
{
processDisplay.SetProcess($"{newColumn.Name} – Format : {oldValueFormat} – not changed",
colindex);
}
else
{
oldColumn.ValueFormat = checkResult.FoundValueFormat;
}
var newValueFormat = checkResult.FoundValueFormat.GetTypeAndFormatDescription();
if (oldValueFormat.Equals(newValueFormat, StringComparison.Ordinal))
{
continue;
}
var msg = $"{newColumn.Name} – Format : {newValueFormat} – updated from {oldValueFormat}";
result.Add(msg);
processDisplay.SetProcess(msg, colindex);
}
else
{
if (!addTextColumns && checkResult.FoundValueFormat.DataType == DataType.String)
{
continue;
}
newColumn.ValueFormat = checkResult.FoundValueFormat;
var msg = $"{newColumn.Name} – Format : {newColumn.GetTypeAndFormatDescription()}";
processDisplay.SetProcess(msg, colindex);
result.Add(msg);
fileSetting.ColumnAdd(newColumn);
}
}
}
// The fileReader does not have the column information yet, let the reader know
fileReader.OverrideColumnFormatFromSetting(fileReader.FieldCount);
// in case its Excel, check all doubles if they could be integer
if (fileSetting is IExcelFile)
{
for (var colindex = 0; colindex < fileReader.FieldCount; colindex++)
{
var oldColumn = fileReader.GetColumn(colindex);
var detect = !(ApplicationSetting.FillGuessSettings.IgnoreIdColums &&
StringUtils.AssumeIDColumn(oldColumn.Name) > 0);
if (oldColumn != null && oldColumn.DataType == DataType.Double)
{
Column newColumn = null;
if (detect)
{
var samples = GetSampleValues(fileReader, ApplicationSetting.FillGuessSettings.CheckedRecords,
colindex, ApplicationSetting.FillGuessSettings.SampleValues, fileSetting.TreatTextAsNull,
processDisplay.CancellationToken);
if (!samples.IsEmpty())
{
var checkResult = GuessNumeric(samples, false, true, processDisplay.CancellationToken);
if (checkResult != null && checkResult.FoundValueFormat.DataType != DataType.Double)
{
newColumn = fileSetting.GetColumn(oldColumn.Name);
if (newColumn == null)
{
newColumn = fileSetting.ColumnAdd(oldColumn);
}
newColumn.DataType = checkResult.FoundValueFormat.DataType;
}
}
}
else
{
newColumn = fileSetting.GetColumn(oldColumn.Name);
if (newColumn == null)
{
newColumn = fileSetting.ColumnAdd(oldColumn);
}
newColumn.DataType = DataType.String;
}
if (newColumn != null)
{
var msg = $"{newColumn.Name} – Overwritten Excel Format : {newColumn.GetTypeAndFormatDescription()}";
processDisplay.SetProcess(msg, colindex);
result.Add(msg);
}
}
}
}
if (ApplicationSetting.FillGuessSettings.DateParts)
{
// Try to find a time for a date if the date does not already have a time
// Case a) TimeFormat has already been recognized
for (var colindex = 0; colindex < fileReader.FieldCount; colindex++)
{
var columnDate = fileReader.GetColumn(colindex);
// Possibly add Time Zone
if (columnDate.DataType == DataType.DateTime && string.IsNullOrEmpty(columnDate.TimeZonePart))
{
for (var coltimeZone = 0; coltimeZone < fileReader.FieldCount; coltimeZone++)
{
var columnTimeZone = fileReader.GetColumn(coltimeZone);
var colName = columnTimeZone.Name.NoSpecials().ToUpperInvariant();
if (columnTimeZone.DataType != DataType.String && columnTimeZone.DataType != DataType.Integer ||
colName != "TIMEZONE" && colName != "TIMEZONEID" && colName != "TIME ZONE" &&
colName != "TIME ZONE ID")
{
continue;
}
columnDate.TimeZonePart = columnTimeZone.Name;
result.Add($"{columnDate.Name} – Added Time Zone : {columnTimeZone.Name}");
}
}
if (columnDate.DataType != DataType.DateTime || !string.IsNullOrEmpty(columnDate.TimePart) ||
columnDate.ValueFormat.DateFormat.IndexOfAny(new[] { ':', 'h', 'H', 'm', 's', 't' }) != -1)
{
continue;
}
// We have a date column without time
for (var coltime = 0; coltime < fileReader.FieldCount; coltime++)
{
var columnTime = fileReader.GetColumn(coltime);
if (columnTime.DataType != DataType.DateTime || !string.IsNullOrEmpty(columnDate.TimePart) ||
columnTime.ValueFormat.DateFormat.IndexOfAny(new[] { '/', 'y', 'M', 'd' }) != -1)
{
continue;
}
// We now have a time column,
// checked if the names somehow make sense
if (!columnDate.Name.NoSpecials().ToUpperInvariant().Replace("DATE", string.Empty).Equals(columnTime.Name.NoSpecials().ToUpperInvariant().Replace("TIME", string.Empty), StringComparison.Ordinal))
{
continue;
}
columnDate.TimePart = columnTime.Name;
columnDate.TimePartFormat = columnTime.ValueFormat.DateFormat;
result.Add($"{columnDate.Name} – Added Time Part : {columnTime.Name}");
}
}
// Case b) TimeFormat has not been recognized (e.G. all values are 08:00) only look in adjacent fields
for (var colindex = 0; colindex < fileReader.FieldCount; colindex++)
{
var columnDate = fileReader.GetColumn(colindex);
if (columnDate.DataType != DataType.DateTime || !string.IsNullOrEmpty(columnDate.TimePart) ||
columnDate.ValueFormat.DateFormat.IndexOfAny(new[] { ':', 'h', 'H', 'm', 's', 't' }) != -1)
{
continue;
}
if (colindex + 1 < fileReader.FieldCount)
{
var columnTime = fileReader.GetColumn(colindex + 1);
if (columnTime.DataType == DataType.String && columnDate.Name.NoSpecials().ToUpperInvariant()
.Replace("DATE", string.Empty)
.Equals(columnTime.Name.NoSpecials().ToUpperInvariant().Replace("TIME", string.Empty),
StringComparison.OrdinalIgnoreCase))
{
columnDate.TimePart = columnTime.Name;
{
var samples = GetSampleValues(fileReader, 1, colindex + 1, 1, fileSetting.TreatTextAsNull,
processDisplay.CancellationToken);
var first = samples.FirstOrDefault();
if (first != null)
{
if (first.Length == 8 || first.Length == 5)
{
columnTime.DataType = DataType.DateTime;
var val = new ValueFormat(DataType.DateTime)
{
DateFormat = first.Length == 8 ? "HH:mm:ss" : "HH:mm"
};
columnTime.ValueFormat = val;
fileSetting.ColumnAdd(columnTime);
result.Add($"{columnTime.Name} – Format : {columnTime.GetTypeAndFormatDescription()}");
}
}
}
result.Add($"{columnDate.Name} – Added Time Part : {columnTime.Name}");
continue;
}
}
if (colindex <= 0)
{
continue;
}
{
var columnTime = fileReader.GetColumn(colindex - 1);
if (columnTime.DataType != DataType.String ||
!columnDate.Name.NoSpecials().ToUpperInvariant().Replace("DATE", string.Empty).Equals(columnTime.Name.NoSpecials().ToUpperInvariant().Replace("TIME", string.Empty), StringComparison.Ordinal))
{
continue;
}
columnDate.TimePart = columnTime.Name;
{
var samples = GetSampleValues(fileReader, 1, colindex - 1, 1, fileSetting.TreatTextAsNull,
processDisplay.CancellationToken);
var first = samples.FirstOrDefault();
if (first != null)
{
if (first.Length == 8 || first.Length == 5)
{
var val = new ValueFormat(DataType.DateTime)
{
DateFormat = first.Length == 8 ? "HH:mm:ss" : "HH:mm"
};
fileSetting.ColumnAdd(columnTime);
columnTime.ValueFormat = val;
result.Add($"{columnTime.Name} – Format : {columnTime.GetTypeAndFormatDescription()}");
}
}
}
result.Add($"{columnDate.Name} – Added Time Part : {columnTime.Name}");
}
}
}
// Sort the columns in fileSetting by order in file
fileSetting.SortColumnByName(columnNamesInFile);
}
}
finally
{
if (resetSkipRows)
{
fileSetting.SkipRows = 0;
}
}
return(result);
}
protected BaseFileWriter([NotNull] string id,
[NotNull] string fullPath, bool hasFieldHeader, [CanBeNull] IValueFormat valueFormatGeneral = null, [CanBeNull] IFileFormat fileFormat = null, [CanBeNull] string recipient = null,
bool unencrypted = false, [CanBeNull] string identifierInContainer = null,
[CanBeNull] string footer = null, [CanBeNull] string header = null,
[CanBeNull] IEnumerable <IColumn> columnDefinition = null, [NotNull] string fileSettingDisplay = null,
[CanBeNull] IProcessDisplay processDisplay = null)
{
m_FullPath = fullPath;
var fileName = FileSystemUtils.GetFileName(fullPath);
ColumnHeader = hasFieldHeader;
if (valueFormatGeneral != null)
{
ValueFormatGeneral = new ImmutableValueFormat(valueFormatGeneral.DataType, valueFormatGeneral.DateFormat, valueFormatGeneral.DateSeparator,
valueFormatGeneral.TimeSeparator, valueFormatGeneral.NumberFormat, valueFormatGeneral.GroupSeparator, valueFormatGeneral.DecimalSeparator, valueFormatGeneral.True,
valueFormatGeneral.False, valueFormatGeneral.DisplayNullAs);
}
else
{
ValueFormatGeneral = new ImmutableValueFormat();
}
if (fileFormat != null)
{
FileFormat = new ImmutableFileFormat(fileFormat.IsFixedLength, fileFormat.QualifyAlways,
fileFormat.QualifyOnlyIfNeeded, fileFormat.EscapeChar, fileFormat.FieldDelimiterChar, fileFormat.DelimiterPlaceholder,
fileFormat.FieldQualifierChar, fileFormat.QuotePlaceholder, fileFormat.NewLine, fileFormat.NewLinePlaceholder);
}
else
{
FileFormat = new ImmutableFileFormat();
}
ColumnDefinition = columnDefinition?.Select(col => col is ImmutableColumn immutableColumn ? immutableColumn : new ImmutableColumn(col.Name, col.ValueFormat, col.ColumnOrdinal, col.Convert, col.DestinationName, col.Ignore, col.Part, col.PartSplitter, col.PartToEnd, col.TimePart, col.TimePartFormat, col.TimeZonePart)).ToList() ??
new List <ImmutableColumn>();
NewLine = fileFormat.NewLine.NewLineString();
if (!string.IsNullOrEmpty(header))
{
Header = ReplacePlaceHolder(StringUtils.HandleCRLFCombinations(header, NewLine), fileFormat.FieldDelimiterChar.GetDescription(),
fileName, id);
}
else
{
Header = string.Empty;
}
if (!string.IsNullOrEmpty(footer))
{
m_Footer = ReplacePlaceHolder(StringUtils.HandleCRLFCombinations(footer, NewLine), fileFormat.FieldDelimiterChar.GetDescription(),
fileName, id);
}
else
{
m_Footer = string.Empty;
}
m_FileSettingDisplay = fileSettingDisplay ?? string.Empty;
m_Recipient = recipient ?? string.Empty;
m_KeepUnencrypted = unencrypted;
m_IdentifierInContainer = identifierInContainer ?? string.Empty;
Logger.Debug("Created Writer for {filesetting}", m_FileSettingDisplay);
if (processDisplay == null)
{
return;
}
m_ReportProgress = t => processDisplay.SetProcess(t, 0, true);
if (!(processDisplay is IProcessDisplayTime processDisplayTime))
{
return;
}
processDisplayTime.Maximum = 0;
m_SetMaxProcess = l => processDisplayTime.Maximum = l;
}
/// <summary>
/// Initializes a new instance of the <see cref="BaseFileWriter" /> class.
/// </summary>
/// <param name="fileSetting">the file setting with the definition for the file</param>
/// <param name="processDisplay">The process display.</param>
/// <exception cref="ArgumentNullException">fileSetting</exception>
/// <exception cref="ArgumentException">No SQL Reader set</exception>
protected BaseFileWriter([NotNull] IFileSettingPhysicalFile fileSetting, [CanBeNull] IProcessDisplay processDisplay)
: this(fileSetting.ID, fileSetting.FullPath, fileSetting.HasFieldHeader, fileSetting.FileFormat.ValueFormatMutable, fileSetting.FileFormat, fileSetting.Recipient,
fileSetting.KeepUnencrypted, fileSetting.IdentifierInContainer, fileSetting.Footer, fileSetting.Header,
fileSetting.ColumnCollection.ReadonlyCopy(), fileSetting.ToString(), processDisplay)
{
}
public JsonFileReader(IFileSettingPhysicalFile fileSetting,
IProcessDisplay processDisplay)
: this(fileSetting.FullPath,
fileSetting.ColumnCollection, fileSetting.RecordLimit,
fileSetting.TreatNBSPAsSpace) => SetProgressActions(processDisplay);
/// <summary>
/// Shows the tree.
/// </summary>
private void ShowTree(IProcessDisplay process)
{
m_TreeView.BeginUpdate();
try
{
m_TreeView.Nodes.Clear();
if (m_TreeData == null)
{
return;
}
foreach (var treeData in m_TreeData)
{
treeData.Visited = false;
}
process.SetProcess("Adding Tree with children", -1, false);
foreach (var treeData in m_TreeData)
{
process.CancellationToken.ThrowIfCancellationRequested();
Extensions.ProcessUIElements();
if (string.IsNullOrEmpty(treeData.ParentID))
{
AddTreeDataNodeWithChild(treeData, null, process);
}
}
process.SetProcess("Finding Cycles in Hierarchy", -1, true);
var hasCycles = false;
foreach (var treeData in m_TreeData)
{
process.CancellationToken.ThrowIfCancellationRequested();
Extensions.ProcessUIElements();
if (!treeData.Visited)
{
hasCycles = true;
break;
}
}
if (!hasCycles)
{
return;
}
process.SetProcess("Adding Cycles", -1, false);
var rootNode = new TreeNode("Cycles in Hierarchy");
m_TreeView.Nodes.Add(rootNode);
foreach (var treeData in m_TreeData)
{
process.CancellationToken.ThrowIfCancellationRequested();
Extensions.ProcessUIElements();
if (!treeData.Visited)
{
MarkInCycle(treeData, new HashSet <TreeData>());
}
}
foreach (var root in m_TreeData)
{
process.CancellationToken.ThrowIfCancellationRequested();
Extensions.ProcessUIElements();
if (!root.Visited && root.InCycle)
{
AddTreeDataNodeWithChild(root, rootNode, process);
}
}
}
catch
{
m_TreeView.Nodes.Clear();
}
finally
{
m_TreeView.EndUpdate();
}
}
/// <summary>
/// Gets the SQl Info messages and logs them
/// </summary>
/// <param name="process">The process display.</param>
/// <returns></returns>
public static EventHandler <string> GetLogInfoMessage(this IProcessDisplay process) =>
delegate(object sender, string message)
{
Logger.Information("SQL Information: {message}", message);
process?.SetProcess(message, -1, true);
};
/// <summary>
/// Gets the column header of a file
/// </summary>
/// <param name="fileSetting">The file setting.</param>
/// <param name="includeIgnored">Set to <c>true</c> if ignored columns should be listed as well, otherwise they will not be
/// listed</param>
/// <param name="processDisplay">The process display.</param>
/// <returns>
/// An array of string with the column headers
/// </returns>
public static ICollection <string> GetColumnHeader(IFileSetting fileSetting, bool includeIgnored, bool openIfNeeded, IProcessDisplay processDisplay)
{
Contract.Requires(fileSetting != null);
var key = CacheListKeyColumnHeader(fileSetting.ID, includeIgnored);
if (key.Length > 3 && ApplicationSetting.CacheList.TryGet(key, out var retValue))
{
return(retValue);
}
if (!openIfNeeded)
{
return(null);
}
using (var fileReader = fileSetting.GetFileReader())
{
fileReader.ProcessDisplay = processDisplay;
fileReader.Open(false, processDisplay?.CancellationToken ?? CancellationToken.None);
// if teh key was long enough it has been stored
if (key.Length > 3)
{
return(ApplicationSetting.CacheList.Get(key));
}
else
{
var header = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
for (var colindex = 0; colindex < fileReader.FieldCount; colindex++)
{
var col = fileReader.GetColumn(colindex);
if (includeIgnored || !col.Ignore)
{
header.Add(col.Name);
}
}
return(header);
}
}
}
/// <summary>
/// Builds the tree data.
/// </summary>
private void BuildTreeData(string parentCol, string idCol, string display1, string display2,
IProcessDisplay process)
{
var intervalAction = new IntervalAction();
var dataColumnParent = m_DataTable.Columns[parentCol];
if (dataColumnParent == null)
{
throw new ArgumentException($"Could not find column {parentCol}");
}
var dataColumnID = m_DataTable.Columns[idCol];
if (dataColumnID == null)
{
throw new ArgumentException($"Could not find column {idCol}");
}
var dataColumnDisplay1 = string.IsNullOrEmpty(display1) ? null : m_DataTable.Columns[display1];
var dataColumnDisplay2 = string.IsNullOrEmpty(display2) ? null : m_DataTable.Columns[display2];
// Using a dictionary here to speed up lookups
var treeDataDictionary = new Dictionary <string, TreeData>();
var rootDataParentFound = new TreeData("{R}", "Parent found / No Parent");
treeDataDictionary.Add(rootDataParentFound.ID, rootDataParentFound);
var max = 0L;
if (process is IProcessDisplayTime processDisplayTime)
{
max = processDisplayTime.Maximum;
}
var counter = 0;
foreach (var dataRow in m_DataRow)
{
process.CancellationToken.ThrowIfCancellationRequested();
intervalAction.Invoke(
count => process.SetProcess($"Parent found {count}/{max} ", count, false),
counter++);
var id = dataRow[dataColumnID.Ordinal].ToString();
if (string.IsNullOrEmpty(id))
{
continue;
}
var treeData = new TreeData(id, dataColumnDisplay1 != null ? dataColumnDisplay2 != null ? dataRow[dataColumnDisplay1.Ordinal] + " - " + dataRow[dataColumnDisplay2.Ordinal] : dataRow[dataColumnDisplay1.Ordinal].ToString() : id, dataRow[dataColumnParent.Ordinal].ToString());
if (dataColumnDisplay1 != null)
{
treeData.Tag = dataRow[dataColumnDisplay1.Ordinal].ToString();
}
// Store the display
if (!treeDataDictionary.ContainsKey(id))
{
treeDataDictionary.Add(id, treeData);
}
}
// Generate a list of missing parents
var additionalRootNodes = new HashSet <string>();
foreach (var child in treeDataDictionary.Values)
{
if (!string.IsNullOrEmpty(child.ParentID) && !treeDataDictionary.ContainsKey(child.ParentID))
{
additionalRootNodes.Add(child.ParentID);
}
}
var rootDataParentNotFound = new TreeData("{M}", "Parent not found");
if (additionalRootNodes.Count > 0)
{
treeDataDictionary.Add(rootDataParentNotFound.ID, rootDataParentNotFound);
counter = 0;
max = additionalRootNodes.Count;
process.SetMaximum(max);
// Create new entries
foreach (var parentID in additionalRootNodes)
{
process.CancellationToken.ThrowIfCancellationRequested();
intervalAction.Invoke(
count => process.SetProcess($"Parent not found (Step 1) {count}/{max} ", count, false),
counter++);
var childData = new TreeData(parentID, $"{parentID}", rootDataParentNotFound.ID);
treeDataDictionary.Add(parentID, childData);
}
}
max = treeDataDictionary.Values.Count;
process.SetMaximum(max);
counter = 0;
foreach (var child in treeDataDictionary.Values)
{
process.CancellationToken.ThrowIfCancellationRequested();
intervalAction.Invoke(
count => process.SetProcess($"Parent not found (Step 2) {count}/{max} ", count, false),
counter++);
if (string.IsNullOrEmpty(child.ParentID) && child.ID != rootDataParentFound.ID &&
child.ID != rootDataParentNotFound.ID)
{
child.ParentID = rootDataParentFound.ID;
}
}
max = treeDataDictionary.Values.Count;
process.SetMaximum(max);
counter = 0;
// Fill m_Children for the new nodes
foreach (var child in treeDataDictionary.Values)
{
process.CancellationToken.ThrowIfCancellationRequested();
intervalAction.Invoke(
count => process.SetProcess($"Set children {count}/{max} ", count, false),
counter++);
if (!string.IsNullOrEmpty(child.ParentID))
{
treeDataDictionary[child.ParentID].Children.Add(child);
}
}
m_TreeData = treeDataDictionary.Values;
}
/// <summary>
/// Initializes a new instance of the <see cref="CsvFileWriter" /> class.
/// </summary>
/// <param name="file">The file.</param>
/// <param name="processDisplay">The process display.</param>
public CsvFileWriter([NotNull] ICsvFile fileSetting, [CanBeNull] IProcessDisplay processDisplay)
: this(fileSetting.ID, fileSetting.FullPath, fileSetting.HasFieldHeader, fileSetting.FileFormat.ValueFormatMutable, fileSetting.FileFormat, fileSetting.CodePageId,
fileSetting.ByteOrderMark, fileSetting.ToString(), fileSetting.ColumnCollection.ReadonlyCopy(), fileSetting.Recipient, fileSetting.KeepUnencrypted, fileSetting.IdentifierInContainer,
fileSetting.Header, fileSetting.Footer, processDisplay)
{
}
private void CloseWrite()
{
// If we are writing we have possibly two steps,
// a) compress / encrypt the data from temp
// b) upload the data to sFTP
if (ProcessDisplay == null)
{
ProcessDisplay = new DummyProcessDisplay();
}
if (WritePath.AssumeGZip() || WritePath.AssumePgp())
{
try
{
// Compress the file
if (WritePath.AssumeGZip())
{
Log.Debug("Compressing temporary file to GZip file");
using (var inFile = File.OpenRead(TempFile))
{
// Create the compressed file.
using (var outFile = File.Create(WritePath))
{
using (var compress = new System.IO.Compression.GZipStream(outFile, System.IO.Compression.CompressionMode.Compress))
{
var processDispayTime = ProcessDisplay as IProcessDisplayTime;
var inputBuffer = new byte[16384];
var max = (int)(inFile.Length / inputBuffer.Length);
ProcessDisplay.Maximum = max;
var count = 0;
int length;
while ((length = inFile.Read(inputBuffer, 0, inputBuffer.Length)) > 0)
{
compress.Write(inputBuffer, 0, length);
ProcessDisplay.CancellationToken.ThrowIfCancellationRequested();
if (processDispayTime != null)
{
ProcessDisplay.SetProcess(
$"GZip {processDispayTime.TimeToCompletion.PercentDisplay}{processDispayTime.TimeToCompletion.EstimatedTimeRemainingDisplaySeperator}", count);
}
else
{
ProcessDisplay.SetProcess($"GZip {count:N0}/{max:N0}",
count);
}
count++;
}
}
}
}
}
// need to encrypt the file
else if (WritePath.AssumePgp())
{
using (FileStream inputStream = new FileInfo(TempFile).OpenRead(),
output = new FileStream(WritePath.LongPathPrefix(), FileMode.Create))
{
Log.Debug("Encrypting temporary file to PGP file");
ApplicationSetting.ToolSetting.PGPInformation.PgpEncrypt(inputStream, output, Recipient, ProcessDisplay);
}
}
}
finally
{
Log.Debug("Removing temporary file");
File.Delete(TempFile);
}
}
}
