/// <summary> /// Opens an file for writing /// </summary> /// <param name="path">The path.</param> /// <remarks> /// There could be one steps a) in case its to be uploaded a temp file will be created /// </remarks> public static ImprovedStream OpenWrite(string path, IProcessDisplay processDisplay = null, string recipient = null) { var retVal = new ImprovedStream() { WritePath = path.RemovePrefix(), ProcessDisplay = processDisplay, Recipient = recipient }; if (path.AssumePgp() || path.AssumeGZip()) { Log.Debug("Creating temporary file"); retVal.TempFile = Path.GetTempFileName(); // download the file to a temp file retVal.BaseStream = File.Create(retVal.TempFile); } else { FileSystemUtils.FileDelete(path.LongPathPrefix()); retVal.BaseStream = File.Create(path.LongPathPrefix()); } retVal.Stream = retVal.BaseStream; return(retVal); }
/// <summary> /// Adds the tree data node with child's. /// </summary> /// <param name="root">The root.</param> /// <param name="rootNode">The root node.</param> /// <param name="process">Progress display</param> private void AddTreeDataNodeWithChild([NotNull] TreeData root, [CanBeNull] TreeNode rootNode, [NotNull] IProcessDisplay process) { if (process == null) { throw new ArgumentNullException(nameof(process)); } root.Visited = true; var treeNode = new TreeNode(root.NodeTitle) { Tag = root }; if (rootNode == null) { m_TreeView.Nodes.Add(treeNode); } else { rootNode.Nodes.Add(treeNode); } if (root.Children.Count > 0) { treeNode.Nodes.AddRange(BuildSubNodes(root, process)); } }
public CsvFileWriter([NotNull] string id, [NotNull] string fullPath, bool hasFieldHeader, [CanBeNull] IValueFormat valueFormat = null, [CanBeNull] IFileFormat fileFormat = null, int codePageId = 65001, bool byteOrderMark = true, [CanBeNull] string fileSettingDisplay = null, [CanBeNull] IEnumerable <IColumn> columnDefinition = null, [CanBeNull] string recipient = null, bool unencyrpted = false, [CanBeNull] string identifierInContainer = null, [CanBeNull] string header = null, [CanBeNull] string footer = null, [CanBeNull] IProcessDisplay processDisplay = null) : base(id, fullPath, hasFieldHeader, valueFormat, fileFormat, recipient, unencyrpted, identifierInContainer, footer, header, columnDefinition, fileSettingDisplay, processDisplay) { m_CodePageId = codePageId; m_ByteOrderMark = byteOrderMark; m_FieldQualifier = fileFormat.FieldQualifierChar.ToStringHandle0(); m_FieldDelimiter = fileFormat.FieldDelimiterChar.ToStringHandle0(); if (fileFormat.EscapeChar != '\0') { m_QualifyCharArray = new[] { (char)0x0a, (char)0x0d }; m_FieldQualifierEscaped = fileFormat.EscapeChar + m_FieldQualifier; m_FieldDelimiterEscaped = fileFormat.EscapeChar + m_FieldDelimiter; } else { // Delimiters are not escaped m_FieldDelimiterEscaped = m_FieldDelimiter; // but require quoting m_QualifyCharArray = new[] { (char)0x0a, (char)0x0d, fileFormat.FieldDelimiterChar }; // the Qualifier is repeated to so it can be recognized as not to be end the quoting m_FieldQualifierEscaped = m_FieldQualifier + m_FieldQualifier; } }
/// <summary> /// Builds the sub nodes. /// </summary> /// <param name="parent">The parent ID.</param> /// <param name="process">Progress display</param> /// <returns></returns> private TreeNode[] BuildSubNodes([NotNull] TreeData parent, IProcessDisplay process) { if (process == null) { throw new ArgumentNullException(nameof(process)); } var treeNodes = new List <TreeNode>(); foreach (var child in parent.Children) { process.CancellationToken.ThrowIfCancellationRequested(); Extensions.ProcessUIElements(); if (child.Visited) { var treeNode = new TreeNode("Cycle -> " + child.Title) { Tag = child }; treeNodes.Add(treeNode); } else { child.Visited = true; var treeNode = new TreeNode(child.NodeTitle, BuildSubNodes(child, process)) { Tag = child }; treeNodes.Add(treeNode); } } return(treeNodes.ToArray()); }
private static IFileWriter DefaultFileWriter([NotNull] IFileSettingPhysicalFile physicalFile, [CanBeNull] IProcessDisplay processDisplay) { IFileWriter writer = null; switch (physicalFile) { case ICsvFile csv when !csv.JsonFormat: writer = new CsvFileWriter(csv, processDisplay); break; case StructuredFile structuredFile: writer = new StructuredFileWriter(structuredFile, processDisplay); break; } if (writer == null) { throw new NotImplementedException($"Writer for {physicalFile} not found"); } writer.WriteFinished += (sender, args) => { physicalFile.ProcessTimeUtc = DateTime.UtcNow; if (physicalFile.SetLatestSourceTimeForWrite) { new FileSystemUtils.FileInfo(physicalFile.FullPath).LastWriteTimeUtc = physicalFile.LatestSourceTimeUtc; } }; return(writer); }
private async Task GetBatchByTimeSpan(TimeSpan maxDuration, bool restoreError, [NotNull] IProcessDisplay processDisplay, Action <DataTable> action) { if (m_DataReaderWrapper == null) { return; } if (processDisplay == null) { throw new ArgumentNullException(nameof(processDisplay)); } try { processDisplay.SetMaximum(100); var dt = await m_DataReaderWrapper.LoadDataTable(maxDuration, restoreError, (l, i) => processDisplay.SetProcess($"Reading data...\nRecord: {l:N0}", i, false), processDisplay.CancellationToken).ConfigureAwait(false); action.Invoke(dt); await m_RefreshDisplayAsync(FilterType.All, processDisplay.CancellationToken) .ConfigureAwait(false); if (m_DataReaderWrapper.EndOfFile) { Dispose(); } } catch (Exception e) { Logger.Error(e, nameof(TwoStepDataTableLoader) + "." + nameof(GetBatchByTimeSpan)); } }
public static void SetMaximum([CanBeNull] this IProcessDisplay processDisplay, long maximum) { if (processDisplay is IProcessDisplayTime processDisplayTime) { processDisplayTime.Maximum = maximum; } }
/// <summary> /// Initializes a new instance of the <see cref="StructuredFileWriter" /> class. /// </summary> /// <param name="file">The file.</param> /// <param name="processDisplay">The process display.</param> public StructuredFileWriter([NotNull] StructuredFile file, [CanBeNull] IProcessDisplay processDisplay) : base(file, processDisplay) { m_Row = file.Row; m_XMLEncode = file.XMLEncode; m_JSONEncode = file.JSONEncode; }
private static IFileReader DefaultFileReader([NotNull] IFileSetting setting, [CanBeNull] string timeZone, [CanBeNull] IProcessDisplay processDisplay) { switch (setting) { case ICsvFile csv when csv.JsonFormat: return(new JsonFileReader(csv, processDisplay)); case ICsvFile csv: return(new CsvFileReader(csv, processDisplay)); default: throw new NotImplementedException($"Reader for {setting} not found"); } }
/// <summary> /// Copy a file locally and provide progress /// </summary> /// <param name="sourceFile">The file to be copied from</param> /// <param name="destFile">The file to be created / overwritten</param> /// <param name="onlyChanged"> /// Checks if the source file is newer or has a different length, if not file will not be copied, /// </param> /// <param name="processDisplay">A process display</param> public static async Task FileCopy([NotNull] string sourceFile, [NotNull] string destFile, bool onlyChanged, [NotNull] IProcessDisplay processDisplay) { if (onlyChanged) { var fiSource = new FileInfo(sourceFile); var fiDestInfo = new FileInfo(destFile); if (fiDestInfo.Exists && fiSource.LastWriteTimeUtc <= fiDestInfo.LastWriteTimeUtc && fiSource.Length == fiDestInfo.Length) { return; } } if (FileExists(sourceFile)) { FileDelete(destFile); } using (var fromStream = OpenRead(sourceFile)) using (var toStream = OpenWrite(destFile)) { var bytes = new byte[81920]; int bytesRead; long totalReads = 0; long oldMax = 0; if (processDisplay is IProcessDisplayTime processDisplayTime) { oldMax = processDisplayTime.Maximum; processDisplayTime.Maximum = fromStream.Length; } var intervalAction = new IntervalAction(); while ((bytesRead = await fromStream.ReadAsync(bytes, 0, bytes.Length, processDisplay.CancellationToken) .ConfigureAwait(false)) > 0) { processDisplay.CancellationToken.ThrowIfCancellationRequested(); totalReads += bytesRead; await toStream.WriteAsync(bytes, 0, bytesRead, processDisplay.CancellationToken).ConfigureAwait(false); intervalAction.Invoke(pos => processDisplay.SetProcess("Copy file", pos, false), totalReads); } processDisplay.SetMaximum(oldMax); } }
/// <summary> /// Refreshes the settings assuming the file has changed, checks CodePage, Delimiter, Start Row and Header /// </summary> /// <param name="file">The file.</param> /// <param name="display">The display.</param> public static void RefreshCsvFile(this ICsvFile file, IProcessDisplay display) { Contract.Requires(file != null); Contract.Requires(display != null); var root = ApplicationSetting.ToolSetting.RootFolder; file.FileName.GetAbsolutePath(root); display.SetProcess("Checking delimited file"); GuessCodePage(file); if (display.CancellationToken.IsCancellationRequested) { return; } display.SetProcess("Code Page: " + EncodingHelper.GetEncodingName(file.CurrentEncoding.CodePage, true, file.ByteOrderMark)); file.FileFormat.FieldDelimiter = GuessDelimiter(file); if (display.CancellationToken.IsCancellationRequested) { return; } display.SetProcess("Delimiter: " + file.FileFormat.FieldDelimiter); file.SkipRows = GuessStartRow(file); if (display.CancellationToken.IsCancellationRequested) { return; } if (file.SkipRows > 0) { display.SetProcess("Start Row: " + file.SkipRows.ToString(CultureInfo.InvariantCulture)); } file.HasFieldHeader = GuessHasHeader(file, display.CancellationToken); display.SetProcess("Column Header: " + file.HasFieldHeader); }
public async Task StartAsync([NotNull] IFileSetting fileSetting, bool includeError, TimeSpan durationInitial, [NotNull] IProcessDisplay processDisplay, [CanBeNull] EventHandler <WarningEventArgs> addWarning) { m_FileReader = FunctionalDI.GetFileReader(fileSetting, TimeZoneInfo.Local.Id, processDisplay); if (m_FileReader == null) { throw new FileReaderException($"Could not get reader for {fileSetting}"); } RowErrorCollection warningList = null; if (addWarning != null) { warningList = new RowErrorCollection(m_FileReader); m_FileReader.Warning += addWarning; m_FileReader.Warning -= warningList.Add; } Logger.Information("Reading data for display"); await m_FileReader.OpenAsync(processDisplay.CancellationToken).ConfigureAwait(false); if (addWarning != null) { warningList.HandleIgnoredColumns(m_FileReader); warningList.PassWarning += addWarning; } m_DataReaderWrapper = new DataReaderWrapper(m_FileReader, fileSetting.RecordLimit, includeError, fileSetting.DisplayStartLineNo, fileSetting.DisplayRecordNo, fileSetting.DisplayEndLineNo); m_ActionBegin?.Invoke(); await GetBatchByTimeSpan(durationInitial, includeError, processDisplay, m_SetDataTable).ConfigureAwait(false); m_SetLoadNextBatchAsync?.Invoke(process => GetBatchByTimeSpan(TimeSpan.MaxValue, includeError, process, (dt) => m_GetDataTable().Merge(dt))); m_ActionFinished?.Invoke(m_DataReaderWrapper); }
/// <summary> /// Gets the column header of a file /// </summary> /// <param name="fileSetting">The file setting.</param> /// <param name="processDisplay">The get process display.</param> /// <returns> /// An array of string with the column headers where the column is empty /// </returns> public static string[] GetEmptyColumnHeader(IFileSetting fileSetting, IProcessDisplay processDisplay) { Contract.Requires(fileSetting != null); Contract.Requires(processDisplay != null); Contract.Ensures(Contract.Result <string[]>() != null); var emptyColumns = new List <string>(); if (!fileSetting.HasFieldHeader) { return(emptyColumns.ToArray()); } using (var fileReader = fileSetting.GetFileReader()) { Contract.Assume(fileReader != null); fileReader.ProcessDisplay = processDisplay; fileReader.Open(true, processDisplay.CancellationToken); if (fileSetting is CsvFile) { for (var column = 0; column < fileReader.FieldCount; column++) { var col = fileReader.GetColumn(column); if (col.Size == 0) { emptyColumns.Add(col.Name); } } } else { var columnHasData = new HashSet <int>(); for (var row = 0; row < 2000 && fileReader.Read(); row++) { for (var column = 0; column < fileReader.FieldCount; column++) { if (columnHasData.Contains(column)) { continue; } if (fileReader[column].ToString().Length > 0) { columnHasData.Add(column); } } if (columnHasData.Count == fileReader.FieldCount) { break; } } for (var column = 0; column < fileReader.FieldCount; column++) { if (!columnHasData.Contains(column)) { emptyColumns.Add(fileReader.GetName(column)); } } } } return(emptyColumns.ToArray()); }
/// <summary> /// Fills the Column Format for reader fileSettings /// </summary> /// <param name="fileSetting">The file setting to check, and fill</param> /// <param name="addTextColumns">if set to <c>true</c> event string columns are added.</param> /// <param name="processDisplay">The process display.</param> public static IList <string> FillGuessColumnFormatReader(this IFileSetting fileSetting, bool addTextColumns, IProcessDisplay processDisplay) { if (processDisplay == null) { throw new ArgumentNullException(nameof(processDisplay)); } Contract.Requires(fileSetting != null); var result = new List <string>(); // if we should not detect, we can finish if (!ApplicationSetting.FillGuessSettings.DetectBoolean && !ApplicationSetting.FillGuessSettings.DetectGUID && !ApplicationSetting.FillGuessSettings.DectectNumbers && !ApplicationSetting.FillGuessSettings.DetectDateTime && !ApplicationSetting.FillGuessSettings.DectectPercentage && !ApplicationSetting.FillGuessSettings.SerialDateTime) { return(result); } var resetSkipRows = false; try { // Make sure that if we do have a CSV file without header that we will skip the first row that // might contain headers, but its simply set as without headers. if (fileSetting is CsvFile && !fileSetting.HasFieldHeader && fileSetting.SkipRows == 0) { fileSetting.SkipRows = 1; resetSkipRows = true; } var othersValueFormatDate = CommonDateFormat(fileSetting.Column.Select(x => x.ValueFormat)); using (var fileReader = fileSetting.GetFileReader()) { Contract.Assume(fileReader != null); // fileReader.ProcessDisplay = processDisplay; fileReader.Open(false, processDisplay.CancellationToken); if (fileReader.FieldCount == 0 || fileReader.EndOfFile) { return(result); } processDisplay.SetProcess("Getting column headers"); processDisplay.Maximum = fileReader.FieldCount; var columnNamesInFile = new List <string>(); for (var colindex = 0; colindex < fileReader.FieldCount; colindex++) { var newColumn = fileReader.GetColumn(colindex); Contract.Assume(newColumn != null); columnNamesInFile.Add(newColumn.Name); var oldColumn = fileSetting.GetColumn(newColumn.Name); processDisplay.SetProcess(newColumn.Name + " – Getting values", colindex); var samples = GetSampleValues(fileReader, ApplicationSetting.FillGuessSettings.CheckedRecords, colindex, ApplicationSetting.FillGuessSettings.SampleValues, fileSetting.TreatTextAsNull, processDisplay.CancellationToken); if (samples.IsEmpty()) { processDisplay.SetProcess(newColumn.Name + " – No values found", colindex); if (!addTextColumns) { continue; } result.Add($"{newColumn.Name} – No values found – Format : {newColumn.GetTypeAndFormatDescription()}"); fileSetting.ColumnAdd(newColumn); } else { var detect = !(ApplicationSetting.FillGuessSettings.IgnoreIdColums && StringUtils.AssumeIDColumn(newColumn.Name) > 0); if (samples.Count < 10) { processDisplay.SetProcess($"{newColumn.Name} – Only {samples.Count} values found in {ApplicationSetting.FillGuessSettings.CheckedRecords} rows", colindex); } else { processDisplay.SetProcess($"{newColumn.Name} – {samples.Count} values found – Examining format", colindex); } var checkResult = GuessValueFormat(samples, ApplicationSetting.FillGuessSettings.MinSamplesForIntDate, ApplicationSetting.FillGuessSettings.TrueValue, ApplicationSetting.FillGuessSettings.FalseValue, ApplicationSetting.FillGuessSettings.DetectBoolean && detect, ApplicationSetting.FillGuessSettings.DetectGUID && detect, ApplicationSetting.FillGuessSettings.DectectNumbers && detect, ApplicationSetting.FillGuessSettings.DetectDateTime && detect, ApplicationSetting.FillGuessSettings.DectectPercentage && detect, ApplicationSetting.FillGuessSettings.SerialDateTime && detect, ApplicationSetting.FillGuessSettings.CheckNamedDates && detect, othersValueFormatDate, processDisplay.CancellationToken); if (checkResult == null) { if (addTextColumns) { checkResult = new CheckResult { FoundValueFormat = new ValueFormat() } } ; else { continue; } } // if we have a mapping to a template that expects a integer and we only have integers but not enough... if (oldColumn != null) { var oldValueFormat = oldColumn.GetTypeAndFormatDescription(); // if we have a date value format already store this if (othersValueFormatDate == null && checkResult.FoundValueFormat.DataType == DataType.DateTime && checkResult.PossibleMatch) { othersValueFormatDate = checkResult.FoundValueFormat; } if (checkResult.FoundValueFormat.Equals(oldColumn.ValueFormat)) { processDisplay.SetProcess($"{newColumn.Name} – Format : {oldValueFormat} – not changed", colindex); } else { oldColumn.ValueFormat = checkResult.FoundValueFormat; } var newValueFormat = checkResult.FoundValueFormat.GetTypeAndFormatDescription(); if (oldValueFormat.Equals(newValueFormat, StringComparison.Ordinal)) { continue; } var msg = $"{newColumn.Name} – Format : {newValueFormat} – updated from {oldValueFormat}"; result.Add(msg); processDisplay.SetProcess(msg, colindex); } else { if (!addTextColumns && checkResult.FoundValueFormat.DataType == DataType.String) { continue; } newColumn.ValueFormat = checkResult.FoundValueFormat; var msg = $"{newColumn.Name} – Format : {newColumn.GetTypeAndFormatDescription()}"; processDisplay.SetProcess(msg, colindex); result.Add(msg); fileSetting.ColumnAdd(newColumn); } } } // The fileReader does not have the column information yet, let the reader know fileReader.OverrideColumnFormatFromSetting(fileReader.FieldCount); // in case its Excel, check all doubles if they could be integer if (fileSetting is IExcelFile) { for (var colindex = 0; colindex < fileReader.FieldCount; colindex++) { var oldColumn = fileReader.GetColumn(colindex); var detect = !(ApplicationSetting.FillGuessSettings.IgnoreIdColums && StringUtils.AssumeIDColumn(oldColumn.Name) > 0); if (oldColumn != null && oldColumn.DataType == DataType.Double) { Column newColumn = null; if (detect) { var samples = GetSampleValues(fileReader, ApplicationSetting.FillGuessSettings.CheckedRecords, colindex, ApplicationSetting.FillGuessSettings.SampleValues, fileSetting.TreatTextAsNull, processDisplay.CancellationToken); if (!samples.IsEmpty()) { var checkResult = GuessNumeric(samples, false, true, processDisplay.CancellationToken); if (checkResult != null && checkResult.FoundValueFormat.DataType != DataType.Double) { newColumn = fileSetting.GetColumn(oldColumn.Name); if (newColumn == null) { newColumn = fileSetting.ColumnAdd(oldColumn); } newColumn.DataType = checkResult.FoundValueFormat.DataType; } } } else { newColumn = fileSetting.GetColumn(oldColumn.Name); if (newColumn == null) { newColumn = fileSetting.ColumnAdd(oldColumn); } newColumn.DataType = DataType.String; } if (newColumn != null) { var msg = $"{newColumn.Name} – Overwritten Excel Format : {newColumn.GetTypeAndFormatDescription()}"; processDisplay.SetProcess(msg, colindex); result.Add(msg); } } } } if (ApplicationSetting.FillGuessSettings.DateParts) { // Try to find a time for a date if the date does not already have a time // Case a) TimeFormat has already been recognized for (var colindex = 0; colindex < fileReader.FieldCount; colindex++) { var columnDate = fileReader.GetColumn(colindex); // Possibly add Time Zone if (columnDate.DataType == DataType.DateTime && string.IsNullOrEmpty(columnDate.TimeZonePart)) { for (var coltimeZone = 0; coltimeZone < fileReader.FieldCount; coltimeZone++) { var columnTimeZone = fileReader.GetColumn(coltimeZone); var colName = columnTimeZone.Name.NoSpecials().ToUpperInvariant(); if (columnTimeZone.DataType != DataType.String && columnTimeZone.DataType != DataType.Integer || colName != "TIMEZONE" && colName != "TIMEZONEID" && colName != "TIME ZONE" && colName != "TIME ZONE ID") { continue; } columnDate.TimeZonePart = columnTimeZone.Name; result.Add($"{columnDate.Name} – Added Time Zone : {columnTimeZone.Name}"); } } if (columnDate.DataType != DataType.DateTime || !string.IsNullOrEmpty(columnDate.TimePart) || columnDate.ValueFormat.DateFormat.IndexOfAny(new[] { ':', 'h', 'H', 'm', 's', 't' }) != -1) { continue; } // We have a date column without time for (var coltime = 0; coltime < fileReader.FieldCount; coltime++) { var columnTime = fileReader.GetColumn(coltime); if (columnTime.DataType != DataType.DateTime || !string.IsNullOrEmpty(columnDate.TimePart) || columnTime.ValueFormat.DateFormat.IndexOfAny(new[] { '/', 'y', 'M', 'd' }) != -1) { continue; } // We now have a time column, // checked if the names somehow make sense if (!columnDate.Name.NoSpecials().ToUpperInvariant().Replace("DATE", string.Empty).Equals(columnTime.Name.NoSpecials().ToUpperInvariant().Replace("TIME", string.Empty), StringComparison.Ordinal)) { continue; } columnDate.TimePart = columnTime.Name; columnDate.TimePartFormat = columnTime.ValueFormat.DateFormat; result.Add($"{columnDate.Name} – Added Time Part : {columnTime.Name}"); } } // Case b) TimeFormat has not been recognized (e.G. all values are 08:00) only look in adjacent fields for (var colindex = 0; colindex < fileReader.FieldCount; colindex++) { var columnDate = fileReader.GetColumn(colindex); if (columnDate.DataType != DataType.DateTime || !string.IsNullOrEmpty(columnDate.TimePart) || columnDate.ValueFormat.DateFormat.IndexOfAny(new[] { ':', 'h', 'H', 'm', 's', 't' }) != -1) { continue; } if (colindex + 1 < fileReader.FieldCount) { var columnTime = fileReader.GetColumn(colindex + 1); if (columnTime.DataType == DataType.String && columnDate.Name.NoSpecials().ToUpperInvariant() .Replace("DATE", string.Empty) .Equals(columnTime.Name.NoSpecials().ToUpperInvariant().Replace("TIME", string.Empty), StringComparison.OrdinalIgnoreCase)) { columnDate.TimePart = columnTime.Name; { var samples = GetSampleValues(fileReader, 1, colindex + 1, 1, fileSetting.TreatTextAsNull, processDisplay.CancellationToken); var first = samples.FirstOrDefault(); if (first != null) { if (first.Length == 8 || first.Length == 5) { columnTime.DataType = DataType.DateTime; var val = new ValueFormat(DataType.DateTime) { DateFormat = first.Length == 8 ? "HH:mm:ss" : "HH:mm" }; columnTime.ValueFormat = val; fileSetting.ColumnAdd(columnTime); result.Add($"{columnTime.Name} – Format : {columnTime.GetTypeAndFormatDescription()}"); } } } result.Add($"{columnDate.Name} – Added Time Part : {columnTime.Name}"); continue; } } if (colindex <= 0) { continue; } { var columnTime = fileReader.GetColumn(colindex - 1); if (columnTime.DataType != DataType.String || !columnDate.Name.NoSpecials().ToUpperInvariant().Replace("DATE", string.Empty).Equals(columnTime.Name.NoSpecials().ToUpperInvariant().Replace("TIME", string.Empty), StringComparison.Ordinal)) { continue; } columnDate.TimePart = columnTime.Name; { var samples = GetSampleValues(fileReader, 1, colindex - 1, 1, fileSetting.TreatTextAsNull, processDisplay.CancellationToken); var first = samples.FirstOrDefault(); if (first != null) { if (first.Length == 8 || first.Length == 5) { var val = new ValueFormat(DataType.DateTime) { DateFormat = first.Length == 8 ? "HH:mm:ss" : "HH:mm" }; fileSetting.ColumnAdd(columnTime); columnTime.ValueFormat = val; result.Add($"{columnTime.Name} – Format : {columnTime.GetTypeAndFormatDescription()}"); } } } result.Add($"{columnDate.Name} – Added Time Part : {columnTime.Name}"); } } } // Sort the columns in fileSetting by order in file fileSetting.SortColumnByName(columnNamesInFile); } } finally { if (resetSkipRows) { fileSetting.SkipRows = 0; } } return(result); }
protected BaseFileWriter([NotNull] string id, [NotNull] string fullPath, bool hasFieldHeader, [CanBeNull] IValueFormat valueFormatGeneral = null, [CanBeNull] IFileFormat fileFormat = null, [CanBeNull] string recipient = null, bool unencrypted = false, [CanBeNull] string identifierInContainer = null, [CanBeNull] string footer = null, [CanBeNull] string header = null, [CanBeNull] IEnumerable <IColumn> columnDefinition = null, [NotNull] string fileSettingDisplay = null, [CanBeNull] IProcessDisplay processDisplay = null) { m_FullPath = fullPath; var fileName = FileSystemUtils.GetFileName(fullPath); ColumnHeader = hasFieldHeader; if (valueFormatGeneral != null) { ValueFormatGeneral = new ImmutableValueFormat(valueFormatGeneral.DataType, valueFormatGeneral.DateFormat, valueFormatGeneral.DateSeparator, valueFormatGeneral.TimeSeparator, valueFormatGeneral.NumberFormat, valueFormatGeneral.GroupSeparator, valueFormatGeneral.DecimalSeparator, valueFormatGeneral.True, valueFormatGeneral.False, valueFormatGeneral.DisplayNullAs); } else { ValueFormatGeneral = new ImmutableValueFormat(); } if (fileFormat != null) { FileFormat = new ImmutableFileFormat(fileFormat.IsFixedLength, fileFormat.QualifyAlways, fileFormat.QualifyOnlyIfNeeded, fileFormat.EscapeChar, fileFormat.FieldDelimiterChar, fileFormat.DelimiterPlaceholder, fileFormat.FieldQualifierChar, fileFormat.QuotePlaceholder, fileFormat.NewLine, fileFormat.NewLinePlaceholder); } else { FileFormat = new ImmutableFileFormat(); } ColumnDefinition = columnDefinition?.Select(col => col is ImmutableColumn immutableColumn ? immutableColumn : new ImmutableColumn(col.Name, col.ValueFormat, col.ColumnOrdinal, col.Convert, col.DestinationName, col.Ignore, col.Part, col.PartSplitter, col.PartToEnd, col.TimePart, col.TimePartFormat, col.TimeZonePart)).ToList() ?? new List <ImmutableColumn>(); NewLine = fileFormat.NewLine.NewLineString(); if (!string.IsNullOrEmpty(header)) { Header = ReplacePlaceHolder(StringUtils.HandleCRLFCombinations(header, NewLine), fileFormat.FieldDelimiterChar.GetDescription(), fileName, id); } else { Header = string.Empty; } if (!string.IsNullOrEmpty(footer)) { m_Footer = ReplacePlaceHolder(StringUtils.HandleCRLFCombinations(footer, NewLine), fileFormat.FieldDelimiterChar.GetDescription(), fileName, id); } else { m_Footer = string.Empty; } m_FileSettingDisplay = fileSettingDisplay ?? string.Empty; m_Recipient = recipient ?? string.Empty; m_KeepUnencrypted = unencrypted; m_IdentifierInContainer = identifierInContainer ?? string.Empty; Logger.Debug("Created Writer for {filesetting}", m_FileSettingDisplay); if (processDisplay == null) { return; } m_ReportProgress = t => processDisplay.SetProcess(t, 0, true); if (!(processDisplay is IProcessDisplayTime processDisplayTime)) { return; } processDisplayTime.Maximum = 0; m_SetMaxProcess = l => processDisplayTime.Maximum = l; }
/// <summary> /// Initializes a new instance of the <see cref="BaseFileWriter" /> class. /// </summary> /// <param name="fileSetting">the file setting with the definition for the file</param> /// <param name="processDisplay">The process display.</param> /// <exception cref="ArgumentNullException">fileSetting</exception> /// <exception cref="ArgumentException">No SQL Reader set</exception> protected BaseFileWriter([NotNull] IFileSettingPhysicalFile fileSetting, [CanBeNull] IProcessDisplay processDisplay) : this(fileSetting.ID, fileSetting.FullPath, fileSetting.HasFieldHeader, fileSetting.FileFormat.ValueFormatMutable, fileSetting.FileFormat, fileSetting.Recipient, fileSetting.KeepUnencrypted, fileSetting.IdentifierInContainer, fileSetting.Footer, fileSetting.Header, fileSetting.ColumnCollection.ReadonlyCopy(), fileSetting.ToString(), processDisplay) { }
public JsonFileReader(IFileSettingPhysicalFile fileSetting, IProcessDisplay processDisplay) : this(fileSetting.FullPath, fileSetting.ColumnCollection, fileSetting.RecordLimit, fileSetting.TreatNBSPAsSpace) => SetProgressActions(processDisplay);
/// <summary> /// Shows the tree. /// </summary> private void ShowTree(IProcessDisplay process) { m_TreeView.BeginUpdate(); try { m_TreeView.Nodes.Clear(); if (m_TreeData == null) { return; } foreach (var treeData in m_TreeData) { treeData.Visited = false; } process.SetProcess("Adding Tree with children", -1, false); foreach (var treeData in m_TreeData) { process.CancellationToken.ThrowIfCancellationRequested(); Extensions.ProcessUIElements(); if (string.IsNullOrEmpty(treeData.ParentID)) { AddTreeDataNodeWithChild(treeData, null, process); } } process.SetProcess("Finding Cycles in Hierarchy", -1, true); var hasCycles = false; foreach (var treeData in m_TreeData) { process.CancellationToken.ThrowIfCancellationRequested(); Extensions.ProcessUIElements(); if (!treeData.Visited) { hasCycles = true; break; } } if (!hasCycles) { return; } process.SetProcess("Adding Cycles", -1, false); var rootNode = new TreeNode("Cycles in Hierarchy"); m_TreeView.Nodes.Add(rootNode); foreach (var treeData in m_TreeData) { process.CancellationToken.ThrowIfCancellationRequested(); Extensions.ProcessUIElements(); if (!treeData.Visited) { MarkInCycle(treeData, new HashSet <TreeData>()); } } foreach (var root in m_TreeData) { process.CancellationToken.ThrowIfCancellationRequested(); Extensions.ProcessUIElements(); if (!root.Visited && root.InCycle) { AddTreeDataNodeWithChild(root, rootNode, process); } } } catch { m_TreeView.Nodes.Clear(); } finally { m_TreeView.EndUpdate(); } }
/// <summary> /// Gets the SQl Info messages and logs them /// </summary> /// <param name="process">The process display.</param> /// <returns></returns> public static EventHandler <string> GetLogInfoMessage(this IProcessDisplay process) => delegate(object sender, string message) { Logger.Information("SQL Information: {message}", message); process?.SetProcess(message, -1, true); };
/// <summary> /// Gets the column header of a file /// </summary> /// <param name="fileSetting">The file setting.</param> /// <param name="includeIgnored">Set to <c>true</c> if ignored columns should be listed as well, otherwise they will not be /// listed</param> /// <param name="processDisplay">The process display.</param> /// <returns> /// An array of string with the column headers /// </returns> public static ICollection <string> GetColumnHeader(IFileSetting fileSetting, bool includeIgnored, bool openIfNeeded, IProcessDisplay processDisplay) { Contract.Requires(fileSetting != null); var key = CacheListKeyColumnHeader(fileSetting.ID, includeIgnored); if (key.Length > 3 && ApplicationSetting.CacheList.TryGet(key, out var retValue)) { return(retValue); } if (!openIfNeeded) { return(null); } using (var fileReader = fileSetting.GetFileReader()) { fileReader.ProcessDisplay = processDisplay; fileReader.Open(false, processDisplay?.CancellationToken ?? CancellationToken.None); // if teh key was long enough it has been stored if (key.Length > 3) { return(ApplicationSetting.CacheList.Get(key)); } else { var header = new HashSet <string>(StringComparer.OrdinalIgnoreCase); for (var colindex = 0; colindex < fileReader.FieldCount; colindex++) { var col = fileReader.GetColumn(colindex); if (includeIgnored || !col.Ignore) { header.Add(col.Name); } } return(header); } } }
/// <summary> /// Builds the tree data. /// </summary> private void BuildTreeData(string parentCol, string idCol, string display1, string display2, IProcessDisplay process) { var intervalAction = new IntervalAction(); var dataColumnParent = m_DataTable.Columns[parentCol]; if (dataColumnParent == null) { throw new ArgumentException($"Could not find column {parentCol}"); } var dataColumnID = m_DataTable.Columns[idCol]; if (dataColumnID == null) { throw new ArgumentException($"Could not find column {idCol}"); } var dataColumnDisplay1 = string.IsNullOrEmpty(display1) ? null : m_DataTable.Columns[display1]; var dataColumnDisplay2 = string.IsNullOrEmpty(display2) ? null : m_DataTable.Columns[display2]; // Using a dictionary here to speed up lookups var treeDataDictionary = new Dictionary <string, TreeData>(); var rootDataParentFound = new TreeData("{R}", "Parent found / No Parent"); treeDataDictionary.Add(rootDataParentFound.ID, rootDataParentFound); var max = 0L; if (process is IProcessDisplayTime processDisplayTime) { max = processDisplayTime.Maximum; } var counter = 0; foreach (var dataRow in m_DataRow) { process.CancellationToken.ThrowIfCancellationRequested(); intervalAction.Invoke( count => process.SetProcess($"Parent found {count}/{max} ", count, false), counter++); var id = dataRow[dataColumnID.Ordinal].ToString(); if (string.IsNullOrEmpty(id)) { continue; } var treeData = new TreeData(id, dataColumnDisplay1 != null ? dataColumnDisplay2 != null ? dataRow[dataColumnDisplay1.Ordinal] + " - " + dataRow[dataColumnDisplay2.Ordinal] : dataRow[dataColumnDisplay1.Ordinal].ToString() : id, dataRow[dataColumnParent.Ordinal].ToString()); if (dataColumnDisplay1 != null) { treeData.Tag = dataRow[dataColumnDisplay1.Ordinal].ToString(); } // Store the display if (!treeDataDictionary.ContainsKey(id)) { treeDataDictionary.Add(id, treeData); } } // Generate a list of missing parents var additionalRootNodes = new HashSet <string>(); foreach (var child in treeDataDictionary.Values) { if (!string.IsNullOrEmpty(child.ParentID) && !treeDataDictionary.ContainsKey(child.ParentID)) { additionalRootNodes.Add(child.ParentID); } } var rootDataParentNotFound = new TreeData("{M}", "Parent not found"); if (additionalRootNodes.Count > 0) { treeDataDictionary.Add(rootDataParentNotFound.ID, rootDataParentNotFound); counter = 0; max = additionalRootNodes.Count; process.SetMaximum(max); // Create new entries foreach (var parentID in additionalRootNodes) { process.CancellationToken.ThrowIfCancellationRequested(); intervalAction.Invoke( count => process.SetProcess($"Parent not found (Step 1) {count}/{max} ", count, false), counter++); var childData = new TreeData(parentID, $"{parentID}", rootDataParentNotFound.ID); treeDataDictionary.Add(parentID, childData); } } max = treeDataDictionary.Values.Count; process.SetMaximum(max); counter = 0; foreach (var child in treeDataDictionary.Values) { process.CancellationToken.ThrowIfCancellationRequested(); intervalAction.Invoke( count => process.SetProcess($"Parent not found (Step 2) {count}/{max} ", count, false), counter++); if (string.IsNullOrEmpty(child.ParentID) && child.ID != rootDataParentFound.ID && child.ID != rootDataParentNotFound.ID) { child.ParentID = rootDataParentFound.ID; } } max = treeDataDictionary.Values.Count; process.SetMaximum(max); counter = 0; // Fill m_Children for the new nodes foreach (var child in treeDataDictionary.Values) { process.CancellationToken.ThrowIfCancellationRequested(); intervalAction.Invoke( count => process.SetProcess($"Set children {count}/{max} ", count, false), counter++); if (!string.IsNullOrEmpty(child.ParentID)) { treeDataDictionary[child.ParentID].Children.Add(child); } } m_TreeData = treeDataDictionary.Values; }
/// <summary> /// Initializes a new instance of the <see cref="CsvFileWriter" /> class. /// </summary> /// <param name="file">The file.</param> /// <param name="processDisplay">The process display.</param> public CsvFileWriter([NotNull] ICsvFile fileSetting, [CanBeNull] IProcessDisplay processDisplay) : this(fileSetting.ID, fileSetting.FullPath, fileSetting.HasFieldHeader, fileSetting.FileFormat.ValueFormatMutable, fileSetting.FileFormat, fileSetting.CodePageId, fileSetting.ByteOrderMark, fileSetting.ToString(), fileSetting.ColumnCollection.ReadonlyCopy(), fileSetting.Recipient, fileSetting.KeepUnencrypted, fileSetting.IdentifierInContainer, fileSetting.Header, fileSetting.Footer, processDisplay) { }
private void CloseWrite() { // If we are writing we have possibly two steps, // a) compress / encrypt the data from temp // b) upload the data to sFTP if (ProcessDisplay == null) { ProcessDisplay = new DummyProcessDisplay(); } if (WritePath.AssumeGZip() || WritePath.AssumePgp()) { try { // Compress the file if (WritePath.AssumeGZip()) { Log.Debug("Compressing temporary file to GZip file"); using (var inFile = File.OpenRead(TempFile)) { // Create the compressed file. using (var outFile = File.Create(WritePath)) { using (var compress = new System.IO.Compression.GZipStream(outFile, System.IO.Compression.CompressionMode.Compress)) { var processDispayTime = ProcessDisplay as IProcessDisplayTime; var inputBuffer = new byte[16384]; var max = (int)(inFile.Length / inputBuffer.Length); ProcessDisplay.Maximum = max; var count = 0; int length; while ((length = inFile.Read(inputBuffer, 0, inputBuffer.Length)) > 0) { compress.Write(inputBuffer, 0, length); ProcessDisplay.CancellationToken.ThrowIfCancellationRequested(); if (processDispayTime != null) { ProcessDisplay.SetProcess( $"GZip {processDispayTime.TimeToCompletion.PercentDisplay}{processDispayTime.TimeToCompletion.EstimatedTimeRemainingDisplaySeperator}", count); } else { ProcessDisplay.SetProcess($"GZip {count:N0}/{max:N0}", count); } count++; } } } } } // need to encrypt the file else if (WritePath.AssumePgp()) { using (FileStream inputStream = new FileInfo(TempFile).OpenRead(), output = new FileStream(WritePath.LongPathPrefix(), FileMode.Create)) { Log.Debug("Encrypting temporary file to PGP file"); ApplicationSetting.ToolSetting.PGPInformation.PgpEncrypt(inputStream, output, Recipient, ProcessDisplay); } } } finally { Log.Debug("Removing temporary file"); File.Delete(TempFile); } } }