public SourceAccess([NotNull] string fileName, bool isReading = true, [CanBeNull] string id = null,
[CanBeNull] string recipient = null, bool keepEnencyrpted = false)
{
FullPath = fileName;
Reading = isReading;
Identifier = id ?? FileSystemUtils.GetShortDisplayFileName(fileName, 40);
Recipient = recipient ?? string.Empty;
KeepEnencyrpted = keepEnencyrpted;
LeaveOpen = false;
FileType = FromExtension(fileName);
switch (FileType)
{
case FileTypeEnum.Zip when !isReading:
IdentifierInContainer = FileSystemUtils.GetFileName(fileName).ReplaceCaseInsensitive(".zip", "");
break;
// for PGP we need a password/ passphrase for Zip we might need one later
case FileTypeEnum.Pgp when isReading:
EncryptedPassphrase = FunctionalDI.GetEncryptedPassphraseForFile(fileName);
break;
}
if (!isReading && KeepEnencyrpted)
{
// remove entension
var split = FileSystemUtils.SplitPath(fileName);
var fn = Path.Combine(split.DirectoryName, split.FileNameWithoutExtension);
m_OpenStream = GetOpenStreamFunc(fn, false);
}
else
{
m_OpenStream = GetOpenStreamFunc(fileName, isReading);
}
}
public async Task <long> WriteAsync([CanBeNull] IFileReader reader, CancellationToken token)
{
if (reader == null)
{
return(-1);
}
HandleWriteStart();
m_SetMaxProcess?.Invoke(-1);
try
{
var sourceAccess = new SourceAccess(m_FullPath, false, recipient: m_Recipient, keepEnencyrpted: m_KeepUnencrypted);
if (!string.IsNullOrEmpty(m_IdentifierInContainer))
{
sourceAccess.IdentifierInContainer = m_IdentifierInContainer;
}
using (var improvedStream = FunctionalDI.OpenStream(sourceAccess))
await WriteReaderAsync(reader, improvedStream as Stream, token).ConfigureAwait(false);
}
catch (Exception exc)
{
Logger.Error(exc, "Could not write file {filename}", FileSystemUtils.GetShortDisplayFileName(m_FullPath));
throw new FileWriterException($"Could not write file '{FileSystemUtils.GetShortDisplayFileName(m_FullPath)}'\n{exc.SourceExceptionMessage()}",
exc);
}
finally
{
Logger.Debug("Finished writing {filesetting} Records: {records}", m_FileSettingDisplay, Records);
WriteFinished?.Invoke(this, null);
}
return(Records);
}
/// <summary>
/// Create a source access based on a stream
/// </summary>
/// <param name="stream">The source stream, it must support seek if its a read stream</param>
/// <param name="isReading"><c>true</c> if used for reading</param>
/// <param name="type">The type of the contents in the stream</param>
public SourceAccess([NotNull] Stream stream, FileTypeEnum type = FileTypeEnum.Stream)
{
if (!stream.CanSeek)
{
throw new ArgumentException("Source stream must support seek to be used for SourceAccess", nameof(stream));
}
LeaveOpen = true;
m_OpenStream = () => stream;
FileType = type;
Reading = true;
FullPath = string.Empty;
// Overwrite in case we can get more information
if (stream is FileStream fs)
{
Identifier = FileSystemUtils.GetShortDisplayFileName(fs.Name);
if ((type == FileTypeEnum.Stream))
{
FileType = FromExtension(fs.Name);
}
}
else
{
Identifier = $"{stream.GetType().Name}_{FileType}";
}
}
public CsvFileReader(ICsvFile fileSetting)
: base(fileSetting)
{
m_CsvFile = fileSetting;
if (string.IsNullOrEmpty(m_CsvFile.FileName))
{
throw new ApplicationException("FileName must be set");
}
if (ApplicationSetting.RemoteFileHandler != null && string.IsNullOrEmpty(m_CsvFile?.RemoteFileName))
{
if (!FileSystemUtils.FileExists(m_CsvFile.FullPath))
{
throw new FileNotFoundException(
$"The file '{FileSystemUtils.GetShortDisplayFileName(m_CsvFile.FileName, 80)}' does not exist or is not accessible.", m_CsvFile.FileName);
}
}
if (m_CsvFile.FileFormat.FieldDelimiterChar == c_Cr ||
m_CsvFile.FileFormat.FieldDelimiterChar == c_Lf ||
m_CsvFile.FileFormat.FieldDelimiterChar == ' ' ||
m_CsvFile.FileFormat.FieldDelimiterChar == '\0')
{
throw new ApplicationException(
"The field delimiter character is invalid, please use something else than CR, LF or Space");
}
if (m_CsvFile.FileFormat.FieldDelimiterChar == m_CsvFile.FileFormat.EscapeCharacterChar)
{
throw new ApplicationException(
$"The escape character is invalid, please use something else than the field delimiter character {FileFormat.GetDescription(m_CsvFile.FileFormat.EscapeCharacter)}.");
}
m_HasQualifier |= m_CsvFile.FileFormat.FieldQualifierChar != '\0';
if (!m_HasQualifier)
{
return;
}
if (m_CsvFile.FileFormat.FieldQualifierChar == m_CsvFile.FileFormat.FieldDelimiterChar)
{
throw new ArgumentOutOfRangeException(
$"The text quoting and the field delimiter characters of a delimited file cannot be the same. {m_CsvFile.FileFormat.FieldDelimiterChar}");
}
if (m_CsvFile.FileFormat.FieldQualifierChar == c_Cr || m_CsvFile.FileFormat.FieldQualifierChar == c_Lf)
{
throw new ApplicationException(
"The text quoting characters is invalid, please use something else than CR or LF");
}
}
public JsonFileReader([NotNull] string fileName,
[CanBeNull] IEnumerable <IColumn> columnDefinition = null,
long recordLimit = 0,
bool treatNbspAsSpace = false, bool trim = false,
string treatTextAsNull = null) :
this(columnDefinition, recordLimit, treatNbspAsSpace, trim, treatTextAsNull, fileName)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentException("File can not be null or empty", nameof(fileName));
}
if (!FileSystemUtils.FileExists(fileName))
{
throw new FileNotFoundException($"The file '{FileSystemUtils.GetShortDisplayFileName(fileName)}' does not exist or is not accessible.", fileName);
}
}
/// <summary>
/// Get the stream, in case of a read stream it's attempted to be at the beginning of the stream
/// </summary>
/// <returns></returns>
public Stream OpenStream()
{
var stream = m_OpenStream.Invoke();
if (LeaveOpen && Reading && stream.Position != 0)
{
stream.Seek(0, SeekOrigin.Begin);
}
// in case the SourceAccess initialized with a function, the stream is only known now...
if (Identifier.Length == 0)
{
Identifier = (stream is FileStream fs)
? FileSystemUtils.GetShortDisplayFileName(fs.Name)
: $"{stream.GetType().Name}_{FileType}";
}
return(stream);
}
public override async Task OpenAsync(CancellationToken token)
{
Logger.Information("Opening JSON file {filename}", FileName);
await BeforeOpenAsync(
$"Opening JSON file {FileSystemUtils.GetShortDisplayFileName(FileName)}")
.ConfigureAwait(false);
Retry:
try
{
ResetPositionToStartOrOpen();
var line = GetNextRecord(true, token);
// get column names for some time
var colNames = new Dictionary <string, DataType>();
var stopwatch = new Stopwatch();
// read additional rows to see if we have some extra columns
while (line != null)
{
foreach (var keyValue in line)
{
if (!colNames.ContainsKey(keyValue.Key))
{
colNames.Add(keyValue.Key, keyValue.Value?.GetType().GetDataType() ?? DataType.String);
}
}
if (stopwatch.ElapsedMilliseconds > 200)
{
break;
}
line = GetNextRecord(true, token);
}
InitColumn(colNames.Count);
ParseColumnName(colNames.Select(colValue => colValue.Key), colNames.Select(colValue => colValue.Value));
FinishOpen();
ResetPositionToStartOrOpen();
}
catch (Exception ex)
{
if (ShouldRetry(ex, token))
{
goto Retry;
}
Close();
var appEx = new FileReaderException(
"Error opening structured text file for reading.\nPlease make sure the file does exist, is of the right type and is not locked by another process.",
ex);
HandleError(-1, appEx.ExceptionMessages());
HandleReadFinished();
throw appEx;
}
finally
{
HandleShowProgress("");
}
}
/// <summary>
/// Writes the file ans displays performance information
/// </summary>
/// <param name="fileSetting">The file setting.</param>
/// <param name="showSummary">
/// Flag indicating that a message Box should be displayed. If <c>true</c> a message box will be
/// shown
/// </param>
/// <param name="cancellationToken">A CancellationToken</param>
/// <param name="fileSettingSourcesCurrent">The file setting sources current.</param>
/// <param name="settingLaterThanSources">The setting later than sources.</param>
/// <param name="ask">if set to <c>true</c> [ask].</param>
/// <returns>
/// Number of record written to file
/// </returns>
public static long WriteFileWithInfo(this IFileSetting fileSetting, bool showSummary,
FileSettingChecker fileSettingSourcesCurrent,
bool ask, bool onlyOlder, CancellationToken cancellationToken)
{
if (fileSetting == null)
{
return(0);
}
long written = 0;
var fi = FileSystemUtils.FileInfo(fileSetting.FullPath);
var dir = FileSystemUtils.GetDirectoryName(fi.FullName);
if (!FileSystemUtils.DirectoryExists(dir))
{
if (_MessageBox.Show(null,
$"The directory {dir.RemovePrefix()} does not exist, should it be created?",
"Directory", MessageBoxButtons.OKCancel,
MessageBoxIcon.Question) == DialogResult.OK)
{
FileSystemUtils.CreateDirectory(dir);
}
else
{
return(0);
}
}
var fileInfo = new FileInfo(fileSetting.FullPath);
if (fileInfo.Exists)
{
fileSetting.FileLastWriteTimeUtc = fi.LastWriteTimeUtc;
}
var stringBuilder = new System.Text.StringBuilder();
using (var processDisplay = fileSetting.GetProcessDisplay(null, true, cancellationToken))
{
fileSettingSourcesCurrent?.Invoke(fileSetting, processDisplay);
if (onlyOlder && fileSetting.SettingLaterThanSources(processDisplay.CancellationToken))
{
return(0);
}
fileSetting.FullPath.DeleteFileQuestion(ask);
var errors = new RowErrorCollection(50);
var writer = fileSetting.GetFileWriter(processDisplay.CancellationToken);
writer.ProcessDisplay = processDisplay;
writer.Warning += errors.Add;
written = writer.Write();
var hasIssues = !string.IsNullOrEmpty(writer.ErrorMessage) || (errors.CountRows > 0);
if (showSummary || hasIssues)
{
fi = FileSystemUtils.FileInfo(fileSetting.FullPath);
// if all source settings are file settings, get the latest file time and set this fileTime
var latest = DateTime.MinValue;
fileSetting.GetSourceFileSettings(delegate(IFileSetting setting)
{
if (!(setting is IFileSettingRemoteDownload))
{
if (latest < setting.FileLastWriteTimeUtc)
{
latest = setting.FileLastWriteTimeUtc;
}
}
else
{
var fiSrc = FileSystemUtils.FileInfo(setting.FullPath);
if (fiSrc.Exists && latest < fiSrc.LastWriteTimeUtc)
{
latest = fiSrc.LastWriteTimeUtc;
}
}
return(false);
}, cancellationToken);
stringBuilder.Append($"Finished writing file\r\rRecords: {written:N0}\rFile size: {fi.Length / 1048576.0:N} MB");
if (latest < DateTime.MaxValue && latest > DateTime.MinValue)
{
stringBuilder.Append($"\rTime adjusted to latest source file: {latest.ToLocalTime():D}");
fi.LastWriteTimeUtc = latest;
fileSetting.FileLastWriteTimeUtc = latest;
}
if (hasIssues)
{
stringBuilder.Append("\rIssues:\r");
}
if (!string.IsNullOrEmpty(writer.ErrorMessage))
{
stringBuilder.Append(writer.ErrorMessage);
}
if (errors.CountRows > 0)
{
stringBuilder.Append(errors.DisplayByRecordNumber);
}
}
}
if (stringBuilder.Length > 0)
{
_MessageBox.Show(null, stringBuilder.ToString(), FileSystemUtils.GetShortDisplayFileName(fileSetting.FileName, 80), MessageBoxButtons.OK);
}
return(written);
}
public static string GetProcessDisplayTitle(this IFileSetting fileSetting)
{
return(string.IsNullOrEmpty(fileSetting.ID)
? FileSystemUtils.GetShortDisplayFileName(fileSetting.FileName, 80)
: fileSetting.ID);
}
/// <summary>
/// CTOR CsvTextDisplay
/// </summary>
public FormCsvTextDisplay([NotNull] string fullPath)
{
m_FullPath = fullPath ?? throw new ArgumentNullException(nameof(fullPath));
InitializeComponent();
Text = FileSystemUtils.GetShortDisplayFileName(m_FullPath);
}
