public static void Add(FileControlBlock file) { waitinglistMutex.WaitOne(); file.Key = key; waitingList.Add(key, file); key += 10000; waitinglistMutex.ReleaseMutex(); }
public Download(FileControlBlock file) { this.File = file; TaskStatusDetectionThread = new Thread(TaskStatusDetection); TaskStatusDetectionThread.Start(); }
/// <summary> /// Retrieve all files from the specified location (matching specified pattern), and import their contents /// into the provided database using the stored procedure(s) specified /// </summary> public override async Task <TaskResult> ExecuteTask(TaskParameters parameters) { var result = new TaskResult(); var dateTimeNow = DateTime.Now; // Use single value throughout for consistency in macro replacements try { #region Retrieve task parameters string connectionString = config.GetConnectionString(parameters.GetString("ConnectionString")); importProcedureName = parameters.GetString("ImportProcedureName"); string importFolder = parameters.GetFolder("ImportFolder", dateTimeNow); string filenameFilter = parameters.GetString("FilenameFilter", null, dateTimeNow); if (string.IsNullOrEmpty(connectionString) || string.IsNullOrEmpty(importProcedureName) || string.IsNullOrEmpty(importFolder) || string.IsNullOrEmpty(filenameFilter)) { throw new ArgumentException("Missing or invalid: one or more of ConnectionString/ImportProcedureName/ImportFolder/FilenameFilter"); } // Retrieve import mode (default is RAW), validate if present: importMode = parameters.GetString("ImportMode")?.ToUpperInvariant(); if (!string.IsNullOrEmpty(importMode)) { if (!ImportModeValidationRegex.IsMatch(importMode)) { throw new ArgumentException("Invalid ImportMode"); } } // Retrieve data parameter name, validate if present: importDataParameterName = parameters.GetString("ImportDataParameterName"); if (!string.IsNullOrEmpty(importDataParameterName)) { if (!importDataParameterName.StartsWith("@")) { throw new ArgumentException("Invalid ImportDataParameterName"); } } // Retrieve imported file archival settings (either folder or rename regex required): string archiveFolder = parameters.GetFolder("ArchiveFolder", dateTimeNow); var archiveRenameRegex = TaskUtilities.General.RegexIfPresent(parameters.GetString("ArchiveRenameRegex"), RegexOptions.IgnoreCase); string archiveRenameReplacement = archiveRenameRegex == null ? null : (parameters.GetString("ArchiveRenameReplacement", null, dateTimeNow) ?? string.Empty); if (string.IsNullOrEmpty(archiveFolder)) { if (archiveRenameRegex == null) { throw new ArgumentException("Either ArchiveFolder or ArchiveRenameRegex settings required"); } } // Create archival folder, if it doesn't already exist: else if (!Directory.Exists(archiveFolder)) { Directory.CreateDirectory(archiveFolder); } // Retrieve optional parameters: importProcedureTimeout = parameters.Get <int>("ImportProcedureTimeout", int.TryParse); importPreProcessorName = parameters.GetString("ImportPreProcessorName"); importPreProcessorTimeout = parameters.Get <int>("ImportPreProcessorTimeout", int.TryParse); importPostProcessorName = parameters.GetString("ImportPostProcessorName"); importPostProcessorTimeout = parameters.Get <int>("ImportPostProcessorTimeout", int.TryParse); importPGPPrivateKeyRing = parameters.GetString("ImportPGPPrivateKeyRing"); importPGPPassphrase = parameters.GetString("ImportPGPPassphrase"); defaultNulls = parameters.GetBool("DefaultNulls"); var filenameRegex = TaskUtilities.General.RegexIfPresent(parameters.GetString("FilenameRegex"), RegexOptions.IgnoreCase); delimiter = parameters.GetString("Delimiter"); if (string.IsNullOrEmpty(delimiter)) { delimiter = ","; } importLineFilterRegex = TaskUtilities.General.RegexIfPresent(parameters.GetString("ImportLineFilterRegex")); bool haltOnImportError = parameters.GetBool("HaltOnImportError"); // Add any parameters to be applied to SQL statements to dictionary: var atparms = parameters.GetKeys().Where(parmname => parmname.StartsWith("@")); foreach (var atparm in atparms) { sqlParameters[atparm] = parameters.GetString(atparm, null, dateTimeNow); } // If custom regex not specified, create one from file filter (this check is performed to avoid false-positives on 8.3 version of filenames): if (filenameRegex == null) { filenameRegex = TaskUtilities.General.RegexFromFileFilter(filenameFilter); } #endregion // Build listing of all matching files in import folder: var fileInfoList = Directory.EnumerateFiles(importFolder, filenameFilter) .Where(fileName => filenameRegex.IsMatch(Path.GetFileName(fileName))) .Select(fileName => new FileInfo(fileName)); if (fileInfoList.Any()) { #region Connect to database and process all files in list await using (var cnn = new SqlConnection(connectionString)) { await cnn.OpenAsync(); // Capture console messages into StringBuilder: var consoleOutput = new StringBuilder(); cnn.InfoMessage += (object obj, SqlInfoMessageEventArgs e) => { consoleOutput.AppendLine(e.Message); }; foreach (var fileInfo in fileInfoList) { using var importFileScope = logger.BeginScope(new Dictionary <string, object>() { ["FileName"] = fileInfo.FullName }); // Determine path to archive file to after completion of import process: var archiveFilePath = TaskUtilities.General.PathCombine(string.IsNullOrEmpty(archiveFolder) ? importFolder : archiveFolder, archiveRenameRegex == null ? fileInfo.Name : archiveRenameRegex.Replace(fileInfo.Name, archiveRenameReplacement)); if (archiveFilePath.Equals(fileInfo.FullName, StringComparison.OrdinalIgnoreCase)) { logger.LogWarning($"Import and archive folders are the same, file will not be archived"); archiveFilePath = null; // Ensure file move is skipped over below } try { // Create fileControlBlock to hold file information and data streams, then pass processing off to // utility function, receiving any output parameters from import procedure and merging into overall // task return value set: using (var fileControlBlock = new FileControlBlock(fileInfo)) { result.MergeReturnValues(await ImportFile(cnn, consoleOutput, fileControlBlock)); } } catch (Exception ex) { if (ex is AggregateException ae) // Exception caught from async task; simplify if possible { ex = TaskUtilities.General.SimplifyAggregateException(ae); } // Log error here (to take advantage of logger scope context) and add exception to result collection: logger.LogError(ex, "Error importing file"); result.AddException(new Exception($"Error importing file {fileInfo.FullName}", ex)); // If we are halting process for individual import errors, throw custom exception to indicate // to outer try block that it can just exit without re-logging: if (haltOnImportError) { archiveFilePath = null; // Prevent finally block from archiving this file throw new TaskExitException(); } } finally { // Unless file archival has been explicitly cancelled, move file to archive (regardless of result): if (archiveFilePath != null) { try { File.Move(fileInfo.FullName, archiveFilePath); logger.LogDebug($"File archived to {archiveFilePath}"); } catch (Exception ex) { // Add exception to response collection, do not re-throw (to avoid losing actual valuable // exception that may currently be throwing to outer block) result.AddException(new Exception($"Error archiving file {fileInfo.FullName} to {archiveFilePath}", ex)); } } // Import procedure should have consumed any console output already; ensure it is cleared for next run: if (consoleOutput.Length > 0) { logger.LogDebug($"Unhandled console output follows:\n{consoleOutput}"); consoleOutput.Clear(); } } } } #endregion } // If this point is reached, consider overall operation successful result.Success = true; } catch (TaskExitException) { // Exception was handled above - just proceed with return below } catch (Exception ex) { if (ex is AggregateException ae) // Exception caught from async task; simplify if possible { ex = TaskUtilities.General.SimplifyAggregateException(ae); } logger.LogError(ex, "Import process failed"); result.AddException(ex); } return(result); }
/// <summary> /// Call import procedure(s), passing source file data into database /// </summary> /// <param name="cnn">Open connection to database</param> /// <param name="fileControlBlock">Object containing FileInfo and an open FileStream for source file</param> /// <returns>Collection of output parameters from import procedure(s)</returns> private async Task <Dictionary <string, string> > ImportFile(SqlConnection cnn, StringBuilder consoleOutput, FileControlBlock fileControlBlock) { var outputParameters = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase); var loggerScope = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase); if (!string.IsNullOrEmpty(importPreProcessorName)) { #region Execute pre-processor try { await using var cmd = new SqlCommand(importPreProcessorName, cnn) { CommandType = CommandType.StoredProcedure }; using var preproclogscope = logger.BeginScope(new Dictionary <string, object>() { ["PreProcessorProcedure"] = importPreProcessorName }); // Apply timeout if configured, derive and bind parameters, execute procedure: if (importPreProcessorTimeout > 0) { cmd.CommandTimeout = (int)importPreProcessorTimeout; } await DeriveAndBindParameters(cmd, fileControlBlock, outputParameters); await cmd.ExecuteNonQueryAsync(); #region Iterate through output parameters int?returnValue = null; foreach (SqlParameter sqlParameter in cmd.Parameters) { if (sqlParameter.Direction == ParameterDirection.ReturnValue) { returnValue = sqlParameter.Value as int?; } else if (sqlParameter.Direction.HasFlag(ParameterDirection.Output)) { // Check for reserved values with special behavior: if (sqlParameter.ParameterName.Equals("@FILEID", StringComparison.OrdinalIgnoreCase)) { if (!loggerScope.ContainsKey("FileID")) { // Set logging scope so subsequent logging for this block will include this value: loggerScope["FileID"] = sqlParameter.Value; fileControlBlock.SetLoggerScope(logger, loggerScope); } } // Save value in output collection (as object, converting DBNull to null): outputParameters[sqlParameter.ParameterName] = sqlParameter.Value == DBNull.Value ? null : sqlParameter.Value; } } #endregion // Validate return code: if (returnValue > 0 && returnValue < 100) // Values 1-99 are reserved for warnings, will not halt import process { logger.LogWarning($"Pre-processor returned non-standard value {returnValue}, proceeding with import"); } else if (returnValue != 0) // Any other non-zero value (including null) will abort process { throw new Exception($"Invalid return value ({returnValue})"); } else { logger.LogDebug("Pre-processor complete"); } } catch (Exception ex) { throw new Exception("Exception executing pre-processor", ex is AggregateException ae ? TaskUtilities.General.SimplifyAggregateException(ae) : ex); } finally { if (consoleOutput.Length > 0) { logger.LogDebug($"Pre-processor console output follows:\n{consoleOutput}"); consoleOutput.Clear(); } } #endregion } #region Execute data import TextFieldParser filecsvreader = null; // Used for CSV import method StreamReader filestreamreader = null; // Used for default import method try { // If PGP decryption of source file is required, update read stream in FileControlBlock now: if (!string.IsNullOrEmpty(importPGPPrivateKeyRing)) { using var privatekeystream = new FileStream(importPGPPrivateKeyRing, FileMode.Open, FileAccess.Read, FileShare.Read); await fileControlBlock.OpenDecryptionStream(privatekeystream, importPGPPassphrase); } // Create and execute import command object await using var cmd = new SqlCommand(importProcedureName, cnn) { CommandType = CommandType.StoredProcedure }; using var proclogscope = logger.BeginScope(new Dictionary <string, object>() { ["ImportProcedure"] = importProcedureName }); // Apply timeout if configured, derive and bind parameters: if (importProcedureTimeout > 0) { cmd.CommandTimeout = (int)importProcedureTimeout; } var dataParameters = await DeriveAndBindParameters(cmd, fileControlBlock, outputParameters); #region Bind data parameter // Choose data parameter to receive input; start by retrieving all parameter names for the appropriate type: var eligibleparms = dataParameters .Where(entry => entry.Value == (importMode == "XML" ? SqlDbType.Xml : SqlDbType.Structured)) .Select(entry => entry.Key) // If importDataParameterName is specified, select matching entry only: .Where(parmname => string.IsNullOrEmpty(importDataParameterName) || parmname.Equals(importDataParameterName, StringComparison.OrdinalIgnoreCase)) // We have to access Count argument more than once, so harden to list to prevent multiple iterations: .ToList(); // If no parameters found (either no inputs of correct type, or specific importDataParameterName not found), throw error: if (eligibleparms.Count == 0) { throw new ArgumentException(string.IsNullOrEmpty(importDataParameterName) ? "No eligible input parameter found" : $"Input parameter name {importDataParameterName} not found"); } else if (eligibleparms.Count > 1) { // If more than one eligible parameter was found, config did not specify; unless we have the specific case where exactly // one parameter has not had a value supplied already, we have no way of deciding which parameter to use: eligibleparms = eligibleparms.Where(parmname => cmd.Parameters[parmname].Value == null).ToList(); if (eligibleparms.Count != 1) { throw new ArgumentException("More than one eligible data parameter found, ImportDataParameterName must be specified"); } } if (importMode == "XML") { cmd.Parameters[eligibleparms[0]].Value = new SqlXml(fileControlBlock.GetStream()); } else if (fileControlBlock.fileInfo.Length > 0) // Ignore empty files { if (importMode == "CSV") // Stream in CSV data { filecsvreader = new TextFieldParser(fileControlBlock.GetStream()) { Delimiters = new string[] { delimiter }, HasFieldsEnclosedInQuotes = true }; cmd.Parameters[eligibleparms[0]].Value = StreamDataTable(csvReader: filecsvreader); } else // Stream in raw format data { filestreamreader = new StreamReader(fileControlBlock.GetStream()); cmd.Parameters[eligibleparms[0]].Value = StreamDataTable(rawReader: filestreamreader); } } #endregion #region Remove any remaining parameters with null unstructured data // Structured parameters cannot be null, and datasets cannot be empty; retrieve all structured data fields // whose parameter value is still null, and remove from parameter collection entirely: var removeparms = dataParameters .Where(entry => entry.Value == SqlDbType.Structured) .Select(entry => entry.Key) .Where(fieldname => cmd.Parameters[fieldname].Value == DBNull.Value); foreach (var removeparm in removeparms) { cmd.Parameters.RemoveAt(removeparm); } #endregion await cmd.ExecuteNonQueryAsync(); #region Iterate through output parameters int?returnValue = null; int?rowsImported = null; foreach (SqlParameter sqlParameter in cmd.Parameters) { if (sqlParameter.Direction == ParameterDirection.ReturnValue) { returnValue = sqlParameter.Value as int?; } else if (sqlParameter.Direction.HasFlag(ParameterDirection.Output)) { // Check for reserved values with special behavior: if (sqlParameter.ParameterName.Equals("@FILEID", StringComparison.OrdinalIgnoreCase)) { if (!loggerScope.ContainsKey("FileID")) { // Update logging scope so subsequent logging for this block will include this value: loggerScope["FileID"] = sqlParameter.Value; fileControlBlock.SetLoggerScope(logger, loggerScope); } } else if (sqlParameter.ParameterName.Equals("@ROWSIMPORTED", StringComparison.OrdinalIgnoreCase)) { rowsImported = sqlParameter.Value as int?; } // Save value in output collection (as object, converting DBNull to null): outputParameters[sqlParameter.ParameterName] = sqlParameter.Value == DBNull.Value ? null : sqlParameter.Value; } } #endregion // Validate return code: if (returnValue > 0 && returnValue < 100) // Values 1-99 are reserved for warnings, will not halt import process { logger.LogWarning(rowsImported == null ? $"Import returned non-standard value {returnValue}, proceeding" : $"Import returned non-standard value {returnValue} with {rowsImported} rows imported, proceeding"); } else if (returnValue != 0) // Any other non-zero value (including null) will abort process { throw new Exception($"Invalid return value ({returnValue})"); } else { logger.LogInformation(rowsImported == null ? "Import complete" : $"Import complete, {rowsImported} rows imported"); } } catch (Exception ex) { throw new Exception("Exception executing import", ex is AggregateException ae ? TaskUtilities.General.SimplifyAggregateException(ae) : ex); } finally { if (consoleOutput.Length > 0) { logger.LogDebug($"Import console output follows:\n{consoleOutput}"); consoleOutput.Clear(); } // Clean up raw and CSV mode reader objects (can't use using statements since these are conditionally initialized // in a different scope from their use, and need to survive until after ExecuteNonQueryAsync call completes) if (filecsvreader != null) { filecsvreader.Close(); filecsvreader = null; } if (filestreamreader != null) { filestreamreader.Dispose(); filestreamreader = null; } } #endregion if (!string.IsNullOrEmpty(importPostProcessorName)) { #region Execute post-processor try { await using var cmd = new SqlCommand(importPostProcessorName, cnn) { CommandType = CommandType.StoredProcedure }; using var postproclogscope = logger.BeginScope(new Dictionary <string, object>() { ["PostProcessorProcedure"] = importPostProcessorName }); // Apply timeout if configured, derive and bind parameters, execute procedure: if (importPostProcessorTimeout > 0) { cmd.CommandTimeout = (int)importPostProcessorTimeout; } await DeriveAndBindParameters(cmd, fileControlBlock, outputParameters); await cmd.ExecuteNonQueryAsync(); #region Iterate through output parameters int?returnValue = null; foreach (SqlParameter sqlParameter in cmd.Parameters) { if (sqlParameter.Direction == ParameterDirection.ReturnValue) { returnValue = sqlParameter.Value as int?; } else if (sqlParameter.Direction.HasFlag(ParameterDirection.Output)) { // Check for reserved values with special behavior: if (sqlParameter.ParameterName.Equals("@FILEID", StringComparison.OrdinalIgnoreCase)) { if (!loggerScope.ContainsKey("FileID")) { // Update logging scope so subsequent logging for this block will include this value: loggerScope["FileID"] = sqlParameter.Value; fileControlBlock.SetLoggerScope(logger, loggerScope); } // Save value in output collection (as object, converting DBNull to null): outputParameters[sqlParameter.ParameterName] = sqlParameter.Value == DBNull.Value ? null : sqlParameter.Value; } } } #endregion // Validate return code: if (returnValue > 0 && returnValue < 100) // Values 1-99 are reserved for warnings, will not halt import process { logger.LogWarning($"Post-processor returned non-standard value {returnValue}"); } else if (returnValue != 0) // Any other non-zero value (including null) will abort process { throw new Exception($"Invalid return value ({returnValue})"); } else { logger.LogDebug("Post-processor complete"); } } catch (Exception ex) { throw new Exception("Exception executing post-processor", ex is AggregateException ae ? TaskUtilities.General.SimplifyAggregateException(ae) : ex); } finally { if (consoleOutput.Length > 0) { logger.LogDebug($"Post-processor console output follows:\n{consoleOutput}"); consoleOutput.Clear(); } } #endregion } // Return output collection (converting objects to strings), to be added to task return values: return(outputParameters.ToDictionary(entry => entry.Key, entry => entry.Value?.ToString())); }
/// <summary> /// Dynamically derive parameters from SQL server, and bind inputs from parameter collection, file control block /// and running collection of output parameters /// </summary> /// <returns> /// A dictionary of potential data parameters (all structured data parameters, add XML inputs if mode is XML) /// </returns> private async Task <Dictionary <string, SqlDbType> > DeriveAndBindParameters(SqlCommand cmd, FileControlBlock fileControlBlock, Dictionary <string, object> outputParameters) { var dataParameters = new Dictionary <string, SqlDbType>(StringComparer.OrdinalIgnoreCase); // Create cache key from database connection string (hashed, to avoid saving passwords/etc in collection) and stored // proc name, check whether parameters have already been derived for this object within current run: string cachekey = $"{cmd.Connection.ConnectionString.GetHashCode()}::{cmd.CommandText}"; if (memorycache.TryGetValue <SqlParameter[]>(cachekey, out var cachedparms)) { // Cached parameters available - clone cached array into destination parameter collection: cmd.Parameters.AddRange(cachedparms.Select(x => ((ICloneable)x).Clone()).Cast <SqlParameter>().ToArray()); } else { // No cached parameters available; derive from server (note this is synchronous, no async version available): SqlCommandBuilder.DeriveParameters(cmd); // Clone parameter collection into cache (with 5 minute TTL) for subsequent calls to this same proc: memorycache.Set( cachekey, cmd.Parameters.Cast <SqlParameter>().Select(x => ((ICloneable)x).Clone()).Cast <SqlParameter>().ToArray(), new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(5)) ); } #region Iterate through parameter collection, applying values from member collection foreach (SqlParameter sqlParameter in cmd.Parameters) { if (sqlParameter.SqlDbType == SqlDbType.Structured) { // Add this parameter to return collection and set input to null (caller must apply valid data or remove): dataParameters[sqlParameter.ParameterName] = SqlDbType.Structured; sqlParameter.Value = DBNull.Value; // Correct for a problem with DeriveParameters, which erroneously adds database name to type: var typeNameParts = sqlParameter.TypeName.Split('.'); if (typeNameParts.Length == 3) // Database name is included in three-part format, strip out { sqlParameter.TypeName = $"{typeNameParts[1]}.{typeNameParts[2]}"; } } else if (sqlParameter.Direction.HasFlag(ParameterDirection.Input)) { // If this is an XML field and we are processing in XML input mode, add to return collection: if (sqlParameter.SqlDbType == SqlDbType.Xml && importMode == "XML") { dataParameters[sqlParameter.ParameterName] = SqlDbType.Xml; } // Check for reserved special parameters to be supplied based on file data: switch (sqlParameter.ParameterName.ToUpperInvariant()) { case "@FILENAME": sqlParameter.Value = fileControlBlock.fileInfo.Name; break; case "@FILEFOLDER": sqlParameter.Value = fileControlBlock.fileInfo.DirectoryName; break; case "@FILEWRITETIME": sqlParameter.Value = fileControlBlock.fileInfo.LastWriteTime; break; case "@FILEWRITETIMEUTC": sqlParameter.Value = fileControlBlock.fileInfo.LastWriteTimeUtc; break; case "@FILESIZE": sqlParameter.Value = fileControlBlock.fileInfo.Length; break; case "@FILEHASH": // Create hash provider, if not already created: if (md5hash == null) { md5hash = IncrementalHash.CreateHash(HashAlgorithmName.MD5); } sqlParameter.Value = await fileControlBlock.GetFileHash(md5hash) ?? DBNull.Value; break; case "@IMPORTSERVER": sqlParameter.Value = Environment.MachineName; break; default: // Unreserved parameter name - supply value from parameters, if possible // This parameter will require an input value if also an output parameter OR we are explicitly defaulting: bool needsDefault = (defaultNulls || sqlParameter.Direction.HasFlag(ParameterDirection.Output)); // Check whether parameter is included in collection of output parameters from previous calls: if (outputParameters.ContainsKey(sqlParameter.ParameterName)) { // Apply value to parameter (replacing null with DBNull): sqlParameter.Value = outputParameters[sqlParameter.ParameterName] ?? DBNull.Value; needsDefault = false; } // Check whether parameter is included in SQL parameter dictionary: else if (sqlParameters.ContainsKey(sqlParameter.ParameterName)) { // Special case - strings are not automatically changed to Guid values, attempt explicit conversion: if (sqlParameter.SqlDbType == SqlDbType.UniqueIdentifier) { if (Guid.TryParse(sqlParameters[sqlParameter.ParameterName], out var guid)) { sqlParameter.Value = guid; needsDefault = false; } } else { // Apply string value to parameter (replacing null with DBNull): sqlParameter.Value = (object)sqlParameters[sqlParameter.ParameterName] ?? DBNull.Value; needsDefault = false; } } // If we still need a default value, set to null (otherwise, any value not set above will be left // unspecified; if stored procedure does not provide a default value, execution will fail and // missing parameter will be indicated by exception string) if (needsDefault) { sqlParameter.Value = DBNull.Value; } break; } ; } } #endregion return(dataParameters); }
public FileControlBlockVO(FileControlBlock fileControlBlock) { name = fileControlBlock.FileName; completed = fileControlBlock.Completed; total = fileControlBlock.Total; }
public Delete(FileControlBlock file) { this.File = file; }