public void CloseLogging() { if (DataLoadInfo != null) { DataLoadInfo.CloseAndMarkComplete(); } }
protected void Execute(out ExtractionPipelineUseCase pipelineUseCase, out IExecuteDatasetExtractionDestination results) { DataLoadInfo d = new DataLoadInfo("Internal", _testDatabaseName, "IgnoreMe", "", true, new DiscoveredServer(UnitTestLoggingConnectionString)); Pipeline pipeline = null; try { pipeline = SetupPipeline(); pipelineUseCase = new ExtractionPipelineUseCase(_request.Configuration.Project, _request, pipeline, d); pipelineUseCase.Execute(new ThrowImmediatelyDataLoadEventListener()); Assert.IsNotEmpty(pipelineUseCase.Source.Request.QueryBuilder.SQL); Assert.IsTrue(pipelineUseCase.ExtractCommand.State == ExtractCommandState.Completed); } finally { if (pipeline != null) { pipeline.DeleteInDatabase(); } } results = pipelineUseCase.Destination; }
protected void Execute(out ExtractionPipelineUseCase pipelineUseCase, out IExecuteDatasetExtractionDestination results) { DataLoadInfo d = new DataLoadInfo("Internal", _testDatabaseName, "IgnoreMe", "", true, new DiscoveredServer(UnitTestLoggingConnectionString)); Pipeline pipeline = null; //because extractable columns is likely to include chi column, it will be removed from the collection (for a substitution identifier) var before = _extractableColumns.ToArray(); try { pipeline = SetupPipeline(); pipelineUseCase = new ExtractionPipelineUseCase(_request.Configuration.Project, _request, pipeline, d); pipelineUseCase.Execute(new ThrowImmediatelyDataLoadEventListener()); Assert.IsNotEmpty(pipelineUseCase.Source.Request.QueryBuilder.SQL); Assert.IsTrue(pipelineUseCase.ExtractCommand.State == ExtractCommandState.Completed); } finally { if (pipeline != null) { pipeline.DeleteInDatabase(); } } results = pipelineUseCase.Destination; _extractableColumns = new List <IColumn>(before); }
public void OnNotify(object sender, NotifyEventArgs e) { if (DataLoadInfo != null) { switch (e.ProgressEventType) { case ProgressEventType.Trace: case ProgressEventType.Debug: break; case ProgressEventType.Information: DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, sender.GetType().Name, e.Message + (e.Exception != null ? "Exception=" + ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true) : "")); break; case ProgressEventType.Warning: DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.GetType().Name, e.Message + (e.Exception != null ? "Exception=" + ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true) : "")); break; case ProgressEventType.Error: DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnTaskFailed, sender.GetType().Name, e.Message); DataLoadInfo.LogFatalError(sender.GetType().Name, e.Exception != null ? ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true) : e.Message); break; default: throw new ArgumentOutOfRangeException(); } } _listener.OnNotify(sender, e); }
public void LogError(string message, Exception exception) { // we are bailing out before the load process has had a chance to create a DataLoadInfo object if (DataLoadInfo == null) { CreateDataLoadInfo(); } DataLoadInfo.LogFatalError(typeof(DataLoadProcess).Name, message + Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(exception, true)); DataLoadInfo.CloseAndMarkComplete(); }
private void StartAuditIfExists(string tableName) { if (LoggingServer != null) { _loggingDatabaseSettings = DataAccessPortal.GetInstance().ExpectServer(LoggingServer, DataAccessContext.Logging); var logManager = new LogManager(_loggingDatabaseSettings); logManager.CreateNewLoggingTaskIfNotExists("Internal"); _dataLoadInfo = (DataLoadInfo)logManager.CreateDataLoadInfo("Internal", GetType().Name, "Loading table " + tableName, "", false); _loggingDatabaseListener = new ToLoggingDatabaseDataLoadEventListener(logManager, _dataLoadInfo); } }
public void LogWarning(string senderName, string message) { if (DataLoadInfo == null) { throw new Exception("Logging hasn't been started for this job (call StartLogging first)"); } if (!DataLoadInfo.IsClosed) { DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, senderName, message); } }
public ExtractionPipelineUseCase(IProject project, IExtractCommand extractCommand, IPipeline pipeline, DataLoadInfo dataLoadInfo) { _dataLoadInfo = dataLoadInfo; ExtractCommand = extractCommand; _pipeline = pipeline; extractCommand.ElevateState(ExtractCommandState.NotLaunched); AddInitializationObject(ExtractCommand); AddInitializationObject(project); AddInitializationObject(_dataLoadInfo); AddInitializationObject(project.DataExportRepository.CatalogueRepository); GenerateContext(); }
public void FataErrorLoggingTest() { DataLoadInfo d = new DataLoadInfo("Internal", "HICSSISLibraryTests.FataErrorLoggingTest", "Test case for fatal error generation", "No rollback is possible/required as no database rows are actually inserted", true, new DiscoveredServer(UnitTestLoggingConnectionString)); DataSource[] ds = new DataSource[] { new DataSource("nothing", DateTime.Now) }; TableLoadInfo t = new TableLoadInfo(d, "Unit test only", "Unit test only", ds, 5); t.Inserts += 3; //simulate that it crashed after 3 d.LogFatalError("HICSSISLibraryTests.FataErrorLoggingTest", "Some terrible event happened"); Assert.IsTrue(d.IsClosed); }
private DataLoadInfo StartAudit() { DataLoadInfo dataLoadInfo; _logManager = _configuration.GetExplicitLoggingDatabaseServerOrDefault(); try { //populate DataLoadInfo object (Audit) dataLoadInfo = new DataLoadInfo(ExecuteDatasetExtractionSource.AuditTaskName, Process.GetCurrentProcess().ProcessName, _configuration.GetLoggingRunName(), "", false, _logManager.Server); } catch (Exception e) { throw new Exception("Problem occurred trying to create Logging Component:" + e.Message + " (check user has access to " + _logManager.Server + " and that the DataLoadTask '" + ExecuteDatasetExtractionSource.AuditTaskName + "' exists)", e); } return(dataLoadInfo); }
public void MD5Test() { string fileContents = "TestStringThatCouldBeSomethingInAFile"; byte[] hashAsBytes; MemoryStream memory = new MemoryStream(); StreamWriter writeToMemory = new StreamWriter(memory); writeToMemory.Write(fileContents); memory.Flush(); memory.Position = 0; using (var md5 = MD5.Create()) { hashAsBytes = md5.ComputeHash(memory); } DataSource[] ds = new DataSource[] { new DataSource("nothing", DateTime.Now) }; ds[0].MD5 = hashAsBytes; //MD5 is a property so confirm write and read are the same - and dont bomb Assert.AreEqual(ds[0].MD5, hashAsBytes); DataLoadInfo d = new DataLoadInfo("Internal", "HICSSISLibraryTests.FataErrorLoggingTest", "Test case for fatal error generation", "No rollback is possible/required as no database rows are actually inserted", true, new DiscoveredServer(UnitTestLoggingConnectionString)); TableLoadInfo t = new TableLoadInfo(d, "Unit test only", "Unit test only", ds, 5); t.Inserts += 5; //simulate that it crashed after 3 t.CloseAndArchive(); d.CloseAndMarkComplete(); }
private ExtractCommandState ExtractSupportingSql(SupportingSQLTable sql, IDataLoadEventListener listener, DataLoadInfo dataLoadInfo) { try { var tempDestination = new DataTableUploadDestination(); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to download SQL for global SupportingSQL " + sql.Name)); using (var con = sql.GetServer().GetConnection()) { con.Open(); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Connection opened successfully, about to send SQL command " + sql.SQL)); var cmd = DatabaseCommandHelper.GetCommand(sql.SQL, con); var da = DatabaseCommandHelper.GetDataAdapter(cmd); var sw = new Stopwatch(); sw.Start(); DataTable dt = new DataTable(); da.Fill(dt); dt.TableName = GetTableName(_destinationDatabase.Server.GetQuerySyntaxHelper().GetSensibleTableNameFromString(sql.Name)); var tableLoadInfo = dataLoadInfo.CreateTableLoadInfo("", dt.TableName, new[] { new DataSource(sql.SQL, DateTime.Now) }, -1); tableLoadInfo.Inserts = dt.Rows.Count; listener.OnProgress(this, new ProgressEventArgs("Reading from SupportingSQL " + sql.Name, new ProgressMeasurement(dt.Rows.Count, ProgressType.Records), sw.Elapsed)); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Decided on the following destination table name for SupportingSQL: " + dt.TableName)); tempDestination.AllowResizingColumnsAtUploadTime = true; tempDestination.PreInitialize(GetDestinationDatabase(listener), listener); tempDestination.ProcessPipelineData(dt, listener, new GracefulCancellationToken()); tempDestination.Dispose(listener, null); //end auditing it tableLoadInfo.CloseAndArchive(); if (_request is ExtractDatasetCommand) { var result = (_request as ExtractDatasetCommand).CumulativeExtractionResults; var supplementalResult = result.AddSupplementalExtractionResult(sql.SQL, sql); supplementalResult.CompleteAudit(this.GetType(), TargetDatabaseServer.ID + "|" + GetDatabaseName() + "|" + dt.TableName, dt.Rows.Count); } else { var extractGlobalsCommand = (_request as ExtractGlobalsCommand); Debug.Assert(extractGlobalsCommand != null, "extractGlobalsCommand != null"); var result = new SupplementalExtractionResults(extractGlobalsCommand.RepositoryLocator.DataExportRepository, extractGlobalsCommand.Configuration, sql.SQL, sql); result.CompleteAudit(this.GetType(), TargetDatabaseServer.ID + "|" + GetDatabaseName() + "|" + dt.TableName, dt.Rows.Count); extractGlobalsCommand.ExtractionResults.Add(result); } } } catch (Exception e) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Extraction of SupportingSQL " + sql + " failed ", e)); return(ExtractCommandState.Crashed); } return(ExtractCommandState.Completed); }
protected bool TryExtractSupportingSQLTable(SupportingSQLTable sql, DirectoryInfo directory, IExtractionConfiguration configuration, IDataLoadEventListener listener, DataLoadInfo dataLoadInfo) { try { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Preparing to extract Supporting SQL " + sql + " to directory " + directory.FullName)); Stopwatch sw = new Stopwatch(); sw.Start(); //start auditing it as a table load string target = Path.Combine(directory.FullName, sql.Name + ".csv"); var tableLoadInfo = dataLoadInfo.CreateTableLoadInfo("", target, new[] { new DataSource(sql.SQL, DateTime.Now) }, -1); TryExtractSupportingSQLTableImpl(sql, directory, configuration, listener, out int sqlLinesWritten, out string description); sw.Stop(); //end auditing it tableLoadInfo.Inserts = sqlLinesWritten; tableLoadInfo.CloseAndArchive(); if (_request is ExtractDatasetCommand) { var result = (_request as ExtractDatasetCommand).CumulativeExtractionResults; var supplementalResult = result.AddSupplementalExtractionResult(sql.SQL, sql); supplementalResult.CompleteAudit(this.GetType(), description, sqlLinesWritten); } else { var extractGlobalsCommand = (_request as ExtractGlobalsCommand); Debug.Assert(extractGlobalsCommand != null, "extractGlobalsCommand != null"); var result = new SupplementalExtractionResults(extractGlobalsCommand.RepositoryLocator.DataExportRepository, extractGlobalsCommand.Configuration, sql.SQL, sql); result.CompleteAudit(this.GetType(), description, sqlLinesWritten); extractGlobalsCommand.ExtractionResults.Add(result); } listener.OnProgress(this, new ProgressEventArgs("Extract " + sql, new ProgressMeasurement(sqlLinesWritten, ProgressType.Records), sw.Elapsed)); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Extracted " + sqlLinesWritten + " records from SupportingSQL " + sql + " into directory " + directory.FullName)); return(true); } catch (Exception e) { if (e is SqlException) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Failed to run extraction SQL (make sure to fully specify all database/table/column objects completely):" + Environment.NewLine + sql.SQL, e)); } else { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Failed to extract " + sql + " into directory " + directory.FullName, e)); } return(false); } }
private void ExtractGlobals(ExtractGlobalsCommand request, IDataLoadEventListener listener, DataLoadInfo dataLoadInfo) { var globalsDirectory = request.GetExtractionDirectory(); if (CleanExtractionFolderBeforeExtraction) { globalsDirectory.Delete(true); globalsDirectory.Create(); } foreach (var doc in request.Globals.Documents) { request.Globals.States[doc] = TryExtractSupportingDocument(doc, globalsDirectory, listener) ? ExtractCommandState.Completed : ExtractCommandState.Crashed; } foreach (var sql in request.Globals.SupportingSQL) { request.Globals.States[sql] = TryExtractSupportingSQLTable(sql, globalsDirectory, request.Configuration, listener, dataLoadInfo) ? ExtractCommandState.Completed : ExtractCommandState.Crashed; } }
public virtual void PreInitialize(DataLoadInfo value, IDataLoadEventListener listener) { _dataLoadInfo = value; }
private void ExtractLookupTableSql(BundledLookupTable lookup, IDataLoadEventListener listener, DataLoadInfo dataLoadInfo) { try { var tempDestination = new DataTableUploadDestination(); var server = DataAccessPortal.GetInstance().ExpectServer(lookup.TableInfo, DataAccessContext.DataExport); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to download SQL for lookup " + lookup.TableInfo.Name)); using (var con = server.GetConnection()) { con.Open(); var sqlString = "SELECT * FROM " + lookup.TableInfo.Name; listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Connection opened successfully, about to send SQL command: " + sqlString)); var cmd = DatabaseCommandHelper.GetCommand(sqlString, con); var da = DatabaseCommandHelper.GetDataAdapter(cmd); var sw = new Stopwatch(); sw.Start(); DataTable dt = new DataTable(); da.Fill(dt); dt.TableName = GetTableName(_destinationDatabase.Server.GetQuerySyntaxHelper().GetSensibleTableNameFromString(lookup.TableInfo.Name)); var tableLoadInfo = dataLoadInfo.CreateTableLoadInfo("", dt.TableName, new[] { new DataSource(sqlString, DateTime.Now) }, -1); tableLoadInfo.Inserts = dt.Rows.Count; listener.OnProgress(this, new ProgressEventArgs("Reading from Lookup " + lookup.TableInfo.Name, new ProgressMeasurement(dt.Rows.Count, ProgressType.Records), sw.Elapsed)); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Decided on the following destination table name for Lookup: " + dt.TableName)); tempDestination.AllowResizingColumnsAtUploadTime = true; tempDestination.PreInitialize(GetDestinationDatabase(listener), listener); tempDestination.ProcessPipelineData(dt, listener, new GracefulCancellationToken()); tempDestination.Dispose(listener, null); //end auditing it tableLoadInfo.CloseAndArchive(); if (_request is ExtractDatasetCommand) { var result = (_request as ExtractDatasetCommand).CumulativeExtractionResults; var supplementalResult = result.AddSupplementalExtractionResult("SELECT * FROM " + lookup.TableInfo.Name, lookup.TableInfo); supplementalResult.CompleteAudit(this.GetType(), TargetDatabaseServer.ID + "|" + GetDatabaseName() + "|" + dt.TableName, dt.Rows.Count); } } } catch (Exception e) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Extraction of Lookup " + lookup.TableInfo.Name + " failed ", e)); throw; } }
public void ExtractGlobals(ExtractGlobalsCommand request, IDataLoadEventListener listener, DataLoadInfo dataLoadInfo) { var globalsToExtract = request.Globals; if (globalsToExtract.Any()) { var globalsDirectory = request.GetExtractionDirectory(); if (CleanExtractionFolderBeforeExtraction) { globalsDirectory.Delete(true); globalsDirectory.Create(); } foreach (var sql in globalsToExtract.SupportingSQL) { ExtractSupportingSql(sql, listener, dataLoadInfo); } foreach (var doc in globalsToExtract.Documents) { ExtractSupportingDocument(globalsDirectory, doc, listener); } } }