private void CloseFile(IDataLoadEventListener listener) { //we never even started or have already closed if (!haveOpened || _fileAlreadyClosed) { return; } _fileAlreadyClosed = true; try { //whatever happens in the writing block, make sure to at least attempt to close off the file _output.Close(); GC.Collect(); //prevents file locks from sticking around //close audit object - unless it was prematurely closed e.g. by a failure somewhere if (!TableLoadInfo.IsClosed) { TableLoadInfo.CloseAndArchive(); } // also close off the cumulative extraction result var result = ((IExtractDatasetCommand)_request).CumulativeExtractionResults; if (result != null) { result.CompleteAudit(this.GetType(), GetDestinationDescription(), LinesWritten); } } catch (Exception e) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Error when trying to close csv file", e)); } }
public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener job, GracefulCancellationToken cancellationToken) { _request.ElevateState(ExtractCommandState.WritingToFile); if (!haveWrittenBundleContents && _request is ExtractDatasetCommand) { WriteBundleContents(((ExtractDatasetCommand)_request).DatasetBundle, job, cancellationToken); } if (_request is ExtractGlobalsCommand) { ExtractGlobals((ExtractGlobalsCommand)_request, job, _dataLoadInfo); return(null); } stopwatch.Start(); if (!haveOpened) { haveOpened = true; _output.Open(); _output.WriteHeaders(toProcess); LinesWritten = 0; //create an audit object TableLoadInfo = new TableLoadInfo(_dataLoadInfo, "", OutputFile, new DataSource[] { new DataSource(_request.DescribeExtractionImplementation(), DateTime.Now) }, -1); } foreach (DataRow row in toProcess.Rows) { _output.Append(row); LinesWritten++; if (TableLoadInfo.IsClosed) { throw new Exception("TableLoadInfo was closed so could not write number of rows (" + LinesWritten + ") to audit object - most likely the extraction crashed?"); } else { TableLoadInfo.Inserts = LinesWritten; } if (LinesWritten % 1000 == 0) { job.OnProgress(this, new ProgressEventArgs("Write to file " + OutputFile, new ProgressMeasurement(LinesWritten, ProgressType.Records), stopwatch.Elapsed)); } } job.OnProgress(this, new ProgressEventArgs("Write to file " + OutputFile, new ProgressMeasurement(LinesWritten, ProgressType.Records), stopwatch.Elapsed)); stopwatch.Stop(); _output.Flush(); return(null); }
public override void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny) { if (_destination != null) { _destination.Dispose(listener, pipelineFailureExceptionIfAny); //if the extraction failed, the table didn't exist in the destination (i.e. the table was created during the extraction) and we are to DropTableIfLoadFails if (pipelineFailureExceptionIfAny != null && _tableDidNotExistAtStartOfLoad && DropTableIfLoadFails) { if (_destinationDatabase != null) { var tbl = _destinationDatabase.ExpectTable(_toProcess.TableName); if (tbl.Exists()) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "DropTableIfLoadFails is true so about to drop table " + tbl)); tbl.Drop(); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Dropped table " + tbl)); } } } if (pipelineFailureExceptionIfAny == null && _request.IsBatchResume && MakeFinalTableDistinctWhenBatchResuming && _destinationDatabase != null && _toProcess != null) { var tbl = _destinationDatabase.ExpectTable(_toProcess.TableName); if (tbl.Exists()) { // if there is no primary key then failed batches may have introduced duplication if (!tbl.DiscoverColumns().Any(p => p.IsPrimaryKey)) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Making {tbl} distinct incase there are duplicate rows from bad batch resumes")); tbl.MakeDistinct(50000000); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Finished distincting {tbl}")); } } } } TableLoadInfo?.CloseAndArchive(); // also close off the cumulative extraction result if (_request is ExtractDatasetCommand) { var result = ((IExtractDatasetCommand)_request).CumulativeExtractionResults; if (result != null && _toProcess != null) { result.CompleteAudit(this.GetType(), GetDestinationDescription(), TableLoadInfo.Inserts, _request.IsBatchResume, pipelineFailureExceptionIfAny != null); } } }
public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { _request.ElevateState(ExtractCommandState.WritingToFile); _toProcess = toProcess; _destinationDatabase = GetDestinationDatabase(listener); //give the data table the correct name if (toProcess.ExtendedProperties.ContainsKey("ProperlyNamed") && toProcess.ExtendedProperties["ProperlyNamed"].Equals(true)) { _isTableAlreadyNamed = true; } _toProcess.TableName = GetTableName(); if (_destination == null) { _destination = PrepareDestination(listener, toProcess); } if (TableLoadInfo == null) { TableLoadInfo = new TableLoadInfo(_dataLoadInfo, "", _toProcess.TableName, new[] { new DataSource(_request.DescribeExtractionImplementation(), DateTime.Now) }, -1); } if (TableLoadInfo.IsClosed) // Maybe it was open and it creashed? { throw new Exception("TableLoadInfo was closed so could not write number of rows (" + toProcess.Rows.Count + ") to audit object - most likely the extraction crashed?"); } if (_request is ExtractDatasetCommand && !haveExtractedBundledContent) { WriteBundleContents(((ExtractDatasetCommand)_request).DatasetBundle, listener, cancellationToken); } if (_request is ExtractGlobalsCommand) { ExtractGlobals((ExtractGlobalsCommand)_request, listener, _dataLoadInfo); return(null); } _destination.ProcessPipelineData(toProcess, listener, cancellationToken); TableLoadInfo.Inserts += toProcess.Rows.Count; return(null); }
public void FataErrorLoggingTest() { DataLoadInfo d = new DataLoadInfo("Internal", "HICSSISLibraryTests.FataErrorLoggingTest", "Test case for fatal error generation", "No rollback is possible/required as no database rows are actually inserted", true, new DiscoveredServer(UnitTestLoggingConnectionString)); DataSource[] ds = new DataSource[] { new DataSource("nothing", DateTime.Now) }; TableLoadInfo t = new TableLoadInfo(d, "Unit test only", "Unit test only", ds, 5); t.Inserts += 3; //simulate that it crashed after 3 d.LogFatalError("HICSSISLibraryTests.FataErrorLoggingTest", "Some terrible event happened"); Assert.IsTrue(d.IsClosed); }
/// <summary> /// Extracts the rows in <paramref name="toProcess"/> to the extraction destination /// </summary> /// <param name="toProcess"></param> /// <param name="job"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public virtual DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener job, GracefulCancellationToken cancellationToken) { _request.ElevateState(ExtractCommandState.WritingToFile); if (!haveWrittenBundleContents && _request is ExtractDatasetCommand) { WriteBundleContents(((ExtractDatasetCommand)_request).DatasetBundle, job, cancellationToken); haveWrittenBundleContents = true; } if (_request is ExtractGlobalsCommand) { ExtractGlobals((ExtractGlobalsCommand)_request, job, _dataLoadInfo); return(null); } stopwatch.Start(); if (!haveOpened) { haveOpened = true; LinesWritten = 0; Open(toProcess, job, cancellationToken); //create an audit object TableLoadInfo = new TableLoadInfo(_dataLoadInfo, "", OutputFile, new DataSource[] { new DataSource(_request.DescribeExtractionImplementation(), DateTime.Now) }, -1); } WriteRows(toProcess, job, cancellationToken, stopwatch); if (TableLoadInfo.IsClosed) { throw new Exception("TableLoadInfo was closed so could not write number of rows (" + LinesWritten + ") to audit object - most likely the extraction crashed?"); } else { TableLoadInfo.Inserts = LinesWritten; } Flush(job, cancellationToken, stopwatch); stopwatch.Stop(); return(null); }