/// <summary> /// Extracts the rows in <paramref name="toProcess"/> to the extraction destination /// </summary> /// <param name="toProcess"></param> /// <param name="job"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public virtual DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener job, GracefulCancellationToken cancellationToken) { _request.ElevateState(ExtractCommandState.WritingToFile); if (!haveWrittenBundleContents && _request is ExtractDatasetCommand) { WriteBundleContents(((ExtractDatasetCommand)_request).DatasetBundle, job, cancellationToken); haveWrittenBundleContents = true; } if (_request is ExtractGlobalsCommand) { ExtractGlobals((ExtractGlobalsCommand)_request, job, _dataLoadInfo); return(null); } stopwatch.Start(); if (!haveOpened) { haveOpened = true; LinesWritten = 0; Open(toProcess, job, cancellationToken); //create an audit object TableLoadInfo = new TableLoadInfo(_dataLoadInfo, "", OutputFile, new DataSource[] { new DataSource(_request.DescribeExtractionImplementation(), DateTime.Now) }, -1); } WriteRows(toProcess, job, cancellationToken, stopwatch); if (TableLoadInfo.IsClosed) { throw new Exception("TableLoadInfo was closed so could not write number of rows (" + LinesWritten + ") to audit object - most likely the extraction crashed?"); } else { TableLoadInfo.Inserts = LinesWritten; } Flush(job, cancellationToken, stopwatch); stopwatch.Stop(); return(null); }
public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { _request.ElevateState(ExtractCommandState.WritingToFile); _toProcess = toProcess; _destinationDatabase = GetDestinationDatabase(listener); //give the data table the correct name if (toProcess.ExtendedProperties.ContainsKey("ProperlyNamed") && toProcess.ExtendedProperties["ProperlyNamed"].Equals(true)) { _isTableAlreadyNamed = true; } _toProcess.TableName = GetTableName(); if (_destination == null) { _destination = PrepareDestination(listener, toProcess); } if (TableLoadInfo == null) { TableLoadInfo = new TableLoadInfo(_dataLoadInfo, "", _toProcess.TableName, new[] { new DataSource(_request.DescribeExtractionImplementation(), DateTime.Now) }, -1); } if (TableLoadInfo.IsClosed) // Maybe it was open and it creashed? { throw new Exception("TableLoadInfo was closed so could not write number of rows (" + toProcess.Rows.Count + ") to audit object - most likely the extraction crashed?"); } if (_request is ExtractDatasetCommand && !haveExtractedBundledContent) { WriteBundleContents(((ExtractDatasetCommand)_request).DatasetBundle, listener, cancellationToken); } if (_request is ExtractGlobalsCommand) { ExtractGlobals((ExtractGlobalsCommand)_request, listener, _dataLoadInfo); return(null); } _destination.ProcessPipelineData(toProcess, listener, cancellationToken); TableLoadInfo.Inserts += toProcess.Rows.Count; return(null); }