public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { base.PrimeOutput(outputs, outputIDs, buffers); PipelineBuffer buffer = buffers[0]; var dataTable = LoadDataTable(DataTableXml); foreach (DataRow dataRow in dataTable.Rows) { buffer.AddRow(); foreach (var columnMapping in _bufferColumnMapping) { // TODO: Update to use the correct data type methods if (dataRow[columnMapping.Key] == DBNull.Value) { buffer.SetNull(columnMapping.Value); } else { buffer[columnMapping.Value] = dataRow[columnMapping.Key]; } } } buffer.SetEndOfRowset(); }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { base.PrimeOutput(outputs, outputIDs, buffers); IDTSOutput100 output = ComponentMetaData.OutputCollection.FindObjectByID(outputIDs[0]); PipelineBuffer buffer = buffers[0]; String filePathVar = ComponentMetaData.CustomPropertyCollection[_FILE_PATH_VARIABLE_FIELD].Value.ToString(); String sheetNameVar = ComponentMetaData.CustomPropertyCollection[_SHEET_NAME_FIELD].Value.ToString(); DataTable dt = GetDataTableWithInputVar(filePathVar, sheetNameVar); foreach (DataRow row in dt.Rows) { buffer.AddRow(); for (int x = 0; x < mapOutputColsToBufferCols.Length; x++) { if (row.IsNull(x)) { buffer.SetNull(mapOutputColsToBufferCols[x]); } else { buffer[mapOutputColsToBufferCols[x]] = row[x]; } } } buffer.SetEndOfRowset(); }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { //System.Diagnostics.Debugger.Launch(); base.PrimeOutput(outputs, outputIDs, buffers); IDTSOutput100 output = ComponentMetaData.OutputCollection.FindObjectByID(outputIDs[0]); PipelineBuffer buffer = buffers[0]; DataTable dt = new DataTable(); dt = GetData(ComponentMetaData.CustomPropertyCollection["FetchXML"].Value.ToString(), false); foreach (DataRow row in dt.Rows) { buffer.AddRow(); for (int x = 0; x < mapOutputColsToBufferCols.Length; x++) { if (row.IsNull(x)) { buffer.SetNull(mapOutputColsToBufferCols[x]); } else { buffer[mapOutputColsToBufferCols[x]] = row[x]; } } } buffer.SetEndOfRowset(); }
public override void ProcessInput(int inputID, PipelineBuffer inputBuffer) { base.ProcessInput(inputID, inputBuffer); while (inputBuffer.NextRow()) { var inputString = inputBuffer.GetString(0); var order = new Order() { OrderCode = inputString, Product = new Product() { Name = inputString + "Product" } }; var outputXml = XmlSerializer.XmlSerialize(order); var outputBytes = Encoding.UTF8.GetBytes(outputXml); outputBuffer.AddRow(); outputBuffer.AddBlobData(0, outputBytes); } if (inputBuffer.EndOfRowset) { outputBuffer.SetEndOfRowset(); } }
public override void ProcessInput(int inputID, PipelineBuffer inputBuffer) { while (inputBuffer.NextRow()) { var parameters = GetWebServiceParameters(inputBuffer); var response = WebService.read(X3WebService.Context, PublicIdentifierValue, parameters); var xml = new XmlDocument(); xml.LoadXml(response.resultXml); var responseXmlWrapper = new X3ObjectResponseWrapper(xml); outputBuffer.AddRow(); foreach (var groupTag in responseXmlWrapper.ResultGroups) { foreach (var fieldTag in groupTag.Fields) { var columnName = GenerateOutputColumnName(groupTag.Id, fieldTag.Name); var outputCollumnIndex = GetOutputColumnIndex(columnName); var outputColumn = GetOutputColumn(columnName); SetColumnData(outputBuffer, outputCollumnIndex, fieldTag.Value, outputColumn.DataType); } } } if (inputBuffer.EndOfRowset) { outputBuffer.SetEndOfRowset(); } }
private void AddOutputRow(ref PipelineBuffer inputbuffer) { // For some RESON I STILL DO NOT UNDERSTAND, THIS METHOD FALLS in a sort of race condition. // The following is the worst workaroud ever, but needed ito quickly address the problem. while (true) { try { _outputBuffer.AddRow(); break; } catch (Exception e) { ComponentMetaData.FireWarning(ComponentConstants.RUNTIME_GENERIC_ERROR, ComponentMetaData.Name, "Outputrow was not ready. ", null, 0); throw e; } } // Copy the inputs into outputs for (var i = 0; i < _startOfJsonColIndex; i++) { if (inputbuffer[i] is BlobColumn) { _outputBuffer.AddBlobData(i, inputbuffer.GetBlobData(i, 0, (int)inputbuffer.GetBlobLength(i))); } else { _outputBuffer[i] = inputbuffer[i]; } } }
public override void ProcessInput(int inputID, PipelineBuffer inputBuffer) { base.ProcessInput(inputID, inputBuffer); var jsSerializer = new JavaScriptSerializer(); while (inputBuffer.NextRow()) { var inputBytes = inputBuffer.GetBlobData(0, 0, (int)inputBuffer.GetBlobLength(0)); var inputXml = Encoding.UTF8.GetString(inputBytes); var inputObject = XmlSerializer.XmlDeserialize <Order>(inputXml); var serializedObject = jsSerializer.Serialize(inputObject); if (serializedObject.Length > 4000 - 4) { serializedObject = serializedObject.Substring(0, 4000 - 4) + " ..."; } outputBuffer.AddRow(); outputBuffer.SetString(0, serializedObject); } if (inputBuffer.EndOfRowset) { outputBuffer.SetEndOfRowset(); } }
/// <summary> /// Write a json document to the output buffers /// </summary> /// <param name="id">The document key</param> /// <param name="json">The JSON representation of the document</param> /// <param name="model">A model used to decompose the document</param> /// <param name="outputIDs">Indexes of the output buffers</param> /// <param name="buffers">The Pipline Buffers</param> private void writeDocToBuffers(string id, string json, JSONDataModel model, int[] outputIDs, PipelineBuffer[] buffers) { // Deserialize the document JsonObject jp = new JsonObject(json); // For each model table foreach (DataTable modelTable in model.Tables) { // Obtain the output IDTSOutput100 output = ComponentMetaData.OutputCollection[modelTable.TableName]; // Continue if output is not connected if (Array.IndexOf(outputIDs, output.ID) == -1) { continue; } // Obtain the buffer tho write to PipelineBuffer buffer = buffers[Array.IndexOf(outputIDs, output.ID)]; // If we are processing a table that has indexes on the second column // this must be a child table if (modelTable.Rows.Count > 2 && modelTable.Rows[1]["xpath"].ToString().StartsWith("#")) { // Go and process the related table processChildTable(id, jp, model, modelTable.TableName, outputIDs, buffers); } else { // Add a new row to the buffer buffer.AddRow(); // For every column definition in teh model foreach (DataRow row in modelTable.Rows) { // Find the column index that the buffer will recognize int columnIndex = columnIndexes[modelTable.TableName][row["shortColumn"].ToString()]; // Get the model datatype Type dataType = Type.GetType(row["datatype"].ToString()); // And XPath string xpath = row["xpath"].ToString(); // Handle special case of DOC_ID if (xpath.Equals("#ID")) { buffer.SetString(columnIndex, id); } else { // Write all other fields as usual writeDataToBuffer(jp, buffer, columnIndex, dataType, xpath); } } } } }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { IDTSOutput100 output = ComponentMetaData.OutputCollection[0]; PipelineBuffer buffer = buffers[0]; try { foreach (ManagementObject row in ExecWQL(false)) { object[] row_values = new object[m_columnInfo.Count]; int i = 0; foreach (ColumnInfo ci in m_columnInfo) { PropertyData wmiColumn = row.Properties[ci.ColumnName]; row_values[i++] = row[wmiColumn.Name]; } // lets unwind wmi row ( unwind array in any column ) // we use the same approach to handle WMI arrays as WMI ODBC driver // for details see msdn : // http://msdn.microsoft.com/en-us/library/aa392328(VS.85).aspx#_hmm_mapping_wmi_arrays_to_odbc foreach (object[] unwinded_row in RowUnwinder.UnwindRow(row_values)) { buffer.AddRow(); i = 0; foreach (ColumnInfo ci in m_columnInfo) { PropertyData wmiColumn = row.Properties[ci.ColumnName]; SetBufferColumn(buffer, row, wmiColumn, ci.BufferColumnIndex, unwinded_row[i++]); } } } // set end of data on all of the buffers buffer.SetEndOfRowset(); } catch (Exception e) { bool bCancel; ErrorSupport.FireErrorWithArgs(HResults.DTS_E_PRIMEOUTPUTFAILED, out bCancel, ComponentMetaData.IdentificationString, e.Message); throw new PipelineComponentHResultException(HResults.DTS_E_PRIMEOUTPUTFAILED); } }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { //On récupère le buffer mémoire dans lequel seront stockées les lignes PipelineBuffer buffer = buffers[0]; //Récupération de la chaine de connexion au stockage blob Azure string storageConnectionString = (string)this.ComponentMetaData.CustomPropertyCollection["StorageConnectionString"].Value; string tableName = (string)this.ComponentMetaData.CustomPropertyCollection["TableName"].Value; //Connexion au stockage CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(storageConnectionString); CloudTableClient tableClient = cloudStorageAccount.CreateCloudTableClient(); CloudTable table = tableClient.GetTableReference(tableName); //Récupération des lignes TableQuery <DynamicTableEntity> query = new TableQuery <DynamicTableEntity>(); var azureTable = table.ExecuteQuery(query); foreach (DynamicTableEntity row in azureTable) { //Ajout d'une ligne au buffer buffer.AddRow(); var properties = row.Properties; int i = 0; //Itération sur chaques colonnes foreach (var col in properties) { if (col.Value != null) { buffer[i] = col.Value.PropertyAsObject; } else { buffer.SetNull(i); } i++; } } //Envoie du buffer buffer.SetEndOfRowset(); }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { IDTSOutput100 output = ComponentMetaData.OutputCollection[0]; PipelineBuffer buffer = buffers[0]; BasicGetResult result = consumerChannel.BasicGet(queueName, true); while (result != null) { string messageContent = Encoding.UTF8.GetString(result.Body); buffer.AddRow(); buffer[0] = messageContent; buffer[1] = result.RoutingKey; result = consumerChannel.BasicGet(queueName, true); } buffer.SetEndOfRowset(); }
public override void ProcessInput(int inputID, PipelineBuffer inputBuffer) { base.ProcessInput(inputID, inputBuffer); while (inputBuffer.NextRow()) { var inputBytes = inputBuffer.GetBlobData(0, 0, (int)inputBuffer.GetBlobLength(0)); var inputXml = Encoding.UTF8.GetString(inputBytes); var inputObject = XmlSerializer.XmlDeserialize <TSource>(inputXml); var outputObject = ProcessInput(inputObject); var outputXml = XmlSerializer.XmlSerialize(outputObject); var outputBytes = Encoding.UTF8.GetBytes(outputXml); outputBuffer.AddRow(); outputBuffer.AddBlobData(0, outputBytes); } if (inputBuffer.EndOfRowset) { outputBuffer.SetEndOfRowset(); } }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { IDTSOutput100 output = ComponentMetaData.OutputCollection[0]; PipelineBuffer buffer = buffers[0]; object message; bool success; while (queueConsumer.IsRunning) { try { success = queueConsumer.Queue.Dequeue(100, out message); } catch (Exception) { break; } if (success) { BasicDeliverEventArgs e = (BasicDeliverEventArgs)message; var messageContent = System.Text.Encoding.UTF8.GetString(e.Body); buffer.AddRow(); buffer[0] = messageContent; buffer[1] = e.RoutingKey; } else { break; } } buffer.SetEndOfRowset(); }
/// <summary> /// Will perform the user-specified behaviour when a processing error occurs /// </summary> /// <param name="disposition">How the error should be handled</param> /// <param name="defaultBuffer">The default output buffer</param> /// <param name="errorBuffer">The error output buffer</param> /// <param name="failingColumnInfo">The information for the problematic column</param> /// <param name="ex">The exception caught from processing (optional)</param> private void HandleProcessingError(DTSRowDisposition disposition, PipelineBuffer defaultBuffer, PipelineBuffer errorBuffer, ColumnInfo failingColumnInfo, Exception ex) { switch (disposition) { case DTSRowDisposition.RD_RedirectRow: if (errorBuffer == null) { throw new InvalidOperationException("There must be an error output defined if redirection was specified"); } // Add a row to the error buffer. errorBuffer.AddRow(); // Get the values from the default buffer // and copy them to the error buffer. var errorOutputColumns = GetErrorOutputColumns().ToArray(); foreach (IDTSOutputColumn100 column in errorOutputColumns) { ColumnInfo copiedColumnInfo = GetColumnInfo(column.Name); if (copiedColumnInfo != null) { errorBuffer[copiedColumnInfo.ErrorOuputBufferColumnIndex] = defaultBuffer[copiedColumnInfo.OuputBufferColumnIndex]; } } // Set the error information. int errorCode = (ex == null ? 0 : System.Runtime.InteropServices.Marshal.GetHRForException(ex)); errorBuffer.SetErrorInfo(errorOutputID, errorCode, failingColumnInfo.OutputColumn.LineageID); // Remove the row that was added to the default buffer. defaultBuffer.RemoveRow(); break; case DTSRowDisposition.RD_FailComponent: throw new Exception(String.Format("There was an issue with column: {0}", failingColumnInfo.OutputColumn.Name), ex); } }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { base.PrimeOutput(outputs, outputIDs, buffers); IDTSOutput100 output = ComponentMetaData.OutputCollection.FindObjectByID(outputIDs[0]); PipelineBuffer buffer = buffers[0]; DataTable dt = GetRSSDataTable(ComponentMetaData.CustomPropertyCollection["RSS Path"].Value.ToString()); foreach (DataRow row in dt.Rows) { buffer.AddRow(); for (int x = 0; x < mapOutputColsToBufferCols.Length; x++) { if (row.IsNull(x)) buffer.SetNull(mapOutputColsToBufferCols[x]); else buffer[mapOutputColsToBufferCols[x]] = row[x]; } } buffer.SetEndOfRowset(); }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { //Do some stuff base.PrimeOutput(outputs, outputIDs, buffers); //SSIS STUFF IDTSOutput100 output = ComponentMetaData.OutputCollection.FindObjectByID(outputIDs[0]); PipelineBuffer buffer = buffers[0]; string path = ComponentMetaData.CustomPropertyCollection["FTP Path"].Value; int nrOfDays = ComponentMetaData.CustomPropertyCollection["FTP Files Not Older Then (Days)"].Value; //Get path part of file path string pathPart = System.IO.Path.GetDirectoryName(path); //ftp://ftp.matchit.no/Maintenance/Archive/*.* FileStruct[] files = _ftpHelper.GetFiles(path); foreach (FileStruct file in files) { //SKIP IF FILS IS OLDER THEN x DAYS if (file.CreateTime < DateTime.Now.AddDays(-nrOfDays)) { continue; } //BUILD THE PATH FOR EACH FILE string filePath = System.IO.Path.Combine(pathPart, file.Name); //Switch "\" with "/" filePath = filePath.Replace(@"\", @"/"); UriBuilder builder = new UriBuilder(); builder.Scheme = "ftp"; builder.Host = _ftpHelper.HostName; builder.Path = filePath; //GOT THE PATH String url = builder.ToString(); //GET THE STREAM FtpWebRequest request = (FtpWebRequest)WebRequest.Create(url); request.Method = WebRequestMethods.Ftp.DownloadFile; //Login string usr = ComponentMetaData.CustomPropertyCollection["FTP User"].Value; string pwd = ComponentMetaData.CustomPropertyCollection["FTP Password"].Value; request.Credentials = new NetworkCredential(usr, pwd); FtpWebResponse response = (FtpWebResponse)request.GetResponse(); //StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, true); Stream responseStream = response.GetResponseStream(); StreamReader reader = new StreamReader(responseStream); //Get lines to skip int rowsToSkip = ComponentMetaData.CustomPropertyCollection["Skip rows"].Value; //READ THE STREAM int currentLineNumber = 0; while (!reader.EndOfStream) { try { //Read line string line = reader.ReadLine(); //Skip x number of rows if (rowsToSkip > currentLineNumber) { currentLineNumber++; continue; } //splitt line string separator = ComponentMetaData.CustomPropertyCollection["Separator"].Value; char c = separator.ToCharArray()[0]; string[] values = line.Split(c); //create output buffer buffer.AddRow(); int i = 0; foreach (CustomDataColumn column in _columnsList) { switch (column.Type) { case "I4": buffer[i] = int.Parse(values[i]); break; case "WSTR": buffer[i] = values[i]; break; case "DBTIMESTAMP": string dateFormatString = column.Length; //For DateTime "length" is used for describing datetime format buffer[i] = DateTime.ParseExact(values[i], dateFormatString, CultureInfo.InvariantCulture); break; default: throw new NotImplementedException("Error, " + column.Type + " is not implemented"); } i++; } currentLineNumber++; } catch (Exception e) { buffer.DirectErrorRow(0, 1, currentLineNumber); } } } buffer.SetEndOfRowset(); }
private void ParseTheFileAndAddToBuffer(string filename, PipelineBuffer buffer) { TextReader tr = File.OpenText(filename); IDTSOutput100 output = ComponentMetaData.OutputCollection["Component Output"]; IDTSOutputColumnCollection100 cols = output.OutputColumnCollection; IDTSOutputColumn100 col; string s = tr.ReadLine(); int i = 0; while (s != null) { if (s.StartsWith("<START>")) buffer.AddRow(); if (s.StartsWith("Name:")) { col = cols["Name"]; i = BufferManager.FindColumnByLineageID(output.Buffer, col.LineageID); string value = s.Substring(5); buffer.SetString(i, value); } if (s.StartsWith("Age:")) { col = cols["Age"]; i = BufferManager.FindColumnByLineageID(output.Buffer, col.LineageID); Int32 value; if (s.Substring(4).Trim() == "") value = 0; else value = Convert.ToInt32(s.Substring(4).Trim()); buffer.SetInt32(i, value); } if (s.StartsWith("Married:")) { col = cols["Married"]; bool value; i = BufferManager.FindColumnByLineageID(output.Buffer, col.LineageID); if (s.Substring(8).Trim() == "") value = true; else value = s.Substring(8).Trim() != "1" ? false : true; buffer.SetBoolean(i, value); } if (s.StartsWith("Salary:")) { col = cols["Salary"]; Decimal value; i = BufferManager.FindColumnByLineageID(output.Buffer, col.LineageID); if (s.Substring(7).Trim() == "") value = 0M; else value = Convert.ToDecimal(s.Substring(8).Trim()); buffer.SetDecimal(i, value); } s = tr.ReadLine(); } tr.Close(); }
/// <summary> /// This is where the data is loaded into the output buffer /// </summary> /// <param name="outputs"></param> /// <param name="outputIDs"></param> /// <param name="buffers"></param> public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { string sharepointUrl = (string)ComponentMetaData.CustomPropertyCollection[C_SHAREPOINTSITEURL].Value; string sharepointList = (string)ComponentMetaData.CustomPropertyCollection[C_SHAREPOINTLISTNAME].Value; string sharepointListView = (string)ComponentMetaData.CustomPropertyCollection[C_SHAREPOINTLISTVIEWNAME].Value; XElement camlQuery = XElement.Parse((string)ComponentMetaData.CustomPropertyCollection[C_CAMLQUERY].Value); short batchSize = (short)ComponentMetaData.CustomPropertyCollection[C_BATCHSIZE].Value; Enums.TrueFalseValue isRecursive = (Enums.TrueFalseValue)ComponentMetaData.CustomPropertyCollection[C_ISRECURSIVE].Value; Enums.TrueFalseValue includeFolders = (Enums.TrueFalseValue)ComponentMetaData.CustomPropertyCollection[C_INCLUDEFOLDERS].Value; PipelineBuffer outputBuffer = buffers[0]; // Get the field names from the output collection var fieldNames = (from col in ComponentMetaData.OutputCollection[0].OutputColumnCollection.Cast <IDTSOutputColumn>() select(string) col.CustomPropertyCollection[0].Value); // Load the data from sharepoint System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch(); timer.Start(); var listData = SharePointUtility.ListServiceUtility.GetListItemData( new Uri(sharepointUrl), _credentials, sharepointList, sharepointListView, fieldNames, camlQuery, isRecursive == Enums.TrueFalseValue.True ? true : false, batchSize); timer.Stop(); bool fireAgain = false; int actualRowCount = 0; foreach (var row in listData) { // Determine if we should continue based on if this is a folder item or not (filter can be pushed up to CAML if // perf becomes an issue) bool canContinue = true; if ((row.ContainsKey("ContentType")) && (row["ContentType"] == "Folder") && (includeFolders == Enums.TrueFalseValue.False)) { canContinue = false; } if (canContinue) { actualRowCount++; outputBuffer.AddRow(); foreach (var fieldName in _bufferLookup.Keys) { if (row.ContainsKey(fieldName)) { switch (_bufferLookupDataType[fieldName]) { case DataType.DT_NTEXT: outputBuffer.AddBlobData(_bufferLookup[fieldName], Encoding.Unicode.GetBytes(row[fieldName].ToString())); break; case DataType.DT_WSTR: outputBuffer.SetString(_bufferLookup[fieldName], row[fieldName]); break; case DataType.DT_R8: outputBuffer.SetDouble(_bufferLookup[fieldName], double.Parse(row[fieldName], _culture)); break; case DataType.DT_I4: outputBuffer.SetInt32(_bufferLookup[fieldName], int.Parse(row[fieldName], _culture)); break; case DataType.DT_BOOL: outputBuffer.SetBoolean(_bufferLookup[fieldName], (int.Parse(row[fieldName], _culture) == 1)); break; case DataType.DT_GUID: outputBuffer.SetGuid(_bufferLookup[fieldName], new Guid(row[fieldName])); break; case DataType.DT_DBTIMESTAMP: outputBuffer.SetDateTime(_bufferLookup[fieldName], DateTime.Parse(row[fieldName], _culture)); break; } } else { switch (_bufferLookupDataType[fieldName]) { case DataType.DT_NTEXT: outputBuffer.AddBlobData(_bufferLookup[fieldName], Encoding.Unicode.GetBytes(String.Empty)); break; case DataType.DT_WSTR: outputBuffer.SetString(_bufferLookup[fieldName], String.Empty); break; case DataType.DT_BOOL: outputBuffer.SetBoolean(_bufferLookup[fieldName], false); break; default: outputBuffer.SetNull(_bufferLookup[fieldName]); break; } } } } } string infoMsg = string.Format(CultureInfo.InvariantCulture, "Loaded {0} records from list '{1}' at '{2}'. Elapsed time is {3}ms", actualRowCount, sharepointList, sharepointUrl, timer.ElapsedMilliseconds); ComponentMetaData.FireInformation(0, ComponentMetaData.Name, infoMsg, "", 0, ref fireAgain); ComponentMetaData.IncrementPipelinePerfCounter( DTS_PIPELINE_CTR_ROWSREAD, (uint)actualRowCount); outputBuffer.SetEndOfRowset(); }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { IDTSOutput100 output = ComponentMetaData.OutputCollection[0]; PipelineBuffer buffer = buffers[0]; // A ListRowsContinuationToken class encapsulates the partition and row key ListRowsContinuationToken continuationToken = null; string strSource = (string)this.ComponentMetaData.CustomPropertyCollection["TableName"].Value; string srcTable = string.Empty; string srcPartitionKey = string.Empty; if (strSource != string.Empty) { string[] str1 = strSource.Split(';'); srcTable = str1[0].Split('=')[1]; srcPartitionKey = str1[1].Split('=')[1]; } do { var allItems = this.context.CreateQuery <GenericEntity>(srcTable).Where(item => item.PartitionKey == srcPartitionKey); var query = allItems as DataServiceQuery <GenericEntity>; if (continuationToken != null) { query = query.AddQueryOption("NextPartitionKey", continuationToken.PartitionKey); if (continuationToken.RowKey != null) { query = query.AddQueryOption("NextRowKey", continuationToken.RowKey); } } var response = query.Execute() as QueryOperationResponse; if (response.Headers.ContainsKey("x-ms-continuation-NextPartitionKey")) { continuationToken = new ListRowsContinuationToken(); continuationToken.PartitionKey = response.Headers["x-ms-continuation-NextPartitionKey"]; if (response.Headers.ContainsKey("x-ms-continuation-NextRowKey")) { continuationToken.RowKey = response.Headers["x-ms-continuation-NextRowKey"]; } } else { continuationToken = null; } foreach (var item in allItems) { buffer.AddRow(); for (int x = 0; x < columnInformation.Count; x++) { var ci = (ColumnInfo)columnInformation[x]; var value = item[ci.ColumnName].Value; if (value != null) { buffer[ci.BufferColumnIndex] = value; } else { buffer.SetNull(ci.BufferColumnIndex); } } } }while (continuationToken != null); buffer.SetEndOfRowset(); }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { //identify buffers PipelineBuffer errorBuffer = null; PipelineBuffer defaultBuffer = null; for (int x = 0; x < outputs; x++) { if (outputIDs[x] == errorOutputID) { errorBuffer = buffers[x]; } else { defaultBuffer = buffers[x]; } } //get ogrlayer and ogrlayer feature definition Layer OGRLayer; bool isSQLLayer = (!(ComponentMetaData.CustomPropertyCollection["SQL Statement"].Value == null || ComponentMetaData.CustomPropertyCollection["SQL Statement"].Value.ToString() == string.Empty)); if (isSQLLayer) { OGRLayer = getSQLLayer(); } else { OGRLayer = getLayer(); } Feature OGRFeature; FeatureDefn OGRFeatureDef = OGRLayer.GetLayerDefn(); //initialize columnInfo object columnInfo ci = new columnInfo(); //for each row in ogrlayer add row to output buffer while ((OGRFeature = OGRLayer.GetNextFeature()) != null) { try { defaultBuffer.AddRow(); //set buffer column values for (int i = 0; i < this.columnInformation.Count; i++) { ci = (columnInfo)this.columnInformation[i]; if (ci.geom) { Geometry geom = OGRFeature.GetGeometryRef(); if (geom != null) { byte[] geomBytes = new byte[geom.WkbSize()]; geom.ExportToWkb(geomBytes); defaultBuffer.AddBlobData(ci.bufferColumnIndex, geomBytes); } } else { int OGRFieldIndex = OGRFeatureDef.GetFieldIndex(ci.columnName); FieldDefn OGRFieldDef = OGRFeatureDef.GetFieldDefn(OGRFieldIndex); FieldType OGRFieldType = OGRFieldDef.GetFieldType(); //declare datetime variables int pnYear, pnMonth, pnDay, pnHour, pnMinute, pnTZFlag; float pnSecond; DateTime dt; TimeSpan ts; switch (OGRFieldType) { //case FieldType.OFTBinary: // break; case FieldType.OFTDate: OGRFeature.GetFieldAsDateTime(OGRFieldIndex, out pnYear, out pnMonth, out pnDay, out pnHour, out pnMinute, out pnSecond, out pnTZFlag); dt = new DateTime(pnYear, pnMonth, pnDay); defaultBuffer.SetDate(ci.bufferColumnIndex, dt); break; case FieldType.OFTDateTime: OGRFeature.GetFieldAsDateTime(OGRFieldIndex, out pnYear, out pnMonth, out pnDay, out pnHour, out pnMinute, out pnSecond, out pnTZFlag); dt = new DateTime(pnYear, pnMonth, pnDay, pnHour, pnMinute, (int)pnSecond); //set time zone? defaultBuffer.SetDateTime(ci.bufferColumnIndex, dt); break; case FieldType.OFTInteger: defaultBuffer.SetInt32(ci.bufferColumnIndex, OGRFeature.GetFieldAsInteger(OGRFieldIndex)); break; case FieldType.OFTInteger64: defaultBuffer.SetInt64(ci.bufferColumnIndex, OGRFeature.GetFieldAsInteger(OGRFieldIndex)); break; case FieldType.OFTReal: defaultBuffer.SetDouble(ci.bufferColumnIndex, OGRFeature.GetFieldAsDouble(OGRFieldIndex)); break; case FieldType.OFTTime: OGRFeature.GetFieldAsDateTime(OGRFieldIndex, out pnYear, out pnMonth, out pnDay, out pnHour, out pnMinute, out pnSecond, out pnTZFlag); ts = new TimeSpan(pnHour, pnMinute, (int)pnSecond); defaultBuffer.SetTime(ci.bufferColumnIndex, ts); break; case FieldType.OFTString: default: defaultBuffer.SetString(ci.bufferColumnIndex, OGRFeature.GetFieldAsString(OGRFieldIndex)); break; } } } } catch (Exception ex) { //redirect to error buffer if (ci.errorDisposition == DTSRowDisposition.RD_RedirectRow) { // Add a row to the error buffer. errorBuffer.AddRow(); // Set the error information. int errorCode = System.Runtime.InteropServices.Marshal.GetHRForException(ex); errorBuffer.SetErrorInfo(errorOutputID, errorCode, ci.lineageID); // Remove the row that was added to the default buffer. defaultBuffer.RemoveRow(); } //fail component else if (ci.errorDisposition == DTSRowDisposition.RD_FailComponent || ci.errorDisposition == DTSRowDisposition.RD_NotUsed) { ComponentMetaData.FireError(0, "primeoutput failure", ex.ToString(), string.Empty, 0, out cancel); throw; } } } //set end of rowset for buffers if (defaultBuffer != null) { defaultBuffer.SetEndOfRowset(); } if (errorBuffer != null) { errorBuffer.SetEndOfRowset(); } //clean up layer object if (isSQLLayer) { this.OGRDataSource.ReleaseResultSet(OGRLayer); } }
/// <summary> /// Called at run time for source components and transformation components with /// asynchronous outputs to let these components add rows to the output buffers. /// </summary> /// <param name="outputs">The number of elements in the outputIDs and buffers arrays.</param> /// <param name="outputIDs">An array of Microsoft.SqlServer.Dts.Pipeline.Wrapper.IDTSOutput100 ID's.</param> /// <param name="buffers">An array of Microsoft.SqlServer.Dts.Pipeline.PipelineBuffer objects.</param> public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { // Determine which buffer is for regular output, and which is for error output PipelineBuffer errorBuffer = null; PipelineBuffer defaultBuffer = null; for (int outputIndex = 0; outputIndex < outputs; outputIndex++) { if (outputIDs[outputIndex] == errorOutputID) { errorBuffer = buffers[outputIndex]; } else { defaultBuffer = buffers[outputIndex]; } } var cursor = GetCollectionCursor(ComponentMetaData.CustomPropertyCollection[COLLECTION_NAME_PROP_NAME].Value); var defaultOutputColumns = GetDefaultOutputColumns().ToArray(); foreach (BsonDocument document in cursor) { ColumnInfo failingColumnInfo = null; try { defaultBuffer.AddRow(); foreach (ColumnInfo columnInfo in this.columnInformata) { failingColumnInfo = columnInfo; if (document.Contains(columnInfo.ColumnName) && document[columnInfo.ColumnName] != null) { if (document.GetValue(columnInfo.ColumnName).IsBsonNull) { defaultBuffer.SetNull(columnInfo.OuputBufferColumnIndex); } else { var value = GetValue(document, columnInfo); try { defaultBuffer[columnInfo.OuputBufferColumnIndex] = value; } catch (DoesNotFitBufferException ex) { if (failingColumnInfo.OutputColumn.TruncationRowDisposition == DTSRowDisposition.RD_IgnoreFailure) { if (value is string) { defaultBuffer[columnInfo.OuputBufferColumnIndex] = value.ToString().Substring(0, columnInfo.OutputColumn.Length); } else { ComponentMetaData.FireWarning(0, "MongoDataSource", string.Format("Truncation of column {0} failed, as truncation of type {1} currently unsupported.", columnInfo.OutputColumn.Name, value.GetType().FullName), String.Empty, 0); } } else { throw ex; } } } } else { defaultBuffer.SetNull(columnInfo.OuputBufferColumnIndex); } } } catch (Exception ex) { DTSRowDisposition disposition = DTSRowDisposition.RD_NotUsed; if (ex is DoesNotFitBufferException) { disposition = failingColumnInfo.OutputColumn.TruncationRowDisposition; } else { disposition = failingColumnInfo.OutputColumn.ErrorRowDisposition; } HandleProcessingError(disposition, defaultBuffer, errorBuffer, failingColumnInfo, ex); } } if (defaultBuffer != null) { defaultBuffer.SetEndOfRowset(); } if (errorBuffer != null) { errorBuffer.SetEndOfRowset(); } }
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers) { IDTSOutput100 output = ComponentMetaData.OutputCollection[0]; PipelineBuffer buffer = buffers[0]; try { string lastId = string.Empty; while (true) { int count = 0; Entity[] entities = GetEntities(false, lastId); // Walk the rows in the DataReader, // and add them to the output buffer. foreach (Entity e in entities) { count++; lastId = e.Id; // Add a row to the output buffer. buffer.AddRow(); for (int x = 0; x < this._columnInfo.Count; x++) { ColumnInfo ci = this._columnInfo[x]; if (ci.ColumnName == DefaultIdColumnName) { buffer[ci.BufferColumnIndex] = e.Id; } else { object value = null; if (e.Properties.TryGetValue(ci.ColumnName, out value)) { buffer[ci.BufferColumnIndex] = value; } else { // NULL value for this column buffer.SetNull(ci.BufferColumnIndex); } } } } // Cloud DB starts to page at 500 entities // If we have less than 500, break out of the loop if (count < 500) { break; } } // Notify the data flow that we are finished adding rows to the output. } catch (Exception e) { ComponentMetaData.FireError(0, ComponentMetaData.Name, e.Message, string.Empty, 0, out this._cancel); throw; } finally { buffer.SetEndOfRowset(); } }
/// <summary> /// Will perform the user-specified behaviour when a processing error occurs /// </summary> /// <param name="disposition">How the error should be handled</param> /// <param name="defaultBuffer">The default output buffer</param> /// <param name="errorBuffer">The error output buffer</param> /// <param name="failingColumnInfo">The information for the problematic column</param> /// <param name="ex">The exception caught from processing (optional)</param> private void HandleProcessingError(DTSRowDisposition disposition, PipelineBuffer defaultBuffer, PipelineBuffer errorBuffer, ColumnInfo failingColumnInfo, Exception ex) { switch (disposition) { case DTSRowDisposition.RD_RedirectRow: if (errorBuffer == null) throw new InvalidOperationException("There must be an error output defined if redirection was specified"); // Add a row to the error buffer. errorBuffer.AddRow(); // Get the values from the default buffer // and copy them to the error buffer. var errorOutputColumns = GetErrorOutputColumns().ToArray(); foreach (IDTSOutputColumn100 column in errorOutputColumns) { ColumnInfo copiedColumnInfo = GetColumnInfo(column.Name); if (copiedColumnInfo != null) errorBuffer[copiedColumnInfo.ErrorOuputBufferColumnIndex] = defaultBuffer[copiedColumnInfo.OuputBufferColumnIndex]; } // Set the error information. int errorCode = (ex == null ? 0 : System.Runtime.InteropServices.Marshal.GetHRForException(ex)); errorBuffer.SetErrorInfo(errorOutputID, errorCode, failingColumnInfo.OutputColumn.LineageID); // Remove the row that was added to the default buffer. defaultBuffer.RemoveRow(); break; case DTSRowDisposition.RD_FailComponent: throw new Exception(String.Format("There was an issue with column: {0}", failingColumnInfo.OutputColumn.Name), ex); } }
/// <summary> /// Writes out a child table. Uses an iterative approach. Would have been much more /// readable if this was recursive, but we really should not be using it. /// </summary> /// <param name="id">The document key</param> /// <param name="jp">A deserialized object holding the document</param> /// <param name="model">A model used to decompose the document</param> /// <param name="tableName">The name of the child table</param> /// <param name="outputIDs">Indexes of the output buffers</param> /// <param name="buffers">The Pipline Buffers</param> void processChildTable(string id, JsonObject jp, JSONDataModel model, string tableName, int[] outputIDs, PipelineBuffer[] buffers) { // Obtain references to the model table, output and buffer DataTable modelTable = model.Tables[tableName]; IDTSOutput100 output = ComponentMetaData.OutputCollection[modelTable.TableName]; PipelineBuffer buffer = buffers[Array.IndexOf(outputIDs, output.ID)]; // The counters will track how many levels of nesting we have int counters = 0; // We will first represent all records that need to be written out as XPaths List <List <string> > xPathRows = new List <List <string> >(); // List <string> xPathRow = new List <string>(); // Collect counters and initial row XPaths foreach (DataRow row in modelTable.Rows) { string xPath = row["xpath"].ToString(); if (xPath.Equals("#ID")) { // Nothing for now } else if (xPath.StartsWith("#")) { counters++; } xPathRow.Add(xPath); } // Add the initial Row xPathRows.Add(xPathRow); // Iteratively expand each row based on the indexes found for (int i = 0; i < counters; i++) { xPathRows = expandRows(jp, i, xPathRows, modelTable); } // Iterate over the expanded rows. They should contain // full XPaths to be used to obtain the values for each cell foreach (List <string> row in xPathRows) { // Add a row to the buffer buffer.AddRow(); // Keep track of which column we are looking at int colIdx = 0; // For each column definition in the model foreach (DataRow modelCol in modelTable.Rows) { // Get the name and the index string col = modelCol["shortColumn"].ToString(); int columnIndex = columnIndexes[modelTable.TableName][col]; // Handle special case of DOC_ID if (row[colIdx].Equals("#ID")) { buffer.SetString(columnIndex, id); } // Handle special case of child table indexes else if (row[colIdx].StartsWith("#")) { // Obtain the nesting index from the XPath by looking at the first index found int counterIndex = int.Parse(row[colIdx].Substring(1)); // Extract the current value of the index Match m = Regex.Match(row[row.Count - 1], "\\[([0-9]+)\\]"); long index = long.Parse(m.Groups[counterIndex + 1].Value); // and write it out to the buffer buffer.SetInt64(columnIndex, index); } else { // Write all other fields as usual Type dataType = Type.GetType(modelCol["datatype"].ToString()); string xpath = row[colIdx]; writeDataToBuffer(jp, buffer, columnIndex, dataType, xpath); } // Increment to look at the next column colIdx++; } } }