public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
base.PrimeOutput(outputs, outputIDs, buffers);
PipelineBuffer buffer = buffers[0];
var dataTable = LoadDataTable(DataTableXml);
foreach (DataRow dataRow in dataTable.Rows)
{
buffer.AddRow();
foreach (var columnMapping in _bufferColumnMapping)
{
// TODO: Update to use the correct data type methods
if (dataRow[columnMapping.Key] == DBNull.Value)
{
buffer.SetNull(columnMapping.Value);
}
else
{
buffer[columnMapping.Value] = dataRow[columnMapping.Key];
}
}
}
buffer.SetEndOfRowset();
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
base.PrimeOutput(outputs, outputIDs, buffers);
IDTSOutput100 output = ComponentMetaData.OutputCollection.FindObjectByID(outputIDs[0]);
PipelineBuffer buffer = buffers[0];
String filePathVar = ComponentMetaData.CustomPropertyCollection[_FILE_PATH_VARIABLE_FIELD].Value.ToString();
String sheetNameVar = ComponentMetaData.CustomPropertyCollection[_SHEET_NAME_FIELD].Value.ToString();
DataTable dt = GetDataTableWithInputVar(filePathVar, sheetNameVar);
foreach (DataRow row in dt.Rows)
{
buffer.AddRow();
for (int x = 0; x < mapOutputColsToBufferCols.Length; x++)
{
if (row.IsNull(x))
{
buffer.SetNull(mapOutputColsToBufferCols[x]);
}
else
{
buffer[mapOutputColsToBufferCols[x]] = row[x];
}
}
}
buffer.SetEndOfRowset();
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
//System.Diagnostics.Debugger.Launch();
base.PrimeOutput(outputs, outputIDs, buffers);
IDTSOutput100 output = ComponentMetaData.OutputCollection.FindObjectByID(outputIDs[0]);
PipelineBuffer buffer = buffers[0];
DataTable dt = new DataTable();
dt = GetData(ComponentMetaData.CustomPropertyCollection["FetchXML"].Value.ToString(), false);
foreach (DataRow row in dt.Rows)
{
buffer.AddRow();
for (int x = 0; x < mapOutputColsToBufferCols.Length; x++)
{
if (row.IsNull(x))
{
buffer.SetNull(mapOutputColsToBufferCols[x]);
}
else
{
buffer[mapOutputColsToBufferCols[x]] = row[x];
}
}
}
buffer.SetEndOfRowset();
}
public override void ProcessInput(int inputID, PipelineBuffer inputBuffer)
{
base.ProcessInput(inputID, inputBuffer);
while (inputBuffer.NextRow())
{
var inputString = inputBuffer.GetString(0);
var order = new Order()
{
OrderCode = inputString,
Product = new Product()
{
Name = inputString + "Product"
}
};
var outputXml = XmlSerializer.XmlSerialize(order);
var outputBytes = Encoding.UTF8.GetBytes(outputXml);
outputBuffer.AddRow();
outputBuffer.AddBlobData(0, outputBytes);
}
if (inputBuffer.EndOfRowset)
{
outputBuffer.SetEndOfRowset();
}
}
public override void ProcessInput(int inputID, PipelineBuffer inputBuffer)
{
while (inputBuffer.NextRow())
{
var parameters = GetWebServiceParameters(inputBuffer);
var response = WebService.read(X3WebService.Context, PublicIdentifierValue, parameters);
var xml = new XmlDocument();
xml.LoadXml(response.resultXml);
var responseXmlWrapper = new X3ObjectResponseWrapper(xml);
outputBuffer.AddRow();
foreach (var groupTag in responseXmlWrapper.ResultGroups)
{
foreach (var fieldTag in groupTag.Fields)
{
var columnName = GenerateOutputColumnName(groupTag.Id, fieldTag.Name);
var outputCollumnIndex = GetOutputColumnIndex(columnName);
var outputColumn = GetOutputColumn(columnName);
SetColumnData(outputBuffer, outputCollumnIndex, fieldTag.Value, outputColumn.DataType);
}
}
}
if (inputBuffer.EndOfRowset)
{
outputBuffer.SetEndOfRowset();
}
}
private void AddOutputRow(ref PipelineBuffer inputbuffer)
{
// For some RESON I STILL DO NOT UNDERSTAND, THIS METHOD FALLS in a sort of race condition.
// The following is the worst workaroud ever, but needed ito quickly address the problem.
while (true)
{
try
{
_outputBuffer.AddRow();
break;
}
catch (Exception e)
{
ComponentMetaData.FireWarning(ComponentConstants.RUNTIME_GENERIC_ERROR, ComponentMetaData.Name, "Outputrow was not ready. ", null, 0);
throw e;
}
}
// Copy the inputs into outputs
for (var i = 0; i < _startOfJsonColIndex; i++)
{
if (inputbuffer[i] is BlobColumn)
{
_outputBuffer.AddBlobData(i, inputbuffer.GetBlobData(i, 0, (int)inputbuffer.GetBlobLength(i)));
}
else
{
_outputBuffer[i] = inputbuffer[i];
}
}
}
public override void ProcessInput(int inputID, PipelineBuffer inputBuffer)
{
base.ProcessInput(inputID, inputBuffer);
var jsSerializer = new JavaScriptSerializer();
while (inputBuffer.NextRow())
{
var inputBytes = inputBuffer.GetBlobData(0, 0, (int)inputBuffer.GetBlobLength(0));
var inputXml = Encoding.UTF8.GetString(inputBytes);
var inputObject = XmlSerializer.XmlDeserialize <Order>(inputXml);
var serializedObject = jsSerializer.Serialize(inputObject);
if (serializedObject.Length > 4000 - 4)
{
serializedObject = serializedObject.Substring(0, 4000 - 4) + " ...";
}
outputBuffer.AddRow();
outputBuffer.SetString(0, serializedObject);
}
if (inputBuffer.EndOfRowset)
{
outputBuffer.SetEndOfRowset();
}
}
/// <summary>
/// Write a json document to the output buffers
/// </summary>
/// <param name="id">The document key</param>
/// <param name="json">The JSON representation of the document</param>
/// <param name="model">A model used to decompose the document</param>
/// <param name="outputIDs">Indexes of the output buffers</param>
/// <param name="buffers">The Pipline Buffers</param>
private void writeDocToBuffers(string id, string json, JSONDataModel model, int[] outputIDs, PipelineBuffer[] buffers)
{
// Deserialize the document
JsonObject jp = new JsonObject(json);
// For each model table
foreach (DataTable modelTable in model.Tables)
{
// Obtain the output
IDTSOutput100 output = ComponentMetaData.OutputCollection[modelTable.TableName];
// Continue if output is not connected
if (Array.IndexOf(outputIDs, output.ID) == -1)
{
continue;
}
// Obtain the buffer tho write to
PipelineBuffer buffer = buffers[Array.IndexOf(outputIDs, output.ID)];
// If we are processing a table that has indexes on the second column
// this must be a child table
if (modelTable.Rows.Count > 2 && modelTable.Rows[1]["xpath"].ToString().StartsWith("#"))
{
// Go and process the related table
processChildTable(id, jp, model, modelTable.TableName, outputIDs, buffers);
}
else
{
// Add a new row to the buffer
buffer.AddRow();
// For every column definition in teh model
foreach (DataRow row in modelTable.Rows)
{
// Find the column index that the buffer will recognize
int columnIndex = columnIndexes[modelTable.TableName][row["shortColumn"].ToString()];
// Get the model datatype
Type dataType = Type.GetType(row["datatype"].ToString());
// And XPath
string xpath = row["xpath"].ToString();
// Handle special case of DOC_ID
if (xpath.Equals("#ID"))
{
buffer.SetString(columnIndex, id);
}
else
{
// Write all other fields as usual
writeDataToBuffer(jp, buffer, columnIndex, dataType, xpath);
}
}
}
}
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
IDTSOutput100 output = ComponentMetaData.OutputCollection[0];
PipelineBuffer buffer = buffers[0];
try
{
foreach (ManagementObject row in ExecWQL(false))
{
object[] row_values = new object[m_columnInfo.Count];
int i = 0;
foreach (ColumnInfo ci in m_columnInfo)
{
PropertyData wmiColumn = row.Properties[ci.ColumnName];
row_values[i++] = row[wmiColumn.Name];
}
// lets unwind wmi row ( unwind array in any column )
// we use the same approach to handle WMI arrays as WMI ODBC driver
// for details see msdn :
// http://msdn.microsoft.com/en-us/library/aa392328(VS.85).aspx#_hmm_mapping_wmi_arrays_to_odbc
foreach (object[] unwinded_row in RowUnwinder.UnwindRow(row_values))
{
buffer.AddRow();
i = 0;
foreach (ColumnInfo ci in m_columnInfo)
{
PropertyData wmiColumn = row.Properties[ci.ColumnName];
SetBufferColumn(buffer, row, wmiColumn, ci.BufferColumnIndex, unwinded_row[i++]);
}
}
}
// set end of data on all of the buffers
buffer.SetEndOfRowset();
}
catch (Exception e)
{
bool bCancel;
ErrorSupport.FireErrorWithArgs(HResults.DTS_E_PRIMEOUTPUTFAILED,
out bCancel, ComponentMetaData.IdentificationString, e.Message);
throw new PipelineComponentHResultException(HResults.DTS_E_PRIMEOUTPUTFAILED);
}
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
//On récupère le buffer mémoire dans lequel seront stockées les lignes
PipelineBuffer buffer = buffers[0];
//Récupération de la chaine de connexion au stockage blob Azure
string storageConnectionString = (string)this.ComponentMetaData.CustomPropertyCollection["StorageConnectionString"].Value;
string tableName = (string)this.ComponentMetaData.CustomPropertyCollection["TableName"].Value;
//Connexion au stockage
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudTableClient tableClient = cloudStorageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference(tableName);
//Récupération des lignes
TableQuery <DynamicTableEntity> query = new TableQuery <DynamicTableEntity>();
var azureTable = table.ExecuteQuery(query);
foreach (DynamicTableEntity row in azureTable)
{
//Ajout d'une ligne au buffer
buffer.AddRow();
var properties = row.Properties;
int i = 0;
//Itération sur chaques colonnes
foreach (var col in properties)
{
if (col.Value != null)
{
buffer[i] = col.Value.PropertyAsObject;
}
else
{
buffer.SetNull(i);
}
i++;
}
}
//Envoie du buffer
buffer.SetEndOfRowset();
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
IDTSOutput100 output = ComponentMetaData.OutputCollection[0];
PipelineBuffer buffer = buffers[0];
BasicGetResult result = consumerChannel.BasicGet(queueName, true);
while (result != null)
{
string messageContent = Encoding.UTF8.GetString(result.Body);
buffer.AddRow();
buffer[0] = messageContent;
buffer[1] = result.RoutingKey;
result = consumerChannel.BasicGet(queueName, true);
}
buffer.SetEndOfRowset();
}
public override void ProcessInput(int inputID, PipelineBuffer inputBuffer)
{
base.ProcessInput(inputID, inputBuffer);
while (inputBuffer.NextRow())
{
var inputBytes = inputBuffer.GetBlobData(0, 0, (int)inputBuffer.GetBlobLength(0));
var inputXml = Encoding.UTF8.GetString(inputBytes);
var inputObject = XmlSerializer.XmlDeserialize <TSource>(inputXml);
var outputObject = ProcessInput(inputObject);
var outputXml = XmlSerializer.XmlSerialize(outputObject);
var outputBytes = Encoding.UTF8.GetBytes(outputXml);
outputBuffer.AddRow();
outputBuffer.AddBlobData(0, outputBytes);
}
if (inputBuffer.EndOfRowset)
{
outputBuffer.SetEndOfRowset();
}
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
IDTSOutput100 output = ComponentMetaData.OutputCollection[0];
PipelineBuffer buffer = buffers[0];
object message;
bool success;
while (queueConsumer.IsRunning)
{
try
{
success = queueConsumer.Queue.Dequeue(100, out message);
}
catch (Exception)
{
break;
}
if (success)
{
BasicDeliverEventArgs e = (BasicDeliverEventArgs)message;
var messageContent = System.Text.Encoding.UTF8.GetString(e.Body);
buffer.AddRow();
buffer[0] = messageContent;
buffer[1] = e.RoutingKey;
}
else
{
break;
}
}
buffer.SetEndOfRowset();
}
/// <summary>
/// Will perform the user-specified behaviour when a processing error occurs
/// </summary>
/// <param name="disposition">How the error should be handled</param>
/// <param name="defaultBuffer">The default output buffer</param>
/// <param name="errorBuffer">The error output buffer</param>
/// <param name="failingColumnInfo">The information for the problematic column</param>
/// <param name="ex">The exception caught from processing (optional)</param>
private void HandleProcessingError(DTSRowDisposition disposition, PipelineBuffer defaultBuffer, PipelineBuffer errorBuffer, ColumnInfo failingColumnInfo, Exception ex)
{
switch (disposition)
{
case DTSRowDisposition.RD_RedirectRow:
if (errorBuffer == null)
{
throw new InvalidOperationException("There must be an error output defined if redirection was specified");
}
// Add a row to the error buffer.
errorBuffer.AddRow();
// Get the values from the default buffer
// and copy them to the error buffer.
var errorOutputColumns = GetErrorOutputColumns().ToArray();
foreach (IDTSOutputColumn100 column in errorOutputColumns)
{
ColumnInfo copiedColumnInfo = GetColumnInfo(column.Name);
if (copiedColumnInfo != null)
{
errorBuffer[copiedColumnInfo.ErrorOuputBufferColumnIndex] = defaultBuffer[copiedColumnInfo.OuputBufferColumnIndex];
}
}
// Set the error information.
int errorCode = (ex == null ? 0 : System.Runtime.InteropServices.Marshal.GetHRForException(ex));
errorBuffer.SetErrorInfo(errorOutputID, errorCode, failingColumnInfo.OutputColumn.LineageID);
// Remove the row that was added to the default buffer.
defaultBuffer.RemoveRow();
break;
case DTSRowDisposition.RD_FailComponent:
throw new Exception(String.Format("There was an issue with column: {0}", failingColumnInfo.OutputColumn.Name), ex);
}
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
base.PrimeOutput(outputs, outputIDs, buffers);
IDTSOutput100 output = ComponentMetaData.OutputCollection.FindObjectByID(outputIDs[0]);
PipelineBuffer buffer = buffers[0];
DataTable dt = GetRSSDataTable(ComponentMetaData.CustomPropertyCollection["RSS Path"].Value.ToString());
foreach (DataRow row in dt.Rows)
{
buffer.AddRow();
for (int x = 0; x < mapOutputColsToBufferCols.Length; x++)
{
if (row.IsNull(x))
buffer.SetNull(mapOutputColsToBufferCols[x]);
else
buffer[mapOutputColsToBufferCols[x]] = row[x];
}
}
buffer.SetEndOfRowset();
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
//Do some stuff
base.PrimeOutput(outputs, outputIDs, buffers);
//SSIS STUFF
IDTSOutput100 output = ComponentMetaData.OutputCollection.FindObjectByID(outputIDs[0]);
PipelineBuffer buffer = buffers[0];
string path = ComponentMetaData.CustomPropertyCollection["FTP Path"].Value;
int nrOfDays = ComponentMetaData.CustomPropertyCollection["FTP Files Not Older Then (Days)"].Value;
//Get path part of file path
string pathPart = System.IO.Path.GetDirectoryName(path);
//ftp://ftp.matchit.no/Maintenance/Archive/*.*
FileStruct[] files = _ftpHelper.GetFiles(path);
foreach (FileStruct file in files)
{
//SKIP IF FILS IS OLDER THEN x DAYS
if (file.CreateTime < DateTime.Now.AddDays(-nrOfDays))
{
continue;
}
//BUILD THE PATH FOR EACH FILE
string filePath = System.IO.Path.Combine(pathPart, file.Name);
//Switch "\" with "/"
filePath = filePath.Replace(@"\", @"/");
UriBuilder builder = new UriBuilder();
builder.Scheme = "ftp";
builder.Host = _ftpHelper.HostName;
builder.Path = filePath;
//GOT THE PATH
String url = builder.ToString();
//GET THE STREAM
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(url);
request.Method = WebRequestMethods.Ftp.DownloadFile;
//Login
string usr = ComponentMetaData.CustomPropertyCollection["FTP User"].Value;
string pwd = ComponentMetaData.CustomPropertyCollection["FTP Password"].Value;
request.Credentials = new NetworkCredential(usr, pwd);
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
//StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, true);
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream);
//Get lines to skip
int rowsToSkip = ComponentMetaData.CustomPropertyCollection["Skip rows"].Value;
//READ THE STREAM
int currentLineNumber = 0;
while (!reader.EndOfStream)
{
try
{
//Read line
string line = reader.ReadLine();
//Skip x number of rows
if (rowsToSkip > currentLineNumber)
{
currentLineNumber++;
continue;
}
//splitt line
string separator = ComponentMetaData.CustomPropertyCollection["Separator"].Value;
char c = separator.ToCharArray()[0];
string[] values = line.Split(c);
//create output buffer
buffer.AddRow();
int i = 0;
foreach (CustomDataColumn column in _columnsList)
{
switch (column.Type)
{
case "I4":
buffer[i] = int.Parse(values[i]);
break;
case "WSTR":
buffer[i] = values[i];
break;
case "DBTIMESTAMP":
string dateFormatString = column.Length; //For DateTime "length" is used for describing datetime format
buffer[i] = DateTime.ParseExact(values[i], dateFormatString, CultureInfo.InvariantCulture);
break;
default:
throw new NotImplementedException("Error, " + column.Type + " is not implemented");
}
i++;
}
currentLineNumber++;
}
catch (Exception e)
{
buffer.DirectErrorRow(0, 1, currentLineNumber);
}
}
}
buffer.SetEndOfRowset();
}
private void ParseTheFileAndAddToBuffer(string filename, PipelineBuffer buffer)
{
TextReader tr = File.OpenText(filename);
IDTSOutput100 output =
ComponentMetaData.OutputCollection["Component Output"];
IDTSOutputColumnCollection100 cols =
output.OutputColumnCollection;
IDTSOutputColumn100 col;
string s = tr.ReadLine();
int i = 0;
while (s != null)
{
if (s.StartsWith("<START>"))
buffer.AddRow();
if (s.StartsWith("Name:"))
{
col = cols["Name"];
i = BufferManager.FindColumnByLineageID(output.Buffer,
col.LineageID);
string value = s.Substring(5);
buffer.SetString(i, value);
}
if (s.StartsWith("Age:"))
{
col = cols["Age"];
i = BufferManager.FindColumnByLineageID(output.Buffer,
col.LineageID);
Int32 value;
if (s.Substring(4).Trim() == "")
value = 0;
else
value = Convert.ToInt32(s.Substring(4).Trim());
buffer.SetInt32(i, value);
}
if (s.StartsWith("Married:"))
{
col = cols["Married"];
bool value;
i = BufferManager.FindColumnByLineageID(output.Buffer,
col.LineageID);
if (s.Substring(8).Trim() == "")
value = true;
else
value = s.Substring(8).Trim() != "1" ? false : true;
buffer.SetBoolean(i, value);
}
if (s.StartsWith("Salary:"))
{
col = cols["Salary"];
Decimal value;
i = BufferManager.FindColumnByLineageID(output.Buffer,
col.LineageID);
if (s.Substring(7).Trim() == "")
value = 0M;
else
value = Convert.ToDecimal(s.Substring(8).Trim());
buffer.SetDecimal(i, value);
}
s = tr.ReadLine();
}
tr.Close();
}
/// <summary>
/// This is where the data is loaded into the output buffer
/// </summary>
/// <param name="outputs"></param>
/// <param name="outputIDs"></param>
/// <param name="buffers"></param>
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
string sharepointUrl = (string)ComponentMetaData.CustomPropertyCollection[C_SHAREPOINTSITEURL].Value;
string sharepointList = (string)ComponentMetaData.CustomPropertyCollection[C_SHAREPOINTLISTNAME].Value;
string sharepointListView = (string)ComponentMetaData.CustomPropertyCollection[C_SHAREPOINTLISTVIEWNAME].Value;
XElement camlQuery = XElement.Parse((string)ComponentMetaData.CustomPropertyCollection[C_CAMLQUERY].Value);
short batchSize = (short)ComponentMetaData.CustomPropertyCollection[C_BATCHSIZE].Value;
Enums.TrueFalseValue isRecursive = (Enums.TrueFalseValue)ComponentMetaData.CustomPropertyCollection[C_ISRECURSIVE].Value;
Enums.TrueFalseValue includeFolders = (Enums.TrueFalseValue)ComponentMetaData.CustomPropertyCollection[C_INCLUDEFOLDERS].Value;
PipelineBuffer outputBuffer = buffers[0];
// Get the field names from the output collection
var fieldNames = (from col in
ComponentMetaData.OutputCollection[0].OutputColumnCollection.Cast <IDTSOutputColumn>()
select(string) col.CustomPropertyCollection[0].Value);
// Load the data from sharepoint
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
var listData = SharePointUtility.ListServiceUtility.GetListItemData(
new Uri(sharepointUrl), _credentials, sharepointList, sharepointListView, fieldNames, camlQuery,
isRecursive == Enums.TrueFalseValue.True ? true : false, batchSize);
timer.Stop();
bool fireAgain = false;
int actualRowCount = 0;
foreach (var row in listData)
{
// Determine if we should continue based on if this is a folder item or not (filter can be pushed up to CAML if
// perf becomes an issue)
bool canContinue = true;
if ((row.ContainsKey("ContentType")) &&
(row["ContentType"] == "Folder") &&
(includeFolders == Enums.TrueFalseValue.False))
{
canContinue = false;
}
if (canContinue)
{
actualRowCount++;
outputBuffer.AddRow();
foreach (var fieldName in _bufferLookup.Keys)
{
if (row.ContainsKey(fieldName))
{
switch (_bufferLookupDataType[fieldName])
{
case DataType.DT_NTEXT:
outputBuffer.AddBlobData(_bufferLookup[fieldName],
Encoding.Unicode.GetBytes(row[fieldName].ToString()));
break;
case DataType.DT_WSTR:
outputBuffer.SetString(_bufferLookup[fieldName], row[fieldName]);
break;
case DataType.DT_R8:
outputBuffer.SetDouble(_bufferLookup[fieldName], double.Parse(row[fieldName], _culture));
break;
case DataType.DT_I4:
outputBuffer.SetInt32(_bufferLookup[fieldName], int.Parse(row[fieldName], _culture));
break;
case DataType.DT_BOOL:
outputBuffer.SetBoolean(_bufferLookup[fieldName], (int.Parse(row[fieldName], _culture) == 1));
break;
case DataType.DT_GUID:
outputBuffer.SetGuid(_bufferLookup[fieldName], new Guid(row[fieldName]));
break;
case DataType.DT_DBTIMESTAMP:
outputBuffer.SetDateTime(_bufferLookup[fieldName], DateTime.Parse(row[fieldName], _culture));
break;
}
}
else
{
switch (_bufferLookupDataType[fieldName])
{
case DataType.DT_NTEXT:
outputBuffer.AddBlobData(_bufferLookup[fieldName],
Encoding.Unicode.GetBytes(String.Empty));
break;
case DataType.DT_WSTR:
outputBuffer.SetString(_bufferLookup[fieldName], String.Empty);
break;
case DataType.DT_BOOL:
outputBuffer.SetBoolean(_bufferLookup[fieldName], false);
break;
default:
outputBuffer.SetNull(_bufferLookup[fieldName]);
break;
}
}
}
}
}
string infoMsg = string.Format(CultureInfo.InvariantCulture,
"Loaded {0} records from list '{1}' at '{2}'. Elapsed time is {3}ms",
actualRowCount,
sharepointList,
sharepointUrl,
timer.ElapsedMilliseconds);
ComponentMetaData.FireInformation(0, ComponentMetaData.Name, infoMsg, "", 0, ref fireAgain);
ComponentMetaData.IncrementPipelinePerfCounter(
DTS_PIPELINE_CTR_ROWSREAD, (uint)actualRowCount);
outputBuffer.SetEndOfRowset();
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
IDTSOutput100 output = ComponentMetaData.OutputCollection[0];
PipelineBuffer buffer = buffers[0];
// A ListRowsContinuationToken class encapsulates the partition and row key
ListRowsContinuationToken continuationToken = null;
string strSource = (string)this.ComponentMetaData.CustomPropertyCollection["TableName"].Value;
string srcTable = string.Empty;
string srcPartitionKey = string.Empty;
if (strSource != string.Empty)
{
string[] str1 = strSource.Split(';');
srcTable = str1[0].Split('=')[1];
srcPartitionKey = str1[1].Split('=')[1];
}
do
{
var allItems = this.context.CreateQuery <GenericEntity>(srcTable).Where(item => item.PartitionKey == srcPartitionKey);
var query = allItems as DataServiceQuery <GenericEntity>;
if (continuationToken != null)
{
query = query.AddQueryOption("NextPartitionKey", continuationToken.PartitionKey);
if (continuationToken.RowKey != null)
{
query = query.AddQueryOption("NextRowKey", continuationToken.RowKey);
}
}
var response = query.Execute() as QueryOperationResponse;
if (response.Headers.ContainsKey("x-ms-continuation-NextPartitionKey"))
{
continuationToken = new ListRowsContinuationToken();
continuationToken.PartitionKey = response.Headers["x-ms-continuation-NextPartitionKey"];
if (response.Headers.ContainsKey("x-ms-continuation-NextRowKey"))
{
continuationToken.RowKey = response.Headers["x-ms-continuation-NextRowKey"];
}
}
else
{
continuationToken = null;
}
foreach (var item in allItems)
{
buffer.AddRow();
for (int x = 0; x < columnInformation.Count; x++)
{
var ci = (ColumnInfo)columnInformation[x];
var value = item[ci.ColumnName].Value;
if (value != null)
{
buffer[ci.BufferColumnIndex] = value;
}
else
{
buffer.SetNull(ci.BufferColumnIndex);
}
}
}
}while (continuationToken != null);
buffer.SetEndOfRowset();
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
//identify buffers
PipelineBuffer errorBuffer = null;
PipelineBuffer defaultBuffer = null;
for (int x = 0; x < outputs; x++)
{
if (outputIDs[x] == errorOutputID)
{
errorBuffer = buffers[x];
}
else
{
defaultBuffer = buffers[x];
}
}
//get ogrlayer and ogrlayer feature definition
Layer OGRLayer;
bool isSQLLayer = (!(ComponentMetaData.CustomPropertyCollection["SQL Statement"].Value == null || ComponentMetaData.CustomPropertyCollection["SQL Statement"].Value.ToString() == string.Empty));
if (isSQLLayer)
{
OGRLayer = getSQLLayer();
}
else
{
OGRLayer = getLayer();
}
Feature OGRFeature;
FeatureDefn OGRFeatureDef = OGRLayer.GetLayerDefn();
//initialize columnInfo object
columnInfo ci = new columnInfo();
//for each row in ogrlayer add row to output buffer
while ((OGRFeature = OGRLayer.GetNextFeature()) != null)
{
try
{
defaultBuffer.AddRow();
//set buffer column values
for (int i = 0; i < this.columnInformation.Count; i++)
{
ci = (columnInfo)this.columnInformation[i];
if (ci.geom)
{
Geometry geom = OGRFeature.GetGeometryRef();
if (geom != null)
{
byte[] geomBytes = new byte[geom.WkbSize()];
geom.ExportToWkb(geomBytes);
defaultBuffer.AddBlobData(ci.bufferColumnIndex, geomBytes);
}
}
else
{
int OGRFieldIndex = OGRFeatureDef.GetFieldIndex(ci.columnName);
FieldDefn OGRFieldDef = OGRFeatureDef.GetFieldDefn(OGRFieldIndex);
FieldType OGRFieldType = OGRFieldDef.GetFieldType();
//declare datetime variables
int pnYear, pnMonth, pnDay, pnHour, pnMinute, pnTZFlag;
float pnSecond;
DateTime dt;
TimeSpan ts;
switch (OGRFieldType)
{
//case FieldType.OFTBinary:
// break;
case FieldType.OFTDate:
OGRFeature.GetFieldAsDateTime(OGRFieldIndex, out pnYear, out pnMonth, out pnDay, out pnHour, out pnMinute, out pnSecond, out pnTZFlag);
dt = new DateTime(pnYear, pnMonth, pnDay);
defaultBuffer.SetDate(ci.bufferColumnIndex, dt);
break;
case FieldType.OFTDateTime:
OGRFeature.GetFieldAsDateTime(OGRFieldIndex, out pnYear, out pnMonth, out pnDay, out pnHour, out pnMinute, out pnSecond, out pnTZFlag);
dt = new DateTime(pnYear, pnMonth, pnDay, pnHour, pnMinute, (int)pnSecond);
//set time zone?
defaultBuffer.SetDateTime(ci.bufferColumnIndex, dt);
break;
case FieldType.OFTInteger:
defaultBuffer.SetInt32(ci.bufferColumnIndex, OGRFeature.GetFieldAsInteger(OGRFieldIndex));
break;
case FieldType.OFTInteger64:
defaultBuffer.SetInt64(ci.bufferColumnIndex, OGRFeature.GetFieldAsInteger(OGRFieldIndex));
break;
case FieldType.OFTReal:
defaultBuffer.SetDouble(ci.bufferColumnIndex, OGRFeature.GetFieldAsDouble(OGRFieldIndex));
break;
case FieldType.OFTTime:
OGRFeature.GetFieldAsDateTime(OGRFieldIndex, out pnYear, out pnMonth, out pnDay, out pnHour, out pnMinute, out pnSecond, out pnTZFlag);
ts = new TimeSpan(pnHour, pnMinute, (int)pnSecond);
defaultBuffer.SetTime(ci.bufferColumnIndex, ts);
break;
case FieldType.OFTString:
default:
defaultBuffer.SetString(ci.bufferColumnIndex, OGRFeature.GetFieldAsString(OGRFieldIndex));
break;
}
}
}
}
catch (Exception ex)
{
//redirect to error buffer
if (ci.errorDisposition == DTSRowDisposition.RD_RedirectRow)
{
// Add a row to the error buffer.
errorBuffer.AddRow();
// Set the error information.
int errorCode = System.Runtime.InteropServices.Marshal.GetHRForException(ex);
errorBuffer.SetErrorInfo(errorOutputID, errorCode, ci.lineageID);
// Remove the row that was added to the default buffer.
defaultBuffer.RemoveRow();
}
//fail component
else if (ci.errorDisposition == DTSRowDisposition.RD_FailComponent || ci.errorDisposition == DTSRowDisposition.RD_NotUsed)
{
ComponentMetaData.FireError(0, "primeoutput failure", ex.ToString(), string.Empty, 0, out cancel);
throw;
}
}
}
//set end of rowset for buffers
if (defaultBuffer != null)
{
defaultBuffer.SetEndOfRowset();
}
if (errorBuffer != null)
{
errorBuffer.SetEndOfRowset();
}
//clean up layer object
if (isSQLLayer)
{
this.OGRDataSource.ReleaseResultSet(OGRLayer);
}
}
/// <summary>
/// Called at run time for source components and transformation components with
/// asynchronous outputs to let these components add rows to the output buffers.
/// </summary>
/// <param name="outputs">The number of elements in the outputIDs and buffers arrays.</param>
/// <param name="outputIDs">An array of Microsoft.SqlServer.Dts.Pipeline.Wrapper.IDTSOutput100 ID's.</param>
/// <param name="buffers">An array of Microsoft.SqlServer.Dts.Pipeline.PipelineBuffer objects.</param>
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
// Determine which buffer is for regular output, and which is for error output
PipelineBuffer errorBuffer = null;
PipelineBuffer defaultBuffer = null;
for (int outputIndex = 0; outputIndex < outputs; outputIndex++)
{
if (outputIDs[outputIndex] == errorOutputID)
{
errorBuffer = buffers[outputIndex];
}
else
{
defaultBuffer = buffers[outputIndex];
}
}
var cursor = GetCollectionCursor(ComponentMetaData.CustomPropertyCollection[COLLECTION_NAME_PROP_NAME].Value);
var defaultOutputColumns = GetDefaultOutputColumns().ToArray();
foreach (BsonDocument document in cursor)
{
ColumnInfo failingColumnInfo = null;
try
{
defaultBuffer.AddRow();
foreach (ColumnInfo columnInfo in this.columnInformata)
{
failingColumnInfo = columnInfo;
if (document.Contains(columnInfo.ColumnName) && document[columnInfo.ColumnName] != null)
{
if (document.GetValue(columnInfo.ColumnName).IsBsonNull)
{
defaultBuffer.SetNull(columnInfo.OuputBufferColumnIndex);
}
else
{
var value = GetValue(document, columnInfo);
try
{
defaultBuffer[columnInfo.OuputBufferColumnIndex] = value;
}
catch (DoesNotFitBufferException ex)
{
if (failingColumnInfo.OutputColumn.TruncationRowDisposition == DTSRowDisposition.RD_IgnoreFailure)
{
if (value is string)
{
defaultBuffer[columnInfo.OuputBufferColumnIndex] = value.ToString().Substring(0, columnInfo.OutputColumn.Length);
}
else
{
ComponentMetaData.FireWarning(0,
"MongoDataSource",
string.Format("Truncation of column {0} failed, as truncation of type {1} currently unsupported.", columnInfo.OutputColumn.Name, value.GetType().FullName),
String.Empty, 0);
}
}
else
{
throw ex;
}
}
}
}
else
{
defaultBuffer.SetNull(columnInfo.OuputBufferColumnIndex);
}
}
}
catch (Exception ex)
{
DTSRowDisposition disposition = DTSRowDisposition.RD_NotUsed;
if (ex is DoesNotFitBufferException)
{
disposition = failingColumnInfo.OutputColumn.TruncationRowDisposition;
}
else
{
disposition = failingColumnInfo.OutputColumn.ErrorRowDisposition;
}
HandleProcessingError(disposition, defaultBuffer, errorBuffer, failingColumnInfo, ex);
}
}
if (defaultBuffer != null)
{
defaultBuffer.SetEndOfRowset();
}
if (errorBuffer != null)
{
errorBuffer.SetEndOfRowset();
}
}
public override void PrimeOutput(int outputs, int[] outputIDs, PipelineBuffer[] buffers)
{
IDTSOutput100 output = ComponentMetaData.OutputCollection[0];
PipelineBuffer buffer = buffers[0];
try
{
string lastId = string.Empty;
while (true)
{
int count = 0;
Entity[] entities = GetEntities(false, lastId);
// Walk the rows in the DataReader,
// and add them to the output buffer.
foreach (Entity e in entities)
{
count++;
lastId = e.Id;
// Add a row to the output buffer.
buffer.AddRow();
for (int x = 0; x < this._columnInfo.Count; x++)
{
ColumnInfo ci = this._columnInfo[x];
if (ci.ColumnName == DefaultIdColumnName)
{
buffer[ci.BufferColumnIndex] = e.Id;
}
else
{
object value = null;
if (e.Properties.TryGetValue(ci.ColumnName, out value))
{
buffer[ci.BufferColumnIndex] = value;
}
else
{
// NULL value for this column
buffer.SetNull(ci.BufferColumnIndex);
}
}
}
}
// Cloud DB starts to page at 500 entities
// If we have less than 500, break out of the loop
if (count < 500)
{
break;
}
}
// Notify the data flow that we are finished adding rows to the output.
}
catch (Exception e)
{
ComponentMetaData.FireError(0, ComponentMetaData.Name, e.Message, string.Empty, 0, out this._cancel);
throw;
}
finally
{
buffer.SetEndOfRowset();
}
}
/// <summary>
/// Will perform the user-specified behaviour when a processing error occurs
/// </summary>
/// <param name="disposition">How the error should be handled</param>
/// <param name="defaultBuffer">The default output buffer</param>
/// <param name="errorBuffer">The error output buffer</param>
/// <param name="failingColumnInfo">The information for the problematic column</param>
/// <param name="ex">The exception caught from processing (optional)</param>
private void HandleProcessingError(DTSRowDisposition disposition, PipelineBuffer defaultBuffer, PipelineBuffer errorBuffer, ColumnInfo failingColumnInfo, Exception ex)
{
switch (disposition)
{
case DTSRowDisposition.RD_RedirectRow:
if (errorBuffer == null) throw new InvalidOperationException("There must be an error output defined if redirection was specified");
// Add a row to the error buffer.
errorBuffer.AddRow();
// Get the values from the default buffer
// and copy them to the error buffer.
var errorOutputColumns = GetErrorOutputColumns().ToArray();
foreach (IDTSOutputColumn100 column in errorOutputColumns)
{
ColumnInfo copiedColumnInfo = GetColumnInfo(column.Name);
if (copiedColumnInfo != null)
errorBuffer[copiedColumnInfo.ErrorOuputBufferColumnIndex] = defaultBuffer[copiedColumnInfo.OuputBufferColumnIndex];
}
// Set the error information.
int errorCode = (ex == null ? 0 : System.Runtime.InteropServices.Marshal.GetHRForException(ex));
errorBuffer.SetErrorInfo(errorOutputID, errorCode, failingColumnInfo.OutputColumn.LineageID);
// Remove the row that was added to the default buffer.
defaultBuffer.RemoveRow();
break;
case DTSRowDisposition.RD_FailComponent:
throw new Exception(String.Format("There was an issue with column: {0}", failingColumnInfo.OutputColumn.Name), ex);
}
}
/// <summary>
/// Writes out a child table. Uses an iterative approach. Would have been much more
/// readable if this was recursive, but we really should not be using it.
/// </summary>
/// <param name="id">The document key</param>
/// <param name="jp">A deserialized object holding the document</param>
/// <param name="model">A model used to decompose the document</param>
/// <param name="tableName">The name of the child table</param>
/// <param name="outputIDs">Indexes of the output buffers</param>
/// <param name="buffers">The Pipline Buffers</param>
void processChildTable(string id, JsonObject jp, JSONDataModel model, string tableName, int[] outputIDs, PipelineBuffer[] buffers)
{
// Obtain references to the model table, output and buffer
DataTable modelTable = model.Tables[tableName];
IDTSOutput100 output = ComponentMetaData.OutputCollection[modelTable.TableName];
PipelineBuffer buffer = buffers[Array.IndexOf(outputIDs, output.ID)];
// The counters will track how many levels of nesting we have
int counters = 0;
// We will first represent all records that need to be written out as XPaths
List <List <string> > xPathRows = new List <List <string> >();
//
List <string> xPathRow = new List <string>();
// Collect counters and initial row XPaths
foreach (DataRow row in modelTable.Rows)
{
string xPath = row["xpath"].ToString();
if (xPath.Equals("#ID"))
{
// Nothing for now
}
else if (xPath.StartsWith("#"))
{
counters++;
}
xPathRow.Add(xPath);
}
// Add the initial Row
xPathRows.Add(xPathRow);
// Iteratively expand each row based on the indexes found
for (int i = 0; i < counters; i++)
{
xPathRows = expandRows(jp, i, xPathRows, modelTable);
}
// Iterate over the expanded rows. They should contain
// full XPaths to be used to obtain the values for each cell
foreach (List <string> row in xPathRows)
{
// Add a row to the buffer
buffer.AddRow();
// Keep track of which column we are looking at
int colIdx = 0;
// For each column definition in the model
foreach (DataRow modelCol in modelTable.Rows)
{
// Get the name and the index
string col = modelCol["shortColumn"].ToString();
int columnIndex = columnIndexes[modelTable.TableName][col];
// Handle special case of DOC_ID
if (row[colIdx].Equals("#ID"))
{
buffer.SetString(columnIndex, id);
}
// Handle special case of child table indexes
else if (row[colIdx].StartsWith("#"))
{
// Obtain the nesting index from the XPath by looking at the first index found
int counterIndex = int.Parse(row[colIdx].Substring(1));
// Extract the current value of the index
Match m = Regex.Match(row[row.Count - 1], "\\[([0-9]+)\\]");
long index = long.Parse(m.Groups[counterIndex + 1].Value);
// and write it out to the buffer
buffer.SetInt64(columnIndex, index);
}
else
{
// Write all other fields as usual
Type dataType = Type.GetType(modelCol["datatype"].ToString());
string xpath = row[colIdx];
writeDataToBuffer(jp, buffer, columnIndex, dataType, xpath);
}
// Increment to look at the next column
colIdx++;
}
}
}
