/// <summary>
/// Calculates hash of Memaory Stream ands based ont he HashColumnInformation and stores the calculated hash into the pipeline output column
/// </summary>
/// <param name="hCol">HashColumnInfo for hash calculation</param>
/// <param name="ms">Memory stream to calculate the hash</param>
/// <param name="buffer">bufefr to store the calculated hash</param>
public static void CalculateHashAndStoreValue(HashColumnsTransformation.HashColumnInfo hCol, HashMemoryBuffers mb, PipelineBuffer buffer)
{
byte[] hash;
MemoryStream ms = mb.MemoryStream;
ms.Position = 2; //Set Position to 0 prior computing hash to move right after the unicode characters to not include them in the hash calculation
//Caculate Hash
hash = hCol.HashAlgorithm.ComputeHash(ms);
//Store the Hash into the Output HashColumn
if (hCol.OutputDataType == DataType.DT_BYTES)
{
buffer.SetBytes(hCol.Index, hash);
}
else
{
string hashStr = BitConverter.ToString(hash).Replace("-", string.Empty);
buffer.SetString(hCol.Index, hashStr);
}
if (ms.Capacity > MemoryStreamShringTreshod)
{
ms.Position = 0;
ms.SetLength(0);
ms.Capacity = MemoryStreamInitialSize;
}
}
public HashThreadState(HashColumnsTransformation.HashColumnInfo hCol, List <HashColumnsTransformation.InputBufferColumnInfo> inputBufferColumns, PipelineBuffer buffer, HashMemoryBuffers mb, ManualResetEvent resetEvent)
{
this.HashColumnInfo = hCol;
this.InputBufferColumns = inputBufferColumns;
this.PipelineBuffer = buffer;
this.MemoryBuffers = mb;
this.ResetEvent = resetEvent;
}
public static void BuildAndCalculateHash(HashColumnsTransformation.HashColumnInfo hCol, List <HashColumnsTransformation.InputBufferColumnInfo> inputBufferColumns, PipelineBuffer buffer, HashMemoryBuffers mb)
{
//Set length of the Stream to 2.
//We are settiong the length to 2 tyo keep the two unicode identification bytes on the beginning as StreamWriter writes the bytes only once
mb.MemoryStream.SetLength(2);
//Write the input columns to Memory Stream
HashColumnsTransformationHelper.BuildHashMemoryStream(hCol, inputBufferColumns, buffer, mb);
//Calculate the Hash ans store it into Output columns
HashColumnsTransformationHelper.CalculateHashAndStoreValue(hCol, mb, buffer);
}
private static void WriteColumnToStreamOriginal(HashColumnsTransformation.InputBufferColumnInfo bci, PipelineBuffer buffer, HashMemoryBuffers mb)
{
int ci = bci.Index;
byte[] bdata = null;
byte[] decimalArray = mb.DecimalArray; //Array for storing decimal numbers
BinaryWriter bw = mb.BinaryWriter;
StreamWriter sw = mb.StreamWriter;
BufferColumn col = buffer.GetColumnInfo(bci.Index);
bw.Write((int)col.DataType); //write data type
if (buffer.IsNull(bci.Index))
{
bw.Write((byte)1); //Write 1 representing NULL
bw.Write(0); //write length of 0 for NULL
}
else
{
bw.Write((byte)0); //write 0 representing NOT NULL
//Get buffer data
lock (bci)
{
switch (col.DataType)
{
case DataType.DT_BOOL:
bdata = BitConverter.GetBytes(buffer.GetBoolean(ci));
break;
case DataType.DT_BYTES:
bdata = buffer.GetBytes(ci);
break;
case DataType.DT_IMAGE:
case DataType.DT_NTEXT:
case DataType.DT_TEXT:
bdata = buffer.GetBlobData(ci, 0, (int)buffer.GetBlobLength(ci));
break;
case DataType.DT_CY:
case DataType.DT_DECIMAL:
case DataType.DT_NUMERIC:
var ia = decimal.GetBits(buffer.GetDecimal(ci));
for (int j = 0; j < 4; j++)
{
int k = 4 * j;
decimalArray[k] = (byte)(ia[j] & 0xFF);
decimalArray[k + 1] = (byte)(ia[j] >> 8 & 0xFF);
decimalArray[k + 2] = (byte)(ia[j] >> 16 & 0xFF);
decimalArray[k + 3] = (byte)(ia[j] >> 24 & 0xFF);
}
bdata = decimalArray;
break;
case DataType.DT_DATE:
case DataType.DT_DBTIMESTAMP:
case DataType.DT_DBTIMESTAMP2:
bdata = BitConverter.GetBytes(buffer.GetDateTime(ci).ToBinary());
break;
case DataType.DT_FILETIME:
bdata = BitConverter.GetBytes(buffer.GetInt64(ci));
break;
case DataType.DT_DBDATE:
bw.Write(buffer.GetDate(ci).ToBinary());
break;
case DataType.DT_DBTIME:
case DataType.DT_DBTIME2:
bdata = BitConverter.GetBytes(DateTime.MinValue.Add(buffer.GetTime(ci)).ToBinary());
break;
case DataType.DT_DBTIMESTAMPOFFSET:
var dtoffset = buffer.GetDateTimeOffset(ci);
BitConverter.GetBytes(dtoffset.DateTime.ToBinary()).CopyTo(bdata, 0);
BitConverter.GetBytes(DateTime.MinValue.Add(dtoffset.Offset).ToBinary()).CopyTo(bdata, 8);
bdata = decimalArray;
break;
case DataType.DT_EMPTY:
case DataType.DT_NULL:
bdata = new byte[0];
break;
case DataType.DT_GUID:
bdata = buffer.GetGuid(ci).ToByteArray();
break;
case DataType.DT_I1:
bdata = BitConverter.GetBytes(buffer.GetSByte(ci));
break;
case DataType.DT_I2:
bdata = BitConverter.GetBytes(buffer.GetInt16(ci));
break;
case DataType.DT_I4:
bdata = BitConverter.GetBytes(buffer.GetInt32(ci));
break;
case DataType.DT_I8:
bdata = BitConverter.GetBytes(buffer.GetInt64(ci));
break;
case DataType.DT_R4:
bdata = BitConverter.GetBytes(buffer.GetSingle(ci));
break;
case DataType.DT_R8:
bdata = BitConverter.GetBytes(buffer.GetDouble(ci));
break;
case DataType.DT_UI1:
bdata = BitConverter.GetBytes(buffer.GetByte(ci));
break;
case DataType.DT_UI2:
bdata = BitConverter.GetBytes(buffer.GetUInt16(ci));
break;
case DataType.DT_UI4:
bdata = BitConverter.GetBytes(buffer.GetUInt32(ci));
break;
case DataType.DT_UI8:
bdata = BitConverter.GetBytes(buffer.GetUInt64(ci));
break;
case DataType.DT_STR:
case DataType.DT_WSTR:
bdata = Encoding.Unicode.GetBytes(buffer.GetString(ci));
break;
default:
bdata = new byte[0];
break;
}
}
if (bdata != null)
{
bw.Write(bdata.Length); //write length of buffer
bw.Write(bdata); //write bufferdata;
}
}
}
private static void WriteColumnToStreamUnicodeDelimited(int columnPosition, HashColumnsTransformation.HashColumnInfo hCol, HashColumnsTransformation.InputBufferColumnInfo bci, PipelineBuffer buffer, HashMemoryBuffers mb)
{
int ci = bci.Index;
byte[] bdata = null;
string strData = null;
bool writeLen = false;
bool isNull;
bool trim = false;
StreamWriter sw = mb.StreamWriter;
BufferColumn col = buffer.GetColumnInfo(bci.Index);
isNull = buffer.IsNull(ci);
//When not first field, write field delimiter
if (columnPosition > 0)
{
sw.Write(hCol.HashFieldsDelimiter);
}
if (isNull)
{
if (hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelimitedNullSafe ||
hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelmitedSafe)
{ //Safe handling: write 1 indicating null and field delimiter.
sw.Write(1);
sw.Write(hCol.HashFieldsDelimiter);
}
else //Non Safe handling = write replacement value
{
sw.Write(hCol.NullReplacement);
}
sw.Flush();
return; //return. no need for other processing as null value is stored in the field.
}
else if (hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelimitedNullSafe ||
hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelmitedSafe)
{ //Saefe handling: write 0 indicating non null value and field delimiter. Field value will be after the delimiter
sw.Write(0);
sw.Write(hCol.HashFieldsDelimiter);
}
//Get buffer data
lock (bci)
{
switch (col.DataType)
{
case DataType.DT_BOOL:
sw.Write(buffer.GetBoolean(ci) ? 1 : 0);
break;
case DataType.DT_BYTES:
bdata = buffer.GetBytes(ci);
break;
case DataType.DT_IMAGE:
bdata = buffer.GetBlobData(ci, 0, (int)buffer.GetBlobLength(ci));
break;
case DataType.DT_CY:
case DataType.DT_DECIMAL:
case DataType.DT_NUMERIC:
strData = buffer.GetDecimal(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_DATE:
strData = buffer.GetDateTime(ci).ToString("yyyy-MM-dd HH");
break;
case DataType.DT_DBTIMESTAMP:
strData = buffer.GetDateTime(ci).ToString("yyyy-MM-dd HH:mm:ss.fff");
break;
case DataType.DT_DBTIMESTAMP2:
strData = buffer.GetDateTime(ci).ToString("yyyy-MM-dd HH:mm:ss.fffffff");
break;
case DataType.DT_FILETIME:
sw.Write(buffer.GetInt64(ci));
break;
case DataType.DT_DBDATE:
strData = buffer.GetDate(ci).ToString("yyyy-MM-dd");
break;
#if NET35
case DataType.DT_DBTIME:
strData = new DateTime(buffer.GetTime(ci).Ticks).ToString("HH:mm:ss");
break;
case DataType.DT_DBTIME2:
strData = new DateTime(buffer.GetTime(ci).Ticks).ToString("HH:mm:ss.fffffff");
break;
#else
case DataType.DT_DBTIME:
strData = buffer.GetTime(ci).ToString("HH:mm:ss");
break;
case DataType.DT_DBTIME2:
strData = buffer.GetTime(ci).ToString("HH:mm:ss.fffffff");
break;
#endif
case DataType.DT_DBTIMESTAMPOFFSET:
strData = buffer.GetDateTimeOffset(ci).ToString("yyyy-MM-dd HH:mm:ss.fffffff zzz");
break;
case DataType.DT_EMPTY:
case DataType.DT_NULL:
bdata = new byte[0];
break;
case DataType.DT_GUID:
strData = buffer.GetGuid(ci).ToString();
break;
case DataType.DT_I1:
sw.Write(buffer.GetSByte(ci));
break;
case DataType.DT_I2:
sw.Write(buffer.GetInt16(ci));
break;
case DataType.DT_I4:
sw.Write(buffer.GetInt32(ci));
break;
case DataType.DT_I8:
sw.Write(buffer.GetInt64(ci));
break;
case DataType.DT_R4:
sw.Write(buffer.GetSingle(ci));
break;
case DataType.DT_R8:
sw.Write(buffer.GetDouble(ci));
break;
case DataType.DT_UI1:
sw.Write(buffer.GetByte(ci));
break;
case DataType.DT_UI2:
sw.Write(buffer.GetUInt16(ci));
break;
case DataType.DT_UI4:
sw.Write(buffer.GetUInt32(ci));
break;
case DataType.DT_UI8:
sw.Write(buffer.GetUInt64(ci));
break;
case DataType.DT_NTEXT:
case DataType.DT_TEXT:
case DataType.DT_STR:
case DataType.DT_WSTR:
trim = true;
strData = buffer.GetString(ci);
if (hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelmitedSafe)
{
writeLen = true;
}
break;
default:
bdata = new byte[0];
break;
}
}
if (bdata != null)
{
strData = BitConverter.ToString(bdata).Replace("-", "");
}
if (strData != null)
{
if (trim)
{
strData = TrimString(hCol, strData);
}
if (writeLen)
{
sw.Write(strData.Length);
sw.Write(hCol.HashFieldsDelimiter);
}
sw.Write(strData);
}
sw.Flush();
}
private static void WriteColumnToStreamBinary(HashColumnsTransformation.HashColumnInfo hCol, HashColumnsTransformation.InputBufferColumnInfo bci, PipelineBuffer buffer, HashMemoryBuffers mb, StreamWriter sw)
{
int ci = bci.Index;
byte[] bdata = null;
byte[] decimalArray = new byte[16]; //Array for storing decimal numbers
string asciiStr = null;
BinaryWriter bw = mb.BinaryWriter;
BufferColumn col = buffer.GetColumnInfo(bci.Index);
bw.Write((byte)0);//Write byte (0) as start of field;
if (buffer.IsNull(bci.Index))
{
bw.Write((byte)1); //Write 1 representing NULL
bw.Write(0); //write length of 0 for NULL
}
else
{
bw.Write((byte)0); //write 0 representing NOT NULL
//Get buffer data
lock (bci)
{
switch (col.DataType)
{
case DataType.DT_BOOL:
bdata = BitConverter.GetBytes(buffer.GetBoolean(ci));
break;
case DataType.DT_BYTES:
bdata = buffer.GetBytes(ci);
break;
case DataType.DT_IMAGE:
bdata = buffer.GetBlobData(ci, 0, (int)buffer.GetBlobLength(ci));
break;
case DataType.DT_CY:
case DataType.DT_DECIMAL:
case DataType.DT_NUMERIC:
bdata = Encoding.ASCII.GetBytes(buffer.GetDecimal(ci).ToString(CultureInfo.InvariantCulture));
break;
case DataType.DT_DATE:
case DataType.DT_DBTIMESTAMP:
case DataType.DT_DBTIMESTAMP2:
bw.Write(buffer.GetDateTime(ci).ToBinary());
break;
case DataType.DT_FILETIME:
bw.Write(buffer.GetInt64(ci));
break;
case DataType.DT_DBDATE:
bw.Write(buffer.GetDate(ci).ToBinary());
break;
case DataType.DT_DBTIME:
case DataType.DT_DBTIME2:
bw.Write(buffer.GetTime(ci).Ticks);
break;
case DataType.DT_DBTIMESTAMPOFFSET:
var dtoffset = buffer.GetDateTimeOffset(ci);
BitConverter.GetBytes(dtoffset.DateTime.ToBinary()).CopyTo(decimalArray, 0);
BitConverter.GetBytes(dtoffset.Offset.Ticks).CopyTo(decimalArray, 8);
bw.Write(decimalArray);
break;
case DataType.DT_EMPTY:
case DataType.DT_NULL:
bdata = new byte[0];
break;
case DataType.DT_GUID:
bw.Write(Encoding.ASCII.GetBytes(buffer.GetGuid(ci).ToString()));
break;
case DataType.DT_I1:
asciiStr = buffer.GetSByte(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_I2:
asciiStr = buffer.GetInt16(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_I4:
asciiStr = buffer.GetInt32(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_I8:
asciiStr = buffer.GetInt64(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_R4:
asciiStr = buffer.GetSingle(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_R8:
asciiStr = buffer.GetDouble(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_UI1:
asciiStr = buffer.GetByte(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_UI2:
asciiStr = buffer.GetUInt16(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_UI4:
asciiStr = buffer.GetUInt32(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_UI8:
asciiStr = buffer.GetUInt64(ci).ToString(CultureInfo.InvariantCulture);
break;
case DataType.DT_NTEXT:
case DataType.DT_TEXT:
case DataType.DT_STR:
case DataType.DT_WSTR:
bdata = Encoding.Unicode.GetBytes(TrimString(hCol, buffer.GetString(ci)));
break;
default:
bdata = new byte[0];
break;
}
}
if (asciiStr != null)
{
bdata = Encoding.ASCII.GetBytes(asciiStr);
}
if (bdata != null)
{
bw.Write(bdata.Length); //write length of buffer
bw.Write(bdata); //write bufferdata;
}
}
}
/// <summary>
/// Builds Memory Stream data for Hashing from Input Columns
/// </summary>
/// <param name="hCol">HashColumnInfo</param>
/// <param name="inputBufferColumns">InputBuferColumnInfo</param>
/// <param name="buffer">input Buffer</param>
/// <param name="mb">Memory Buffers</param>
public static void BuildHashMemoryStream(HashColumnsTransformation.HashColumnInfo hCol, List <HashColumnsTransformation.InputBufferColumnInfo> inputBufferColumns, PipelineBuffer buffer, HashMemoryBuffers mb)
{
MemoryStream ms = mb.MemoryStream;
StreamWriter sw = mb.StreamWriter;
for (int i = 0; i < hCol.HashInputColumns.Count; i++)
{
int colIdx = hCol.HashInputColumns[i];
HashColumnsTransformation.InputBufferColumnInfo bci = inputBufferColumns[colIdx];
switch (hCol.HashImplmentationType)
{
case HashColumnsTransformation.HashImplementationType.BinarySafe:
WriteColumnToStreamBinary(hCol, bci, buffer, mb, sw);
break;
case HashColumnsTransformation.HashImplementationType.UnicodeStringDelmited:
WriteColumnToStreamUnicodeDelimited(i, hCol, bci, buffer, mb);
break;
default:
WriteColumnToStreamOriginal(bci, buffer, mb);
break;
}
}
}
