/// <summary> /// Calculates hash of Memaory Stream ands based ont he HashColumnInformation and stores the calculated hash into the pipeline output column /// </summary> /// <param name="hCol">HashColumnInfo for hash calculation</param> /// <param name="ms">Memory stream to calculate the hash</param> /// <param name="buffer">bufefr to store the calculated hash</param> public static void CalculateHashAndStoreValue(HashColumnsTransformation.HashColumnInfo hCol, HashMemoryBuffers mb, PipelineBuffer buffer) { byte[] hash; MemoryStream ms = mb.MemoryStream; ms.Position = 2; //Set Position to 0 prior computing hash to move right after the unicode characters to not include them in the hash calculation //Caculate Hash hash = hCol.HashAlgorithm.ComputeHash(ms); //Store the Hash into the Output HashColumn if (hCol.OutputDataType == DataType.DT_BYTES) { buffer.SetBytes(hCol.Index, hash); } else { string hashStr = BitConverter.ToString(hash).Replace("-", string.Empty); buffer.SetString(hCol.Index, hashStr); } if (ms.Capacity > MemoryStreamShringTreshod) { ms.Position = 0; ms.SetLength(0); ms.Capacity = MemoryStreamInitialSize; } }
/// <summary> /// Builds Memory Stream data for Hashing from Input Columns /// </summary> /// <param name="hCol">HashColumnInfo</param> /// <param name="inputBufferColumns">InputBuferColumnInfo</param> /// <param name="buffer">input Buffer</param> /// <param name="mb">Memory Buffers</param> public static void BuildHashMemoryStream(HashColumnsTransformation.HashColumnInfo hCol, List <HashColumnsTransformation.InputBufferColumnInfo> inputBufferColumns, PipelineBuffer buffer, HashMemoryBuffers mb) { MemoryStream ms = mb.MemoryStream; StreamWriter sw = mb.StreamWriter; for (int i = 0; i < hCol.HashInputColumns.Count; i++) { int colIdx = hCol.HashInputColumns[i]; HashColumnsTransformation.InputBufferColumnInfo bci = inputBufferColumns[colIdx]; switch (hCol.HashImplmentationType) { case HashColumnsTransformation.HashImplementationType.BinarySafe: WriteColumnToStreamBinary(hCol, bci, buffer, mb, sw); break; case HashColumnsTransformation.HashImplementationType.UnicodeStringDelmited: WriteColumnToStreamUnicodeDelimited(i, hCol, bci, buffer, mb); break; default: WriteColumnToStreamOriginal(bci, buffer, mb); break; } } }
public HashThreadState(HashColumnsTransformation.HashColumnInfo hCol, List <HashColumnsTransformation.InputBufferColumnInfo> inputBufferColumns, PipelineBuffer buffer, HashMemoryBuffers mb, ManualResetEvent resetEvent) { this.HashColumnInfo = hCol; this.InputBufferColumns = inputBufferColumns; this.PipelineBuffer = buffer; this.MemoryBuffers = mb; this.ResetEvent = resetEvent; }
public static void BuildAndCalculateHash(HashColumnsTransformation.HashColumnInfo hCol, List <HashColumnsTransformation.InputBufferColumnInfo> inputBufferColumns, PipelineBuffer buffer, HashMemoryBuffers mb) { //Set length of the Stream to 2. //We are settiong the length to 2 tyo keep the two unicode identification bytes on the beginning as StreamWriter writes the bytes only once mb.MemoryStream.SetLength(2); //Write the input columns to Memory Stream HashColumnsTransformationHelper.BuildHashMemoryStream(hCol, inputBufferColumns, buffer, mb); //Calculate the Hash ans store it into Output columns HashColumnsTransformationHelper.CalculateHashAndStoreValue(hCol, mb, buffer); }
private static string TrimString(HashColumnsTransformation.HashColumnInfo hcol, string str) { switch (hcol.StringTrimming) { case HashColumnsTransformation.StringTrimming.None: return(str); case HashColumnsTransformation.StringTrimming.Right: return(str.TrimEnd()); case HashColumnsTransformation.StringTrimming.Left: return(str.TrimStart()); case HashColumnsTransformation.StringTrimming.Full: return(str.Trim()); default: return(str); } }
private static void WriteColumnToStreamUnicodeDelimited(int columnPosition, HashColumnsTransformation.HashColumnInfo hCol, HashColumnsTransformation.InputBufferColumnInfo bci, PipelineBuffer buffer, HashMemoryBuffers mb) { int ci = bci.Index; byte[] bdata = null; string strData = null; bool writeLen = false; bool isNull; bool trim = false; StreamWriter sw = mb.StreamWriter; BufferColumn col = buffer.GetColumnInfo(bci.Index); isNull = buffer.IsNull(ci); //When not first field, write field delimiter if (columnPosition > 0) { sw.Write(hCol.HashFieldsDelimiter); } if (isNull) { if (hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelimitedNullSafe || hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelmitedSafe) { //Safe handling: write 1 indicating null and field delimiter. sw.Write(1); sw.Write(hCol.HashFieldsDelimiter); } else //Non Safe handling = write replacement value { sw.Write(hCol.NullReplacement); } sw.Flush(); return; //return. no need for other processing as null value is stored in the field. } else if (hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelimitedNullSafe || hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelmitedSafe) { //Saefe handling: write 0 indicating non null value and field delimiter. Field value will be after the delimiter sw.Write(0); sw.Write(hCol.HashFieldsDelimiter); } //Get buffer data lock (bci) { switch (col.DataType) { case DataType.DT_BOOL: sw.Write(buffer.GetBoolean(ci) ? 1 : 0); break; case DataType.DT_BYTES: bdata = buffer.GetBytes(ci); break; case DataType.DT_IMAGE: bdata = buffer.GetBlobData(ci, 0, (int)buffer.GetBlobLength(ci)); break; case DataType.DT_CY: case DataType.DT_DECIMAL: case DataType.DT_NUMERIC: strData = buffer.GetDecimal(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_DATE: strData = buffer.GetDateTime(ci).ToString("yyyy-MM-dd HH"); break; case DataType.DT_DBTIMESTAMP: strData = buffer.GetDateTime(ci).ToString("yyyy-MM-dd HH:mm:ss.fff"); break; case DataType.DT_DBTIMESTAMP2: strData = buffer.GetDateTime(ci).ToString("yyyy-MM-dd HH:mm:ss.fffffff"); break; case DataType.DT_FILETIME: sw.Write(buffer.GetInt64(ci)); break; case DataType.DT_DBDATE: strData = buffer.GetDate(ci).ToString("yyyy-MM-dd"); break; #if NET35 case DataType.DT_DBTIME: strData = new DateTime(buffer.GetTime(ci).Ticks).ToString("HH:mm:ss"); break; case DataType.DT_DBTIME2: strData = new DateTime(buffer.GetTime(ci).Ticks).ToString("HH:mm:ss.fffffff"); break; #else case DataType.DT_DBTIME: strData = buffer.GetTime(ci).ToString("HH:mm:ss"); break; case DataType.DT_DBTIME2: strData = buffer.GetTime(ci).ToString("HH:mm:ss.fffffff"); break; #endif case DataType.DT_DBTIMESTAMPOFFSET: strData = buffer.GetDateTimeOffset(ci).ToString("yyyy-MM-dd HH:mm:ss.fffffff zzz"); break; case DataType.DT_EMPTY: case DataType.DT_NULL: bdata = new byte[0]; break; case DataType.DT_GUID: strData = buffer.GetGuid(ci).ToString(); break; case DataType.DT_I1: sw.Write(buffer.GetSByte(ci)); break; case DataType.DT_I2: sw.Write(buffer.GetInt16(ci)); break; case DataType.DT_I4: sw.Write(buffer.GetInt32(ci)); break; case DataType.DT_I8: sw.Write(buffer.GetInt64(ci)); break; case DataType.DT_R4: sw.Write(buffer.GetSingle(ci)); break; case DataType.DT_R8: sw.Write(buffer.GetDouble(ci)); break; case DataType.DT_UI1: sw.Write(buffer.GetByte(ci)); break; case DataType.DT_UI2: sw.Write(buffer.GetUInt16(ci)); break; case DataType.DT_UI4: sw.Write(buffer.GetUInt32(ci)); break; case DataType.DT_UI8: sw.Write(buffer.GetUInt64(ci)); break; case DataType.DT_NTEXT: case DataType.DT_TEXT: case DataType.DT_STR: case DataType.DT_WSTR: trim = true; strData = buffer.GetString(ci); if (hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelmitedSafe) { writeLen = true; } break; default: bdata = new byte[0]; break; } } if (bdata != null) { strData = BitConverter.ToString(bdata).Replace("-", ""); } if (strData != null) { if (trim) { strData = TrimString(hCol, strData); } if (writeLen) { sw.Write(strData.Length); sw.Write(hCol.HashFieldsDelimiter); } sw.Write(strData); } sw.Flush(); }
private static void WriteColumnToStreamBinary(HashColumnsTransformation.HashColumnInfo hCol, HashColumnsTransformation.InputBufferColumnInfo bci, PipelineBuffer buffer, HashMemoryBuffers mb, StreamWriter sw) { int ci = bci.Index; byte[] bdata = null; byte[] decimalArray = new byte[16]; //Array for storing decimal numbers string asciiStr = null; BinaryWriter bw = mb.BinaryWriter; BufferColumn col = buffer.GetColumnInfo(bci.Index); bw.Write((byte)0);//Write byte (0) as start of field; if (buffer.IsNull(bci.Index)) { bw.Write((byte)1); //Write 1 representing NULL bw.Write(0); //write length of 0 for NULL } else { bw.Write((byte)0); //write 0 representing NOT NULL //Get buffer data lock (bci) { switch (col.DataType) { case DataType.DT_BOOL: bdata = BitConverter.GetBytes(buffer.GetBoolean(ci)); break; case DataType.DT_BYTES: bdata = buffer.GetBytes(ci); break; case DataType.DT_IMAGE: bdata = buffer.GetBlobData(ci, 0, (int)buffer.GetBlobLength(ci)); break; case DataType.DT_CY: case DataType.DT_DECIMAL: case DataType.DT_NUMERIC: bdata = Encoding.ASCII.GetBytes(buffer.GetDecimal(ci).ToString(CultureInfo.InvariantCulture)); break; case DataType.DT_DATE: case DataType.DT_DBTIMESTAMP: case DataType.DT_DBTIMESTAMP2: bw.Write(buffer.GetDateTime(ci).ToBinary()); break; case DataType.DT_FILETIME: bw.Write(buffer.GetInt64(ci)); break; case DataType.DT_DBDATE: bw.Write(buffer.GetDate(ci).ToBinary()); break; case DataType.DT_DBTIME: case DataType.DT_DBTIME2: bw.Write(buffer.GetTime(ci).Ticks); break; case DataType.DT_DBTIMESTAMPOFFSET: var dtoffset = buffer.GetDateTimeOffset(ci); BitConverter.GetBytes(dtoffset.DateTime.ToBinary()).CopyTo(decimalArray, 0); BitConverter.GetBytes(dtoffset.Offset.Ticks).CopyTo(decimalArray, 8); bw.Write(decimalArray); break; case DataType.DT_EMPTY: case DataType.DT_NULL: bdata = new byte[0]; break; case DataType.DT_GUID: bw.Write(Encoding.ASCII.GetBytes(buffer.GetGuid(ci).ToString())); break; case DataType.DT_I1: asciiStr = buffer.GetSByte(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_I2: asciiStr = buffer.GetInt16(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_I4: asciiStr = buffer.GetInt32(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_I8: asciiStr = buffer.GetInt64(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_R4: asciiStr = buffer.GetSingle(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_R8: asciiStr = buffer.GetDouble(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_UI1: asciiStr = buffer.GetByte(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_UI2: asciiStr = buffer.GetUInt16(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_UI4: asciiStr = buffer.GetUInt32(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_UI8: asciiStr = buffer.GetUInt64(ci).ToString(CultureInfo.InvariantCulture); break; case DataType.DT_NTEXT: case DataType.DT_TEXT: case DataType.DT_STR: case DataType.DT_WSTR: bdata = Encoding.Unicode.GetBytes(TrimString(hCol, buffer.GetString(ci))); break; default: bdata = new byte[0]; break; } } if (asciiStr != null) { bdata = Encoding.ASCII.GetBytes(asciiStr); } if (bdata != null) { bw.Write(bdata.Length); //write length of buffer bw.Write(bdata); //write bufferdata; } } }