/// <summary>
        /// Calculates hash of Memaory Stream ands based ont he HashColumnInformation and stores the calculated hash into the pipeline output column
        /// </summary>
        /// <param name="hCol">HashColumnInfo for hash calculation</param>
        /// <param name="ms">Memory stream to calculate the hash</param>
        /// <param name="buffer">bufefr to store the calculated hash</param>
        public static void CalculateHashAndStoreValue(HashColumnsTransformation.HashColumnInfo hCol, HashMemoryBuffers mb, PipelineBuffer buffer)
        {
            byte[]       hash;
            MemoryStream ms = mb.MemoryStream;

            ms.Position = 2; //Set Position to 0 prior computing hash to move right after the unicode characters to not include them in the hash calculation
                             //Caculate Hash
            hash = hCol.HashAlgorithm.ComputeHash(ms);

            //Store the Hash into the Output HashColumn
            if (hCol.OutputDataType == DataType.DT_BYTES)
            {
                buffer.SetBytes(hCol.Index, hash);
            }
            else
            {
                string hashStr = BitConverter.ToString(hash).Replace("-", string.Empty);
                buffer.SetString(hCol.Index, hashStr);
            }

            if (ms.Capacity > MemoryStreamShringTreshod)
            {
                ms.Position = 0;
                ms.SetLength(0);
                ms.Capacity = MemoryStreamInitialSize;
            }
        }
        /// <summary>
        /// Builds Memory Stream data for Hashing from Input Columns
        /// </summary>
        /// <param name="hCol">HashColumnInfo</param>
        /// <param name="inputBufferColumns">InputBuferColumnInfo</param>
        /// <param name="buffer">input Buffer</param>
        /// <param name="mb">Memory Buffers</param>
        public static void BuildHashMemoryStream(HashColumnsTransformation.HashColumnInfo hCol, List <HashColumnsTransformation.InputBufferColumnInfo> inputBufferColumns, PipelineBuffer buffer, HashMemoryBuffers mb)
        {
            MemoryStream ms = mb.MemoryStream;
            StreamWriter sw = mb.StreamWriter;

            for (int i = 0; i < hCol.HashInputColumns.Count; i++)
            {
                int colIdx = hCol.HashInputColumns[i];
                HashColumnsTransformation.InputBufferColumnInfo bci = inputBufferColumns[colIdx];

                switch (hCol.HashImplmentationType)
                {
                case HashColumnsTransformation.HashImplementationType.BinarySafe:
                    WriteColumnToStreamBinary(hCol, bci, buffer, mb, sw);
                    break;

                case HashColumnsTransformation.HashImplementationType.UnicodeStringDelmited:
                    WriteColumnToStreamUnicodeDelimited(i, hCol, bci, buffer, mb);
                    break;

                default:
                    WriteColumnToStreamOriginal(bci, buffer, mb);
                    break;
                }
            }
        }
 public HashThreadState(HashColumnsTransformation.HashColumnInfo hCol, List <HashColumnsTransformation.InputBufferColumnInfo> inputBufferColumns, PipelineBuffer buffer, HashMemoryBuffers mb, ManualResetEvent resetEvent)
 {
     this.HashColumnInfo     = hCol;
     this.InputBufferColumns = inputBufferColumns;
     this.PipelineBuffer     = buffer;
     this.MemoryBuffers      = mb;
     this.ResetEvent         = resetEvent;
 }
        public static void BuildAndCalculateHash(HashColumnsTransformation.HashColumnInfo hCol, List <HashColumnsTransformation.InputBufferColumnInfo> inputBufferColumns, PipelineBuffer buffer, HashMemoryBuffers mb)
        {
            //Set length of the Stream to 2.
            //We are settiong the length to 2 tyo keep the two unicode identification bytes on the beginning as StreamWriter writes the bytes only once
            mb.MemoryStream.SetLength(2);

            //Write the input columns to Memory Stream
            HashColumnsTransformationHelper.BuildHashMemoryStream(hCol, inputBufferColumns, buffer, mb);

            //Calculate the Hash ans store it into Output columns
            HashColumnsTransformationHelper.CalculateHashAndStoreValue(hCol, mb, buffer);
        }
        private static string TrimString(HashColumnsTransformation.HashColumnInfo hcol, string str)
        {
            switch (hcol.StringTrimming)
            {
            case HashColumnsTransformation.StringTrimming.None:
                return(str);

            case HashColumnsTransformation.StringTrimming.Right:
                return(str.TrimEnd());

            case HashColumnsTransformation.StringTrimming.Left:
                return(str.TrimStart());

            case HashColumnsTransformation.StringTrimming.Full:
                return(str.Trim());

            default:
                return(str);
            }
        }
        private static void WriteColumnToStreamUnicodeDelimited(int columnPosition, HashColumnsTransformation.HashColumnInfo hCol, HashColumnsTransformation.InputBufferColumnInfo bci, PipelineBuffer buffer, HashMemoryBuffers mb)
        {
            int ci = bci.Index;

            byte[] bdata    = null;
            string strData  = null;
            bool   writeLen = false;
            bool   isNull;
            bool   trim = false;

            StreamWriter sw = mb.StreamWriter;

            BufferColumn col = buffer.GetColumnInfo(bci.Index);

            isNull = buffer.IsNull(ci);

            //When not first field, write field delimiter
            if (columnPosition > 0)
            {
                sw.Write(hCol.HashFieldsDelimiter);
            }

            if (isNull)
            {
                if (hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelimitedNullSafe ||
                    hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelmitedSafe)
                { //Safe handling: write 1 indicating null and field delimiter.
                    sw.Write(1);
                    sw.Write(hCol.HashFieldsDelimiter);
                }
                else //Non Safe handling = write replacement value
                {
                    sw.Write(hCol.NullReplacement);
                }
                sw.Flush();
                return; //return. no need for other processing as null value is stored in the field.
            }
            else if (hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelimitedNullSafe ||
                     hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelmitedSafe)
            { //Saefe handling: write 0 indicating non null value and field delimiter. Field value will be after the delimiter
                sw.Write(0);
                sw.Write(hCol.HashFieldsDelimiter);
            }

            //Get buffer data
            lock (bci)
            {
                switch (col.DataType)
                {
                case DataType.DT_BOOL:
                    sw.Write(buffer.GetBoolean(ci) ? 1 : 0);
                    break;

                case DataType.DT_BYTES:
                    bdata = buffer.GetBytes(ci);
                    break;

                case DataType.DT_IMAGE:
                    bdata = buffer.GetBlobData(ci, 0, (int)buffer.GetBlobLength(ci));
                    break;

                case DataType.DT_CY:
                case DataType.DT_DECIMAL:
                case DataType.DT_NUMERIC:
                    strData = buffer.GetDecimal(ci).ToString(CultureInfo.InvariantCulture);
                    break;

                case DataType.DT_DATE:
                    strData = buffer.GetDateTime(ci).ToString("yyyy-MM-dd HH");
                    break;

                case DataType.DT_DBTIMESTAMP:
                    strData = buffer.GetDateTime(ci).ToString("yyyy-MM-dd HH:mm:ss.fff");
                    break;

                case DataType.DT_DBTIMESTAMP2:
                    strData = buffer.GetDateTime(ci).ToString("yyyy-MM-dd HH:mm:ss.fffffff");
                    break;

                case DataType.DT_FILETIME:
                    sw.Write(buffer.GetInt64(ci));
                    break;

                case DataType.DT_DBDATE:
                    strData = buffer.GetDate(ci).ToString("yyyy-MM-dd");
                    break;

#if NET35
                case DataType.DT_DBTIME:
                    strData = new DateTime(buffer.GetTime(ci).Ticks).ToString("HH:mm:ss");
                    break;

                case DataType.DT_DBTIME2:
                    strData = new DateTime(buffer.GetTime(ci).Ticks).ToString("HH:mm:ss.fffffff");
                    break;
#else
                case DataType.DT_DBTIME:
                    strData = buffer.GetTime(ci).ToString("HH:mm:ss");
                    break;

                case DataType.DT_DBTIME2:
                    strData = buffer.GetTime(ci).ToString("HH:mm:ss.fffffff");
                    break;
#endif
                case DataType.DT_DBTIMESTAMPOFFSET:
                    strData = buffer.GetDateTimeOffset(ci).ToString("yyyy-MM-dd HH:mm:ss.fffffff zzz");
                    break;

                case DataType.DT_EMPTY:
                case DataType.DT_NULL:
                    bdata = new byte[0];
                    break;

                case DataType.DT_GUID:
                    strData = buffer.GetGuid(ci).ToString();
                    break;

                case DataType.DT_I1:
                    sw.Write(buffer.GetSByte(ci));
                    break;

                case DataType.DT_I2:
                    sw.Write(buffer.GetInt16(ci));
                    break;

                case DataType.DT_I4:
                    sw.Write(buffer.GetInt32(ci));
                    break;

                case DataType.DT_I8:
                    sw.Write(buffer.GetInt64(ci));
                    break;

                case DataType.DT_R4:
                    sw.Write(buffer.GetSingle(ci));
                    break;

                case DataType.DT_R8:
                    sw.Write(buffer.GetDouble(ci));
                    break;

                case DataType.DT_UI1:
                    sw.Write(buffer.GetByte(ci));
                    break;

                case DataType.DT_UI2:
                    sw.Write(buffer.GetUInt16(ci));
                    break;

                case DataType.DT_UI4:
                    sw.Write(buffer.GetUInt32(ci));
                    break;

                case DataType.DT_UI8:
                    sw.Write(buffer.GetUInt64(ci));
                    break;

                case DataType.DT_NTEXT:
                case DataType.DT_TEXT:
                case DataType.DT_STR:
                case DataType.DT_WSTR:
                    trim    = true;
                    strData = buffer.GetString(ci);
                    if (hCol.HashImplmentationType == HashColumnsTransformation.HashImplementationType.UnicodeStringDelmitedSafe)
                    {
                        writeLen = true;
                    }
                    break;

                default:
                    bdata = new byte[0];
                    break;
                }
            }

            if (bdata != null)
            {
                strData = BitConverter.ToString(bdata).Replace("-", "");
            }

            if (strData != null)
            {
                if (trim)
                {
                    strData = TrimString(hCol, strData);
                }

                if (writeLen)
                {
                    sw.Write(strData.Length);
                    sw.Write(hCol.HashFieldsDelimiter);
                }
                sw.Write(strData);
            }

            sw.Flush();
        }
        private static void WriteColumnToStreamBinary(HashColumnsTransformation.HashColumnInfo hCol, HashColumnsTransformation.InputBufferColumnInfo bci, PipelineBuffer buffer, HashMemoryBuffers mb, StreamWriter sw)
        {
            int ci = bci.Index;

            byte[] bdata        = null;
            byte[] decimalArray = new byte[16]; //Array for storing decimal numbers
            string asciiStr     = null;

            BinaryWriter bw = mb.BinaryWriter;

            BufferColumn col = buffer.GetColumnInfo(bci.Index);

            bw.Write((byte)0);//Write byte (0) as start of field;

            if (buffer.IsNull(bci.Index))
            {
                bw.Write((byte)1); //Write 1 representing NULL
                bw.Write(0);       //write length of 0 for NULL
            }
            else
            {
                bw.Write((byte)0); //write 0 representing NOT NULL

                //Get buffer data
                lock (bci)
                {
                    switch (col.DataType)
                    {
                    case DataType.DT_BOOL:
                        bdata = BitConverter.GetBytes(buffer.GetBoolean(ci));
                        break;

                    case DataType.DT_BYTES:
                        bdata = buffer.GetBytes(ci);
                        break;

                    case DataType.DT_IMAGE:
                        bdata = buffer.GetBlobData(ci, 0, (int)buffer.GetBlobLength(ci));
                        break;

                    case DataType.DT_CY:
                    case DataType.DT_DECIMAL:
                    case DataType.DT_NUMERIC:
                        bdata = Encoding.ASCII.GetBytes(buffer.GetDecimal(ci).ToString(CultureInfo.InvariantCulture));
                        break;

                    case DataType.DT_DATE:
                    case DataType.DT_DBTIMESTAMP:
                    case DataType.DT_DBTIMESTAMP2:
                        bw.Write(buffer.GetDateTime(ci).ToBinary());
                        break;

                    case DataType.DT_FILETIME:
                        bw.Write(buffer.GetInt64(ci));
                        break;

                    case DataType.DT_DBDATE:
                        bw.Write(buffer.GetDate(ci).ToBinary());
                        break;

                    case DataType.DT_DBTIME:
                    case DataType.DT_DBTIME2:
                        bw.Write(buffer.GetTime(ci).Ticks);
                        break;

                    case DataType.DT_DBTIMESTAMPOFFSET:
                        var dtoffset = buffer.GetDateTimeOffset(ci);
                        BitConverter.GetBytes(dtoffset.DateTime.ToBinary()).CopyTo(decimalArray, 0);
                        BitConverter.GetBytes(dtoffset.Offset.Ticks).CopyTo(decimalArray, 8);
                        bw.Write(decimalArray);
                        break;

                    case DataType.DT_EMPTY:
                    case DataType.DT_NULL:
                        bdata = new byte[0];
                        break;

                    case DataType.DT_GUID:
                        bw.Write(Encoding.ASCII.GetBytes(buffer.GetGuid(ci).ToString()));
                        break;

                    case DataType.DT_I1:
                        asciiStr = buffer.GetSByte(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_I2:
                        asciiStr = buffer.GetInt16(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_I4:
                        asciiStr = buffer.GetInt32(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_I8:
                        asciiStr = buffer.GetInt64(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_R4:
                        asciiStr = buffer.GetSingle(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_R8:
                        asciiStr = buffer.GetDouble(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_UI1:
                        asciiStr = buffer.GetByte(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_UI2:
                        asciiStr = buffer.GetUInt16(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_UI4:
                        asciiStr = buffer.GetUInt32(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_UI8:
                        asciiStr = buffer.GetUInt64(ci).ToString(CultureInfo.InvariantCulture);
                        break;

                    case DataType.DT_NTEXT:
                    case DataType.DT_TEXT:
                    case DataType.DT_STR:
                    case DataType.DT_WSTR:
                        bdata = Encoding.Unicode.GetBytes(TrimString(hCol, buffer.GetString(ci)));
                        break;

                    default:
                        bdata = new byte[0];
                        break;
                    }
                }

                if (asciiStr != null)
                {
                    bdata = Encoding.ASCII.GetBytes(asciiStr);
                }

                if (bdata != null)
                {
                    bw.Write(bdata.Length); //write length of buffer
                    bw.Write(bdata);        //write bufferdata;
                }
            }
        }