/// <summary> /// This method converts the given array of time series values (array of double precision /// floats) to a BLOB (byte array). It also sets the computes the checksum from the resultant /// BLOB, and then sets the Checksum and ValueBlob properties of the given ITimeSeriesTrace /// object accordingly. /// </summary> /// <param name="valueArray">The array of time series values to convert into a BLOB</param> /// <param name="compressionCode">a generation number that indicates what compression technique to use</param> /// <param name="traceObject">object whose TraceNumber property will be used to compute the checksum, /// and whose properties will be assigned by this method</param> /// <returns>The BLOB that is created from valueArray</returns> public static unsafe byte[] ConvertArrayToBlobRegular( double[] valueArray, int compressionCode, ITimeSeriesTrace traceObject) { // The number of bytes required for the BLOB int nBin = traceObject.TimeStepCount * sizeof(double); // Allocate an array for the BLOB Byte[] blobData = new Byte[nBin]; // Copy the array of doubles that was passed to the method into the byte array. // The byte array becomes the BLOB. Buffer.BlockCopy(valueArray, 0, blobData, 0, nBin); // Compute the checksum using the uncompressed BLOB. During development, it was // demonstrated that the checksum would be computed faster on the compressed BLOB. // However, this could make it difficult to upgrade the compression algorithm in the // future, because the checksum value would be dependent on the compression algorithm. Byte[] checksum = ComputeTraceChecksum(traceObject.TraceNumber, blobData); Boolean checksumChanged = (MurmurHash.ByteArraysAreEqual(traceObject.Checksum, checksum) == false); // If the checksum did not change, then we will not assign any properties to the traceObject. // The result will be that we will return the original ValueBlob. If the checksum did change, // then we compute a new compressed ValueBlob and assign the new values. if (checksumChanged) { traceObject.Checksum = checksum; // the BLOB is stored in a compressed form, so our last step is to compress it traceObject.ValueBlob = CompressBlob(blobData, compressionCode); } return(traceObject.ValueBlob); }
/// <summary> /// This method adds the data contained in the traceObject parameter into a new row in a DataTable, /// which is suitable for quick insertion--at a later time--to the trace table. This method doesn't /// change any data in the database--it is assumed that method TSConnection.CommitNewTraceWrites /// will be called later in order to send the changes to the database. The properties of the /// traceObject, including the checksum and BLOB, should be populated before calling this method /// </summary> /// <param name="traceObject">ITimeSeriesTrace object containing the properties that are to /// be inserted to the trace table</param> private unsafe void WriteTrace(ITimeSeriesTrace traceObject) { DataTable dataTable; // Attempt to get the existing DataTable object from the collection that is kept by // the TSConnection object. If this fails, then we'll create a new DataTable. if (TSConnection.BulkCopyDataTables.TryGetValue(TraceTableName, out dataTable) == false) { // Create the DataTable object and add columns that match the columns // of the database table. dataTable = new DataTable(); dataTable.Columns.Add("TimeSeries_Id", typeof(int)); dataTable.Columns.Add("TraceNumber", typeof(int)); dataTable.Columns.Add("TimeStepCount", typeof(int)); dataTable.Columns.Add("EndDate", typeof(DateTime)); dataTable.Columns.Add("ValueBlob", typeof(byte[])); dataTable.Columns.Add("Checksum", typeof(byte[])); // Add the DataTable to a collection that is kept in the TSConnection object. TSConnection.BulkCopyDataTables.Add(TraceTableName, dataTable); } // Add the trace as a new DataRow object in the DataTable. In the 'Add' method, // the parameters must be entered in the same order as the columns were created // in the code immediately above. dataTable.Rows.Add(Id, traceObject.TraceNumber, traceObject.TimeStepCount, traceObject.EndDate, traceObject.ValueBlob, traceObject.Checksum); }
private unsafe void WriteOneTraceParam(int id, ITimeSeriesTrace traceObject, SqlCommand cmd) { cmd.Parameters["@TimeSeries_Id"].Value = id; cmd.Parameters["@TraceNumber"].Value = 26; cmd.Parameters["@ValueBlob"].Value = traceObject.ValueBlob; cmd.Parameters["@Checksum"].Value = traceObject.ValueBlob; cmd.ExecuteNonQuery(); }
/// <summary> /// This method converts a List of TimeSeriesValue objects into a BLOB (byte array) of /// time series values and computes a checksum from the BLOB. This method assigns the new values /// of the ValueBlob, Checksum, EndDate, and TimeStepCount to the object given in the traceObject /// parameter. The TraceNumber property of the traceObject parameter must be set before calling /// this method. /// /// The entire List is converted into the BLOB--i.e., the method does not take any /// parameters for limiting the size of the List that is created. This method will /// throw exceptions if the meta-parameters that are passed in are not consistent /// with the List of TimeSeriesValue objects. /// </summary> /// <param name="timeStepUnit">TSDateCalculator.TimeStepUnitCode value for /// Minute, Hour, Day, Week, Month, Year, or Irregular</param> /// <param name="timeStepQuantity">The number of the given unit that defines the time step. /// For instance, if the time step is 6 hours long, then this value is 6.</param> /// <param name="timeStepCount">The number of time steps stored in the BLOB</param> /// <param name="blobStartDate">Date of the first time step in the BLOB</param> /// <param name="blobEndDate">Date of the last time step in the BLOB</param> /// <param name="dateValueList">A List of TimeSeriesValue objects that will be converted to a BLOB</param> /// <param name="traceObject">an object which contains the a TraceNumber property that is used to /// compute the checksum. The computed BLOB and checksum are both saved to the appropriate properties /// of this object.</param> /// <param name="compressionCode">a generation number that indicates what compression technique to use</param> /// <returns>The BLOB (byte array) of time series values that was created from dateValueList</returns> public byte[] ConvertListToBlobWithChecksum( TSDateCalculator.TimeStepUnitCode timeStepUnit, short timeStepQuantity, int timeStepCount, DateTime blobStartDate, DateTime blobEndDate, List <TimeSeriesValue> dateValueList, ITimeSeriesTrace traceObject, out int compressionCode) { // Error checks if (dateValueList.Count != timeStepCount) { throw new TSLibraryException(ErrCode.Enum.Checksum_Improper_Count); } if (dateValueList[0].Date != blobStartDate) { throw new TSLibraryException(ErrCode.Enum.Checksum_Improper_StartDate); } if (dateValueList.Last().Date != blobEndDate) { throw new TSLibraryException(ErrCode.Enum.Checksum_Improper_EndDate); } // When compressing, we always use the latest compression method compressionCode = TSBlobCoder.currentCompressionCode; // Assign properties to the Trace object if (traceObject.EndDate != blobEndDate) { traceObject.EndDate = blobEndDate; } if (traceObject.TimeStepCount != timeStepCount) { traceObject.TimeStepCount = timeStepCount; } // Convert the List dateValueList into a BLOB. if (timeStepUnit == TSDateCalculator.TimeStepUnitCode.Irregular) { // IRREGULAR TIME SERIES // The method in TSBlobCoder can only process an array of TSDateValueStruct. Therefore // we convert the List of objects to an Array of struct instances. TSDateValueStruct[] dateValueArray = dateValueList.Select(tsv => (TSDateValueStruct)tsv).ToArray(); // Let the method in TSBlobCoder class do all the work TSBlobCoder.ConvertArrayToBlobIrregular(dateValueArray, compressionCode, traceObject); } else { // REGULAR TIME SERIES // The method in TSBlobCoder can only process an array of double values. Therefore // we convert the List of date/value objects to an Array values. double[] valueArray = dateValueList.Select(dv => dv.Value).ToArray(); // Let the method in TSBlobCoder class do all the work TSBlobCoder.ConvertArrayToBlobRegular(valueArray, compressionCode, traceObject); } return(traceObject.ValueBlob); }
private unsafe void WriteOneTrace(ITimeSeriesTrace traceObject) { // SQL statement that gives us a resultset for the DataTable object. Note that // this query is rigged so that it will always return 0 records. This is because // we only want the resultset to define the fields of the DataTable object. String comm = BuildStringForEmptyTraceDataTable(); // SqlDataAdapter object will use the query to fill the DataTable using (SqlDataAdapter adp = new SqlDataAdapter(comm, Connx)) { // SqlCommandBuilder object must be instantiated in order for us to call // the Update method of the SqlDataAdapter. Interestingly, we only need to // instantiate this object--we don't need to use it in any other way. using (SqlCommandBuilder bld = new SqlCommandBuilder(adp)) { DataTable dTable = new DataTable(); // Execute the query to fill the DataTable object try { adp.Fill(dTable); } catch (Exception e) { // The query failed throw new TSLibraryException(ErrCode.Enum.Could_Not_Open_Table, "Table '" + TableName + "' could not be opened using query:\n\n" + comm, e); } // DataRow object represents the current row of the DataTable object, which in turn // represents a record that we will add to the database table. DataRow currentRow = dTable.NewRow(); // transfer all of the data into the DataRow object currentRow["TimeSeries_Id"] = 1; currentRow["TraceNumber"] = traceObject.TraceNumber; currentRow["ValueBlob"] = traceObject.ValueBlob; currentRow["Checksum"] = traceObject.Checksum; dTable.Rows.Add(currentRow); // Save the DataRow object to the database adp.Update(dTable); dTable.Dispose(); } } }
/// <summary> /// This method converts the given array of time series values (date/value pairs stored in /// TSDateValueStruct) to a BLOB (byte array). It also sets the computes the checksum from the /// resultant BLOB, and then sets the Checksum and ValueBlob properties of the given /// ITimeSeriesTrace object accordingly. /// </summary> /// <param name="dateValueArray">The array of time series values to convert into a BLOB</param> /// <param name="compressionCode">a generation number that indicates what compression technique to use</param> /// <param name="traceObject">object whose TraceNumber property will be used to compute the checksum, /// and whose properties will be assigned by this method</param> /// <returns>The BLOB that is created from dateValueArray</returns> public static unsafe byte[] ConvertArrayToBlobIrregular( TSDateValueStruct[] dateValueArray, int compressionCode, ITimeSeriesTrace traceObject) { // The number of bytes required for the BLOB int nBin = traceObject.TimeStepCount * sizeof(TSDateValueStruct); // Allocate an array for the BLOB Byte[] blobData = new Byte[nBin]; // MemoryStream and BinaryWriter objects enable copying of data to the BLOB using (MemoryStream blobStream = new MemoryStream(blobData)) using (BinaryWriter blobWriter = new BinaryWriter(blobStream)) { // Loop through the entire array for (int i = 0; i < traceObject.TimeStepCount; i++) { // write the value to the BLOB as DATE followed by VALUE blobWriter.Write(dateValueArray[i].Date.ToBinary()); blobWriter.Write(dateValueArray[i].Value); } } // Compute the checksum using the uncompressed BLOB. During development, it was // demonstrated that the checksum would be computed faster on the compressed BLOB. // However, this could make it difficult to upgrade the compression algorithm in the // future, because the checksum value would be dependent on the compression algorithm. Byte[] checksum = ComputeTraceChecksum(traceObject.TraceNumber, blobData); Boolean checksumChanged = (MurmurHash.ByteArraysAreEqual(traceObject.Checksum, checksum) == false); // If the checksum did not change, then we will not assign any properties to the traceObject. // The result will be that we will return the original ValueBlob. If the checksum did change, // then we compute a new compressed ValueBlob and assign the new values. if (checksumChanged) { traceObject.Checksum = checksum; // the BLOB is stored in a compressed form, so our last step is to compress it traceObject.ValueBlob = CompressBlob(blobData, compressionCode); } return(traceObject.ValueBlob); }
// The series of tests below is for ConvertListToBlobWithChecksum() // // This method is reused by the actual test methods that follow public Boolean ComputeTestChecksums( TSDateCalculator.TimeStepUnitCode u1, short q1, List<TimeSeriesValue> list1, ITimeSeriesTrace trace1, TSDateCalculator.TimeStepUnitCode u2, short q2, List<TimeSeriesValue> list2, ITimeSeriesTrace trace2) { TSLibrary tsLib = new TSLibrary(); int compressionCode; tsLib.ConvertListToBlobWithChecksum( u1, q1, list1.Count, list1[0].Date, list1[list1.Count - 1].Date, list1, trace1, out compressionCode); tsLib.ConvertListToBlobWithChecksum( u2, q2, list2.Count, list2[0].Date, list2[list2.Count - 1].Date, list2, trace2, out compressionCode); Assert.IsTrue(trace1.Checksum.Length == 16); Assert.IsTrue(trace2.Checksum.Length == 16); for (int i = 0; i < trace2.Checksum.Length; i++) if (trace1.Checksum[i] != trace2.Checksum[i]) return false; return true; }
// The series of tests below is for ConvertListToBlobWithChecksum() // // This method is reused by the actual test methods that follow public Boolean ComputeTestChecksums( TSDateCalculator.TimeStepUnitCode u1, short q1, List <TimeSeriesValue> list1, ITimeSeriesTrace trace1, TSDateCalculator.TimeStepUnitCode u2, short q2, List <TimeSeriesValue> list2, ITimeSeriesTrace trace2) { TSLibrary tsLib = new TSLibrary(); int compressionCode; tsLib.ConvertListToBlobWithChecksum( u1, q1, list1.Count, list1[0].Date, list1[list1.Count - 1].Date, list1, trace1, out compressionCode); tsLib.ConvertListToBlobWithChecksum( u2, q2, list2.Count, list2[0].Date, list2[list2.Count - 1].Date, list2, trace2, out compressionCode); Assert.IsTrue(trace1.Checksum.Length == 16); Assert.IsTrue(trace2.Checksum.Length == 16); for (int i = 0; i < trace2.Checksum.Length; i++) { if (trace1.Checksum[i] != trace2.Checksum[i]) { return(false); } } return(true); }
/// <summary> /// This method converts a List of TimeSeriesValue objects into a BLOB (byte array) of /// time series values and computes a checksum from the BLOB. This method assigns the new values /// of the ValueBlob, Checksum, EndDate, and TimeStepCount to the object given in the traceObject /// parameter. The TraceNumber property of the traceObject parameter must be set before calling /// this method. /// /// The entire List is converted into the BLOB--i.e., the method does not take any /// parameters for limiting the size of the List that is created. This method will /// throw exceptions if the meta-parameters that are passed in are not consistent /// with the List of TimeSeriesValue objects. /// </summary> /// <param name="timeStepUnit">TSDateCalculator.TimeStepUnitCode value for /// Minute, Hour, Day, Week, Month, Year, or Irregular</param> /// <param name="timeStepQuantity">The number of the given unit that defines the time step. /// For instance, if the time step is 6 hours long, then this value is 6.</param> /// <param name="timeStepCount">The number of time steps stored in the BLOB</param> /// <param name="blobStartDate">Date of the first time step in the BLOB</param> /// <param name="blobEndDate">Date of the last time step in the BLOB</param> /// <param name="dateValueList">A List of TimeSeriesValue objects that will be converted to a BLOB</param> /// <param name="traceObject">an object which contains the a TraceNumber property that is used to /// compute the checksum. The computed BLOB and checksum are both saved to the appropriate properties /// of this object.</param> /// <param name="compressionCode">a generation number that indicates what compression technique to use</param> /// <returns>The BLOB (byte array) of time series values that was created from dateValueList</returns> public byte[] ConvertListToBlobWithChecksum( TSDateCalculator.TimeStepUnitCode timeStepUnit, short timeStepQuantity, int timeStepCount, DateTime blobStartDate, DateTime blobEndDate, List<TimeSeriesValue> dateValueList, ITimeSeriesTrace traceObject, out int compressionCode) { // Error checks if (dateValueList.Count != timeStepCount) throw new TSLibraryException(ErrCode.Enum.Checksum_Improper_Count); if (dateValueList[0].Date != blobStartDate) throw new TSLibraryException(ErrCode.Enum.Checksum_Improper_StartDate); if (dateValueList.Last().Date != blobEndDate) throw new TSLibraryException(ErrCode.Enum.Checksum_Improper_EndDate); // When compressing, we always use the latest compression method compressionCode = TSBlobCoder.currentCompressionCode; // Assign properties to the Trace object if (traceObject.EndDate != blobEndDate) traceObject.EndDate = blobEndDate; if (traceObject.TimeStepCount != timeStepCount) traceObject.TimeStepCount = timeStepCount; // Convert the List dateValueList into a BLOB. if (timeStepUnit == TSDateCalculator.TimeStepUnitCode.Irregular) { // IRREGULAR TIME SERIES // The method in TSBlobCoder can only process an array of TSDateValueStruct. Therefore // we convert the List of objects to an Array of struct instances. TSDateValueStruct[] dateValueArray = dateValueList.Select(tsv => (TSDateValueStruct)tsv).ToArray(); // Let the method in TSBlobCoder class do all the work TSBlobCoder.ConvertArrayToBlobIrregular(dateValueArray, compressionCode, traceObject); } else { // REGULAR TIME SERIES // The method in TSBlobCoder can only process an array of double values. Therefore // we convert the List of date/value objects to an Array values. double[] valueArray = dateValueList.Select(dv => dv.Value).ToArray(); // Let the method in TSBlobCoder class do all the work TSBlobCoder.ConvertArrayToBlobRegular(valueArray, compressionCode, traceObject); } return traceObject.ValueBlob; }