/// <summary> /// This method writes a new record to the database table for a regular time series. /// The method will record extra parameters (other than those that are saved /// as class-level properties of this object) into the database record using the strings /// in the method parameters extraParamNames and extraParamValues. This method does not /// make any changes to the trace table. /// </summary> /// <param name="doWriteToDB">true if the method should actually save the timeseries to the database</param> /// <param name="tsImport">TSImport object into which the method will record values that it has computed. /// If this parameter is null, then the method will skip the recording of such paramters to an object.</param> /// <param name="timeStepUnit">TSDateCalculator.TimeStepUnitCode value for Minute,Hour,Day,Week,Month, or Year</param> /// <param name="timeStepQuantity">The number of the given unit that defines the time step. /// For instance, if the time step is 6 hours long, then this value is 6.</param> /// <param name="timeStepCount">The number of time steps in the time series</param> /// <param name="outStartDate">date of the first time step in the series</param> /// <param name="extraParamNames">A list of field names that the the method should fill, in addition /// to the fields that the TimeSeriesLibrary is designed to maintain. Every item in this list must /// be matched to an item in extraParamValues.</param> /// <param name="extraParamValues">A list of field values that the the method should fill, in addition /// to the fields that the TimeSeriesLibrary is designed to maintain. Every item in this list must /// be matched to an item in extraParamNames.</param> /// <returns>the primary key Id value of the new record that was created</returns> public unsafe int WriteParametersRegular( bool doWriteToDB, TSImport tsImport, short timeStepUnit, short timeStepQuantity, int timeStepCount, DateTime outStartDate, String extraParamNames, String extraParamValues) { ErrorCheckWriteValues(doWriteToDB, tsImport); // The method's parameters are used to compute the meta-parameters of this time series TSParameters.SetParametersRegular( (TSDateCalculator.TimeStepUnitCode)timeStepUnit, timeStepQuantity, timeStepCount, outStartDate, // new time series are always compressed by the current compression technique TSBlobCoder.currentCompressionCode); IsInitialized = true; // Compute the Checksum for this time series ensemble. Because this is a newly // written series, there are not yet any traces to incorporate into the checksum // (presumably those will be added later). Checksum = TSBlobCoder.ComputeChecksum(TSParameters, new List <ITimeSeriesTrace>()); // WriteParameters method will handle all of the database interaction if (doWriteToDB) { WriteParameters(extraParamNames, extraParamValues); } return(Id); }
/// <summary> /// This method contains error checks on the input parameters of this class's methods /// WriteValuesRegular() and WriteValuesIrregular(), so both of those methods call /// this private method before they undertake any other operations. /// </summary> /// <param name="doWriteToDB">true if the 'Write' method should actually save the timeseries to the database</param> /// <param name="tsImport">TSImport object into which the method will record values that it has computed. /// If this parameter is null, then the method will skip the recording of such paramters to an object.</param> private void ErrorCheckWriteValues(bool doWriteToDB, TSImport tsImport) { // TODO: create an error code and throw exception if (doWriteToDB && (ParametersTableName == null || TSConnection.Connection == null || TraceTableName == null)) { } }
/// <summary> /// This method prepares a new record for the trace table for a regular time step series. /// The method converts the given valueArray into the BLOB that is actually stored in /// the table. The method computes the checksum of the trace, and computes a new checksum /// for the parameters table to reflect the fact that a new trace has been added to the ensemble. /// For both the insertion to the trace table and the update to the parameters table, this method /// only stores changes in DataTable objects--nothing is changed in the database. In order for /// the changes to be sent to the database, the method TSConnection.CommitNewTraceWrites must /// be called after WriteTraceRegular has been called for all new traces. /// </summary> /// <param name="id">identifying primary key value of the the parameters table for the record /// that this trace belongs to</param> /// <param name="doWriteToDB">true if the method should actually save the timeseries to the database</param> /// <param name="tsImport">TSImport object into which the method will record values that it has computed. /// If this parameter is null, then the method will skip the recording of such paramters to an object.</param> /// <param name="traceNumber">number of the trace to write</param> /// <param name="valueArray">The array of values to be written to the database</param> public unsafe void WriteTraceRegular(int id, bool doWriteToDB, TSImport tsImport, int traceNumber, double[] valueArray) { // Initialize class fields other than the BLOB of data values if (!IsInitialized) { Initialize(id, true); } // This method can only process regular-time-step series if (TimeStepUnit == TSDateCalculator.TimeStepUnitCode.Irregular) { throw new TSLibraryException(ErrCode.Enum.Record_Not_Regular, String.Format("The method can only process regular time series, but" + "the record with Id {0} is irregular.", id)); } // Create a trace object int timeStepCount = valueArray.Count(); ITimeSeriesTrace traceObject = new TSTrace { TraceNumber = traceNumber, TimeStepCount = timeStepCount, EndDate = TSDateCalculator.IncrementDate(BlobStartDate, TimeStepUnit, TimeStepQuantity, timeStepCount - 1) }; if (tsImport != null) { tsImport.TraceList.Add(traceObject); } else { TraceList.Add(traceObject); } // Convert the array of double values into a byte array...a BLOB TSBlobCoder.ConvertArrayToBlobRegular(valueArray, CompressionCode, traceObject); // Create a new record for the trace table // (but for now it is only stored in a DataTable object) if (doWriteToDB) { WriteTrace(traceObject); } // Compute a new checksum for the parameters table // (but for now it is only stored in a DataTable object) UpdateParametersChecksum(doWriteToDB, tsImport); }
/// <summary> /// This computes a new value for the Checksum field of the parameters table. It does not save /// this change to the database, but to a DataTable object. It is assumed that method /// TSConnection.CommitNewTraceWrites will be called later in order to send the changes to the /// database. If parameter 'toWriteToDB' is false, then this method can simply save the /// new Checksum value to the object given in the 'tsImport' parameter. /// </summary> /// <param name="toWriteToDB">true if the method should actually /// save the timeseries to the database</param> /// <param name="tsImport">TSImport object into which the method will record values /// that it has computed. If this parameter is null, then the method will skip the recording /// of such paramters to an object.</param> private void UpdateParametersChecksum(Boolean toWriteToDB, TSImport tsImport) { // The collection in variable 'traceObjects' contains one item for each trace for this // time series. The primary purpose of the list is to store the checksum for each trace, // since the checksum of the timeseries is computed from the list of checksums from each // of its traces. List <ITimeSeriesTrace> traceObjects; if (tsImport != null) { traceObjects = tsImport.TraceList; } else { traceObjects = TraceList; } // Compute the new checksum of the ensemble Checksum = TSBlobCoder.ComputeChecksum(TimeStepUnit, TimeStepQuantity, BlobStartDate, traceObjects); if (toWriteToDB) { DataTable dataTable; // Attempt to get the existing DataTable object from the collection that is kept by // the TSConnection object. If this fails, then we'll create a new DataTable. if (TSConnection.BulkCopyDataTables.TryGetValue(ParametersTableName, out dataTable) == false) { // Create the DataTable object and add columns that match the columns // of the database table. dataTable = new DataTable(); dataTable.Columns.Add("Id", typeof(int)); dataTable.Columns.Add("Checksum", typeof(byte[])); // Add the DataTable to a collection that is kept in the TSConnection object. TSConnection.BulkCopyDataTables.Add(ParametersTableName, dataTable); } dataTable.Rows.Add(Id, Checksum); } }
/// <summary> /// This method reads the given XML file and stores each of the timeseries described /// therein to the database. For each timeseries that it stores, it adds an item to /// list 'tsImportList', which records metadata of the timeseries that is not processed /// directly by TimeSeriesLibrary. Therefore, the process that calls TimeSeriesLibrary /// can process tsImportList to complete the importation of the timeseries. /// </summary> /// <param name="xmlFileName">Name of the file that will be read. If xmlText is null, /// then this parameter must be non-null, and vice-versa.</param> /// <param name="xmlText">The text of an XML file that will be read. If xmlFileName is null, /// then this parameter must be non-null, and vice-versa.</param> /// <param name="tsImportList">List of TSImport objects that this function records for /// each series that is imported. This method appends the list.</param> /// <param name="shouldStoreToDatabase">If true, then this method will write the timeseries from /// the XML file to database. If false, then this method does not write to database.</param> /// <param name="shouldRecordDetails">If true, then this method stores the BLOB and detailed /// elements to the list of TSImport objects. If false, then this method does not store /// the BLOB to the TSImport object, and all fields that TimeSeriesLibrary does not process /// are stored to the TSImport object's UnprocessedElements field.</param> /// <returns>The number of time series records that were successfully stored</returns> public int ReadAndStore( String xmlFileName, String xmlText, List <TSImport> tsImportList, Boolean shouldStoreToDatabase, Boolean shouldRecordDetails) { String s; // ephemeral String object int numTs = 0; // The # of time series successfuly processed by this method TSDateCalculator.TimeStepUnitCode TimeStepUnit = TSDateCalculator.TimeStepUnitCode.Day; // to be read from XML short TimeStepQuantity = 1; // to be read from XML DateTime StartDate = _defaultTime; // to be read from XML double[] valueArray = null; // to be read from XML var elementNames = new List <String> { "TimeSeries", "Pattern" }; // Flags will indicate if the XML is missing any data Boolean foundTimeStepUnit, foundTimeStepQuantity, foundStartDate, foundValueArray; // Error checks if (xmlFileName == null && xmlText == null) { throw new TSLibraryException(ErrCode.Enum.Xml_Memory_File_Exclusion, "The method's xmlFileName and xmlText parameters can not both be null."); } if (xmlFileName != null && xmlText != null) { throw new TSLibraryException(ErrCode.Enum.Xml_Memory_File_Exclusion, "The method's xmlFileName and xmlText parameters can not both be non-null."); } if (shouldStoreToDatabase && TSConnection == null) { throw new TSLibraryException(ErrCode.Enum.Xml_Connection_Not_Initialized, "The method is directed to store results to database, " + "but a database connection has not been assigned in the constructor."); } // Initialize a Stream object for the XmlReader object to read from. This method can // be called with either the file name of an XML file, or with a string containing the // complete text of the XML to be parsed. The type of Stream object that we initialize // depends on which parameter the method was called with. Stream xmlStream; if (xmlFileName == null) { xmlStream = new MemoryStream(Encoding.ASCII.GetBytes(xmlText)); ReportedFileName = "The given XML text "; } else { xmlStream = new FileStream(xmlFileName, FileMode.Open); ReportedFileName = "The XML file '" + xmlFileName + "' "; } try { // Loop once for "TimeSeries" and once for "Pattern". Note that the approach of looping // through the entire file twice seems undesirable, but no better option was evident because // methods of XmlReader such as 'ReadToNextSibling' do not have an equivalent that could // seek *either* "TimeSeries" or "Pattern". foreach (String elementName in elementNames) { Boolean isPattern = elementName == "Pattern"; // Start at the beginning of the XML file xmlStream.Seek(0, SeekOrigin.Begin); // This XmlReader object opens the XML file and parses it for us. The 'using' // statement ensures that the XmlReader's resources are properly disposed. using (XmlReader xmlReader = XmlReader.Create(xmlStream)) { // All of the data that we'll read is contained inside an element named 'Import' try { if (!xmlReader.ReadToFollowing("Import")) { throw new TSLibraryException(ErrCode.Enum.Xml_File_Empty, ReportedFileName + "does not contain an <Import> element."); } } catch { throw new TSLibraryException(ErrCode.Enum.Xml_File_Empty, ReportedFileName + "does not contain an <Import> element."); } // The file must contain at least one element named 'TimeSeries'. Move to the first // such element now. if (!xmlReader.ReadToDescendant(elementName)) { // if no such element is found then there is nothing to process in this iteration // of the loop. After the loop is finished, an exception is thrown if no elements // were read. continue; } // do-while loop through all elements named 'TimeSeries'. There will be one iteration // of this loop for each timeseries in the XML file. do { // Get a new XmlReader object that can not read outside // of the current 'TimeSeries' element. XmlReader oneSeriesXmlReader = xmlReader.ReadSubtree(); // A new TSImport object will store properties of this time series // that the TimeSeriesLibrary is not designed to handle. TSImport tsImport = new TSImport(shouldRecordDetails) { IsPattern = isPattern }; // Flags will indicate if the XML is missing any data foundTimeStepUnit = foundTimeStepQuantity = foundStartDate = foundValueArray = false; // This default trace number will be used if the "Trace" attribute is not used on a <Data> // element. The default trace value may be reassigned if a <TraceNumber> element is read. int defaultTraceNumber = 1; // Collection of Strings that hold the unparsed DataSeries. The key of the // Dictionary is the trace number of the DataSeries Dictionary <int, String> DataStrings = new Dictionary <int, String>(); // advance the reader past the outer element oneSeriesXmlReader.ReadStartElement(); // Read one timeseries from XML while (oneSeriesXmlReader.Read()) { Boolean binaryEncoded = false; // If the current position of the reader is on an element's start tag (e.g. <Name>) if (oneSeriesXmlReader.NodeType == XmlNodeType.Element) { // Note that XML standard is case sensitive switch (oneSeriesXmlReader.Name) { case "Name": // <Name> is not processed by TimeSeriesLibrary. Record it on a list // so another module can process the <Name> field. Presumably, // the process will distinguish timeseries based on the <Name> field. tsImport.Name = oneSeriesXmlReader.ReadElementContentAsString(); break; case "StartDate": // TimeSeriesLibrary will store <StartDate> to the data table s = oneSeriesXmlReader.ReadElementContentAsString(); if (!TimeExtensions.TryParse(s, out StartDate, _defaultTime)) { throw new TSLibraryException(ErrCode.Enum.Xml_File_StartDate_Unreadable, ReportedFileName + "contains unreadable StartDate element value " + s); } foundStartDate = true; break; case "TimeStepUnit": // TimeSeriesLibrary will store <TimeStepUnit> to the data table s = oneSeriesXmlReader.ReadElementContentAsString(); TimeStepUnit = ParseTimeStepUnit(s); foundTimeStepUnit = true; // If it is an irregular time series if (TimeStepUnit == TSDateCalculator.TimeStepUnitCode.Irregular) { // <TimeStepQuantity> and <StartDate> are unnecessary // and irrelevant to irregular time series foundTimeStepQuantity = true; foundStartDate = true; } break; case "TimeStepQuantity": // TimeSeriesLibrary will store <TimeStepQuantity> to the data table s = oneSeriesXmlReader.ReadElementContentAsString(); TimeStepQuantity = ParseTimeStepQuantity(s); foundTimeStepQuantity = true; break; case "Data": // <Data> may have a TraceNumber attribute int traceNumber = 0; s = oneSeriesXmlReader.GetAttribute("Trace"); if (int.TryParse(s, out traceNumber) == false) { traceNumber = 0; } if (DataStrings.ContainsKey(traceNumber)) { throw new TSLibraryException(ErrCode.Enum.Xml_File_Inconsistent, ReportedFileName + "contains a time series with more than " + "one trace number " + traceNumber.ToString()); } // <Data> contains a whitespace-deliminted string of values // that comprise the time series s = oneSeriesXmlReader.ReadElementContentAsString(); // add the unparsed string to a dictionary // where the trace number is the dictionary key DataStrings.Add(traceNumber, s); foundValueArray = true; break; case "Encoding": // The default is that <Data> contains decimal text. // However, the <Encoding> element may specify that it is // Base64 encoded. s = oneSeriesXmlReader.ReadElementContentAsString(); if (s == "Base64" || s == "base64") { binaryEncoded = true; } break; case "Apart": // <Apart> contains the A part of record name from a HECDSS file tsImport.SetAPart(oneSeriesXmlReader); break; case "Bpart": // <Bpart> contains the B part of record name from a HECDSS file tsImport.SetBPart(oneSeriesXmlReader); break; case "Cpart": // <Cpart> contains the C part of record name from a HECDSS file tsImport.SetCPart(oneSeriesXmlReader); break; case "Epart": // <Epart> contains the E part of record name from a HECDSS file tsImport.SetEPart(oneSeriesXmlReader); break; case "Units": // <Units> contains the name of the units of measurement for the time series values tsImport.SetUnits(oneSeriesXmlReader); break; case "TimeSeriesType": // <TimeSeriesType> contains the text name of the time series type, // [PER-AVER | PER-CUM | INST-VAL | INST-CUM] tsImport.SetTimeSeriesType(oneSeriesXmlReader); break; case "TraceNumber": // <TraceNumber> contains the trace number for an ensemble defaultTraceNumber = tsImport.GetTraceNumber(oneSeriesXmlReader); break; case "MultiplicationFactor": tsImport.SetMultiplicationFactor(oneSeriesXmlReader); break; default: // Any other tags are simply copied to the String object // 'UnprocessedElements'. Here they are stored with // the enclosing tags (e.g. "<Units>CFS</Units>"). tsImport.AddUnprocessedElement(oneSeriesXmlReader.ReadOuterXml()); break; } } } // This XmlReader object was created with the ReadSubtree() method so that it would only // be able to read the current time series element. We have now reached the end of the // time series element, so the XmlReader should be closed. oneSeriesXmlReader.Close(); // The record can not be saved to the table if information for some of the fields is missing. // These flags indicate whether each of the required fields was found in the XML file. if (!(foundTimeStepUnit && foundTimeStepQuantity && foundStartDate && foundValueArray)) { // One or more required fields were missing, so we'll throw an exception. String errorList, nameString; if (tsImport.Name == "") { nameString = "unnamed time series"; } else { nameString = "time series named '" + tsImport.Name + "'"; } errorList = "Some required subelements were missing from " + nameString + " in " + ReportedFileName + "\n"; if (!foundStartDate) { errorList += "\n<StartDate> was not found"; } if (!foundTimeStepUnit) { errorList += "\n<TimeStepUnit> was not found"; } if (!foundTimeStepQuantity) { errorList += "\n<TimeStepQuantity> was not found"; } if (!foundValueArray) { errorList += "\n<Data> was not found"; } throw new TSLibraryException(ErrCode.Enum.Xml_File_Incomplete, errorList); } // Now that we've established that all fields have been read, we can parse the // string of timeseries values into an array, and save the array to the database. if (TimeStepUnit == TSDateCalculator.TimeStepUnitCode.Irregular) { // IRREGULAR TIME SERIES // The TS object is used to save one record to the database table TS ts = new TS(TSConnection, TableName, TraceTableName); foreach (KeyValuePair <int, String> keyValuePair in DataStrings) { int traceNumber = keyValuePair.Key; if (traceNumber == 0) { traceNumber = defaultTraceNumber; } // Split the big data string into an array of strings. // The date/time/value triplets will be all collated together. String[] stringArray = keyValuePair.Value .Split(new char[] { }, StringSplitOptions.RemoveEmptyEntries); // We'll use this date/value structure to build each item of the date/value array TSDateValueStruct tsv = new TSDateValueStruct(); // allocate the array of date/value pairs TSDateValueStruct[] dateValueArray = new TSDateValueStruct[stringArray.Length / 3]; // Loop through the array of strings, 3 elements at a time for (int i = 2; i < stringArray.Length; i += 3) { s = stringArray[i - 2] + " " + stringArray[i - 1]; tsv.Date = DateTime.Parse(s); tsv.Value = double.Parse(stringArray[i]); dateValueArray[i / 3] = tsv; } if (tsImport.TraceList.Count == 0) { // Ignore whatever was entered in the StartDate element, since it // might conflict with the date/value entries StartDate = dateValueArray[0].Date; // Write parameters to the database and record values in the TSImport object ts.WriteParametersIrregular(shouldStoreToDatabase, tsImport, dateValueArray.Count(), StartDate, dateValueArray.Last().Date, null, null); } else { if (StartDate != ts.BlobStartDate) { throw new TSLibraryException(ErrCode.Enum.Xml_File_Inconsistent, ReportedFileName + "contains a time series " + "with traces that do not overlay each other."); } } ts.WriteTraceIrregular(0, shouldStoreToDatabase, tsImport, traceNumber, dateValueArray); } tsImport.RecordFromTS(ts); // Done with the TS object. ts = null; } else { // REGULAR TIME SERIES // The TS object is used to save one record to the database table TS ts = new TS(TSConnection, TableName, TraceTableName); foreach (KeyValuePair <int, String> keyValuePair in DataStrings) { int traceNumber = keyValuePair.Key; if (traceNumber == 0) { traceNumber = defaultTraceNumber; } // Fancy LINQ statement turns the String object into an array of double[] valueArray = keyValuePair.Value .Split(new char[] { }, StringSplitOptions.RemoveEmptyEntries) .Select(z => double.Parse(z)).ToArray(); if (tsImport.TraceList.Count == 0) { // Write to the database and record values in the TSImport object ts.WriteParametersRegular(shouldStoreToDatabase, tsImport, (short)TimeStepUnit, TimeStepQuantity, valueArray.Count(), StartDate, null, null); } ts.WriteTraceRegular(0, shouldStoreToDatabase, tsImport, traceNumber, valueArray); } tsImport.RecordFromTS(ts); // Done with the TS object. ts = null; } // the TSImport object contains data for this timeseries that TSLibrary does not process. // Add the TSImport object to a list that the calling process can read and use. tsImportList.Add(tsImport); numTs++; } while (xmlReader.ReadToNextSibling(elementName)); } } if (numTs == 0) { throw new TSLibraryException(ErrCode.Enum.Xml_File_Empty, ReportedFileName + " does not contain any " + elementNames.Select(ss => "<" + ss + ">").ToStringWithConjunc("or") + " element".Pluralize(elementNames.Count) + "."); } } catch (XmlException e) { // An XmlException was caught somewhere in the lifetime of the object xmlReader, // so we can presumably say there was an error in how the XML file was formatted. // The information from the XmlException object is included in the error message // that we throw here, and the XmlException is included as an inner exception. throw new TSLibraryException(ErrCode.Enum.Xml_File_Malformed, ReportedFileName + "is malformed.\n\n" + e.Message, e); } finally { // Disposing the Stream object ensures that the file is closed, if applicable. // This is put into the 'finally' clause so that we can ensure that the Stream // is closed no matter how we exit this method. xmlStream.Dispose(); } return(numTs); }