/// <summary>
/// This method writes a new record to the database table for a regular time series.
/// The method will record extra parameters (other than those that are saved
/// as class-level properties of this object) into the database record using the strings
/// in the method parameters extraParamNames and extraParamValues. This method does not
/// make any changes to the trace table.
/// </summary>
/// <param name="doWriteToDB">true if the method should actually save the timeseries to the database</param>
/// <param name="tsImport">TSImport object into which the method will record values that it has computed.
/// If this parameter is null, then the method will skip the recording of such paramters to an object.</param>
/// <param name="timeStepUnit">TSDateCalculator.TimeStepUnitCode value for Minute,Hour,Day,Week,Month, or Year</param>
/// <param name="timeStepQuantity">The number of the given unit that defines the time step.
/// For instance, if the time step is 6 hours long, then this value is 6.</param>
/// <param name="timeStepCount">The number of time steps in the time series</param>
/// <param name="outStartDate">date of the first time step in the series</param>
/// <param name="extraParamNames">A list of field names that the the method should fill, in addition
/// to the fields that the TimeSeriesLibrary is designed to maintain. Every item in this list must
/// be matched to an item in extraParamValues.</param>
/// <param name="extraParamValues">A list of field values that the the method should fill, in addition
/// to the fields that the TimeSeriesLibrary is designed to maintain. Every item in this list must
/// be matched to an item in extraParamNames.</param>
/// <returns>the primary key Id value of the new record that was created</returns>
public unsafe int WriteParametersRegular(
bool doWriteToDB, TSImport tsImport,
short timeStepUnit, short timeStepQuantity,
int timeStepCount, DateTime outStartDate,
String extraParamNames, String extraParamValues)
{
ErrorCheckWriteValues(doWriteToDB, tsImport);
// The method's parameters are used to compute the meta-parameters of this time series
TSParameters.SetParametersRegular(
(TSDateCalculator.TimeStepUnitCode)timeStepUnit, timeStepQuantity,
timeStepCount, outStartDate,
// new time series are always compressed by the current compression technique
TSBlobCoder.currentCompressionCode);
IsInitialized = true;
// Compute the Checksum for this time series ensemble. Because this is a newly
// written series, there are not yet any traces to incorporate into the checksum
// (presumably those will be added later).
Checksum = TSBlobCoder.ComputeChecksum(TSParameters, new List <ITimeSeriesTrace>());
// WriteParameters method will handle all of the database interaction
if (doWriteToDB)
{
WriteParameters(extraParamNames, extraParamValues);
}
return(Id);
}
/// <summary>
/// This method contains error checks on the input parameters of this class's methods
/// WriteValuesRegular() and WriteValuesIrregular(), so both of those methods call
/// this private method before they undertake any other operations.
/// </summary>
/// <param name="doWriteToDB">true if the 'Write' method should actually save the timeseries to the database</param>
/// <param name="tsImport">TSImport object into which the method will record values that it has computed.
/// If this parameter is null, then the method will skip the recording of such paramters to an object.</param>
private void ErrorCheckWriteValues(bool doWriteToDB, TSImport tsImport)
{
// TODO: create an error code and throw exception
if (doWriteToDB && (ParametersTableName == null ||
TSConnection.Connection == null ||
TraceTableName == null))
{
}
}
/// <summary>
/// This method prepares a new record for the trace table for a regular time step series.
/// The method converts the given valueArray into the BLOB that is actually stored in
/// the table. The method computes the checksum of the trace, and computes a new checksum
/// for the parameters table to reflect the fact that a new trace has been added to the ensemble.
/// For both the insertion to the trace table and the update to the parameters table, this method
/// only stores changes in DataTable objects--nothing is changed in the database. In order for
/// the changes to be sent to the database, the method TSConnection.CommitNewTraceWrites must
/// be called after WriteTraceRegular has been called for all new traces.
/// </summary>
/// <param name="id">identifying primary key value of the the parameters table for the record
/// that this trace belongs to</param>
/// <param name="doWriteToDB">true if the method should actually save the timeseries to the database</param>
/// <param name="tsImport">TSImport object into which the method will record values that it has computed.
/// If this parameter is null, then the method will skip the recording of such paramters to an object.</param>
/// <param name="traceNumber">number of the trace to write</param>
/// <param name="valueArray">The array of values to be written to the database</param>
public unsafe void WriteTraceRegular(int id,
bool doWriteToDB, TSImport tsImport,
int traceNumber,
double[] valueArray)
{
// Initialize class fields other than the BLOB of data values
if (!IsInitialized)
{
Initialize(id, true);
}
// This method can only process regular-time-step series
if (TimeStepUnit == TSDateCalculator.TimeStepUnitCode.Irregular)
{
throw new TSLibraryException(ErrCode.Enum.Record_Not_Regular,
String.Format("The method can only process regular time series, but" +
"the record with Id {0} is irregular.", id));
}
// Create a trace object
int timeStepCount = valueArray.Count();
ITimeSeriesTrace traceObject = new TSTrace
{
TraceNumber = traceNumber,
TimeStepCount = timeStepCount,
EndDate = TSDateCalculator.IncrementDate(BlobStartDate,
TimeStepUnit, TimeStepQuantity, timeStepCount - 1)
};
if (tsImport != null)
{
tsImport.TraceList.Add(traceObject);
}
else
{
TraceList.Add(traceObject);
}
// Convert the array of double values into a byte array...a BLOB
TSBlobCoder.ConvertArrayToBlobRegular(valueArray, CompressionCode, traceObject);
// Create a new record for the trace table
// (but for now it is only stored in a DataTable object)
if (doWriteToDB)
{
WriteTrace(traceObject);
}
// Compute a new checksum for the parameters table
// (but for now it is only stored in a DataTable object)
UpdateParametersChecksum(doWriteToDB, tsImport);
}
/// <summary>
/// This computes a new value for the Checksum field of the parameters table. It does not save
/// this change to the database, but to a DataTable object. It is assumed that method
/// TSConnection.CommitNewTraceWrites will be called later in order to send the changes to the
/// database. If parameter 'toWriteToDB' is false, then this method can simply save the
/// new Checksum value to the object given in the 'tsImport' parameter.
/// </summary>
/// <param name="toWriteToDB">true if the method should actually
/// save the timeseries to the database</param>
/// <param name="tsImport">TSImport object into which the method will record values
/// that it has computed. If this parameter is null, then the method will skip the recording
/// of such paramters to an object.</param>
private void UpdateParametersChecksum(Boolean toWriteToDB, TSImport tsImport)
{
// The collection in variable 'traceObjects' contains one item for each trace for this
// time series. The primary purpose of the list is to store the checksum for each trace,
// since the checksum of the timeseries is computed from the list of checksums from each
// of its traces.
List <ITimeSeriesTrace> traceObjects;
if (tsImport != null)
{
traceObjects = tsImport.TraceList;
}
else
{
traceObjects = TraceList;
}
// Compute the new checksum of the ensemble
Checksum = TSBlobCoder.ComputeChecksum(TimeStepUnit, TimeStepQuantity,
BlobStartDate, traceObjects);
if (toWriteToDB)
{
DataTable dataTable;
// Attempt to get the existing DataTable object from the collection that is kept by
// the TSConnection object. If this fails, then we'll create a new DataTable.
if (TSConnection.BulkCopyDataTables.TryGetValue(ParametersTableName, out dataTable) == false)
{
// Create the DataTable object and add columns that match the columns
// of the database table.
dataTable = new DataTable();
dataTable.Columns.Add("Id", typeof(int));
dataTable.Columns.Add("Checksum", typeof(byte[]));
// Add the DataTable to a collection that is kept in the TSConnection object.
TSConnection.BulkCopyDataTables.Add(ParametersTableName, dataTable);
}
dataTable.Rows.Add(Id, Checksum);
}
}
/// <summary>
/// This method reads the given XML file and stores each of the timeseries described
/// therein to the database. For each timeseries that it stores, it adds an item to
/// list 'tsImportList', which records metadata of the timeseries that is not processed
/// directly by TimeSeriesLibrary. Therefore, the process that calls TimeSeriesLibrary
/// can process tsImportList to complete the importation of the timeseries.
/// </summary>
/// <param name="xmlFileName">Name of the file that will be read. If xmlText is null,
/// then this parameter must be non-null, and vice-versa.</param>
/// <param name="xmlText">The text of an XML file that will be read. If xmlFileName is null,
/// then this parameter must be non-null, and vice-versa.</param>
/// <param name="tsImportList">List of TSImport objects that this function records for
/// each series that is imported. This method appends the list.</param>
/// <param name="shouldStoreToDatabase">If true, then this method will write the timeseries from
/// the XML file to database. If false, then this method does not write to database.</param>
/// <param name="shouldRecordDetails">If true, then this method stores the BLOB and detailed
/// elements to the list of TSImport objects. If false, then this method does not store
/// the BLOB to the TSImport object, and all fields that TimeSeriesLibrary does not process
/// are stored to the TSImport object's UnprocessedElements field.</param>
/// <returns>The number of time series records that were successfully stored</returns>
public int ReadAndStore(
String xmlFileName, String xmlText,
List <TSImport> tsImportList,
Boolean shouldStoreToDatabase, Boolean shouldRecordDetails)
{
String s; // ephemeral String object
int numTs = 0; // The # of time series successfuly processed by this method
TSDateCalculator.TimeStepUnitCode TimeStepUnit = TSDateCalculator.TimeStepUnitCode.Day; // to be read from XML
short TimeStepQuantity = 1; // to be read from XML
DateTime StartDate = _defaultTime; // to be read from XML
double[] valueArray = null; // to be read from XML
var elementNames = new List <String> {
"TimeSeries", "Pattern"
};
// Flags will indicate if the XML is missing any data
Boolean foundTimeStepUnit, foundTimeStepQuantity, foundStartDate, foundValueArray;
// Error checks
if (xmlFileName == null && xmlText == null)
{
throw new TSLibraryException(ErrCode.Enum.Xml_Memory_File_Exclusion,
"The method's xmlFileName and xmlText parameters can not both be null.");
}
if (xmlFileName != null && xmlText != null)
{
throw new TSLibraryException(ErrCode.Enum.Xml_Memory_File_Exclusion,
"The method's xmlFileName and xmlText parameters can not both be non-null.");
}
if (shouldStoreToDatabase && TSConnection == null)
{
throw new TSLibraryException(ErrCode.Enum.Xml_Connection_Not_Initialized,
"The method is directed to store results to database, " +
"but a database connection has not been assigned in the constructor.");
}
// Initialize a Stream object for the XmlReader object to read from. This method can
// be called with either the file name of an XML file, or with a string containing the
// complete text of the XML to be parsed. The type of Stream object that we initialize
// depends on which parameter the method was called with.
Stream xmlStream;
if (xmlFileName == null)
{
xmlStream = new MemoryStream(Encoding.ASCII.GetBytes(xmlText));
ReportedFileName = "The given XML text ";
}
else
{
xmlStream = new FileStream(xmlFileName, FileMode.Open);
ReportedFileName = "The XML file '" + xmlFileName + "' ";
}
try
{
// Loop once for "TimeSeries" and once for "Pattern". Note that the approach of looping
// through the entire file twice seems undesirable, but no better option was evident because
// methods of XmlReader such as 'ReadToNextSibling' do not have an equivalent that could
// seek *either* "TimeSeries" or "Pattern".
foreach (String elementName in elementNames)
{
Boolean isPattern = elementName == "Pattern";
// Start at the beginning of the XML file
xmlStream.Seek(0, SeekOrigin.Begin);
// This XmlReader object opens the XML file and parses it for us. The 'using'
// statement ensures that the XmlReader's resources are properly disposed.
using (XmlReader xmlReader = XmlReader.Create(xmlStream))
{
// All of the data that we'll read is contained inside an element named 'Import'
try
{
if (!xmlReader.ReadToFollowing("Import"))
{
throw new TSLibraryException(ErrCode.Enum.Xml_File_Empty,
ReportedFileName + "does not contain an <Import> element.");
}
}
catch
{
throw new TSLibraryException(ErrCode.Enum.Xml_File_Empty,
ReportedFileName + "does not contain an <Import> element.");
}
// The file must contain at least one element named 'TimeSeries'. Move to the first
// such element now.
if (!xmlReader.ReadToDescendant(elementName))
{
// if no such element is found then there is nothing to process in this iteration
// of the loop. After the loop is finished, an exception is thrown if no elements
// were read.
continue;
}
// do-while loop through all elements named 'TimeSeries'. There will be one iteration
// of this loop for each timeseries in the XML file.
do
{
// Get a new XmlReader object that can not read outside
// of the current 'TimeSeries' element.
XmlReader oneSeriesXmlReader = xmlReader.ReadSubtree();
// A new TSImport object will store properties of this time series
// that the TimeSeriesLibrary is not designed to handle.
TSImport tsImport = new TSImport(shouldRecordDetails)
{
IsPattern = isPattern
};
// Flags will indicate if the XML is missing any data
foundTimeStepUnit = foundTimeStepQuantity = foundStartDate = foundValueArray = false;
// This default trace number will be used if the "Trace" attribute is not used on a <Data>
// element. The default trace value may be reassigned if a <TraceNumber> element is read.
int defaultTraceNumber = 1;
// Collection of Strings that hold the unparsed DataSeries. The key of the
// Dictionary is the trace number of the DataSeries
Dictionary <int, String> DataStrings = new Dictionary <int, String>();
// advance the reader past the outer element
oneSeriesXmlReader.ReadStartElement();
// Read one timeseries from XML
while (oneSeriesXmlReader.Read())
{
Boolean binaryEncoded = false;
// If the current position of the reader is on an element's start tag (e.g. <Name>)
if (oneSeriesXmlReader.NodeType == XmlNodeType.Element)
{
// Note that XML standard is case sensitive
switch (oneSeriesXmlReader.Name)
{
case "Name":
// <Name> is not processed by TimeSeriesLibrary. Record it on a list
// so another module can process the <Name> field. Presumably,
// the process will distinguish timeseries based on the <Name> field.
tsImport.Name = oneSeriesXmlReader.ReadElementContentAsString();
break;
case "StartDate":
// TimeSeriesLibrary will store <StartDate> to the data table
s = oneSeriesXmlReader.ReadElementContentAsString();
if (!TimeExtensions.TryParse(s, out StartDate, _defaultTime))
{
throw new TSLibraryException(ErrCode.Enum.Xml_File_StartDate_Unreadable,
ReportedFileName
+ "contains unreadable StartDate element value " + s);
}
foundStartDate = true;
break;
case "TimeStepUnit":
// TimeSeriesLibrary will store <TimeStepUnit> to the data table
s = oneSeriesXmlReader.ReadElementContentAsString();
TimeStepUnit = ParseTimeStepUnit(s);
foundTimeStepUnit = true;
// If it is an irregular time series
if (TimeStepUnit == TSDateCalculator.TimeStepUnitCode.Irregular)
{
// <TimeStepQuantity> and <StartDate> are unnecessary
// and irrelevant to irregular time series
foundTimeStepQuantity = true;
foundStartDate = true;
}
break;
case "TimeStepQuantity":
// TimeSeriesLibrary will store <TimeStepQuantity> to the data table
s = oneSeriesXmlReader.ReadElementContentAsString();
TimeStepQuantity = ParseTimeStepQuantity(s);
foundTimeStepQuantity = true;
break;
case "Data":
// <Data> may have a TraceNumber attribute
int traceNumber = 0;
s = oneSeriesXmlReader.GetAttribute("Trace");
if (int.TryParse(s, out traceNumber) == false)
{
traceNumber = 0;
}
if (DataStrings.ContainsKey(traceNumber))
{
throw new TSLibraryException(ErrCode.Enum.Xml_File_Inconsistent,
ReportedFileName + "contains a time series with more than "
+ "one trace number " + traceNumber.ToString());
}
// <Data> contains a whitespace-deliminted string of values
// that comprise the time series
s = oneSeriesXmlReader.ReadElementContentAsString();
// add the unparsed string to a dictionary
// where the trace number is the dictionary key
DataStrings.Add(traceNumber, s);
foundValueArray = true;
break;
case "Encoding":
// The default is that <Data> contains decimal text.
// However, the <Encoding> element may specify that it is
// Base64 encoded.
s = oneSeriesXmlReader.ReadElementContentAsString();
if (s == "Base64" || s == "base64")
{
binaryEncoded = true;
}
break;
case "Apart": // <Apart> contains the A part of record name from a HECDSS file
tsImport.SetAPart(oneSeriesXmlReader); break;
case "Bpart": // <Bpart> contains the B part of record name from a HECDSS file
tsImport.SetBPart(oneSeriesXmlReader); break;
case "Cpart": // <Cpart> contains the C part of record name from a HECDSS file
tsImport.SetCPart(oneSeriesXmlReader); break;
case "Epart": // <Epart> contains the E part of record name from a HECDSS file
tsImport.SetEPart(oneSeriesXmlReader); break;
case "Units": // <Units> contains the name of the units of measurement for the time series values
tsImport.SetUnits(oneSeriesXmlReader); break;
case "TimeSeriesType":
// <TimeSeriesType> contains the text name of the time series type,
// [PER-AVER | PER-CUM | INST-VAL | INST-CUM]
tsImport.SetTimeSeriesType(oneSeriesXmlReader); break;
case "TraceNumber": // <TraceNumber> contains the trace number for an ensemble
defaultTraceNumber = tsImport.GetTraceNumber(oneSeriesXmlReader); break;
case "MultiplicationFactor":
tsImport.SetMultiplicationFactor(oneSeriesXmlReader); break;
default:
// Any other tags are simply copied to the String object
// 'UnprocessedElements'. Here they are stored with
// the enclosing tags (e.g. "<Units>CFS</Units>").
tsImport.AddUnprocessedElement(oneSeriesXmlReader.ReadOuterXml());
break;
}
}
}
// This XmlReader object was created with the ReadSubtree() method so that it would only
// be able to read the current time series element. We have now reached the end of the
// time series element, so the XmlReader should be closed.
oneSeriesXmlReader.Close();
// The record can not be saved to the table if information for some of the fields is missing.
// These flags indicate whether each of the required fields was found in the XML file.
if (!(foundTimeStepUnit && foundTimeStepQuantity && foundStartDate && foundValueArray))
{
// One or more required fields were missing, so we'll throw an exception.
String errorList, nameString;
if (tsImport.Name == "")
{
nameString = "unnamed time series";
}
else
{
nameString = "time series named '" + tsImport.Name + "'";
}
errorList = "Some required subelements were missing from " + nameString
+ " in " + ReportedFileName + "\n";
if (!foundStartDate)
{
errorList += "\n<StartDate> was not found";
}
if (!foundTimeStepUnit)
{
errorList += "\n<TimeStepUnit> was not found";
}
if (!foundTimeStepQuantity)
{
errorList += "\n<TimeStepQuantity> was not found";
}
if (!foundValueArray)
{
errorList += "\n<Data> was not found";
}
throw new TSLibraryException(ErrCode.Enum.Xml_File_Incomplete, errorList);
}
// Now that we've established that all fields have been read, we can parse the
// string of timeseries values into an array, and save the array to the database.
if (TimeStepUnit == TSDateCalculator.TimeStepUnitCode.Irregular)
{
// IRREGULAR TIME SERIES
// The TS object is used to save one record to the database table
TS ts = new TS(TSConnection, TableName, TraceTableName);
foreach (KeyValuePair <int, String> keyValuePair in DataStrings)
{
int traceNumber = keyValuePair.Key;
if (traceNumber == 0)
{
traceNumber = defaultTraceNumber;
}
// Split the big data string into an array of strings.
// The date/time/value triplets will be all collated together.
String[] stringArray = keyValuePair.Value
.Split(new char[] { }, StringSplitOptions.RemoveEmptyEntries);
// We'll use this date/value structure to build each item of the date/value array
TSDateValueStruct tsv = new TSDateValueStruct();
// allocate the array of date/value pairs
TSDateValueStruct[] dateValueArray = new TSDateValueStruct[stringArray.Length / 3];
// Loop through the array of strings, 3 elements at a time
for (int i = 2; i < stringArray.Length; i += 3)
{
s = stringArray[i - 2] + " " + stringArray[i - 1];
tsv.Date = DateTime.Parse(s);
tsv.Value = double.Parse(stringArray[i]);
dateValueArray[i / 3] = tsv;
}
if (tsImport.TraceList.Count == 0)
{
// Ignore whatever was entered in the StartDate element, since it
// might conflict with the date/value entries
StartDate = dateValueArray[0].Date;
// Write parameters to the database and record values in the TSImport object
ts.WriteParametersIrregular(shouldStoreToDatabase, tsImport,
dateValueArray.Count(), StartDate, dateValueArray.Last().Date,
null, null);
}
else
{
if (StartDate != ts.BlobStartDate)
{
throw new TSLibraryException(ErrCode.Enum.Xml_File_Inconsistent,
ReportedFileName + "contains a time series "
+ "with traces that do not overlay each other.");
}
}
ts.WriteTraceIrregular(0, shouldStoreToDatabase, tsImport,
traceNumber, dateValueArray);
}
tsImport.RecordFromTS(ts);
// Done with the TS object.
ts = null;
}
else
{
// REGULAR TIME SERIES
// The TS object is used to save one record to the database table
TS ts = new TS(TSConnection, TableName, TraceTableName);
foreach (KeyValuePair <int, String> keyValuePair in DataStrings)
{
int traceNumber = keyValuePair.Key;
if (traceNumber == 0)
{
traceNumber = defaultTraceNumber;
}
// Fancy LINQ statement turns the String object into an array of double[]
valueArray = keyValuePair.Value
.Split(new char[] { }, StringSplitOptions.RemoveEmptyEntries)
.Select(z => double.Parse(z)).ToArray();
if (tsImport.TraceList.Count == 0)
{
// Write to the database and record values in the TSImport object
ts.WriteParametersRegular(shouldStoreToDatabase, tsImport,
(short)TimeStepUnit, TimeStepQuantity, valueArray.Count(),
StartDate, null, null);
}
ts.WriteTraceRegular(0, shouldStoreToDatabase, tsImport,
traceNumber, valueArray);
}
tsImport.RecordFromTS(ts);
// Done with the TS object.
ts = null;
}
// the TSImport object contains data for this timeseries that TSLibrary does not process.
// Add the TSImport object to a list that the calling process can read and use.
tsImportList.Add(tsImport);
numTs++;
} while (xmlReader.ReadToNextSibling(elementName));
}
}
if (numTs == 0)
{
throw new TSLibraryException(ErrCode.Enum.Xml_File_Empty,
ReportedFileName + " does not contain any "
+ elementNames.Select(ss => "<" + ss + ">").ToStringWithConjunc("or")
+ " element".Pluralize(elementNames.Count) + ".");
}
}
catch (XmlException e)
{
// An XmlException was caught somewhere in the lifetime of the object xmlReader,
// so we can presumably say there was an error in how the XML file was formatted.
// The information from the XmlException object is included in the error message
// that we throw here, and the XmlException is included as an inner exception.
throw new TSLibraryException(ErrCode.Enum.Xml_File_Malformed,
ReportedFileName + "is malformed.\n\n" + e.Message, e);
}
finally
{
// Disposing the Stream object ensures that the file is closed, if applicable.
// This is put into the 'finally' clause so that we can ensure that the Stream
// is closed no matter how we exit this method.
xmlStream.Dispose();
}
return(numTs);
}
