private static void ReadAllFormats(string watershedName)
{
//TODO - compare validateWatershed data with computed
File.AppendAllText(logFile, NL);
File.AppendAllText(logFile, "---------- Reading Ensembles ----------" + NL);
string fn;
// TimeSeriesOfEnsembleLocations wshedData=null;
DateTime startTime = DateTime.MinValue;
DateTime endTime = DateTime.MaxValue;
// DSS
fn = "ensemble_V7_" + tag + ".dss";
ReadTimed(fn, () =>
{
return(DssEnsemble.Read(watershedName, startTime, endTime, fn));
});
fn = "ensemble_V7_profiles_" + tag + ".dss";
ReadTimed(fn, () =>
{
return(DssEnsemble.ReadTimeSeriesProfiles(watershedName, startTime, endTime, fn));
});
// SQLITE
fn = "ensemble_sqlite_blob_" + tag + ".db";
ReadTimed(fn, () =>
{
return(SqLiteEnsemble.Read(watershedName, startTime, endTime, fn));
});
// Pisces
fn = "ensemble_pisces_" + tag + ".pdb";
ReadTimed(fn, () =>
{
return(SqLiteEnsemble.Read(watershedName, startTime, endTime, fn));
});
// Serial HDF5
fn = "ensemble_serial_1RowPerChunk.h5";
ReadTimed(fn, () =>
{
using (var hr = new H5Reader(fn))
return(HDF5Ensemble.Read(hr, watershedName));
});
// Parallel HDF5
foreach (int c in new[] { 1, 10, -1 })
{
fn = "ensemble_parallel_" + c.ToString() + "RowsPerChunk.h5";
ReadTimed(fn, () =>
{
using (var hr = new H5Reader(fn))
return(HDF5Ensemble.Read(hr, watershedName));
});
}
}
public static void WriteWithDataTable(string filename, Watershed watershed, bool compress = false, bool createPiscesDB = false)
{
var server = SqLiteEnsemble.GetServer(filename);
byte[] uncompressed = null;
Reclamation.TimeSeries.TimeSeriesDatabase db;
int locIdx = 1;
int WatershedFolderIndex = 1;
int scIndex = 0;
int rowCounter = 0;
Reclamation.TimeSeries.TimeSeriesDatabaseDataSet.SeriesCatalogDataTable sc = null;
if (createPiscesDB)
{
db = new Reclamation.TimeSeries.TimeSeriesDatabase(server);
// limit how much we query.
//var where = "id = (select max(id) from seriescatalog) or id = parentid";
var where = "id = (select max(id) from seriescatalog)";
sc = db.GetSeriesCatalog(where);
WatershedFolderIndex = sc.AddFolder(watershed.Name); // creates root level folder
scIndex = WatershedFolderIndex + 2;
}
else
{
}
var timeSeriesTable = GetBlobTable(server);
int index = server.NextID("timeseries_blob", "id");
foreach (Location loc in watershed.Locations)
{
if (createPiscesDB)
{
locIdx = sc.AddFolder(loc.Name, ++scIndex, WatershedFolderIndex);
}
foreach (Forecast f in loc.Forecasts)
{
var t = f.IssueDate;
var timeseries_start_date = f.TimeStamps[0];
index++;
var row = timeSeriesTable.NewRow();// create rows in separate loop first
row["id"] = index;
row["issue_date"] = f.IssueDate;
row["watershed"] = watershed.Name;
row["location_name"] = loc.Name;
row["timeseries_start_date"] = timeseries_start_date;
row["member_length"] = f.Ensemble.GetLength(1);
row["member_count"] = f.Ensemble.GetLength(0);
row["compressed"] = compress ? 1 : 0;
row["byte_value_array"] = ConvertToBytes(f.Ensemble, compress, ref uncompressed);
if (createPiscesDB)
{
string connectionString = "timeseries_blobs.id=" + index
+ ";member_length=" + f.Ensemble.GetLength(1)
+ ";ensemble_member_index={member_index}"
+ ";timeseries_start_date=" + timeseries_start_date.ToString("yyyy-MM-dd HH:mm:ss");
scIndex = AddPiscesSeries(loc.Name, scIndex, sc, f, locIdx, connectionString);
}
timeSeriesTable.Rows.Add(row);
rowCounter++;
if (rowCounter % 1000 == 0)
{
server.SaveTable(timeSeriesTable);
timeSeriesTable.Rows.Clear();
timeSeriesTable.AcceptChanges();
}
}
}
if (createPiscesDB)
{
server.SaveTable(sc);
}
server.SaveTable(timeSeriesTable);
}
private static void WriteAllFormats(Watershed waterShedData, bool delete)
{
File.AppendAllText(logFile, NL);
string fn, dir;
// DSS 6/7
fn = "ensemble_V7_" + tag + ".dss";
if (delete)
{
File.Delete(fn);
}
WriteTimed(fn, tag, () => DssEnsemble.Write(fn, waterShedData));
fn = "ensemble_V7_profiles_" + tag + ".dss";
if (delete)
{
File.Delete(fn);
}
WriteTimed(fn, tag, () => DssEnsemble.WriteToTimeSeriesProfiles(fn, waterShedData));
bool compress = true;
// SQLITE
fn = "ensemble_sqlite_" + tag + ".db";
if (delete)
{
File.Delete(fn);
}
WriteTimed(fn, tag, () =>
{
SqLiteEnsemble.Write(fn, waterShedData, compress, false);
});
fn = "ensemble_pisces_" + tag + ".pdb";
if (delete)
{
File.Delete(fn);
}
WriteTimed(fn, tag, () =>
{
SqLiteEnsemble.WriteWithDataTable(fn, waterShedData, compress, true);
});
// Serial HDF5
fn = "ensemble_serial_1RowPerChunk.h5";
WriteTimed(fn, tag, () =>
{
using (var h5w = new H5Writer(fn))
HDF5Ensemble.Write(h5w, waterShedData);
});
// Parallel HDF5
foreach (int c in new[] { 1, 10, -1 })
{
fn = "ensemble_parallel_" + c.ToString() + "RowsPerChunk.h5";
WriteTimed(fn, tag, () =>
{
using (var h5w = new H5Writer(fn))
HDF5Ensemble.WriteParallel(h5w, waterShedData, c);
});
}
}
