/// <summary/>
public override IEnumerable <IRow> Combine(IRowset left, IRowset right, IUpdatableRow output)
{
var buffer = new List <Tuple <int, string> >();
foreach (var row2 in right.Rows)
{
buffer.Add(Tuple.Create <int, string>(
row2.Get <int>("employee_id"),
row2.Get <string>("employee_name")
));
}
foreach (var row in left.Rows)
{
foreach (var tuple in buffer)
{
if (row.Get <int>("employee_id") == tuple.Item1)
{
output.Set("employee_id", tuple.Item1);
output.Set("employee_name", tuple.Item2);
output.Set("department_name", row.Get <string>("department_name"));
yield return(output.AsReadOnly());
}
}
}
}
public void TestMyProcessorWithCollection()
{
//Schema: "a:int, b:int"
USqlSchema schema = new USqlSchema(
new USqlColumn <int>("a"),
new USqlColumn <int>("b")
);
IUpdatableRow output = new USqlRow(schema, null).AsUpdatable();
//Generate Rowset with specified values
List <object[]> values = new List <object[]> {
new object[2] {
2, 3
},
new object[2] {
10, 20
}
};
IEnumerable <IRow> rows = UnitTestHelper.CreateRowsFromValues(schema, values);
IRowset rowset = UnitTestHelper.GetRowsetFromCollection(rows, output.AsReadOnly());
//Create UDO instance
MyProcessor processor = new MyProcessor(floor: 1, enforce: true);
foreach (IRow r in rowset.Rows)
{
IRow after = processor.Process(r, output);
//Verify result
Assert.IsTrue(after.Get <int>(0) == 2);
Assert.IsTrue(after.Get <int>(1) == 4);
break;
}
}
/// <summary>
///
/// </summary>
/// <param name="input"></param>
/// <param name="output"></param>
/// <returns></returns>
public override IEnumerable<IRow> Reduce(IRowset input, IUpdatableRow output)
{
int count = 0;
int[] colValues = new int[colNames.Length];
foreach (IRow row in input.Rows)
{
if (count == 0)
{
colValues[(int)ColNames.id] = int.Parse(row.Get<string>("id").ToString());
colValues[(int)ColNames.loc] = location.GetValue(row.Get<string>("loc").ToString());
colValues[(int)ColNames.fs] = int.Parse(row.Get<string>("fs").ToString());
colValues[(int)ColNames.tr] = int.Parse(row.Get<string>("tr").ToString());
colValues[(int)ColNames.st] = sevType.GetValue(row.Get<string>("st").ToString());
}
colValues[eventType.GetValue(row.Get<string>("et").ToString())] = 1;
int vol = int.Parse(row.Get<string>("vol").ToString());
colValues[logFeature.GetValue(row.Get<string>("lf").ToString())] = vol;
colValues[resType.GetValue(row.Get<string>("rt").ToString())] = 1;
count++;
}
// Write output
for (int n = (int)ColNames.lf_1; n < colValues.Length; n++)
{
string colName = colNames[n];
output.Set(colName, colValues[n].ToString());
}
yield return output.AsReadOnly();
}
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
int acc = 0;
int max = 0;
foreach (var row in input.Rows)
{
var timestamp = row.Get <DateTime>("timestamp");
var op = row.Get <string>("op");
if (op == "start")
{
acc++;
}
else
{
acc--;
if (acc < 0)
{
acc = 0;
}
}
max = System.Math.Max(max, acc);
}
output.Set <string>("cohort", "FOO");
output.Set <int>("max", max);
yield return(output.AsReadOnly());
}
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
// Cache the rows in the input rowset (should be records for a single vehicle registration)
// Only save rows where the vehicle is not marked as having been recovered
var stolenVehicleRecords = (from row in input.Rows
select new StolenVehicleRecord(
row.Get <string>("VehicleRegistration"),
row.Get <string>("DateStolen"),
row.Get <string>("DateRecovered")
)).ToList();
// If there aren't any items in the stolenVehicleRecords list, then this vehicle is not stolen so skip over it
if (stolenVehicleRecords.Count > 0)
{
// Sort the data in the stolenVehicleRecords list by DateStolen in descending order, so that the most recent record occurs first
stolenVehicleRecords.Sort();
// Retrieve the first record in the stolenVehicleRecords list - this is the most recent record of the vehicle having been stolen
var stolenVehicleRecord = stolenVehicleRecords.First();
// If the record does not have a recovery date, then output it, otherwise the vehicle is considered to have been recovered and is no longer stolen
if (stolenVehicleRecord.DateRecovered == null)
{
output.Set <string>("VehicleRegistration", stolenVehicleRecord.VehicleRegistration);
output.Set <DateTime>("DateStolen", stolenVehicleRecord.DateStolen);
yield return(output.AsReadOnly());
}
}
}
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
// Init aggregation values
int i = 0;
var begin = DateTime.MaxValue; // Dummy value to make compiler happy
var end = DateTime.MinValue; // Dummy value to make compiler happy
// requires that the reducer is PRESORTED on begin and READONLY on the reduce key.
foreach (var row in input.Rows)
{
// Initialize the first interval with the first row if i is 0
if (i == 0)
{
i++; // mark that we handled the first row
begin = row.Get <DateTime>("begin");
end = row.Get <DateTime>("end");
// If the end is just a time and not a date, it can be earlier than the begin, indicating it is on the next day.
// This let's fix up the end to the next day in that case
if (end < begin)
{
end = end.AddDays(1);
}
}
else
{
var b = row.Get <DateTime>("begin");
var e = row.Get <DateTime>("end");
// fix up the date if end is earlier than begin
if (e < b)
{
e = e.AddDays(1);
}
// if the begin is still inside the interval, increase the interval if it is longer
if (b <= end)
{
// if the new end time is later than the current, extend the interval
if (e > end)
{
end = e;
}
}
else // output the previous interval and start a new one
{
output.Set <DateTime>("begin", begin);
output.Set <DateTime>("end", end);
yield return(output.AsReadOnly());
begin = b; end = e;
} // if
} // if
} // foreach
// now output the last interval
output.Set <DateTime>("begin", begin);
output.Set <DateTime>("end", end);
yield return(output.AsReadOnly());
} // Reduce
public override IEnumerable <IRow> Combine(IRowset left, IRowset right, IUpdatableRow output)
{
var ipList = (from ip in right.Rows
select new
{
IPStart = ip.Get <long>("ip_start_int"),
IPEnd = ip.Get <long>("ip_end_int"),
country = ip.Get <string>("country"),
state = ip.Get <string>("state"),
city = ip.Get <string>("city")
}).ToList();
foreach (var row in left.Rows)
{
output.Set <int>("Year", row.Get <int>("Year"));
output.Set <int>("Month", row.Get <int>("Month"));
output.Set <int>("Day", row.Get <int>("Day"));
output.Set <long?>("TotalVisits", row.Get <long?>("TotalVisits"));
long IP = row.Get <long>("IPInt");
string Location = "";
if (ipList != null)
{
var loc = (from w in ipList
where IP >= w.IPStart && IP <= w.IPEnd
select new
{
country = w.country,
state = w.state,
city = w.city
}).ToList();
if ((loc != null) && (loc.Count > 0))
{
if (String.IsNullOrEmpty(loc[0].state))
{
Location = String.Format("{0}, {1}", loc[0].city, loc[0].country);
}
else
{
Location = String.Format("{0}, {1}, {2}", loc[0].city, loc[0].state, loc[0].country);
}
}
}
;
output.Set <string>("Location", Location);
yield return(output.AsReadOnly());
}
}
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
foreach (var row in input.Rows)
{
foreach (var column in output.Schema)
{
if (!column.IsReadOnly)
{
output.Set <string>(column.Name, row.Get <string>(column.Name));
}
}
}
yield return(output.AsReadOnly());
}
public override IEnumerable<IRow> Reduce(IRowset input, IUpdatableRow output)
{
// Init aggregation values
bool first_row_processed = false;
var begin = DateTime.MaxValue; // Dummy value to make compiler happy
var end = DateTime.MinValue; // Dummy value to make compiler happy
// requires that the reducer is PRESORTED on begin and READONLY on the reduce key.
foreach (var row in input.Rows)
{
// Initialize the first interval with the first row if i is 0
if (!first_row_processed)
{
first_row_processed = true; // mark that we handled the first row
begin = row.Get<DateTime>("begin");
end = row.Get<DateTime>("end");
// If the end is just a time and not a date, it can be earlier than the begin, indicating it is on the next day.
// This let's fix up the end to the next day in that case
if (end < begin) { end = end.AddDays(1); }
}
else
{
var b = row.Get<DateTime>("begin");
var e = row.Get<DateTime>("end");
// fix up the date if end is earlier than begin
if (e < b) { e = e.AddDays(1); }
// if the begin is still inside the interval, increase the interval if it is longer
if (b <= end)
{
// if the new end time is later than the current, extend the interval
if (e > end) { end = e; }
}
else // output the previous interval and start a new one
{
output.Set<DateTime>("begin", begin);
output.Set<DateTime>("end", end);
yield return output.AsReadOnly();
begin = b; end = e;
} // if
} // if
} // foreach
// now output the last interval
output.Set<DateTime>("begin", begin);
output.Set<DateTime>("end", end);
yield return output.AsReadOnly();
}
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
// Init aggregation values
var firstRowProcessed = false;
var begin = DateTime.MinValue;
var end = DateTime.MinValue;
var finalvalue = 0.0;
// requires that the reducer is PRESORTED on begin and READONLY on the reduce key.
foreach (var row in input.Rows)
{
if (!firstRowProcessed)
{
firstRowProcessed = true;
begin = row.Get <DateTime>(BeginColName);
end = row.Get <DateTime>(EndColName);
finalvalue = row.Get <double>(ValueColName);
}
else
{
var b = row.Get <DateTime>(BeginColName);
var e = row.Get <DateTime>(EndColName);
var tmpvalue = row.Get <double>(ValueColName);
if ((b - end).TotalSeconds <= _maxDuration)
{
finalvalue += tmpvalue;
}
else
{
output.Set <double>(ValueColName, finalvalue);
output.Set <DateTime>(BeginColName, begin);
output.Set <DateTime>(EndColName, end);
yield return(output.AsReadOnly());
finalvalue = tmpvalue;
begin = b;
}
end = e;
}
}
output.Set <DateTime>(BeginColName, begin);
output.Set <DateTime>(EndColName, end);
output.Set <double>(ValueColName, finalvalue);
yield return(output.AsReadOnly());
}
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
int? maxlatency = null;
string tquery = "", tmarket = "";
foreach (var row in input.Rows)
{
int?l = row.Get <int?>("Latency");
if (maxlatency == null || l > maxlatency)
{
maxlatency = l;
tquery = row.Get <string>("Query");
tmarket = row.Get <string>("Market");
}
}
output.Set("Query", tquery);
output.Set("Market", tmarket);
output.Set("Latency", maxlatency);
yield return(output.AsReadOnly());
}
public void TestMyReducer()
{
USqlColumn <string> col1 = new USqlColumn <string>("Query");
USqlColumn <string> col2 = new USqlColumn <string>("Market");
USqlColumn <int> col3 = new USqlColumn <int>("Latency");
List <IColumn> columns = new List <IColumn> {
col1, col2, col3
};
USqlSchema schema = new USqlSchema(columns);
IUpdatableRow output = new USqlRow(schema, null).AsUpdatable();
//Generate Rowset with specified values
List <object[]> values = new List <object[]> {
new object[3] {
"Query1", "Market1", 1
},
new object[3] {
"Query2", "Market2", 2
},
new object[3] {
"Query3", "Market3", 3
},
new object[3] {
"Query4", "Market4", 4
},
};
IEnumerable <IRow> rows = UnitTestHelper.CreateRowsFromValues(schema, values);
IRowset rowset = UnitTestHelper.GetRowsetFromCollection(rows, output.AsReadOnly());
//Create UDO instance. The reducer should get the largest latency
MyReducer reducer = new MyReducer();
foreach (IRow r in reducer.Reduce(rowset, output))
{
Assert.IsTrue(rowset.Schema.Count == 3);
}
//Make sure the reducer returns the row of the largest latency
Assert.IsTrue(output.Get <int>("Latency") == 4);
}
// Reduce the input rowset to summarize the number of employees in each role in each department
// The data in the input rowset contains the employee ID, employee name, department ID, and comma separated list of roles the employee has performed in that department
// For each department, return:
// DepartmentID int,
// NumberOfAssociates int,
// NumberOfEmployees int,
// NumberOfTeamLeaders int,
// NumberOfManagers int,
// NumberOfVicePresidents int,
// NumberOfPresidents int
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
Dictionary <string, int> roles = new Dictionary <string, int>();
roles["associate"] = 0;
roles["employee"] = 0;
roles["teamleader"] = 0;
roles["manager"] = 0;
roles["vicepresident"] = 0;
roles["president"] = 0;
var groupedRows = (from row in input.Rows
select new
{
dept = row.Get <int>("DepartmentID"),
roles = row.Get <string>("Roles")
}).ToList();
foreach (var row in groupedRows)
{
// Parse the comma separated list of roles into individual values
string[] rolesForEmployee = row.roles.Split(',');
// Iterate through this list of roles for the employee aggregate the role counts
foreach (string role in rolesForEmployee)
{
roles[role]++;
}
}
// Output the aggregate results
output.Set <int>("DepartmentID", groupedRows.First().dept);
output.Set <int>("NumberOfAssociates", roles["associate"]);
output.Set <int>("NumberOfEmployees", roles["employee"]);
output.Set <int>("NumberOfTeamLeaders", roles["teamleader"]);
output.Set <int>("NumberOfManagers", roles["manager"]);
output.Set <int>("NumberOfVicePresidents", roles["vicepresident"]);
output.Set <int>("NumberOfPresidents", roles["president"]);
yield return(output.AsReadOnly());
}
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
var i = 0;
var start = DateTime.MinValue;
var end = DateTime.MaxValue;
foreach (var row in input.Rows)
{
if (i == 0)
{
start = row.Get <DateTime>(Start);
end = row.Get <DateTime>(End);
}
else
{
var nextStart = row.Get <DateTime>(Start);
var nextEnd = row.Get <DateTime>(End);
if (nextStart <= end)
{
if (nextEnd > end)
{
end = nextEnd;
}
}
else
{
output.Set <DateTime>(Start, start);
output.Set <DateTime>(End, end);
yield return(output.AsReadOnly());
start = nextStart; end = nextEnd;
}
}
i++;
}
output.Set <DateTime>(Start, start);
output.Set <DateTime>(End, end);
yield return(output.AsReadOnly());
}
// Combine the data in both rowsets to produce a list of employees and the names of the departments in which they have worked
// The left input data contains the employee ID, employee name, department ID, and comma separated list of roles the employee has performed in that department
// The right input data contains the department name and department ID
// The output data should contains employee ID, employee name, department name, and role
// A single employee will likely generate multiple rows of output
public override IEnumerable <IRow> Combine(IRowset left, IRowset right, IUpdatableRow output)
{
// Read the right rowset containing department details into a local List collection
// (you can only enumerate an IRowset collection once, and we need to perform multiple iterations, so the data must be cached locally)
var rightRowset = (from row in right.Rows
select new
{
deptID = row.Get <int>("DepartmentID"),
deptName = row.Get <string>("DepartmentName")
}).ToList();
// Join the rows in each collection across the Department ID column
foreach (var leftRow in left.Rows)
{
// Find the name for the department
var department = (from deptInfo in rightRowset
where deptInfo.deptID == leftRow.Get <int>("DepartmentID")
select new { id = deptInfo.deptID, name = deptInfo.deptName }).FirstOrDefault();
// Output the employee and department role details
if (department != null)
{
// Split the comma separated list of roles in the employee data into individual values
string[] rolesForEmployee = leftRow.Get <string>("Roles").Split(',');
// Iterate through this list of roles for the employee and output each one in turn
foreach (string role in rolesForEmployee)
{
output.Set <int>("EmpID", leftRow.Get <int>("EmployeeID"));
output.Set <string>("EmpName", leftRow.Get <string>("EmployeeName"));
output.Set <string>("DeptName", department.name);
output.Set <string>("Role", role);
yield return(output.AsReadOnly());
}
}
}
}
public void TestMyProcessorWithFile()
{
//Schema: "a:int, b:int"
USqlColumn <int> col1 = new USqlColumn <int>("a");
USqlColumn <int> col2 = new USqlColumn <int>("b");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
//Generate one row with default values
IUpdatableRow output = new USqlRow(schema, null).AsUpdatable();
//Get upstreams from file
IRowset rowset = UnitTestHelper.GetRowsetFromFile(@"processor.txt", schema, output.AsReadOnly(), discardAdditionalColumns: true, rowDelimiter: null, columnSeparator: '\t');
//Create UDO instance
MyProcessor processor = new MyProcessor(floor: 50, enforce: true);
foreach (IRow r in rowset.Rows)
{
processor.Process(r, output);
}
}
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
var rows = (from r in input.Rows
select new SnotelRow
{
DatePart = r.Get <DateTime>("DatePart"),
Date = r.Get <DateTime>("Date"),
DateString = r.Get <String>("DateString"),
StationName = r.Get <String>("StationName"),
ElevationFt = r.Get <int>("ElevationFt"),
Lat = r.Get <double>("Lat"),
Lon = r.Get <double>("Lon"),
SnowWaterEquivalentIn = r.Get <float?>("SnowWaterEquivalentIn"),
PrecipitationAccumulation = r.Get <float?>("PrecipitationAccumulation"),
SnowDepthIn = r.Get <int?>("SnowDepthIn"),
AirTemperatureObservedF = r.Get <int?>("AirTemperatureObservedF"),
SnotelState = r.Get <String>("SnotelState"),
__fileHour = r.Get <int>("__fileHour"),
__fileDate = r.Get <DateTime>("__fileDate")
}).ToList();
List <double> pointsSwe = new List <double>();
List <double> pointsPrecip = new List <double>();
List <double> pointsSnowDepth = new List <double>();
List <double> pointsAirTemp = new List <double>();
List <double> valuesSwe = new List <double>();
List <double> valuesPrecip = new List <double>();
List <double> valuesSnowDepth = new List <double>();
List <double> valuesAirTemp = new List <double>();
bool interpolateSwe = false;
bool interpolatePrecip = false;
bool interpolateSnowDepth = false;
bool interpolateAirTemp = false;
for (int count = 0; count < rows.Count(); count++)
{
var row = rows.ElementAt(count);
if (row.SnowWaterEquivalentIn != null)
{
pointsSwe.Add(count);
valuesSwe.Add(row.SnowWaterEquivalentIn.Value);
}
else
{
interpolateSwe = true;
}
if (row.PrecipitationAccumulation != null)
{
pointsPrecip.Add(count);
valuesPrecip.Add(row.PrecipitationAccumulation.Value);
}
else
{
interpolatePrecip = true;
}
if (row.SnowDepthIn != null)
{
pointsSnowDepth.Add(count);
valuesSnowDepth.Add(row.SnowDepthIn.Value);
}
else
{
interpolateSnowDepth = true;
}
if (row.AirTemperatureObservedF != null)
{
pointsAirTemp.Add(count);
valuesAirTemp.Add(row.AirTemperatureObservedF.Value);
}
else
{
interpolateAirTemp = true;
}
}
var methodSwe = (pointsSwe.Count > 1 && interpolateSwe ? Interpolate.Linear(pointsSwe, valuesSwe) : null);
var methodPrecip = (pointsPrecip.Count > 1 && interpolatePrecip ? Interpolate.Linear(pointsPrecip, valuesPrecip) : null);
var methodSnowDepth = (pointsSnowDepth.Count > 1 && interpolateSnowDepth ? Interpolate.Linear(pointsSnowDepth, valuesSnowDepth) : null);
var methodAirTemp = (pointsAirTemp.Count > 1 && interpolateAirTemp ? Interpolate.Linear(pointsAirTemp, valuesAirTemp) : null);
for (int count = 0; count < rows.Count(); count++)
{
var row = rows.ElementAt(count);
if (row.SnowWaterEquivalentIn != null)
{
output.Set <float?>("SnowWaterEquivalentIn", row.SnowWaterEquivalentIn.Value);
}
else if (row.SnowWaterEquivalentIn == null && methodSwe != null)
{
float swe = (float)methodSwe.Interpolate(count);
output.Set <float?>("SnowWaterEquivalentIn", swe);
}
else
{
output.Set <float?>("SnowWaterEquivalentIn", null);
}
if (row.PrecipitationAccumulation != null)
{
output.Set <float?>("PrecipitationAccumulation", row.PrecipitationAccumulation.Value);
}
else if (row.PrecipitationAccumulation == null && methodPrecip != null)
{
float precip = (float)methodPrecip.Interpolate(count);
output.Set <float?>("PrecipitationAccumulation", precip);
}
else
{
output.Set <float?>("PrecipitationAccumulation", null);
}
if (row.SnowDepthIn != null)
{
output.Set <int?>("SnowDepthIn", row.SnowDepthIn.Value);
}
else if (row.SnowDepthIn == null && methodSnowDepth != null)
{
int depth = (int)methodSnowDepth.Interpolate(count);
output.Set <int?>("SnowDepthIn", depth);
}
else
{
output.Set <int?>("SnowDepthIn", null);
}
if (row.AirTemperatureObservedF != null)
{
output.Set <int?>("AirTemperatureObservedF", row.AirTemperatureObservedF.Value);
}
else if (row.AirTemperatureObservedF == null && methodAirTemp != null)
{
int temp = (int)methodAirTemp.Interpolate(count);
output.Set <int?>("AirTemperatureObservedF", temp);
}
else
{
output.Set <int?>("AirTemperatureObservedF", null);
}
output.Set <DateTime>("DatePart", row.DatePart);
output.Set <DateTime>("Date", row.Date);
output.Set <String>("DateString", row.DateString);
output.Set <String>("StationName", row.StationName);
output.Set <int>("ElevationFt", row.ElevationFt);
output.Set <double>("Lat", row.Lat);
output.Set <double>("Lon", row.Lon);
output.Set <String>("SnotelState", row.SnotelState);
output.Set <int>("__fileHour", row.__fileHour);
output.Set <DateTime>("__fileDate", row.__fileDate);
yield return(output.AsReadOnly());
}
}
/// <summary>
/// Combine is called once per match on the join clause; its not prefiltered to left or right but gives the full data sets for each maching the join clause
/// </summary>
/// <param name="left"></param>
/// <param name="right"></param>
/// <param name="output"></param>
/// <returns></returns>
public override IEnumerable <IRow> Combine(IRowset left, IRowset right, IUpdatableRow output)
{
var theRight = (from row in right.Rows
select new SnotelRow
{
StationName = row.Get <string>("StationName"),
Lat = row.Get <double>("Lat"),
Lon = row.Get <double>("Lon"),
ElevationFt = row.Get <int>("ElevationFt"),
SnotelState = row.Get <string>("SnotelState")
}).ToList();
foreach (var row in left.Rows)
{
var Lat = row.Get <double>("Lat");
var Lon = row.Get <double>("Lon");
string closestStation = "None";
double distanceToStation = DistanceThresholdKm + 1; //default is just longer than distance threshold
SnotelRow closestRow = null;
//narrow the search range down to just ones within 1 degree lat/lon of the current value
foreach (var subRow in theRight)
{
//Calculate distance
var tmpDistance = DistanceBetweenCoordinates(Lat, Lon, subRow.Lat, subRow.Lon);
//Store value if its bigger than the previous value
if (tmpDistance < DistanceThresholdKm && tmpDistance < distanceToStation)
{
closestStation = subRow.StationName;
distanceToStation = tmpDistance;
closestRow = subRow;
}
}
if (closestRow == null)
{
distanceToStation = 0;
closestRow = new SnotelRow()
{
StationName = "None",
Lat = 0,
Lon = 0,
ElevationFt = 0,
//SnowWaterEquivalentIn = null,
//PrecipitationAccumulation = null,
//SnowDepthIn = null,
//AirTemperatureObservedF = null,
SnotelState = "None"
};
}
output.Set <DateTime>("Date", row.Get <DateTime>("__fileDate"));
output.Set <double>("Lat", row.Get <double>("Lat"));
output.Set <double>("Lon", row.Get <double>("Lon"));
output.Set <string>("StationName", closestStation);
output.Set <float>("DistanceToStationKm", (float)distanceToStation);
output.Set <int>("ElevationFt", closestRow.ElevationFt);
output.Set <double>("SnotelLat", closestRow.Lat);
output.Set <double>("SnotelLon", closestRow.Lon);
output.Set <string>("SnotelState", closestRow.SnotelState);
yield return(output.AsReadOnly());
}
}
public void TestMyCombiner()
{
//Construct left columns
USqlColumn <int> col1 = new USqlColumn <int>("employee_id");
USqlColumn <string> col2 = new USqlColumn <string>("employee_name");
USqlColumn <string> col3 = new USqlColumn <string>("department_name");
//Construct left schema
List <IColumn> columns_left = new List <IColumn> {
col1, col3
};
USqlSchema schema_left = new USqlSchema(columns_left);
//Construct right schema
List <IColumn> columns_right = new List <IColumn> {
col1, col2
};
USqlSchema schema_right = new USqlSchema(columns_right);
//Construct result schema. expected schema: employee_id, employee_name, department_name
List <IColumn> columns_result = new List <IColumn> {
col1, col2, col3
};
USqlSchema schema_result = new USqlSchema(columns_result);
//Construct the output schema which will be needed later
IUpdatableRow output_left = new USqlRow(schema_left, null).AsUpdatable();
IUpdatableRow output_right = new USqlRow(schema_right, null).AsUpdatable();
IUpdatableRow output_result = new USqlRow(schema_result, null).AsUpdatable();
//Generate Rowset with specified values for left input
List <object[]> values_left = new List <object[]> {
new object[2] {
1, "HR"
},
new object[2] {
2, "R&D"
},
new object[2] {
4, "Operation"
},
};
//Generate Rowset with specified values for right input
List <object[]> values_right = new List <object[]> {
new object[2] {
1, "John"
},
new object[2] {
2, "Tom"
},
new object[2] {
3, "Melinda"
},
};
IEnumerable <IRow> rows_left = UnitTestHelper.CreateRowsFromValues(schema_left, values_left);
IEnumerable <IRow> rows_right = UnitTestHelper.CreateRowsFromValues(schema_right, values_right);
IRowset rowset_left = UnitTestHelper.GetRowsetFromCollection(rows_left, output_left.AsReadOnly());
IRowset rowset_right = UnitTestHelper.GetRowsetFromCollection(rows_right, output_right.AsReadOnly());
//Create UDO instance. The combiner will do something like inner join
MyCombiner combiner = new MyCombiner();
IEnumerable <IRow> after = combiner.Combine(rowset_left, rowset_right, output_result);
//Verify result
List <IRow> resultlist = after.ToList();
Assert.IsTrue(resultlist[0].Schema.Count == 3);
//Verify the values. expected to see:
//the first row: 1, "John", "HR"
//the second row: 2, "Tom", "R&D"
//here we only verify the first row
Assert.IsTrue(resultlist[0].Get <int>("employee_id") == 1);
Assert.IsTrue(resultlist[0].Get <string>(1) == "John");
Assert.IsTrue(resultlist[0].Get <string>(2) == "HR");
}
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
foreach (var row in input.Rows)
{
var rowList = new List <float>();
rowList.Add(row.Get <float>("n_f_APCPsurface1HourForecast"));
for (int i = 1; i < 341; i++)
{
rowList.Add(0);
}
cache.Add(rowList);
}
//convert to array of floats
float[][] values = new float[cache.Count][];
for (int i = 0; i < cache.Count; i++)
{
values[i] = cache[i].ToArray();
}
//deploy resource command should place is in working dir
string modelLocation = "ModelAboveV1.bin";
var xgbc = BaseXgbModel.LoadClassifierFromFile(modelLocation);
var predictions = xgbc.Predict(values);
for (int i = 0; i < predictions.Length; i++)
{
output.Set <float>("prediction", predictions[i]);
yield return(output.AsReadOnly());
}
/*
* __fileDate AS Date,
* Lat,
* Lon,
* n_f_APCPsurface1HourForecast,
* n_f_WindSpeed10m1HourForecast AS n_f_10mWindSpeed1HourForecast,
* n_f_APCPsurface2HourForecast,
* n_f_WindSpeed10m2HourForecast AS n_f_10mWindSpeed2HourForecast,
* n_f_APCPsurface3HourForecast,
* n_f_WindSpeed10m3HourForecast AS n_f_10mWindSpeed3HourForecast,
* n_f_APCPsurface4HourForecast,
* n_f_WindSpeed10m4HourForecast AS n_f_10mWindSpeed4HourForecast,
* n_f_APCPsurface5HourForecast,
* n_f_WindSpeed10m5HourForecast AS n_f_10mWindSpeed5HourForecast,
* n_f_APCPsurface6HourForecast,
* n_f_WindSpeed10m6HourForecast AS n_f_10mWindSpeed6HourForecast,
* n_f_APCPsurface7HourForecast,
* n_f_WindSpeed10m7HourForecast AS n_f_10mWindSpeed7HourForecast,
* n_f_APCPsurface8HourForecast,
* n_f_WindSpeed10m8HourForecast AS n_f_10mWindSpeed8HourForecast,
* n_f_APCPsurface9HourForecast,
* n_f_WindSpeed10m9HourForecast AS n_f_10mWindSpeed9HourForecast,
* n_f_APCPsurface10HourForecast,
* n_f_WindSpeed10m10HourForecast AS n_f_10mWindSpeed10HourForecast,
* n_f_APCPsurface11HourForecast,
* n_f_WindSpeed10m11HourForecast AS n_f_10mWindSpeed11HourForecast,
* n_f_APCPsurface12HourForecast,
* n_f_WindSpeed10m12HourForecast AS n_f_10mWindSpeed12HourForecast,
* n_f_APCPsurface13HourForecast,
* n_f_WindSpeed10m13HourForecast AS n_f_10mWindSpeed13HourForecast,
* n_f_APCPsurface14HourForecast,
* n_f_WindSpeed10m14HourForecast AS n_f_10mWindSpeed14HourForecast,
* n_f_APCPsurface15HourForecast,
* n_f_WindSpeed10m15HourForecast AS n_f_10mWindSpeed15HourForecast,
* n_f_APCPsurface16HourForecast,
* n_f_WindSpeed10m16HourForecast AS n_f_10mWindSpeed16HourForecast,
* n_f_APCPsurface17HourForecast,
* n_f_WindSpeed10m17HourForecast AS n_f_10mWindSpeed17HourForecast,
* n_f_APCPsurface18HourForecast,
* n_f_WindSpeed10m18HourForecast AS n_f_10mWindSpeed18HourForecast,
* n_f_APCPsurface19HourForecast,
* n_f_WindSpeed10m19HourForecast AS n_f_10mWindSpeed19HourForecast,
* n_f_APCPsurface20HourForecast,
* n_f_WindSpeed10m20HourForecast AS n_f_10mWindSpeed20HourForecast,
* n_f_APCPsurface21HourForecast,
* n_f_WindSpeed10m21HourForecast AS n_f_10mWindSpeed21HourForecast,
* n_f_APCPsurface22HourForecast,
* n_f_WindSpeed10m22HourForecast AS n_f_10mWindSpeed22HourForecast,
* n_f_APCPsurface23HourForecast,
* n_f_WindSpeed10m23HourForecast AS n_f_10mWindSpeed23HourForecast,
* n_f_MaxTempSurfaceF AS n_f_tempMaxF,
* n_f_MaxWindSpeed10m AS n_f_10mWindSpeedMax,
* n_r_SnowDepthIn AS n_r_snowDepthIn,
* n_f_MinTempSurfaceF AS n_f_tempMinF,
* n_f_AvgTempSurfaceF AS n_f_tempAveF,
* n_f_WindSpeed10m0HourForecast AS n_f_10mWindSpeed,
* n_f_APCPsurface0HourForecast AS n_f_APCPsurface,
* n_r_PrecipincrementSnowIn AS n_r_precipIncrementSnowIn,
* n_r_Prev3DaySnowAccumulation AS n_r_Prev3daySnowAccumulation,
* [n_r_Prev7DaySnowAccumulation]
* AS n_r_Prev7daySnowAccumulation,
* [n_r_Prev3DayMaxTemp] AS n_r_Prev3dayMaxTemp,
* [n_r_Prev3DayMaxWindSpeed10m] AS n_r_Prev3DayMax10mWind,
* [n_r_Prev3DayMinTemp] AS n_r_Prev3dayMinTemp,
* [n_r_Prev7DayMaxTemp] AS n_r_Prev7dayMaxTemp,
* [n_r_Prev7DayMaxWindSpeed10m] AS n_r_Prev7DayMax10mWind,
*
* [n_r_Prev7DayMinTemp] AS n_r_Prev7dayMinTemp,
*
* [n_r_Prev1DayMaxTemp] AS n_r_Prev1dayMaxTemp,
*
* [n_r_Prev1DayMaxWindSpeed10m] AS n_r_Prev1DayMax10mWind,
*
* [n_r_Prev1DayMinTemp] AS n_r_Prev1dayMinTemp,
*
* [n_r_Prev1DayPrecip] AS n_r_Prev1DayPrecip,
* c_r_Prev3DayFreezeThawLikeliness,
* c_r_Prev7DayFreezeThawLikeliness,
*
* [c_r_Prev3DayWindSlabLikeliness] AS c_r_Prev3DayWindSlabLikeliness,
*
* [c_r_Prev7DayWindSlabLikeliness] AS c_r_Prev7DayWindSlabLikeliness,
* n_f_Next24HourChangeInTempFromPrev3DayMax,
* n_f_Next24HourChangeInTempFromPrev1DayMax AS n_f_Next24HoursChangeInTempFromPrev1DayMax,
* c_r_LongTermColdTemps AS c_f_LongTermColdTemps,
* n_r_Prev24HoursPrecipAsRainTotalIn,
* n_r_SNOWDAS_SnowDepth_mm,
* n_r_SNOWDAS_SWE_mm,
* n_r_SNOWDAS_SnowmeltRunoff_micromm,
* n_r_SNOWDAS_Sublimation_micromm,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem,
* n_r_SNOWDAS_SnowpackAveTemp_k,
* n_r_SnowDepthIn1InPast AS n_r_snowDepthIn1InPast,
* n_r_PrecipincrementSnowIn1InPast AS n_r_precipIncrementSnowIn1InPast,
* n_r_Prev3DaySnowAccumulation1InPast AS n_r_Prev3daySnowAccumulation1InPast,
* [n_r_Prev7DaySnowAccumulation1InPast] AS n_r_Prev7daySnowAccumulation1InPast,
* [n_r_Prev3DayMaxTemp1InPast] AS n_r_Prev3dayMaxTemp1InPast,
* [n_r_Prev3DayMaxWindSpeed10m1InPast] AS n_r_Prev3DayMax10mWind1InPast,
* [n_r_Prev3DayMinTemp1InPast] AS n_r_Prev3dayMinTemp1InPast,
* [n_r_Prev7DayMaxTemp1InPast] AS n_r_Prev7dayMaxTemp1InPast,
* [n_r_Prev7DayMaxWindSpeed10m1InPast] AS n_r_Prev7DayMax10mWind1InPast,
*
* [n_r_Prev7DayMinTemp1InPast] AS n_r_Prev7dayMinTemp1InPast,
*
* [n_r_Prev1DayMaxTemp1InPast] AS n_r_Prev1dayMaxTemp1InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m1InPast] AS n_r_Prev1DayMax10mWind1InPast,
*
* [n_r_Prev1DayMinTemp1InPast] AS n_r_Prev1dayMinTemp1InPast,
*
* [n_r_Prev1DayPrecip1InPast] AS n_r_Prev1DayPrecip1InPast,
* c_r_Prev3DayFreezeThawLikeliness1InPast,
* c_r_Prev7DayFreezeThawLikeliness1InPast,
*
* [c_r_Prev3DayWindSlabLikeliness1InPast] AS c_r_Prev3DayWindSlabLikeliness1InPast,
*
* [c_r_Prev7DayWindSlabLikeliness1InPast] AS c_r_Prev7DayWindSlabLikeliness1InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn1InPast,
* n_r_SNOWDAS_SnowDepth_mm1InPast,
* n_r_SNOWDAS_SWE_mm1InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm1InPast,
* n_r_SNOWDAS_Sublimation_micromm1InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem1InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem1InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k1InPast,
* n_r_SnowDepthIn2InPast AS n_r_snowDepthIn2InPast,
* n_r_PrecipincrementSnowIn2InPast AS n_r_precipIncrementSnowIn2InPast,
* n_r_Prev3DaySnowAccumulation2InPast AS n_r_Prev3daySnowAccumulation2InPast,
* [n_r_Prev7DaySnowAccumulation2InPast] AS n_r_Prev7daySnowAccumulation2InPast,
* [n_r_Prev3DayMaxTemp2InPast] AS n_r_Prev3dayMaxTemp2InPast,
* [n_r_Prev3DayMaxWindSpeed10m2InPast] AS n_r_Prev3DayMax10mWind2InPast,
* [n_r_Prev3DayMinTemp2InPast] AS n_r_Prev3dayMinTemp2InPast,
* [n_r_Prev7DayMaxTemp2InPast] AS n_r_Prev7dayMaxTemp2InPast,
* [n_r_Prev7DayMaxWindSpeed10m2InPast] AS n_r_Prev7DayMax10mWind2InPast,
*
* [n_r_Prev7DayMinTemp2InPast] AS n_r_Prev7dayMinTemp2InPast,
*
* [n_r_Prev1DayMaxTemp2InPast] AS n_r_Prev1dayMaxTemp2InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m2InPast] AS n_r_Prev1DayMax10mWind2InPast,
*
* [n_r_Prev1DayMinTemp2InPast] AS n_r_Prev1dayMinTemp2InPast,
*
* [n_r_Prev1DayPrecip2InPast] AS n_r_Prev1DayPrecip2InPast,
* c_r_Prev3DayFreezeThawLikeliness2InPast,
* c_r_Prev7DayFreezeThawLikeliness2InPast,
*
* [c_r_Prev3DayWindSlabLikeliness2InPast] AS c_r_Prev3DayWindSlabLikeliness2InPast,
*
* [c_r_Prev7DayWindSlabLikeliness2InPast] AS c_r_Prev7DayWindSlabLikeliness2InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn2InPast,
* n_r_SNOWDAS_SnowDepth_mm2InPast,
* n_r_SNOWDAS_SWE_mm2InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm2InPast,
* n_r_SNOWDAS_Sublimation_micromm2InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem2InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem2InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k2InPast,
* n_r_SnowDepthIn3InPast AS n_r_snowDepthIn3InPast,
* n_r_PrecipincrementSnowIn3InPast AS n_r_precipIncrementSnowIn3InPast,
* n_r_Prev3DaySnowAccumulation3InPast AS n_r_Prev3daySnowAccumulation3InPast,
* [n_r_Prev7DaySnowAccumulation3InPast] AS n_r_Prev7daySnowAccumulation3InPast,
* [n_r_Prev3DayMaxTemp3InPast] AS n_r_Prev3dayMaxTemp3InPast,
* [n_r_Prev3DayMaxWindSpeed10m3InPast] AS n_r_Prev3DayMax10mWind3InPast,
* [n_r_Prev3DayMinTemp3InPast] AS n_r_Prev3dayMinTemp3InPast,
* [n_r_Prev7DayMaxTemp3InPast] AS n_r_Prev7dayMaxTemp3InPast,
* [n_r_Prev7DayMaxWindSpeed10m3InPast] AS n_r_Prev7DayMax10mWind3InPast,
*
* [n_r_Prev7DayMinTemp3InPast] AS n_r_Prev7dayMinTemp3InPast,
*
* [n_r_Prev1DayMaxTemp3InPast] AS n_r_Prev1dayMaxTemp3InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m3InPast] AS n_r_Prev1DayMax10mWind3InPast,
*
* [n_r_Prev1DayMinTemp3InPast] AS n_r_Prev1dayMinTemp3InPast,
*
* [n_r_Prev1DayPrecip3InPast] AS n_r_Prev1DayPrecip3InPast,
* c_r_Prev3DayFreezeThawLikeliness3InPast,
* c_r_Prev7DayFreezeThawLikeliness3InPast,
*
* [c_r_Prev3DayWindSlabLikeliness3InPast] AS c_r_Prev3DayWindSlabLikeliness3InPast,
*
* [c_r_Prev7DayWindSlabLikeliness3InPast] AS c_r_Prev7DayWindSlabLikeliness3InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn3InPast,
* n_r_SNOWDAS_SnowDepth_mm3InPast,
* n_r_SNOWDAS_SWE_mm3InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm3InPast,
* n_r_SNOWDAS_Sublimation_micromm3InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem3InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem3InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k3InPast,
* n_r_SnowDepthIn4InPast AS n_r_snowDepthIn4InPast,
* n_r_PrecipincrementSnowIn4InPast AS n_r_precipIncrementSnowIn4InPast,
* n_r_Prev3DaySnowAccumulation4InPast AS n_r_Prev3daySnowAccumulation4InPast,
* [n_r_Prev7DaySnowAccumulation4InPast] AS n_r_Prev7daySnowAccumulation4InPast,
* [n_r_Prev3DayMaxTemp4InPast] AS n_r_Prev3dayMaxTemp4InPast,
* [n_r_Prev3DayMaxWindSpeed10m4InPast] AS n_r_Prev3DayMax10mWind4InPast,
* [n_r_Prev3DayMinTemp4InPast] AS n_r_Prev3dayMinTemp4InPast,
* [n_r_Prev7DayMaxTemp4InPast] AS n_r_Prev7dayMaxTemp4InPast,
* [n_r_Prev7DayMaxWindSpeed10m4InPast] AS n_r_Prev7DayMax10mWind4InPast,
*
* [n_r_Prev7DayMinTemp4InPast] AS n_r_Prev7dayMinTemp4InPast,
*
* [n_r_Prev1DayMaxTemp4InPast] AS n_r_Prev1dayMaxTemp4InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m4InPast] AS n_r_Prev1DayMax10mWind4InPast,
*
* [n_r_Prev1DayMinTemp4InPast] AS n_r_Prev1dayMinTemp4InPast,
*
* [n_r_Prev1DayPrecip4InPast] AS n_r_Prev1DayPrecip4InPast,
* c_r_Prev3DayFreezeThawLikeliness4InPast,
* c_r_Prev7DayFreezeThawLikeliness4InPast,
*
* [c_r_Prev3DayWindSlabLikeliness4InPast] AS c_r_Prev3DayWindSlabLikeliness4InPast,
*
* [c_r_Prev7DayWindSlabLikeliness4InPast] AS c_r_Prev7DayWindSlabLikeliness4InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn4InPast,
* n_r_SNOWDAS_SnowDepth_mm4InPast,
* n_r_SNOWDAS_SWE_mm4InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm4InPast,
* n_r_SNOWDAS_Sublimation_micromm4InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem4InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem4InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k4InPast,
* n_r_SnowDepthIn5InPast AS n_r_snowDepthIn5InPast,
* n_r_PrecipincrementSnowIn5InPast AS n_r_precipIncrementSnowIn5InPast,
* n_r_Prev3DaySnowAccumulation5InPast AS n_r_Prev3daySnowAccumulation5InPast,
* [n_r_Prev7DaySnowAccumulation5InPast] AS n_r_Prev7daySnowAccumulation5InPast,
* [n_r_Prev3DayMaxTemp5InPast] AS n_r_Prev3dayMaxTemp5InPast,
* [n_r_Prev3DayMaxWindSpeed10m5InPast] AS n_r_Prev3DayMax10mWind5InPast,
* [n_r_Prev3DayMinTemp5InPast] AS n_r_Prev3dayMinTemp5InPast,
* [n_r_Prev7DayMaxTemp5InPast] AS n_r_Prev7dayMaxTemp5InPast,
* [n_r_Prev7DayMaxWindSpeed10m5InPast] AS n_r_Prev7DayMax10mWind5InPast,
*
* [n_r_Prev7DayMinTemp5InPast] AS n_r_Prev7dayMinTemp5InPast,
*
* [n_r_Prev1DayMaxTemp5InPast] AS n_r_Prev1dayMaxTemp5InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m5InPast] AS n_r_Prev1DayMax10mWind5InPast,
*
* [n_r_Prev1DayMinTemp5InPast] AS n_r_Prev1dayMinTemp5InPast,
*
* [n_r_Prev1DayPrecip5InPast] AS n_r_Prev1DayPrecip5InPast,
* c_r_Prev3DayFreezeThawLikeliness5InPast,
* c_r_Prev7DayFreezeThawLikeliness5InPast,
*
* [c_r_Prev3DayWindSlabLikeliness5InPast] AS c_r_Prev3DayWindSlabLikeliness5InPast,
*
* [c_r_Prev7DayWindSlabLikeliness5InPast] AS c_r_Prev7DayWindSlabLikeliness5InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn5InPast,
* n_r_SNOWDAS_SnowDepth_mm5InPast,
* n_r_SNOWDAS_SWE_mm5InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm5InPast,
* n_r_SNOWDAS_Sublimation_micromm5InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem5InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem5InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k5InPast,
* n_r_SnowDepthIn6InPast AS n_r_snowDepthIn6InPast,
* n_r_PrecipincrementSnowIn6InPast AS n_r_precipIncrementSnowIn6InPast,
* n_r_Prev3DaySnowAccumulation6InPast AS n_r_Prev3daySnowAccumulation6InPast,
* [n_r_Prev7DaySnowAccumulation6InPast] AS n_r_Prev7daySnowAccumulation6InPast,
* [n_r_Prev3DayMaxTemp6InPast] AS n_r_Prev3dayMaxTemp6InPast,
* [n_r_Prev3DayMaxWindSpeed10m6InPast] AS n_r_Prev3DayMax10mWind6InPast,
* [n_r_Prev3DayMinTemp6InPast] AS n_r_Prev3dayMinTemp6InPast,
* [n_r_Prev7DayMaxTemp6InPast] AS n_r_Prev7dayMaxTemp6InPast,
* [n_r_Prev7DayMaxWindSpeed10m6InPast] AS n_r_Prev7DayMax10mWind6InPast,
*
* [n_r_Prev7DayMinTemp6InPast] AS n_r_Prev7dayMinTemp6InPast,
*
* [n_r_Prev1DayMaxTemp6InPast] AS n_r_Prev1dayMaxTemp6InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m6InPast] AS n_r_Prev1DayMax10mWind6InPast,
*
* [n_r_Prev1DayMinTemp6InPast] AS n_r_Prev1dayMinTemp6InPast,
*
* [n_r_Prev1DayPrecip6InPast] AS n_r_Prev1DayPrecip6InPast,
* c_r_Prev3DayFreezeThawLikeliness6InPast,
* c_r_Prev7DayFreezeThawLikeliness6InPast,
*
* [c_r_Prev3DayWindSlabLikeliness6InPast] AS c_r_Prev3DayWindSlabLikeliness6InPast,
*
* [c_r_Prev7DayWindSlabLikeliness6InPast] AS c_r_Prev7DayWindSlabLikeliness6InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn6InPast,
* n_r_SNOWDAS_SnowDepth_mm6InPast,
* n_r_SNOWDAS_SWE_mm6InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm6InPast,
* n_r_SNOWDAS_Sublimation_micromm6InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem6InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem6InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k6InPast,
* n_r_SnowDepthIn7InPast AS n_r_snowDepthIn7InPast,
* n_r_PrecipincrementSnowIn7InPast AS n_r_precipIncrementSnowIn7InPast,
* n_r_Prev3DaySnowAccumulation7InPast AS n_r_Prev3daySnowAccumulation7InPast,
* [n_r_Prev7DaySnowAccumulation7InPast] AS n_r_Prev7daySnowAccumulation7InPast,
* [n_r_Prev3DayMaxTemp7InPast] AS n_r_Prev3dayMaxTemp7InPast,
* [n_r_Prev3DayMaxWindSpeed10m7InPast] AS n_r_Prev3DayMax10mWind7InPast,
* [n_r_Prev3DayMinTemp7InPast] AS n_r_Prev3dayMinTemp7InPast,
* [n_r_Prev7DayMaxTemp7InPast] AS n_r_Prev7dayMaxTemp7InPast,
* [n_r_Prev7DayMaxWindSpeed10m7InPast] AS n_r_Prev7DayMax10mWind7InPast,
*
* [n_r_Prev7DayMinTemp7InPast] AS n_r_Prev7dayMinTemp7InPast,
*
* [n_r_Prev1DayMaxTemp7InPast] AS n_r_Prev1dayMaxTemp7InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m7InPast] AS n_r_Prev1DayMax10mWind7InPast,
*
* [n_r_Prev1DayMinTemp7InPast] AS n_r_Prev1dayMinTemp7InPast,
*
* [n_r_Prev1DayPrecip7InPast] AS n_r_Prev1DayPrecip7InPast,
* c_r_Prev3DayFreezeThawLikeliness7InPast,
* c_r_Prev7DayFreezeThawLikeliness7InPast,
*
* [c_r_Prev3DayWindSlabLikeliness7InPast] AS c_r_Prev3DayWindSlabLikeliness7InPast,
*
* [c_r_Prev7DayWindSlabLikeliness7InPast] AS c_r_Prev7DayWindSlabLikeliness7InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn7InPast,
* n_r_SNOWDAS_SnowDepth_mm7InPast,
* n_r_SNOWDAS_SWE_mm7InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm7InPast,
* n_r_SNOWDAS_Sublimation_micromm7InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem7InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem7InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k7InPast,
* n_r_SnowDepthIn8InPast AS n_r_snowDepthIn8InPast,
* n_r_PrecipincrementSnowIn8InPast AS n_r_precipIncrementSnowIn8InPast,
* n_r_Prev3DaySnowAccumulation8InPast AS n_r_Prev3daySnowAccumulation8InPast,
* [n_r_Prev7DaySnowAccumulation8InPast] AS n_r_Prev7daySnowAccumulation8InPast,
* [n_r_Prev3DayMaxTemp8InPast] AS n_r_Prev3dayMaxTemp8InPast,
* [n_r_Prev3DayMaxWindSpeed10m8InPast] AS n_r_Prev3DayMax10mWind8InPast,
* [n_r_Prev3DayMinTemp8InPast] AS n_r_Prev3dayMinTemp8InPast,
* [n_r_Prev7DayMaxTemp8InPast] AS n_r_Prev7dayMaxTemp8InPast,
* [n_r_Prev7DayMaxWindSpeed10m8InPast] AS n_r_Prev7DayMax10mWind8InPast,
*
* [n_r_Prev7DayMinTemp8InPast] AS n_r_Prev7dayMinTemp8InPast,
*
* [n_r_Prev1DayMaxTemp8InPast] AS n_r_Prev1dayMaxTemp8InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m8InPast] AS n_r_Prev1DayMax10mWind8InPast,
*
* [n_r_Prev1DayMinTemp8InPast] AS n_r_Prev1dayMinTemp8InPast,
*
* [n_r_Prev1DayPrecip8InPast] AS n_r_Prev1DayPrecip8InPast,
* c_r_Prev3DayFreezeThawLikeliness8InPast,
* c_r_Prev7DayFreezeThawLikeliness8InPast,
*
* [c_r_Prev3DayWindSlabLikeliness8InPast] AS c_r_Prev3DayWindSlabLikeliness8InPast,
*
* [c_r_Prev7DayWindSlabLikeliness8InPast] AS c_r_Prev7DayWindSlabLikeliness8InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn8InPast,
* n_r_SNOWDAS_SnowDepth_mm8InPast,
* n_r_SNOWDAS_SWE_mm8InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm8InPast,
* n_r_SNOWDAS_Sublimation_micromm8InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem8InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem8InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k8InPast,
* n_r_SnowDepthIn9InPast AS n_r_snowDepthIn9InPast,
* n_r_PrecipincrementSnowIn9InPast AS n_r_precipIncrementSnowIn9InPast,
* n_r_Prev3DaySnowAccumulation9InPast AS n_r_Prev3daySnowAccumulation9InPast,
* [n_r_Prev7DaySnowAccumulation9InPast] AS n_r_Prev7daySnowAccumulation9InPast,
* [n_r_Prev3DayMaxTemp9InPast] AS n_r_Prev3dayMaxTemp9InPast,
* [n_r_Prev3DayMaxWindSpeed10m9InPast] AS n_r_Prev3DayMax10mWind9InPast,
* [n_r_Prev3DayMinTemp9InPast] AS n_r_Prev3dayMinTemp9InPast,
* [n_r_Prev7DayMaxTemp9InPast] AS n_r_Prev7dayMaxTemp9InPast,
* [n_r_Prev7DayMaxWindSpeed10m9InPast] AS n_r_Prev7DayMax10mWind9InPast,
*
* [n_r_Prev7DayMinTemp9InPast] AS n_r_Prev7dayMinTemp9InPast,
*
* [n_r_Prev1DayMaxTemp9InPast] AS n_r_Prev1dayMaxTemp9InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m9InPast] AS n_r_Prev1DayMax10mWind9InPast,
*
* [n_r_Prev1DayMinTemp9InPast] AS n_r_Prev1dayMinTemp9InPast,
*
* [n_r_Prev1DayPrecip9InPast] AS n_r_Prev1DayPrecip9InPast,
* c_r_Prev3DayFreezeThawLikeliness9InPast,
* c_r_Prev7DayFreezeThawLikeliness9InPast,
*
* [c_r_Prev3DayWindSlabLikeliness9InPast] AS c_r_Prev3DayWindSlabLikeliness9InPast,
*
* [c_r_Prev7DayWindSlabLikeliness9InPast] AS c_r_Prev7DayWindSlabLikeliness9InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn9InPast,
* n_r_SNOWDAS_SnowDepth_mm9InPast,
* n_r_SNOWDAS_SWE_mm9InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm9InPast,
* n_r_SNOWDAS_Sublimation_micromm9InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem9InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem9InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k9InPast,
* n_r_SnowDepthIn10InPast AS n_r_snowDepthIn10InPast,
* n_r_PrecipincrementSnowIn10InPast AS n_r_precipIncrementSnowIn10InPast,
* n_r_Prev3DaySnowAccumulation10InPast AS n_r_Prev3daySnowAccumulation10InPast,
* [n_r_Prev7DaySnowAccumulation10InPast] AS n_r_Prev7daySnowAccumulation10InPast,
* [n_r_Prev3DayMaxTemp10InPast] AS n_r_Prev3dayMaxTemp10InPast,
* [n_r_Prev3DayMaxWindSpeed10m10InPast] AS n_r_Prev3DayMax10mWind10InPast,
* [n_r_Prev3DayMinTemp10InPast] AS n_r_Prev3dayMinTemp10InPast,
* [n_r_Prev7DayMaxTemp10InPast] AS n_r_Prev7dayMaxTemp10InPast,
* [n_r_Prev7DayMaxWindSpeed10m10InPast] AS n_r_Prev7DayMax10mWind10InPast,
*
* [n_r_Prev7DayMinTemp10InPast] AS n_r_Prev7dayMinTemp10InPast,
*
* [n_r_Prev1DayMaxTemp10InPast] AS n_r_Prev1dayMaxTemp10InPast,
*
* [n_r_Prev1DayMaxWindSpeed10m10InPast] AS n_r_Prev1DayMax10mWind10InPast,
*
* [n_r_Prev1DayMinTemp10InPast] AS n_r_Prev1dayMinTemp10InPast,
*
* [n_r_Prev1DayPrecip10InPast] AS n_r_Prev1DayPrecip10InPast,
* c_r_Prev3DayFreezeThawLikeliness10InPast,
* c_r_Prev7DayFreezeThawLikeliness10InPast,
*
* [c_r_Prev3DayWindSlabLikeliness10InPast] AS c_r_Prev3DayWindSlabLikeliness10InPast,
*
* [c_r_Prev7DayWindSlabLikeliness10InPast] AS c_r_Prev7DayWindSlabLikeliness10InPast,
* n_r_Prev24HoursPrecipAsRainTotalIn10InPast,
* n_r_SNOWDAS_SnowDepth_mm10InPast,
* n_r_SNOWDAS_SWE_mm10InPast,
* n_r_SNOWDAS_SnowmeltRunoff_micromm10InPast,
* n_r_SNOWDAS_Sublimation_micromm10InPast,
* n_r_SNOWDAS_SolidPrecip_kgpersquarem10InPast,
* n_r_SNOWDAS_LiquidPrecip_kgpersquarem10InPast,
* n_r_SNOWDAS_SnowpackAveTemp_k10InPast
*/
}
/// <summary>
/// Will reduce a malformed set to one with distinct filehour, lat, lon
/// should reduce on filehour, lat and lon
/// </summary>
/// <param name="input"></param>
/// <param name="output"></param>
/// <returns></returns>
public override IEnumerable <IRow> Reduce(IRowset input, IUpdatableRow output)
{
var rows = (from r in input.Rows
select new NamRow
{
//ParsedHour = DateTime.ParseExact(r.Get<string>("DateString"), "yyyyMMdd HH:00", null).Hour,
DateString = r.Get <string>("DateString"),
Lat = r.Get <double>("Lat"),
Lon = r.Get <double>("Lon"),
APCPsurface = r.Get <double?>("APCPsurface"),
APCPStepSize = r.Get <int?>("APCPStepSize"),
CSNOWsurface = r.Get <int?>("CSNOWsurface"),
CRAINsurface = r.Get <int?>("CRAINsurface"),
TMPsurface = r.Get <double?>("TMPsurface"),
Tmp2mAboveGround = r.Get <double?>("Tmp2mAboveGround"),
RH2mAboveGround = r.Get <double?>("RH2mAboveGround"),
TMP80mAboveGround = r.Get <double?>("TMP80mAboveGround"),
TMPTrop = r.Get <double?>("TMPTrop"),
WindSpeed10m = r.Get <double?>("WindSpeed10m"),
WindDirection10m = r.Get <double?>("WindDirection10m"),
WindSpeed80m = r.Get <double?>("WindSpeed80m"),
WindDirection80m = r.Get <double?>("WindDirection80m"),
WindSpeedTrop = r.Get <double?>("WindSpeedTrop"),
WindDirectionTrop = r.Get <double?>("WindDirectionTrop"),
__fileHour = r.Get <int>("__fileHour")
}).ToList();
NamRow r1 = null;
NamRow r2 = null;
if (rows.Count() == 1)
{
r1 = rows.First();
r2 = rows.First();
}
else if (rows.Count() == 2)
{
r1 = rows.Where(rtmp => rtmp.ParsedHour == 0).First();
r2 = rows.Where(rtmp => rtmp.ParsedHour != 0).First();
}
else
{
throw new ArgumentException(string.Format("Unexpected number of rows, got {0} but expected 1 or 2", rows.Count()));
}
//in the case we are attempting to fix there should be two row per lat/lon
//first get the date/lat/lon and TMPSurface from row with correct hour
output.Set <string>("DateString", r1.DateString);
output.Set <double>("Lat", r1.Lat);
output.Set <double>("Lon", r1.Lon);
output.Set <double?>("TMPsurface", r1.TMPsurface);
output.Set <double?>("APCPsurface", r2.APCPsurface);
output.Set <int?>("APCPStepSize", r2.APCPStepSize);
output.Set <int?>("CSNOWsurface", r2.CSNOWsurface);
output.Set <int?>("CRAINsurface", r2.CRAINsurface);
output.Set <double?>("Tmp2mAboveGround", r2.Tmp2mAboveGround);
output.Set <double?>("RH2mAboveGround", r2.RH2mAboveGround);
output.Set <double?>("TMP80mAboveGround", r2.TMP80mAboveGround);
output.Set <double?>("TMPTrop", r2.TMPTrop);
output.Set <double?>("WindSpeed10m", r2.WindSpeed10m);
output.Set <double?>("WindDirection10m", r2.WindDirection10m);
output.Set <double?>("WindSpeed80m", r2.WindSpeed80m);
output.Set <double?>("WindDirection80m", r2.WindDirection80m);
output.Set <double?>("WindSpeedTrop", r2.WindSpeedTrop);
output.Set <double?>("WindDirectionTrop", r2.WindDirectionTrop);
output.Set <int>("__fileHour", r2.__fileHour);
yield return(output.AsReadOnly());
}
