// Given a table, return a Set of all the distinct input values. This is for tables that have a single INPUT column.
// @param tableId table identifier
// @return A set of unique inputs
private HashSet <String> getAllInputValues(String tableId)
{
HashSet <String> values = new HashSet <String>();
// if the table is not found, return right away with an empty list
//StagingTable table = (getTable(tableId) as StagingTable);
StagingTable table = (StagingTable)(getTable(tableId));
if (table == null)
{
return(values);
}
// find the input key
HashSet <String> inputKeys = new HashSet <String>();
foreach (StagingColumnDefinition col in table.getColumnDefinitions())
{
if (col.getType() == ColumnType.INPUT)
{
inputKeys.Add(col.getKey());
}
}
if (inputKeys.Count != 1)
{
throw new System.InvalidOperationException("Table '" + table.getId() + "' must have one and only one INPUT column.");
}
HashSet <String> .Enumerator enumInput = inputKeys.GetEnumerator();
enumInput.MoveNext();
String inputKey = enumInput.Current;
foreach (StagingTableRow row in table.getTableRows())
{
foreach (StagingRange range in row.getColumnInput(inputKey))
{
if (range.getLow() != null)
{
if (range.getLow() == range.getHigh())
{
values.Add(range.getLow());
}
else
{
int low = Convert.ToInt32(range.getLow());
int high = Convert.ToInt32(range.getHigh());
// add all values in range
for (int i = low; i <= high; i++)
{
values.Add(padStart(i.ToString(), range.getLow().Length, '0'));
}
}
}
}
}
return(values);
}
/**
* Add a table
*/
public void addTable(StagingTable table)
{
initTable(table);
_tables[table.getId()] = table;
foreach (KeyValuePair <string, StagingTable> entry in _tables)
{
_TableKeys.Add(entry.Key);
}
}
// Constructor loads all schemas and sets up table cache
// @param algorithm algorithm
// @param version version
protected StagingFileDataProvider(String algorithm, String version) : base()
{
_algorithm = algorithm;
_version = version;
String basedir = System.IO.Directory.GetCurrentDirectory() + "\\";
if (!Directory.Exists(basedir + "Algorithms\\"))
{
basedir = System.IO.Directory.GetCurrentDirectory() + "\\..\\..\\..\\";
if (System.IO.Directory.GetCurrentDirectory().IndexOf("x64") >= 0)
{
basedir += "\\..\\";
}
basedir += "Resources\\";
}
String directory = "";
// loop over all tables and load them into Map
try
{
directory = basedir + "Algorithms\\" + algorithm.ToLower() + "\\" + version + "\\tables";
foreach (String file in readLines(directory + "\\ids.txt"))
{
if (file.Length != 0)
{
TextReader reader = getStagingInputStream(directory + "\\" + file + ".json");
StagingTable table = new StagingTable();
using (reader)
{
Newtonsoft.Json.JsonSerializer serializer = new Newtonsoft.Json.JsonSerializer();
table = (StagingTable)serializer.Deserialize(reader, typeof(StagingTable));
}
initTable(table);
_tables[table.getId()] = table;
}
}
}
catch (IOException e)
{
throw new System.InvalidOperationException("IOException reading tables: " + e.Message);
}
// loop over all schemas and load them into Map
try
{
directory = basedir + "Algorithms\\" + algorithm.ToLower() + "\\" + version + "\\schemas";
foreach (String file in readLines(directory + "\\ids.txt"))
{
if (file.Length != 0)
{
TextReader reader = getStagingInputStream(directory + "\\" + file + ".json");
StagingSchema schema = new StagingSchema();
using (reader)
{
Newtonsoft.Json.JsonSerializer serializer = new Newtonsoft.Json.JsonSerializer();
schema = (StagingSchema)serializer.Deserialize(reader, typeof(StagingSchema));
}
initSchema(schema);
_schemas[schema.getId()] = schema;
}
}
}
catch (IOException e)
{
throw new System.InvalidOperationException("IOException reading schemas: " + e.Message);
}
GenerateSchemaIds();
GenerateTableIds();
// finally, initialize any caches now that everything else has been set up
invalidateCache();
}
public void testInputTables()
{
HashSet <String> errors = new HashSet <String>();
foreach (String schemaId in _STAGING.getSchemaIds())
{
StagingSchema schema = _STAGING.getSchema(schemaId);
// build a list of input tables that should be excluded
foreach (StagingSchemaInput input in schema.getInputs())
{
if (input.getTable() != null)
{
HashSet <String> inputKeys = new HashSet <String>();
StagingTable table = _STAGING.getTable(input.getTable());
foreach (StagingColumnDefinition def in table.getColumnDefinitions())
{
if (ColumnType.INPUT == def.getType())
{
inputKeys.Add(def.getKey());
}
}
// make sure the input key matches the an input column
if (!inputKeys.Contains(input.getKey()))
{
errors.Add("Input key " + schemaId + ":" + input.getKey() + " does not match validation table " + table.getId() + ": " + inputKeys.ToString());
}
}
}
}
assertNoErrors(errors, "input values and their assocated validation tables");
}
// Initialize data provider
private void init(Stream inStream)
{
HashSet <String> algorithms = new HashSet <String>();
HashSet <String> versions = new HashSet <String>();
using (ZipArchive archive = new ZipArchive(inStream, ZipArchiveMode.Read))
{
foreach (ZipArchiveEntry entry in archive.Entries)
{
if ((entry.Name.Length == 0) || (!entry.Name.EndsWith(".json")))
{
continue;
}
if (entry.FullName.StartsWith("tables"))
{
String s = extractEntry(entry);
StagingTable table = new StagingTable();
table = Newtonsoft.Json.JsonConvert.DeserializeObject <StagingTable>(s);
if (DebugSettings.DEBUG_LOADED_TABLES)
{
Debug.WriteLine("Table: ");
Debug.WriteLine(table.GetDebugString(" "));
}
initTable(table);
algorithms.Add(table.getAlgorithm());
versions.Add(table.getVersion());
_tables[table.getId()] = table;
}
else if (entry.FullName.StartsWith("schemas"))
{
String s = extractEntry(entry);
StagingSchema schema = new StagingSchema();
schema = Newtonsoft.Json.JsonConvert.DeserializeObject <StagingSchema>(s);
if (DebugSettings.DEBUG_LOADED_SCHEMAS)
{
Debug.WriteLine("Schema: ");
Debug.WriteLine(schema.GetDebugString(" "));
}
initSchema(schema);
algorithms.Add(schema.getAlgorithm());
versions.Add(schema.getVersion());
_schemas[schema.getId()] = schema;
}
}
}
// verify that all the algorithm names and versions are consistent
if (algorithms.Count != 1)
{
throw new System.InvalidOperationException("Error initializing provider; only a single algorithm should be included in file");
}
if (versions.Count != 1)
{
throw new System.InvalidOperationException("Error initializing provider; only a single version should be included in file");
}
HashSet <String> .Enumerator enumAlg = algorithms.GetEnumerator();
HashSet <String> .Enumerator enumVer = versions.GetEnumerator();
enumAlg.MoveNext();
enumVer.MoveNext();
_algorithm = enumAlg.Current;
_version = enumVer.Current;
GenerateSchemaIds();
GenerateTableIds();
// finally, initialize any caches now that everything else has been set up
invalidateCache();
}
// Initialize a table.
// @param table table entity
// @return initialized table entity
public static StagingTable initTable(StagingTable table)
{
HashSet <String> extraInputs = new HashSet <String>();
List <List <String> > pTableRawRows = table.getRawRows();
// empty out the parsed rows
List <ITableRow> newTableRows = new List <ITableRow>();
if (pTableRawRows != null)
{
newTableRows.Capacity = pTableRawRows.Count;
}
table.setTableRows(newTableRows);
if (pTableRawRows != null)
{
foreach (List <String> row in pTableRawRows)
{
StagingTableRow tableRowEntity = new StagingTableRow();
// make sure the number of cells in the row matches the number of columns defined
if (table.getColumnDefinitions().Count != row.Count)
{
throw new System.InvalidOperationException("Table '" + table.getId() + "' has a row with " + row.Count + " values but should have " + table.getColumnDefinitions().Count + ": " + row);
}
// loop over the column definitions in order since the data needs to be retrieved by array position
for (int i = 0; i < table.getColumnDefinitions().Count; i++)
{
StagingColumnDefinition col = (StagingColumnDefinition)(table.getColumnDefinitions()[i]);
String cellValue = row[i];
switch (col.getType())
{
case ColumnType.INPUT:
// if there are no ranges in the list, that means the cell was "blank" and is not needed in the table row
List <Range> ranges = splitValues(cellValue);
if (!(ranges.Count == 0))
{
tableRowEntity.addInput(col.getKey(), ranges);
// if there are key references used (values that reference other inputs) like {{key}}, then add them to the extra inputs list
foreach (StagingRange range in ranges)
{
if (DecisionEngineFuncs.isReferenceVariable(range.getLow()))
{
extraInputs.Add(DecisionEngineFuncs.trimBraces(range.getLow()));
}
if (DecisionEngineFuncs.isReferenceVariable(range.getHigh()))
{
extraInputs.Add(DecisionEngineFuncs.trimBraces(range.getHigh()));
}
}
}
break;
case ColumnType.ENDPOINT:
StagingEndpoint endpoint = parseEndpoint(cellValue);
endpoint.setResultKey(col.getKey());
tableRowEntity.addEndpoint(endpoint);
// if there are key references used (values that reference other inputs) like {{key}}, then add them to the extra inputs list
if (EndpointType.VALUE == endpoint.getType() && DecisionEngineFuncs.isReferenceVariable(endpoint.getValue()))
{
extraInputs.Add(DecisionEngineFuncs.trimBraces(endpoint.getValue()));
}
break;
case ColumnType.DESCRIPTION:
// do nothing
break;
default:
throw new System.InvalidOperationException("Table '" + table.getId() + " has an unknown column type: '" + col.getType() + "'");
}
}
tableRowEntity.ConvertColumnInput();
newTableRows.Add(tableRowEntity);
}
}
// add extra inputs, if any; do not include context variables since they are not user input
foreach (String s in Staging.CONTEXT_KEYS)
{
extraInputs.Remove(s);
}
table.setExtraInput(extraInputs.Count == 0 ? null : extraInputs);
table.GenerateInputColumnDefinitions();
return(table);
}
