/// <summary>
/// Generates completion options for a series.
/// </summary>
/// <param name="text"></param>
/// <param name="offset"></param>
/// <param name="tableName"></param>
/// <param name="storage"></param>
/// <returns></returns>
public bool GenerateSeriesCompletions(string text, int offset, string tableName, IStorageReader storage)
{
triggerWord = text?.Substring(0, offset).Split(' ').Last().Replace("[", "").Replace("]", "");
List <string> columnNames = storage.ColumnNames(tableName).ToList();
List <NeedContextItemsArgs.ContextItem> intellisenseOptions = new List <NeedContextItemsArgs.ContextItem>();
foreach (string columnName in columnNames)
{
if (string.IsNullOrEmpty(triggerWord) || string.IsNullOrEmpty(triggerWord.Replace("[", "").Replace("]", "")) || columnName.StartsWith(triggerWord.Replace("[", "").Replace("]", "")))
{
intellisenseOptions.Add(new NeedContextItemsArgs.ContextItem()
{
Name = columnName,
Units = string.Empty,
TypeName = string.Empty,
Descr = string.Empty,
ParamString = string.Empty
});
}
}
if (intellisenseOptions.Any())
{
view.Populate(intellisenseOptions);
}
return(intellisenseOptions.Any());
}
/// <summary>
/// Create series definitions assuming the vary by fields are text fields in the table.
/// </summary>
/// <param name="reader">The reader to read from.</param>
/// <param name="varyByFieldNames">The vary by fields.</param>
/// <param name="whereClauseForInScopeData">An SQL WHERE clause for rows that are in scope.</param>
private List <SeriesDefinition> CreateDefinitionsFromFieldInTable(IStorageReader reader, List <string> varyByFieldNames, string whereClauseForInScopeData)
{
List <SeriesDefinition> definitions = new List <SeriesDefinition>();
var fieldsThatExist = reader.ColumnNames(TableName);
var varyByThatExistInTable = varyByFieldNames.Where(v => fieldsThatExist.Contains(v)).ToList();
var validValuesForEachVaryByField = new List <List <string> >();
foreach (var varyByFieldName in varyByThatExistInTable)
{
var data = reader.GetData(TableName,
fieldNames: new string[] { varyByFieldName },
filter: whereClauseForInScopeData,
distinct: true);
var values = DataTableUtilities.GetColumnAsStrings(data, varyByFieldName).Distinct().ToList();
validValuesForEachVaryByField.Add(values);
}
foreach (var combination in MathUtilities.AllCombinationsOf(validValuesForEachVaryByField.ToArray(), reverse:true))
{
var descriptors = new List <SimulationDescription.Descriptor>();
for (int i = 0; i < combination.Count; i++)
{
descriptors.Add(new SimulationDescription.Descriptor(varyByThatExistInTable[i],
combination[i]));
}
definitions.Add(new SeriesDefinition(this, whereClauseForInScopeData, Filter, descriptors));
}
return(definitions);
}
/// <summary>
/// Create a data view from the specified table and filter.
/// </summary>
/// <param name="simulationZones">The list of simulation / zone pairs.</param>
/// <param name="storage">Storage service</param>
private DataTable GetBaseData(IStorageReader storage, List <SimulationZone> simulationZones)
{
// Get a list of all simulation names in all simulationZones.
List <string> simulationNames = new List <string>();
simulationZones.ForEach(sim => simulationNames.AddRange(sim.simulationNames));
string filter = null;
foreach (string simulationName in simulationNames.Distinct())
{
if (filter != null)
{
filter += ",";
}
filter += "'" + simulationName + "'";
}
filter = "SimulationName in (" + filter + ")";
if (Filter != string.Empty)
{
filter = AddToFilter(filter, Filter);
}
List <string> fieldNames = new List <string>();
if (storage.ColumnNames(TableName).Contains("Zone"))
{
fieldNames.Add("Zone");
}
if (XFieldName != null && !XFieldName.Equals("SimulationName"))
{
fieldNames.Add(XFieldName);
}
if (YFieldName != null && !fieldNames.Contains(YFieldName))
{
fieldNames.Add(YFieldName);
}
if (X2FieldName != null && !fieldNames.Contains(X2FieldName))
{
fieldNames.Add(X2FieldName);
}
if (Y2FieldName != null && !fieldNames.Contains(Y2FieldName))
{
fieldNames.Add(Y2FieldName);
}
// Add in column names from annotation series.
foreach (EventNamesOnGraph annotation in Apsim.Children(this, typeof(EventNamesOnGraph)))
{
fieldNames.Add(annotation.ColumnName);
}
return(storage.GetData(tableName: TableName, fieldNames: fieldNames, filter: filter));
}
/// <summary>
/// Create a data view from the specified table and filter.
/// </summary>
/// <param name="factors">The list of simulation / zone pairs.</param>
/// <param name="storage">Storage service</param>
private DataTable GetBaseData(IStorageReader storage, List <ISimulationGeneratorFactors> factors)
{
List <string> fieldNames = new List <string>();
foreach (ISimulationGeneratorFactors factor in factors)
{
fieldNames.Add(factor.ColumnName);
}
if (XFieldName != null)
{
fieldNames.Add(XFieldName);
}
if (YFieldName != null)
{
fieldNames.Add(YFieldName);
}
if (YFieldName != null)
{
if (storage.ColumnNames(TableName).Contains(YFieldName + "Error"))
{
fieldNames.Add(YFieldName + "Error");
}
}
if (X2FieldName != null)
{
fieldNames.Add(X2FieldName);
}
if (Y2FieldName != null)
{
fieldNames.Add(Y2FieldName);
}
// Add in column names from annotation series.
foreach (EventNamesOnGraph annotation in Apsim.Children(this, typeof(EventNamesOnGraph)))
{
fieldNames.Add(annotation.ColumnName);
}
string filterToUse;
if (Filter == null || Filter == string.Empty)
{
filterToUse = CreateRowFilter(factors);
}
else
{
filterToUse = Filter + " AND (" + CreateRowFilter(factors) + ")";
}
return(storage.GetData(tableName: TableName, checkpointName: Checkpoint, fieldNames: fieldNames.Distinct(), filter: filterToUse));
}
/// <summary>
/// Create an SQL WHERE clause for rows that are in scope.
/// </summary>
/// <param name="reader">The reader to read from.</param>
/// <param name="simulationFilter">The names of simulatiosn that are in scope.</param>
private IEnumerable <string> CreateInScopeWhereClause(IStorageReader reader, List <string> simulationFilter)
{
var fieldsThatExist = reader.ColumnNames(TableName);
if (fieldsThatExist.Contains("SimulationID") || fieldsThatExist.Contains("SimulationName"))
{
// Extract all the simulation names from all descriptions.
return(simulationFilter.Distinct());
}
else
{
return(null);
}
}
/// <summary>
/// Create an SQL WHERE clause for rows that are in scope.
/// </summary>
/// <param name="reader">The reader to read from.</param>
/// <param name="simulationFilter">The names of simulatiosn that are in scope.</param>
private string CreateInScopeWhereClause(IStorageReader reader, List<string> simulationFilter)
{
var fieldsThatExist = reader.ColumnNames(TableName);
if (fieldsThatExist.Contains("SimulationID") || fieldsThatExist.Contains("SimulationName"))
{
// Extract all the simulation names from all descriptions.
var simulationNames = simulationFilter.Distinct();
string whereClause = "SimulationName IN (" +
StringUtilities.Build(simulationNames, ",", "'", "'") +
")";
return whereClause;
}
else if (Filter != string.Empty)
return Filter;
else
return null;
}
/// <summary>Populates the field names in the view.</summary>
private void PopulateFieldNames()
{
Graph parentGraph = series.Parent as Graph;
if (this.seriesView.DataSource != null &&
this.seriesView.DataSource.SelectedValue != string.Empty &&
this.seriesView.DataSource.SelectedValue != null &&
parentGraph != null)
{
List <string> fieldNames = new List <string>();
fieldNames.Add("SimulationName");
fieldNames.AddRange(storage.ColumnNames(seriesView.DataSource.SelectedValue));
fieldNames.Sort();
this.seriesView.X.Values = fieldNames.ToArray();
this.seriesView.Y.Values = fieldNames.ToArray();
this.seriesView.X2.Values = fieldNames.ToArray();
this.seriesView.Y2.Values = fieldNames.ToArray();
}
}
/// <summary>
/// Create an SQL WHERE clause for rows that are in scope.
/// </summary>
/// <param name="reader">The reader to read from.</param>
/// <param name="simulationDescriptions">The simulation descriptions that are in scope.</param>
private string CreateInScopeWhereClause(IStorageReader reader, IEnumerable <SimulationDescription> simulationDescriptions)
{
var fieldsThatExist = reader.ColumnNames(TableName);
if (fieldsThatExist.Contains("SimulationID") || fieldsThatExist.Contains("SimulationName"))
{
// Extract all the simulation names from all descriptions.
var simulationNames = simulationDescriptions.Select(d => d.Name).Distinct();
string whereClause = "SimulationName IN (" +
StringUtilities.Build(simulationNames, ",", "'", "'") +
")";
return(whereClause);
}
else if (Filter != string.Empty)
{
return(Filter);
}
else
{
return(null);
}
}
/// <summary>Reads all data from the specified reader.</summary>
/// <param name="reader">Storage reader.</param>
/// <param name="simulationDescriptions">Complete list of simulation descriptions.</param>
public void ReadData(IStorageReader reader, List <SimulationDescription> simulationDescriptions)
{
if (X != null && Y != null)
{
return;
}
if (series.TableName == null)
{
if (!String.IsNullOrEmpty(XFieldName))
{
X = GetDataFromModels(XFieldName);
}
if (!String.IsNullOrEmpty(YFieldName))
{
Y = GetDataFromModels(YFieldName);
}
if (!String.IsNullOrEmpty(X2FieldName))
{
X2 = GetDataFromModels(X2FieldName);
}
if (!String.IsNullOrEmpty(Y2FieldName))
{
Y2 = GetDataFromModels(Y2FieldName);
}
}
else
{
var fieldsThatExist = reader.ColumnNames(series.TableName);
// If we have descriptors, then use them to filter the data for this series.
string filter = null;
if (Descriptors != null)
{
foreach (var descriptor in Descriptors)
{
if (fieldsThatExist.Contains(descriptor.Name))
{
filter = AddToFilter(filter, descriptor.Name + " = '" + descriptor.Value + "'");
}
else
{
filter = AddSimulationNameClauseToFilter(filter, descriptor, simulationDescriptions);
}
}
// Incorporate our scope filter if we haven't limited filter to particular simulations.
if (!filter.Contains("SimulationName IN"))
{
filter = AddToFilter(filter, scopeFilter);
}
}
else
{
filter = AddToFilter(filter, scopeFilter);
}
if (!string.IsNullOrEmpty(userFilter))
{
filter = AddToFilter(filter, userFilter);
}
// Get a list of fields to read from data store.
var fieldsToRead = new List <string>();
fieldsToRead.Add(XFieldName);
fieldsToRead.Add(YFieldName);
if (X2FieldName != null)
{
fieldsToRead.Add(X2FieldName);
}
if (Y2FieldName != null)
{
fieldsToRead.Add(Y2FieldName);
}
// Add any error fields to the list of fields to read.
var fieldsToAdd = new List <string>();
foreach (var fieldName in fieldsToRead)
{
if (fieldsThatExist.Contains(fieldName + "Error"))
{
fieldsToAdd.Add(fieldName + "Error");
}
}
fieldsToRead.AddRange(fieldsToAdd);
// Add any field names from the filter.
fieldsToRead.AddRange(ExtractFieldNamesFromFilter(filter));
// Add any fields from child graphable models.
foreach (IGraphable series in Apsim.Children(series, typeof(IGraphable)))
{
fieldsToRead.AddRange(series.GetExtraFieldsToRead(this));
}
// Checkpoints don't exist in observed files so don't pass a checkpoint name to
// GetData in this situation.
string localCheckpointName = CheckpointName;
if (!reader.ColumnNames(series.TableName).Contains("CheckpointID"))
{
localCheckpointName = null;
}
// Go get the data.
Data = reader.GetData(series.TableName, localCheckpointName, fieldNames: fieldsToRead.Distinct(), filter: filter);
// Get the units for our x and y variables.
XFieldUnits = reader.Units(series.TableName, XFieldName);
YFieldUnits = reader.Units(series.TableName, YFieldName);
// If data was found, populate our data (e.g. X and Y) properties.
if (Data.Rows.Count > 0)
{
X = GetDataFromTable(Data, XFieldName);
Y = GetDataFromTable(Data, YFieldName);
X2 = GetDataFromTable(Data, X2FieldName);
Y2 = GetDataFromTable(Data, Y2FieldName);
Error = GetErrorDataFromTable(Data, YFieldName);
if (series.Cumulative)
{
Y = MathUtilities.Cumulative(Y as IEnumerable <double>);
}
if (series.CumulativeX)
{
X = MathUtilities.Cumulative(X as IEnumerable <double>);
}
}
}
}
/// <summary>
/// Create a datatable that covers a collection of series definitions.
/// </summary>
/// <param name="storage">A data store reader.</param>
/// <param name="series">A list of series definitions.</param>
/// <param name="simulationDescriptions">A list of simulation descriptions.</param>
/// <returns></returns>
private static void ReadAllData(IStorageReader storage, IEnumerable <SeriesDefinition> series,
List <SimulationDescription> simulationDescriptions)
{
var definitionsToProcess = series.ToList();
// Remove all series that already have data. I'm not sure
// under what circumstance this would happen.
definitionsToProcess.RemoveAll(d => d.X != null && d.Y != null);
// Process and remove all series that have a table name specified.
var definitionsWithNoTable = definitionsToProcess.Where(d => d.Series.TableName == null);
foreach (var d in definitionsWithNoTable)
{
d.GetDataFromModels();
}
definitionsToProcess.RemoveAll(d => definitionsWithNoTable.Contains(d));
var allTableNames = definitionsToProcess.Select(d => d.Series.TableName)
.Distinct();
// Get a list of inscope simulation names.
var allSimulationNamesInScope = new List <string>();
foreach (var d in definitionsToProcess)
{
if (d.InScopeSimulationNames != null)
{
allSimulationNamesInScope.AddRange(d.InScopeSimulationNames);
}
}
var inScopeSimulationNames = allSimulationNamesInScope.Distinct();
if (!inScopeSimulationNames.Any())
{
inScopeSimulationNames = null;
}
foreach (var tableName in allTableNames)
{
var definitionsUsingThisTable = definitionsToProcess.Where(d => d.Series.TableName == tableName);
var checkpointNames = definitionsUsingThisTable.Select(d => d.CheckpointName)
.Distinct();
var fieldsThatExist = storage.ColumnNames(tableName);
var fieldNames = definitionsUsingThisTable.SelectMany(d => d.GetFieldNames(fieldsThatExist))
.Distinct();
// Only attempt to read simulation names if this table actually contains
// a simulation name or ID column. Some tables (ie observed data from excel)
// don't necessarily have these columns.
IEnumerable <string> simulationNames = fieldsThatExist.Contains("SimulationID") || fieldsThatExist.Contains("SimulationName") ? inScopeSimulationNames : null;
foreach (var checkpointName in checkpointNames)
{
var table = storage.GetData(tableName, checkpointName, simulationNames, fieldNames);
// Tell each series definition to read its data.
var definitions = definitionsToProcess.Where(d => d.Series.TableName == tableName && d.CheckpointName == checkpointName);
Parallel.ForEach(definitions, (definition) =>
definition.ReadData(table, simulationDescriptions, storage));
}
}
}
