/// <summary>
///
/// </summary>
/// <param name="csvds">CSV data source</param>
/// <param name="filteringValue"> filtered value</param>
/// <param name="filteringAttribute"> filtering attribute </param>
/// <param name="color"></param>
/// <returns></returns>
View Facet(CSVDataSource csvds, string filteringValue, string filteringAttribute, Color color)
{
//B02598
//B02512
// filters the array on a particular value in another dimension
Filter baseFilter = (array, datasource, value, dimension) =>
{
return(array.Select((b, i) => new { index = i, _base = b })
.Where(b => datasource.getOriginalValuePrecise(csvds[dimension].Data[b.index], dimension).ToString() == value)
.Select(b => b._base).ToArray());
};
Filter identity = (ar, ds, fv, fa) => { return(ar); };
// baseFilter = identity;
var xData = baseFilter(csvds["Lat"].Data, csvds, filteringValue, filteringAttribute);
var yData = baseFilter(csvds["Lon"].Data, csvds, filteringValue, filteringAttribute);
var zData = baseFilter(csvds["Base"].Data, csvds, filteringValue, filteringAttribute);
ViewBuilder vb = new ViewBuilder(MeshTopology.Points, "Uber pick up point visualisation").
initialiseDataView(xData.Length).
setDataDimension(xData, ViewBuilder.VIEW_DIMENSION.X).
setDataDimension(yData, ViewBuilder.VIEW_DIMENSION.Y).
//setDataDimension(zData, ViewBuilder.VIEW_DIMENSION.Z).
setSize(baseFilter(csvds["Date"].Data, csvds, filteringValue, filteringAttribute)).
setColors(xData.Select(x => color).ToArray());
Material mt = IATKUtil.GetMaterialFromTopology(AbstractVisualisation.GeometryType.Points);
mt.SetFloat("_MinSize", 0.01f);
mt.SetFloat("_MaxSize", 0.05f);
return(vb.updateView().apply(gameObject, mt));
}
private void Awake()
{
if (dataSource == null)
{
dataSource = ChartManager.Instance.DataSource;
}
}
/// <summary>
/// Diese Methode färbt falls er bei der Klassifizierung fehelrhaft interpretiert worden ist ein.
/// </summary>
public void colorError(CSVDataSource dataSource)
{
int columnindex = 0;
int j = 0;
foreach (var column in dataSource)
{
if (column.Identifier.Equals(dataSource.categoryColumn))
{
columnindex = j;
}
// dann möchte ich den index und befülle diesen index dann unten
j++;
}
if (!(dataSource.results[index].Equals(dataSource.getOriginalValue(dataSource[columnindex].Data[index], dataSource[columnindex].Identifier).ToString())))
{
Renderer renderer = this.GetComponent <Renderer>();
if (dataSource.resultColor != null)
{
color = Utilities.HexToColor(dataSource.resultColor);
renderer.material.color = color;
}
else
{
color = Color.red;
renderer.material.color = color;
}
}
}
// a space time cube
View Uber(CSVDataSource csvds)
{
// header
// Date,Time,Lat,Lon,Base
Gradient g = new Gradient();
GradientColorKey[] gck = new GradientColorKey[2];
gck[0] = new GradientColorKey(Color.blue, 0);
gck[1] = new GradientColorKey(Color.red, 1);
g.colorKeys = gck;
// create a view builder with the point topology
ViewBuilder vb = new ViewBuilder(MeshTopology.Points, "Uber pick up point visualisation").
initialiseDataView(csvds.DataCount).
setDataDimension(csvds["Lat"].Data, ViewBuilder.VIEW_DIMENSION.X).
setDataDimension(csvds["Base"].Data, ViewBuilder.VIEW_DIMENSION.Y).
setSize(csvds["Base"].Data).
setColors(csvds["Time"].Data.Select(x => g.Evaluate(x)).ToArray());
// initialise the view builder wiith thhe number of data points and parent GameOBject
//Enumerable.Repeat(1f, dataSource[0].Data.Length).ToArray()
Material mt = IATKUtil.GetMaterialFromTopology(AbstractVisualisation.GeometryType.Points);
mt.SetFloat("_MinSize", 0.01f);
mt.SetFloat("_MaxSize", 0.05f);
View v = vb.updateView().apply(gameObject, mt);
return(v);
}
// Use this for initialization
void Start()
{
CSVDataSource csvds = createCSVDataSource(dataSource.text);
Gradient g = new Gradient();
GradientColorKey[] gck = new GradientColorKey[2];
gck[0] = new GradientColorKey(Color.blue, 0);
gck[1] = new GradientColorKey(Color.red, 1);
g.colorKeys = gck;
//stock,date,open
// create a view builder with the point topology
ViewBuilder vb = new ViewBuilder(MeshTopology.Lines, "Uber pick up point visualisation").
initialiseDataView(csvds.DataCount).
setDataDimension(csvds["stock"].Data, ViewBuilder.VIEW_DIMENSION.X).
setDataDimension(csvds["date"].Data, ViewBuilder.VIEW_DIMENSION.Y).
setDataDimension(csvds["open"].Data, ViewBuilder.VIEW_DIMENSION.Z).
setSize(csvds["volume"].Data).
setColors(csvds["volume"].Data.Select(x => g.Evaluate(x)).ToArray()).
createIndicesLinkedTopology(csvds["stock"].Data);
// create a view builder with the point topology
Material mt = IATKUtil.GetMaterialFromTopology(AbstractVisualisation.GeometryType.Lines);
//Material mt = new Material(Shader.Find("IATK/LinesShader"));
mt.mainTexture = Resources.Load("circle-outline-basic") as Texture2D;
mt.renderQueue = 3000;
mt.SetFloat("_MinSize", 0.01f);
mt.SetFloat("_MaxSize", 0.05f);
View view = vb.updateView().apply(gameObject, mt);
}
public override int NumberOfLabels()
{
if (IsDiscreet())
{
if (attributeFilter.minScale > 0.001f || attributeFilter.maxScale < 0.999f)
{
return(0);
}
CSVDataSource csvDataSource = (CSVDataSource)dataSource;
int numValues = csvDataSource.TextualDimensionsListReverse[attributeFilter.Attribute].Count;
if (numValues < (Mathf.CeilToInt(axisLength / 0.075f)))
{
return(numValues);
}
else
{
return(2);
}
}
else
{
return(Mathf.CeilToInt(axisLength / 0.075f));
}
}
public void CSVDataSourceReadStreamTest()
{
var dataStream = new FileStream(_csvDataFilePath, FileMode.Open);
var dataSource = new CSVDataSource(dataStream);
AssertData(dataSource);
}
public void CSVDataSourceReadTextReaderTest()
{
var textReader = File.OpenText(_csvDataFilePath);
var dataSource = new CSVDataSource(textReader);
AssertData(dataSource);
}
/// <summary>
/// Creates the Scatterplots in a subroutine, one per frame.
/// This was done to keep the programm from freezen when creating
/// the Scatterplots.
/// </summary>
/// <param name="scatterplotPrefab"></param>
/// <param name="dataSource"></param>
/// <param name="dimCombinations"></param>
/// <param name="matrixWidth"></param>
/// <returns></returns>
IEnumerator CreateScatterplotsCoroutine(GameObject scatterplotPrefab, CSVDataSource dataSource, int[,] dimCombinations, int matrixWidth)
{
for (int i = 0; dimCombinations.GetLength(0) > i; i++)
{
//Debug.Log("creations scatterplot" + i);
int matrixPosX = i % matrixWidth * 2;
int matrixPosZ = i / matrixWidth * 2;
int xDim = dimCombinations[i, 0];
int yDim = dimCombinations[i, 1];
int zDim = dimCombinations[i, 2];
Transform tempTransform = transform;
tempTransform.position += new Vector3(0f, 0.02f, 0f);
Scatterplot scatterplot = Instantiate(scatterplotPrefab, tempTransform).GetComponent <Scatterplot>();
scatterplot.Initialize(dataSource, matrixPosX, matrixPosZ, pointSize, xDim, yDim, zDim, bigScatterplot);
scatterplots[i] = scatterplot;
if (bigScatterplot)
{
scatterplot.transform.localScale += Vector3.one;
}
else
{
scatterplot.transform.localScale -= Vector3.one / 2;
}
yield return(null);
}
}
// a space time cube
View uber(CSVDataSource csvds)
{
// header
// Date,Time,Lat,Lon,Base
Gradient g = new Gradient();
GradientColorKey[] gck = new GradientColorKey[2];
gck[0] = new GradientColorKey(Color.blue, 0);
gck[1] = new GradientColorKey(Color.red, 1);
g.colorKeys = gck;
// create a view builder with the point topology
ViewBuilder vb = new ViewBuilder(MeshTopology.Points, "Uber pick up point visualisation").
initialiseDataView(csvds.DataCount).
setDataDimension(csvds["Lat"].Data, ViewBuilder.VIEW_DIMENSION.X).
setDataDimension(csvds["Base"].Data, ViewBuilder.VIEW_DIMENSION.Y).
setDataDimension(csvds["Lon"].Data, ViewBuilder.VIEW_DIMENSION.Z).
setSize(csvds["Base"].Data).
setColors(csvds["Time"].Data.Select(x => g.Evaluate(x)).ToArray());
// initialise the view builder wiith thhe number of data points and parent GameOBject
//Enumerable.Repeat(1f, dataSource[0].Data.Length).ToArray()
Material mt = new Material(Shader.Find("IATK/OutlineDots"));
//Material mt = new Material(Shader.Find("IATK/LinesShader"));
mt.mainTexture = Resources.Load("circle-outline-basic") as Texture2D;
mt.renderQueue = 3000;
mt.SetFloat("_MinSize", 0.01f);
mt.SetFloat("_MaxSize", 0.05f);
return(vb.updateView().apply(gameObject, mt));
}
/// <summary>
/// Calls the subroutine which creates the Scatterplots.
/// </summary>
/// <param name="dataSource"></param>
/// <param name="dimCombinations"></param>
private void CreateScatterplots(CSVDataSource dataSource, int[,] dimCombinations)
{
GameObject scatterplotPrefab = Resources.Load("Prefabs/Scatterplot") as GameObject;
int matrixWidth = (int)(Mathf.Sqrt(dimCombinations.GetLength(0)) + 1);
scatterplots = new Scatterplot[dimCombinations.GetLength(0)];
StartCoroutine(CreateScatterplotsCoroutine(scatterplotPrefab, dataSource, dimCombinations, matrixWidth));
}
public void traning(string traningexamplespath)
{
IVersatileDataSource source = new CSVDataSource(traningexamplespath, false, CSVFormat.DecimalPoint);
var data = new VersatileMLDataSet(source);
data.DefineSourceColumn("num1", 0, ColumnType.Continuous);
data.DefineSourceColumn("num2", 1, ColumnType.Continuous);
data.DefineSourceColumn("num1", 2, ColumnType.Continuous);
data.DefineSourceColumn("num1", 3, ColumnType.Continuous);
data.DefineSourceColumn("num1", 4, ColumnType.Continuous);
data.DefineSourceColumn("num1", 5, ColumnType.Continuous);
data.DefineSourceColumn("num1", 6, ColumnType.Continuous);
data.DefineSourceColumn("num1", 7, ColumnType.Continuous);
data.DefineSourceColumn("num1", 8, ColumnType.Continuous);
data.DefineSourceColumn("num1", 9, ColumnType.Continuous);
data.DefineSourceColumn("num1", 10, ColumnType.Continuous);
data.DefineSourceColumn("num1", 11, ColumnType.Continuous);
ColumnDefinition outputColumn = data.DefineSourceColumn("kind", 12, ColumnType.Nominal);
data.Analyze();
data.DefineSingleOutputOthersInput(outputColumn);
var model = new EncogModel(data);
model.SelectMethod(data, MLMethodFactory.TypeFeedforward);
// Send any output to the console.
model.Report = new ConsoleStatusReportable();
// Now normalize the data. Encog will automatically determine the correct normalization
// type based on the model you chose in the last step.
data.Normalize();
model.HoldBackValidation(0.3, true, 1001);
// Choose whatever is the default training type for this model.
model.SelectTrainingType(data);
// Use a 5-fold cross-validated train. Return the best method found.
bestMethod = (IMLRegression)model.Crossvalidate(5, true);
//Console.WriteLine(@"Training error: " + model.CalculateError(bestMethod, model.TrainingDataset));
//Console.WriteLine(@"Validation error: " + model.CalculateError(bestMethod, model.ValidationDataset));
// Display our normalization parameters.
helper = data.NormHelper;
// Console.WriteLine(helper.ToString());
// Display the final model.
//Console.WriteLine(@"Final model: " + bestMethod);
source.Close();
saveNetwork("save.eg");
savehelper("helper.hp");
//EncogFramework.Instance.Shutdown();
}
public void Awake()
{
scatterplotDropdown = GameObject.FindGameObjectWithTag("ScatterplotDropdown").GetComponent <Dropdown>();
dataDropdown = GameObject.FindGameObjectWithTag("DataDropdown").GetComponent <Dropdown>();
clipboardText = GameObject.FindGameObjectWithTag("InfoText").GetComponent <Text>();
dataSource = gameObject.AddComponent <CSVDataSource>();
//loadFilesFromDirectory();
scatterplotDropdown.value = 6;
}
private void AssertData(CSVDataSource dataSource)
{
var csvData = dataSource.FetchData();
Assert.NotNull(csvData);
var castedData = csvData.Data as string[][];
Assert.NotNull(castedData);
Assert.AreEqual(11033, castedData.Count());
Assert.AreEqual("TOA5", castedData[0][0]);
}
/// <summary>
/// Initializes the ScatterplotMatrix.
/// Should always be called after creating this component.
/// </summary>
/// /// <param name="dataSource"></param>
/// /// <param name="dimCombinations"></param>
/// /// <param name="pointSize"></param>
public void Initialize(CSVDataSource dataSource, int[,] dimCombinations, float pointSize, bool bigScatterplot)
{
foreach (int dimi in dimCombinations)
{
//Debug.Log(dimi.ToString());
}
this.pointSize = pointSize;
this.bigScatterplot = bigScatterplot;
CreateScatterplots(dataSource, dimCombinations);
}
public VersatileMLDataSet LoadDataSetFromCsv(string filename)
{
var csvFormatSpec = new CSVFormat('.', '\t');
var dataSource = new CSVDataSource(filename, headers: true, format: csvFormatSpec);
var dataSet = new VersatileMLDataSet(dataSource)
{
NormHelper = { Format = csvFormatSpec }
};
return(dataSet);
}
public override int NumberOfLabels()
{
if (IsDiscreet())
{
CSVDataSource csvdatasoure = (CSVDataSource)dataSource;
return(csvdatasoure.TextualDimensionsListReverse[attributeFilter.Attribute].Count);
}
else
{
return(4);
}
}
void CreateDataSource(string content)
{
if (DataSource != null)
{
Destroy(DataSource);
}
DataSource = gameObject.AddComponent <CSVDataSource>();
DataSource.Load(content);
DataSourceLoaded.Invoke(DataSource);
}
/// <summary>
/// Initializes the ScatterplotMatrix.
/// Should always be called after creating this component.
/// </summary>
/// <param name="dataSource"></param>
/// <param name="matrixPosX"></param>
/// <param name="matrixPosZ"></param>
/// <param name="pointSize"></param>
/// <param name="xDim"></param>
/// <param name="yDim"></param>
/// <param name="zDim"></param>
public void Initialize(CSVDataSource dataSource, float matrixPosX, float matrixPosZ, float pointSize, int xDim, int yDim, int zDim)
{
this.dataSource = dataSource;
this.pointSize = pointSize;
this.xDim = xDim;
this.yDim = yDim;
this.zDim = zDim;
float posOffset = 1;
InitializeAxesLabel();
CreateDataPoints();
transform.Translate(new Vector3(matrixPosX + posOffset * matrixPosX, 0, matrixPosZ + posOffset * matrixPosZ));
}
// This is needed to create the buttons if the script is disabled
protected void Initialise()
{
if (dataSource == null)
{
dataSource = ChartManager.Instance.DataSource;
}
buttons = new List <MenuButton>();
isOpen = false;
CreateButtons();
CloseButtons(0);
isInitialised = true;
}
protected virtual List <string> GetAttributesList()
{
if (dataSource == null)
{
dataSource = ChartManager.Instance.DataSource;
}
List <string> dimensions = new List <string>();
for (int i = 0; i < dataSource.DimensionCount; ++i)
{
dimensions.Add(dataSource[i].Identifier);
}
return(dimensions);
}
void CreateDataSource(string content)
{
if (DataSource != null)
{
Destroy(DataSource);
}
DataSource = gameObject.AddComponent <CSVDataSource>();
DataSource.Load(content);
CreateVisualization(DataSource);
StartCoroutine(CreateTableCoroutine());
//init bounding box if datasource loaded
Boundingbox?.Init();
}
public void OnDataSourceLoaded(CSVDataSource source)
{
foreach (Transform child in transform)
{
DestroyImmediate(child.gameObject);
}
Filters.AddRange(source.Select(x => new DimensionFilter {
Attribute = x.Identifier
}));
//foreach (var filter in Filters)
// Instantiate(DimensionObj, transform)?.Init(this, filter);
Dropdown.ClearOptions();
Dropdown.AddOptions(Filters.Select(x => x.Attribute).ToList());
}
// Use this for initialization
void Start()
{
barList = new List <GameObject>();
int count = 0;
foreach (TextAsset ta in datasets)
{
GameObject go = new GameObject("BarCharts-" + count);
go.transform.parent = this.transform;
CSVDataSource csvdata = createCSVDataSource(ta.text, go);
CreateBarchart(csvdata, go, count);
count++;
barList.Add(go);
}
}
public void TestCSVDataSource_Positive()
{
var source = new CSVDataSource();
source.Load();
var expected = new List <string>()
{
"name,address_line1,address_line2",
"Dave,Market Street,London",
"John,Queen Street,York"
};
var actual = source.Data;
CollectionAssert.AreEqual(expected, actual);
}
public void DisplayTrajectories()
{
// Assignments data parsing
List <List <int> > assignments = ParseListOfListOfInts(assignmentsData);
// Trajectories files data parsing
List <String> fileNames = ParseListOfString(filesData);
// Create Trajectories objects
int nb_files = assignments.Count;
int[] assignments_counter = new int[nb_files];
Array.Clear(assignments_counter, 0, nb_files); // Fills array with 0s
for (int i = 0; i < assignments.Count; i++)
{
GameObject t = Instantiate(trajectoryPrefab, new Vector3(assignments_counter[assignments[i][0]]++, assignments[i][0], 0), Quaternion.identity);
t.transform.parent = transform;
// upload CSV file
CSVDataSource dataSource = t.transform.Find("[IATK] New Data Source").GetComponent <CSVDataSource>();
dataSource.data = Resources.Load(fileNames[i]) as TextAsset;
// set up visualisation
Visualisation dataVisualisation = t.transform.Find("[IATK] New Visualisation").GetComponent <Visualisation>();
dataVisualisation.dataSource = dataSource;
dataVisualisation.CreateVisualisation(AbstractVisualisation.VisualisationTypes.SCATTERPLOT);
dataVisualisation.colour = randomColorFromInt(assignments[i][0]);
dataVisualisation.xDimension = new DimensionFilter {
Attribute = "x"
};
dataVisualisation.yDimension = new DimensionFilter {
Attribute = "y"
};
dataVisualisation.zDimension = new DimensionFilter {
Attribute = "z"
};
dataVisualisation.geometry = AbstractVisualisation.GeometryType.Lines;
dataVisualisation.linkingDimension = "TrajectoryID";
dataVisualisation.theVisualizationObject.UpdateVisualisation(AbstractVisualisation.PropertyType.X);
dataVisualisation.theVisualizationObject.UpdateVisualisation(AbstractVisualisation.PropertyType.Y);
dataVisualisation.theVisualizationObject.UpdateVisualisation(AbstractVisualisation.PropertyType.Z);
dataVisualisation.theVisualizationObject.UpdateVisualisation(AbstractVisualisation.PropertyType.GeometryType);
dataVisualisation.theVisualizationObject.UpdateVisualisation(AbstractVisualisation.PropertyType.LinkingDimension);
}
}
/// <summary>
/// Creates the Scatterplots in a subroutine, one per frame.
/// This was done to keep the programm from freezen when creating
/// the Scatterplots.
/// </summary>
/// <param name="scatterplotPrefab"></param>
/// <param name="dataSource"></param>
/// <param name="dimCombinations"></param>
/// <param name="matrixWidth"></param>
/// <returns></returns>
IEnumerator CreateScatterplotsCoroutine(GameObject scatterplotPrefab, CSVDataSource dataSource, int[,] dimCombinations, int matrixWidth)
{
for (int i = 0; dimCombinations.GetLength(0) > i; ++i)
{
int matrixPosX = i % matrixWidth * 2;
int matrixPosZ = i / matrixWidth * 2;
int xDim = dimCombinations[i, 0];
int yDim = dimCombinations[i, 1];
int zDim = dimCombinations[i, 2];
Scatterplot scatterplot = Instantiate(scatterplotPrefab, transform).GetComponent <Scatterplot>();
scatterplot.Initialize(dataSource, matrixPosX, matrixPosZ, pointSize, xDim, yDim, zDim);
scatterplots[i] = scatterplot;
scatterplot.transform.localScale += Vector3.one;
yield return(null);
}
}
static void Main(string[] args)
{
var network = new BasicNetwork();
network.AddLayer(new BasicLayer(2));
network.AddLayer(new BasicLayer(3));
network.AddLayer(new BasicLayer(1));
network.Structure.FinalizeStructure();
network.Reset();
var trainingDataSource = new CSVDataSource(@"Data\training.csv", true, ',');
//var validationDataSource = new CSVDataSource(@"Data\validation.csv", true, ',');
var trainingSet = new VersatileMLDataSet(trainingDataSource);
//var validationSet = new VersatileMLDataSet(validationDataSource);
trainingSet.Analyze();
trainingSet.Normalize();
var training = new ResilientPropagation(network, trainingSet);
int epoch = 1;
do
{
training.Iteration();
Console.WriteLine($"Epoch #{epoch}. Error: {training.Error}");
epoch++;
}while (training.Error > 0.01);
training.FinishTraining();
Console.WriteLine("Neural Network Results:");
foreach (var pair in trainingSet)
{
var output = network.Compute(pair.Input);
Console.WriteLine($"{pair.Input[0]},{pair.Input[1]}, actual={output[0]}, ideal={pair.Ideal}");
}
EncogFramework.Instance.Shutdown();
}
/// <summary>
/// Returns the data from the CSV-File which this DataPoint represents.
/// The first dimension holds the three columns.
/// The second dimension holds the columns identifier in index 0 and the value in index 1.
/// </summary>
/// <returns></returns>
public string[,] GetData()
{
CSVDataSource dataSource = scatterplot.dataSource;
int xDim = scatterplot.xDim;
int yDim = scatterplot.yDim;
int zDim = scatterplot.zDim;
string[,] data = new string[3, 2];
data[0, 0] = dataSource[xDim].Identifier;
data[0, 1] = dataSource.getOriginalValue(dataSource[xDim].Data[index], dataSource[xDim].Identifier).ToString();
data[1, 0] = dataSource[yDim].Identifier;
data[1, 1] = dataSource.getOriginalValue(dataSource[yDim].Data[index], dataSource[yDim].Identifier).ToString();
data[2, 0] = dataSource[zDim].Identifier;
data[2, 1] = dataSource.getOriginalValue(dataSource[zDim].Data[index], dataSource[zDim].Identifier).ToString();
return(data);
}
/// <summary>
/// Diese Methode färbt den DataPoint falls er selektiert wurde ein.
/// </summary>
public void colorSelected(CSVDataSource dataSource)
{
if (!(dataSource.selectedIndicies is null))
{
Renderer renderer = this.GetComponent <Renderer>();
if (dataSource.selectedIndicies.Contains(index))
{
Debug.LogError(dataSource.selectionColor);
if (dataSource.selectionColor != null)
{
color = Utilities.HexToColor(dataSource.selectionColor);
renderer.material.color = color;
}
else
{
color = Color.black;
renderer.material.color = color;
}
}
}
}
