private static Data.Attribute[] ParseAttributeLine(string line, int attributeIndex)
{
List <Data.Attribute> attributes = new List <Data.Attribute>();
string[] parts = line.Split(' ');
string name = parts[1];
for (int i = 2; i < parts.Length - 1; i++)
{
name += parts[i];
}
if (string.Compare(parts[parts.Length - 1].Trim(), "numeric", true) == 0)
{
Data.Attribute attribute = new Data.Attribute(name, attributeIndex);
attributes.Add(attribute);
}
else
{
string[] values = line.Split(' ');
values = values[values.Length - 1].Trim('{', '}').Split(',');
foreach (string value in values)
{
Data.Attribute attribute = new Data.Attribute(name + "::" + value, attributeIndex);
attributes.Add(attribute);
attributeIndex++;
}
}
return(attributes.ToArray());
}
private static Data.Attribute ParseAttributeLine(string line, int attributeIndex)
{
Data.Attribute attribute=null;
string[] parts = line.Split(' ');
string name=parts[1];
for (int i = 2; i < parts.Length - 1; i++)
name += parts[i];
if (string.Compare(parts[parts.Length - 1].Trim(), "numeric", true) == 0)
{
attribute = new Data.NumericAttribute(name, attributeIndex);
}
else
{
string[] values = line.Split(' ');
values = values[values.Length - 1].Trim('{', '}').Split(',');
attribute = new Data.NominalAttribute(name, attributeIndex, values);
}
return attribute;
}
public void SimplePlan()
{
var catalog = new AggregateCatalog(
new AssemblyCatalog(typeof (ITypedPool).Assembly),
new AssemblyCatalog(typeof (Element).Assembly));
var container = new CompositionContainer(catalog);
container.ComposeExportedValue(container);
var cachier = container.GetExportedValue<ITypedPool>();
var rootElement = new Element { Id = Guid.NewGuid(), Name = "Root" };
var attr1 = new Data.Attribute { Id = Guid.NewGuid(), ElementId = rootElement.Id, Name = "Attr1" };
var attr2 = new Data.Attribute { Id = Guid.NewGuid(), ElementId = rootElement.Id, Name = "Attr2" };
cachier.Post(rootElement.Id, rootElement);
cachier.Post(attr1.Id, attr1);
cachier.Post(attr2.Id, attr2);
var sAttr = new ResourceProjectionElement
{
Name = "Attribute",
FillingInfo = new FillingInfo { { "Name", FillingInfo.FillType.Native } },
ResourceType = "Attribute",
Namespace = "Test"
};
var sElement = new ResourceProjectionElement
{
Name = "Element",
FillingInfo = new FillingInfo { { "Name", FillingInfo.FillType.Native } },
ResourceType = "Element",
Namespace = "Test",
};
var steps = new TestCommendationCollection1(sAttr, new[] { attr1.Id, attr2.Id });
var plan = new TestFramingPlan(sElement, steps);
var builder = container.GetExportedValue<ProjectionBuilder>();
var writer = new TestWriter();
var sb = builder.RealisePlan(plan, rootElement.Id, "Test", 2, new Streamer<StringBuilder>(writer), new Dictionary<string, object>());
var expWriter = new TestWriter();
var expSb = expWriter.Open("Test");
expWriter.StartWritingNode("Element", rootElement.Id);
expWriter.WriteAttribute("Name", rootElement.Name);
expWriter.StartWritingNode("Attribute", attr1.Id);
expWriter.WriteAttribute("Name", attr1.Name);
expWriter.WriteAttribute("DefinitlyNoChildren", "True");
expWriter.EndWritingNode();
expWriter.StartWritingNode("Attribute", attr2.Id);
expWriter.WriteAttribute("Name", attr2.Name);
expWriter.WriteAttribute("DefinitlyNoChildren", "True");
expWriter.EndWritingNode();
expWriter.EndWritingNode();
Assert.AreEqual(expSb.ToString(), sb.ToString());
}
public static IClassifier CreateMetaClassifier(IClassificationAlgorithm algorithm, List <Dataset> trainingsets, List <ClassifierInfo> ensemble)
{
List <Data.Attribute> metaAttributes = new List <Data.Attribute>();
for (int i = 0; i < ensemble.Count; i++)
{
metaAttributes.Add(new Data.Attribute(ensemble[i].Desc, i, new string[] { "Yes", "No" }));
}
Data.Attribute metaClass = new Data.Attribute("Class", ensemble.Count, new string[] { "Yes", "No" });
Metadata metadata = new Metadata("metaClassifier", metaAttributes.ToArray(), metaClass);
List <Example> metaExamples = new List <Example>();
int counter = 0;
foreach (Example example in trainingsets[0])
{
List <int> predictions = new List <int>();
for (int infoIndex = 0; infoIndex < ensemble.Count; infoIndex++)
{
ClassifierInfo info = ensemble[infoIndex];
int predictoin = info.Classifier.Classify(trainingsets[infoIndex][example.Index]).Label;
predictions.Add(predictoin);
}
Example metaExample = new Example(metadata, counter++, predictions.ToArray(), trainingsets[0][example.Index].Label);
metaExamples.Add(metaExample);
}
Dataset metaDataset = new Dataset(metadata);
metaDataset.SetExamples(metaExamples.ToArray());
algorithm.Dataset = metaDataset;
IClassifier classifier = algorithm.CreateClassifier();
return(classifier);
}
private void UpdateValueCounts()
{
for (int attributeIndex = 0; attributeIndex < this._metadata.Attributes.Length; attributeIndex++)
{
Data.Attribute currentAttribute = this._metadata.Attributes[attributeIndex];
for (int valueIndex = 0; valueIndex < currentAttribute.Values.Length; valueIndex++)
{
currentAttribute.ValueCounts[valueIndex] = this.Filter(attributeIndex, valueIndex).Count;
}
for (int valueIndex = 0; valueIndex < this._metadata.Target.Values.Length; valueIndex++)
{
this._metadata.Target.ValueCounts[valueIndex] = this.Filter(valueIndex).Count;
}
}
this._metadata.Size = this._examples.Count;
}
public async Task <JSONResponseVM> AddNewAttribute([FromBody] AttributeVM attr)
{
//check to see if admin
//create new attibute. Have to use Data.Attribute because System.Attribute is also a thing
Data.Attribute att = new Data.Attribute
{
name = attr.name,
attribute_type_id = attr.attribute_type_id,
image_src = attr.image_src,
rarity = attr.rarity,
attribute = await context.AttributeTypes.Where(a => a.attribute_type_id == attr.attribute_type_id).FirstOrDefaultAsync()
};
context.Attributes.Add(att);
context.SaveChanges();
return(new JSONResponseVM {
success = true, message = "Successfully add new attribute: " + att.name
});
}
private static Data.Attribute ParseAttributeLine(string line, int attributeIndex)
{
List <string> values = new List <string>();
string name = "";
string[] parts = line.Split(' ');
if (parts.Length == 3)
{
name = parts[1];
}
else
{
name = parts[1] + parts[2];
}
string[] vals = parts[parts.Length - 1].Split(',');
foreach (string val in vals)
{
values.Add(val.Trim('{', '}', ' '));
}
foreach (string val in values)
{
if (val.Contains("All"))
{
return(null);
}
}
Data.Attribute attribute = new Data.Attribute(name, attributeIndex, values.ToArray());
return(attribute);
}
public void SimplePlan()
{
var catalog = new AggregateCatalog(
new AssemblyCatalog(typeof(ITypedPool).Assembly),
new AssemblyCatalog(typeof(Element).Assembly));
var container = new CompositionContainer(catalog);
container.ComposeExportedValue(container);
var cachier = container.GetExportedValue <ITypedPool>();
var rootElement = new Element {
Id = Guid.NewGuid(), Name = "Root"
};
var attr1 = new Data.Attribute {
Id = Guid.NewGuid(), ElementId = rootElement.Id, Name = "Attr1"
};
var attr2 = new Data.Attribute {
Id = Guid.NewGuid(), ElementId = rootElement.Id, Name = "Attr2"
};
cachier.Post(rootElement.Id, rootElement);
cachier.Post(attr1.Id, attr1);
cachier.Post(attr2.Id, attr2);
var sAttr = new ResourceProjectionElement
{
Name = "Attribute",
FillingInfo = new FillingInfo {
{ "Name", FillingInfo.FillType.Native }
},
ResourceType = "Attribute",
Namespace = "Test"
};
var sElement = new ResourceProjectionElement
{
Name = "Element",
FillingInfo = new FillingInfo {
{ "Name", FillingInfo.FillType.Native }
},
ResourceType = "Element",
Namespace = "Test",
};
var steps = new TestCommendationCollection1(sAttr, new[] { attr1.Id, attr2.Id });
var plan = new TestFramingPlan(sElement, steps);
var builder = container.GetExportedValue <ProjectionBuilder>();
var writer = new TestWriter();
var sb = builder.RealisePlan(plan, rootElement.Id, "Test", 2, new Streamer <StringBuilder>(writer), new Dictionary <string, object>());
var expWriter = new TestWriter();
var expSb = expWriter.Open("Test");
expWriter.StartWritingNode("Element", rootElement.Id);
expWriter.WriteAttribute("Name", rootElement.Name);
expWriter.StartWritingNode("Attribute", attr1.Id);
expWriter.WriteAttribute("Name", attr1.Name);
expWriter.WriteAttribute("DefinitlyNoChildren", "True");
expWriter.EndWritingNode();
expWriter.StartWritingNode("Attribute", attr2.Id);
expWriter.WriteAttribute("Name", attr2.Name);
expWriter.WriteAttribute("DefinitlyNoChildren", "True");
expWriter.EndWritingNode();
expWriter.EndWritingNode();
Assert.AreEqual(expSb.ToString(), sb.ToString());
}
public static EnsemblePerNodeTopDownClassifier CreateEnsembleDSPerNodeTopDownClassifier(List <Data.Dataset> trainingSets, IClassificationAlgorithm algorithm)
{
ClassHierarchy hierarchy = ((Data.HierarchicalAttribute)trainingSets[0].Metadata.Target).Hierarchy;
EnsemblePerNodeTopDownClassifier topDownClassifier = new EnsemblePerNodeTopDownClassifier(hierarchy);
Data.Attribute[] metaAttributes = new Data.Attribute[2];
string[] values = new string[trainingSets.Count];
for (int i = 0; i < metaAttributes.Length; i++)
{
values[i] = trainingSets[i].Metadata.DatasetName;
}
metaAttributes[0] = new Data.Attribute("Dataset", 0, values);
metaAttributes[1] = new Data.Attribute("Prediction", 1, new string[] { "yes", "no" });
Data.Attribute target = new Data.Attribute("class", metaAttributes.Length, new string[] { "yes", "no" });
Metadata meta = new Metadata("metaModel", metaAttributes, target);
Node current = hierarchy.Root;
Queue <Node> queue = new Queue <Node>();
foreach (Node node in hierarchy.GetNodes(current.Children))
{
queue.Enqueue(node);
}
while (queue.Count != 0)
{
current = queue.Dequeue();
Node[] siblings = hierarchy.GetSiblings(current);
List <string> negativeClassValues = new List <string>();
foreach (Node sibling in siblings)
{
negativeClassValues.Add(sibling.Name);
}
Dataset dsLocal = new Dataset(meta.Clone());
int i = 0;
foreach (Dataset trainingSet in trainingSets)
{
Dataset local = trainingSet.GetBinaryFlatLabelSubDataset(current.Name, negativeClassValues.ToArray());
algorithm.TrainingSet = local;
IClassifier classifier = algorithm.CreateClassifier();
topDownClassifier.AddClassifier(current.Name, local.Metadata.DatasetName, classifier);
//Example metaExample=new Example(dsLocal,i,
}
//algorithm.TrainingSet = local;
//IClassifier classifier = algorithm.CreateClassifier();
//topDownClassifier.PutClassifier(current.Name, trainingSet.Metadata.DatasetName, classifier);
//if (current.Children != null)
// foreach (string child in current.Children)
// queue.Enqueue(hierarchy[child]);
}
return(topDownClassifier);
}
public static Data.Dataset LoadHierarchicalDatasetFromTxt(string filePath, bool skipfirstAttribute)
{
StreamReader reader = new StreamReader(filePath);
string datasetName = null;
Data.Metadata metadata = null;
Data.Dataset dataset = null;
List<Data.Attribute> attributes = new List<Data.Attribute>();
List<Data.Example> examples = new List<Data.Example>();
List<Data.Node> nodes = new List<Data.Node>();
int attributeIndex = 0;
int exampleIndex = 0;
string mode = "start";
while (!reader.EndOfStream)
{
string line = reader.ReadLine();
if (!string.IsNullOrEmpty(line))
{
if (line.Contains("%"))
continue;
if (line.Contains("@relation"))
{
datasetName = line.Substring(line.IndexOf(' ')).Trim();
datasetName = datasetName.Contains("-weka") ? datasetName.Substring(0, datasetName.IndexOf("-weka")) : datasetName;
}
else if (line.Contains("@attribute"))
{
Data.Attribute attribute = ParseAttributeLine(line, attributeIndex);
if (attribute != null)
{
attributeIndex++;
attributes.Add(attribute);
}
}
else if (line.Contains("@ontology"))
{
mode = "ontolog";
}
else if (line.Contains("@data"))
{
List<string> classValues = new List<string>();
int counter = 0;
for(int i=1; i<nodes.Count;i++)
{
Data.Node node = nodes[i];
node.ValueIndex = counter;
classValues.Add(node.Name);
counter++;
}
Data.ClassHierarchy classHierarchy = new Data.ClassHierarchy(nodes.ToArray());
Data.HierarchicalAttribute target = new Data.HierarchicalAttribute("class", attributes.Count, classValues.ToArray(), classHierarchy);
metadata = new Data.Metadata(datasetName, attributes.ToArray(), target,true);
dataset = new Data.Dataset(metadata);
mode = "data";
}
else
{
if (mode == "ontolog")
{
Data.Node node = ParseOntologyLine(line);
if (!nodes.Exists(n => n.Name == node.Name))
nodes.Add(node);
}
else
{
Data.Example example = ParseHierarchicalExampleLine(exampleIndex, line, dataset, skipfirstAttribute);
exampleIndex++;
examples.Add(example);
}
}
}
}
dataset.SetExamples(examples.ToArray());
return dataset;
}
public static Data.Dataset LoadDatasetFromArff(string filePath)
{
StreamReader reader = new StreamReader(filePath);
string datasetName = null;
Data.Metadata metadata = null;
Data.Dataset dataset = null;
List<Data.Attribute> attributes = new List<Data.Attribute>();
List<Data.Example> examples = new List<Data.Example>();
int attributeIndex = 0;
int exampleIndex = 0;
List<string> exampleLines = new List<string>();
while (!reader.EndOfStream)
{
string line = reader.ReadLine();
if (!string.IsNullOrEmpty(line))
{
if (line.Contains("%"))
continue;
if (line.Contains("@relation"))
{
datasetName = line.Substring(line.IndexOf(' ')).Trim();
datasetName = datasetName.Contains("-weka") ? datasetName.Substring(0, datasetName.IndexOf("-weka")) : datasetName;
}
else if (line.Contains("@attribute"))
{
Data.Attribute attribute = ParseAttributeLine(line, attributeIndex);
if (attribute != null)
{
attributeIndex++;
attributes.Add(attribute);
}
}
else if (line.Contains("@data"))
{
int targetIndex = attributes.FindIndex(m => string.Compare(m.Name, "Class", true) == 0);
Data.NominalAttribute target = attributes[targetIndex] as Data.NominalAttribute;
attributes.RemoveAt(target.Index);
metadata = new Data.Metadata(datasetName, attributes.ToArray(), target,false);
dataset = new Data.Dataset(metadata);
}
else
{
if (exampleLines.Contains(line))
continue;
Data.Example example = ParseExampleLine(exampleIndex, line, dataset);
exampleIndex++;
examples.Add(example);
exampleLines.Add(line);
}
}
}
dataset.SetExamples(examples.ToArray());
return dataset;
}
private static Data.Example ParseExampleLine(int exampleIndex, string line, Data.Metadata metadata, List <int> nominalAttributesIndexes)
{
string [] parts = line.Split(',');
List <double> values = new List <double>();
int label = 0;
for (; label < metadata.Target.Length; label++)
{
if (metadata.Target[label] == parts[parts.Length - 1])
{
break;
}
}
int offset = 0;
for (int index = 0; index < parts.Length - 1; index++)
{
string stringValue = parts[index];
if (!nominalAttributesIndexes.Contains(index))
{
double value = double.NaN;
if (stringValue != "?")
{
value = double.Parse(stringValue);
}
values.Add(value);
offset++;
}
else
{
Data.Attribute attribute = metadata.Attributes[offset];
string attributeName = attribute.Name.Substring(0, attribute.Name.IndexOf("::"));
while (true)
{
attribute = metadata.Attributes[offset];
if (!attribute.Name.Contains("::"))
{
break;
}
string currentAttributeName = attribute.Name.Substring(0, attribute.Name.IndexOf("::"));
if (currentAttributeName != attributeName)
{
break;
}
string valueName = attribute.Name.Substring(attribute.Name.IndexOf("::") + 2);
if (valueName == stringValue)
{
values.Add(1);
}
else
{
values.Add(0);
}
offset++;
if (offset == metadata.Attributes.Length)
{
break;
}
}
}
}
Data.Example example = new Data.Example(metadata, exampleIndex, values.ToArray(), label);
return(example);
}
