internal Accessor(FieldInfo field, DGet <T, V> dget, DSet <T, V> dset)
: base()
{
_field = field;
_dget = dget;
_dset = dset;
}
private bool Match(Cluster cluster, string matchWord)
{
var name = "Sherlock-Holmes-" + Guid.NewGuid().ToString("D");
var corpus = File.ReadAllLines(Book);
// Store the file to the cluster
var dset = new DSet <string> {
Name = name, Cluster = cluster
};
dset.Store(corpus);
var corpusLines = dset.LoadSource();
var count1 =
corpusLines.SelectMany(SplitWords)
.Where(w => w == matchWord)
.Count();
Console.WriteLine("Counted with DSet: The word {0} occurs {1} times", matchWord, count1);
var count2 =
corpus.SelectMany(SplitWords)
.Where(w => w == matchWord)
.Count();
Console.WriteLine("Counted locally: The word {0} occurs {1} times", matchWord, count2);
return(count1 == count2);
}
// Methods
internal Accessor(PropertyInfo pi, DGet <T, V> dget, DSet <T, V> dset, DRSet <T, V> drset, MetaAccessor <T, V2> storage)
: this(false, pi, storage)
{
this.dget = dget;
this.dset = dset;
this.drset = drset;
}
public override string Accept(DSet type)
{
var x = new StringBuilder();
Append(type.Datas, x);
return(x.ToString());
}
/// <summary>
/// Конструктор для класса, генерирующий
/// Def и Use множества по базовому блоку
/// </summary>
/// <param name="block">Базовый блок</param>
public DUSets(BasicBlock block)
{
this.Block = block;
DSet = new DSet();
USet = new USet();
BuildDUSets();
}
internal Accessor(PropertyInfo property,
DGet <T, V> dget, DSet <T, V> dset)
: base()
{
_property = property;
_dget = dget;
_dset = dset;
}
internal Accessor(PropertyInfo pi, DGet <T, V> dget, DSet <T, V> dset, DRSet <T, V> drset, MetaAccessor <T, V2> storage)
{
this.pi = pi;
this.dget = dget;
this.dset = dset;
this.drset = drset;
this.storage = storage;
}
public override global::System.Data.DataSet Clone()
{
DSet cln = ((DSet)(base.Clone()));
cln.InitVars();
cln.SchemaSerializationMode = this.SchemaSerializationMode;
return(cln);
}
public void Accept(DSet type, StringBuilder line)
{
line.Append('{');
foreach (var d in type.Datas)
{
d.Apply(this, line);
line.Append(',');
}
line.Append('}');
}
public Params RunEpoch(DSet <Example> trainSet, ISGDModel <Params> model, Params curParams)
{
Params accumulatedParams =
trainSet.Fold(
(wb, e) => { return(model.Update(wb, e)); },
model.AddParams,
curParams);
var averagedParams = model.ScaleParams(accumulatedParams, (float)(1.0 / trainSet.NumPartitions));
return(averagedParams);
}
public static global::System.Xml.Schema.XmlSchemaComplexType GetTypedDataSetSchema(global::System.Xml.Schema.XmlSchemaSet xs)
{
DSet ds = new DSet();
global::System.Xml.Schema.XmlSchemaComplexType type = new global::System.Xml.Schema.XmlSchemaComplexType();
global::System.Xml.Schema.XmlSchemaSequence sequence = new global::System.Xml.Schema.XmlSchemaSequence();
global::System.Xml.Schema.XmlSchemaAny any = new global::System.Xml.Schema.XmlSchemaAny();
any.Namespace = ds.Namespace;
sequence.Items.Add(any);
type.Particle = sequence;
global::System.Xml.Schema.XmlSchema dsSchema = ds.GetSchemaSerializable();
if (xs.Contains(dsSchema.TargetNamespace))
{
global::System.IO.MemoryStream s1 = new global::System.IO.MemoryStream();
global::System.IO.MemoryStream s2 = new global::System.IO.MemoryStream();
try {
global::System.Xml.Schema.XmlSchema schema = null;
dsSchema.Write(s1);
for (global::System.Collections.IEnumerator schemas = xs.Schemas(dsSchema.TargetNamespace).GetEnumerator(); schemas.MoveNext();)
{
schema = ((global::System.Xml.Schema.XmlSchema)(schemas.Current));
s2.SetLength(0);
schema.Write(s2);
if ((s1.Length == s2.Length))
{
s1.Position = 0;
s2.Position = 0;
for (; ((s1.Position != s1.Length) &&
(s1.ReadByte() == s2.ReadByte()));)
{
;
}
if ((s1.Position == s1.Length))
{
return(type);
}
}
}
}
finally {
if ((s1 != null))
{
s1.Close();
}
if ((s2 != null))
{
s2.Close();
}
}
}
xs.Add(dsSchema);
return(type);
}
/// <summary>
/// Создает Def и Use множества для базового блока
/// </summary>
private void BuildDUSets()
{
var duLists = new DULists(Block);
foreach (var d in duLists.DList)
{
DSet.Add(d.DefVariable.Name);
}
foreach (var u in duLists.UListNotValid)
{
USet.Add(u.Name);
}
}
/// <summary>
/// Loads a DSet in parallel by splitting the file int numPartitions * cluster.NumNodes,
/// finding the next separator (newline), and reading up to maxLines of each chunk in parallel.
/// The file should be in LibSVM format, i.e.: tab separated with the first column being the label,
/// followed by any number of FeatureIndex:FeatureValue pairs, followed by a newline.
/// The dimension can be passed for convenience, but can also be set by the client later, so long
/// as this is done before trying to any operations on the Example sparse vectors.
/// </summary>
private static DSet <Example> LoadDSet(Cluster cluster, string file, int numPartitions, int maxLines, int dimension)
{
int numNodes = cluster.NumNodes;
var trainSet =
new DSet <Example>()
{
Cluster = cluster, Name = "trainSet", NumParallelExecution = numPartitions, SerializationLimit = int.MaxValue
}
.SourceN(numPartitions, i =>
{
// The dataset is divided in chunks by raw byte position,
// and each partition reads from the next newline until either
// (a) maxLines / numPartions are read or
// (b) it has read bast the raw start pos of the next partition
long trainFileSize = new FileInfo(file).Length;
long byteChunkSize = trainFileSize / (numPartitions * numNodes);
int maxLinesPerPartition = maxLines / (numPartitions * numNodes);
var rawStartPos = i * byteChunkSize;
var stream = new BinaryReader(File.OpenRead(file));
if (i != 0)
{
stream.BaseStream.Seek(rawStartPos, SeekOrigin.Begin);
// position stream after next newline
while (stream.ReadChar() != '\n')
{
;
}
}
var nextPartRawStartPos = rawStartPos + byteChunkSize;
var reader = new StreamReader(stream.BaseStream);
string line;
var lines = new List <Example>();
while (stream.BaseStream.Position < nextPartRawStartPos &&
lines.Count < maxLinesPerPartition &&
(line = reader.ReadLine()) != null)
{
lines.Add(ParseLine(dimension, line));
}
return(lines);
})
.CacheInMemory();
return(trainSet);
}
static void GetProcessInfo(Cluster cluster)
{
var dset = new DSet<int> { Name = Guid.NewGuid().ToString("D"), Cluster = cluster };
var descriptions =
dset
.DistributeN(cluster.NumNodes * 4, Enumerable.Range(0, cluster.NumNodes * 8))
.Select(i =>
{
var machineName = System.Environment.MachineName;
var process = System.Diagnostics.Process.GetCurrentProcess();
var description = $"{machineName}-{process.Id}-{Thread.CurrentThread.ManagedThreadId}-{i}";
return description;
})
.ToIEnumerable()
.ToArray();
foreach (var description in descriptions)
{
Console.WriteLine(description);
}
}
static void GetProcessInfo(Cluster cluster)
{
var dset = new DSet <int> {
Name = Guid.NewGuid().ToString("D"), Cluster = cluster
};
var descriptions =
dset
.DistributeN(cluster.NumNodes * 4, Enumerable.Range(0, cluster.NumNodes * 8))
.Select(i =>
{
var machineName = System.Environment.MachineName;
var process = System.Diagnostics.Process.GetCurrentProcess();
var description = $"{machineName}-{process.Id}-{Thread.CurrentThread.ManagedThreadId}-{i}";
return(description);
})
.ToIEnumerable()
.ToArray();
foreach (var description in descriptions)
{
Console.WriteLine(description);
}
}
private bool Match(Cluster cluster, string matchWord)
{
var name = "Sherlock-Holmes-" + Guid.NewGuid().ToString("D");
var corpus = File.ReadAllLines(Book);
// Store the file to the cluster
var dset = new DSet<string> { Name = name, Cluster = cluster };
dset.Store(corpus);
var corpusLines = dset.LoadSource();
var count1 =
corpusLines.SelectMany(SplitWords)
.Where(w => w == matchWord)
.Count();
Console.WriteLine("Counted with DSet: The word {0} occurs {1} times", matchWord, count1);
var count2 =
corpus.SelectMany(SplitWords)
.Where(w => w == matchWord)
.Count();
Console.WriteLine("Counted locally: The word {0} occurs {1} times", matchWord, count2);
return count1 == count2;
}
public float GetAccuracy(DSet <Example> dataset, ISGDModel <Params> model, Params curParams)
{
int hits = dataset.Fold((c, ex) => model.Predict(curParams, ex) == ex.Label ? c + 1 : c, (c1, c2) => c1 + c2, 0);
return((float)hits / dataset.Count());
}
public static global::System.Xml.Schema.XmlSchemaComplexType GetTypedTableSchema(global::System.Xml.Schema.XmlSchemaSet xs)
{
global::System.Xml.Schema.XmlSchemaComplexType type = new global::System.Xml.Schema.XmlSchemaComplexType();
global::System.Xml.Schema.XmlSchemaSequence sequence = new global::System.Xml.Schema.XmlSchemaSequence();
DSet ds = new DSet();
global::System.Xml.Schema.XmlSchemaAny any1 = new global::System.Xml.Schema.XmlSchemaAny();
any1.Namespace = "http://www.w3.org/2001/XMLSchema";
any1.MinOccurs = new decimal(0);
any1.MaxOccurs = decimal.MaxValue;
any1.ProcessContents = global::System.Xml.Schema.XmlSchemaContentProcessing.Lax;
sequence.Items.Add(any1);
global::System.Xml.Schema.XmlSchemaAny any2 = new global::System.Xml.Schema.XmlSchemaAny();
any2.Namespace = "urn:schemas-microsoft-com:xml-diffgram-v1";
any2.MinOccurs = new decimal(1);
any2.ProcessContents = global::System.Xml.Schema.XmlSchemaContentProcessing.Lax;
sequence.Items.Add(any2);
global::System.Xml.Schema.XmlSchemaAttribute attribute1 = new global::System.Xml.Schema.XmlSchemaAttribute();
attribute1.Name = "namespace";
attribute1.FixedValue = ds.Namespace;
type.Attributes.Add(attribute1);
global::System.Xml.Schema.XmlSchemaAttribute attribute2 = new global::System.Xml.Schema.XmlSchemaAttribute();
attribute2.Name = "tableTypeName";
attribute2.FixedValue = "dtDataTable";
type.Attributes.Add(attribute2);
type.Particle = sequence;
global::System.Xml.Schema.XmlSchema dsSchema = ds.GetSchemaSerializable();
if (xs.Contains(dsSchema.TargetNamespace))
{
global::System.IO.MemoryStream s1 = new global::System.IO.MemoryStream();
global::System.IO.MemoryStream s2 = new global::System.IO.MemoryStream();
try {
global::System.Xml.Schema.XmlSchema schema = null;
dsSchema.Write(s1);
for (global::System.Collections.IEnumerator schemas = xs.Schemas(dsSchema.TargetNamespace).GetEnumerator(); schemas.MoveNext();)
{
schema = ((global::System.Xml.Schema.XmlSchema)(schemas.Current));
s2.SetLength(0);
schema.Write(s2);
if ((s1.Length == s2.Length))
{
s1.Position = 0;
s2.Position = 0;
for (; ((s1.Position != s1.Length) &&
(s1.ReadByte() == s2.ReadByte()));)
{
;
}
if ((s1.Position == s1.Length))
{
return(type);
}
}
}
}
finally {
if ((s1 != null))
{
s1.Close();
}
if ((s2 != null))
{
s2.Close();
}
}
}
xs.Add(dsSchema);
return(type);
}
static void Main(string[] args)
{
Arguments arg;
if ((arg = Arguments.Parse(args)) == null)
{
return;
}
var sw = System.Diagnostics.Stopwatch.StartNew();
Console.Write("Initializing/Connecting to cluster... ");
Logger.ParseArgs(new string[] { "-log", "DistributedSGD.log" });
Prajna.Core.Environment.Init();
var cluster = new Cluster(arg.Cluster);
Console.WriteLine($"done. (took: {sw.Elapsed})");
sw.Restart();
Console.Write("Loading dataset(s)... ");
DSet <Example> trainSet = LoadDSet(cluster, arg.TrainFile, arg.NumPartitions, arg.NumTrain > 0 ? arg.NumTrain : int.MaxValue, 0);
DSet <Example> testSet = arg.TestFile == null ? null : LoadDSet(cluster, arg.TestFile, arg.NumPartitions, arg.NumTest > 0 ? arg.NumTest : int.MaxValue, 0);
Func <DSet <Example>, int> getDimension = ds => ds.Fold((max, ex) => Math.Max(ex.Features.Indices.Max(), max), Math.Max, 0) + 1;
var dimension = Math.Max(getDimension(trainSet), arg.TestFile == null ? 0 : getDimension(testSet));
Console.WriteLine($"done. (took: {sw.Elapsed})");
SetDimension(trainSet, dimension);
Console.WriteLine($"Train Count: {trainSet.Count()}");
float trainPrior = GetPrior(trainSet);
Console.WriteLine($"Train Prior: {trainPrior}");
if (arg.TestFile != null)
{
SetDimension(testSet, dimension);
Console.WriteLine($"Test Count: {testSet.Count()}");
float testPrior = GetPrior(testSet);
Console.WriteLine($"Test Prior: {testPrior}");
}
ILossFunction loss;
switch (arg.Loss.ToLower())
{
case "logistic":
loss = new LogisticLoss();
break;
case "hinge":
loss = new HingeLoss();
break;
default:
Console.WriteLine($"Unrecognized loss function: {arg.Loss}. Supported values: Hinge, Logistic.");
return;
}
if (arg.ModelOut != null)
{
if (!Directory.Exists(Path.GetDirectoryName(arg.ModelOut)))
{
Console.WriteLine($"Directory {Path.GetDirectoryName(arg.ModelOut)} not found.");
return;
}
}
var model = new LinearModel(new HingeLoss(), arg.LearningRate, arg.L2, arg.L1);
var initialParams = new WeightsAndBias(new float[dimension], 0.0f);
WeightsAndBias finalParams = RunSGD(trainSet, testSet, model, initialParams, DistributedSGD <WeightsAndBias> .Instance, arg.NumEpochs);
if (arg.ModelOut != null)
{
Console.WriteLine();
Console.Write("Done training. Saving model... ");
sw.Restart();
using (var writer = new StreamWriter(arg.ModelOut))
{
writer.WriteLine("Dimension:");
writer.WriteLine(finalParams.Weights.Length);
writer.WriteLine("Weights:");
foreach (var w in finalParams.Weights)
{
writer.WriteLine(w);
}
writer.WriteLine("Bias:");
writer.WriteLine(finalParams.Bias);
}
Console.WriteLine($"done. (took: {sw.Elapsed})");
}
}
/// <summary>
/// The "prior" of a dataset is simply the number of positive examples divided by the size of the dataset.
/// </summary>
/// <param name="examples"></param>
/// <returns></returns>
private static float GetPrior(DSet <Example> examples)
{
long hits = examples.Fold((c, ex) => ex.Label == 1.0f ? c + 1 : c, (c1, c2) => c1 + c2, 0);
return((float)hits / examples.Count());
}
static void SetDimension(DSet <Example> ds, int dim)
{
ds.Iter(ex => ex.Features.Dimension = dim);
}
public bool Accept(DSet type)
{
return(type.Datas.Count == 0);
}
public void Accept(DSet type, TType x, List <ResourceInfo> y)
{
Accept(type.Datas, type.Type.ElementType, y);
}
public void Accept(DSet type, DefField x, List <ResourceInfo> y)
{
Accept(x, type.Datas, type.Type.ElementType, y);
}
public void Accept(DSet type, DefAssembly x)
{
throw new NotImplementedException();
}
public void Accept(DSet type, RawTextTable x)
{
}
public Evaluator(Interpreter interpreter)
{
i = interpreter;
interpreter.evaluator = this;
var throwStatement = new Operation("throw")
{
association = Operation.Association.Right,
unaryFunction = (right) =>
{
throw i.Exception(right.ToString());
}
};
var tryCatch = new Operation("try")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
int depth = i.GetLocation().depth;
try
{
i.ExecuteBlock();
i.Eat("catch");
i.EatUntilToken("l curly");
i.SkipBlock();
}
catch (InterpreterException e)
{
i.SkipToDepth(depth);
i.Eat("catch");
string exceptionName = null;
string lineName = null;
string columnName = null;
if (i.CurrentToken == "of")
{
i.Eat();
exceptionName = i.GetIdentifier();
i.heap.DeclareReadOnly(exceptionName, new DException(e.message.ToDString()));
if (i.CurrentToken == "comma")
{
i.Eat();
lineName = i.GetIdentifier();
i.heap.DeclareReadOnly(lineName, e.location.line.ToDNumber());
}
if (i.CurrentToken == "comma")
{
i.Eat();
columnName = i.GetIdentifier();
i.heap.DeclareReadOnly(columnName, e.location.column.ToDNumber());
}
}
i.ExecuteBlock();
if (exceptionName != null)
{
i.heap.DeleteLocal(exceptionName);
}
if (lineName != null)
{
i.heap.DeleteLocal(lineName);
}
if (columnName != null)
{
i.heap.DeleteLocal(columnName);
}
}
return(dvoid);
}
};
var include = new Operation("include")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
var filename = Evaluate <DString>().ToString();
if (i.includes.Contains(filename))
{
throw new InterpreterException(i.CurrentToken, "File has already been included: " + filename);
}
i.TryEat("semicolon");
var tokens = Lexer.ScanFile(filename);
tokens.RemoveAll(x => x == "eof");
i.tokens.InsertRange(i.pointer, tokens);
i.includes.Add(filename);
return(dvoid);
}
};
var charLiteral = new Operation("char")
{
eatOperator = false,
association = Operation.Association.None,
unaryFunction = (none) =>
{
return(i.Eat <char>().ToDChar());
}
};
var baseLiteral = new Operation("base")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
var super = i.functionOwners.Peek().GetMember("base");
lastVariable = super;
if (i.CurrentToken == "l bracket")
{
i.Eat();
var p = i.GetParameters();
((DFunction)super.Value.GetMemberValue("constructor")).Call(p);
}
return(lastVariable.Value);
}
};
var thisLiteral = new Operation("this")
{
association = Operation.Association.None,
unaryFunction = (none) => {
lastVariable = new ReadOnlyVariable(i.functionOwners.Peek(), i.depth);
return(lastVariable.Value);
}
};
var structureLiteral = new Operation("structure")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
DTemplate super = null;
if (i.CurrentToken == "extending")
{
i.Eat();
var extending = Evaluate();
if (extending is DTemplate)
{
super = (DTemplate)extending;
}
else
{
throw new InterpreterException(i.CurrentToken, "Expected a strong template or a weak template");
}
}
var structure = new DWeakTemplate()
{
i = i,
definition = i.GetLocation(),
super = super
};
while (i.CurrentToken != "l curly")
{
i.Eat();
}
i.SkipBlock();
return(structure);
}
};
var funcLiteral = new Operation("function")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
i.TryEat("of");
var location = i.GetLocation();
DWeakFunction func = new DWeakFunction()
{
i = i,
location = location
};
while (i.CurrentToken != "l curly")
{
i.Eat();
}
i.SkipBlock();
return(func);
}
};
var tableLiteral = new Operation("table")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
i.UnsafeEat("l curly");
var table = new DTable();
var members = new Dictionary <string, Member>();
var properties = new Dictionary <string, WeakProperty>();
while (i.CurrentToken != "r curly")
{
if (i.CurrentToken == "get")
{
i.Eat();
string name = i.GetIdentifier();
DWeakFunction getter = (DWeakFunction)funcLiteral.unaryFunction.Invoke(null);
getter.owner = table;
properties.Add(name, new WeakProperty(getter));
}
else if (i.CurrentToken == "set")
{
i.Eat();
string name = i.GetIdentifier();
if (!properties.ContainsKey(name))
{
throw i.Exception("Must declare a getter for " + name + " before declaring a setter.");
}
DWeakFunction setter = (DWeakFunction)funcLiteral.unaryFunction.Invoke(null);
setter.owner = table;
properties[name].setter = setter;
}
else if (i.CurrentToken == "with")
{
i.Eat();
string name = i.GetIdentifier();
DObject value = new DUndefined();
if (i.CurrentToken == "equals")
{
i.Eat();
value = Evaluate();
if (value.GetType() == typeof(DWeakFunction))
{
((DWeakFunction)value).owner = table;
}
}
members.Add(name, new Member(value));
}
else if (i.CurrentToken == "constructor")
{
i.Eat();
DWeakFunction ctor = (DWeakFunction)funcLiteral.unaryFunction.Invoke(null);
ctor.owner = table;
members.Add("constructor", new ReadOnlyMember(ctor));
}
else if (i.CurrentToken == "function")
{
i.Eat();
string name = i.GetIdentifier();
DWeakFunction ctor = (DWeakFunction)funcLiteral.unaryFunction.Invoke(null);
ctor.owner = table;
members.Add(name, new Member(ctor));
}
else
{
throw i.Exception("Unexpected token: " + i.CurrentToken.Type);
}
i.TryEat("semicolon");
}
properties.ToList().ForEach(x => members.Add(x.Key, x.Value));
table.SetMembers(members);
i.UnsafeEat("r curly");
return(table);
}
};
tableOp = tableLiteral;
var returnStatement = new Operation("return")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
i.returnValue = Evaluate();
return(dvoid);
}
};
var continueStatement = new Operation("continue")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
i.shouldContinue = true;
return(dvoid);
}
};
var breakStatement = new Operation("break")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
i.shouldBreak = true;
return(dvoid);
}
};
var forEachLoop = new Operation("for")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
i.Eat("each");
string varName = i.GetIdentifier();
i.Eat("in");
DSet set = Evaluate <DSet>();
var j = i.heap.DeclareLocal(varName);
var beginning = i.GetLocation();
i.BeginLoop(beginning);
foreach (var item in set.items)
{
if (i.shouldBreak)
{
break;
}
j.Value = item.Value;
i.shouldContinue = false;
i.ExecuteLoop();
i.Goto(beginning);
}
i.heap.DeleteLocal(varName);
i.EndLoop();
return(dvoid);
}
};
var forLoop = new Operation("from")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
int from = Evaluate <DNumber>().ToInt();
i.Eat("to");
int to = Evaluate <DNumber>().ToInt();
int increment;
if (from > to)
{
increment = -1;
}
else
{
increment = 1;
}
if (i.CurrentToken == "by")
{
i.Eat();
increment = Evaluate <DNumber>().ToInt();
}
i.Eat("with");
string varName = i.GetIdentifier();
var j = i.heap.DeclareLocal(varName);
var beginning = i.GetLocation();
i.BeginLoop(beginning);
for (int k = from; increment > 0 ? k <to : k> to; k += increment)
{
if (i.shouldBreak)
{
break;
}
j.Value = k.ToDNumber();
i.ExecuteLoop();
i.Goto(beginning);
}
i.heap.DeleteLocal(varName);
i.EndLoop();
return(dvoid);
}
};
var untilLoop = new Operation("until")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
var beginning = i.GetLocation();
i.BeginLoop(beginning);
while (!i.GetCondition())
{
if (i.shouldBreak)
{
break;
}
i.ExecuteLoop();
i.Goto(beginning);
}
i.EndLoop();
return(dvoid);
}
};
/*
* var instantiation = new Operation("new")
* {
* association = Operation.Association.None,
* unaryFunction = (none) =>
* {
* var template = Evaluate<DTemplate>();
* i.Eat("l bracket");
* var args = i.GetParameters();
* return template.Instantiate(args);
* }
* };
*/
var instantiation = new Operation("new")
{
association = Operation.Association.Right,
unaryFunction = (right) =>
{
//var template = Evaluate<DTemplate>();
i.Eat("l bracket");
var args = i.GetParameters();
return(DObject.AssertType <DTemplate>(right).Instantiate(args));
}
};
var whileLoop = new Operation("while")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
var beginning = i.GetLocation();
i.BeginLoop(beginning);
while (i.GetCondition())
{
if (i.shouldBreak)
{
break;
}
i.ExecuteLoop();
i.Goto(beginning);
}
i.EndLoop();
return(dvoid);
}
};
var ifStatement = new Operation("if")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
bool satisfied = i.GetCondition();
if (satisfied)
{
i.ExecuteBlock();
}
else
{
i.SkipBlock();
}
while (i.CurrentToken == "else" && i.NextToken == "if")
{
i.Eat();
i.Eat();
bool meetsCondition = i.GetCondition();
if (satisfied || !meetsCondition)
{
i.SkipBlock();
}
else
{
satisfied = true;
i.ExecuteBlock();
}
}
if (i.CurrentToken == "else")
{
i.Eat();
if (satisfied)
{
i.SkipBlock();
}
else
{
i.ExecuteBlock();
}
}
return(dvoid);
}
};
var openBracket = new Operation("l curly")
{
eatOperator = false,
association = Operation.Association.None,
unaryFunction = (none) =>
{
i.Eat();
return(dvoid);
}
};
var closeBracket = new Operation("r curly")
{
eatOperator = false,
association = Operation.Association.None,
unaryFunction = (none) =>
{
i.Eat();
return(dvoid);
}
};
var preInc = new Operation("increment")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
var right = Evaluate();
return(lastVariable.Value = (DObject.AssertType <DNumber>(lastVariable.Value).ToFloat() + 1).ToDNumber());
}
};
var incAdd = new Operation("add")
{
association = Operation.Association.Left,
unaryFunction = (left) =>
{
return(lastVariable.Value = lastVariable.Value.OpADD(Evaluate()));
}
};
var incSub = new Operation("subtract")
{
association = Operation.Association.Left,
unaryFunction = (left) =>
{
return(lastVariable.Value = lastVariable.Value.OpSUB(Evaluate()));
}
};
var postInc = new Operation("increment")
{
association = Operation.Association.Left,
unaryFunction = (left) =>
{
return(lastVariable.Value = (DObject.AssertType <DNumber>(lastVariable.Value).ToFloat() + 1).ToDNumber());
}
};
var preDec = new Operation("decrement")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
var right = Evaluate();
return(lastVariable.Value = (DObject.AssertType <DNumber>(lastVariable.Value).ToFloat() - 1).ToDNumber());
}
};
var postDec = new Operation("decrement")
{
association = Operation.Association.Left,
unaryFunction = (left) =>
{
return(lastVariable.Value = (DObject.AssertType <DNumber>(lastVariable.Value).ToFloat() - 1).ToDNumber());
}
};
var semicolon = new Operation("semicolon")
{
association = Operation.Association.None,
unaryFunction = (none) => dvoid
};
var declaration = new Operation("var")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
if (i.CurrentToken == "structure")
{
i.Eat();
string name = i.GetIdentifier();
DWeakTemplate value = (DWeakTemplate)structureLiteral.unaryFunction(null);
i.heap.DeclareLocal(name, value);
}
else if (i.CurrentToken == "table")
{
i.Eat();
string name = i.GetIdentifier();
DTable value = (DTable)tableLiteral.unaryFunction(null);
i.heap.DeclareLocal(name, value);
}
else if (i.CurrentToken == "function")
{
i.Eat();
string name = i.GetIdentifier();
DWeakFunction value = (DWeakFunction)funcLiteral.unaryFunction(null);
i.heap.DeclareLocal(name, value);
}
else
{
string name = i.Eat <string>("identifier");
DObject value = DUndefined.instance;
if (i.CurrentToken == "equals")
{
i.Eat();
value = Evaluate();
}
i.heap.DeclareLocal(name, value);
}
return(dvoid);
}
};
var assignment = new Operation("equals")
{
association = Operation.Association.Left,
unaryFunction = (left) =>
{
var v = lastVariable;
return(v.Value = Evaluate());
}
};
var identifier = new Operation("identifier")
{
eatOperator = false,
association = Operation.Association.None,
unaryFunction = (none) =>
{
string name = i.Eat <string>("identifier");
if (!i.heap.Exists(name))
{
throw new InterpreterException(i.PreviousToken, "Variable does not exist in this scope: " + name);
}
lastVariable = i.heap.Get(name);
if (i.CurrentToken == "equals")
{
return(DUndefined.instance);
}
return(lastVariable.Value);
}
};
var conditional = new Operation("question mark")
{
association = Operation.Association.Left,
unaryFunction = (left) =>
{
var ifTrue = Evaluate();
i.Eat("else");
var ifFalse = Evaluate();
if (DObject.AssertType <DBool>(left).ToBool())
{
return(ifTrue);
}
else
{
return(ifFalse);
}
}
};
var traverse = new Operation("dot")
{
association = Operation.Association.Left,
unaryFunction = (left) =>
{
var name = i.GetIdentifier();
var type = left.GetType();
if (left.HasMember(name))
{
lastVariable = left.GetMember(name);
if (i.CurrentToken == "equals")
{
return(DUndefined.instance);
}
return(lastVariable.Value);
}
else if (StrongTypeRegistry.strongFunctions.ContainsKey(type))
{
var methods = StrongTypeRegistry.strongFunctions[type];
if (methods.ContainsKey(name))
{
var method = methods[name];
lastVariable = null;
return(new DStrongFunction(method, left));
}
else if (left.HasMember("base"))
{
i.pointer -= 2; // recurse back to the dot, with base being the new left side
return(left.GetMemberValue("base"));
}
}
else if (left.HasMember("base"))
{
i.pointer -= 2; // recurse back to the dot, with base being the new left side
return(left.GetMemberValue("base"));
}
throw new InterpreterException(i.CurrentToken, "Object (" + type.Name + ") does not contain member: " + name);
}
};
var methodCall = new Operation("l bracket")
{
association = Operation.Association.Left,
unaryFunction = (left) => {
var parameters = i.GetParameters();
if (left is DFunction)
{
return(((DFunction)left).Call(parameters));
}
else if (left is DTemplate)
{
return(((DTemplate)left).Instantiate(parameters));
}
else
{
throw i.Exception("Can only invoke a function or a template, not a " + left.GetType().Name);
}
}
};
/*var eof = new Operation("eof")
* {
* eatOperator = false,
* association = Operation.Association.None,
* unaryFunction = (none) => DUndefined.instance
* };*/
var arrayLiteral = new Operation("l square")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
List <DObject> p = new List <DObject>();
if (i.CurrentToken == "r square")
{
i.Eat();
return(new DSet(new List <DObject>()));
}
bool firstTime = true;
do
{
if (!firstTime)
{
i.Eat("comma");
}
p.Add(Evaluate());
firstTime = false;
}while (i.CurrentToken == "comma");
i.Eat("r square");
return(new DSet(p));
}
};
var undefinedLiteral = new Operation("undefined")
{
association = Operation.Association.None,
unaryFunction = (none) => DUndefined.instance
};
var numberLiteral = new Operation("number")
{
eatOperator = false,
association = Operation.Association.None,
unaryFunction = (none) => i.Eat <float>().ToDNumber()
};
var stringLiteral = new Operation("string")
{
eatOperator = false,
association = Operation.Association.None,
unaryFunction = (none) => i.Eat <string>().ToDString()
};
var boolLiteral = new Operation("bool")
{
eatOperator = false,
association = Operation.Association.None,
unaryFunction = (none) => i.Eat <bool>().ToDBool()
};
var index = new Operation("l square")
{
association = Operation.Association.Left,
unaryFunction = (left) =>
{
lastVariable = left.OpINDEX(Evaluate());
i.Eat("r square");
return(lastVariable.Value);
}
};
var bracket = new Operation("l bracket")
{
association = Operation.Association.None,
unaryFunction = (none) =>
{
var result = Evaluate();
i.Eat("r bracket");
return(result);
}
};
var neg = new Operation("minus")
{
association = Operation.Association.Right,
unaryFunction = (right) => right.OpNEG()
};
var pow = new Operation("exponent")
{
binaryFunction = (left, right) => left.OpPOW(right)
};
var mul = new Operation("multiply")
{
binaryFunction = (left, right) => left.OpMUL(right)
};
var div = new Operation("divide")
{
binaryFunction = (left, right) => left.OpDIV(right)
};
var mod = new Operation("modulus")
{
binaryFunction = (left, right) => left.OpMOD(right)
};
var add = new Operation("plus")
{
binaryFunction = (left, right) => left.OpADD(right)
};
var sub = new Operation("minus")
{
binaryFunction = (left, right) => left.OpSUB(right)
};
var not = new Operation("not")
{
association = Operation.Association.Right,
unaryFunction = (right) => DObject.AssertType <DBool>(right).OpNOT()
};
var eq = new Operation("double equal")
{
binaryFunction = (left, right) => left.OpEQ(right)
};
var neq = new Operation("not equal")
{
binaryFunction = (left, right) => left.OpNEQ(right)
};
var gr = new Operation("greater than")
{
binaryFunction = (left, right) => left.OpGR(right)
};
var ls = new Operation("less than")
{
binaryFunction = (left, right) => left.OpLS(right)
};
var geq = new Operation("greater or equal")
{
binaryFunction = (left, right) => left.OpGEQ(right)
};
var leq = new Operation("less or equal")
{
binaryFunction = (left, right) => left.OpLEQ(right)
};
var and = new Operation("and")
{
binaryFunction = (left, right) => DObject.AssertType <DBool>(left).OpAND(DObject.AssertType <DBool>(right))
};
var or = new Operation("or")
{
binaryFunction = (left, right) => {
if (left is DUndefined)
{
return(right);
}
else if (left.GetType() == typeof(DBool) && right.GetType() == typeof(DBool))
{
return(((DBool)left).OpOR((DBool)right));
}
return(left);
//return DObject.AssertType<DBool>(left).OpOR(DObject.AssertType<DBool>(right));
}
};
//operations.Add(eof);
void register(Operation op)
{
precedences.Last().Add(op);
}
void precedence()
{
precedences.Add(new List <Operation>());
}
precedence();
register(thisLiteral);
register(baseLiteral);
register(bracket);
register(undefinedLiteral);
register(arrayLiteral);
register(numberLiteral);
register(stringLiteral);
register(boolLiteral);
register(tableLiteral);
register(charLiteral);
//register(instantiation);
precedence();
register(identifier);
register(methodCall);
register(traverse);
register(index);
precedence();
register(assignment);
register(preInc);
register(postInc);
register(preDec);
register(postDec);
register(incAdd);
register(incSub);
precedence();
register(neg);
register(not);
precedence();
register(pow);
precedence();
register(mul);
register(div);
register(mod);
precedence();
register(add);
register(sub);
precedence();
register(eq);
register(neq);
register(gr);
register(ls);
register(geq);
register(leq);
precedence();
register(and);
precedence();
register(or);
precedence();
register(conditional);
precedence();
register(semicolon);
register(declaration);
register(closeBracket);
register(openBracket);
register(ifStatement);
register(whileLoop);
register(untilLoop);
register(forEachLoop);
register(continueStatement);
register(breakStatement);
register(funcLiteral);
register(returnStatement);
register(forLoop);
register(structureLiteral);
register(include);
register(tryCatch);
register(throwStatement);
}
static void Main(string[] args)
{
Assembly assem = Assembly.GetExecutingAssembly();
bool hasEmbeddedFile = assem.GetManifestResourceNames().Contains("DormezInterpreter.program.dmz");
List <Token> tokens;
if (hasEmbeddedFile)
{
string embeddedCode;
using (Stream stream = assem.GetManifestResourceStream("DormezInterpreter.program.dmz"))
{
using (var reader = new StreamReader(stream))
{
embeddedCode = reader.ReadToEnd();
}
}
tokens = Lexer.ScanString(embeddedCode);
}
else if (args.Length > 0)
{
string filename = args[0];
if (!File.Exists(filename))
{
Console.WriteLine("File does not exist: " + filename);
Console.ReadKey();
return;
}
tokens = Lexer.ScanFile(filename);
}
else
{
Console.WriteLine("Proper usage: DormezInterpreter.exe <filename> [arg1] [arg2] ...");
Console.ReadKey();
return;
}
var interpreter = new Interpreter(tokens);
var evaluator = new Evaluator(interpreter);
try
{
var argList = args.ToList();
if (!hasEmbeddedFile && args.Length > 0)
{
argList.RemoveAt(0);
}
var dArgs = new DSet(argList.Select(x => x.ToDString()));
interpreter.heap.DeclareGlobalVariable("args", dArgs);
interpreter.Execute();
}
catch (InterpreterException e)
{
Console.WriteLine("FATAL ERROR @ " + e.Message);
Console.ReadKey();
}
}
public int Accept(DSet data, TType type, Title x)
{
SetTitleValue(x, data.Apply(ToExcelStringVisitor.Ins, type.GetTag("sep")));
return(1);
}
public abstract string Accept(DSet type);
public void Accept(DSet type, StringBuilder x)
{
Append(type.Datas, x);
}
public void Accept(DSet type, ByteBuf x)
{
WriteList(type.Datas, x);
}
