public static FastReadPlan RenderFastReadPlan(Workspace Home, HScriptParser.Crudam_read_fastContext context)
{
// Get the data source //
DataSet data = VisitorHelper.GetData(Home, context.full_table_name());
string alias =
(context.K_AS() != null)
? context.IDENTIFIER().GetText()
: data.Name;
// Create a record register //
StreamRegister memory = new StreamRegister(null);
// Create expression visitor //
ExpressionVisitor exp_vis = new ExpressionVisitor(new MemoryStruct(true), Home, alias, data.Columns, memory);
// Where clause //
Predicate where = VisitorHelper.GetWhere(exp_vis, context.where_clause());
// Create a reader //
RecordReader reader = data.OpenReader(where);
// Attach the reader to the register //
memory.BaseStream = reader;
// Get the fields being returned //
FNodeSet nodes = VisitorHelper.GetReturnStatement(exp_vis, context.return_action().expression_or_wildcard_set());
// Get the output cursor from the return statement //
RecordWriter writer = VisitorHelper.GetWriter(Home, nodes.Columns, context.return_action());
return new FastReadPlan(data, where, nodes, writer);
}
// Get the data //
public static DataSet GetData(Workspace Enviro, HScriptParser.Full_table_nameContext context)
{
// Get the name //
string t_name = context.table_name().IDENTIFIER().GetText();
// Global context //
if (context.K_GLOBAL() != null)
{
if (Enviro.ChunkHeap.Exists(t_name))
return Enviro.ChunkHeap[t_name];
throw new HScriptCompileException("Global chunk '{0}' does not exist", t_name);
}
// Table context //
if (context.database_name() != null)
{
string d_base = context.database_name().IDENTIFIER().GetText();
if (Enviro.Exists(d_base, t_name))
return Enviro.GetStaticTable(d_base, t_name);
throw new HScriptCompileException("Table '{0}' does not exist", t_name);
}
throw new HScriptCompileException("Data '{0}' does not exist in memory or on disk", t_name);
}
public CreateChunkPlan(string Name, Schema Columns, Workspace Space)
{
this._name = Name;
this._columns = Columns;
this._space = Space;
this.Name = "CREATE_CHUNK";
}
static void Main(string[] args)
{
string Notifier = "";
Stopwatch sw = new Stopwatch();
sw.Start();
long bytes = GC.GetTotalMemory(false);
//double[][] data = new double[4][];
//data[0] = new double[] {0D, 0D, 0D};
//data[1] = new double[] {1D, 0D, 1D};
//data[2] = new double[] {0D, 1D, 1D};
//data[3] = new double[] {1D, 1D, 0D};
//Matrix m = new Matrix(4, 3);
//m[0] = data[0];
//m[1] = data[1];
//m[2] = data[2];
//m[3] = data[3];
//Thoroughbred.WarAdmiral.LinkUpdate rule = Thoroughbred.WarAdmiral.LinkUpdateFactory.Generate("b;0.70;0.0");
//Thoroughbred.WarAdmiral.LinkUpdate rule = Thoroughbred.WarAdmiral.LinkUpdateFactory.Generate("m");
//Thoroughbred.WarAdmiral.LinkUpdate rule = Thoroughbred.WarAdmiral.LinkUpdateFactory.Generate("r");
//Thoroughbred.WarAdmiral.LinkUpdate rule = Thoroughbred.WarAdmiral.LinkUpdateFactory.Generate("r+");
//Thoroughbred.WarAdmiral.LinkUpdate rule = Thoroughbred.WarAdmiral.LinkUpdateFactory.Generate("r-");
//Thoroughbred.WarAdmiral.LinkUpdate rule = Thoroughbred.WarAdmiral.LinkUpdateFactory.Generate("ir+");
//Thoroughbred.WarAdmiral.LinkUpdate rule = Thoroughbred.WarAdmiral.LinkUpdateFactory.Generate("ir-");
//Thoroughbred.WarAdmiral.LinkUpdate rule = Thoroughbred.WarAdmiral.LinkUpdateFactory.Generate("test");
//Thoroughbred.WarAdmiral.Network net = new Thoroughbred.WarAdmiral.Network(rule, data);
//net.AddDataLayer(new int[] { 0, 1 }, true);
//net.AddHiddenLayer(2, true, new Numerics.BinarySigmoid());
//net.AddOutputLayer(new int[] { 2 }, new Numerics.BinarySigmoid());
//net.Construct();
//net.Render();
//net.PrintPredictions(data);
//Console.WriteLine(net.Statistics());
//Console.WriteLine("--------------------------------------");
//Thoroughbred.ManOWar.SimpleNetwork sim = new SimpleNetwork();
//sim.TrainOne(m);
//sim.PrintGradients();
Workspace space = new Workspace(@"C:\Users\pwdlu_000\Documents\Equus\X_Data\Temp_Database\");
HScriptProcessor runner = new HScriptProcessor(space);
string script = File.ReadAllText(@"C:\Users\pwdlu_000\Documents\Equus\Equus\HScript\TestScript.txt");
runner.Execute(script);
bytes = GC.GetTotalMemory(false) - bytes;
Console.WriteLine("Disk Reads: {0} ::::: Disk Writes: {1}", BinarySerializer.DiskReads, BinarySerializer.DiskWrites);
Console.WriteLine(":::::: Complete {0} : {1}kb ::::::", sw.Elapsed.ToString(), Math.Round(((double)bytes) / (1024.00), 2));
Notifier = Console.ReadLine();
}
public Procedure(string Name, string Description, Workspace UseSpace, HParameterSet UseParameters)
{
this._Name = Name;
this._ProcedureDescription = Description;
this._Parameters = UseParameters;
this._Home = UseSpace;
this._ParametersMap = new Dictionary<string, ProcedureParameterMetaData>(StringComparer.OrdinalIgnoreCase);
}
// Reads //
public static StagedReadName RenderStagedReadPlan(Workspace Home, HScriptParser.Crudam_readContext context)
{
// Get the data source //
DataSet data = VisitorHelper.GetData(Home, context.full_table_name());
string alias =
(context.K_AS() != null)
? context.IDENTIFIER().GetText()
: data.Name;
// Create a local heap to work off of //
MemoryStruct local_heap = new MemoryStruct(true);
// Create a record register //
StreamRegister memory = new StreamRegister(null);
// Create expression visitor //
ExpressionVisitor exp_vis = new ExpressionVisitor(local_heap, Home, alias, data.Columns, memory);
// Where clause //
Predicate where = VisitorHelper.GetWhere(exp_vis, context.where_clause());
// Create a reader //
RecordReader reader = data.OpenReader(where);
// Attach the reader to the register //
memory.BaseStream = reader;
// Create the action visitor //
ActionVisitor act_vis = new ActionVisitor(Home, local_heap, exp_vis);
// Get the declarations //
if (context.crudam_declare_many() != null)
{
VisitorHelper.AllocateMemory(Home, local_heap, exp_vis, context.crudam_declare_many());
}
// Get the initial actionset //
TNode pre_run =
(context.init_action() != null)
? act_vis.ToNode(context.init_action().query_action())
: new TNodeNothing(null);
// Get the main actionset //
TNode run = act_vis.ToNode(context.main_action().query_action());
// Get the final actionset //
TNode post_run =
(context.final_action() != null)
? act_vis.ToNode(context.final_action().query_action())
: new TNodeNothing(null);
return new StagedReadName(reader, pre_run, run, post_run);
}
public static DataSet GetData(Workspace Enviro, FNodeSet Nodes, HScriptParser.Return_actionContext context)
{
// Get the table name //
string name = context.full_table_name().table_name().IDENTIFIER().GetText();
string db =
(context.full_table_name().database_name() == null)
? "global"
: context.full_table_name().database_name().GetText();
// Figure out if we need to append //
bool appendto =
(context.K_INSERT() != null)
? true
: false;
// Global -- Append //
if (context.full_table_name().database_name() == null && appendto)
{
if (Enviro.ChunkHeap.Exists(name))
return Enviro.ChunkHeap[name];
throw new HScriptCompileException(string.Format("Chunk '{0}' does not exist", name));
}
// Static -- Append //
if (appendto)
{
string fullname = db + "." + name;
if (Enviro.Exists(db, name))
return Enviro.GetStaticTable(db, name);
throw new HScriptCompileException(string.Format("Table '{0}' does not exist", fullname));
}
// Global -- Create New //
if (context.full_table_name().database_name() == null)
{
RecordSet data = new RecordSet(Nodes.Columns);
Enviro.ChunkHeap.Reallocate(name, data);
return data;
}
// Static -- Create New //
string dir = Enviro.Connections[db];
Table t = new Table(dir, name, Nodes.Columns);
return t;
}
// Delete //
public static DeletePlan RenderDeletePlan(Workspace Home, HScriptParser.Crudam_deleteContext context)
{
// Get the data source //
DataSet data = VisitorHelper.GetData(Home, context.full_table_name());
// Create expression visitor //
ExpressionVisitor exp_vis = new ExpressionVisitor(null, Home, data.Name, data.Columns, null);
// Get where //
Predicate where = VisitorHelper.GetWhere(exp_vis, context.where_clause());
return new DeletePlan(data, where);
}
public static void AllocateMemory(Workspace Home, MemoryStruct Heap, ExpressionVisitor Evaluator, HScriptParser.DeclareMatrix2DContext context)
{
string name = context.IDENTIFIER().GetText();
CellAffinity type = GetAffinity(context.type());
int rows = (int)Evaluator.ToNode(context.expression()[0]).Evaluate().valueINT;
int cols = (int)Evaluator.ToNode(context.expression()[1]).Evaluate().valueINT;
CellMatrix mat = new CellMatrix(rows, cols, type);
Heap.Arrays.Reallocate(name, mat);
}
public static DataSet CreateData(Workspace Enviro, Schema Columns, HScriptParser.Full_table_nameContext context)
{
// Get the name //
string t_name = context.table_name().IDENTIFIER().GetText();
// Global context //
if (context.database_name() == null)
{
RecordSet rs = new RecordSet(Columns);
Enviro.ChunkHeap.Reallocate(t_name, rs);
return rs;
}
// Table context //
if (context.database_name() != null)
{
string d_base = context.database_name().IDENTIFIER().GetText();
if (!Enviro.Connections.Exists(d_base))
throw new HScriptCompileException("Connection to '{0}' does not exist", d_base);
string dir = Enviro.Connections[d_base];
Table t = new Table(dir, t_name, Columns);
return t;
}
throw new HScriptCompileException("Cannot create data '{0}'", t_name);
}
// Heap allocations //
public static void AllocateMemory(Workspace Home, MemoryStruct Heap, ExpressionVisitor Evaluator, HScriptParser.DeclareScalarContext context)
{
string name = context.IDENTIFIER().GetText();
CellAffinity type = GetAffinity(context.type());
Cell value = (context.ASSIGN() != null) ? Evaluator.ToNode(context.expression()).Evaluate() : new Cell(type);
Heap.Scalars.Reallocate(name, value);
}
internal static DeclareMatrixNode RenderDeclareNode(Workspace Home, ExpressionVisitor Evaluator, HScriptParser.DeclareMatrixLiteralContext context)
{
string name = context.IDENTIFIER().GetText();
MatrixVisitor MEvaluator = new MatrixVisitor(Home, Home.GlobalHeap, Evaluator);
MNode node = MEvaluator.ToMatrix(context.matrix_expression());
return new DeclareMatrixNode(Home.GlobalHeap, name, node);
}
// Create data //
public static bool DataExists(Workspace Enviro, HScriptParser.Full_table_nameContext context)
{
// Get the name //
string t_name = context.table_name().IDENTIFIER().GetText();
// Global context //
if (context.database_name() == null)
{
return Enviro.ChunkHeap.Exists(t_name);
}
// Table context //
if (context.database_name() != null)
{
string d_base = context.database_name().IDENTIFIER().GetText();
return Enviro.Exists(d_base, t_name);
}
return false;
}
public static Lambda RenderLambda(Workspace Home, HScriptParser.Lambda_unitContext context)
{
if (context is HScriptParser.LambdaGenericContext)
return VisitorHelper.RenderLambda(Home, context as HScriptParser.LambdaGenericContext);
else
return VisitorHelper.RenderLambda(Home, context as HScriptParser.LambdaGradientContext);
}
public static void AllocateMemory(Workspace Home, MemoryStruct Heap, ExpressionVisitor Evaluator, HScriptParser.Crudam_declare_manyContext context)
{
foreach (HScriptParser.Declare_genericContext ctx in context.declare_generic())
{
AllocateMemory(Home, Heap, Evaluator, ctx);
}
}
// Update //
public static UpdatePlan RenderUpdatePlan(Workspace Home, HScriptParser.Crudam_updateContext context)
{
// Get the data source //
DataSet data = VisitorHelper.GetData(Home, context.full_table_name());
// Create expression visitor //
ExpressionVisitor exp_vis = new ExpressionVisitor(null, Home, data.Name, data.Columns, null);
// Get where //
Predicate where = VisitorHelper.GetWhere(exp_vis, context.where_clause());
// Create the key and fnodeset //
Key keys = new Key();
FNodeSet expressions = new FNodeSet();
foreach (HScriptParser.Update_unitContext ctx in context.update_unit())
{
keys.Add(data.Columns.ColumnIndex(ctx.IDENTIFIER().GetText()));
expressions.Add(exp_vis.ToNode(ctx.expression()));
}
return new UpdatePlan(data, keys, expressions, where);
}
public static void CommandRun(string[] args)
{
// Communicate //
Console.WriteLine("Welcome to Horse!");
string temp = null;
if (args.Length == 0)
{
Console.WriteLine("Please enter a temp directory:");
temp = Console.ReadLine();
}
else
{
temp = args[0];
}
// Run each command //
Workspace space = new Workspace(temp);
HScriptProcessor runner = new HScriptProcessor(space);
int runs = 0;
string Command = CommOkGo;
while (Command != CommExit)
{
Console.WriteLine("Enter Horse Command:");
Command = Console.ReadLine();
while (Command.Length < 4)
{
Console.WriteLine("Command is invalid. Please enter a new command");
Command = Console.ReadLine();
}
string MajorCommand = Command.Substring(0, 4).ToUpper();
string MinorCommand = Command.Substring(4, Command.Length - 4).Trim();
// Check for exit //
if (MajorCommand == CommExit)
{
return;
}
// File run //
else if (MajorCommand == CommRunFile)
{
string file = File.ReadAllText(MinorCommand);
runner.Execute(file);
}
// Command line run //
else if (MajorCommand == CommRunCommand)
{
runner.Execute(MinorCommand);
}
// Invalid command //
else
{
Console.WriteLine("Command '{0}' is not valid", MajorCommand);
}
runs++;
}
}
// Create //
public static CreateTablePlan RenderCreatePlan(Workspace Home, HScriptParser.Crudam_create_tableContext context)
{
// Build visitor //
ExpressionVisitor exp_vis = new ExpressionVisitor(null, Home);
// Build the schema //
Schema columns = new Schema();
foreach (HScriptParser.Create_table_unitContext ctx in context.create_table_unit())
{
columns.Add(
ctx.IDENTIFIER().GetText(),
VisitorHelper.GetAffinity(ctx.type()),
(ctx.expression() == null) ? true : exp_vis.ToNode(ctx.expression()).Evaluate().valueBOOL,
VisitorHelper.GetSize(ctx.type(), true));
}
string name = context.full_table_name().table_name().GetText();
string db = context.full_table_name().database_name().GetText();
string db_path = Home.Connections[db];
long chunk_size =
(context.create_table_size() == null)
? RecordSet.EstimateMaxRecords(columns)
: exp_vis.ToNode(context.create_table_size().expression()).Evaluate().valueINT;
return new CreateTablePlan(db_path, name, columns, (int)chunk_size);
}
// Action Node //
public static ActionPlan RenderActionPlan(Workspace Home, HScriptParser.Query_actionContext context)
{
ExpressionVisitor exp_vis = new ExpressionVisitor(null, Home);
ActionVisitor act_vis = new ActionVisitor(Home, null, exp_vis);
TNode act = act_vis.ToNode(context);
return new ActionPlan(act);
}
internal static DeclareMatrixNode RenderDeclareNode(Workspace Home, ExpressionVisitor Evaluator, HScriptParser.DeclareMatrix2DContext context)
{
string name = context.IDENTIFIER().GetText();
int row = (int)Evaluator.ToNode(context.expression()[0]).Evaluate().valueINT;
int col = (int)Evaluator.ToNode(context.expression()[1]).Evaluate().valueINT;
CellAffinity affinity = VisitorHelper.GetAffinity(context.type());
MNodeLiteral m = new MNodeLiteral(null, new Gidran.CellMatrix(row, 1, affinity));
return new DeclareMatrixNode(Home.GlobalHeap, name, m);
}
public static void AllocateMemory(Workspace Home, MemoryStruct Heap, ExpressionVisitor Evaluator, HScriptParser.DeclareMatrixLiteralContext context)
{
string name = context.IDENTIFIER().GetText();
CellAffinity type = GetAffinity(context.type());
MatrixVisitor vis = new MatrixVisitor(Home, Heap, Evaluator);
CellMatrix mat = vis.ToMatrix(context.matrix_expression()).Evaluate();
Heap.Arrays.Reallocate(name, mat);
}
public static Lambda RenderLambda(Workspace Home, HScriptParser.LambdaGradientContext context)
{
string NewLambdaName = context.IDENTIFIER()[0].GetText();
string BaseLambdaName = context.IDENTIFIER()[1].GetText();
string GradientVariable = context.IDENTIFIER()[2].GetText();
Lambda mu = Home.Lambdas[BaseLambdaName];
Lambda mu_prime = mu.Gradient(NewLambdaName, GradientVariable);
return mu_prime;
}
public static void AllocateMemory(Workspace Home, MemoryStruct Heap, ExpressionVisitor Evaluator, HScriptParser.Declare_genericContext context)
{
if (context is HScriptParser.DeclareScalarContext)
{
VisitorHelper.AllocateMemory(Home, Heap, Evaluator, context as HScriptParser.DeclareScalarContext);
return;
}
if (context is HScriptParser.DeclareMatrix1DContext)
{
VisitorHelper.AllocateMemory(Home, Heap, Evaluator, context as HScriptParser.DeclareMatrix1DContext);
return;
}
if (context is HScriptParser.DeclareMatrix2DContext)
{
VisitorHelper.AllocateMemory(Home, Heap, Evaluator, context as HScriptParser.DeclareMatrix2DContext);
return;
}
if (context is HScriptParser.DeclareMatrixLiteralContext)
{
VisitorHelper.AllocateMemory(Home, Heap, Evaluator, context as HScriptParser.DeclareMatrixLiteralContext);
return;
}
}
public static DeclarePlan RenderDeclarePlan(Workspace Home, HScriptParser.Crudam_declare_manyContext context)
{
ExpressionVisitor eval = new ExpressionVisitor(null, Home);
DeclarePlan plan = new DeclarePlan(Home.GlobalHeap);
foreach (HScriptParser.Declare_genericContext ctx in context.declare_generic())
{
if (ctx is HScriptParser.DeclareScalarContext)
plan.Add(RenderDeclareNode(Home, eval, ctx as HScriptParser.DeclareScalarContext));
else if (ctx is HScriptParser.DeclareMatrix1DContext)
plan.Add(RenderDeclareNode(Home, eval, ctx as HScriptParser.DeclareMatrix1DContext));
else if (ctx is HScriptParser.DeclareMatrix2DContext)
plan.Add(RenderDeclareNode(Home, eval, ctx as HScriptParser.DeclareMatrix2DContext));
else if (ctx is HScriptParser.DeclareMatrixLiteralContext)
plan.Add(RenderDeclareNode(Home, eval, ctx as HScriptParser.DeclareMatrixLiteralContext));
}
return plan;
}
// Lambdas //
public static Lambda RenderLambda(Workspace Home, HScriptParser.LambdaGenericContext context)
{
// Get the name //
string name = context.IDENTIFIER()[0].GetText();
// Get all the pointers //
List<string> pointers = new List<string>();
for (int i = 1; i < context.IDENTIFIER().Count; i++)
{
string var_name = context.IDENTIFIER()[i].GetText();
pointers.Add(var_name);
}
// Get the expression //
PointerExpressionVisitor pev = new PointerExpressionVisitor(pointers, Home);
// Build the node //
FNode node = pev.ToNode(context.expression());
// Build the lambda //
Lambda mu = new Lambda(name, node, pointers);
return mu;
}
internal static DeclareScalarNode RenderDeclareNode(Workspace Home, ExpressionVisitor Evaluator, HScriptParser.DeclareScalarContext context)
{
string name = context.IDENTIFIER().GetText();
FNode node = Evaluator.ToNode(context.expression());
return new DeclareScalarNode(Home.GlobalHeap, name, node);
}
public HScriptProcessor(Workspace UseHome)
{
this.Home = UseHome;
this.Commands = new List<HScriptParser.CommandContext>();
this._CompileErrorMessages = new List<string>();
}
public static CreateChunkPlan RenderCreateChunk(Workspace Home, HScriptParser.Crudam_create_tableContext context)
{
ExpressionVisitor exp_vis = new ExpressionVisitor(null, Home);
// Build the schema //
Schema columns = new Schema();
foreach (HScriptParser.Create_table_unitContext ctx in context.create_table_unit())
{
columns.Add(
ctx.IDENTIFIER().GetText(),
VisitorHelper.GetAffinity(ctx.type()),
(ctx.expression() == null) ? true : exp_vis.ToNode(ctx.expression()).Evaluate().valueBOOL,
VisitorHelper.GetSize(ctx.type(), true));
}
string name = context.full_table_name().table_name().IDENTIFIER().GetText();
return new CreateChunkPlan(name, columns, Home);
}
// Parameters //
public static HParameterSet GetHParameter(Workspace Home, HScriptParser.Hparameter_setContext context)
{
// Create a set //
HParameterSet parmset = new HParameterSet();
// Step one: cycle through all parameters and look for any tables //
DataSet first_data = null;
List<HScriptParser.HparameterContext> param_context = new List<HScriptParser.HparameterContext>();
foreach (HScriptParser.HparameterContext ctx in context.hparameter())
{
if (ctx.full_table_name() != null)
{
DataSet d = VisitorHelper.GetData(Home, ctx.full_table_name());
if (first_data == null)
first_data = d;
parmset.Add(ctx.SCALAR().GetText(), d);
}
else
param_context.Add(ctx);
}
ExpressionVisitor exp_vis =
first_data != null
? new ExpressionVisitor(null, Home, first_data.Name, first_data.Columns, new StaticRegister(null))
: new ExpressionVisitor(null, Home);
MatrixVisitor mat_vis = new MatrixVisitor(Home, null, exp_vis);
foreach (HScriptParser.HparameterContext ctx in param_context)
{
string name = ctx.SCALAR().GetText();
if (ctx.expression() != null)
parmset.Add(name, exp_vis.ToNode(ctx.expression()));
else if (ctx.expression_alias_list() != null)
parmset.Add(name, exp_vis.ToNodes(ctx.expression_alias_list()));
else if (ctx.lambda_unit() != null)
parmset.Add(name, VisitorHelper.RenderLambda(Home, ctx.lambda_unit()));
else if (ctx.matrix_expression() != null)
parmset.Add(name, mat_vis.ToMatrix(ctx.matrix_expression()));
else if (ctx.K_OUT() != null)
parmset.Add(name, Home.GlobalHeap.Scalars, ctx.IDENTIFIER().GetText());
}
return parmset;
}
public TNodePrint(TNode Parent, Workspace Home, FNodeSet Expressions)
: base(Parent)
{
this._expressions = Expressions;
this._home = Home;
}
