/// <summary>
/// Generate the module and its implementation.
/// </summary>
/// <param name="writer">The writer.</param>
/// <param name="prefix">The module prefix.</param>
/// <param name="regenerate">The command string used to generate.</param>
/// <param name="component">The component.</param>
/// <param name="moduleId"></param>
/// <param name="moduleName"></param>
/// <param name="moduleOwner"></param>
/// <param name="moduleVersion"></param>
/// <param name="moduleState"></param>
/// <param name="moduleType"></param>
/// <param name="moduleDeterminism"></param>
/// <param name="moduleCategory"></param>
/// <param name="exclude">The set of parameters to exclude</param>
/// <param name="namespaces">The set of extra namespaces</param>
public void Generate(IndentedTextWriter writer, string prefix, string regenerate, ComponentCatalog.LoadableClassInfo component,
string moduleId, string moduleName, string moduleOwner, string moduleVersion, string moduleState, string moduleType, string moduleDeterminism, string moduleCategory,
HashSet <string> exclude, HashSet <string> namespaces)
{
Contracts.AssertValue(exclude);
Name = moduleName;
Owner = moduleOwner;
Version = moduleVersion;
State = moduleState;
ModuleType = moduleType;
Determinism = moduleDeterminism;
Category = moduleCategory;
Exclude = exclude;
Namespaces = namespaces;
GenerateHeader(writer, regenerate);
using (writer.Nest())
{
GenerateClassName(writer, prefix, component);
using (writer.Nest())
GenerateContent(writer, prefix, component, moduleId);
GenerateFooter(writer);
}
GenerateFooter(writer);
}
private static void ShowMetadata(IndentedTextWriter itw, Schema schema, int col, bool showVals)
{
Contracts.AssertValue(itw);
Contracts.AssertValue(schema);
Contracts.Assert(0 <= col && col < schema.ColumnCount);
using (itw.Nest())
{
foreach (var kvp in schema.GetMetadataTypes(col).OrderBy(p => p.Key))
{
Contracts.AssertNonEmpty(kvp.Key);
Contracts.AssertValue(kvp.Value);
var type = kvp.Value;
itw.Write("Metadata '{0}': {1}", kvp.Key, type);
if (showVals)
{
if (!type.IsVector)
{
ShowMetadataValue(itw, schema, col, kvp.Key, type);
}
else
{
ShowMetadataValueVec(itw, schema, col, kvp.Key, type);
}
}
itw.WriteLine();
}
}
}
private static void ShowMetadata(IndentedTextWriter itw, Schema schema, int col, bool showVals)
{
Contracts.AssertValue(itw);
Contracts.AssertValue(schema);
Contracts.Assert(0 <= col && col < schema.Count);
using (itw.Nest())
{
foreach (var metaColumn in schema[col].Metadata.Schema.OrderBy(mcol => mcol.Name))
{
var type = metaColumn.Type;
itw.Write("Metadata '{0}': {1}", metaColumn.Name, type);
if (showVals)
{
if (!(type is VectorType vectorType))
{
ShowMetadataValue(itw, schema, col, metaColumn.Name, type);
}
else
{
ShowMetadataValueVec(itw, schema, col, metaColumn.Name, vectorType);
}
}
itw.WriteLine();
}
}
/// <summary>
/// Process a script to be parsed (from the input resource).
/// </summary>
private static void Process(IndentedTextWriter wrt, string text, ArgsBase defaults)
{
var env = new MLContext(seed: 42);
using (wrt.Nest())
{
var args1 = defaults.Clone();
using (wrt.Nest())
{
if (!CmdParser.ParseArguments(env, text, args1, s => wrt.WriteLine("*** {0}", s)))
{
wrt.WriteLine("*** Failed!");
}
}
string str1 = args1.ToString();
wrt.WriteLine("ToString: {0}", str1);
string settings1 = CmdParser.GetSettings(env, args1, defaults, SettingsFlags.None);
wrt.WriteLine("Settings: {0}", settings1);
var args2 = defaults.Clone();
using (wrt.Nest())
{
if (!CmdParser.ParseArguments(env, settings1, args2, s => wrt.WriteLine("*** BUG: {0}", s)))
{
wrt.WriteLine("*** BUG: parsing result of GetSettings failed!");
}
}
string str2 = args2.ToString();
if (str1 != str2)
{
wrt.WriteLine("*** BUG: ToString Mismatch: {0}", str2);
}
string settings2 = CmdParser.GetSettings(env, args2, defaults, SettingsFlags.None);
if (settings1 != settings2)
{
wrt.WriteLine("*** BUG: Settings Mismatch: {0}", settings2);
}
}
}
/// <summary>
/// Try produce indented string for command line.
/// </summary>
/// <param name="text">command line</param>
/// <param name="itw">indenting text writer</param>
/// <returns>true if we was able to produce indented string without any problem</returns>
private static bool TryProduceIndentString(string text, IndentedTextWriter itw)
{
string[] tokens;
if (!CmdParser.LexString(text, out tokens))
{
return(false);
}
for (var i = 0; i < tokens.Length;)
{
//We in last token, or next token don't equal to '='.
if (i + 1 == tokens.Length || tokens[i + 1] != "=")
{
itw.WriteLine(tokens[i++]);
}
else
{
itw.Write(tokens[i++]);
itw.Write(tokens[i++]);
// We have something like "name =" which is invalid.
if (i >= tokens.Length)
{
return(false);
}
//We have something like "name = value {options}".
if (i + 1 < tokens.Length && tokens[i + 1].StartsWith("{") && tokens[i + 1].EndsWith("}"))
{
itw.Write(tokens[i++]);
itw.WriteLine("{");
using (itw.Nest())
{
var str = CmdLexer.UnquoteValue(tokens[i++]);
// REVIEW: Probably we shouldn't give up if we have problem within one of the token
//and we need return partially indented string.
bool success = TryProduceIndentString(str, itw);
if (!success)
{
return(false);
}
}
itw.WriteLine("}");
}
//We have something like "name = value".
else
{
itw.WriteLine(tokens[i++]);
}
}
}
return(true);
}
private void ListKinds(IndentedTextWriter writer)
{
var sigs = _env.ComponentCatalog.GetAllSignatureTypes()
.Select(ComponentCatalog.SignatureToString)
.OrderBy(x => x);
writer.WriteLine("Available component kinds:");
using (writer.Nest())
{
foreach (var sig in sigs)
{
writer.WriteLine(sig);
}
}
}
protected override void GenerateMethodSignature(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("/// <summary>");
w.WriteLine("/// Creates a {0}", component.LoadNames[0]);
w.WriteLine("/// </summary>");
w.WriteLine("/// <param name=\"env\">The environment</param>");
w.WriteLine("/// <param name=\"data\">The data set</param>");
w.WriteLine("/// <returns>The transformed data.</returns>");
w.WriteLine("public IDataView Create{0}{1}Impl(", prefix, component.LoadNames[0]);
using (w.Nest())
{
w.WriteLine("IHostEnvironment env,");
w.WriteLine("IDataView data)");
}
}
protected override void GenerateMethodSignature(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.Write("public static Tuple<ITrainer> Create{0}{1}(", prefix, component.LoadNames[0]);
using (w.Nest())
{
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray();
var pre = "";
foreach (var arg in arguments)
{
GenerateMethodSignature(w, arg, null, null, null, ref pre, "");
}
w.WriteLine(")");
}
}
protected static void GenerateProperty(IndentedTextWriter w, string typeName, string argName, string defVal,
bool isBool, string helpText)
{
var help = helpText ?? argName;
help = help.Replace("&", "&").Replace("<", "<").Replace(">", ">");
w.WriteLine("/// <summary> Gets or sets {0}{1} </summary>", isBool ? "a value indicating whether " : "", help);
w.WriteLine("public {0} {1}", typeName, Capitalize(argName));
w.WriteLine("{");
using (w.Nest())
{
w.WriteLine("get {{ return {0}; }}", argName);
w.WriteLine("set {{ {0} = value; }}", argName);
}
w.WriteLine("}");
w.WriteLine();
}
protected override void GenerateMethodSignature(IndentedTextWriter w, string prefix,
ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("public static Tuple<IDataView, DataTransform> Create{0}{1}(", prefix, component.LoadNames[0]);
using (w.Nest())
{
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
w.WriteLine("[DataLabInputPort(FriendlyName = \"IDataView\", DisplayName = \"IDataView\", IsOptional = false, DataTypes = WellKnownDataTypeIds.IDataViewDotNet, Description = \"Input data (IDataView)\")]");
w.Write("IDataView data");
var pre = ",";
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateMethodSignature(w, arg, null, null, null, ref pre, "");
}
w.WriteLine(")");
}
}
protected override void GenerateImplBody(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
w.WriteLine("var args = new {0}();", GetCSharpTypeName(component.ArgType));
w.WriteLine("var defs = new {0}();", GetCSharpTypeName(component.ArgType));
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray();
foreach (var arg in arguments)
{
GenerateImplBody(w, arg, "");
}
w.WriteLine("return new Tuple<string, string>(\"{0}\", CmdParser.GetSettings(args, defs));", component.LoadNames[0]);
}
w.WriteLine("}");
}
protected override void GenerateImplCall(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
var className = prefix + component.LoadNames[0];
w.WriteLine("var builder = new {0}();", className);
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateImplCall(w, arg, "");
}
w.WriteLine("var env = new LocalEnvironment(1, verbose: true);");
w.WriteLine("var view = builder.Create{0}{1}Impl(env, data);", prefix, component.LoadNames[0]);
w.WriteLine("return new Tuple<IDataView, DataTransform>(view, new DataTransform(view));");
}
w.WriteLine("}");
}
protected override void GenerateImplCall(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
var className = prefix + component.LoadNames[0];
w.WriteLine("var builder = new {0}();", className);
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray();
foreach (var arg in arguments)
{
GenerateImplCall(w, arg, "");
}
w.WriteLine("var learner = builder.GetTlcSettings();");
w.WriteLine("return new TlcTrainer(learner.Item1, learner.Item2);");
}
w.WriteLine("}");
}
protected override void GenerateModuleAttribute(IndentedTextWriter w, string prefix,
ComponentCatalog.LoadableClassInfo component, string moduleId)
{
if (!string.IsNullOrEmpty(prefix))
{
prefix += " ";
}
w.WriteLine("[DataLabModule(FriendlyName = \"{0}{1}\",", prefix, component.UserName);
using (w.Nest())
{
var desc = component.Summary ?? component.LoadNames[0];
w.WriteLine("Description = \"{0}\",", desc.Replace("\n", "\\n").Replace("\r", "\\r"));
w.WriteLine("IsBlocking = true,");
w.WriteLine("IsDeterministic = true,");
w.WriteLine("Version = \"2.0\",");
w.WriteLine("Owner = \"Microsoft Corporation\",");
w.WriteLine("FamilyId = \"{{{0}}}\",", moduleId.ToUpperInvariant());
w.WriteLine("ReleaseState = States.Alpha)]");
}
}
private void GenerateHeader(IndentedTextWriter w, string regenerate)
{
w.WriteLine("//------------------------------------------------------------------------------");
w.WriteLine("// <auto-generated>");
w.WriteLine("// This code was generated by a tool.");
w.WriteLine("//");
w.WriteLine("// Changes to this file may cause incorrect behavior and will be lost if");
w.WriteLine("// the code is regenerated.");
w.WriteLine("// </auto-generated>");
w.WriteLine("//------------------------------------------------------------------------------");
w.WriteLine("// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
w.WriteLine("// DO NOT MODIFY THIS FILE");
w.WriteLine("// This file is generated from the TLC components, please don't modify.");
w.WriteLine("// The following command was used to generate this file: " + regenerate);
w.WriteLine("// Version used to generate this file: " + Assembly.GetExecutingAssembly().GetName().Version);
w.WriteLine("// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
w.WriteLine();
w.WriteLine("namespace Microsoft.Analytics.Platform.ML.IDVUtils");
w.WriteLine("{");
using (w.Nest())
GenerateUsings(w);
}
private void DumpErrors(IndentedTextWriter wrt, List <int> lineMap, int lineMin,
string fileName, string phase, List <Error> errors)
{
Contracts.AssertValue(wrt);
Contracts.AssertValue(lineMap);
Contracts.AssertNonEmpty(phase);
Contracts.AssertNonEmpty(errors);
using (wrt.Nest())
{
foreach (var err in errors)
{
var tok = err.Token;
Contracts.AssertValue(tok);
var pos = new LambdaParser.SourcePos(lineMap, tok.Span, lineMin);
wrt.Write("{0}({1},{2})-({3},{4}): ",
fileName, pos.LineMin, pos.ColumnMin, pos.LineLim, pos.ColumnLim);
wrt.Write("error: ");
wrt.WriteLine(err.GetMessage());
}
}
}
private void ShowUsage(IndentedTextWriter writer, string kind, string summary, string loadName,
IReadOnlyList <string> loadNames, object args, int?columns)
{
_env.Assert(loadName == loadNames[0]);
writer.WriteLine("Help for {0}: '{1}'", kind, loadName);
using (writer.Nest())
ShowAliases(writer, loadNames);
writer.WriteLine();
ShowFormattedSummary(writer, summary, columns);
if (args == null)
{
writer.WriteLine("Component '{0}' is not configurable", loadName);
writer.WriteLine();
}
else
{
writer.WriteLine(CmdParser.ArgumentsUsage(_env, args.GetType(), args, false, columns));
}
}
private static void ShowMetadataValueVec <T>(IndentedTextWriter itw, Schema schema, int col, string kind, ColumnType type)
{
Contracts.AssertValue(itw);
Contracts.AssertValue(schema);
Contracts.Assert(0 <= col && col < schema.Count);
Contracts.AssertNonEmpty(kind);
Contracts.AssertValue(type);
Contracts.Assert(type.IsVector);
Contracts.Assert(type.ItemType.RawType == typeof(T));
var conv = Conversions.Instance.GetStringConversion <T>(type.ItemType);
var value = default(VBuffer <T>);
schema[col].Metadata.GetValue(kind, ref value);
itw.Write(": Length={0}, Count={0}", value.Length, value.GetValues().Length);
using (itw.Nest())
{
var sb = default(StringBuilder);
int count = 0;
foreach (var item in value.Items())
{
if ((count % 10) == 0)
{
itw.WriteLine();
}
else
{
itw.Write(", ");
}
var val = item.Value;
conv(in val, ref sb);
itw.Write("[{0}] '{1}'", item.Key, sb);
count++;
}
}
}
protected override void GenerateModuleAttribute(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component, string moduleId)
{
if (!string.IsNullOrEmpty(prefix))
{
prefix += " ";
}
w.WriteLine("[DataLabModule(FriendlyName = \"{0}{1}\",", prefix, component.UserName);
using (w.Nest())
{
var desc = component.Summary ?? component.LoadNames[0];
w.WriteLine("Description = \"{0}\",", desc.Replace("\n", "\\n").Replace("\r", "\\r"));
string cat;
if (component.IsOfType(typeof(SignatureBinaryClassifierTrainer)) ||
component.IsOfType(typeof(SignatureMultiClassClassifierTrainer)))
{
cat = @"Machine Learning\Initialize Model\Classification";
}
else if (component.IsOfType(typeof(SignatureRegressorTrainer)))
{
cat = @"Machine Learning\Initialize Model\Regression";
}
else if (component.IsOfType(typeof(SignatureAnomalyDetectorTrainer)))
{
cat = @"Machine Learning\Initialize Model\Anomaly Detection";
}
else
{
cat = @"Machine Learning\Initialize Model";
}
w.WriteLine("Category = @\"{0}\",", cat);
w.WriteLine("IsBlocking = true,");
w.WriteLine("IsDeterministic = true,");
w.WriteLine("Version = \"2.0\",");
w.WriteLine("Owner = \"Microsoft Corporation\",");
w.WriteLine("FamilyId = \"{{{0}}}\",", Guid.NewGuid().ToString().ToUpperInvariant());
w.WriteLine("ReleaseState = States.Alpha)]");
}
}
protected override void GenerateImplBody(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
if (component.ArgType == null)
{
var call = GenerateCall(component);
w.WriteLine("return {0}(env, data);", call);
}
else
{
w.WriteLine("var args = new {0}();", GetCSharpTypeName(component.ArgType));
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateImplBody(w, arg, "");
}
var call = GenerateCall(component);
w.WriteLine("return {0}(args, env, data);", call);
}
}
w.WriteLine("}");
}
protected override void GenerateContent(IndentedTextWriter writer, string prefix,
ComponentCatalog.LoadableClassInfo component, string moduleId)
{
writer.WriteLine("[Module(");
_compName = prefix + component.LoadNames[0];
var name = Name ?? PrettyPrintDisplayName(component.LoadNames[0]);
using (writer.Nest())
{
writer.WriteLine("Name = \"{0}\",", name);
writer.WriteLine("FamilyId = \"{0}\",", moduleId);
writer.WriteLine("Owner = \"{0}\",", Owner);
writer.WriteLine("ReleaseVersion = \"{0}\",", Version);
writer.WriteLine("State = ModuleState.{0},", State);
writer.WriteLine("Type = ModuleType.{0},", ModuleType);
writer.WriteLine("Determinism = Determinism.{0},", Determinism);
writer.WriteLine("Category = @\"{0}\")]", Category);
}
writer.WriteLine("[Obsolete]");
writer.WriteLine("public static IModule Create{0}(", _compName);
using (writer.Nest())
{
writer.WriteLine("[Help(Display = @\"Dataset\", ToolTip = @\"Input dataset\")]");
writer.WriteLine("[ModuleInputPort]");
writer.WriteLine("IDataView idataset,");
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateMethodSignature(writer, arg, null, null, null, "");
}
writer.WriteLine("[Help(Display = @\"Results dataset\", ToolTip = @\"Transformed dataset\")]");
writer.WriteLine("[ModuleOutputPort]");
writer.WriteLine("IDataView odataset,");
writer.WriteLine("[Help(Display = @\"{0}\", ToolTip = @\"{0}\")]", name);
writer.WriteLine("[ModuleOutputPort]");
writer.WriteLine("DataTransform otransform,");
writer.WriteLine("[Context]");
writer.WriteLine("IContext context)");
}
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("var instance = new {0}Module();", _compName);
writer.WriteLine();
writer.WriteLine("var ports = new Dictionary<string, object> { { \"idataset\", idataset } };");
writer.WriteLine("var parameters = new Dictionary<string, object>");
writer.WriteLine("{");
using (writer.Nest())
{
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateDictionaryEntry(writer, arg, "");
}
}
writer.WriteLine("};");
writer.WriteLine();
writer.WriteLine("instance.Context = context;");
writer.WriteLine("instance.SetInputPorts(ports);");
writer.WriteLine("instance.SetParameters(parameters);");
writer.WriteLine();
writer.WriteLine("return instance;");
}
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("[Obsolete]");
writer.WriteLine("public class {0}Module : ModuleBase", _compName);
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("private Dictionary<string, object> parameters;");
writer.WriteLine("private Dictionary<string, object> ports;");
writer.WriteLine();
writer.WriteLine("public override Dictionary<string, object> Run()");
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("var view = ConstructTransform((IDataView)ports[\"idataset\"]);");
writer.WriteLine("return new Dictionary<string, object> { { \"odataset\", view }, { \"otransform\", new DataTransform(view) } };");
}
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("public override void SetParameters(Dictionary<string, object> parameters)");
writer.WriteLine("{");
using (writer.Nest())
writer.WriteLine("this.parameters = parameters;");
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("public override void SetInputPorts(Dictionary<string, object> ports)");
writer.WriteLine("{");
using (writer.Nest())
writer.WriteLine("this.ports = ports;");
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("public override Dictionary<string, object> ComputeSchema(Dictionary<string, object> inputports)");
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("var view = ConstructTransform((IDataView)inputports[\"idataset\"]);");
writer.WriteLine("return new Dictionary<string, object> { { \"odataset\", view.Schema } };");
}
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("private IDataView ConstructTransform(IDataView input)");
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("var builder = new {0}();", _compName);
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateImplCall(writer, arg, null, null, null, "");
}
writer.WriteLine("return builder.Create{0}Impl(Host, input);", _compName);
}
writer.WriteLine("}");
}
writer.WriteLine("}");
}
private void ShowComponents(IndentedTextWriter writer)
{
Type typeSig;
Type typeRes;
string kind;
if (_kind == null)
{
// Show commands.
typeSig = typeof(SignatureCommand);
typeRes = typeof(ICommand);
kind = "Command";
writer.WriteLine("Available commands:");
}
else
{
kind = _kind.ToLowerInvariant();
var sigs = _env.ComponentCatalog.GetAllSignatureTypes();
typeSig = sigs.FirstOrDefault(t => ComponentCatalog.SignatureToString(t).ToLowerInvariant() == kind);
if (typeSig == null)
{
typeSig = sigs.FirstOrDefault(t => ComponentCatalog.SignatureToString(t).StartsWithInvariantCultureIgnoreCase(kind));
if (typeSig == null)
{
writer.WriteLine("Couldn't find kind '{0}'", kind);
ListKinds(writer);
return;
}
}
typeRes = typeof(object);
writer.WriteLine("Available components for kind '{0}':", ComponentCatalog.SignatureToString(typeSig));
}
var infos = _env.ComponentCatalog.GetAllDerivedClasses(typeRes, typeSig)
.Where(x => !x.IsHidden)
.OrderBy(x => x.LoadNames[0].ToLowerInvariant());
using (writer.Nest())
{
var components = new List <Component>();
foreach (var info in infos)
{
_env.Assert(info.LoadNames.Count > 0);
writer.Write("{0}", info.LoadNames[0]);
if (!string.IsNullOrWhiteSpace(info.UserName))
{
writer.Write(": {0}", info.UserName);
}
writer.WriteLine();
using (writer.Nest())
ShowAliases(writer, info.LoadNames);
components.Add(new Component(kind, info, info.CreateArguments()));
}
if (components.Count > 0)
{
Serialize(components);
}
}
}
private void Run(string name)
{
string outDir = "ExprParser";
string text = GetResText(InResName(name));
string inName = name + "Input.txt";
string outName = name + "Output.txt";
string outNameAssem = name + "Output.Assem.txt";
string outPath = DeleteOutputPath(outDir, outName);
string outPathAssem = DeleteOutputPath(outDir, outNameAssem);
using (var wr = OpenWriter(outPath))
{
var wrt = new IndentedTextWriter(wr, " ");
// Individual scripts are separated by $.
// Inputs start after #.
int count = 0;
int ichLim = 0;
int lineLim = 1;
for (; ichLim < text.Length; ichLim++)
{
int ichMin = ichLim;
int lineMin = lineLim;
while (ichLim < text.Length && text[ichLim] != '$')
{
if (text[ichLim] == '\n')
{
lineLim++;
}
ichLim++;
}
while (ichMin < ichLim && char.IsWhiteSpace(text[ichMin]))
{
if (text[ichMin] == '\n')
{
lineMin++;
}
ichMin++;
}
if (ichMin >= ichLim)
{
continue;
}
// Process the script.
count++;
string scriptName = string.Format("Script {0}, lines {1} to {2}", count, lineMin, lineLim);
wrt.WriteLine("===== Start {0} =====", scriptName);
var types = ParseTypes(text, ref ichMin, ichLim);
int ichLimChars = text.IndexOf('#', ichMin);
if (ichLimChars < 0 || ichLimChars >= ichLim)
{
ichLimChars = ichLim;
}
else
{
Contracts.Assert(ichMin < ichLimChars && ichLimChars < ichLim);
}
CharCursor chars = new CharCursor(text, ichMin, ichLimChars);
Delegate del = null;
lock (_sync)
{
try
{
LambdaNode node;
List <Error> errors;
List <int> lineMap;
var perm = Utils.GetIdentityPermutation(types.Length);
using (wrt.Nest())
{
node = LambdaParser.Parse(out errors, out lineMap, chars, perm, types);
}
Check(node != null, "Null LambdaNode");
if (Utils.Size(errors) > 0)
{
DumpErrors(wrt, lineMap, lineMin, inName, "Parsing", errors);
goto LDone;
}
LambdaBinder.Run(Env, ref errors, node, msg => wr.WriteLine(msg));
if (Utils.Size(errors) > 0)
{
DumpErrors(wrt, lineMap, lineMin, inName, "Binding", errors);
goto LDone;
}
wrt.WriteLine("Binding succeeded. Output type: {0}", node.ResultType);
del = LambdaCompiler.Compile(out errors, node);
Contracts.Assert(TestFuncs1.Writer == null);
TestFuncs1.Writer = wr;
if (Utils.Size(errors) > 0)
{
DumpErrors(wrt, lineMap, lineMin, inName, "Compiling", errors);
goto LDone;
}
wrt.WriteLine("Compiling succeeded.");
if (ichLimChars < ichLim)
{
Evaluate(wrt, del, node.ResultType, types, text, ichLimChars + 1, ichLim);
}
}
catch (Exception ex)
{
if (!ex.IsMarked())
{
wrt.WriteLine("Unknown Exception: {0}!", del != null ? del.GetMethodInfo().DeclaringType : (object)"<null>");
}
wrt.WriteLine("Exception: {0}", ex.Message);
}
finally
{
TestFuncs1.Writer = null;
}
LDone:
wrt.WriteLine("===== End {0} =====", scriptName);
}
}
}
CheckEquality(outDir, outName, digitsOfPrecision: 6);
Done();
}
private static void PrintSchema(TextWriter writer, Arguments args, Schema schema, ITransposeDataView transposeDataView)
{
Contracts.AssertValue(writer);
Contracts.AssertValue(args);
Contracts.AssertValue(schema);
Contracts.AssertValueOrNull(transposeDataView);
#if !CORECLR
if (args.ShowJson)
{
writer.WriteLine("Json Schema not supported.");
return;
}
#endif
int colLim = schema.Count;
var itw = new IndentedTextWriter(writer, " ");
itw.WriteLine("{0} columns:", colLim);
using (itw.Nest())
{
var names = default(VBuffer <ReadOnlyMemory <char> >);
for (int col = 0; col < colLim; col++)
{
var name = schema[col].Name;
var type = schema[col].Type;
var slotType = transposeDataView?.GetSlotType(col);
itw.WriteLine("{0}: {1}{2}", name, type, slotType == null ? "" : " (T)");
bool metaVals = args.ShowMetadataValues;
if (metaVals || args.ShowMetadataTypes)
{
ShowMetadata(itw, schema, col, metaVals);
continue;
}
if (!args.ShowSlots)
{
continue;
}
if (!type.IsKnownSizeVector())
{
continue;
}
ColumnType typeNames;
if ((typeNames = schema[col].Metadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.SlotNames)?.Type) == null)
{
continue;
}
if (typeNames.GetVectorSize() != type.GetVectorSize() || !(typeNames.GetItemType() is TextType))
{
Contracts.Assert(false, "Unexpected slot names type");
continue;
}
schema[col].Metadata.GetValue(MetadataUtils.Kinds.SlotNames, ref names);
if (names.Length != type.GetVectorSize())
{
Contracts.Assert(false, "Unexpected length of slot names vector");
continue;
}
using (itw.Nest())
{
bool verbose = args.Verbose ?? false;
foreach (var kvp in names.Items(all: verbose))
{
if (verbose || !kvp.Value.IsEmpty)
{
itw.WriteLine("{0}:{1}", kvp.Key, kvp.Value);
}
}
}
}
}
}
