public static void GenerateLoaderAddInputMethod(IndentedTextWriter writer, string className)
{
//Constructor.
writer.WriteLine("[Obsolete]");
writer.WriteLine("[JsonIgnore]");
writer.WriteLine("private string _inputFilePath = null;");
writer.WriteLine($"public {className}(string filePath)");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine("_inputFilePath = filePath;");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLine("");
//SetInput.
writer.WriteLine("[Obsolete]");
writer.WriteLine($"public void SetInput(IHostEnvironment env, Experiment experiment)");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine("IFileHandle inputFile = new SimpleFileHandle(env, _inputFilePath, false, false);");
writer.WriteLine("experiment.SetInput(InputFile, inputFile);");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLine("");
//GetInputData
writer.WriteLine("[Obsolete]");
writer.WriteLine("public Var<IDataView> GetInputData() => null;");
writer.WriteLine("");
//Apply.
writer.WriteLine("[Obsolete]");
writer.WriteLine($"public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment)");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine("Contracts.Assert(previousStep == null);");
writer.WriteLine("");
writer.WriteLine($"return new {className}PipelineStep(experiment.Add(this));");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLine("");
//Pipelinestep class.
writer.WriteLine("[Obsolete]");
writer.WriteLine($"private class {className}PipelineStep : ILearningPipelineDataStep");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine($"public {className}PipelineStep (Output output)");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine("Data = output.Data;");
writer.WriteLine("Model = null;");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
writer.WriteLine("public Var<IDataView> Data { get; }");
writer.WriteLine("public Var<ITransformModel> Model { get; }");
writer.Outdent();
writer.WriteLine("}");
}
public static void GenerateMethod(IndentedTextWriter writer, string className, string defaultNamespace)
{
var inputOuputClassName = defaultNamespace + className;
writer.WriteLine($"public {inputOuputClassName}.Output Add({inputOuputClassName} input)");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine($"var output = new {inputOuputClassName}.Output();");
writer.WriteLine("Add(input, output);");
writer.WriteLine("return output;");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
writer.WriteLine($"public void Add({inputOuputClassName} input, {inputOuputClassName}.Output output)");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine($"_jsonNodes.Add(Serialize(\"{className}\", input, output));");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
}
private void GenerateInputOutput(IndentedTextWriter writer, ComponentCatalog.EntryPointInfo entryPointInfo, ComponentCatalog catalog)
{
var classAndMethod = CSharpGeneratorUtils.GetEntryPointMetadata(entryPointInfo);
writer.WriteLine($"namespace Legacy.{classAndMethod.Namespace}");
writer.WriteLine("{");
writer.Indent();
GenerateInput(writer, entryPointInfo, catalog);
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
}
private Type GenerateManyToOneColumn(IndentedTextWriter writer, string className, Type columnType,
System.Reflection.FieldInfo fieldInfo, ArgumentAttribute inputAttr, Type type, bool isArray)
{
var fieldName = CSharpGeneratorUtils.Capitalize(inputAttr.Name ?? fieldInfo.Name);
var apiName = _generatedClasses.GetApiName(type, "");
writer.WriteLine($"public {className}()");
writer.WriteLine("{");
writer.WriteLine("}");
writer.WriteLine("");
writer.WriteLine($"public {className}(string output{fieldName}, params string[] input{fieldName}s)");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine($"Add{fieldName}(output{fieldName}, input{fieldName}s);");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLine("");
writer.WriteLine($"public void Add{fieldName}(string name, params string[] source)");
writer.WriteLine("{");
writer.Indent();
if (isArray)
{
writer.WriteLine($"var list = {fieldName} == null ? new List<{apiName}>() : new List<{apiName}>({fieldName});");
writer.WriteLine($"list.Add(ManyToOneColumn<{apiName}>.Create(name, source));");
writer.WriteLine($"{fieldName} = list.ToArray();");
}
else
{
writer.WriteLine($"{fieldName} = ManyToOneColumn<{apiName}>.Create(name, source);");
}
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
Contracts.Assert(columnType == null);
columnType = type;
return(columnType);
}
private void GenerateComponent(IndentedTextWriter writer, ComponentCatalog.ComponentInfo component, ComponentCatalog catalog)
{
GenerateEnums(writer, component.ArgumentType, "Runtime");
writer.WriteLineNoTabs();
GenerateClasses(writer, component.ArgumentType, catalog, "Runtime");
writer.WriteLineNoTabs();
CSharpGeneratorUtils.GenerateSummary(writer, component.Description);
writer.WriteLine($"public sealed class {CSharpGeneratorUtils.GetComponentName(component)} : {component.Kind}");
writer.WriteLine("{");
writer.Indent();
GenerateInputFields(writer, component.ArgumentType, catalog, "Runtime");
writer.WriteLine($"internal override string ComponentName => \"{component.Name}\";");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
}
private void GenerateInput(IndentedTextWriter writer, ComponentCatalog.EntryPointInfo entryPointInfo, ComponentCatalog catalog)
{
var entryPointMetadata = CSharpGeneratorUtils.GetEntryPointMetadata(entryPointInfo);
string classBase = "";
if (entryPointInfo.InputKinds != null)
{
classBase += $" : {string.Join(", ", entryPointInfo.InputKinds.Select(CSharpGeneratorUtils.GetCSharpTypeName))}";
if (entryPointInfo.InputKinds.Any(t => typeof(ITrainerInput).IsAssignableFrom(t) || typeof(ITransformInput).IsAssignableFrom(t)))
{
classBase += ", Microsoft.ML.Legacy.ILearningPipelineItem";
}
}
GenerateEnums(writer, entryPointInfo.InputType, _defaultNamespace + entryPointMetadata.Namespace);
writer.WriteLineNoTabs();
GenerateClasses(writer, entryPointInfo.InputType, catalog, _defaultNamespace + entryPointMetadata.Namespace);
CSharpGeneratorUtils.GenerateSummary(writer, entryPointInfo.Description, entryPointInfo.XmlInclude);
if (entryPointInfo.ObsoleteAttribute != null)
{
writer.WriteLine($"[Obsolete(\"{entryPointInfo.ObsoleteAttribute.Message}\")]");
}
else
{
writer.WriteLine("[Obsolete]");
}
writer.WriteLine($"public sealed partial class {entryPointMetadata.ClassName}{classBase}");
writer.WriteLine("{");
writer.Indent();
writer.WriteLineNoTabs();
if (entryPointInfo.InputKinds != null && entryPointInfo.InputKinds.Any(t => typeof(Legacy.ILearningPipelineLoader).IsAssignableFrom(t)))
{
CSharpGeneratorUtils.GenerateLoaderAddInputMethod(writer, entryPointMetadata.ClassName);
}
GenerateColumnAddMethods(writer, entryPointInfo.InputType, catalog, entryPointMetadata.ClassName, out Type transformType);
writer.WriteLineNoTabs();
GenerateInputFields(writer, entryPointInfo.InputType, catalog, _defaultNamespace + entryPointMetadata.Namespace);
writer.WriteLineNoTabs();
GenerateOutput(writer, entryPointInfo, out HashSet <string> outputVariableNames);
GenerateApplyFunction(writer, entryPointMetadata.ClassName, transformType, outputVariableNames, entryPointInfo.InputKinds);
writer.Outdent();
writer.WriteLine("}");
}
private void GenerateOutput(IndentedTextWriter writer, ComponentCatalog.EntryPointInfo entryPointInfo, out HashSet <string> outputVariableNames)
{
outputVariableNames = new HashSet <string>();
string classBase = "";
if (entryPointInfo.OutputKinds != null)
{
classBase = $" : {string.Join(", ", entryPointInfo.OutputKinds.Select(CSharpGeneratorUtils.GetCSharpTypeName))}";
}
writer.WriteLine("[Obsolete]");
writer.WriteLine($"public sealed class Output{classBase}");
writer.WriteLine("{");
writer.Indent();
var outputType = entryPointInfo.OutputType;
if (outputType.IsGenericType && outputType.GetGenericTypeDefinition() == typeof(CommonOutputs.MacroOutput <>))
{
outputType = outputType.GetGenericTypeArgumentsEx()[0];
}
foreach (var fieldInfo in outputType.GetFields())
{
var outputAttr = fieldInfo.GetCustomAttributes(typeof(TlcModule.OutputAttribute), false)
.FirstOrDefault() as TlcModule.OutputAttribute;
if (outputAttr == null)
{
continue;
}
CSharpGeneratorUtils.GenerateSummary(writer, outputAttr.Desc);
var outputTypeString = CSharpGeneratorUtils.GetOutputType(fieldInfo.FieldType);
outputVariableNames.Add(CSharpGeneratorUtils.Capitalize(outputAttr.Name ?? fieldInfo.Name));
writer.WriteLine($"public {outputTypeString} {CSharpGeneratorUtils.Capitalize(outputAttr.Name ?? fieldInfo.Name)} {{ get; set; }} = new {outputTypeString}();");
writer.WriteLineNoTabs();
}
writer.Outdent();
writer.WriteLine("}");
}
public void Dispose()
{
_writer.Outdent();
_writer = null;
}
public static void GenerateFooter(IndentedTextWriter writer)
{
writer.Outdent();
writer.WriteLine("}");
}
private static void GenerateApplyFunction(IndentedTextWriter writer, string className, Type type,
HashSet <string> outputVariableNames, Type[] inputKinds)
{
if (inputKinds == null)
{
return;
}
bool isTransform = false;
bool isCalibrator = false;
if (inputKinds.Any(t => typeof(ITransformInput).IsAssignableFrom(t)))
{
isTransform = true;
}
else if (!inputKinds.Any(t => typeof(ITrainerInput).IsAssignableFrom(t)))
{
return;
}
if (inputKinds.Any(t => typeof(ICalibratorInput).IsAssignableFrom(t)))
{
isCalibrator = true;
}
writer.WriteLine("[Obsolete]");
if (isTransform)
{
writer.WriteLine("public Var<IDataView> GetInputData() => Data;");
}
else
{
writer.WriteLine("public Var<IDataView> GetInputData() => TrainingData;");
}
writer.WriteLine("");
writer.WriteLine("[Obsolete]");
writer.WriteLine("public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment)");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine("if (previousStep != null)");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine("if (!(previousStep is ILearningPipelineDataStep dataStep))");
writer.WriteLine("{");
writer.Indent();
writer.WriteLine("throw new InvalidOperationException($\"{ nameof(" + className + ")} only supports an { nameof(ILearningPipelineDataStep)} as an input.\");");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
if (isTransform)
{
writer.WriteLine("Data = dataStep.Data;");
}
else
{
writer.WriteLine("TrainingData = dataStep.Data;");
}
writer.Outdent();
writer.WriteLine("}");
string pipelineStep = $"{className}PipelineStep";
writer.WriteLine($"Output output = experiment.Add(this);");
writer.WriteLine($"return new {pipelineStep}(output);");
writer.Outdent();
writer.WriteLine("}");
//Pipeline step.
writer.WriteLineNoTabs();
writer.WriteLine("[Obsolete]");
if (isTransform && !isCalibrator)
{
writer.WriteLine($"private class {pipelineStep} : ILearningPipelineDataStep");
}
else
{
writer.WriteLine($"private class {pipelineStep} : ILearningPipelinePredictorStep");
}
writer.WriteLine("{");
writer.Indent();
writer.WriteLine("[Obsolete]");
writer.WriteLine($"public {pipelineStep}(Output output)");
writer.WriteLine("{");
writer.Indent();
if (isTransform && !isCalibrator)
{
writer.WriteLine("Data = output.OutputData;");
if (outputVariableNames.Contains("Model"))
{
writer.WriteLine("Model = output.Model;");
}
}
else
{
writer.WriteLine("Model = output.PredictorModel;");
}
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
if (isTransform && !isCalibrator)
{
writer.WriteLine("[Obsolete]");
writer.WriteLine("public Var<IDataView> Data { get; }");
writer.WriteLine("[Obsolete]");
writer.WriteLine("public Var<TransformModel> Model { get; }");
}
else
{
writer.WriteLine("[Obsolete]");
writer.WriteLine("public Var<PredictorModel> Model { get; }");
}
writer.Outdent();
writer.WriteLine("}");
}
private void GenerateClasses(IndentedTextWriter writer, Type inputType, ComponentCatalog catalog, string currentNamespace)
{
foreach (var fieldInfo in inputType.GetFields())
{
var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute;
if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly)
{
continue;
}
var type = fieldInfo.FieldType;
type = CSharpGeneratorUtils.ExtractOptionalOrNullableType(type);
if (type.IsArray)
{
type = type.GetElementType();
}
if (type == typeof(JArray) || type == typeof(JObject))
{
continue;
}
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Var <>))
{
continue;
}
if (type == typeof(CommonInputs.IEvaluatorInput))
{
continue;
}
if (type == typeof(CommonOutputs.IEvaluatorOutput))
{
continue;
}
var typeEnum = TlcModule.GetDataType(type);
if (typeEnum != TlcModule.DataKind.Unknown)
{
continue;
}
if (_generatedClasses.IsGenerated(type.FullName))
{
continue;
}
GenerateEnums(writer, type, currentNamespace);
GenerateClasses(writer, type, catalog, currentNamespace);
var apiName = _generatedClasses.GetApiName(type, currentNamespace);
string classBase = "";
if (type.IsSubclassOf(typeof(OneToOneColumn)))
{
classBase = $" : OneToOneColumn<{apiName}>, IOneToOneColumn";
}
else if (type.IsSubclassOf(typeof(ManyToOneColumn)))
{
classBase = $" : ManyToOneColumn<{apiName}>, IManyToOneColumn";
}
writer.WriteLine("[Obsolete]");
writer.WriteLine($"public sealed partial class {apiName}{classBase}");
writer.WriteLine("{");
writer.Indent();
_generatedClasses.MarkAsGenerated(type.FullName);
GenerateInputFields(writer, type, catalog, currentNamespace);
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
}
}
private void GenerateEnums(IndentedTextWriter writer, Type inputType, string currentNamespace)
{
foreach (var fieldInfo in inputType.GetFields())
{
var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute;
if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly)
{
continue;
}
var type = CSharpGeneratorUtils.ExtractOptionalOrNullableType(fieldInfo.FieldType);
if (_generatedClasses.IsGenerated(type.FullName))
{
continue;
}
if (!type.IsEnum)
{
var typeEnum = TlcModule.GetDataType(type);
if (typeEnum == TlcModule.DataKind.Unknown)
{
GenerateEnums(writer, type, currentNamespace);
}
continue;
}
var enumType = Enum.GetUnderlyingType(type);
writer.WriteLine("[Obsolete]");
var apiName = _generatedClasses.GetApiName(type, currentNamespace);
if (enumType == typeof(int))
{
writer.WriteLine($"public enum {apiName}");
}
else
{
Contracts.Assert(enumType == typeof(byte));
writer.WriteLine($"public enum {apiName} : byte");
}
_generatedClasses.MarkAsGenerated(type.FullName);
writer.WriteLine("{");
writer.Indent();
var names = Enum.GetNames(type);
var values = Enum.GetValues(type);
var lines = new List <string>();
for (int i = 0; i < names.Length; i++)
{
var name = names[i];
if (type.GetField(name).GetCustomAttribute <HideEnumValueAttribute>() != null)
{
continue;
}
var value = values.GetValue(i);
if (enumType == typeof(int))
{
lines.Add($"{name} = {(int)value}");
}
else
{
Contracts.Assert(enumType == typeof(byte));
lines.Add($"{name} = {(byte)value}");
}
}
for (int i = 0; i < lines.Count - 1; i++)
{
writer.WriteLine($"{lines[i]},");
}
writer.WriteLine($"{lines[lines.Count-1]}");
writer.Outdent();
writer.WriteLine("}");
writer.WriteLineNoTabs();
}
}