public void AzureImageCodeGeneratorTest()
{
(var pipeline, var columnInference) = GetMockedAzureImagePipelineAndInference();
var setting = new CodeGeneratorSettings()
{
TrainDataset = @"/path/to/dataset",
ModelPath = @"/path/to/model",
MlTask = TaskKind.MulticlassClassification,
OutputName = @"CodeGenTest",
OutputBaseDir = @"/path/to/codegen",
LabelName = "Label",
Target = GenerateTarget.ModelBuilder,
StablePackageVersion = "stableversion",
UnstablePackageVersion = "unstableversion",
OnnxModelPath = @"/path/to/onnxModel",
IsAzureAttach = true,
IsImage = true,
ClassificationLabel = new string[] { "label1", "label2", "label3" },
};
var codeGen = new AzureAttachCodeGenenrator(pipeline, columnInference, setting);
foreach (var project in codeGen.ToSolution())
{
foreach (var projectFile in project)
{
NamerFactory.AdditionalInformation = projectFile.Name;
Approvals.Verify(((ICSharpFile)projectFile).File);
}
}
}
public void AzureCodeGeneratorTest()
{
(var pipeline, var columnInference, var mapping) = GetMockedAzurePipelineAndInference();
var setting = new CodeGeneratorSettings()
{
TrainDataset = @"\path\to\file",
ModelPath = @"\path\to\model",
MlTask = TaskKind.MulticlassClassification,
OutputName = @"test",
OutputBaseDir = @"\path\to\test",
LabelName = "label",
Target = GenerateTarget.ModelBuilder,
StablePackageVersion = "StablePackageVersion",
UnstablePackageVersion = "UnstablePackageVersion",
OnnxModelPath = @"\path\to\onnx",
IsAzureAttach = true,
IsImage = false,
OnnxInputMapping = mapping,
};
var codeGen = new AzureAttachCodeGenenrator(pipeline, columnInference, setting);
foreach (var project in codeGen.ToSolution())
{
foreach (var projectFile in project)
{
NamerFactory.AdditionalInformation = projectFile.Name;
Approvals.Verify(((ICSharpFile)projectFile).File);
}
}
}
private static void WarnCompileFailed(CodeGeneratorSettings settings, string slnPath, string dialogTitle)
{
string logPath = Path.GetFullPath(Path.Combine(settings.GetManagedBinDir(), "PluginInstaller", "build.log"));
if (string.IsNullOrEmpty(slnPath))
{
slnPath = Path.GetFullPath(Path.Combine(settings.GetManagedModulesDir(), "UnrealEngine.sln"));
}
FMessage.OpenDialog("Compile failed.\nBuild log: '" + logPath + "'\nsln: '" + slnPath + "'");
}
public AzureAttachCodeGenenrator(Pipeline pipeline, ColumnInferenceResults columnInferenceResults, CodeGeneratorSettings options)
{
_pipeline = pipeline;
_settings = options;
_columnInferenceResult = columnInferenceResults;
Name = _settings.OutputName;
var namespaceValue = Utilities.Utils.Normalize(_settings.OutputName);
AzureAttachConsoleApp = new AzureAttachConsoleAppCodeGenerator(_pipeline, _columnInferenceResult, _settings, namespaceValue);
AzureAttachModel = new AzureAttachModelCodeGenerator(_pipeline, _columnInferenceResult, _settings, namespaceValue);
}
internal ViewModel(GraphQLDocument query, CodeGeneratorSettings settings)
{
this.query = query;
this.Namespace = settings.Namespace;
this.ClassName = settings.ClassName;
this.OutputPath = settings.OutputPath;
this.RootPath = settings.RootPath;
this.typeLookup = new Dictionary <string, TypeViewModel>();
this.inputTypeLookup = new Dictionary <IGraphQLType, string>();
this.typeCollection = new List <TypeViewModel>();
// lets name each set
this.operationCollection = new List <OperationViewModel>();
foreach (Operation op in query.Operations)
{
List <NamedTypeViewModel> argCollection = new List <NamedTypeViewModel>();
foreach (KeyValuePair <string, ValueTypeReference> arg in op.Paramaters)
{
argCollection.Add(new NamedTypeViewModel
{
Name = arg.Key,
Type = GenerateTypeReference(arg.Value)
});
}
string name = Path.GetFileNameWithoutExtension(op.Path);
string opName = $"{name}_{op.Name }".Trim('_');
OperationViewModel opVM = new OperationViewModel()
{
Name = opName,
OperationName = op.Name ?? opName,
Arguments = argCollection,
QueryFragment = op.Query,
NamedQuery = op.Name ?? string.Empty,
ResultType = new TypeReferenceModel
{
IsCollection = false,
CanCollectionBeNull = false,
CanValueBeNull = false,
TypeName = GenerateType(op.Selection, opName)
}
};
// collect all fragments reference from this query and grab thier nodes query strings too
opVM.Query = FullQuery(opVM);
this.operationCollection.Add(opVM);
}
}
internal ViewModel(GraphQLDocument query, CodeGeneratorSettings settings)
{
Namespace = settings.Namespace;
ClassName = settings.ClassName;
this.typeLookup = new Dictionary <string, TypeViewModel>();
this.inputTypeLookup = new Dictionary <IGraphQLType, string>();
this.typeCollection = new List <TypeViewModel>();
// lets name each set
this.operationCollection = new List <OperationViewModel>();
foreach (var op in query.Operations)
{
List <NamedTypeViewModel> argCollection = new List <NamedTypeViewModel>();
foreach (var arg in op.Paramaters)
{
argCollection.Add(new NamedTypeViewModel
{
Name = arg.Key,
Type = GenerateTypeReference(arg.Value)
});
}
var name = Path.GetFileNameWithoutExtension(op.Path);
var opName = $"{name}_{op.Name }".Trim('_');
var opVM = new OperationViewModel()
{
Name = opName,
Arguments = argCollection,
Query = op.Query,
NamedQuery = op.Name ?? string.Empty,
ResultType = new TypeReferenceModel
{
IsCollection = false,
CanCollectionBeNull = false,
CanValueBeNull = false,
TypeName = GenerateType(op.Selection, opName)
}
};
operationCollection.Add(opVM);
}
// lets convert each to a type with a back track to lookup a type based on unique reference
// I need type definitions for Operation Result and Argument types graphs
}
private static void GenerateAndCompileMissingAssemblies()
{
string projectFileName = FPaths.ProjectFilePath;
if (!FBuild.WithEditor || string.IsNullOrEmpty(projectFileName))
{
return;
}
// NOTE: This is called before ManagedUnrealModuleInfo.Load / ManagedUnrealTypes.Load so some things may not be accessible.
// Maybe call load then unload after this is finished?
// NOTE: Most of the functions called here use FSlowTask but the UI isn't available until the engine is fully initialized.
// Instead use FFeedbackContext to display a progress bar (FDesktopPlatformModule::Get()->GetNativeFeedbackContext())
// TODO: Redirect some of the FSlowTask messages to the log, so that the "show log" button displays useful info?
codeGenContext = FFeedbackContext.GetGetDesktopFeedbackContext();
string dialogTitle = "USharp";
CodeGeneratorSettings settings = new CodeGeneratorSettings();
string engineWrapperDllPath = Path.Combine(settings.GetManagedModulesDir(), "bin", "Debug", "UnrealEngine.dll");
string engineWrapperSlnPath = Path.Combine(settings.GetManagedModulesDir(), "UnrealEngine.sln");
bool compileEngineWrapperCode = false;
if (!File.Exists(engineWrapperSlnPath))
{
if (FMessage.OpenDialog(EAppMsgType.YesNo, "C# engine wrapper code not found. Generate it now?", dialogTitle) == EAppReturnType.Yes)
{
codeGenContext.BeginSlowTask("Generating C# engine wrapper code (this might take a while...)", true);
CodeGenerator.GenerateCode(new string[] { "modules" });
codeGenContext.EndSlowTask();
compileEngineWrapperCode = true;
}
}
if (compileEngineWrapperCode || (!File.Exists(engineWrapperDllPath) && File.Exists(engineWrapperSlnPath)))
{
if (compileEngineWrapperCode ||
FMessage.OpenDialog(EAppMsgType.YesNo, "C# engine wrapper code isn't compiled. Compile it now?", dialogTitle) == EAppReturnType.Yes)
{
codeGenContext.BeginSlowTask("Compiling C# engine wrapper code (this might take a while...)", true);
bool compiled = CodeGenerator.CompileGeneratedCode();
codeGenContext.EndSlowTask();
if (!compiled)
{
WarnCompileFailed(settings, null, dialogTitle);
}
}
}
string projectName = Path.GetFileNameWithoutExtension(projectFileName);
string gameSlnPath = Path.Combine(settings.GetManagedDir(), projectName + ".Managed.sln");
string gameDllPath = Path.Combine(FPaths.ProjectDir, "Binaries", "Managed", projectName + ".Managed.dll");
if (!File.Exists(gameSlnPath) &&
FMessage.OpenDialog(EAppMsgType.YesNo, "USharp is enabled but the C# game project files weren't found. Generate them now?", dialogTitle) == EAppReturnType.Yes)
{
TemplateProjectGenerator.Generate();
}
if (File.Exists(gameSlnPath) && !File.Exists(gameDllPath) &&
FMessage.OpenDialog(EAppMsgType.YesNo, "C# game project code isn't compiled. Compile it now?", dialogTitle) == EAppReturnType.Yes)
{
codeGenContext.BeginSlowTask("Compiling C# game project code (this might take a while...)", true);
bool compiled = CodeGenerator.CompileCode(gameSlnPath, null);
codeGenContext.EndSlowTask();
if (!compiled)
{
WarnCompileFailed(settings, null, dialogTitle);
}
}
codeGenContext = null;
}
public AzureAttachConsoleAppCodeGenerator(Pipeline pipeline, ColumnInferenceResults columnInferenceResults, CodeGeneratorSettings options, string namespaceValue)
{
_pipeline = pipeline;
_settings = options;
_columnInferenceResult = columnInferenceResults;
_nameSpaceValue = namespaceValue;
Name = $"{_settings.OutputName}.ConsoleApp";
var(_, _, PreTrainerTransforms, _) = _pipeline.GenerateTransformsAndTrainers();
ModelBuilder = new CSharpCodeFile()
{
File = new AzureModelBuilder()
{
Path = _settings.TrainDataset,
HasHeader = _columnInferenceResult.TextLoaderOptions.HasHeader,
Separator = _columnInferenceResult.TextLoaderOptions.Separators.FirstOrDefault(),
PreTrainerTransforms = PreTrainerTransforms,
AllowQuoting = _columnInferenceResult.TextLoaderOptions.AllowQuoting,
AllowSparse = _columnInferenceResult.TextLoaderOptions.AllowSparse,
Namespace = _nameSpaceValue,
Target = _settings.Target,
OnnxModelPath = _settings.OnnxModelPath,
MLNetModelpath = _settings.ModelPath,
}.TransformText(),
Name = "ModelBuilder.cs",
};
PredictProject = new CSharpProjectFile()
{
File = new PredictProject()
{
Namespace = _nameSpaceValue,
IncludeMklComponentsPackage = false,
IncludeLightGBMPackage = false,
IncludeFastTreePackage = false,
IncludeImageTransformerPackage = _settings.IsImage,
IncludeImageClassificationPackage = false,
IncludeOnnxPackage = true,
IncludeResNet18Package = false,
IncludeRecommenderPackage = false,
StablePackageVersion = _settings.StablePackageVersion,
UnstablePackageVersion = _settings.UnstablePackageVersion,
}.TransformText(),
Name = $"{_settings.OutputName}.ConsoleApp.csproj",
};
var columns = _columnInferenceResult.TextLoaderOptions.Columns;
var featuresList = columns.Where((str) => str.Name != _settings.LabelName).Select((str) => str.Name).ToList();
var sampleResult = Utils.GenerateSampleData(_settings.TrainDataset, _columnInferenceResult);
PredictProgram = new CSharpCodeFile()
{
File = new PredictProgram()
{
TaskType = _settings.MlTask.ToString(),
LabelName = _settings.LabelName,
Namespace = _nameSpaceValue,
AllowQuoting = _columnInferenceResult.TextLoaderOptions.AllowQuoting,
AllowSparse = _columnInferenceResult.TextLoaderOptions.AllowSparse,
HasHeader = _columnInferenceResult.TextLoaderOptions.HasHeader,
Separator = _columnInferenceResult.TextLoaderOptions.Separators.FirstOrDefault(),
Target = _settings.Target,
Features = featuresList,
SampleData = sampleResult,
}.TransformText(),
Name = "Program.cs",
};
}
public AzureAttachModelCodeGenerator(Pipeline pipeline, ColumnInferenceResults columnInferenceResults, CodeGeneratorSettings options, string namespaceValue)
{
_pipeline = pipeline;
_settings = options;
_columnInferenceResult = columnInferenceResults;
_nameSpaceValue = namespaceValue;
Name = $"{_settings.OutputName}.Model";
ModelInputClass = new CSharpCodeFile()
{
File = new ModelInputClass()
{
Namespace = _nameSpaceValue,
ClassLabels = Utilities.Utils.GenerateClassLabels(_columnInferenceResult, _settings.OnnxInputMapping),
Target = _settings.Target
}.TransformText(),
Name = "ModelInput.cs",
};
var labelType = _columnInferenceResult.TextLoaderOptions.Columns.Where(t => t.Name == _settings.LabelName).First().DataKind;
Type labelTypeCsharp = Utils.GetCSharpType(labelType);
AzureImageModelOutputClass = new CSharpCodeFile()
{
File = new AzureImageModelOutputClass()
{
Namespace = _nameSpaceValue,
Target = _settings.Target,
Labels = _settings.ClassificationLabel,
}.TransformText(),
Name = "ModelOutput.cs",
};
AzureObjectDetectionModelOutputClass = new CSharpCodeFile()
{
File = new AzureObjectDetectionModelOutputClass()
{
Namespace = _nameSpaceValue,
Target = _settings.Target,
Labels = _settings.ObjectLabel,
}.TransformText(),
Name = "ModelOutput.cs",
};
ModelProject = new CSharpProjectFile()
{
File = new ModelProject()
{
IncludeFastTreePackage = false,
IncludeImageClassificationPackage = false,
IncludeImageTransformerPackage = _settings.IsImage,
IncludeLightGBMPackage = false,
IncludeMklComponentsPackage = false,
IncludeOnnxModel = true,
IncludeOnnxRuntime = _settings.IsObjectDetection,
IncludeRecommenderPackage = false,
StablePackageVersion = _settings.StablePackageVersion,
UnstablePackageVersion = _settings.UnstablePackageVersion,
OnnxRuntimePackageVersion = _settings.OnnxRuntimePacakgeVersion,
Target = _settings.Target,
}.TransformText(),
Name = $"{ _settings.OutputName }.Model.csproj",
};
ConsumeModel = new CSharpCodeFile()
{
File = new ConsumeModel()
{
Namespace = _nameSpaceValue,
Target = _settings.Target,
MLNetModelName = _settings.ModelName,
OnnxModelName = _settings.OnnxModelName,
IsAzureImage = _settings.IsAzureAttach && _settings.IsImage,
IsAzureObjectDetection = _settings.IsObjectDetection && _settings.IsAzureAttach,
}.TransformText(),
Name = "ConsumeModel.cs",
};
}
public AzureAttachModelCodeGenerator(Pipeline pipeline, ColumnInferenceResults columnInferenceResults, CodeGeneratorSettings options, string namespaceValue)
{
_pipeline = pipeline;
_settings = options;
_columnInferenceResult = columnInferenceResults;
_nameSpaceValue = namespaceValue;
Name = $"{_settings.OutputName}.Model";
ModelInputClass = new CSharpCodeFile()
{
File = new ModelInputClass()
{
Namespace = _nameSpaceValue,
ClassLabels = Utilities.Utils.GenerateClassLabels(_columnInferenceResult, _settings.OnnxInputMapping),
Target = _settings.Target
}.TransformText(),
Name = "ModelInput.cs",
};
var labelType = _columnInferenceResult.TextLoaderOptions.Columns.Where(t => t.Name == _settings.LabelName).First().DataKind;
Type labelTypeCsharp = Utils.GetCSharpType(labelType);
ModelOutputClass = new CSharpCodeFile()
{
File = new ModelOutputClass()
{
Namespace = _nameSpaceValue,
Target = _settings.Target,
TaskType = _settings.MlTask.ToString(),
PredictionLabelType = labelTypeCsharp.Name,
}.TransformText(),
Name = "ModelOutput.cs",
};
NormalizeMapping = new CSharpCodeFile()
{
File = new NormalizeMapping()
{
Target = _settings.Target,
Namespace = _nameSpaceValue,
}.TransformText(),
Name = "NormalizeMapping.cs",
};
ModelProject = new CSharpProjectFile()
{
File = new ModelProject()
{
IncludeFastTreePackage = false,
IncludeImageClassificationPackage = false,
IncludeImageTransformerPackage = _settings.IsImage,
IncludeLightGBMPackage = false,
IncludeMklComponentsPackage = false,
IncludeOnnxModel = true,
IncludeRecommenderPackage = false,
StablePackageVersion = _settings.StablePackageVersion,
UnstablePackageVersion = _settings.UnstablePackageVersion,
}.TransformText(),
Name = $"{ _settings.OutputName }.Model.csproj",
};
LabelMapping = new CSharpCodeFile()
{
File = new LabelMapping()
{
Target = _settings.Target,
Namespace = _nameSpaceValue,
LabelMappingInputLabelType = typeof(Int64).Name,
PredictionLabelType = labelTypeCsharp.Name,
TaskType = _settings.MlTask.ToString(),
}.TransformText(),
Name = "LabelMapping.cs",
};
ImageLabelMapping = new CSharpCodeFile()
{
File = new ImageLabelMapping()
{
Target = _settings.Target,
Namespace = _nameSpaceValue,
Labels = _settings.ClassificationLabel,
}.TransformText(),
Name = "LabelMapping.cs",
};
ConsumeModel = new CSharpCodeFile()
{
File = new ConsumeModel()
{
Namespace = _nameSpaceValue,
Target = _settings.Target,
HasLabelMapping = true,
HasNormalizeMapping = _settings.IsImage,
MLNetModelpath = _settings.ModelPath,
}.TransformText(),
Name = "ConsumeModel.cs",
};
}
public GraphQLDocument(GraphQLParser.AST.GraphQLDocument ast, IEnumerable <LocatedNamedSource> queryParts, CodeGeneratorSettings settings)
{
this.ast = ast;
this.QueryParts = queryParts;
this.settings = settings;
List <object> items = ast.Definitions.Select(this.Visit).ToList();
this.Operations = items.OfType <Operation>().ToList();
this.types = items.OfType <IGraphQLType>().GroupBy(x => x.Name).Select(x => x.First()).ToList();
this.astPrinter = new AstPrinter(settings.TypeNameDirective);
foreach (IGraphQLInitter i in items.OfType <IGraphQLInitter>().Where(x => !(x is Operation)))
{
i.Resolve(this);
}
foreach (Operation i in items.OfType <Operation>())
{
i.Resolve(this);
}
}
public static void InitializeApplication()
{
CodeGeneratorSettings.AddGlobalImport("QuickCodeLearning.Customers.Utilities");
}
