public async Task <ImportMetadata> UpdateImportMetadataAsync(ImportMetadata metadata)
{
using (var cn = new SqlConnection(dbOptions.ConnectionString))
{
var grid = await cn.QueryMultipleAsync(
Sql.UpdateImportMetadata,
new
{
id = metadata.Id,
sourceId = metadata.SourceId,
type = metadata.Type,
structure = metadata.StructureJson,
constraints = ResourceConstraintTable.From(metadata),
user = user.UUID,
groups = GroupMembership.From(user),
admin = user.IsAdmin
},
commandType : CommandType.StoredProcedure,
commandTimeout : dbOptions.DefaultTimeout
);
var updated = DbReader.Read(grid);
return(updated);
}
}
public void ShouldCorrectlySetColumnDefinitionsWithOnlyHeaders()
{
var metadata = new ImportMetadata
{
ColumnDefinitions = new List <ColumnDefinition>
{
new ColumnDefinition
{
Name = "Date"
},
new ColumnDefinition
{
Name = "Category"
},
new ColumnDefinition
{
Name = "Expense"
}
}
};
var command = new DetermineColumnDefinitionsCommand();
command.Execute(metadata);
metadata.ColumnDefinitions[0].DataType.Should().Be(ColumnDataType.Text);
metadata.ColumnDefinitions[0].MLType.Should().Be(ColumnMLType.Categorical);
metadata.ColumnDefinitions[1].DataType.Should().Be(ColumnDataType.Text);
metadata.ColumnDefinitions[1].MLType.Should().Be(ColumnMLType.Categorical);
metadata.ColumnDefinitions[2].DataType.Should().Be(ColumnDataType.Text);
metadata.ColumnDefinitions[2].MLType.Should().Be(ColumnMLType.Categorical);
}
/// <summary>
/// Public wrapper used to execute an instantiated ImportMetadata geoprocessing tool.
/// The wrapper's try/catch block contains an error message identifying the type of tool.
///param name="gpTool" the geoprocessing tool
///param name="xmlFilePaths" the xml file paths
/// </summary>
static public int Execute(ImportMetadata gpTool, out string[] xmlFilePaths)
{
// Instantiate a new string array and set the anticipated paths
// into the array. Invoking method is responsible for cleanup
// once it has processed the files generated by this method.
xmlFilePaths = new string[3];
xmlFilePaths[0] = gpTool.Source_Metadata.ToString(); // Original XML
xmlFilePaths[1] = gpTool.Output_Metadata.ToString(); // Translated XML
try
{
Execute(gpTool as IGPProcess);
}
catch (Exception ex)
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("Unexpected exception generated by geoprocessing tool");
sb.AppendLine("Metadata Translator (Conversion Tools - Metadata)");
sb.AppendLine("");
sb.AppendLine(ex.Message);
MessageBox.Show(sb.ToString(), "Translation Failure",
MessageBoxButtons.OK, MessageBoxIcon.Warning);
retVal = -1;
}
return(retVal);
}
public void FixPaths()
{
var paths = JsonConvert.DeserializeObject <PathPoint[][]>(Paths);
PathsFixed = new PathPoint[paths.Count()][];
string[] param = ImportMetadata.Split(';');
var floorParams = param.Select(p => Convert.ToDouble(p, CultureInfo.InvariantCulture)).ToList();
int i = 0;
foreach (var item in paths)
{
PathsFixed[i] = new PathPoint[2];
PathsFixed[i][0] = new PathPoint
{
X = (double)((-item[0].X + floorParams[0]) / (floorParams[0] - floorParams[2]) * Width - Width / 2),
Y = (double)((-item[0].Y + floorParams[1]) / (floorParams[1] - floorParams[3]) * Height - Height / 2)
};
PathsFixed[i][1] = new PathPoint
{
X = (double)((-item[1].X + floorParams[0]) / (floorParams[0] - floorParams[2]) * Width - Width / 2),
Y = (double)((-item[1].Y + floorParams[1]) / (floorParams[1] - floorParams[3]) * Height - Height / 2)
};
i++;
}
}
public ImportMetadata Execute(string content)
{
// TODO: Use TextFieldParser when client-side Blazor supports .NET Core 3.
// NOTE: Use Visual Studio 2019 16.3+
using TextReader fileReader = new StringReader(content);
var csv = new CsvParser(fileReader);
var headers = csv.Read();
var firstRow = csv.Read();
var metadata = new ImportMetadata();
for (int i = 0; i < headers.Length; ++i)
{
var column = new ColumnDefinition
{
Name = headers[i],
FirstValue = firstRow?[i]
};
metadata.ColumnDefinitions.Add(column);
}
return(metadata);
}
public async Task <ActionResult <ImportMetadata> > GetMetadata(string id, [FromServices] DataImporter importer)
{
try
{
if (!importOptions.REDCap.Enabled)
{
return(NotFound());
}
ImportMetadata meta = null;
var isGuid = Guid.TryParse(id, out var guidId);
if (isGuid)
{
// Get by leaf internal Id
meta = await importer.GetImportMetadata(guidId);
}
else
{
// Get by sourceId
meta = await importer.GetImportMetadata(id);
}
if (meta == null)
{
return(NotFound());
}
return(Ok(meta));
}
catch (Exception ex)
{
log.LogError("Failed get import metadata. Error:{Error}", ex.ToString());
return(StatusCode(StatusCodes.Status500InternalServerError));
}
}
public void Write(ChunkWriter writer)
{
ulong infoChunkLength = (ulong)((2U * 4U) + 4U + ((Name != null) ? Encoding.BigEndianUnicode.GetByteCount(Name) : 0));
ulong total = 16U + infoChunkLength;
if (Elements != null)
{
foreach (GfxElement element in Elements)
{
total += element.ChunkSize();
}
}
if (ImportMetadata != null)
{
total += ImportMetadata.ChunkSize();
}
writer.OpenChunk(ChunkType.GfxElementSet, total);
writer.OpenChunk(ChunkType.GfxElementSetInfo, infoChunkLength);
writer.Write(ElementWidth);
writer.Write(ElementHeight);
writer.Write(Name);
writer.CloseChunk();
if (Elements != null)
{
foreach (GfxElement element in Elements)
{
element.Write(writer);
}
}
if (ImportMetadata != null)
{
ImportMetadata.Write(writer);
}
writer.CloseChunk();
}
public void ShouldCorrectlySetColumnDefinitions()
{
var metadata = new ImportMetadata
{
ColumnDefinitions = new List <ColumnDefinition>
{
new ColumnDefinition
{
Name = "Date",
FirstValue = "17/05/2018"
},
new ColumnDefinition
{
Name = "Category",
FirstValue = "🛫 Transport"
},
new ColumnDefinition
{
Name = "Expense",
FirstValue = "12.23"
}
}
};
var command = new DetermineColumnDefinitionsCommand();
command.Execute(metadata);
metadata.ColumnDefinitions[0].DataType.Should().Be(ColumnDataType.Text);
metadata.ColumnDefinitions[0].MLType.Should().Be(ColumnMLType.Categorical);
metadata.ColumnDefinitions[1].DataType.Should().Be(ColumnDataType.Text);
metadata.ColumnDefinitions[1].MLType.Should().Be(ColumnMLType.Categorical);
metadata.ColumnDefinitions[2].DataType.Should().Be(ColumnDataType.Number);
metadata.ColumnDefinitions[2].MLType.Should().Be(ColumnMLType.Numeric);
}
public bool Intercept(MemberInfo member, ImportMetadata metadata, object target, ref object value)
{
Contract.Requires<ArgumentNullException>(member != null, "member");
Contract.Requires<ArgumentNullException>(metadata != null, "metadata");
Contract.Requires<ArgumentNullException>(target != null, "target");
return false;
}
private void LoadData()
{
var directoryPath = ConfigurationManager.AppSettings.Get("RepositoryDir");
var imp = new ImportMetadata();
_allItems = imp.ImportMetafiles(directoryPath, new XmlSer());
FilteredMetadataItems = _allItems;
}
public ImportInterceptorHelper([NotNull] IImportInterceptor interceptor, [NotNull] MemberInfo member,
[NotNull] ImportMetadata metadata, [NotNull] object target)
{
_interceptor = interceptor;
_member = member;
_metadata = metadata;
_target = target;
}
protected override void ReadSpecific(BinaryReader reader)
{
var count = reader.ReadVarUint32();
for (var i = 0; i < count; i++)
{
Imports.Add(ImportMetadata.Read(reader));
}
}
public override void Inject(object target, IContainer container, ImportMetadata metadata, IImportInterceptor interceptor, ErrorTracer errorTracer, BuildParameter[] parameters)
{
Contract.Requires<ArgumentNullException>(target != null, "target");
Contract.Requires<ArgumentNullException>(container != null, "container");
Contract.Requires<ArgumentNullException>(metadata != null, "metadata");
Contract.Requires<ArgumentNullException>(errorTracer != null, "errorTracer");
throw new NotImplementedException();
}
/// <summary>
/// Imports xml metadata document
/// </summary>
/// <remarks>
/// used to import xml metadata document.
/// </remarks>
/// <param name="sourceXml">Metadata to be converted</param>
/// <param name="mdFileID">Unique file identifier</param>
/// <returns>Returns true if execution was successful.</returns>
private bool ImportMetadata(string sourceXml, ref String xmlFileName)
{
ImportMetadata gpTool = GpHelper.xmlImporter(sourceXml, xmlFileName);
// Request that the GpHelper execute the instantiated tool. A wrapper is
// used to abstract error handling and allow a simple integer success code
// equivalency check to determine success/failure.
string[] xmlFilePaths;
return(GpHelper.Execute(gpTool, out xmlFilePaths) == 0);
}
/// <summary>
/// Initialisiert eine neue Instanz der <see cref="EventMemberInjector" /> Klasse.
/// </summary>
/// <param name="metadata">
/// The metadata.
/// </param>
/// <param name="manager">
/// The manager.
/// </param>
/// <param name="member">
/// The member.
/// </param>
public EventMemberInjector(ImportMetadata metadata, IEventManager manager, MemberInfo member)
{
Contract.Requires<ArgumentNullException>(metadata != null, "metadata");
Contract.Requires<ArgumentNullException>(manager != null, "manager");
Contract.Requires<ArgumentNullException>(member != null, "member");
_metadata = metadata;
_manager = manager;
_member = member;
}
public ProxyGenerator Generate(ExportMetadata metadata, ImportMetadata[] imports, out IImportInterceptor interceptor)
{
Contract.Requires<ArgumentNullException>(metadata != null, "metadata");
Contract.Requires<ArgumentNullException>(imports != null, "imports");
Contract.Ensures(Contract.Result<ProxyGenerator>() != null);
interceptor = null;
return null;
}
public bool Intercept(MemberInfo member, ImportMetadata metadata, object target, ref object value)
{
bool returnValue = true;
foreach (var importInterceptor in _interceptors.Where(importInterceptor => returnValue))
{
returnValue = importInterceptor.Intercept(member, metadata, target, ref value);
}
return returnValue;
}
public void ImportMetadata_DoesDirectoryExist_ReturnsNull()
{
//arrange
const string path = "C:\\einTestPfad";
ImportMetadata import = new ImportMetadata();
//act
List <MetadataItem> result = import.ImportMetafiles(path, new XmlSer());
//assert
Assert.That(result, Is.Null);
}
public void ImportMetadata_DirectoryExist_ReturnsEmpty()
{
//arrange
const string path = "Testordner\\Testempty";
ImportMetadata import = new ImportMetadata();
//act
List <MetadataItem> result = import.ImportMetafiles(path, new XmlSer());
//assert
Assert.That(result, Is.Empty);
}
public void ImportMetadata_ImportList_ReturnsList()
{
//arrange
const string path = "Testordner";
ImportMetadata import = new ImportMetadata();
//act
List <MetadataItem> result = import.ImportMetafiles(path, new XmlSerStub(true));
//assert
Assert.That(result, Is.EqualTo(new List <MetadataItem> {
null, null, null
}));
}
public void ReceiveMetadataByCustomType()
{
ContainerConfiguration cfg = new ContainerConfiguration().WithAssembly(Assembly.GetExecutingAssembly());
ReflectionAndComposition r = new ReflectionAndComposition();
using (CompositionHost container = cfg.CreateContainer())
{
container.SatisfyImports(r);
}
ImportMetadata meta = r.CurPlayer.Metadata;
Console.WriteLine($"{nameof(ImportMetadata.MaxTracks)}:" +
$"{meta.MaxTracks}\n{nameof(ImportMetadata.Skin)}:" +
$"{meta.Skin}");
}
/// <summary>
/// Returns an instantiated ImportMetadata geoprocessing tool with input
/// and output parameters set to the specified values.
/// param name="inFile" the input file
/// param name="xmlFileName" the xml file name
/// </summary>
static public ImportMetadata xmlImporter(string inFile, String xmlFileName)
{
// Assign/Create the temporary files for translation output
string outFile = xmlFileName;
string logFile = Path.GetTempFileName();
// Instantiate the geoprocessing tool and define its parameters
// Usage: MDImporter_conversion <source> <output>
ImportMetadata mdImporter = new ImportMetadata();
mdImporter.Source_Metadata = inFile;
mdImporter.Output_Metadata = outFile;
return(mdImporter);
}
public ProxyGenerator Generate(ExportMetadata metadata, ImportMetadata[] imports, out IImportInterceptor interceptor)
{
interceptor = null;
if (!typeof (ModelBase).IsAssignableFrom(metadata.Export.ImplementType)) return _genericGenerator;
var targetImports =
imports.Where(meta => meta.Metadata.ContainsKey(EnablePropertyInheritanceMetadataName))
.Where(m => (bool) m.Metadata[EnablePropertyInheritanceMetadataName])
.ToArray();
if (targetImports.Length == 0) return _genericGenerator;
interceptor = new PropertyImportInterceptor(targetImports);
return _genericGenerator;
}
public void ShouldGenerateAutoMLCode()
{
string csContent = LoadFileContent("Data/AutoMLCode.txt");
var metadata = new ImportMetadata
{
ColumnDefinitions = new List <ColumnDefinition>
{
new ColumnDefinition
{
Name = "Category",
DataType = ColumnDataType.Text,
MLType = ColumnMLType.Label
},
new ColumnDefinition
{
Name = "Description",
DataType = ColumnDataType.Text,
MLType = ColumnMLType.Text
},
new ColumnDefinition
{
Name = "Account",
DataType = ColumnDataType.Text,
MLType = ColumnMLType.Categorical
},
new ColumnDefinition
{
Name = "Date",
DataType = ColumnDataType.Text,
MLType = ColumnMLType.Ignored
},
new ColumnDefinition
{
Name = "Expense",
DataType = ColumnDataType.Text,
MLType = ColumnMLType.Numeric
}
}
};
var generateModelCodeQuery = new AutoMLCodeQuery();
var code = generateModelCodeQuery.Execute(metadata);
// This make sure it passes on all platforms.
code.Replace("\r", string.Empty).Should().Be(csContent.Replace("\r", string.Empty));
}
public void Execute(ImportMetadata metadata)
{
foreach (var column in metadata.ColumnDefinitions)
{
bool isNumeric = !string.IsNullOrWhiteSpace(column.FirstValue) && double.TryParse(column.FirstValue, out _);
if (isNumeric)
{
column.DataType = ColumnDataType.Number;
column.MLType = ColumnMLType.Numeric;
}
else
{
column.DataType = ColumnDataType.Text;
column.MLType = ColumnMLType.Categorical;
}
}
}
public override void Inject(object target, IContainer container, ImportMetadata metadata, IImportInterceptor interceptor, ErrorTracer errorTracer, BuildParameter[] parameters)
{
errorTracer.Phase = "EventManager Inject " + metadata.ContractName;
try
{
var eventInfo = _member as EventInfo;
if (eventInfo != null) _manager.AddPublisher(_metadata.ContractName, eventInfo, target, errorTracer);
var method = _member as MethodInfo;
if (method != null) _manager.AddEventHandler(_metadata.ContractName, method, target, errorTracer);
}
catch (Exception e)
{
errorTracer.Exceptional = true;
errorTracer.Exception = e;
}
}
public async Task <ImportMetadata> CreateImportMetadataAsync(ImportMetadata metadata)
{
using (var cn = new SqlConnection(dbOptions.ConnectionString))
{
var grid = await cn.QueryMultipleAsync(
Sql.CreateImportMetadata,
new
{
sourceId = metadata.SourceId,
type = metadata.Type,
structure = metadata.StructureJson,
constraints = ResourceConstraintTable.From(metadata),
user = user.UUID
},
commandType : CommandType.StoredProcedure,
commandTimeout : dbOptions.DefaultTimeout
);
var created = DbReader.Read(grid);
return(created);
}
}
public string Execute(ImportMetadata metadata, string sampleNamespace, string modelsNamespace)
{
string autoMlCode = _autoMLCodeQuery.Execute(metadata);
autoMlCode = autoMlCode
.TrimEnd('\r', '\n')
.Replace("\n", "\n ");
return($@"using Microsoft.ML;
using Microsoft.ML.AutoML;
using Microsoft.ML.Data;
using {modelsNamespace};
namespace {sampleNamespace}
{{
public static class Program
{{
public static void Main(string[] args)
{{
string pathToCsv = ""[path-to-csv]"";
var mlContext = new MLContext();
IDataView trainingDataView = mlContext.Data.LoadFromTextFile<ModelInput>(
path: pathToCsv,
hasHeader: true,
separatorChar: ',',
allowQuoting: true,
allowSparse: false);
var model = AutoTrain(mlContext, trainingDataView, 5);
// Save training model to disk.
mlContext.Model.Save(model, trainingDataView.Schema, ""MLModel.zip"");
}}
{autoMlCode}
}}
}}
");
}
public string Execute(ImportMetadata metadata, string sampleNamespace, string modelsNamespace)
{
string autoMlCode = _autoMLCodeQuery.Execute(metadata);
autoMlCode = autoMlCode
.TrimEnd('\r', '\n')
.Replace("\n", "\n ");
return($@"using Microsoft.ML;
using Microsoft.ML.AutoML;
using Microsoft.ML.Data;
using System;
using {modelsNamespace}
namespace {sampleNamespace}
{{
public static class Program
{{
public static void Main(string[] args)
{{
var mlContext = new MLContext();
// Load model & create prediction engine
string modelPath = AppDomain.CurrentDomain.BaseDirectory + ""MLModel.zip"";
ITransformer mlModel = mlContext.Model.Load(modelPath, out var modelInputSchema);
var predEngine = mlContext.Model.CreatePredictionEngine<ModelInput, ModelOutput>(mlModel);
// TODO: Generate an input
ModelInput input = new ModelInput();
// Use model to make prediction on input data
ModelOutput result = predEngine.Predict(input);
Console.WriteLine(""Predicted value: "" + result.Prediction);
}}
}}
}}
");
}
public string Execute(ImportMetadata importMetadata)
{
string label = importMetadata.ColumnDefinitions
.Where(c => c.MLType == ColumnMLType.Label)
.Select(c => c.Name)
.Single();
string columnDefinitionCode = $@" var columnInfo = new ColumnInformation {{ LabelColumnName = ""{label}"" }};
";
foreach (var column in importMetadata.ColumnDefinitions.Where(c => c.MLType != ColumnMLType.Label && c.MLType != ColumnMLType.Ignored && c.MLType != ColumnMLType.Undefined))
{
string mlType = column.MLType.ToString();
columnDefinitionCode += @$ " columnInfo.{mlType}ColumnNames.Add(" "{column.Name}" ");
";
}
string template = $@"public static ITransformer AutoTrain(MLContext mlContext, IDataView trainingDataView, uint maxTimeInSec)
{{
{columnDefinitionCode}
var experimentSettings = new MulticlassExperimentSettings()
{{
MaxExperimentTimeInSeconds = maxTimeInSec,
OptimizingMetric = MulticlassClassificationMetric.MacroAccuracy
}};
var experiment = mlContext.Auto().CreateMulticlassClassificationExperiment(experimentSettings);
var result = experiment.Execute(trainingDataView, columnInfo);
// Final model that can be used and/or evaluated.
return result.BestRun.Model;
}}
";
return(template);
}
public ProxyGenerator Generate(ExportMetadata metadata, ImportMetadata[] imports, out IImportInterceptor interceptor)
{
interceptor = null;
return GenericGenerator;
}
/// <summary>
/// Returns an instantiated ImportMetadata geoprocessing tool with input
/// and output parameters set to the specified values.
/// param name="inFile" the input file
/// param name="xmlFileName" the xml file name
/// </summary>
static public ImportMetadata xmlImporter(string inFile, String xmlFileName)
{
// Assign/Create the temporary files for translation output
string outFile = xmlFileName;
string logFile = Path.GetTempFileName();
// Instantiate the geoprocessing tool and define its parameters
// Usage: MDImporter_conversion <source> <output>
ImportMetadata mdImporter = new ImportMetadata();
mdImporter.Source_Metadata = inFile;
mdImporter.Output_Metadata = outFile;
return mdImporter;
}
/// <summary>
/// Public wrapper used to execute an instantiated ImportMetadata geoprocessing tool.
/// The wrapper's try/catch block contains an error message identifying the type of tool.
///param name="gpTool" the geoprocessing tool
///param name="xmlFilePaths" the xml file paths
/// </summary>
static public int Execute(ImportMetadata gpTool, out string[] xmlFilePaths)
{
// Instantiate a new string array and set the anticipated paths
// into the array. Invoking method is responsible for cleanup
// once it has processed the files generated by this method.
xmlFilePaths = new string[3];
xmlFilePaths[0] = gpTool.Source_Metadata.ToString(); // Original XML
xmlFilePaths[1] = gpTool.Output_Metadata.ToString(); // Translated XML
try
{
Execute(gpTool as IGPProcess);
}
catch (Exception ex)
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("Unexpected exception generated by geoprocessing tool");
sb.AppendLine("Metadata Translator (Conversion Tools - Metadata)");
sb.AppendLine("");
sb.AppendLine(ex.Message);
MessageBox.Show(sb.ToString(), "Translation Failure",
MessageBoxButtons.OK, MessageBoxIcon.Warning);
retVal = -1;
}
return retVal;
}
public bool Intercept(MemberInfo member, ImportMetadata metadata, object target, ref object value)
{
if (!_metadatas.Contains(metadata)) return true;
var viewModel = (ViewModelBase) target;
var model = value as ModelBase;
if (model == null) return true;
viewModel.RegisterInheritedModel(metadata.ContractName, model);
return true;
}
