public static OrionQuery CreateDataDescriptionQuery(DataIntegration ign, IEnumerable <ModelTarget> targets)
{
var qr = new OrionQuery(OrionOp.AnalyzeFile);
var data = new JObject();
data["src"] = ign.Collection;
data["src_type"] = "collection";
data["targets"] = CreateTargetsDef(targets.ToArray());
data["formatting"] = new JObject();
qr["params"] = data;
return(qr);
}
public void TriggerSAPSyncToDownloadToSQL()
{
IntegrationLibrary.library.DataIntegration di = new IntegrationLibrary.library.DataIntegration();
di.EI_CONN_STRING = ConfigurationManager.AppSettings["IntegrationConnStr"].ToString();
di.jobs_id = "1";
di.variables = new string[0];
di.execute();
di = new DataIntegration();
di.EI_CONN_STRING = ConfigurationManager.AppSettings["IntegrationConnStr"].ToString();
di.jobs_id = "2";
di.variables = new string[0];
di.execute();
}
public DonutBuilder(DataIntegration integration, IRedisCacher cacher, IServiceProvider serviceProvider)
{
_tContext = typeof(TContext);
_serviceProvider = serviceProvider;
var assembly = Assembly.GetExecutingAssembly();
var resourceName = "Donut.Lex.Templates.Donutfile.txt";
_cacher = cacher;
_integration = integration;
using (Stream stream = assembly.GetManifestResourceStream(resourceName))
using (StreamReader reader = new StreamReader(stream))
{
_template = reader.ReadToEnd();
}
}
public static DonutScript Create(
string donutName,
IEnumerable <ModelTarget> targets,
DataIntegration integration)
{
var script = new DonutScript()
{
Type = new ScriptTypeInfo(donutName), Targets = targets
};
ValidateIntegrations(integration);
script.AddIntegrations(integration);
ValidateIntegrations(integration);
return(script);
}
public static void SetEncoding(DataIntegration ign, FieldDefinition targetField, Type opsEncoding)
{
var ctor = opsEncoding.GetConstructor(new Type[] { typeof(FieldEncodingOptions) });
FieldEncoding encoding = ctor.Invoke(new object[] { new FieldEncodingOptions
{
Integration = ign
} }) as FieldEncoding;
if (encoding == null)
{
throw new Exception($"Could not create encoding for field: {targetField.Name}");
}
targetField.DataEncoding = encoding.Encoding;
targetField.Extras = new FieldExtras();
//targetField.Extras.Field = targetField;
}
/// <summary>
/// TODO: Simplify
/// </summary>
/// <param name="model"></param>
/// <param name="ign"></param>
/// <param name="trainingTasks"></param>
/// <returns></returns>
public static OrionQuery CreateTrainQuery(
Model model,
DataIntegration ign,
IEnumerable <TrainingTask> trainingTasks)
{
var rootIntegration = ign;
var qr = new OrionQuery(OrionOp.Train);
var parameters = new JObject();
var models = new JObject();
var dataOptions = new JObject();
var autoModel = new JObject();
trainingTasks = trainingTasks.ToList();
var flags = GetModelDataFlags(model, null);
parameters["client"] = model.User.UserName;
parameters["experiment_name"] = $"Model{model.Id}";
//Todo update..
parameters["targets"] = CreateTargetsDef(model.Targets.ToArray());
parameters["tasks"] = new JArray(trainingTasks.Select(x => x.Id).ToArray());
parameters["script"] = GetTrainingScript(trainingTasks);
models["auto"] = autoModel; // GridSearchCV - param_grid
var sourceCol = ign.GetModelSourceCollection(model);
dataOptions["db"] = sourceCol;
dataOptions["start"] = null;
dataOptions["end"] = null;
dataOptions["scoring"] = "auto";
dataOptions["collection"] = flags["collection"];
//Get fields from the mongo collection, these would be already generated features, so it's safe to use them
BsonDocument featuresDoc = MongoHelper.GetCollection(sourceCol).AsQueryable().FirstOrDefault();
var fields = GetFieldsList(featuresDoc, rootIntegration);
dataOptions["fields"] = fields;
parameters["models"] = models;
parameters["options"] = dataOptions;
parameters["model_id"] = model.Id;
qr["params"] = parameters;
return(qr);
}
/// <summary>
///
/// </summary>
/// <param name="donutName"></param>
/// <param name="target"></param>
/// <param name="integration"></param>
/// <param name="featureBodies"></param>
/// <returns></returns>
public static DonutScript CreateWithFeatures(string donutName, IEnumerable <ModelTarget> targets, DataIntegration integration, params string[] featureBodies)
{
var script = Create(donutName, targets, integration);
ValidateIntegrations(integration);
var tokenizer = new FeatureToolsTokenizer(integration);
int i = 0;
foreach (var fstring in featureBodies)
{
if (string.IsNullOrEmpty(fstring))
{
continue;
}
var featureName = $"f_{i}";
try
{
var parser = new DonutSyntaxReader(tokenizer.Tokenize(fstring));
IExpression expFeatureBody = parser.ReadExpression();
if (expFeatureBody == null)
{
continue;
}
if (targets != null && targets.Any(x => x.Column.Name == expFeatureBody.ToString()))
{
featureName = targets.First().Column.Name;
}
var expFeature = new AssignmentExpression(new NameExpression(featureName), expFeatureBody);
script.Features.Add(expFeature);
}
catch (Exception ex)
{
throw new FeatureGenerationFailed(featureName, fstring, ex);
}
i++;
}
return(script);
}
public static OrionQuery CreatePredictionQuery(Model model, DataIntegration getRootIntegration)
{
throw new NotImplementedException();
}
private static JArray GetFieldsList(IEnumerable <BsonElement> featureFields, DataIntegration rootIntegration)
{
var output = new JArray();
foreach (var field in featureFields)
{
var jsfld = new JObject();
jsfld["name"] = field.Name;
var matchingField = rootIntegration.Fields.FirstOrDefault(x => x.Name == field.Name);
if (matchingField != null)
{
jsfld["id"] = matchingField.Id;
}
var isEncoded = matchingField != null && matchingField.DataEncoding != FieldDataEncoding.None;
if (isEncoded)
{
var encodedFields = GetEncodedFieldSubfields(rootIntegration, matchingField);
foreach (var encFld in encodedFields)
{
encFld["is_key"] = rootIntegration.DataIndexColumn != null &&
!string.IsNullOrEmpty(rootIntegration.DataIndexColumn) &&
encFld["name"].ToString() == rootIntegration.DataIndexColumn;
encFld["type"] = "float";
//if (encFld.Value.IsDateTime) jsfld["type"] = "datetime";
//else if (encFld.Value.IsString) jsfld["type"] = "str";
output.Add(encFld);
}
}
else
{
jsfld["is_key"] = rootIntegration.DataIndexColumn != null &&
!string.IsNullOrEmpty(rootIntegration.DataIndexColumn) &&
field.Name == rootIntegration.DataIndexColumn;
jsfld["type"] = "float";
if (field.Value.IsDateTime)
{
jsfld["type"] = "datetime";
}
else if (field.Value.IsString)
{
jsfld["type"] = "str";
}
output.Add(jsfld);
}
}
return(output);
}
