public static OrionQuery CreateDataDescriptionQuery(DataIntegration ign, IEnumerable <ModelTarget> targets) { var qr = new OrionQuery(OrionOp.AnalyzeFile); var data = new JObject(); data["src"] = ign.Collection; data["src_type"] = "collection"; data["targets"] = CreateTargetsDef(targets.ToArray()); data["formatting"] = new JObject(); qr["params"] = data; return(qr); }
public void TriggerSAPSyncToDownloadToSQL() { IntegrationLibrary.library.DataIntegration di = new IntegrationLibrary.library.DataIntegration(); di.EI_CONN_STRING = ConfigurationManager.AppSettings["IntegrationConnStr"].ToString(); di.jobs_id = "1"; di.variables = new string[0]; di.execute(); di = new DataIntegration(); di.EI_CONN_STRING = ConfigurationManager.AppSettings["IntegrationConnStr"].ToString(); di.jobs_id = "2"; di.variables = new string[0]; di.execute(); }
public DonutBuilder(DataIntegration integration, IRedisCacher cacher, IServiceProvider serviceProvider) { _tContext = typeof(TContext); _serviceProvider = serviceProvider; var assembly = Assembly.GetExecutingAssembly(); var resourceName = "Donut.Lex.Templates.Donutfile.txt"; _cacher = cacher; _integration = integration; using (Stream stream = assembly.GetManifestResourceStream(resourceName)) using (StreamReader reader = new StreamReader(stream)) { _template = reader.ReadToEnd(); } }
public static DonutScript Create( string donutName, IEnumerable <ModelTarget> targets, DataIntegration integration) { var script = new DonutScript() { Type = new ScriptTypeInfo(donutName), Targets = targets }; ValidateIntegrations(integration); script.AddIntegrations(integration); ValidateIntegrations(integration); return(script); }
public static void SetEncoding(DataIntegration ign, FieldDefinition targetField, Type opsEncoding) { var ctor = opsEncoding.GetConstructor(new Type[] { typeof(FieldEncodingOptions) }); FieldEncoding encoding = ctor.Invoke(new object[] { new FieldEncodingOptions { Integration = ign } }) as FieldEncoding; if (encoding == null) { throw new Exception($"Could not create encoding for field: {targetField.Name}"); } targetField.DataEncoding = encoding.Encoding; targetField.Extras = new FieldExtras(); //targetField.Extras.Field = targetField; }
/// <summary> /// TODO: Simplify /// </summary> /// <param name="model"></param> /// <param name="ign"></param> /// <param name="trainingTasks"></param> /// <returns></returns> public static OrionQuery CreateTrainQuery( Model model, DataIntegration ign, IEnumerable <TrainingTask> trainingTasks) { var rootIntegration = ign; var qr = new OrionQuery(OrionOp.Train); var parameters = new JObject(); var models = new JObject(); var dataOptions = new JObject(); var autoModel = new JObject(); trainingTasks = trainingTasks.ToList(); var flags = GetModelDataFlags(model, null); parameters["client"] = model.User.UserName; parameters["experiment_name"] = $"Model{model.Id}"; //Todo update.. parameters["targets"] = CreateTargetsDef(model.Targets.ToArray()); parameters["tasks"] = new JArray(trainingTasks.Select(x => x.Id).ToArray()); parameters["script"] = GetTrainingScript(trainingTasks); models["auto"] = autoModel; // GridSearchCV - param_grid var sourceCol = ign.GetModelSourceCollection(model); dataOptions["db"] = sourceCol; dataOptions["start"] = null; dataOptions["end"] = null; dataOptions["scoring"] = "auto"; dataOptions["collection"] = flags["collection"]; //Get fields from the mongo collection, these would be already generated features, so it's safe to use them BsonDocument featuresDoc = MongoHelper.GetCollection(sourceCol).AsQueryable().FirstOrDefault(); var fields = GetFieldsList(featuresDoc, rootIntegration); dataOptions["fields"] = fields; parameters["models"] = models; parameters["options"] = dataOptions; parameters["model_id"] = model.Id; qr["params"] = parameters; return(qr); }
/// <summary> /// /// </summary> /// <param name="donutName"></param> /// <param name="target"></param> /// <param name="integration"></param> /// <param name="featureBodies"></param> /// <returns></returns> public static DonutScript CreateWithFeatures(string donutName, IEnumerable <ModelTarget> targets, DataIntegration integration, params string[] featureBodies) { var script = Create(donutName, targets, integration); ValidateIntegrations(integration); var tokenizer = new FeatureToolsTokenizer(integration); int i = 0; foreach (var fstring in featureBodies) { if (string.IsNullOrEmpty(fstring)) { continue; } var featureName = $"f_{i}"; try { var parser = new DonutSyntaxReader(tokenizer.Tokenize(fstring)); IExpression expFeatureBody = parser.ReadExpression(); if (expFeatureBody == null) { continue; } if (targets != null && targets.Any(x => x.Column.Name == expFeatureBody.ToString())) { featureName = targets.First().Column.Name; } var expFeature = new AssignmentExpression(new NameExpression(featureName), expFeatureBody); script.Features.Add(expFeature); } catch (Exception ex) { throw new FeatureGenerationFailed(featureName, fstring, ex); } i++; } return(script); }
public static OrionQuery CreatePredictionQuery(Model model, DataIntegration getRootIntegration) { throw new NotImplementedException(); }
private static JArray GetFieldsList(IEnumerable <BsonElement> featureFields, DataIntegration rootIntegration) { var output = new JArray(); foreach (var field in featureFields) { var jsfld = new JObject(); jsfld["name"] = field.Name; var matchingField = rootIntegration.Fields.FirstOrDefault(x => x.Name == field.Name); if (matchingField != null) { jsfld["id"] = matchingField.Id; } var isEncoded = matchingField != null && matchingField.DataEncoding != FieldDataEncoding.None; if (isEncoded) { var encodedFields = GetEncodedFieldSubfields(rootIntegration, matchingField); foreach (var encFld in encodedFields) { encFld["is_key"] = rootIntegration.DataIndexColumn != null && !string.IsNullOrEmpty(rootIntegration.DataIndexColumn) && encFld["name"].ToString() == rootIntegration.DataIndexColumn; encFld["type"] = "float"; //if (encFld.Value.IsDateTime) jsfld["type"] = "datetime"; //else if (encFld.Value.IsString) jsfld["type"] = "str"; output.Add(encFld); } } else { jsfld["is_key"] = rootIntegration.DataIndexColumn != null && !string.IsNullOrEmpty(rootIntegration.DataIndexColumn) && field.Name == rootIntegration.DataIndexColumn; jsfld["type"] = "float"; if (field.Value.IsDateTime) { jsfld["type"] = "datetime"; } else if (field.Value.IsString) { jsfld["type"] = "str"; } output.Add(jsfld); } } return(output); }