public PipelineModel ToPipelineModel(PipelineEntity pipeline)
{
PipelineModel pipelineModel = new PipelineModel();
pipelineModel.Id = pipeline.id;
pipelineModel.CompanyId = pipeline.company_id;
pipelineModel.Connections = new List <PipelineItemConnectionModel>();
foreach (PipelineItemEntity pipelineItem in pipeline.PipelineItems)
{
PipelineItemConnectionModel connectionModel = new PipelineItemConnectionModel();
connectionModel.PipelineItem = pipelineItem.ToModel <PipelineItemModel>();
connectionModel.InputPipelineItems = ModelEntityMapper.Mapper.Map <ICollection <PipelineItemModel> >(
pipelineItem.InputPipelineItems
);
connectionModel.OutputPipelineItems = ModelEntityMapper.Mapper.Map <ICollection <PipelineItemModel> >(
pipelineItem.OutputPipelineItems
);
connectionModel.InputStorageCells = ModelEntityMapper.Mapper.Map <ICollection <StorageCellModel> >(
pipelineItem.InputStorageCells
);
connectionModel.OutputStorageCells = ModelEntityMapper.Mapper.Map <ICollection <StorageCellModel> >(
pipelineItem.OutputStorageCells
);
pipelineModel.Connections.Add(connectionModel);
}
return(pipelineModel);
}
public void TestPipelineFit()
{
DataFrame input = _spark.Sql("SELECT array('hello', 'I', 'AM', 'a', 'string', 'TO', " +
"'TOKENIZE') as input from range(100)");
const string inputColumn = "input";
const string outputColumn = "output";
const double minDf = 1;
const double minTf = 10;
const int vocabSize = 10000;
CountVectorizer countVectorizer = new CountVectorizer()
.SetInputCol(inputColumn)
.SetOutputCol(outputColumn)
.SetMinDF(minDf)
.SetMinTF(minTf)
.SetVocabSize(vocabSize);
var stages = new JavaPipelineStage[] {
countVectorizer
};
Pipeline pipeline = new Pipeline().SetStages(stages);
PipelineModel pipelineModel = pipeline.Fit(input);
DataFrame output = pipelineModel.Transform(input);
Assert.IsType <StructType>(pipelineModel.TransformSchema(input.Schema()));
Assert.IsType <DataFrame>(output);
using (var tempDirectory = new TemporaryDirectory())
{
string savePath = Path.Join(tempDirectory.Path, "pipeline");
pipeline.Save(savePath);
Pipeline loadedPipeline = Pipeline.Load(savePath);
Assert.Equal(pipeline.Uid(), loadedPipeline.Uid());
string writePath = Path.Join(tempDirectory.Path, "pipelineWithWrite");
pipeline.Write().Save(writePath);
Pipeline loadedPipelineWithRead = pipeline.Read().Load(writePath);
Assert.Equal(pipeline.Uid(), loadedPipelineWithRead.Uid());
}
TestFeatureBase(pipeline, "stages", stages);
}
public PipelineModel GetAzurePipeline()
{
var myPipeLine = new PipelineModel
{
Name = _name,
Type = "Microsoft.DataFactory/factories/pipelines",
Properties = new PropertiesModel
{
Activities = _activityModel.ToArray(),
PropertyDescription = _propertyDescription,
Folder = new Folder
{
Name = _folder
},
Annotations = new string[] { }
}
};
return(myPipeLine);
}
private PipelineMetric Monitor(PipelineModel pipeline, IStorage storage)
{
var defaultMetrics = new List <MetricType> {
MetricType.ThreadCount, MetricType.QueueSize, MetricType.ProcessedEvents
};
var server = pipeline.ServerName ?? Environment.MachineName;
var heartbeat = storage.Get <ServerModel>(new StorageKey($"server:{server.ToLower()}"));
var statistics = new StatisticsApi(server, pipeline.PipelineId, storage);
var metrics = new PipelineMetric(pipeline.PipelineId, server, heartbeat != null ? DateTime.Parse(heartbeat.Heartbeat) : (DateTime?)null);
metrics.AddMetric("Server", "Server", server);
metrics.AddMetric("Heartbeat", "Heartbeat", heartbeat != null ? DateTime.Parse(heartbeat.Heartbeat).ToString("yyyy.MM.dd HH:mm:ss") : "");
foreach (var key in defaultMetrics)
{
var metric = statistics.FirstOrDefault(s => s.Key == key);
if (metric == null)
{
metric = new Metric(key, "", "");
}
metrics.AddMetric(key.ToString(), metric?.Title, metric?.Value);
}
foreach (var metric in statistics)
{
if (defaultMetrics.Contains(metric.Key))
{
continue;
}
switch (metric.Key)
{
case MetricType.EventLog:
//LogEventStatisticWriter.cs
if (metric.Value is IEnumerable <LogEvent> logs)
{
foreach (var log in logs.OrderByDescending(l => l.Timestamp).Take(Options.LogCount))
{
metrics.AddElement(metric.Key.ToString(), metric.Title, new DashboardLog
{
Timestamp = log.Timestamp,
Source = log.Source,
Level = log.Level.ToString(),
Message = log.Message
});
}
}
break;
case MetricType.WorkersLog:
//WorkersLogMetricCounter.cs
if (metric.Value is IEnumerable <ActiveWorkersLogMessage> workers)
{
foreach (var worker in workers.OrderByDescending(w => w.Timestamp).Take(Options.WorkersLogCount))
{
metrics.AddElement(metric.Key.ToString(), metric.Title, new TimelineLog <int>
{
Timestamp = worker.Timestamp,
Value = worker.ActiveWorkers
});
}
}
break;
default:
metrics.AddMetric(metric.Key.ToString(), metric.Title, metric.Value ?? 0);
break;
}
}
return(metrics);
}
public void SetPipelineModel(PipelineModel model)
{
Settings.Add(PipelineModel, model);
}
public ActionResult ShowFormAddPipeline(long pipelineId)
{
AddPipelineViewModel addPipeline = new AddPipelineViewModel();
addPipeline.Pipeline = new PipelineModel()
{
};
var param = new BasicParamModel()
{
FilterField = string.Empty,
PageNumber = 1,
PageSize = int.MaxValue,
LanguageId = CurrentUser.LanguageId,
RoleId = 1,
UserId = CurrentUser.UserId,
DbName = CurrentUser.DbName
};
List <PipelineStepModel> pipelineSteps = new List <PipelineStepModel>();
PipelineModel pipeline = new PipelineModel();
if (pipelineId != 0)
{
var pipelineStepResponse = this._pipelineService.GetPipelineStepByPipelineId(pipelineId);
if (!string.IsNullOrEmpty(pipelineStepResponse))
{
var result = JsonConvert.DeserializeObject <HrmResultModel <PipelineStepModel> >(pipelineStepResponse);
if (!CheckPermission(result))
{
//return to Access Denied
}
else
{
pipelineSteps = result.Results;
}
}
var pipelineResponse = this._pipelineService.GetPipelineById(pipelineId);
if (!string.IsNullOrEmpty(pipelineResponse))
{
var result = JsonConvert.DeserializeObject <HrmResultModel <PipelineModel> >(pipelineResponse);
if (!CheckPermission(result))
{
//return to Access Denied
}
else
{
pipeline = result.Results.FirstOrDefault();
}
}
}
if (pipelineSteps.Count() == 0)
{
List <MasterDataModel> masterDatas = new List <MasterDataModel>();
var responseMasterData = this._masterDataService.GetAllMasterDataByName(MasterGroup.PipelinePosition, _languageId);
if (responseMasterData != null)
{
var resultMasterData = JsonConvert.DeserializeObject <HrmResultModel <MasterDataModel> >(responseMasterData);
if (!CheckPermission(resultMasterData))
{
//return to Access Denied
}
else
{
masterDatas = resultMasterData.Results;
}
}
foreach (var item in masterDatas)
{
pipelineSteps.Add(new PipelineStepModel()
{
PositionId = item.Id,
PositionName = item.Name
});
}
}
addPipeline.Pipeline = pipeline;
addPipeline.PipelineStep = pipelineSteps;
if (addPipeline.Menu == null)
{
var menu_vm = new Hrm.Framework.ViewModels.MenuViewModel();
menu_vm.Menus = new List <MenuModel>();
var response = _menuService.GetMenu();
if (response != null)
{
var result = JsonConvert.DeserializeObject <HrmResultModel <MenuModel> >(response);
if (!CheckPermission(result))
{
//return to Access Denied
}
else
{
menu_vm.Menus = result.Results;
addPipeline.Menu = JsonConvert.DeserializeObject <List <dynamic> >(JsonConvert.SerializeObject(menu_vm.Menus));
}
}
}
return(PartialView(UrlHelpers.TemplateAdmin("Pipeline", "_AddPipeline.cshtml"), addPipeline));
}
