/// <summary> /// Gets the vectors with label identifier as dimension value. If vector is not found in this dictionary, it will set labelID to 0, for incorrect it will set 1 and for correct 2 /// </summary> /// <param name="CompleteDataSet">The complete data set.</param> /// <param name="criteria">The criteria.</param> /// <returns></returns> public FeatureVectorWithLabelIDSet GetVectorsWithLabelID(List <String> CompleteDataSet, Double criteria = 0.5, List <String> labels = null) { if (CompleteDataSet == null) { CompleteDataSet = new List <string>(); } labels = SpaceLabel.SetDefaultLabelList(CompleteDataSet.Any(), labels); CompleteDataSet.AddRange(this.Select(x => x.Key)); //if (labels==null) //{ // labels = new List<string>(); // labels.Add(SpaceLabel.UNKNOWN); // labels.Add(SpaceLabel.INCORRECT); // labels.Add(SpaceLabel.CORRECT); //} Int32 l_unknown = labels.IndexOf(SpaceLabel.UNKNOWN); Int32 l_correct = labels.IndexOf(SpaceLabel.CORRECT); Int32 l_incorrect = labels.IndexOf(SpaceLabel.INCORRECT); var output = new FeatureVectorWithLabelIDSet(); output.DoAutoSetUnknownLabels = false; foreach (String id in CompleteDataSet) { Int32 l = l_unknown; FeatureVectorWithLabelID fv_id = null; if (ContainsKey(id)) { if (this[id].dimensions[0] < criteria) { l = l_incorrect; } else { l = l_correct; } fv_id = new FeatureVectorWithLabelID(this[id], l); } else { if (l_unknown > -1) { fv_id = new FeatureVectorWithLabelID(new FeatureVector(id), l); } } output.Add(fv_id); } return(output); }
public IActionResult Post([FromBody] PipelineViewModel value) { var _id = Guid.Parse(userManager.GetUserId(User)); Profile profile = dataCenterContext.Profiles.Where(x => x.Id == _id).First(); Algorithm algorithm = dataCenterContext.Algorithms.Find(Guid.Parse(value.AlgorithmId)); CompleteDataSet completeDataSet = dataCenterContext.CompleteDataSets.Find(Guid.Parse(value.DataSetId)); Pipeline pipeline = new Pipeline { Algorithm = algorithm, Id = Guid.NewGuid(), Name = value.Name, Description = value.Description, NumberOfContainers = value.NumberOfContainers, GetCompleteDataSet = completeDataSet }; List <PipelineParameter> parameters = new List <PipelineParameter>(); foreach (var parameter in value.Parameters) { AlgorithmParameters algorithmParameter = dataCenterContext.AlgorithmParameters.Find(Guid.Parse(parameter.Id)); parameters.Add(new PipelineParameter { Id = Guid.NewGuid(), Pipeline = pipeline, Value = parameter.Value, AlgorithmParameter = algorithmParameter }); } pipeline.PipelineParameters = parameters; ProfilePipeline profilePipeline = new ProfilePipeline { Pipeline = pipeline, Profile = profile }; dataCenterContext.ProfilePipeline.Add(profilePipeline); dataCenterContext.SaveChanges(); RabbitMqService rabbitMqService = new RabbitMqService(); rabbitMqService.SendMessage(pipeline.Id + "~" + algorithm.MasterImage); for (int i = 0; i < pipeline.NumberOfContainers - 1; i++) { rabbitMqService.SendMessage(pipeline.Id + "~" + algorithm.SlaveImage); } return(Ok()); }
public IActionResult Post([FromBody] CompleteDataSetViewModel value) { var _id = Guid.Parse(userManager.GetUserId(User)); Profile profile = dataCenterContext.Profiles.Where(x => x.Id == _id).First(); CompleteDataSet completeDataSet = new CompleteDataSet { Id = Guid.NewGuid(), Description = value.Description, Name = value.Name, XComponentDataSet = dataCenterContext.DataSets.Find(Guid.Parse(value.XComponentId)), YComponentDataSet = dataCenterContext.DataSets.Find(Guid.Parse(value.YComponentId)) }; ProfileCompleteDataSet profileCompleteDataSet = new ProfileCompleteDataSet { CompleteDataSet = completeDataSet, Profile = profile }; dataCenterContext.ProfilesCompleteDataSets.Add(profileCompleteDataSet); dataCenterContext.SaveChanges(); return(Ok()); }
static void Main(string[] args) { string url = args[0]; string pipelineId = args[1]; IConfiguration Configuration = BuildConfiguration(); Credentials credentials = new Credentials { UserName = $"{Configuration["username-athena"]}", Password = $"{Configuration["password-athena"]}" }; WebHelper webHelper = new WebHelper() { Credentials = credentials, PipelineId = pipelineId, Url = url }; webHelper.FetchJWT(); Pipeline pipeline = webHelper.GetPipeline(); CompleteDataSet completeDataSet = webHelper.GetCompleteDataSet(pipeline.dataSetId); Dictionary <string, string> masterParameters = BuildMasterParameters(pipeline.parameters); string ftpUrl = "ftp" + ":" + url.Split(':')[1]; FtpLayer ftpLayer = new FtpLayer(ftpUrl, $"{Configuration["username-ftp"]}", $"{Configuration["password-ftp"]}"); ftpLayer.DownloadFile(completeDataSet.xComponentId); ftpLayer.DownloadFile(completeDataSet.yComponentId); NeuralNetworkMaster master = new NeuralNetworkMaster { PipelineId = pipelineId, NumberOfSlaves = pipeline.numberOfContainers - 1, InputLayerSize = int.Parse(masterParameters["InputLayerSize"]), HiddenLayerSize = int.Parse(masterParameters["HiddenLayerSize"]), HiddenLayerLength = int.Parse(masterParameters["HiddenLayerLength"]), OutputLayerSize = int.Parse(masterParameters["OutputLayerSize"]), Lambda = int.Parse(masterParameters["Lambda"]), Epoch = int.Parse(masterParameters["Epoch"]), XFileName = completeDataSet.xComponentId, YFileName = completeDataSet.yComponentId, Configuration = Configuration, WebHelper = webHelper }; master.Train(); StringBuilder files = new StringBuilder(); int hiddenLayerLength = int.Parse(masterParameters["HiddenLayerLength"]); for (int i = 0; i < (hiddenLayerLength + 1); i++) { ftpLayer.UploadFile(pipelineId + "_Theta_" + i); files.Append(pipelineId + "_Theta_" + i + ".csv,"); } files.Remove(files.Length - 1, 1); webHelper.PostResult(files.ToString()); }