/// <summary>
/// Gets the vectors with label identifier as dimension value. If vector is not found in this dictionary, it will set labelID to 0, for incorrect it will set 1 and for correct 2
/// </summary>
/// <param name="CompleteDataSet">The complete data set.</param>
/// <param name="criteria">The criteria.</param>
/// <returns></returns>
public FeatureVectorWithLabelIDSet GetVectorsWithLabelID(List <String> CompleteDataSet, Double criteria = 0.5, List <String> labels = null)
{
if (CompleteDataSet == null)
{
CompleteDataSet = new List <string>();
}
labels = SpaceLabel.SetDefaultLabelList(CompleteDataSet.Any(), labels);
CompleteDataSet.AddRange(this.Select(x => x.Key));
//if (labels==null)
//{
// labels = new List<string>();
// labels.Add(SpaceLabel.UNKNOWN);
// labels.Add(SpaceLabel.INCORRECT);
// labels.Add(SpaceLabel.CORRECT);
//}
Int32 l_unknown = labels.IndexOf(SpaceLabel.UNKNOWN);
Int32 l_correct = labels.IndexOf(SpaceLabel.CORRECT);
Int32 l_incorrect = labels.IndexOf(SpaceLabel.INCORRECT);
var output = new FeatureVectorWithLabelIDSet();
output.DoAutoSetUnknownLabels = false;
foreach (String id in CompleteDataSet)
{
Int32 l = l_unknown;
FeatureVectorWithLabelID fv_id = null;
if (ContainsKey(id))
{
if (this[id].dimensions[0] < criteria)
{
l = l_incorrect;
}
else
{
l = l_correct;
}
fv_id = new FeatureVectorWithLabelID(this[id], l);
}
else
{
if (l_unknown > -1)
{
fv_id = new FeatureVectorWithLabelID(new FeatureVector(id), l);
}
}
output.Add(fv_id);
}
return(output);
}
public IActionResult Post([FromBody] PipelineViewModel value)
{
var _id = Guid.Parse(userManager.GetUserId(User));
Profile profile = dataCenterContext.Profiles.Where(x => x.Id == _id).First();
Algorithm algorithm = dataCenterContext.Algorithms.Find(Guid.Parse(value.AlgorithmId));
CompleteDataSet completeDataSet = dataCenterContext.CompleteDataSets.Find(Guid.Parse(value.DataSetId));
Pipeline pipeline = new Pipeline
{
Algorithm = algorithm,
Id = Guid.NewGuid(),
Name = value.Name,
Description = value.Description,
NumberOfContainers = value.NumberOfContainers,
GetCompleteDataSet = completeDataSet
};
List <PipelineParameter> parameters = new List <PipelineParameter>();
foreach (var parameter in value.Parameters)
{
AlgorithmParameters algorithmParameter = dataCenterContext.AlgorithmParameters.Find(Guid.Parse(parameter.Id));
parameters.Add(new PipelineParameter
{
Id = Guid.NewGuid(),
Pipeline = pipeline,
Value = parameter.Value,
AlgorithmParameter = algorithmParameter
});
}
pipeline.PipelineParameters = parameters;
ProfilePipeline profilePipeline = new ProfilePipeline
{
Pipeline = pipeline,
Profile = profile
};
dataCenterContext.ProfilePipeline.Add(profilePipeline);
dataCenterContext.SaveChanges();
RabbitMqService rabbitMqService = new RabbitMqService();
rabbitMqService.SendMessage(pipeline.Id + "~" + algorithm.MasterImage);
for (int i = 0; i < pipeline.NumberOfContainers - 1; i++)
{
rabbitMqService.SendMessage(pipeline.Id + "~" + algorithm.SlaveImage);
}
return(Ok());
}
public IActionResult Post([FromBody] CompleteDataSetViewModel value)
{
var _id = Guid.Parse(userManager.GetUserId(User));
Profile profile = dataCenterContext.Profiles.Where(x => x.Id == _id).First();
CompleteDataSet completeDataSet = new CompleteDataSet
{
Id = Guid.NewGuid(),
Description = value.Description,
Name = value.Name,
XComponentDataSet = dataCenterContext.DataSets.Find(Guid.Parse(value.XComponentId)),
YComponentDataSet = dataCenterContext.DataSets.Find(Guid.Parse(value.YComponentId))
};
ProfileCompleteDataSet profileCompleteDataSet = new ProfileCompleteDataSet
{
CompleteDataSet = completeDataSet,
Profile = profile
};
dataCenterContext.ProfilesCompleteDataSets.Add(profileCompleteDataSet);
dataCenterContext.SaveChanges();
return(Ok());
}
static void Main(string[] args)
{
string url = args[0];
string pipelineId = args[1];
IConfiguration Configuration = BuildConfiguration();
Credentials credentials = new Credentials
{
UserName = $"{Configuration["username-athena"]}",
Password = $"{Configuration["password-athena"]}"
};
WebHelper webHelper = new WebHelper()
{
Credentials = credentials,
PipelineId = pipelineId,
Url = url
};
webHelper.FetchJWT();
Pipeline pipeline = webHelper.GetPipeline();
CompleteDataSet completeDataSet = webHelper.GetCompleteDataSet(pipeline.dataSetId);
Dictionary <string, string> masterParameters = BuildMasterParameters(pipeline.parameters);
string ftpUrl = "ftp" + ":" + url.Split(':')[1];
FtpLayer ftpLayer = new FtpLayer(ftpUrl, $"{Configuration["username-ftp"]}", $"{Configuration["password-ftp"]}");
ftpLayer.DownloadFile(completeDataSet.xComponentId);
ftpLayer.DownloadFile(completeDataSet.yComponentId);
NeuralNetworkMaster master = new NeuralNetworkMaster
{
PipelineId = pipelineId,
NumberOfSlaves = pipeline.numberOfContainers - 1,
InputLayerSize = int.Parse(masterParameters["InputLayerSize"]),
HiddenLayerSize = int.Parse(masterParameters["HiddenLayerSize"]),
HiddenLayerLength = int.Parse(masterParameters["HiddenLayerLength"]),
OutputLayerSize = int.Parse(masterParameters["OutputLayerSize"]),
Lambda = int.Parse(masterParameters["Lambda"]),
Epoch = int.Parse(masterParameters["Epoch"]),
XFileName = completeDataSet.xComponentId,
YFileName = completeDataSet.yComponentId,
Configuration = Configuration,
WebHelper = webHelper
};
master.Train();
StringBuilder files = new StringBuilder();
int hiddenLayerLength = int.Parse(masterParameters["HiddenLayerLength"]);
for (int i = 0; i < (hiddenLayerLength + 1); i++)
{
ftpLayer.UploadFile(pipelineId + "_Theta_" + i);
files.Append(pipelineId + "_Theta_" + i + ".csv,");
}
files.Remove(files.Length - 1, 1);
webHelper.PostResult(files.ToString());
}
