public async Task <NlpDoc> Predict(AgentBase agent, AiRequest request)
{
// load model
var dir = Path.Combine(request.AgentDir, request.Model);
Console.WriteLine($"Load model from {dir}");
var metaJson = File.ReadAllText(Path.Combine(dir, "model-meta.json"));
var meta = JsonConvert.DeserializeObject <ModelMetaData>(metaJson);
// Get NLP Provider
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var providerPipe = meta.Pipeline.First();
var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpProvider;
provider.Configuration = config.GetSection(meta.Platform);
var data = new NlpDoc
{
Sentences = new List <NlpDocSentence>
{
new NlpDocSentence
{
Text = request.Text
}
}
};
await provider.Load(agent, providerPipe);
meta.Pipeline.RemoveAt(0);
var settings = new PipeSettings
{
ModelDir = dir,
ProjectDir = request.AgentDir
};
// pipe process
var pipelines = config.GetValue <String>($"{meta.BotEngine}:pipe")
.Split(',')
.Select(x => x.Trim())
.ToList();
for (int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++)
{
var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpPredict;
pipe.Configuration = config.GetSection(meta.BotEngine).GetSection(pipelines[pipeIdx]);
pipe.Settings = settings;
var pipeModel = meta.Pipeline.FirstOrDefault(x => x.Name == pipelines[pipeIdx]);
await pipe.Predict(agent, data, pipeModel);
}
Console.WriteLine($"Prediction result:", Color.Green);
Console.WriteLine(JsonConvert.SerializeObject(data, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
return(data);
}
public async Task <NlpDoc> Predict(Agent agent, AIRequest request)
{
// load model
var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
Console.WriteLine($"Load model from {dir}");
var metaJson = File.ReadAllText(Path.Join(dir, "metadata.json"));
var meta = JsonConvert.DeserializeObject <ModelMetaData>(metaJson);
// Get NLP Provider
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var providerPipe = meta.Pipeline.First();
var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpPipeline;
provider.Configuration = config.GetSection(meta.Platform);
var data = new NlpDoc
{
Sentences = new List <NlpDocSentence>
{
new NlpDocSentence
{
Text = request.Query.FirstOrDefault()
}
}
};
await provider.Train(agent, data, providerPipe);
meta.Pipeline.RemoveAt(0);
var settings = new PipeSettings
{
ModelDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id),
PredictDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "PredictFiles", agent.Id),
AlgorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
};
if (!Directory.Exists(settings.PredictDir))
{
Directory.CreateDirectory(settings.PredictDir);
}
// pipe process
meta.Pipeline.ForEach(async pipeMeta =>
{
var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
pipe.Settings = settings;
await pipe.Predict(agent, data, pipeMeta);
});
Console.WriteLine(JsonConvert.SerializeObject(data, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
return(data);
}
public async Task <ModelMetaData> Train(Agent agent, BotTrainOptions options)
{
var data = new NlpDoc();
// Get NLP Provider
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var platform = config.GetSection($"BotPlatform").Value;
string providerName = config.GetSection($"{platform}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpProvider;
provider.Configuration = config.GetSection(platform);
var pipeModel = new PipeModel
{
Name = providerName,
Class = provider.ToString(),
Meta = new JObject(),
Time = DateTime.UtcNow
};
await provider.Load(agent, pipeModel);
var settings = new PipeSettings
{
ProjectDir = options.AgentDir,
AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
};
settings.ModelDir = Path.Combine(options.AgentDir, options.Model);
if (!Directory.Exists(settings.ProjectDir))
{
Directory.CreateDirectory(settings.ProjectDir);
}
if (!Directory.Exists(settings.TempDir))
{
Directory.CreateDirectory(settings.TempDir);
}
if (!Directory.Exists(settings.ModelDir))
{
Directory.CreateDirectory(settings.ModelDir);
}
var meta = new ModelMetaData
{
Platform = platform,
Language = agent.Language,
TrainingDate = DateTime.UtcNow,
Version = config.GetValue <String>($"Version"),
Pipeline = new List <PipeModel>()
{
pipeModel
},
Model = settings.ModelDir
};
// pipe process
var pipelines = provider.Configuration.GetValue <String>($"Pipe:train")
.Split(',')
.Select(x => x.Trim())
.ToList();
for (int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++)
{
var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpTrain;
// set configuration to current section
pipe.Configuration = provider.Configuration.GetSection(pipelines[pipeIdx]);
pipe.Settings = settings;
pipeModel = new PipeModel
{
Name = pipelines[pipeIdx],
Class = pipe.ToString(),
Time = DateTime.UtcNow
};
meta.Pipeline.Add(pipeModel);
await pipe.Train(agent, data, pipeModel);
}
// save model meta data
var metaJson = JsonConvert.SerializeObject(meta, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
File.WriteAllText(Path.Combine(settings.ModelDir, "model-meta.json"), metaJson);
Console.WriteLine(metaJson);
return(meta);
}
public async Task <ModelMetaData> Train(Agent agent)
{
/*agent.Intents = dc.Table<Intent>()
* .Include(x => x.Contexts)
* .Include(x => x.Responses).ThenInclude(x => x.Contexts)
* .Include(x => x.Responses).ThenInclude(x => x.Parameters).ThenInclude(x => x.Prompts)
* .Include(x => x.Responses).ThenInclude(x => x.Messages)
* .Include(x => x.UserSays).ThenInclude(x => x.Data)
* .Where(x => x.AgentId == agentId)
* .ToList();*/
var data = new NlpDoc();
// Get NLP Provider
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var platform = config.GetSection($"BotPlatform").Value;
string providerName = config.GetSection($"{platform}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpProvider;
provider.Configuration = config.GetSection(platform);
var pipeModel = new PipeModel
{
Name = providerName,
Class = provider.ToString(),
Meta = new JObject(),
Time = DateTime.UtcNow
};
await provider.Load(agent, pipeModel);
var settings = new PipeSettings
{
ProjectDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id),
AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
};
settings.ModelDir = Path.Combine(settings.ProjectDir, "model" + DateTime.UtcNow.ToString("MMddyyyyHHmm"));
if (!Directory.Exists(settings.ProjectDir))
{
Directory.CreateDirectory(settings.ProjectDir);
}
if (!Directory.Exists(settings.TempDir))
{
Directory.CreateDirectory(settings.TempDir);
}
if (!Directory.Exists(settings.ModelDir))
{
Directory.CreateDirectory(settings.ModelDir);
}
var meta = new ModelMetaData
{
Platform = platform,
Language = agent.Language,
TrainingDate = DateTime.UtcNow,
Version = config.GetValue <String>($"Version"),
Pipeline = new List <PipeModel>()
{
pipeModel
},
Model = settings.ModelDir
};
// pipe process
var pipelines = provider.Configuration.GetValue <String>($"Pipe:train")
.Split(',')
.Select(x => x.Trim())
.ToList();
pipelines.ForEach(async pipeName =>
{
var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpTrain;
pipe.Configuration = provider.Configuration;
pipe.Settings = settings;
pipeModel = new PipeModel
{
Name = pipeName,
Class = pipe.ToString(),
Time = DateTime.UtcNow
};
meta.Pipeline.Add(pipeModel);
await pipe.Train(agent, data, pipeModel);
});
// save model meta data
var metaJson = JsonConvert.SerializeObject(meta, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
File.WriteAllText(Path.Combine(settings.ModelDir, "metadata.json"), metaJson);
Console.WriteLine(metaJson);
return(meta);
}