public async Task <TrainingCorpus> ExtractorCorpus(TAgent agent)
{
var corpus = new TrainingCorpus
{
Entities = new List <TrainingEntity>(),
UserSays = new List <TrainingIntentExpression <TrainingIntentExpressionPart> >()
};
agent.Entities.ForEach(entity =>
{
corpus.Entities.Add(new TrainingEntity
{
Entity = entity.Name,
Values = entity.Entries.Select(x => new TrainingEntitySynonym
{
Value = x.Value,
Synonyms = x.Synonyms.Select(y => y.Synonym).ToList()
}).ToList()
});
});
agent.Intents.ForEach(intent =>
{
// filter unexpected intents
if (intent.Name != "Default Fallback Intent")
{
// caculate contexts hash
intent.ContextHash = String.Join('_', intent.Contexts.OrderBy(x => x.Name).Select(x => x.Name)).GetMd5Hash();
intent.UserSays.ForEach(say => {
corpus.UserSays.Add(new TrainingIntentExpression <TrainingIntentExpressionPart>
{
Intent = intent.Name,
Text = String.Join("", say.Data.Select(x => x.Text)),
Entities = say.Data.Where(x => !String.IsNullOrEmpty(x.Meta))
.Select(x => new TrainingIntentExpressionPart
{
Value = x.Text,
Entity = x.Meta,
Start = x.Start
})
.ToList(),
ContextHash = intent.ContextHash
});
});
}
});
return(corpus);
}
public async Task <TrainingCorpus> ExtractorCorpus(TAgent agent)
{
var corpus = new TrainingCorpus()
{
Entities = new List <TrainingEntity>(),
UserSays = new List <TrainingIntentExpression <TrainingIntentExpressionPart> >()
};
List <string> entities = new List <string>();
// generate entity list
agent.Intents.ForEach(intent =>
{
intent.Entities.ForEach(entity =>
{
if (!entities.Contains(entity.Entity))
{
corpus.Entities.Add(new TrainingEntity
{
Entity = entity.Entity,
Values = new List <TrainingEntitySynonym>
{
new TrainingEntitySynonym
{
Value = entity.Value,
Synonyms = new List <string>
{
entity.Value
}
}
}
});
entities.Add(entity.Entity);
}
});
});
agent.Intents.ForEach(intent =>
{
var express = new TrainingIntentExpression <TrainingIntentExpressionPart>()
{
Text = intent.Text,
Intent = intent.Intent,
Entities = intent.Entities
};
corpus.UserSays.Add(express);
});
return(corpus);
}
public async Task <ModelMetaData> Train(TAgent agent, TrainingCorpus corpus)
{
string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id);
var model = "model_" + DateTime.UtcNow.ToString("yyyyMMdd");
var trainer = new BotTrainer();
agent.Corpus = corpus;
var trainOptions = new BotTrainOptions
{
AgentDir = agentDir,
Model = model
};
var info = await trainer.Train(agent, trainOptions);
return(info);
}
public virtual async Task <ModelMetaData> Train(TAgent agent, TrainingCorpus corpus, BotTrainOptions options)
{
if (String.IsNullOrEmpty(options.AgentDir))
{
options.AgentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id);
}
if (String.IsNullOrEmpty(options.Model))
{
options.Model = "model_" + DateTime.UtcNow.ToString("yyyyMMdd");
}
ModelMetaData meta = null;
// train by contexts
corpus.UserSays.GroupBy(x => x.ContextHash).Select(g => new
{
Context = g.Key,
Corpus = new TrainingCorpus
{
Entities = corpus.Entities,
UserSays = corpus.UserSays.Where(x => x.ContextHash == g.Key).ToList()
}
})
.ToList()
.ForEach(async c =>
{
var trainer = new BotTrainer(settings);
agent.Corpus = c.Corpus;
meta = await trainer.Train(agent, new BotTrainOptions
{
AgentDir = options.AgentDir,
Model = options.Model + $"{Path.DirectorySeparatorChar}{c.Context}"
});
});
meta.Pipeline.Clear();
meta.Model = options.Model;
return(meta);
}
public async Task <TrainingCorpus> ExtractorCorpus(TAgent agent)
{
var corpus = new TrainingCorpus
{
Entities = new List <TrainingEntity>(),
UserSays = new List <TrainingIntentExpression <TrainingIntentExpressionPart> >()
};
agent.Entities.ForEach(entity =>
{
corpus.Entities.Add(new TrainingEntity
{
Entity = entity.Name,
Values = entity.Entries.Select(x => new TrainingEntitySynonym
{
Value = x.Value,
Synonyms = x.Synonyms.Select(y => y.Synonym).ToList()
}).ToList()
});
});
agent.Intents.ForEach(intent =>
{
intent.UserSays.ForEach(say => {
corpus.UserSays.Add(new TrainingIntentExpression <TrainingIntentExpressionPart>
{
Intent = intent.Name,
Text = String.Join("", say.Data.Select(x => x.Text)),
Entities = say.Data.Where(x => !String.IsNullOrEmpty(x.Meta))
.Select(x => new TrainingIntentExpressionPart
{
Value = x.Text,
Entity = x.Meta,
Start = x.Start
})
.ToList()
});
});
});
return(corpus);
}
public virtual async Task <ModelMetaData> Train(TAgent agent, TrainingCorpus corpus, BotTrainOptions options)
{
if (String.IsNullOrEmpty(options.AgentDir))
{
options.AgentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id);
}
if (String.IsNullOrEmpty(options.Model))
{
options.Model = "model_" + DateTime.UtcNow.ToString("yyyyMMdd");
}
var trainer = new BotTrainer(settings);
agent.Corpus = corpus;
var info = await trainer.Train(agent, options);
return(info);
}
public async Task <TrainingCorpus> ExtractorCorpus(TAgent agent)
{
var corpus = new TrainingCorpus();
corpus.Entities = agent.Entities.Select(x => new TrainingEntity
{
Entity = x.EntityName,
Values = x.Examples.Select(y => new TrainingEntitySynonym
{
Value = y.Value,
Synonyms = y.Synonyms
}).ToList()
}).ToList();
corpus.UserSays = new List <TrainingIntentExpression <TrainingIntentExpressionPart> >();
foreach (DomainModel domain in agent.Domains)
{
foreach (IntentModel intent in domain.Intents)
{
foreach (IntentExampleModel example in intent.Examples)
{
var say = new TrainingIntentExpression <TrainingIntentExpressionPart>()
{
Intent = intent.IntentName,
Text = example.UserSays,
Entities = example.Entities.Select(x => new TrainingIntentExpressionPart
{
Entity = x.Entity,
Start = x.Start,
Value = x.Value
}).ToList()
};
corpus.UserSays.Add(say);
}
}
}
return(corpus);
}
