public static RasaTrainingData GrabCorpus(this Agent agent, Database dc)
{
var trainingData = new RasaTrainingData
{
Entities = new List <RasaTraningEntity>(),
UserSays = new List <RasaIntentExpression>()
};
var expressParts = new List <IntentExpressionPart>();
var intents = dc.Table <Intent>()
.Include(x => x.Contexts)
.Include(x => x.UserSays).ThenInclude(say => say.Data)
.Where(x => x.UserSays.Count > 0)
.ToList();
intents.ForEach(intent =>
{
intent.UserSays.ForEach(exp =>
{
var say = new RasaIntentExpression
{
Intent = intent.Name,
Text = String.Join("", exp.Data.OrderBy(x => x.UpdatedTime).Select(x => x.Text)),
};
// convert entity format
exp.Data.Where(x => !String.IsNullOrEmpty(x.Meta))
.ToList()
.ForEach(x =>
{
int start = say.Text.IndexOf(x.Text);
var part = new RasaIntentExpressionPart
{
Value = x.Text,
Entity = x.Alias,
Start = start,
End = start + x.Text.Length
};
if (say.Entities == null)
{
say.Entities = new List <RasaIntentExpressionPart>();
}
say.Entities.Add(part);
// assemble entity synonmus
if (!trainingData.Entities.Any(y => y.EntityType == x.Alias && y.EntityValue == x.Text))
{
var allSynonyms = (from e in dc.Table <Entity>()
join ee in dc.Table <EntityEntry>() on e.Id equals ee.EntityId
join ees in dc.Table <EntityEntrySynonym>() on ee.Id equals ees.EntityEntryId
where e.Name == x.Alias && ee.Value == x.Text & ees.Synonym != x.Text
select ees.Synonym).ToList();
var te = new RasaTraningEntity
{
EntityType = x.Alias,
EntityValue = x.Text,
Synonyms = allSynonyms
};
trainingData.Entities.Add(te);
}
});
trainingData.UserSays.Add(say);
});
});
return(trainingData);
}
private void ImportIntentUserSays(RasaIntentExpression intent, List <RasaIntentExpression> sentences)
{
var intents = new List <RasaIntentExpression>();
var userSays = sentences.Where(x => x.Intent == intent.Intent).ToList();
userSays.ForEach(say =>
{
var expression = new IntentExpression();
say.Entities = say.Entities.OrderBy(x => x.Start).ToList();
expression.Data = new List <IntentExpressionPart>();
int pos = 0;
for (int entityIdx = 0; entityIdx < say.Entities.Count; entityIdx++)
{
var entity = say.Entities[entityIdx];
// previous
if (entity.Start > 0)
{
expression.Data.Add(new IntentExpressionPart
{
Text = say.Text.Substring(pos, entity.Start - pos),
Start = pos
});
}
// self
expression.Data.Add(new IntentExpressionPart
{
Alias = entity.Entity,
Meta = entity.Entity,
Text = say.Text.Substring(entity.Start, entity.Value.Length),
Start = entity.Start
});
pos = entity.End + 1;
if (pos < say.Text.Length && entityIdx == say.Entities.Count - 1)
{
// end
expression.Data.Add(new IntentExpressionPart
{
Text = say.Text.Substring(pos),
Start = pos
});
}
}
if (say.Entities.Count == 0)
{
expression.Data.Add(new IntentExpressionPart
{
Text = say.Text.Substring(pos)
});
}
int second = 0;
expression.Data.ForEach(x => x.UpdatedTime = DateTime.UtcNow.AddSeconds(second++));
intents.Add(say);
});
}
