public TaskDisplay(int taskId, string charName, DateTime matureTime, TaskKind kind)
{
this.TaskId = taskId;
this.CharName = charName;
this.MatureTime = matureTime;
this.Kind = kind;
}
public override IEnumerable <SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
{
var newColumns = new List <string>();
foreach (var column in columns)
{
if (!column.Type.GetItemType().IsBool() || column.Purpose != ColumnPurpose.NumericFeature)
{
continue;
}
newColumns.Add(column.ColumnName);
}
if (newColumns.Count() > 0)
{
var newColumnsArr = newColumns.ToArray();
yield return(TypeConvertingExtension.CreateSuggestedTransform(Context, newColumnsArr, newColumnsArr));
}
}
public override IEnumerable <SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
{
bool foundCat = false;
bool foundCatHash = false;
var catColumnsNew = new List <string>();
var catHashColumnsNew = new List <string>();
foreach (var column in columns)
{
if (column.Purpose != ColumnPurpose.CategoricalFeature)
{
continue;
}
if (column.Dimensions.Cardinality != null && column.Dimensions.Cardinality < 100)
{
foundCat = true;
catColumnsNew.Add(column.ColumnName);
}
else
{
foundCatHash = true;
catHashColumnsNew.Add(column.ColumnName);
}
}
if (foundCat)
{
var catColumnsArr = catColumnsNew.ToArray();
yield return(OneHotEncodingExtension.CreateSuggestedTransform(Context, catColumnsArr, catColumnsArr));
}
if (foundCatHash)
{
var catHashColumnsNewArr = catHashColumnsNew.ToArray();
yield return(OneHotHashEncodingExtension.CreateSuggestedTransform(Context, catHashColumnsNewArr, catHashColumnsNewArr));
}
var transformedColumns = new List <string>();
transformedColumns.AddRange(catColumnsNew);
transformedColumns.AddRange(catHashColumnsNew);
}
public List<long> GetTaskKeys(uint charIdx, TaskKind taskKind)
{
List<long> taskKeys = new List<long>(10);
if (taskKind == TaskKind.Invocation) {
foreach (var kvp in this.Queue) {
if (kvp.Value.CharIdx == charIdx && kvp.Value.Kind == taskKind) {
taskKeys.Add(kvp.Key);
}
}
} else if (taskKind == TaskKind.Profession) {
foreach (var kvp in this.Queue) {
if (kvp.Value.CharIdx == charIdx && kvp.Value.Kind == taskKind) {
taskKeys.Add(kvp.Key);
}
}
}
return taskKeys;
}
/// <summary>
/// Given a predictor type, return a set of all permissible trainers (with their sweeper params, if defined).
/// </summary>
/// <returns>Array of viable learners.</returns>
public static IEnumerable <SuggestedTrainer> AllowedTrainers(MLContext mlContext, TaskKind task,
ColumnInformation columnInfo, IEnumerable <TrainerName> trainerAllowList)
{
var trainerExtensions = TrainerExtensionCatalog.GetTrainers(task, trainerAllowList, columnInfo);
var trainers = new List <SuggestedTrainer>();
foreach (var trainerExtension in trainerExtensions)
{
var learner = new SuggestedTrainer(mlContext, trainerExtension, columnInfo);
trainers.Add(learner);
}
return(trainers.ToArray());
}
private static IEnumerable <SuggestedTransform> InferLabelTransforms(MLContext context, TaskKind task,
DatasetColumnInfo[] columns)
{
var inferredTransforms = new List <SuggestedTransform>();
if (task != TaskKind.MulticlassClassification)
{
return(inferredTransforms);
}
// If label column type wasn't originally key type,
// convert predicted label column back from key to value.
// (Non-key label column was converted to key, b/c multiclass trainers only
// accept label columns that are key type)
var labelColumn = columns.First(c => c.Purpose == ColumnPurpose.Label);
if (!labelColumn.Type.IsKey())
{
inferredTransforms.Add(KeyToValueMappingExtension.CreateSuggestedTransform(context, DefaultColumnNames.PredictedLabel, DefaultColumnNames.PredictedLabel));
}
return(inferredTransforms);
}
public static (PipelineRunResult[] allPipelines, PipelineRunResult bestPipeline) Fit(IDataView trainData,
IDataView validationData, string label, AutoFitSettings settings, TaskKind task, OptimizingMetric metric,
IEnumerable <(string, ColumnPurpose)> purposeOverrides, IDebugLogger debugLogger)
public override IEnumerable <SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
{
foreach (var column in columns)
{
if (!column.Type.GetItemType().IsText() || column.Purpose != ColumnPurpose.ImagePath)
{
continue;
}
var columnDestSuffix = "_featurized";
string columnDestRenamed = $"{column.ColumnName}{columnDestSuffix}";
yield return(RawByteImageLoading.CreateSuggestedTransform(Context, column.ColumnName, columnDestRenamed));
}
}
private static void ValidateColumnInformation(IDataView trainData, ColumnInformation columnInformation, TaskKind task)
{
ValidateColumnInformation(columnInformation);
ValidateTrainDataColumn(trainData, columnInformation.LabelColumnName, LabelColumnPurposeName, GetAllowedLabelTypes(task));
ValidateTrainDataColumn(trainData, columnInformation.ExampleWeightColumnName, WeightColumnPurposeName);
ValidateTrainDataColumn(trainData, columnInformation.SamplingKeyColumnName, SamplingKeyColumnPurposeName);
ValidateTrainDataColumn(trainData, columnInformation.UserIdColumnName, UserIdColumnPurposeName);
ValidateTrainDataColumn(trainData, columnInformation.ItemIdColumnName, ItemIdColumnPurposeName);
ValidateTrainDataColumn(trainData, columnInformation.GroupIdColumnName, GroupIdColumnPurposeName);
ValidateTrainDataColumns(trainData, columnInformation.CategoricalColumnNames, CategoricalColumnPurposeName,
new DataViewType[] { NumberDataViewType.Single, TextDataViewType.Instance });
ValidateTrainDataColumns(trainData, columnInformation.NumericColumnNames, NumericColumnPurposeName,
new DataViewType[] { NumberDataViewType.Single, BooleanDataViewType.Instance });
ValidateTrainDataColumns(trainData, columnInformation.TextColumnNames, TextColumnPurposeName,
new DataViewType[] { TextDataViewType.Instance });
ValidateTrainDataColumns(trainData, columnInformation.IgnoredColumnNames, IgnoredColumnPurposeName);
}
public static (PipelineRunResult[] allPipelines, PipelineRunResult bestPipeline) Fit(IDataView trainData,
IDataView validationData, string label, InferredColumn[] inferredColumns, AutoFitSettings settings,
TaskKind task, OptimizingMetric metric, IDebugLogger debugLogger)
{
// hack: init new MLContext
var mlContext = new MLContext();
// infer pipelines
var optimizingMetricfInfo = new OptimizingMetricInfo(metric);
var autoFitter = new AutoFitter(mlContext, optimizingMetricfInfo, settings, task,
label, ToInternalColumnPurposes(inferredColumns),
trainData, validationData, debugLogger);
var allPipelines = autoFitter.Fit(1);
var bestScore = allPipelines.Max(p => p.Score);
var bestPipeline = allPipelines.First(p => p.Score == bestScore);
return(allPipelines, bestPipeline);
}
/// <summary>
/// Given a predictor type & target max num of iterations, return a set of all permissible trainers (with their sweeper params, if defined).
/// </summary>
/// <returns>Array of viable learners.</returns>
public static IEnumerable <SuggestedTrainer> AllowedTrainers(MLContext mlContext, TaskKind task,
int maxIterations)
{
var trainerExtensions = TrainerExtensionCatalog.GetTrainers(task, maxIterations);
var trainers = new List <SuggestedTrainer>();
foreach (var trainerExtension in trainerExtensions)
{
var learner = new SuggestedTrainer(mlContext, trainerExtension);
trainers.Add(learner);
}
return(trainers.ToArray());
}
/// <summary>
/// Removes a task from the queue and adds a new task with the same charIdx, kind, id, and bonus for a later time.
/// </summary>
/// <param name="intr"></param>
/// <param name="charIdx"></param>
/// <param name="taskKind"></param>
/// <param name="taskId"></param>
/// <param name="bonusFactor"></param>
/// <param name="incrementTaskId"></param>
/// <param name="force">If set to true, advances a task even if it has not matured.</param>
private void AdvanceTask(Interactor intr, uint charIdx, TaskKind taskKind, int taskId,
float bonusFactor, bool incrementTaskId, bool force)
{
// CHECK TO SEE IF THAT TASK IS ALREADY IN QUEUE,
// IF NOT ADD
// IF SO, CHECK TO SEE IF THAT TASK HAS MATURED
// IF IT HAS MATURED, ADVANCE
intr.Log(LogEntryType.Info, "Advancing task: "
+ "[charIdx: " + charIdx
+ ", taskKind: " + taskKind.ToString()
+ ", taskId: " + taskId + "].");
var nowTicks = DateTime.Now.AddSeconds(1).Ticks;
long taskKey = 0;
bool keyExists = this.TryGetTaskKey(charIdx, taskKind, taskId, out taskKey);
if (keyExists) {
intr.Log(LogEntryType.Debug, "Queue entry found for task: " +
"taskId: " + taskId.ToString() + ", " +
"taskKind: " + taskKind.ToString() + ", " +
"charIdx: " + charIdx.ToString() + ". ");
if (taskKey < nowTicks || force) { // MATURE or FORCED
if (taskKey < nowTicks) {
intr.Log(LogEntryType.Debug, "Task is mature. Removing and adding anew ...");
} else {
intr.Log(LogEntryType.Debug, "Forcing task advancement. Removing and adding anew ...");
}
//intr.Log(LogEntryType.Debug, "Removing old task.");
Queue.Remove(taskKey);
if (taskKind == TaskKind.Invocation) {
intr.Log(LogEntryType.Debug, "Queuing subsequent invocation task.");
var invokesToday = (incrementTaskId) ? taskId + 1 : taskId;
this.QueueSubsequentInvocationTask(intr, charIdx, invokesToday);
} else if (taskKind == TaskKind.Profession) {
intr.Log(LogEntryType.Debug, "Queuing subsequent professions task.");
this.QueueSubsequentProfessionTask(intr, charIdx, taskId, bonusFactor);
}
} else { // NOT MATURE
intr.Log(LogEntryType.Debug, "Task is not mature: taskKey: " + taskKey +
", nowTicks: " + nowTicks + ".");
}
} else { // DOESN'T EXIST YET
intr.Log(LogEntryType.Info, "Key not found for task.");
if (taskKind == TaskKind.Invocation) {
this.QueueSubsequentInvocationTask(intr, charIdx, 1);
} else if (taskKind == TaskKind.Profession) {
this.QueueSubsequentProfessionTask(intr, charIdx, taskId, bonusFactor);
}
}
}
public bool TryGetTaskKey(uint charIdx, TaskKind taskKind, int taskId, out long taskKey)
{
List<long> taskKeys = new List<long>(10);
if (taskKind == TaskKind.Invocation) {
foreach (var kvp in this.Queue) {
if (kvp.Value.CharIdx == charIdx && kvp.Value.Kind == taskKind) {
taskKeys.Add(kvp.Key);
}
}
} else if (taskKind == TaskKind.Profession) {
foreach (var kvp in this.Queue) {
if (kvp.Value.CharIdx == charIdx && kvp.Value.Kind == taskKind && kvp.Value.TaskId == taskId) {
taskKeys.Add(kvp.Key);
}
}
}
if (taskKeys.Count > 1) {
throw new Exception("TaskQueue::GetTaskKey(): Queue consistency error. Found " + taskKeys.Count
+ " keys for charIdx: " + charIdx
+ ", taskKind: " + taskKind.ToString()
+ ", taskId: " + taskId + ".");
} else if (taskKeys.Count == 1) {
taskKey = taskKeys[0];
return true;
} else {
taskKey = 0;
return false;
}
}
public bool TryGetTaskKey(uint charIdx, TaskKind taskKind, out long taskKey)
{
if (taskKind == TaskKind.Profession) {
throw new Exception("TaskQueue::GetTaskKey(): Profession tasks must specify a taskId as the third parameter");
} else {
return this.TryGetTaskKey(charIdx, taskKind, 999, out taskKey);
}
}
public override IEnumerable <SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
{
var featureCols = new List <string>();
foreach (var column in columns)
{
if (!column.Type.GetItemType().IsText() || column.Purpose != ColumnPurpose.TextFeature)
{
continue;
}
var columnDestSuffix = "_tf";
var columnNameSafe = column.ColumnName;
string columnDestRenamed = $"{columnNameSafe}{columnDestSuffix}";
featureCols.Add(columnDestRenamed);
yield return(TextFeaturizingExtension.CreateSuggestedTransform(Context, columnNameSafe, columnDestRenamed));
}
}
public static void ValidateSamplingKey(string samplingKeyColumnName, string groupIdColumnName, TaskKind task)
{
if (task == TaskKind.Ranking && samplingKeyColumnName != null && samplingKeyColumnName != groupIdColumnName)
{
throw new ArgumentException($"If provided, {nameof(samplingKeyColumnName)} must be the same as {nameof(groupIdColumnName)} for Ranking Experiments", samplingKeyColumnName);
}
}
public override IEnumerable <SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
{
var columnsWithMissing = new List <string>();
foreach (var column in columns)
{
if (column.Type.GetItemType() == NumberDataViewType.Single &&
column.Purpose == ColumnPurpose.NumericFeature &&
column.Dimensions.HasMissing == true)
{
columnsWithMissing.Add(column.ColumnName);
}
}
if (columnsWithMissing.Any())
{
var columnsArr = columnsWithMissing.ToArray();
var indicatorColNames = GetNewColumnNames(columnsArr.Select(c => $"{c}_MissingIndicator"), columns).ToArray();
yield return(MissingValueIndicatingExtension.CreateSuggestedTransform(Context, columnsArr, indicatorColNames));
yield return(TypeConvertingExtension.CreateSuggestedTransform(Context, indicatorColNames, indicatorColNames));
yield return(MissingValueReplacingExtension.CreateSuggestedTransform(Context, columnsArr, columnsArr));
}
}
public abstract IEnumerable <SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task);
public MenuItem(int num, TaskKind kind)
{
this.kind = kind;
this.num = num;
}
public override IEnumerable <SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
{
if (task != TaskKind.MulticlassClassification)
{
yield break;
}
var lastLabelColId = Array.FindLastIndex(columns, x => x.Purpose == ColumnPurpose.Label);
if (lastLabelColId < 0)
{
yield break;
}
var col = columns[lastLabelColId];
if (!col.Type.IsKey())
{
yield return(ValueToKeyMappingExtension.CreateSuggestedTransform(Context, col.ColumnName,
col.ColumnName));
}
}
public static IEnumerable <SuggestedTransform> InferTransforms(MLContext context, TaskKind task, DatasetColumnInfo[] columns)
{
var suggestedTransforms = new List <SuggestedTransform>();
suggestedTransforms.AddRange(InferLabelTransforms(context, task, columns));
return(suggestedTransforms);
}
public override IEnumerable <SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
{
if (task != TaskKind.Recommendation)
{
yield break;
}
foreach (var column in columns)
{
if (column.Purpose == ColumnPurpose.UserId ||
column.Purpose == ColumnPurpose.ItemId)
{
yield return(ValueToKeyMappingExtension.CreateSuggestedTransform(Context, column.ColumnName, column.ColumnName));
}
}
}
public bool TryGetCharTask(uint charIdx, TaskKind kind, int taskId, out GameTask task)
{
//var charTasksNode = (XmlElement)CharNode(charIdx).SelectSingleNode("Tasks");
var charTasksNode = CharTasksNode(charIdx);
var taskNodeName = GameTask.GenXmlNodeName(kind, taskId);
var taskNode = (XmlElement)charTasksNode.SelectSingleNode(taskNodeName);
if (taskNode != null) {
try {
task = GameTask.FromXmlElement(taskNode);
return true;
} catch (Exception) { }
}
task = new GameTask();
return false;
}
public override IEnumerable <SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
{
if (task != TaskKind.Ranking)
{
yield break;
}
foreach (var column in columns)
{
if (column.Purpose == ColumnPurpose.GroupId && !column.Type.IsKey())
{
yield return(HashingExtension.CreateSuggestedTransform(Context, column.ColumnName, column.ColumnName));
}
}
}
public static IEnumerable <SuggestedTransform> InferTransformsPostTrainer(MLContext context, TaskKind task, DatasetColumnInfo[] columns)
{
return(TransformPostTrainerInference.InferTransforms(context, task, columns));
}
public DateTime CalculateTaskMatureTime(DateTime startTime, uint charIdx, TaskKind kind, int taskId, float bonusFactor)
{
int durationMins = 0;
switch (kind) {
case TaskKind.Invocation:
durationMins = InvokeDelayMinutes[taskId];
break;
case TaskKind.Profession:
float timeFactor = 1.0f / (1.0f + bonusFactor);
durationMins = (int)(ProfessionTasksRef.ProfessionTaskDurationMinutes[taskId] * timeFactor);
break;
default:
return DateTime.Now;
}
DateTime taskMatureTime = startTime
.AddMinutes(durationMins)
.AddTicks(10000 * charIdx)
.AddTicks(10 * taskId);
Interactor.Logger.Debug("Task duration (charIdx: {0}, startTime: {1}, kind: {2}, taskId: {3}, bonusFactor {4}): {5} min.",
charIdx, startTime, kind, taskId, bonusFactor, durationMins);
return taskMatureTime;
}
