public void RunBostonTest()
{
var workdir = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "testbed", "boston");
var dsPath = Path.Combine(workdir, "housing.data");
try
{
DownloadHelpers.DownloadDataset(
"https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data",
dsPath);
}
catch (WebException)
{
Assert.Fail("Failed to download Boston dataset");
}
var newDsPath = Path.Combine(workdir, "housing.csv");
File.WriteAllText(newDsPath, "");
File.AppendAllLines(
newDsPath,
File.ReadLines(dsPath).Select(x => Regex.Replace(x.Trim(), @"\s+", ","))
);
dsPath = newDsPath;
MLContext mlContext = new MLContext();
var model = new CatBoostModel(
Path.Combine(workdir, "boston_housing_model.cbm"),
"MedV"
);
IDataView dataView = mlContext.Data.LoadFromTextFile <BostonDataPoint>(dsPath, hasHeader: false, separatorChar: ',');
IEnumerable <BostonDataPoint> dataPoints = mlContext.Data.CreateEnumerable <BostonDataPoint>(dataView, reuseRowObject: false);
var predictions = model.Transform(dataView);
IEnumerable <CatBoostValuePrediction> predictionsBatch = mlContext.Data.CreateEnumerable <CatBoostValuePrediction>(predictions, reuseRowObject: false);
var deltas = dataPoints
.Zip(predictionsBatch, Tuple.Create)
.Zip(Enumerable.Range(1, predictionsBatch.Count()), Tuple.Create)
.Select(rec => new
{
Index = rec.Item2,
LogDelta = Math.Abs(rec.Item1.Item2.OutputValues[0] - Math.Log(rec.Item1.Item1.MedV)),
Pred = Math.Exp(rec.Item1.Item2.OutputValues[0]),
Target = rec.Item1.Item1.MedV
});
int totalErrors = deltas.Where(x => x.LogDelta >= .4).Count();
Assert.IsTrue(
totalErrors <= 7,
$"Boston test crashed: expected <= 7 errors, got {totalErrors} error(s) on samples {{" +
string.Join(
", ",
deltas.Where(x => x.LogDelta >= .4).Take(8).Select(x => x.Index + 1)
) + ", ...}"
);
}
public void RunIrisTest()
{
var workdir = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "testbed", "iris");
var dsPath = Path.Combine(workdir, "iris.data");
try
{
DownloadHelpers.DownloadDataset(
"https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data",
dsPath);
}
catch (WebException)
{
Assert.Fail("Failed to download Iris dataset");
}
MLContext mlContext = new MLContext();
var model = new CatBoostModel(
Path.Combine(workdir, "iris_model.cbm"),
"IrisType"
);
IDataView dataView = mlContext.Data.LoadFromTextFile <IrisDataPoint>(dsPath, hasHeader: false, separatorChar: ',');
IEnumerable <IrisDataPoint> dataPoints = mlContext.Data.CreateEnumerable <IrisDataPoint>(dataView, reuseRowObject: false);
var predictions = model.Transform(dataView);
IEnumerable <CatBoostValuePrediction> predictionsBatch = mlContext.Data.CreateEnumerable <CatBoostValuePrediction>(predictions, reuseRowObject: false);
string[] targetLabelList = new string[] { "Iris-setosa", "Iris-versicolor", "Iris-virginica" };
int i = 0;
List <int> failedIds = new List <int>();
foreach (var xy in dataPoints.Zip(predictionsBatch, Tuple.Create))
{
++i;
(var x, var y) = xy;
int argmax = Enumerable.Range(0, y.OutputValues.Length).Select(j => Tuple.Create(y.OutputValues[j], j)).Max().Item2;
string predLabel = targetLabelList[argmax];
if (predLabel != x.IrisType)
{
failedIds.Add(i);
}
}
Assert.IsTrue(failedIds.Count == 0, $"Iris test crashed on sample #{string.Join(",", failedIds.Select(x => x.ToString()).ToArray())}");
}
public void RunMushroom()
{
var workdir = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "testbed", "mushrooms");
var dsPath = Path.Combine(workdir, "mushrooms.csv");
try
{
DownloadHelpers.DownloadDataset(
"https://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.data",
dsPath);
}
catch (WebException)
{
Assert.Fail("Failed to download Mushroom dataset");
}
MLContext mlContext = new MLContext();
var model = new CatBoostModel(
Path.Combine(workdir, "mushroom_model.cbm"),
"Class"
);
IDataView dataView = mlContext.Data.LoadFromTextFile <MushroomDataPoint>(dsPath, hasHeader: false, separatorChar: ',');
IEnumerable <MushroomDataPoint> dataPoints = mlContext.Data.CreateEnumerable <MushroomDataPoint>(dataView, reuseRowObject: false);
var predictions = model.Transform(dataView);
IEnumerable <CatBoostValuePrediction> predictionsBatch = mlContext.Data.CreateEnumerable <CatBoostValuePrediction>(predictions, reuseRowObject: false);
string[] targetLabelList = new string[] { "e", "p" };
int i = 0;
foreach (var xy in dataPoints.Zip(predictionsBatch, Tuple.Create))
{
(var x, var y) = xy;
int argmax = (y.OutputValues[0] > 0) ? 1 : 0;
string predLabel = targetLabelList[argmax];
Assert.IsTrue(
predLabel == x.Class,
$"Mushroom test crashed on sample {i + 1}"
);
}
}
public void RunIrisTest()
{
var workdir = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "testbed", "iris");
var dsPath = Path.Combine(workdir, "iris.data");
try {
DownloadHelpers.DownloadDataset(
"https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data",
dsPath);
} catch (WebException) {
Assert.Fail("Failed to download Iris dataset");
}
var df = Frame.ReadCsv(dsPath, hasHeaders: false);
df.RenameColumns(new Collection <string>
{
"sepal length", "sepal width", "petal length", "petal width", "target"
});
var target = df.Rows.Select(obj => obj.Value["target"]).Values.Select(x => (string)x).ToArray();
df.DropColumn("target");
var data = df.ToArray2D <float>();
var model = new CatBoostModelEvaluator(Path.Combine(workdir, "iris_model.cbm"));
model.CatFeaturesIndices = new Collection <int> {
};
double[,] res = model.EvaluateBatch(data, new string[df.RowCount, 0]);
string[] targetLabelList = new string[] { "Iris-setosa", "Iris-versicolor", "Iris-virginica" };
string errors = "";
for (int i = 0; i < res.GetLength(0); ++i)
{
int argmax = Enumerable.Range(0, res.GetLength(1)).Select(j => Tuple.Create(res[i, j], j)).Max().Item2;
string predLabel = targetLabelList[argmax];
if (predLabel != target[i])
{
errors += $"#{i + 1} ";
}
}
Assert.AreEqual(errors.Length, 1, $"Iris test failed on samples: #{errors}");
}
public void RunMushroomTest()
{
var workdir = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "testbed", "mushrooms");
var dsPath = Path.Combine(workdir, "mushrooms.csv");
try
{
DownloadHelpers.DownloadDataset(
"https://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.data",
dsPath);
}
catch (WebException)
{
Assert.Fail("Failed to download Mushroom dataset");
}
var df = Frame.ReadCsv(dsPath, hasHeaders: false);
var target = df.Rows.Select(obj => obj.Value["Column1"]).Values.Select(x => (string)x).ToArray();
df.DropColumn("Column1");
var data = df.ToArray2D <string>();
var model = new CatBoostModelEvaluator(Path.Combine(workdir, "mushroom_model.cbm"));
model.CatFeaturesIndices = Enumerable.Range(0, df.ColumnCount).ToList();
double[,] res = model.EvaluateBatch(new float[df.RowCount, 0], data);
string[] targetLabelList = new string[] { "e", "p" };
for (int i = 0; i < res.GetLength(0); ++i)
{
int argmax = res[i, 0] > 0 ? 1 : 0;
string predLabel = targetLabelList[argmax];
Assert.IsTrue(
predLabel == target[i],
$"Mushroom test crashed on sample {i + 1}"
);
}
}
public void RunBostonTest()
{
var workdir = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "testbed", "boston");
var dsPath = Path.Combine(workdir, "housing.data");
try
{
DownloadHelpers.DownloadDataset(
"https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data",
dsPath);
}
catch (WebException)
{
Assert.Fail("Failed to download Boston dataset");
}
List <float[]> featureList = new List <float[]>();
List <double> targetList = new List <double>();
using (TextReader textReader = new StreamReader(dsPath))
{
while (textReader.Peek() != -1)
{
var tokens = textReader.ReadLine().Split(' ').ToList().Where(x => x != "");
targetList.Add(double.Parse(tokens.Last()));
featureList.Add(tokens.SkipLast(1).Select(x => float.Parse(x)).ToArray());
}
}
if (featureList.Where(x => x.Length != featureList.First().Length).Any())
{
throw new InvalidDataException("Inconsistent column count in housing.data");
}
double[] target = targetList.ToArray();
float[,] features = new float[featureList.Count, featureList.First().Length];
for (int i = 0; i < featureList.Count; ++i)
{
for (int j = 0; j < featureList.First().Length; ++j)
{
features[i, j] = featureList[i][j];
}
}
var model = new CatBoostModelEvaluator(Path.Combine(workdir, "boston_housing_model.cbm"));
model.CatFeaturesIndices = new Collection <int> {
};
double[,] res = model.EvaluateBatch(features, new string[featureList.Count, 0]);
var deltas = Enumerable.Range(0, featureList.Count).Select(i => new
{
Index = i + 1,
LogDelta = Math.Abs(res[i, 0] - Math.Log(target[i])),
Pred = Math.Exp(res[i, 0]),
Target = target[i]
});
int totalErrors = deltas.Where(x => x.LogDelta >= .4).Count();
Assert.IsTrue(
totalErrors <= 7,
$"Boston test crashed: expected <= 7 errors, got {totalErrors} error(s) on samples {{" +
string.Join(
", ",
deltas.Where(x => x.LogDelta >= .4).Take(8).Select(x => x.Index + 1)
) + ", ...}"
);
}