public void Test_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
{
// This benchmark is profiling bulk scoring speed and not training speed.
string cmd = @"Test data=" + _mslrWeb10k_Test + " in=" + _modelPath_MSLR;
var environment = EnvironmentFactory.CreateRankingEnvironment <RankerEvaluator, TextLoader, HashingTransformer, FastTreeRankingTrainer>();
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
}
public void Test_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
{
// This benchmark is profiling bulk scoring speed and not training speed.
string cmd = @"Test data=" + _mslrWeb10k_Test + " in=" + _modelPath_MSLR;
var environment = EnvironmentFactory.CreateRankingEnvironment <RankerEvaluator, TextLoader, HashingTransformer, FastTreeRankingTrainer, FastTreeRankingModelParameters>();
cmd.ExecuteMamlCommand(environment);
}
public void Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron()
{
// This benchmark is profiling bulk scoring speed and not training speed.
string modelpath = Path.Combine(Path.GetDirectoryName(typeof(MulticlassClassificationTest).Assembly.Location), @"WikiModel.fold000.zip");
string cmd = @"Test data=" + _dataPathWiki + " in=" + modelpath;
var environment = EnvironmentFactory.CreateClassificationEnvironment <TextLoader, OneHotEncodingTransformer, AveragedPerceptronTrainer, LinearBinaryModelParameters>();
cmd.ExecuteMamlCommand(environment);
}
public void Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron()
{
// This benchmark is profiling bulk scoring speed and not training speed.
string modelpath = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.fold000.zip");
string cmd = @"Test data=" + _dataPath_Wiki + " in=" + modelpath;
var environment = EnvironmentFactory.CreateClassificationEnvironment <TextLoader, OneHotEncodingTransformer, AveragedPerceptronTrainer>();
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
}
public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
{
string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
" eval=RankingEvaluator{t=10}" +
" data=" + _mslrWeb10k_Train +
" loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
" xf=HashTransform{col=GroupId} xf=NAHandleTransform{col=Features}" +
" tr=FastTreeRanking{}";
var environment = EnvironmentFactory.CreateRankingEnvironment <RankerEvaluator, TextLoader, HashingTransformer, FastTreeRankingTrainer, FastTreeRankingModelParameters>();
cmd.ExecuteMamlCommand(environment);
}
public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
{
string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
" eval=RankingEvaluator{t=10}" +
" data=" + _mslrWeb10k_Train +
" loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
" xf=HashTransform{col=GroupId} xf=NAHandleTransform{col=Features}" +
" tr=FastTreeRanking{}";
var environment = EnvironmentFactory.CreateRankingEnvironment <RankerEvaluator, TextLoader, HashingTransformer, FastTreeRankingTrainer>();
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
}
public void CV_Multiclass_WikiDetox_BigramsAndTrichar_LightGBMMulticlass()
{
string cmd = @"CV k=5 data=" + _dataPathWiki +
" loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+}" +
" xf=Convert{col=logged_in type=R4}" +
" xf=CategoricalTransform{col=ns}" +
" xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
" xf=Concat{col=Features:FeaturesText,logged_in,ns}" +
" tr=LightGBMMulticlass{iter=10}";
var environment = EnvironmentFactory.CreateClassificationEnvironment <TextLoader, OneHotEncodingTransformer, LightGbmMulticlassTrainer, OneVersusAllModelParameters>();
cmd.ExecuteMamlCommand(environment);
}
public void CV_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron()
{
string cmd = @"CV k=5 data=" + _dataPath_Wiki +
" loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+}" +
" xf=Convert{col=logged_in type=R4}" +
" xf=CategoricalTransform{col=ns}" +
" xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
" xf=Concat{col=Features:FeaturesText,logged_in,ns}" +
" tr=OVA{p=AveragedPerceptron{iter=10}}";
var environment = EnvironmentFactory.CreateClassificationEnvironment <TextLoader, OneHotEncodingTransformer, AveragedPerceptronTrainer>();
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
}
public void CV_Multiclass_WikiDetox_WordEmbeddings_SDCAMC()
{
string cmd = @"CV k=5 data=" + _dataPathWiki +
" tr=SDCAMC" +
" loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+}" +
" xf=Convert{col=logged_in type=R4}" +
" xf=CategoricalTransform{col=ns}" +
" xf=TextTransform{col=FeaturesText:comment tokens=+ wordExtractor={} charExtractor={}}" +
" xf=WordEmbeddingsTransform{col=FeaturesWordEmbedding:FeaturesText_TransformedText model=FastTextWikipedia300D}" +
" xf=Concat{col=Features:FeaturesWordEmbedding,logged_in,ns}";
var environment = EnvironmentFactory.CreateClassificationEnvironment <TextLoader, OneHotEncodingTransformer, SdcaMaximumEntropyMulticlassTrainer, MaximumEntropyModelParameters>();
cmd.ExecuteMamlCommand(environment);
}
public void CV_Multiclass_WikiDetox_WordEmbeddings_OVAAveragedPerceptron()
{
string cmd = @"CV k=5 data=" + _dataPath_Wiki +
" tr=OVA{p=AveragedPerceptron{iter=10}}" +
" loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+}" +
" xf=Convert{col=logged_in type=R4}" +
" xf=CategoricalTransform{col=ns}" +
" xf=TextTransform{col=FeaturesText:comment tokens=+ wordExtractor=NGramExtractorTransform{ngram=2}}" +
" xf=WordEmbeddingsTransform{col=FeaturesWordEmbedding:FeaturesText_TransformedText model=FastTextWikipedia300D}" +
" xf=Concat{col=Features:FeaturesText,FeaturesWordEmbedding,logged_in,ns}";
var environment = EnvironmentFactory.CreateClassificationEnvironment <TextLoader, OneHotEncodingTransformer, AveragedPerceptronTrainer, LinearBinaryModelParameters>();
cmd.ExecuteMamlCommand(environment);
}
public void CV_Multiclass_WikiDetox_WordEmbeddings_SDCAMC()
{
string cmd = @"CV k=5 data=" + _dataPath_Wiki +
" tr=SDCAMC" +
" loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+}" +
" xf=Convert{col=logged_in type=R4}" +
" xf=CategoricalTransform{col=ns}" +
" xf=TextTransform{col=FeaturesText:comment tokens=+ wordExtractor={} charExtractor={}}" +
" xf=WordEmbeddingsTransform{col=FeaturesWordEmbedding:FeaturesText_TransformedText model=FastTextWikipedia300D}" +
" xf=Concat{col=Features:FeaturesWordEmbedding,logged_in,ns}";
using (var environment = EnvironmentFactory.CreateClassificationEnvironment <TextLoader, CategoricalTransform, SdcaMultiClassTrainer>())
{
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
}
}
public void SetupScoringSpeedTests()
{
_dataPathWiki = GetBenchmarkDataPathAndEnsureData(TestDatasets.WikiDetox.trainFilename, TestDatasets.WikiDetox.path);
if (!File.Exists(_dataPathWiki))
{
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPathWiki));
}
_modelPathWiki = Path.Combine(Path.GetDirectoryName(typeof(MulticlassClassificationTest).Assembly.Location), @"WikiModel.zip");
string cmd = @"CV k=5 data=" + _dataPathWiki +
" loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4}" +
" xf=CategoricalTransform{col=ns}" +
" xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
" xf=Concat{col=Features:FeaturesText,logged_in,ns}" +
" tr=OVA{p=AveragedPerceptron{iter=10}}" +
" out={" + _modelPathWiki + "}";
var environment = EnvironmentFactory.CreateClassificationEnvironment <TextLoader, OneHotEncodingTransformer, AveragedPerceptronTrainer, LinearBinaryModelParameters>();
cmd.ExecuteMamlCommand(environment);
}
public void SetupScoringSpeedTests()
{
_dataPath_Wiki = Path.GetFullPath(TestDatasets.WikiDetox.trainFilename);
if (!File.Exists(_dataPath_Wiki))
{
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPath_Wiki));
}
_modelPath_Wiki = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.zip");
string cmd = @"CV k=5 data=" + _dataPath_Wiki +
" loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4}" +
" xf=CategoricalTransform{col=ns}" +
" xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
" xf=Concat{col=Features:FeaturesText,logged_in,ns}" +
" tr=OVA{p=AveragedPerceptron{iter=10}}" +
" out={" + _modelPath_Wiki + "}";
var environment = EnvironmentFactory.CreateClassificationEnvironment <TextLoader, OneHotEncodingTransformer, AveragedPerceptronTrainer>();
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
}
public void SetupScoringSpeedTests()
{
_mslrWeb10k_Test = Path.GetFullPath(TestDatasets.MSLRWeb.testFilename);
_mslrWeb10k_Validate = Path.GetFullPath(TestDatasets.MSLRWeb.validFilename);
_mslrWeb10k_Train = Path.GetFullPath(TestDatasets.MSLRWeb.trainFilename);
if (!File.Exists(_mslrWeb10k_Test))
{
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Test));
}
if (!File.Exists(_mslrWeb10k_Validate))
{
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Validate));
}
if (!File.Exists(_mslrWeb10k_Train))
{
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Train));
}
_modelPath_MSLR = Path.Combine(Directory.GetCurrentDirectory(), @"FastTreeRankingModel.zip");
string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
" eval=RankingEvaluator{t=10}" +
" data=" + _mslrWeb10k_Train +
" loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
" xf=HashTransform{col=GroupId}" +
" xf=NAHandleTransform{col=Features}" +
" tr=FastTreeRanking{}" +
" out={" + _modelPath_MSLR + "}";
using (var environment = EnvironmentFactory.CreateRankingEnvironment <RankerEvaluator, TextLoader, HashTransformer, FastTreeRankingTrainer>())
{
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
}
}
public void SetupScoringSpeedTests()
{
_mslrWeb10k_Test = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.testFilename);
_mslrWeb10k_Validate = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.validFilename);
_mslrWeb10k_Train = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.trainFilename);
if (!File.Exists(_mslrWeb10k_Test))
{
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Test));
}
if (!File.Exists(_mslrWeb10k_Validate))
{
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Validate));
}
if (!File.Exists(_mslrWeb10k_Train))
{
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Train));
}
_modelPath_MSLR = Path.Combine(Path.GetDirectoryName(typeof(RankingTest).Assembly.Location), "FastTreeRankingModel.zip");
string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
" eval=RankingEvaluator{t=10}" +
" data=" + _mslrWeb10k_Train +
" loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
" xf=HashTransform{col=GroupId}" +
" xf=NAHandleTransform{col=Features}" +
" tr=FastTreeRanking{}" +
" out={" + _modelPath_MSLR + "}";
var environment = EnvironmentFactory.CreateRankingEnvironment <RankerEvaluator, TextLoader, HashingTransformer, FastTreeRankingTrainer, FastTreeRankingModelParameters>();
cmd.ExecuteMamlCommand(environment);
}
