public void CanHandleUtf8() { var fastText = new FastTextWrapper(loggerFactory: _loggerFactory); string outPath = Path.Combine(_tempDir, "rus"); fastText.Supervised("data.rus.txt", outPath, FastTextArgs.SupervisedDefaults()); var labels = fastText.GetLabels(); labels.Length.Should().Be(2); labels.Should().Contain(new[] { "__label__оператор", "__label__выход" }); var pred = fastText.PredictSingle("Позови оператора"); pred.Probability.Should().BeGreaterThan(0); pred.Label.Should().Be("__label__оператор"); var sourceWords = File.ReadAllText("data.rus.txt") .Split(new[] { " ", "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries) .Where(x => !x.StartsWith("__label__")) .Distinct().ToArray(); var nn = fastText.GetNN("оператор", 2); nn.Length.Should().Be(2); sourceWords.Should().Contain(nn.Select(x => x.Label)); foreach (var prediction in nn) { prediction.Probability.Should().BeGreaterThan(0); } }
static void Main(string[] args) { Log.Logger = new LoggerConfiguration() .MinimumLevel.Debug() .WriteTo.Console(theme: ConsoleTheme.None) .CreateLogger(); var log = Log.ForContext <Program>(); var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); Directory.CreateDirectory(tempDir); log.Information($"Temp dir: {tempDir}"); string outPath = Path.Combine(tempDir, "cooking.bin"); var fastText = new FastTextWrapper(loggerFactory: new LoggerFactory(new[] { new SerilogLoggerProvider() })); var ftArgs = FastTextArgs.SupervisedDefaults(); ftArgs.epoch = 15; ftArgs.lr = 1; ftArgs.dim = 300; ftArgs.wordNgrams = 2; ftArgs.minn = 3; ftArgs.maxn = 6; fastText.Supervised("cooking.train.txt", outPath, ftArgs); }
static void Main(string[] args) { Log.Logger = new LoggerConfiguration() .MinimumLevel.Debug() .WriteTo.Console(theme: ConsoleTheme.None) .CreateLogger(); var log = Log.ForContext <Program>(); var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); Directory.CreateDirectory(tempDir); log.Information($"Temp dir: {tempDir}"); string outPath = Path.Combine(tempDir, "cooking.bin"); var fastText = new FastTextWrapper(loggerFactory: new LoggerFactory(new[] { new SerilogLoggerProvider() })); var ftArgs = FastTextArgs.SupervisedDefaults(); ftArgs.epoch = 15; ftArgs.lr = 1; ftArgs.dim = 300; ftArgs.wordNgrams = 2; ftArgs.minn = 3; ftArgs.maxn = 6; fastText.Supervised("cooking.train.txt", outPath, ftArgs); try { File.Delete("_debug.txt"); } catch { } var result = fastText.TestInternal("cooking.valid.txt", 1, 0.0f, true); log.Information($"Results:\n\tPrecision: {result.GlobalMetrics.GetPrecision()}" + $"\n\tRecall: {result.GlobalMetrics.GetRecall()}" + $"\n\tF1: {result.GlobalMetrics.GetF1()}"); var curve = result.GetPrecisionRecallCurve(); var(_, debugCurve) = TestResult.LoadDebugResult("_debug.txt", fastText.GetLabels()); string plotPath = PlotCurves(tempDir, new [] { curve, debugCurve }); log.Information($"Precision-Recall plot: {plotPath}"); Console.WriteLine("\nPress any key to exit."); Console.ReadKey(); Directory.Delete(tempDir, true); }
public void CanTrainModelWithOldApi() { var fastText = new FastTextWrapper(loggerFactory: _loggerFactory); string outPath = Path.Combine(_tempDir, "cooking"); fastText.Train("cooking.train.txt", outPath, FastTextArgs.SupervisedDefaults()); CheckLabels(fastText.GetLabels()); File.Exists(outPath + ".bin").Should().BeTrue(); File.Exists(outPath + ".vec").Should().BeTrue(); }
public void CanGetDefaultSupervisedArgs() { var args = FastTextArgs.SupervisedDefaults(); args.bucket.Should().Be(2000000); args.dim.Should().Be(100); args.loss.Should().Be(LossName.Softmax); args.model.Should().Be(ModelName.Supervised); args.minCount.Should().Be(1); args.minn.Should().Be(0); args.maxn.Should().Be(0); args.lr.Should().BeApproximately(0.1d, 10e-5); }
public void CantUsePretrainedVectorsWithDifferentDimension() { var fastText = new FastTextWrapper(loggerFactory: _loggerFactory); string outPath = Path.Combine(_tempDir, "cooking"); var args = FastTextArgs.SupervisedDefaults(); args.PretrainedVectors = "cooking.unsup.300.vec"; fastText.Invoking(x => x.Supervised("cooking.train.txt", outPath, args)) .Should().Throw <NativeLibraryException>() .WithMessage("Dimension of pretrained vectors (300) does not match dimension (100)!"); }
public void CanTrainSupervisedWithNoLogging() { var fastText = new FastTextWrapper(); string outPath = Path.Combine(_tempDir, "cooking"); fastText.Supervised("cooking.train.txt", outPath, FastTextArgs.SupervisedDefaults()); fastText.IsModelReady().Should().BeTrue(); fastText.GetModelDimension().Should().Be(100); CheckLabels(fastText.GetLabels()); File.Exists(outPath + ".bin").Should().BeTrue(); File.Exists(outPath + ".vec").Should().BeTrue(); }
public SupervisedModelFixture() { _tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); Directory.CreateDirectory(_tempDir); FastText = new FastTextWrapper(); string outPath = Path.Combine(_tempDir, "cooking"); FastText.Supervised("cooking.train.txt", outPath, FastTextArgs.SupervisedDefaults()); FastText.IsModelReady().Should().BeTrue(); ModelPath = outPath + ".bin"; File.Exists(ModelPath).Should().BeTrue(); File.Exists(outPath + ".vec").Should().BeTrue(); }
public void CanUsePretrainedVectorsForSupervisedModel() { var fastText = new FastTextWrapper(loggerFactory: _loggerFactory); string outPath = Path.Combine(_tempDir, "cooking"); var args = FastTextArgs.SupervisedDefaults(); args.PretrainedVectors = "cooking.unsup.300.vec"; args.dim = 300; fastText.Supervised("cooking.train.txt", outPath, args); fastText.IsModelReady().Should().BeTrue(); fastText.GetModelDimension().Should().Be(300); CheckLabels(fastText.GetLabels()); File.Exists(outPath + ".bin").Should().BeTrue(); File.Exists(outPath + ".vec").Should().BeTrue(); }