public void Train(string checkpoint, string run, int?counter, dynamic sessionConfig = null, CancellationToken cancellation = default) { Session session = sessionConfig is null ? Session.NewDyn(config : sessionConfig) : new Session(); using (session.StartUsing()) { var context = tf.placeholder(tf.int32, new TensorShape(this.batchSize, null)); var output = Gpt2Model.Model(this.hParams, input: context); Tensor labels = context[.., 1..];
public override int Run(string[] remainingArguments) { this.CheckRequiredArguments(); if (remainingArguments.Length < 1) { throw new ArgumentNullException("dataset"); } string datasetName = remainingArguments[0]; string checkpoint = Gpt2Checkpoints.ProcessCheckpointConfig( gpt2Root: Environment.CurrentDirectory, checkpoint: this.Checkpoint, modelName: this.ModelName, runName: this.RunName); var encoder = Gpt2Encoder.LoadEncoder(this.ModelName); string searchPattern = this.Include ?? "*"; var dataset = searchPattern.EndsWith("*.csv") ? LoadCsv(encoder, root: datasetName, field: this.ColumnName) : Gpt2Dataset.LoadDataset(encoder, path: datasetName, pattern: searchPattern); if (dataset.Count == 0) { Console.Error.WriteLine("The dataset is empty!"); return(-1); } var hParams = Gpt2Model.LoadHParams(this.ModelName); var random = this.Seed is null ? new Random() : new Random(this.Seed.Value); var stop = new CancellationTokenSource(); Console.CancelKeyPress += delegate { stop.Cancel(); }; dynamic config = config_pb2.ConfigProto.CreateInstance(); config.gpu_options.allow_growth = true; new Gpt2Trainer(dataset, encoder, hParams, this.BatchSize, this.SampleLength, random) { SaveEvery = this.SaveEvery, SampleNum = this.SampleNum, SampleEvery = this.SampleEvery, } .Train(checkpoint, this.RunName, sessionConfig: config, counter: checkpoint == "fresh" ? 1 : (int?)null, cancellation: stop.Token); return(0); }
/// <summary> /// Interactively run the model /// </summary> /// <param name="modelName">Which model to use</param> /// <param name="checkpoint">Which checkpoint to load</param> /// <param name="seed">Seed for random number generators, fix seed to reproduce results</param> /// <param name="sampleCount">Number of samples to return total</param> /// <param name="batchSize">Number of batches (only affects speed/memory). Must divide sampleCount.</param> /// <param name="length">Number of tokens in generated text, if null (default), is /// determined by model hyperparameters</param> /// <param name="temperature">randomness in boltzmann distribution. /// Lower temperature results in less random completions. As the /// temperature approaches zero, the model will become deterministic and /// repetitive. Higher temperature results in more random completions.</param> /// <param name="topK">Controls diversity. 1 means only 1 word is /// considered for each step (token), resulting in deterministic completions, /// while 40 means 40 words are considered at each step. 0 (default) is a /// special setting meaning no restrictions. 40 generally is a good value. /// </param> public static void Run(string modelName = "117M", string checkpoint = null, int?seed = null, int sampleCount = 1, int batchSize = 1, int?length = null, float temperature = 1, int topK = 0) { if (sampleCount % batchSize != 0) { throw new ArgumentException(); } var encoder = Gpt2Encoder.LoadEncoder(modelName); var hParams = Gpt2Model.LoadHParams(modelName); int nCtx = ((dynamic)hParams).n_ctx; if (length is null) { length = nCtx; } else if (length > nCtx) { throw new ArgumentException("Can't get samples longer than window size: " + hParams.get("n_ctx")); } new Session(graph: new Graph()).UseSelf(sess => { var context = tf.placeholder(tf.int32, new TensorShape(batchSize, null)); tf.set_random_seed(seed); var output = Gpt2Sampler.SampleSequence( hParams: hParams, length: length.Value, context: context, batchSize: batchSize, temperature: temperature, topK: topK); var saver = new Saver(); checkpoint = checkpoint ?? tf.train.latest_checkpoint(Path.Combine("models", modelName)); saver.restore(sess, checkpoint); bool interrupted = false; Console.CancelKeyPress += (object sender, ConsoleCancelEventArgs args) => Volatile.Write(ref interrupted, args.Cancel = true); while (!interrupted) { string text; do { Console.Write("Model prompt >>> "); text = Console.ReadLine(); if (Volatile.Read(ref interrupted)) { break; } if (string.IsNullOrEmpty(text)) { Console.WriteLine("Prompt should not be empty"); } } while (!Volatile.Read(ref interrupted) && string.IsNullOrEmpty(text)); if (Volatile.Read(ref interrupted)) { break; } var contextTokens = encoder.Encode(text); if (!tf.test.is_gpu_available() && contextTokens.Count >= length.Value) { Console.Error.WriteLine(); Console.Error.WriteLine("Prompt is too long."); Console.Error.WriteLine(); continue; } int generated = 0; foreach (var _ in Enumerable.Range(0, sampleCount / batchSize)) { ndarray <int> @out; try { @out = sess.run(output, feed_dict: new Dictionary <object, object> { [context] = Enumerable.Repeat(contextTokens, batchSize).ToArray(), })[.., contextTokens.Count..]; } catch (InvalidArgumentError ex) { throw new ArgumentOutOfRangeException( "Unable to generate sequence of desired length. " + "Try lowering length by passing -l (-sample-length) parameter. " + "Current length: " + length.Value, innerException: ex); } foreach (int i in Enumerable.Range(0, batchSize)) { generated++; var part = (ndarray <int>)@out[i]; text = encoder.Decode(part); Console.WriteLine($"{Delimiter} SAMPLE {generated} {Delimiter}"); Console.WriteLine(text); } } Console.Write(Delimiter); Console.WriteLine(Delimiter); } }); }