private LdaTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, ctx, input, TestType)
{
Host.AssertValue(ctx);
// *** Binary format ***
// <prefix handled in static Create method>
// <base>
// ldaState[num infos]: The LDA parameters
// Note: infos.length would be just one in most cases.
_exes = new ColInfoEx[Infos.Length];
_ldas = new LdaState[Infos.Length];
_types = new ColumnType[Infos.Length];
for (int i = 0; i < _ldas.Length; i++)
{
_ldas[i] = new LdaState(Host, ctx);
_exes[i] = _ldas[i].InfoEx;
_types[i] = new VectorType(NumberType.Float, _ldas[i].InfoEx.NumTopic);
}
using (var ent = ctx.Repository.OpenEntryOrNull("model", WordTopicModelFilename))
{
_saveText = ent != null;
}
Metadata.Seal();
}
public void Dispose()
{
if (State is object)
{
State.Dispose();
State = null;
}
}
public void Train(IEnumerable <IDocument> documents, int threads, IEnumerable <string> stopwords = null)
{
var stopWords = new HashSet <uint>((stopwords ?? StopWords.Snowball.For(Language)).Select(s => Hash(s.AsSpan())));
if (Data.NumberOfTopics <= 1)
{
throw new Exception($"Invalid number of topics ({nameof(Data)}.{nameof(Data.NumberOfTopics)}), must be > 1");
}
var state = new LdaState(Data, threads);
var(count, corpusSize) = InitializeVocabulary(documents, stopWords);
if (count == 0 || corpusSize == 0)
{
throw new Exception("Empty corpus, nothing to train LDA model");
}
var vocabulary = new ConcurrentDictionary <int, string>();
state.AllocateDataMemory(count, corpusSize);
foreach (var doc in documents)
{
GetTokensAndFrequencies(doc, vocabulary, stopWords, out var tokenCount, out var tokenIndices, out var tokenFrequencies);
if (tokenCount >= Data.MinimumTokenCountPerDocument)
{
var docIndex = state.FeedTrain(Data, tokenIndices, tokenCount, tokenFrequencies);
}
ArrayPool <int> .Shared.Return(tokenIndices);
ArrayPool <double> .Shared.Return(tokenFrequencies);
}
state.CompleteTrain();
state.ReadModelFromTrainedLDA(Data);
Data.Vocabulary = vocabulary;
Data.StopWords = stopWords;
State = state;
}
private void Train(IChannel ch, IDataView trainingData, LdaState[] states)
{
Host.AssertValue(ch);
ch.AssertValue(trainingData);
ch.AssertValue(states);
ch.Assert(states.Length == Infos.Length);
bool[] activeColumns = new bool[trainingData.Schema.ColumnCount];
int[] numVocabs = new int[Infos.Length];
for (int i = 0; i < Infos.Length; i++)
{
activeColumns[Infos[i].Source] = true;
numVocabs[i] = 0;
}
//the current lda needs the memory allocation before feedin data, so needs two sweeping of the data,
//one for the pre-calc memory, one for feedin data really
//another solution can be prepare these two value externally and put them in the beginning of the input file.
long[] corpusSize = new long[Infos.Length];
int[] numDocArray = new int[Infos.Length];
using (var cursor = trainingData.GetRowCursor(col => activeColumns[col]))
{
var getters = new ValueGetter <VBuffer <Double> > [Utils.Size(Infos)];
for (int i = 0; i < Infos.Length; i++)
{
corpusSize[i] = 0;
numDocArray[i] = 0;
getters[i] = RowCursorUtils.GetVecGetterAs <Double>(NumberType.R8, cursor, Infos[i].Source);
}
VBuffer <Double> src = default(VBuffer <Double>);
long rowCount = 0;
while (cursor.MoveNext())
{
++rowCount;
for (int i = 0; i < Infos.Length; i++)
{
int docSize = 0;
getters[i](ref src);
// compute term, doc instance#.
for (int termID = 0; termID < src.Count; termID++)
{
int termFreq = GetFrequency(src.Values[termID]);
if (termFreq < 0)
{
// Ignore this row.
docSize = 0;
break;
}
if (docSize >= _exes[i].NumMaxDocToken - termFreq)
{
break; //control the document length
}
//if legal then add the term
docSize += termFreq;
}
// Ignore empty doc
if (docSize == 0)
{
continue;
}
numDocArray[i]++;
corpusSize[i] += docSize * 2 + 1; // in the beggining of each doc, there is a cursor variable
// increase numVocab if needed.
if (numVocabs[i] < src.Length)
{
numVocabs[i] = src.Length;
}
}
}
for (int i = 0; i < Infos.Length; ++i)
{
if (numDocArray[i] != rowCount)
{
ch.Assert(numDocArray[i] < rowCount);
ch.Warning($"Column '{Infos[i].Name}' has skipped {rowCount - numDocArray[i]} of {rowCount} rows either empty or with negative, non-finite, or fractional values.");
}
}
}
// Initialize all LDA states
for (int i = 0; i < Infos.Length; i++)
{
var state = new LdaState(Host, _exes[i], numVocabs[i]);
if (numDocArray[i] == 0 || corpusSize[i] == 0)
{
throw ch.Except("The specified documents are all empty in column '{0}'.", Infos[i].Name);
}
state.AllocateDataMemory(numDocArray[i], corpusSize[i]);
states[i] = state;
}
using (var cursor = trainingData.GetRowCursor(col => activeColumns[col]))
{
int[] docSizeCheck = new int[Infos.Length];
// This could be optimized so that if multiple trainers consume the same column, it is
// fed into the train method once.
var getters = new ValueGetter <VBuffer <Double> > [Utils.Size(Infos)];
for (int i = 0; i < Infos.Length; i++)
{
docSizeCheck[i] = 0;
getters[i] = RowCursorUtils.GetVecGetterAs <Double>(NumberType.R8, cursor, Infos[i].Source);
}
VBuffer <Double> src = default(VBuffer <Double>);
while (cursor.MoveNext())
{
for (int i = 0; i < Infos.Length; i++)
{
getters[i](ref src);
docSizeCheck[i] += states[i].FeedTrain(Host, ref src);
}
}
for (int i = 0; i < Infos.Length; i++)
{
Host.Assert(corpusSize[i] == docSizeCheck[i]);
states[i].CompleteTrain();
}
}
}
