/// <summary>
/// Create unigram table for random sub-sampling.
/// Frequent words will occupy more positions in the table.
/// </summary>
public static int[] GetUnigramTable(this WordCollection wordCollection, int tableSize, double power = 0.75)
{
if (wordCollection.GetNumberOfUniqueWords() == 0)
{
// maybe throw?
return(Array.Empty <int>());
}
var table = new int[tableSize];
var sumOfOccurenceOfWordsRaisedToPower = wordCollection.GetSumOfOccurenceOfWordsRaisedToPower(power);
var words = wordCollection.GetWords().ToArray();
var indexOfCurrentWord = -1;
var highestPositionOfWordInTable = -1;
for (var tablePosition = 0; tablePosition < tableSize; tablePosition++)
{
if (tablePosition > highestPositionOfWordInTable)
{
indexOfCurrentWord++;
highestPositionOfWordInTable += (int)Math.Ceiling(Math.Pow(wordCollection.GetOccurrenceOfWord(words[indexOfCurrentWord]), power) / sumOfOccurenceOfWordsRaisedToPower * tableSize);
}
table[tablePosition] = indexOfCurrentWord;
if (indexOfCurrentWord >= wordCollection.GetNumberOfUniqueWords())
{
indexOfCurrentWord = wordCollection.GetNumberOfUniqueWords() - 1;
}
}
return(table);
}
private static void IterateQueue(WordCollection wordCollection, List <Node> queue)
{
var numberOfInteriorNodes = 0;
for (var a = 0; a < wordCollection.GetNumberOfUniqueWords() - 1; a++)
{
var node = CreateInteriorNode(queue, numberOfInteriorNodes);
numberOfInteriorNodes++;
InsertNodeInQueue(queue, node);
}
}
public void Setup(string trainingFileLocation, int dimensions = 50, int minWordOccurrences = 5)
{
_trainingFileHandler = new FileHandler(trainingFileLocation);
WordCollection = _trainingFileHandler.GetWordDictionaryFromFile(MaxCodeLength);
WordCollection.RemoveWordsWithCountLessThanMinCount(minWordOccurrences);
_table = WordCollection.GetUnigramTable(TableSize);
var inputLayer = new Layer(WordCollection.GetNumberOfUniqueWords(), Array.Empty <Layer>(), ActivationFunctionType.RELU, InitialisationFunctionType.None);
var hiddenLayer = new Layer(dimensions, new[] { inputLayer }, ActivationFunctionType.Linear, InitialisationFunctionType.RandomWeighted, false);
NeuralNetwork = new Layer(WordCollection.GetNumberOfUniqueWords(), new[] { hiddenLayer }, ActivationFunctionType.Sigmoid, InitialisationFunctionType.HeEtAl, false);
NeuralNetwork.Initialise(new Random());
GC.Collect();
}
public void WriteWordEmbeddings(string embeddingsFileLocation)
{
var words = WordCollection.GetWords().ToArray();
var hiddenLayer = NeuralNetwork.PreviousLayers[0];
var inputLayer = hiddenLayer.PreviousLayers[0];
var wordEmbeddings = new List <WordEmbedding>();
for (var i = 0; i < WordCollection.GetNumberOfUniqueWords(); i++)
{
wordEmbeddings.Add(new WordEmbedding(words[i], hiddenLayer.Nodes.Select(hiddenNode => hiddenNode.Weights[inputLayer.Nodes[i]].Value).ToArray()));
}
using var fileStream = new FileStream(embeddingsFileLocation, FileMode.OpenOrCreate, FileAccess.Write);
_ = fileStream.Seek(0, SeekOrigin.End);
using var writer = new StreamWriter(fileStream, Encoding.UTF8);
wordEmbeddings.WriteEmbeddingToStream(writer);
}
