public Seq2SeqClassificationModel(int hiddenDim, int srcEmbeddingDim, int tgtEmbeddingDim, int encoderLayerDepth, int decoderLayerDepth, int multiHeadNum,
EncoderTypeEnums encoderType, DecoderTypeEnums decoderType, Vocab srcVocab, Vocab tgtVocab, Vocab clsVocab, bool enableCoverageModel, bool sharedEmbeddings, bool enableSegmentEmbeddings, bool enableTagEmbeddings, int maxSegmentNum)
: base(hiddenDim, srcEmbeddingDim, tgtEmbeddingDim, encoderLayerDepth, decoderLayerDepth, multiHeadNum, encoderType, decoderType, srcVocab, tgtVocab, enableCoverageModel, sharedEmbeddings, enableSegmentEmbeddings,
enableTagEmbeddings, maxSegmentNum, false)
{
ClsVocab = clsVocab;
}
public double[,] ToVector(List <HuffmanNode> nodes)
{
if (nodes == null)
{
throw new ArgumentNullException(nameof(nodes));
}
var oneHot = new double[1, Vocab.GetLengthLeafs()];
foreach (var node in nodes)
{
var nodeOneHot = MatrixOps.OneHotVector(node.Index, Vocab.GetLengthLeafs());
for (var j = 0; j < nodeOneHot.CountOfColumns(); j++)
{
oneHot[0, j] += nodeOneHot[0, j];
}
}
//skip-gram output is actually as many '1's as context words
if (IsCbow)
{
oneHot.ApplyToEach(v => v / nodes.Count);
}
return(oneHot);
}
/// <summary>Implementation of the abstract method SetUpGrid.</summary>
protected override void SetUpGrid()
{
int width = this.tileset.SizeWidth();
int height = this.tileset.SizeHeight();
int spriteSize = this.tileset.SpriteSize();
if (control != null)
{
control.Controls.Clear();
}
grid = new List <Button>();
// Verify all the options to see if throws any exception.
try
{
for (int y = 0; y < height; y += spriteSize)
{
for (int x = 0; x < width; x += spriteSize)
{
Button btn = NewButton(null, spriteSize);
btn.Click += new EventHandler(ButtonClickEventHandler);
grid.Add(btn);
if (control != null)
{
control.Controls.Add(btn);
btn.Location = new Point(x, y);
}
}
}
}
catch (IndexOutOfRangeException) {
throw new ConvertException(Vocab.GetText("sizeNotMatchErrorMsg"));
}
}
public void BuildComplexContextSimply()
{
// given
const string expected = @"
{
'@base': 'http://example.com/',
'@vocab': 'http://schema.org/',
'dcterms': 'http://purl.org/dc/terms/',
'xsd': 'http://www.w3.org/2001/XMLSchema#',
'title': 'dcterms:title',
'creator': {
'@id': 'dcterms:creator',
'@type': '@id'
},
'medium': {
'@id': 'dcterms:medium',
'@container': '@set',
'@type': '@vocab'
},
'date': {
'@id': 'dcterms:date',
'@type': 'xsd:date'
},
'@language': 'en',
'label': {
'@id': 'http://www.w3.org/2004/02/skos/core#prefLabel',
'@language': null
},
'altLabel': {
'@id': 'http://www.w3.org/2004/02/skos/core#altLabel',
'@container': '@language',
'@type': 'xsd:string'
}
}";
// when
var context = new JObject(
Base.Is("http://example.com/"),
Vocab.Is(new Uri("http://schema.org/")),
"dcterms".IsPrefixOf("http://purl.org/dc/terms/"),
"xsd".IsPrefixOf(new Uri("http://www.w3.org/2001/XMLSchema#")),
"title".IsProperty("dcterms:title"),
"creator".IsProperty("dcterms:creator")
.Type().Id(),
"medium".IsProperty("dcterms:medium")
.Container().Set()
.Type().Vocab(),
"date".IsProperty("dcterms:date")
.Type().Is("xsd:date"),
"en".IsLanguage(),
"label".IsProperty("http://www.w3.org/2004/02/skos/core#prefLabel")
.Language(null),
"altLabel".IsProperty("http://www.w3.org/2004/02/skos/core#altLabel")
.Container().Language()
.Type().Is("xsd:string"));
// then
context = JObject.Parse(context.ToString()); // DeepEqual fails otherwise
Assert.That(JToken.DeepEquals(context, JObject.Parse(expected)), "Actual context was {0}", context);
}
/// <summary>
/// Build vocabulary from training corpus
/// For return vocabs: (source vocab, target vocab, classification vocab)
/// </summary>
/// <param name="vocabSize"></param>
public (Vocab, Vocab, Vocab) BuildVocabs(int srcVocabSize = 45000, int tgtVocabSize = 45000, bool sharedVocab = false)
{
if (sharedVocab && (srcVocabSize != tgtVocabSize))
{
throw new ArgumentException($"Vocab size must be equal if sharedVocab is true. Src Vocab Size = '{srcVocabSize}', Tgt Vocab Size = '{tgtVocabSize}'");
}
foreach (var sntPairBatch in this)
{
CorpusBatch.CountSntPairTokens(sntPairBatch.SntPairs);
}
CorpusBatch.ReduceSrcTokensToSingleGroup();
if (sharedVocab)
{
CorpusBatch.MergeTokensCountSrcTgt(0, 1);
}
(var srcVocabs, var tgtVocabs) = CorpusBatch.GenerateVocabs(srcVocabSize, tgtVocabSize);
Vocab srcVocab = srcVocabs[0];
Vocab clsVocab = tgtVocabs[0];
Vocab tgtVocab = tgtVocabs[1];
return(srcVocab, tgtVocab, clsVocab);
}
public SeqClassification(SeqClassificationOptions options, Vocab srcVocab = null, List <Vocab> clsVocabs = null)
: base(options.DeviceIds, options.ProcessorType, options.ModelFilePath, options.MemoryUsageRatio, options.CompilerOptions, options.ValidIntervalHours, updateFreq: options.UpdateFreq)
{
m_shuffleType = options.ShuffleType;
m_options = options;
// Model must exist if current task is not for training
if ((m_options.Task != ModeEnums.Train) && !File.Exists(m_options.ModelFilePath))
{
throw new FileNotFoundException($"Model '{m_options.ModelFilePath}' doesn't exist.");
}
if (File.Exists(m_options.ModelFilePath))
{
if (srcVocab != null || clsVocabs != null)
{
throw new ArgumentException($"Model '{m_options.ModelFilePath}' exists and it includes vocabulary, so input vocabulary must be null.");
}
m_modelMetaData = LoadModelImpl_WITH_CONVERT(CreateTrainableParameters);
//m_modelMetaData = LoadModelImpl();
//---LoadModel_As_BinaryFormatter( CreateTrainableParameters );
}
else
{
m_modelMetaData = new SeqClassificationModel(options.HiddenSize, options.EmbeddingDim, options.EncoderLayerDepth, options.MultiHeadNum,
options.EncoderType, srcVocab, clsVocabs, options.EnableSegmentEmbeddings, options.EnableTagEmbeddings, options.MaxSegmentNum);
//Initializng weights in encoders and decoders
CreateTrainableParameters(m_modelMetaData);
}
m_modelMetaData.ShowModelInfo();
}
public void ParseDirectory(String path)
{
FileList = Directory.EnumerateFiles(path).Select(x => Path.GetFileName(x)).ToList();
IEnumerable <char> distinctSymbols = FileList.
Aggregate <String>((x, y) => x + y).
ToCharArray().
Distinct().
Where(x => !Char.IsLetter(x)).
OrderByDescending(x => x);
List <string[]> splitNames = FileList.Select(x => x.Split(distinctSymbols.ToArray <char>(), StringSplitOptions.RemoveEmptyEntries)).ToList();
IEnumerable <IEnumerable <Tuple <string, string> > > splitSingleNames = splitNames.Select(x => x.Select(a => Tuple.Create("", a.ToLower())));
IEnumerable <IEnumerable <Tuple <string, string> > > splitPairNames = splitNames.Select(x => x.Zip(x.Skip(1), (a, b) => Tuple.Create(a.ToLower(), b.ToLower())));
IEnumerable <IEnumerable <Tuple <string, string> > > union = splitPairNames;//splitSingleNames.Zip(splitPairNames, (a, b) => a.Concat(b));
HashSet <Tuple <string, string> > vocabSet = new HashSet <Tuple <string, string> >();
foreach (IEnumerable <Tuple <string, string> > tokens in union)
{
foreach (Tuple <string, string> token in tokens)
{
vocabSet.Add(token);
}
}
Vocab = vocabSet.ToList();
ParsedNames = union.Select(x => x.Select(y => Vocab.FindIndex(z => z.Item1.ToLower() == y.Item1.ToLower() && z.Item2.ToLower() == y.Item2.ToLower())).ToList()).ToList();
IEnumerable <double> zeros = Enumerable.Repeat(0.0, Vocab.Count);
double[] v = ParsedNames
.Select(x => zeros.Select((a, i) => x.Contains(i) ? /*IndexLookup.IsNumeric(Vocab[i].Item2)?5.0:*/ 15.0 : 0.0))
.Aggregate((x, y) => x.Concat(y))
.ToArray();
Input = CreateMatrix.Dense(Vocab.Count, ParsedNames.Count, v).Transpose().NormalizeRows(2);
}
public FilePredictor(Options options, DeviceDescriptor device = null)
{
Options = options ?? throw new ArgumentNullException("No options provided");
Device = device ?? DeviceDescriptor.UseDefaultDevice();
Vocab = new Vocab(options);
GetOrCreateModel();
}
public double[,] ToVector(HuffmanNode node)
{
if (node == null)
{
throw new ArgumentNullException(nameof(node));
}
return(MatrixOps.OneHotVector(node.Index, Vocab.GetLengthLeafs()));
}
public async Task <ActionResult <ResultModel> > Put([FromBody] Vocab vocab)
{
var nameIdentifier = HttpContext.User.Claims.Where(x => x.Type == ClaimTypes.NameIdentifier).FirstOrDefault();
if (nameIdentifier == null)
{
return(new ResultModel()
{
Code = 200
,
Message = "User ID is not in header"
});
}
;
var user = await userManager.FindByIdAsync(nameIdentifier.Value);
user.lastVocabUpdateDate = ConvertToTimestamp(DateTime.UtcNow);
await userManager.UpdateAsync(user);
Vocab foundVocab = dbContext.Vocabs.Where(x => x.Id == vocab.Id).FirstOrDefault();
if (foundVocab == null)
{
return(new ResultModel()
{
Code = 1999
,
Message = "Vocab doesn't exist."
});
}
else
{
foundVocab.Word = vocab.Word;
foundVocab.Meaning = vocab.Meaning;
foundVocab.Example = vocab.Example;
dbContext.Vocabs.Update(foundVocab);
int result = dbContext.SaveChanges();
if (result == 1)
{
return(new ResultModel()
{
Code = 200
,
Message = ""
});
}
else
{
return(new ResultModel()
{
Code = 1999
,
Message = "Update failed."
});
}
}
}
public DSSMModel(string name, string commonModelPath, Vocab vocab, int sourceWindowSize, int targetWindowSize, bool compatibilityMode)
{
this.Name = name;
this.SourceModel = new DNN();
this.SourceModel.LoadModelV0(commonModelPath, compatibilityMode);
this.TargetModel = this.SourceModel;
this.Vocabulary = vocab;
this.SourceWindowSize = sourceWindowSize;
this.TargetWindowSize = targetWindowSize;
}
public void ValidatesToFalse()
{
var vocab = new Vocab {
Soruce = "Unit", Target = "Einheit"
};
var validator = new StrictAnswerValidator();
var result = validator.Validate(vocab, "FALSE");
Assert.False(result);
}
private void LoadDictionary()
{
string address = @"Exam1_TestData\TD2\dictionary.txt";
var lines = File.ReadAllLines(address);
foreach (var line in lines)
{
Vocab.Add(line);
}
}
static void Run_PyScript()
{
ProcessStartInfo pyStart = new ProcessStartInfo();
pyStart.FileName = @"./Python36/python.exe";
pyStart.Arguments = @"./PyScript.py";
pyStart.UseShellExecute = false;
pyStart.CreateNoWindow = true;
pyStart.RedirectStandardError = true;
pyStart.RedirectStandardOutput = true;
Console.WriteLine("CSV reader started");
using (Process process = Process.Start(pyStart))
{
using (StreamReader reader = process.StandardOutput)
{
var stdout = reader.ReadToEnd();
var stderr = process.StandardError.ReadToEnd();
if (stderr != "")
{
Console.WriteLine(stderr);
}
else
{
var DataSet = stdout.Split(')');
for (uint i = 1; i < DataSet.Length; i++)
{
var sentTxt = DataSet[i].Split('(')[0].Split(',');
var cls = DataSet[i].Split('(')[1].Replace("\r", string.Empty).Replace(" ", string.Empty).Replace("\n", string.Empty);
for (uint j = 0; j < sentTxt.Length; j++)
{
var word = sentTxt[j];
//var typ = word.Split('#')[word.Split('#').Length - 1];
if (Data.ContainsKey(cls))
{
Data[cls].Add(word);
}
else
{
Data.Add(cls, new List <string> {
word
});
}
Vocab.Add(word);
}
}
Console.WriteLine("CSV reader Executed");
}
}
}
}
protected internal virtual HuffmanNode GetTargetNode(int?fromCorpusPosition = null)
{
var pos = fromCorpusPosition ?? CurrentCorpusPosition;
var targetWord = GetCorpusWordAt(pos);
if (String.IsNullOrWhiteSpace(targetWord))
{
return(null);
}
return(Vocab.GetNodeByWord(targetWord));
}
public void ValidatesToTrue()
{
var input = "Einheit";
var vocab = new Vocab {
Soruce = "Unit", Target = "Einheit"
};
var validator = new StrictAnswerValidator();
var result = validator.Validate(vocab, input);
Assert.True(result);
}
public void IsCaseSensetive()
{
var input = "ABC";
var vocab = new Vocab {
Soruce = "Unit", Target = input
};
var validator = new StrictAnswerValidator();
var result = validator.Validate(vocab, input.ToLower());
Assert.False(result);
}
public void CanHandleSpacesAndSpecialChars()
{
var input = "/ _!#-$%&/{=";
var vocab = new Vocab {
Soruce = "Unit", Target = input
};
var validator = new StrictAnswerValidator();
var result = validator.Validate(vocab, input);
Assert.True(result);
}
public void CanHandleUTF8()
{
var input = "öüä";
var vocab = new Vocab {
Soruce = "Unit", Target = input
};
var validator = new StrictAnswerValidator();
var result = validator.Validate(vocab, input);
Assert.True(result);
}
public void TestVocabBuildInTokens()
{
Vocab vocab = new Vocab("enuSpm.vocab");
Assert.IsTrue((int)SENTTAGS.START == vocab.GetWordIndex(BuildInTokens.BOS));
Assert.IsTrue((int)SENTTAGS.END == vocab.GetWordIndex(BuildInTokens.EOS));
Assert.IsTrue((int)SENTTAGS.UNK == vocab.GetWordIndex(BuildInTokens.UNK));
Assert.IsTrue(vocab.GetString((int)SENTTAGS.START) == BuildInTokens.BOS);
Assert.IsTrue(vocab.GetString((int)SENTTAGS.END) == BuildInTokens.EOS);
Assert.IsTrue(vocab.GetString((int)SENTTAGS.UNK) == BuildInTokens.UNK);
}
private void HandleCurrent(Vocab vocab)
{
_ui.HandleCurrent(_iterator);
if (_validator.Validate(vocab, _ui.GetUserInput(_iterator)))
{
_calculator.increaseCorrectAnswersByOne();
}
else
{
_calculator.increaseWrongAnswersByOne();
}
}
/// <summary>if the image is convertible to MV tileset.</summary>
/// <param name="img">Image to be checked.</param>
/// <returns>Return true if the image is convertible and false if not.</returns>
protected virtual bool IsConvertible(Image img)
{
if (inputTileset.GetType() != typeof(Maker.XP_Tile))
{
if (img.Width != inputTileset.SizeWidth() || img.Height != inputTileset.SizeHeight())
{
System.Windows.Forms.MessageBox.Show(Vocab.GetText("sizeErrorMsg"));
return(false);
}
}
return(true);
}
public void Update(System.Guid id, Vocab.Product product)
{
System.Collections.Generic.IDictionary<string, object> uriArguments = new System.Collections.Generic.Dictionary<string, object>();
var accept = new string[0];
var contentType = new string[] {
"application/rdf+xml",
"text/turtle",
"application/ld+json",
"application/owl+xml" };
uriArguments["id"] = id;
Call(Verb.PUT, "/api/product/{id}", accept, contentType, uriArguments, product);
}
/// <summary>Implementation of the abstract method SetUpGrid.</summary>
protected override void SetUpGrid()
{
int width = tileset.SizeWidth();
int height = tileset.SizeHeight();
int spriteSize = tileset.SpriteSize();
if (width == -1)
{
width = tilesetImage.Width; // custom
}
if (height == -1)
{
height = tilesetImage.Height; // XP and custom
}
if (spriteSize <= 0)
{
throw new ConvertException(Vocab.GetText("sizeIsZeroErrorMsg"));
}
Bitmap[] tiles = SplitImageInSprites(tilesetImage, tileset.SpriteSize());
if (control != null)
{
control.Controls.Clear();
}
grid = new List <Button>();
int i = 0;
// Verify all the options to see if throws any exception.
try
{
for (int y = 0; y < height; y += spriteSize)
{
for (int x = 0; x < width; i++, x += spriteSize)
{
Button btn = NewButton(tiles[i], spriteSize);
btn.Click += new EventHandler(ButtonClickEventHandler);
grid.Add(btn);
if (control != null)
{
control.Controls.Add(btn);
btn.Location = new Point(x, y);
}
}
}
}
catch (IndexOutOfRangeException)
{
throw new ConvertException(Vocab.GetText("sizeNotMatchErrorMsg"));
}
}
public static async Task <HttpResponseMessage> SaveVocab(string uri, Vocab newVocab)
{
string json = JsonConvert.SerializeObject(newVocab);
var content = new StringContent(json.ToString(), Encoding.UTF8, "application/json");
var request = new HttpRequestMessage();
request.RequestUri = new Uri(uri);
request.Method = HttpMethod.Post;
request.Content = content;
System.Diagnostics.Debug.WriteLine(content);
return(await httpClient.SendAsync(request));
}
/// <summary>if the image is convertible to MV tileset.</summary>
/// <param name="img">Image to be checked.</param>
/// <returns>Return true if the image is convertible and false if not.</returns>
protected virtual bool IsConvertible(Image img)
{
if (inputTileset.TilesetName() != Tileset.XP_Tile.Name)
{
if (img.Width != inputTileset.SizeWidth() || img.Height != inputTileset.SizeHeight())
{
System.Console.WriteLine($"{img.Width}x{img.Height} != {inputTileset.SizeWidth()}x{inputTileset.SizeHeight()} | input tileset: {inputTileset}");
System.Windows.Forms.MessageBox.Show(Vocab.GetText("sizeNotMatchErrorMsg"));
return(false);
}
}
return(true);
}
protected internal virtual void InitWiWo(int v = 0)
{
v = v > 0 ? v : Vocab.GetLengthLeafs();
var n = Size;
//https://github.com/ronxin/wevi/blob/master/js/vector_math.js @ function get_random_init_weight(hidden_size)
//values are always less than 0.1
double GetInitWeight(double d) => (d - 0.5D) / n;
WI = MatrixOps.RandomMatrix(v, n, GetInitWeight);
System.Threading.Thread.Sleep(50);
WO = MatrixOps.RandomMatrix(n, v, GetInitWeight);
}
/// <summary>if the image is convertible to MV tileset.</summary>
/// <param name="img">Image to be checked.</param>
/// <returns>Return true if the image is convertible and false if not.</returns>
protected override bool IsConvertible(Image img)
{
if (Maker.Custom.SPRITE_SIZE <= 0)
{
throw new ConvertException(Vocab.GetText("sizeIsZeroErrorMsg"));
}
if (Maker.Custom.SPRITE_SIZE >= img.Width || Maker.Custom.SPRITE_SIZE >= img.Height)
{
throw new ConvertException(Vocab.GetText("sizeOutOfRangeErrorMsg"));
}
return(true);
}
/// <summary>if the image is convertible to MV tileset.</summary>
/// <param name="img">Image to be checked.</param>
/// <returns>Return true if the image is convertible and false if not.</returns>
protected override bool IsConvertible(Image img)
{
if (inputTileset.TileSize() <= 0)
{
throw new ConvertException(Vocab.GetText("sizeIsZeroErrorMsg"));
}
if (inputTileset.TileSize() >= img.Width || inputTileset.TileSize() >= img.Height)
{
throw new ConvertException(Vocab.GetText("sizeOutOfRangeErrorMsg"));
}
return(true);
}
public List <Vocab> CreatVocab(String[] vocab)
{
List <Vocab> vocabList = new List <Vocab>();
char[] delimiters = new char[] { '=' };
foreach (string i in vocab)
{
String[] wordsPerLine = i.Split(delimiters, StringSplitOptions.RemoveEmptyEntries);
Vocab v = new Vocab();
v.position = Int32.Parse(wordsPerLine[0]);
v.word = wordsPerLine[1];
vocabList.Add(v);
}
return(vocabList);
}
public void BuilderTest()
{
Vocab voc = new Vocab();
voc.Add("and", 1);
voc.Add("end", 3);
voc.Add("old", 2);
voc.Add("the", 6);
voc.Add("them", 5);
voc.Add("then", 4);
voc.Range = ('a', 'z');
voc.Name = "en";
Index.Builder bldr = new Index.Builder("BuilderTest").AddVoc(voc);
bldr.AddDoc("A", "");
string[] words = { "and", "tupman", "everybody", "old" };
List <int>[] pos = new List <int> [words.Length];
for (int q = 0; q < pos.Length; q++)
{
pos[q] = new List <int>();
}
foreach (Match match in Regex.Matches(TestText1.ToLower(), @"\b\w+\b"))
{
for (int q = 0; q < pos.Length; q++)
{
if (match.Value.Equals(words[q]))
{
pos[q].Add(match.Index);
}
}
bldr.AddWord(match.Value, (ulong)match.Index);
}
bldr.EndPage("1");
using (Index index = bldr.Build())
{
for (int q = 0; q < pos.Length; q++)
{
Index.SearchResult res = index.Search(words[q]);
Assert.NotNull(res);
Assert.Equal(1, res.foundPages.Count);
Assert.True(pos[q].SequenceEqual(res.foundPages[0].pos));
}
}
Directory.Delete("BuilderTest", true);
}
public System.Guid Create(Vocab.Product product)
{
System.Collections.Generic.IDictionary<string, object> uriArguments = new System.Collections.Generic.Dictionary<string, object>();
var accept = new string[] {
"application/rdf+xml",
"text/turtle",
"application/ld+json",
"application/owl+xml" };
var contentType = new string[] {
"application/rdf+xml",
"text/turtle",
"application/ld+json",
"application/owl+xml" };
var result = Call<System.Guid>(Verb.POST, "/api/product#POSTProduct", accept, contentType, uriArguments, product);
return result;
}
internal static HandleRef getCPtr(Vocab obj)
{
return (obj == null) ? new HandleRef(null, IntPtr.Zero) : obj.swigCPtr;
}
