internal void LinkAsNext(VectorNode tcpVNode)
{
tcpVNode.next = this.next;
this.next = tcpVNode;
tcpVNode.prev = this;
tcpVNode.next.prev = tcpVNode;
}
public void Run(IDictionary <string, string> args, ILogger logger)
{
var dataDirectory = args["dataDirectory"];
var collection = args["collection"];
var documentId = long.Parse(args["documentId"]);
var select = new HashSet <string>(args["select"].Split(new char[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries));
var collectionId = collection.ToHash();
var model = new BagOfCharsModel();
using (var sessionFactory = new SessionFactory(dataDirectory, logger))
using (var documents = new DocumentStreamSession(sessionFactory))
using (var documentReader = new DocumentReader(collectionId, sessionFactory))
{
var doc = documents.ReadDocument((collectionId, documentId), select, documentReader);
foreach (var field in doc.Fields)
{
var tokens = model.Tokenize(field.Value.ToString());
var tree = new VectorNode();
foreach (var token in tokens)
{
tree.MergeOrAdd(new VectorNode(token), model);
}
Console.WriteLine(field.Name);
Console.WriteLine(PathFinder.Visualize(tree));
Console.WriteLine(string.Join('\n', tokens));
}
}
}
/*
* internal void LinkAsNext(Bytes data, uint length) {
* VectorNode next = new VectorNode(data, length);
* next.next = this.next;
* this.next = next;
* next.prev = this;
* next.next.prev = next;
* }
*/
internal void LinkAsPrev(VectorNode tcpVNode)
{
tcpVNode.prev = this.prev;
this.prev = tcpVNode;
tcpVNode.next = this;
tcpVNode.prev.next = tcpVNode;
}
private int AddDocument(string documentId, string[] input, VectorTree tree)
{
var docId = Hash(documentId);
var dic = new SortedList <double, string>();
var count = tree.Count;
foreach (var word in input)
{
if (string.IsNullOrWhiteSpace(word))
{
continue;
}
tree.GetNode(0, 0).Add(word, docId);
var wordvec = new VectorNode(word).TermVector;
if (wordvec.CosAngle(tree.Find(0, 0, word).TermVector) < VectorNode.IdenticalAngle)
{
throw new Exception("error");
}
//Console.Write("{0} [", word);
//foreach (var c in word.Components())
//{
// Console.Write("{0}:{1}, ", c.Key, c.Value);
//}
//Console.WriteLine("]");
}
return(tree.Count - count);
}
private static void Print(VectorNode tree)
{
var diagram = PathFinder.Visualize(tree);
File.WriteAllText(@"c:\temp\imagemodeltesttree.txt", diagram);
Debug.WriteLine(diagram);
}
static void Main(string[] args)
{
var doc1 = new VectorNode(args[0]);
var doc2 = new VectorNode(args[1]);
var angle = doc1.Vector.CosAngle(doc2.Vector);
Console.WriteLine($"similarity: {angle}");
}
internal VectorNode BoxedUnlink()
{
this.prev.next = this.next;
this.next.prev = this.prev;
this.next = this;
this.prev = this;
return(this);
}
/*
* internal VectorNode(Bytes arg, uint length) {
* this.next = this;
* this.prev = this;
* this.data = arg;
* this.startOffset = 0;
* this.length = length;
* }
*/
internal Bytes Unlink()
{
this.prev.next = this.next;
this.next.prev = this.prev;
this.next = this;
this.prev = this;
return(this.data);
}
public override INode <AstNode> VisitVector(NotQuiteLisp.Parser.NQLParser.VectorContext context)
{
var children = GetChildAstNodes(context);
var vectorNode = new VectorNode(children);
return(vectorNode);
}
private static void CompareBaseless(string first, string second, IStringModel model)
{
var doc1 = new VectorNode(model.Tokenize(first.ToCharArray()).First());
var doc2 = new VectorNode(model.Tokenize(second.ToCharArray()).First());
var angle = model.CosAngle(doc1.Vector, doc2.Vector);
Console.WriteLine($"similarity (baseless): {angle}");
}
public void Put(VectorNode tree)
{
var column = _index.GetOrAdd(tree.KeyId.Value, key => new VectorNode());
foreach (var node in PathFinder.All(tree))
{
_indexingStrategy.ExecutePut <T>(column, new VectorNode(node.Vector, docIds: node.DocIds));
}
}
private static void Compare(string first, string second, IStringModel model)
{
var baseVectorComponents = new List <float>(model.VectorWidth);
var baseVectors = new List <IVector>();
for (int i = 0; i < model.VectorWidth; i++)
{
baseVectorComponents.Add(i == 0 ? 1 : Convert.ToSingle(Math.Log10(i)));
var bvecs = new List <float>(model.VectorWidth);
for (int y = 0; y < model.VectorWidth; y++)
{
float value;
if (y == i)
{
value = 1;
}
else
{
value = 0;
}
bvecs.Add(value);
}
baseVectors.Add(new IndexedVector(bvecs, model.VectorWidth));
}
var bvector = new IndexedVector(baseVectorComponents, model.VectorWidth);
var doc1 = new VectorNode(model.Tokenize(first.ToCharArray()).First());
var doc2 = new VectorNode(model.Tokenize(second.ToCharArray()).First());
var angles1 = new List <float>();
var angles2 = new List <float>();
foreach (var bvec in baseVectors)
{
angles1.Add(Convert.ToSingle(model.CosAngle(doc1.Vector, bvec)));
angles2.Add(Convert.ToSingle(model.CosAngle(doc2.Vector, bvec)));
}
var docVector1 = new IndexedVector(angles1, model.VectorWidth);
var docVector2 = new IndexedVector(angles2, model.VectorWidth);
var angle = model.CosAngle(docVector1, docVector2);
var angle1 = model.CosAngle(docVector1, bvector);
var angle2 = model.CosAngle(docVector2, bvector);
Console.WriteLine($"similarity: {angle}");
Console.WriteLine($"bvector similarity 1: {angle1}");
Console.WriteLine($"bvector similarity 2: {angle2}");
Console.WriteLine($"base vector similarity: {Math.Min(angle1, angle2) / Math.Max(angle1, angle2)}");
}
public VectorQueueByte()
{
size = 0;
count = 0;
VectorNode node = this.listHead = new VectorNode();
this.listTail = node;
currentTxBuff = null;
currentTxBufferOffset = 0;
currentTxTotalOffset = 0;
}
public void Analyze <T>(IModel <T> model)
{
var tokens = model.Tokenize((T)Value);
Tree = new VectorNode();
foreach (var token in tokens)
{
model.ExecutePut <string>(Tree, new VectorNode(token, keyId: KeyId));
}
_tokens = GetTokens();
}
public Bytes ExtractHead()
{
if (this.Empty)
{
return(null);
}
VectorNode candidate = this.listHead.next;
//VTable.Assert(candidate.length != 0);
size -= (uint)candidate.data.Length;
count--;
return(candidate.BoxedUnlink().data);
}
static void Main(string[] args)
{
var model = new BocModel();
if (args[0] == "-b")
{
var root = new VectorNode();
while (true)
{
Console.WriteLine("enter word:");
var command = Console.ReadLine();
if (string.IsNullOrWhiteSpace(command))
{
break;
}
GraphBuilder.Add(root, new VectorNode(model.Tokenize(command).Embeddings[0]), model);
}
Console.WriteLine(root.Visualize());
while (true)
{
Console.WriteLine("enter query:");
var command = Console.ReadLine();
if (string.IsNullOrWhiteSpace(command))
{
break;
}
var hit = PathFinder.ClosestMatch(root, model.Tokenize(command).Embeddings[0], model);
Console.WriteLine($"{hit.Score} {hit.Node}");
}
}
else
{
var doc1 = new VectorNode(model.Tokenize(args[0]).Embeddings[0]);
var doc2 = new VectorNode(model.Tokenize(args[1]).Embeddings[0]);
var angle = model.CosAngle(doc1.Vector, doc2.Vector);
Console.WriteLine($"similarity: {angle}");
}
}
// public void Release() {}
// public void Acquire() {}
public void Dispose()
{
VectorNode current = this.listHead.next;
while (current != this.listHead)
{
// temporary hack until we fix the upcast in receiver context
Bytes data = current.data;
if (data != null)
{
//delete data;
}
current = current.next;
}
this.listTail = this.listHead = new VectorNode();
}
static void Main(string[] args)
{
var document1 = args[0];
var document2 = args[1];
var docNode1 = new VectorNode(document1);
var docNode2 = new VectorNode(document2);
var docAngle = docNode1.Vector.CosAngle(docNode2.Vector);
if (docAngle >= VectorNode.TermIdenticalAngle)
{
Console.Write("{0} is very similar to {1}", document1, document2);
}
else
{
var tokenizer = new LatinTokenizer();
var index1 = new VectorNode();
var tokens = tokenizer.Tokenize(document1);
foreach (var token in tokens.Tokens)
{
var termVector = tokens.ToCharVector(token.offset, token.length);
index1.Add(new VectorNode(termVector), VectorNode.TermIdenticalAngle, VectorNode.TermFoldAngle, new MemoryStream());
}
float score = 0;
var count = 0;
var tokens2 = tokenizer.Tokenize(document2);
foreach (var token in tokens2.Tokens)
{
var termVector = tokens.ToCharVector(token.offset, token.length);
var node = index1.ClosestMatch(new VectorNode(termVector), VectorNode.TermFoldAngle);
score += node.Score;
count++;
}
var similarity = (score / count) * 100;
Console.WriteLine("{0} is {1}% similar to {2}", document1, similarity, document2);
Console.Read();
}
}
static void Main(string[] args)
{
var document1 = args[0];
var document2 = args[1];
var docNode1 = new VectorNode(document1);
var docNode2 = new VectorNode(document2);
var docAngle = docNode1.TermVector.CosAngle(docNode2.TermVector);
if (docAngle >= VectorNode.IdenticalAngle)
{
Console.Write("{0} is very similar to {1}", document1, document2);
}
else
{
var tokenizer = new LatinTokenizer();
var index1 = new VectorNode();
foreach (var token in tokenizer.Tokenize(document1))
{
index1.Add(new VectorNode(token), new MemoryStream());
}
float score = 0;
var count = 0;
foreach (var token in tokenizer.Tokenize(document2))
{
var node = index1.ClosestMatch(new VectorNode(token), skipDirtyNodes: false);
score += node.Score;
count++;
}
var similarity = (score / count) * 100;
Console.WriteLine("{0} is {1}% similar to {2}", document1, similarity, document2);
Console.Read();
}
}
private static void RunInteractiveGraphBuilder(IStringModel model)
{
var root = new VectorNode();
while (true)
{
Console.WriteLine("enter text:");
var command = Console.ReadLine();
if (string.IsNullOrWhiteSpace(command))
{
break;
}
var node = new VectorNode(model.Tokenize(command.ToCharArray()).First());
GraphBuilder.MergeOrAdd(root, node, model, model.FoldAngle, model.IdenticalAngle);
}
Console.WriteLine(root.Visualize());
while (true)
{
Console.WriteLine("enter query:");
var command = Console.ReadLine();
if (string.IsNullOrWhiteSpace(command))
{
break;
}
var hit = PathFinder.ClosestMatch(root, model.Tokenize(command.ToCharArray()).First(), model);
Console.WriteLine($"{hit.Score} {hit.Node}");
}
}
public Term(long keyId, VectorNode node)
{
KeyId = keyId;
Node = node;
}
public void ExecutePut <T>(VectorNode column, VectorNode node)
{
column.MergeOrAddConcurrent(node, this);
}
public static void DeserializeUnorderedFile(
Stream indexStream,
Stream vectorStream,
VectorNode root,
(float identicalAngle, float foldAngle) similarity)
public void ExecutePut <T>(VectorNode column, VectorNode node)
{
column.Build(node, this);
}
private static void Print(VectorNode tree)
{
var diagram = PathFinder.Visualize(tree);
File.WriteAllText(@"c:\temp\mnisttree.txt", diagram);
}
static void Main(string[] args)
{
var angle = new VectorNode(args[0]).TermVector.CosAngle(new VectorNode(args[1]).TermVector);
Console.WriteLine(angle);
}
private static void Print(VectorNode tree)
{
File.WriteAllText(@"c:\temp\mnisttree.txt", PathFinder.Visualize(tree));
}
public Term(object key, VectorNode node)
{
Key = key;
KeyHash = key.ToHash();
Node = node;
}
public WebresourceNode AddSingleNode(Webresource resource, string[] nameParts, FolderNode folder = null)
{
var fileName = nameParts.Last();
WebresourceType type = WebresourceType.Auto;
if (resource.Type != 0)
{
type = (WebresourceType)resource.Type;
}
if (type == WebresourceType.Auto)
{
if (fileName.IndexOf(".", StringComparison.Ordinal) < 0)
{
if (resource.Type == 0)
{
return(null);
}
type = (WebresourceType)resource.Type;
}
else
{
type = Webresource.GetTypeFromExtension(fileName
.Split(new[] { '.' }, StringSplitOptions.RemoveEmptyEntries).Last());
}
}
WebresourceNode node = null;
switch (type)
{
case WebresourceType.WebPage:
node = new WebpageNode(resource, Settings);
break;
case WebresourceType.Css:
node = new CssNode(resource, Settings);
break;
case WebresourceType.Data:
node = new DataNode(resource, Settings);
break;
case WebresourceType.Gif:
node = new GifNode(resource, Settings);
break;
case WebresourceType.Ico:
node = new IcoNode(resource, Settings);
break;
case WebresourceType.Jpg:
node = new JpgNode(resource, Settings);
break;
case WebresourceType.Png:
node = new PngNode(resource, Settings);
break;
case WebresourceType.Resx:
node = new ResxNode(resource, Settings);
break;
case WebresourceType.Script:
node = new JavaScriptNode(resource, Settings);
break;
case WebresourceType.Silverlight:
node = new SilverlightNode(resource, Settings);
break;
case WebresourceType.Vector:
node = new VectorNode(resource, Settings);
break;
case WebresourceType.Xsl:
node = new XslNode(resource, Settings);
break;
}
resource.Node = node;
if (folder != null && node != null)
{
node.ForeColor = resource.State == WebresourceState.Saved ? Color.Blue :
resource.State == WebresourceState.Draft ? Color.Red : Color.Black;
folder.Nodes.Add(node);
(node.Parent as FolderNode)?.SetFolderColor();
}
return(node);
}
public GraphInfo(long keyId, VectorNode graph)
{
_keyId = keyId;
_graph = graph;
}
