public void LineNumbers()
{
string textfile = "";
using (StreamReader fs = new StreamReader(@"C:\Новая папка\1.txt"))
{
while (true)
{
string temp = fs.ReadLine();
if (temp == null)
{
break;
}
textfile += temp;
}
}
string[] words = new[] { "hello", "word" };
AhoCorasick.Trie <int> trie = new AhoCorasick.Trie <int>();
for (int i = 0; i < words.Length; i++)
{
trie.Add(words[i], i);
}
trie.Build();
int[] lines = trie.Find(textfile).ToArray();
Assert.AreEqual(5, lines.Length);
// Assert.AreEqual(1, lines[0]);
// Assert.AreEqual(1, lines[1]);
}
static void Main(string[] args)
{
Stopwatch sw = new Stopwatch();
sw.Start();
//untuk true / false
string[] text = "ini laptop thomas".Split(' ');
AhoCorasick.Trie <string, bool> trie = new AhoCorasick.Trie <string, bool>();
trie.Add(new[] { "thomasa" }, true);
trie.Build();
/*
* string[] masuk = "one two three four".Split(' ');
*
* AhoCorasick.Trie<string, bool> trie = new AhoCorasick.Trie<string, bool>();
*
* string text = System.IO.File.ReadAllText(@"C:\Users\Thomas Yap\Documents\CUDA Examples\testaho\lorem 1000words.txt");
*
* trie.Build();
*/
Console.WriteLine(trie.Find(text).Any());
sw.Stop();
Console.WriteLine("Elapsed={0}", sw.Elapsed);
Console.ReadKey();
}
public void findsome()
{
string textfile = "";
using (StreamReader fs = new StreamReader(@"C:\Новая папка\1.txt"))
{
while (true)
{
string temp = fs.ReadLine();
if (temp == null)
{
break;
}
textfile += temp;
}
}
AhoCorasick.Trie trie = new AhoCorasick.Trie();
trie.Add("hello");
// trie.Add("word");
trie.Build();
string[] matches = trie.Find(textfile).ToArray();
Assert.AreEqual(5, matches.Length);
Assert.AreEqual("hello", matches[0]);
// Assert.AreEqual("hellonull",matches[1]);
// Assert.AreEqual("word", matches[1]);
}
public static List <string> Match(AhoCorasick.Trie trie, List <string> hashtags, string text)
{
List <int> positions = new List <int>();
foreach (string position in trie.Find(text))
{
positions.Add(Int16.Parse(position));
}
//check if it is non alfa char
var verifiedWords = new List <string>();
if (positions.Count() == 0)
{
return(verifiedWords);
}
foreach (int wordNo in positions)
{
string word = hashtags.ElementAt(wordNo);
int startingPosition = text.IndexOf(word);
int endingPosition = startingPosition + word.Count();
//If beggining or end of text, assume its not english letter
bool front = startingPosition == 0 || (startingPosition != -1 && !Search.IsEnglishLetter(text.ElementAt(startingPosition - 1)));
bool end = endingPosition == text.Count() || (endingPosition != -1 && !Search.IsEnglishLetter(text.ElementAt(endingPosition)));
if (front && end)
{
verifiedWords.Add(word);
}
}
return(verifiedWords);
}
public void Words()
{
string[] text = "hello:hello:wor:ddsdsdf:word:hello".Split(':');
AhoCorasick.Trie <string, bool> trie = new AhoCorasick.Trie <string, bool>();
trie.Add(new[] { "wol" }, true);
trie.Build();
Assert.IsFalse(trie.Find(text).Any());
}
public void Words()
{
string[] text = "one two three four".Split(' ');
AhoCorasick.Trie <string, bool> trie = new AhoCorasick.Trie <string, bool>();
trie.Add(new[] { "three", "four" }, true);
trie.Build();
Assert.IsTrue(trie.Find(text).Any());
}
public void Contains()
{
string text = "hello and welcome to this beautiful world!";
AhoCorasick.Trie trie = new AhoCorasick.Trie();
trie.Add("hello");
trie.Add("world");
trie.Build();
Assert.IsTrue(trie.Find(text).Any());
}
public void Contains()
{
string text = "hello and welcome to this beautiful world!";
AhoCorasick.Trie trie = new AhoCorasick.Trie();
trie.Add("hello");
trie.Add("world");
trie.Build();
Assert.IsTrue(trie.Find(text).Any());
}
public void HelloWorld()
{
string text = "hello and welcome to this beautiful wo5rld!";
AhoCorasick.Trie trie = new AhoCorasick.Trie();
trie.Add("hello");
trie.Add("worldddddd");
trie.Build();
string[] matches = trie.Find(text).ToArray();
Assert.AreEqual(2, matches.Length);
Assert.AreEqual("hello", matches[0]);
Assert.AreEqual("world5dd", matches[1]);
}
public void HelloWorld()
{
string text = "hello and welcome to this beautiful world!";
AhoCorasick.Trie trie = new AhoCorasick.Trie();
trie.Add("hello");
trie.Add("world");
trie.Build();
string[] matches = trie.Find(text).ToArray();
Assert.AreEqual(2, matches.Length);
Assert.AreEqual("hello", matches[0]);
Assert.AreEqual("world", matches[1]);
}
public void HelloWorld()
{
string text = "hello and welcome to this beautiful world!";
var trie = new AhoCorasick.Trie();
trie.Add("hello");
trie.Add("world");
trie.Build();
var matches = trie.Find(text).ToArray();
Assert.AreEqual(2, matches.Length);
Assert.AreEqual(Tuple.Create("hello", 4), matches[0]);
Assert.AreEqual(Tuple.Create("world", 40), matches[1]);
}
public void LineNumbers()
{
string text = "world, i hello you!";
string[] words = new[] { "hello", "world" };
AhoCorasick.Trie<int> trie = new AhoCorasick.Trie<int>();
for (int i = 0; i < words.Length; i++)
trie.Add(words[i], i);
trie.Build();
int[] lines = trie.Find(text).ToArray();
Assert.AreEqual(2, lines.Length);
Assert.AreEqual(1, lines[0]);
Assert.AreEqual(0, lines[1]);
}
public void WordsAndIds()
{
string text = "hello and welcome to this beautiful world!";
var trie = new AhoCorasick.Trie <Tuple <string, int> >();
trie.Add("hello", Tuple.Create("hello", 123));
trie.Add("world", Tuple.Create("world", 456));
trie.Build();
var matches = trie.Find(text).ToArray();
Assert.AreEqual(2, matches.Length);
Assert.AreEqual(Tuple.Create(Tuple.Create("hello", 123), 4), matches[0]);
Assert.AreEqual(Tuple.Create(Tuple.Create("world", 456), 40), matches[1]);
}
// remember to check for the same key added before adding when counting words!
public static void Main()
{
AhoCorasick.Trie trie = new AhoCorasick.Trie();
// add words
trie.Add("hello");
trie.Add("world");
// build search tree
trie.Build();
string text = "hello and welcome to this beautiful world world hello!";
// find words
foreach (string word in trie.Find(text))
{
Console.WriteLine(word);
}
}
public void LineNumbers()
{
string text = "world, i hello you!";
string[] words = new[] { "hello", "world" };
AhoCorasick.Trie <int> trie = new AhoCorasick.Trie <int>();
for (int i = 0; i < words.Length; i++)
{
trie.Add(words[i], i);
}
trie.Build();
int[] lines = trie.Find(text).ToArray();
Assert.AreEqual(2, lines.Length);
Assert.AreEqual(1, lines[0]);
Assert.AreEqual(0, lines[1]);
}
private void NonRegexSearch(SearchFile sf, string line, int iLine, bool caseSensitive)
{
//Find.
bool bFound = false;
lock (MatchedFiles_Lock)
{
IEnumerable <string> ret = _objFileContentsSearchTrie.Find(caseSensitive ? line : line.ToLower());
if (ret.Any())
{
if (sf.MatchedLines == null)
{
sf.MatchedLines = new List <SearchLine>();
}
sf.MatchedLines.Add(new SearchLine(iLine, line));
bFound = true;
}
}
if (bFound)
{
AddMatch(sf);
}
}
/// <summary>
///
/// </summary>
/// <param name="path"></param>
/// <param name="trie"></param>
/// <param name="capitalizedHashtags"></param>
/// <param name="hashtags"></param>
public static void Match(string path, AhoCorasick.Trie trie, List <string> hashtags)
{
DirectoryInfo rootFolder = new DirectoryInfo(path);
var files = rootFolder.EnumerateFiles("*.json", SearchOption.AllDirectories);
tags = hashtags.ToDictionary(x => x, x => 0);
Parallel.ForEach(files, new ParallelOptions {
MaxDegreeOfParallelism = 16
}, (file1) =>
{
int found = 0;
int matched = 0;
Console.WriteLine("Reading " + file1.FullName);
var jsonText = System.IO.File.ReadAllText(file1.FullName);
IList <_Tweet> tweets;
try
{
tweets = JsonConvert.DeserializeObject <IList <_Tweet> >(jsonText);
} catch (Exception E)
{
return;
}
var matchedTweets = new List <string>();
for (var i = 0; i < tweets.Count; i++)
{
String text = tweets[i].Text.ToLower();
if (tweets[i] == null)
{
continue;
}
List <int> positions = new List <int>();
foreach (string position in trie.Find(text))
{
positions.Add(Int16.Parse(position));
}
found++;
if (positions.Count() == 0)
{
continue;
}
//check if it is non alfa char
var verifiedWords = new List <string>();
foreach (int wordNo in positions)
{
string word = hashtags.ElementAt(wordNo);
int startingPosition = text.IndexOf(word);
int endingPosition = startingPosition + word.Count();
//If beggining or end of text, assume its not english letter
bool front = startingPosition == 0 || (startingPosition != -1 && !IsEnglishLetter(text.ElementAt(startingPosition - 1)));
bool end = endingPosition == text.Count() || (endingPosition != -1 && !IsEnglishLetter(text.ElementAt(endingPosition)));
if (front && end)
{
verifiedWords.Add(word);
}
else
{
continue;
}
}
if (verifiedWords.Count == 0)
{
continue;
}
matched++;
matchedTweets.Add(JsonConvert.SerializeObject(tweets[i]));
//Check if what happens here is correct
foreach (string word in verifiedWords)
{
lock (keywordLock)
{
tags[word] += 1;
}
}
}
Interlocked.Add(ref TweetsFound, found);
Interlocked.Add(ref TweetsMatched, matched);
System.IO.File.WriteAllLines(file1.DirectoryName + "/Matched" + file1.Name + ".txt", matchedTweets);
});
bool mergeIntoOneFile = true;
if (mergeIntoOneFile)
{
MergeSearchResults(path, "output.txt");
}
using (StreamWriter sw = new StreamWriter(path + "/Search_stats.txt"))
{
sw.Write(JsonConvert.SerializeObject(new Stats
{
Found = TweetsFound,
Matched = TweetsMatched,
Results = tags
}, Formatting.Indented));
// sw.Write(JsonConvert.SerializeObject(objectToSerialize, Formatting.Indented));
Interlocked.Increment(ref TweetsFound);
}
}
public void Words()
{
string[] text = "one two three four".Split(' ');
AhoCorasick.Trie<string, bool> trie = new AhoCorasick.Trie<string, bool>();
trie.Add(new[] { "three", "four" }, true);
trie.Build();
Assert.IsTrue(trie.Find(text).Any());
}