public void Search_UnContiguousSublist()
{
Assert.IsTrue(Naive.Search_UnContiguousSublist(new List <int> {
10, 3, 4, 1, 7, 8
}, new List <int> {
1, 7
}));
Assert.IsTrue(Naive.Search_UnContiguousSublist(new List <int> {
10, 3, 4, 1, 7, 8
}, new List <int> {
1, 8
}));
Assert.IsFalse(Naive.Search_UnContiguousSublist(new List <int> {
10, 3, 4, 1, 7, 8
}, new List <int> {
1, 80
}));
Assert.IsTrue(Naive.Search_UnContiguousSublist(new List <int> {
1, 10, 3, 4, 1, 7, 8
}, new List <int> {
1, 7
}));
Assert.IsTrue(Naive.Search_UnContiguousSublist(new List <int> {
10, 3, 4, 1, 7, 8
}, new List <int> {
}));
Assert.IsTrue(Naive.Search_UnContiguousSublist(new List <int> {
}, new List <int> {
}));
Assert.IsFalse(Naive.Search_UnContiguousSublist(new List <int> {
}, new List <int> {
1
}));
}
public static void GenUrlLiftRatio()
{
double matchBase = 506000;
double ranBase = 330000.0 * 20000;
string matchfile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\common_user_stat\url02.csv";
string ranfile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\common_user_stat\url_random_2w.csv";
string outfile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\common_user_stat\url_lift.csv";
Dictionary <string, double> url2ratio_match = LoadUrl2Cnt(matchfile, matchBase, 20);
Dictionary <string, double> url2ratio_ran = LoadUrl2Cnt(ranfile, ranBase, 0);
Dictionary <string, double> url2lift = new Dictionary <string, double>();
foreach (var pair in url2ratio_match)
{
if (url2ratio_ran.ContainsKey(pair.Key))
{
url2lift.Add(pair.Key, pair.Value / url2ratio_ran[pair.Key]);
}
else
{
url2lift.Add(pair.Key, 19.9);
}
}
Naive.OutputSortedDict(url2lift, outfile);
}
public static void GenUriStat()
{
string infile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\data-train-dca\urls.csv";
string[] outfiles =
{
@"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\url_freq_dep00.csv",
@"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\url_freq_dep01.csv",
@"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\url_freq_dep02.csv",
@"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\url_freq_dep03.csv"
};
Dictionary <string, int>[] url2freq00 = new Dictionary <string, int> [4];
for (int i = 0; i < 4; i++)
{
url2freq00[i] = new Dictionary <string, int>();
}
using (StreamReader rd = new StreamReader(infile))
{
int cnt = 0;
string content = null;
while ((content = rd.ReadLine()) != null)
{
if (cnt++ % 10000 == 0)
{
Console.WriteLine(cnt);
}
string[] words = content.Split(',');
string url = words[1];
if (url.IndexOf("?") > 0)
{
url = url.Substring(0, url.IndexOf("?"));
}
int dep = 0;
int slash_idx = url.IndexOf("/");
while (dep <= 3)
{
AddDict(url2freq00[dep], url.Substring(0, slash_idx < 0 ? url.Length : slash_idx));
dep++;
if (slash_idx < 0)
{
break;
}
slash_idx = url.IndexOf("/", slash_idx + 1);
}
}
}
for (int i = 0; i < 4; i++)
{
Naive.OutputSortedDict(url2freq00[i], outfiles[i]);
}
}
static void Main(string[] args)
{
foreach (var r in Naive.NaiveStringMatcher("abcaaaaaabc", "abc"))
{
Console.WriteLine(r);
}
Console.WriteLine();
foreach (var r in KnutMorrisPratt.KmpSearch("abcaaaaaabc", "abc"))
{
Console.WriteLine(r);
}
}
private int GetOffset(IMemoryPattern pattern)
{
switch (pattern.Algorithm)
{
case PatternScannerAlgorithm.BoyerMooreHorspool:
return(BoyerMooreHorspool.IndexOf(Data, pattern.GetBytes().ToArray()));
case PatternScannerAlgorithm.Naive:
return(Naive.GetIndexOf(pattern, Data, _module));
}
throw new NotImplementedException("GetOffset encountered an unknown PatternScannerAlgorithm: " +
pattern.Algorithm + ".");
}
public static void Stat02()
{
Dictionary <string, int> fid2usercnt = new Dictionary <string, int>();
string infile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\data-train-dca\facts.json";
string outfile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\fid2usercnt.csv";
using (StreamReader rd = new StreamReader(infile))
{
int factcnt = 0;
string content = null;
while ((content = rd.ReadLine()) != null)
{
if (factcnt++ % 10000 == 0)
{
Console.WriteLine(factcnt);
}
Facts ss = JsonConvert.DeserializeObject <Facts>(content);
HashSet <string> cfidset = new HashSet <string>();
foreach (var re in ss.facts)
{
if (cfidset.Contains(re.fid))
{
//.WriteLine("fid hit!!");
}
else
{
cfidset.Add(re.fid);
}
}
foreach (var cfid in cfidset)
{
if (!fid2usercnt.ContainsKey(cfid))
{
fid2usercnt.Add(cfid, 1);
}
else
{
fid2usercnt[cfid]++;
}
}
}
}
Naive.OutputSortedDict(fid2usercnt, outfile);
}
public void TestNaive()
{
//Full text test
Assert.AreEqual(true, Naive.Run("ABCDEFG", "ABCDEFG"));
//Leading percent test
Assert.AreEqual(true, Naive.Run("%EFG", "ABCDEFG"));
//Trailing percent test
Assert.AreEqual(true, Naive.Run("ABC%", "ABCDEFG"));
//Middle percent test
Assert.AreEqual(true, Naive.Run("ABC%EFG", "ABCDEFG"));
//Leading underscore test
Assert.AreEqual(true, Naive.Run("_BCDEFG", "ABCDEFG"));
//Trailing underscore test
Assert.AreEqual(true, Naive.Run("ABCDEF_", "ABCDEFG"));
//Middle underscore test
Assert.AreEqual(true, Naive.Run("ABC_EFG", "ABCDEFG"));
//Mixed wild card test
Assert.AreEqual(true, Naive.Run("_BCD%_", "ABCDEFG"));
//Repeated character test
Assert.AreEqual(true, Naive.Run("ABC%DEF", "ABCXXDDEF"));
//Repeated pattern test
Assert.AreEqual(true, Naive.Run("ABC%FGHI%FGHI", "ABCFGHIFGHIFGHI"));
//Multiple percent test
Assert.AreEqual(true, Naive.Run("ABC%EF%GHI%MNQ%XYZ", "ABCDEFGHIJKLMNQRSTUVWXYZ"));
//Search length test
Assert.AreEqual(true, Naive.Run("ABC%DEF", "ABCDEF"));
//Search length test 2
Assert.AreEqual(false, Naive.Run("ABC%DEFG", "ABCDEF"));
//All percent test
Assert.AreEqual(true, Naive.Run("%%%%%%", "ABCDEF"));
//Leading percent underscore mismatch test
Assert.AreEqual(false, Naive.Run("%_HIJKL", "ABCDEFG"));
//Character following percent is last character in string mismatch test
Assert.AreEqual(false, Naive.Run("%GABCDE", "ABCDEFG"));
}
public static void StatUserFidCnt()
{
Dictionary <string, int> user2factcnt = new Dictionary <string, int>();
int factcnt = 0;
string infile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\data-train-dca\facts.json";
using (StreamReader rd = new StreamReader(infile))
{
factcnt = 0;
string content = null;
while ((content = rd.ReadLine()) != null)
{
if (factcnt++ % 10000 == 0)
{
Console.WriteLine(factcnt);
}
Facts ss = JsonConvert.DeserializeObject <Facts>(content);
if (!user2factcnt.ContainsKey(ss.uid))
{
user2factcnt.Add(ss.uid, ss.facts.Count);
}
else
{
user2factcnt[ss.uid] += ss.facts.Count;
}
}
}
Console.WriteLine("user cnt : {0}", user2factcnt.Count);
Console.WriteLine("user max fact cnt : {0}", user2factcnt.Max(a => a.Value));
Naive.OutputSortedDict(user2factcnt, @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\user2fidcnt.csv");
}
private void Init(string pred)
{
stExpr = pred;
algorithm = new UnaryAlgorithm(algoNaive = new Naive(pred));
}
private void Init(Map m, int[] a)
{
attrsOrig = new int[a.Length];
a.CopyTo(attrsOrig, 0);
algorithm = new UnaryAlgorithm(algoNaive = new Naive(m, a));
}
// Runs random games
private static void RunRandomGame()
{
int choice = GetChoice("1: Graphic run\n2: Test runs");
if (choice == 1) // graphic game
{
GameEngine game = new GameEngine();
Naive naive = new Naive(game);
naive.RunRandomPlay(true);
}
else // test runs
{
int runs = GetChoice("Choose number of runs: ");
StreamWriter writer = new StreamWriter(RANDOM_LOG_FILE, true);
Dictionary<int, int> highTileCount = new Dictionary<int, int>() { { 64, 0 }, { 128, 0 }, { 256, 0 }, { 512, 0 }, { 1024, 0 }, { 2048, 0 }, { 4096, 0 }, { 8192, 0 }, { 16384, 0 }, { 32768, 0 } };
int totalScore = 0;
for (int i = 0; i < runs; i++)
{
GameEngine game = new GameEngine();
Naive naive = new Naive(game);
State endState = naive.RunRandomPlay(false);
// note highest tile and points
int highestTile = BoardHelper.HighestTile(endState.Board);
int points = endState.Points;
totalScore += points;
// write stats
String stats = i + ":\t" + highestTile + "\t" + points + "\t";
Console.WriteLine(stats);
writer.WriteLine(stats);
// keep track of high cards
List<int> keys = new List<int>(highTileCount.Keys);
for (int j = 0; j < keys.Count; j++)
{
if (highestTile >= keys[j]) highTileCount[keys[j]]++;
}
Thread.Sleep(1000);
}
writer.Close();
Console.WriteLine(GetStatistics(highTileCount, runs, totalScore));
}
Console.ReadLine();
}
public static void UrlComAnalysis_random()
{
var fid2url = Loader.LoadFid2Url();
string outfile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\common_user_stat\url_random_2w.csv";
var fact2url = Loader.LoadFid2Url(1);
var user2facts = Loader.LoadUserFacts();
Dictionary <string, HashSet <string> > user2urls = new Dictionary <string, HashSet <string> >();
List <string> uid_set = new List <string>(user2facts.Keys);
foreach (var uid in uid_set)
{
var cset = new HashSet <string>();
foreach (var fact in user2facts[uid].facts)
{
if (fact2url.ContainsKey(fact.fid))
{
if (!cset.Contains(fact2url[fact.fid]))
{
cset.Add(fact2url[fact.fid]);
}
}
}
user2urls.Add(uid, cset);
}
Dictionary <string, int> url02cnt = new Dictionary <string, int>();
long N = 33000 * 20000;
int len = uid_set.Count;
Random rng = new Random((int)DateTime.Now.Ticks);
for (long i = 0; i < N; i++)
{
if (i % 10000L == 0)
{
Console.WriteLine(i);
}
int a = rng.Next(len);
int b = rng.Next(len);
if (a == b)
{
continue;
}
string[] words = new string[] { uid_set[a], uid_set[b] };
if (user2urls.ContainsKey(words[0]) && user2urls.ContainsKey(words[1]))
{
foreach (var url in user2urls[words[0]])
{
if (user2urls[words[1]].Contains(url))
{
if (!url02cnt.ContainsKey(url))
{
url02cnt.Add(url, 1);
}
else
{
url02cnt[url]++;
}
}
}
}
}
Naive.OutputSortedDict(url02cnt, outfile);
}
public static void UrlComAnalysis()
{
var fid2url = Loader.LoadFid2Url();
string gtfile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\data-train-dca\train.csv";
string outfile = @"\\mlsdata\e$\Users\v-lianji\others\CIKM16\my\common_user_stat\url02.csv";
var user2matches = Loader.LoadGroundTruth(gtfile);
var user2fact = Loader.LoadUserFacts(user2matches);
var fact2url = Loader.LoadFid2Url();
Dictionary <string, HashSet <string> > user2urls = new Dictionary <string, HashSet <string> >();
int target_dep = 1;
HashSet <string> uid_set = new HashSet <string>(user2matches.Keys);
user2matches.Clear();
user2matches = null;
foreach (var uid in uid_set)
{
user2urls.Add(uid, new HashSet <string>());
foreach (var fact in user2fact[uid].facts)
{
string target_url = ExtractUrlWithDep(target_dep, fact2url[fact.fid]);
if (target_url == null)
{
continue;
}
if (!user2urls[uid].Contains(target_url))
{
user2urls[uid].Add(target_url);
}
}
}
Dictionary <string, int> url02cnt = new Dictionary <string, int>();
using (StreamReader rd = new StreamReader(gtfile))
{
string content = null;
while ((content = rd.ReadLine()) != null)
{
string[] words = content.Split(',');
if (user2urls.ContainsKey(words[0]) && user2urls.ContainsKey(words[1]))
{
foreach (var url in user2urls[words[0]])
{
if (user2urls[words[1]].Contains(url))
{
if (!url02cnt.ContainsKey(url))
{
url02cnt.Add(url, 1);
}
else
{
url02cnt[url]++;
}
}
}
}
}
}
Naive.OutputSortedDict(url02cnt, outfile);
}
/// <summary>
/// Constructor which create new model.
/// </summary>
public NaiveController()
{
model = new Naive();
}
