public void AddToLeaderboard()
{
// First, order current Leaderboard just in case
OrderLeaderboard();
// Max of 10 rankers on leaderboard, so check if current player should enter
if (currentLeaderboard?.Count >= 10)
{
// If current player is better than the last place, change players
if (currentPlayerAttempt.rankerFinalScore < currentLeaderboard[9].rankerFinalScore &&
currentPlayerAttempt.rankerFinalScore > 0)
{
Ranker removedPlayer = currentLeaderboard[9];
currentLeaderboard.Remove(removedPlayer);
currentLeaderboard.Add(currentPlayerAttempt);
// Order once again after the new player is added
OrderLeaderboard();
}
}
else
{
// Add player to leaderboard
if (currentPlayerAttempt.rankerFinalScore > 0)
{
currentLeaderboard.Add(currentPlayerAttempt);
}
// Order once again after the new player is added
OrderLeaderboard();
}
SaveLeaderboardFile();
}
// Saves the current won game
public void SavePlayerAttempt(int finalPlayerScore, float timeElapsed)
{
currentPlayerAttempt = new Ranker();
currentPlayerAttempt.rankerName = PlayerPrefs.GetString("PlayerName");
currentPlayerAttempt.rankerFinalScore = finalPlayerScore;
currentPlayerAttempt.rankerTimeElapsed = timeElapsed;
}
public void PerformMissionAboveQueueCapacityWithUnskilledTeamShouldFailOldestPutAllOthersOnHold()
{
// Arrange
var soldier = new Ranker("Soldier", 20, 100, 100);
this.army.AddSoldier(soldier);
var mission1 = new Easy(soldier.OverallSkill);
var mission2 = new Medium(soldier.OverallSkill * 100);
var mission3 = new Hard(soldier.OverallSkill * 100);
var mission4 = new Hard(soldier.OverallSkill * 100);
this.missionController.Missions.Enqueue(mission1);
this.missionController.Missions.Enqueue(mission2);
this.missionController.Missions.Enqueue(mission3);
// Act
var result = this.missionController.PerformMission(mission4).Trim();
var expectedResult = new StringBuilder()
.AppendLine($"Mission declined - {mission1.Name}")
.AppendLine($"Mission on hold - {mission2.Name}")
.AppendLine($"Mission on hold - {mission3.Name}")
.AppendLine($"Mission on hold - {mission4.Name}")
.ToString()
.Trim();
// Assert
Assert.AreEqual(1, this.missionController.FailedMissionCounter, "Incorrect FailedMissionsCounter!");
Assert.AreEqual(0, this.missionController.SuccessMissionCounter, "Incorrect SuccessMissionCounter!");
Assert.AreEqual(3, this.missionController.Missions.Count, "Incorrect Mission count!");
Assert.AreEqual(expectedResult, result, "Incorrect Result msg!");
}
public void PerformMissionAboveQueueCapacityWithSkilledTeamShouldFailOldestExecuteNext()
{
// Arrange
var soldier = new Ranker("Soldier", 20, 100, 100);
this.army.AddSoldier(soldier);
this.wareHouse.AddAmmunitions(nameof(AutomaticMachine), 1000);
this.wareHouse.AddAmmunitions(nameof(Gun), 1000);
this.wareHouse.AddAmmunitions(nameof(Helmet), 1000);
var mission1 = new Easy(soldier.OverallSkill);
var mission2 = new Medium(soldier.OverallSkill);
var mission3 = new Hard(soldier.OverallSkill * 100);
var mission4 = new Hard(soldier.OverallSkill * 100);
this.missionController.Missions.Enqueue(mission1);
this.missionController.Missions.Enqueue(mission2);
this.missionController.Missions.Enqueue(mission3);
// Act
var result = this.missionController.PerformMission(mission4).Trim();
var expectedResult = new StringBuilder()
.AppendLine($"Mission declined - {mission1.Name}")
.AppendLine($"Mission completed - {mission2.Name}")
.ToString()
.Trim();
// Assert
Assert.AreEqual(1, this.missionController.FailedMissionCounter, "Incorrect FailedMissionsCounter!");
Assert.AreEqual(1, this.missionController.SuccessMissionCounter, "Incorrect SuccessMissionCounter!");
Assert.AreEqual(2, this.missionController.Missions.Count, "Incorrect Mission count!");
Assert.AreEqual(true, result.StartsWith(expectedResult), "Incorrect Result msg!");
}
private static void UnitySortBucket(BucketSlices slices)
{
var ranks = new NativeArray <Ranker>(slices.count, Allocator.Temp, NativeArrayOptions.UninitializedMemory);
for (int i = 0; i < slices.count; i++)
{
ranks[i] = new Ranker
{
index = i,
key = slices.xmins[i]
};
}
ranks.Sort();
NativeArray <float> xminBackup = new NativeArray <float>(slices.count, Allocator.Temp, NativeArrayOptions.UninitializedMemory);
NativeArray <float> xmaxBackup = new NativeArray <float>(slices.count, Allocator.Temp, NativeArrayOptions.UninitializedMemory);
NativeArray <float4> yzminmaxBackup = new NativeArray <float4>(slices.count, Allocator.Temp, NativeArrayOptions.UninitializedMemory);
NativeArray <ColliderBody> bodyBackup = new NativeArray <ColliderBody>(slices.count, Allocator.Temp, NativeArrayOptions.UninitializedMemory);
slices.xmins.CopyTo(xminBackup);
slices.xmaxs.CopyTo(xmaxBackup);
slices.yzminmaxs.CopyTo(yzminmaxBackup);
slices.bodies.CopyTo(bodyBackup);
for (int i = 0; i < slices.count; i++)
{
int src = ranks[i].index;
slices.xmins[i] = xminBackup[src];
slices.xmaxs[i] = xmaxBackup[src];
slices.yzminmaxs[i] = yzminmaxBackup[src];
slices.bodies[i] = bodyBackup[src];
}
}
private static void UnitySortBucket(NativeSlice <int> unsortedSrcIndices, NativeArray <float> xmins)
{
var count = unsortedSrcIndices.Length;
if (count <= 1)
{
return;
}
var ranks = new NativeArray <Ranker>(count, Allocator.Temp, NativeArrayOptions.UninitializedMemory);
for (int i = 0; i < count; i++)
{
ranks[i] = new Ranker
{
index = unsortedSrcIndices[i],
key = xmins[unsortedSrcIndices[i]]
};
}
ranks.Sort();
for (int i = 0; i < count; i++)
{
unsortedSrcIndices[i] = ranks[i].index;
}
}
public RankingGeneral(CompetitionClass competition)
{
InitializeComponent();
comp = competition;
r = new Ranker(comp);
rankPicker.ItemsSource = DataConstants.rankPickerText;
}
private void saveRanking()
{
var name = PlayerPrefs.GetString(@"name");
var score = HighScore.Value;
var ranker = new Ranker(name, score);
Ranking.Save(ranker);
}
private void saveRanking()
{
var name = PlayerPrefs.GetString(PlayerPrefsKey.PlayerName);
var score = highScoreManager.Value;
var ranker = new Ranker(name, score);
rankingManager.Save(highScoreManager.RankingType, ranker);
}
private void saveRanking()
{
var name = PlayerPrefs.GetString(PlayerPrefsKey.PlayerName);
var score = highScore.Value;
var ranker = new Ranker(name, score);
ranking.Save(RankingType.StackMode, ranker);
}
public MainForm()
{
InitializeComponent();
vexDbApi = new VexDbApi();
ranker = new Ranker();
//init the table
InitRankingsTable();
}
public async Task OnTurn(ITurnContext context)
{
if (context.Activity.Type == ActivityTypes.Message)
{
var msg = context.Activity;
if (msg.Attachments?.Count > 0)
{
var http = new HttpClient();
var str = await http.GetStreamAsync(msg.Attachments[0].ContentUrl);
var cli = new OcrClient(Config.VisionApiKey, "https://westus.api.cognitive.microsoft.com/vision/v1.0/");
var res = await cli.StartOcrAsync(str);
res = await cli.GetOcrResult(res);
// await context.SendActivity(res);
// Do prediction locally
dynamic js = JsonConvert.DeserializeObject(res);
var lines = new List <string>();
foreach (dynamic x in js.recognitionResult.lines)
{
lines.Add(x.text.ToString());
}
await context.SendActivity(lines.Aggregate((a, b) => $"{a}\r\n{b}"));
lines = lines.Select(x => TextUtils.Unspacify(TextUtils.DigiTrim(x))).Where(x => x.Length >= 2).ToList();
if (lines == null || lines.Count < 1)
{
await context.SendActivity("Sorry, nothing suitable found");
}
else
{
var DR = new Ranker <string>(TextMetrics.Date);
var AR = new Ranker <string>(TextMetrics.Amount);
await context.SendActivity($"Date={DR.Top(lines)}, Amount={AR.Top(lines)}");
}
// Call flask web server
var cnt = new StringContent(res, Encoding.UTF8, "application/json");
var resp = await http.PostAsync("http://127.0.0.1:5000/api/", cnt);
res = await resp.Content.ReadAsStringAsync();
js = JsonConvert.DeserializeObject(res);
await context.SendActivity($"Date={js.Date}, Amount={js.Amount}");
}
else
{
await context.SendActivity("Please attach a photo of a receipt");
}
}
}
public void NestedCopyTest()
{
Dictionary <string, int> a = new Dictionary <string, int>();
a["1+1"] = 2;
a["2+2"] = 4;
Dictionary <string, int> b = Ranker.NestedCopy(a);
a["1+1"] = 3;
Console.WriteLine(a["1+1"]);
Console.WriteLine(b["1+1"]);
Assert.AreNotEqual(a["1+1"], b["1+1"]);
}
IEnumerator Start()
{
using (var www = new WWW("http://127.0.0.1:8000/getscore/"))
{
yield return(www);
var response = www.text;
var result = Halak.JValue.Parse(response);
Ranker[] data = new Ranker[9];
var length = 0;
for (int i = 0; i < data.Length; i++)
{
HighScore[i].text = (i + 1).ToString() + " NON 0";
}
foreach (var rankerValue in result.Array())
{
var ranker = new Ranker(
rankerValue["rank"],
rankerValue["name"],
rankerValue["score"],
rankerValue["date"]);
data[length] = new Ranker(ranker.Rank, ranker.Name, ranker.Score, ranker.Date);
length++;
}
for (int i = 0; i < length; i++)
{
for (int x = 0; x < length; x++)
{
Ranker temp = new Ranker();
if (data[x].Score < data[x + 1].Score)
{
temp = data[x];
data[x] = data[x + 1];
data[x + 1] = data[x];
}
x++;
}
}
for (int i = 0; i < length; i++)
{
HighScore[i].text = (i + 1).ToString() + " " + data[i].Name + " " + data[i].Score.ToString();
}
}
}
public async Task DoThing()
{
var clientApiFactory = new CfbApiClientFactory();
var driveResolver = new DriveResolver(clientApiFactory);
var gameResolver = new GameResolver(clientApiFactory);
var teamResolver = new TeamResolver(clientApiFactory);
var ranker = new Ranker(driveResolver, gameResolver, teamResolver);
var rank = await ranker.Rank();
Assert.True(rank.Length > 100);
}
void Update()
{
//Fill & sort the list in order
for (int i = 0; i < currentRacers; i++)
{
racerRanks[i].racer = racerStats[i].gameObject;
racerRanks[i].raceCompletion = racerStats[i].raceCompletion - ((float)racerStats[i].GetComponent <Statistics>().rank / 1000);
racerRanks[i].speedRecord = racerRanks[i].racer.GetComponent <Statistics>().speedRecord;
}
Ranker m_ranker = new Ranker();
racerRanks.Sort(m_ranker);
racerRanks.Reverse();
}
/**
* uses a filter
*/
public static void useFilter(Instances data)
{
weka.filters.supervised.attribute.AttributeSelection filter = new weka.filters.supervised.attribute.AttributeSelection();
InfoGainAttributeEval eval = new InfoGainAttributeEval();
Ranker search = new Ranker();
search.setNumToSelect(1000);
search.setThreshold(-1.7976931348623157E308d);
filter.setEvaluator(eval);
filter.setSearch(search);
filter.setInputFormat(data);
Instances newData = Filter.useFilter(data, filter);
Console.WriteLine(newData);
}
public void TestMissionControllerMissionSuccessful()
{
IMission currentMission = new Easy(10);
this.wareHouse.AddAmmunition(new AutomaticMachine("AutomaticMachine"), 4);
this.wareHouse.AddAmmunition(new AutomaticMachine("Gun"), 4);
this.wareHouse.AddAmmunition(new AutomaticMachine("Helmet"), 4);
ISoldier soldier = new Ranker("Ivan", 28, 55, 100);
this.wareHouse.EquipSoldier(soldier);
this.army.AddSoldier(soldier);
this.missionController = new MissionController(this.army, this.wareHouse);
var expectedOutput = String.Format(OutputMessages.MissionSuccessful, this.mission.Name) + "\r\n";
Assert.AreEqual(this.missionController.PerformMission(currentMission), expectedOutput);
}
public void CompleteMission()
{
IMission mission = new Medium(40);
this.wareHouse.AddAmmunitions("Gun", 10);
this.wareHouse.AddAmmunitions("AutomaticMachine", 10);
this.wareHouse.AddAmmunitions("Helmet", 10);
var soldier = new Ranker("Soldier", 30, 50, 50);
this.army.AddSoldier(soldier);
this.wareHouse.EquipArmy(army);
string message = $"Mission completed - {mission.Name}";
Assert.That(missionController.PerformMission(mission).Trim(), Is.EqualTo(message));
}
public void PerformMissionSuccessfullyIncreasesSucceededMissionsCounter()
{
var mission = new Easy(20);
this.wareHouse.AddAmmunitions("Gun", 10);
this.wareHouse.AddAmmunitions("AutomaticMachine", 10);
this.wareHouse.AddAmmunitions("Helmet", 10);
var soldier3 = new Ranker("Soldier 3", 30, 50, 50);
this.army.AddSoldier(soldier3);
this.wareHouse.EquipArmy(this.army);
var message = this.sut.PerformMission(mission).Trim();
Assert.AreEqual(1, this.sut.SuccessMissionCounter);
}
public void PerformMissionWithEnoughSoldiersReturnsCorrectMessage()
{
var mission = new Easy(20);
this.wareHouse.AddAmmunitions("Gun", 10);
this.wareHouse.AddAmmunitions("AutomaticMachine", 10);
this.wareHouse.AddAmmunitions("Helmet", 10);
var soldier3 = new Ranker("Soldier 3", 30, 50, 50);
this.army.AddSoldier(soldier3);
this.wareHouse.EquipArmy(this.army);
var message = this.sut.PerformMission(mission).Trim();
Assert.AreEqual($"Mission completed - {mission.Name}", message);
}
public void TestMissionCompleteMessage()
{
IMission mission = new Easy(20);
this.wareHouse.AddAmmunitions("Gun", 2);
this.wareHouse.AddAmmunitions("AutomaticMachine", 2);
this.wareHouse.AddAmmunitions("Helmet", 2);
ISoldier soldier = new Ranker("Ivan", 20, 50, 50);
this.army.AddSoldier(soldier);
this.wareHouse.EquipArmy(this.army);
string message = this.missionController.PerformMission(mission).Trim();
Assert.That(message, Is.EqualTo($"Mission completed - {mission.Name}"));
}
public ActionResult algoStart()
{
//Algo start
Ranker ranker = new Ranker(db);
ranker.start();
RankingDates dates = db.Dates.ToList().Last();
if (dates.FirstRankingDate == "false")
{
dates.FirstRankingDate = "true";
db.Entry(dates).State = EntityState.Modified;
db.SaveChanges();
}
else
{
if (dates.FirstRankingDate == "true" && dates.SecondRankingDate == "false")
{
dates.SecondRankingDate = "true";
db.Entry(dates).State = EntityState.Modified;
db.SaveChanges();
}
else
{
if (dates.SecondRankingDate == "true" && dates.ThirdRankingDate == "false")
{
dates.ThirdRankingDate = "true";
db.Entry(dates).State = EntityState.Modified;
db.SaveChanges();
}
}
}
//return RedirectToAction("Index", "StudentRankingInformation");
var redirectUrl = new UrlHelper(Request.RequestContext).Action("Index", "ProgrammeRankList");
return(Json(new { Url = redirectUrl }));
}
public void TestMissionControllerMissionDeclined()
{
IMission currentMission = new Easy(1000);
this.wareHouse.AddAmmunition(new AutomaticMachine("AutomaticMachine"), 4);
this.wareHouse.AddAmmunition(new AutomaticMachine("Gun"), 4);
this.wareHouse.AddAmmunition(new AutomaticMachine("Helmet"), 4);
ISoldier soldier = new Ranker("Ivan", 28, 55, 100);
this.wareHouse.EquipSoldier(soldier);
this.army.AddSoldier(soldier);
this.missionController = new MissionController(this.army, this.wareHouse);
this.missionController.PerformMission(currentMission);
this.missionController.PerformMission(currentMission);
this.missionController.PerformMission(currentMission);
var expectedOutput = String.Format(OutputMessages.MissionDeclined, this.mission.Name);
var actualOutput = this.missionController.PerformMission(currentMission).Split(new char[] { '\r', '\n' })[0];
Assert.AreEqual(actualOutput, expectedOutput);
}
private Hit ConvertDocumentIdToHit(WeightedPosting posting, ParsedQuery query)
{
string formattedDocumentId = string.Format("{0:D6}", posting.PostingId);
var document = new Document(formattedDocumentId);
var snippet = new SnippetGenerator(query, document).GenerateSnippet();
return
(new Hit
{
DocumentId = formattedDocumentId,
Description = GetDescription(posting.PostingId, string.Format("№{0}", formattedDocumentId)),
DebugInfo =
new[]
{
string.Format("tf-idf = {0:F5} = {1}", Ranker.GetTfIdf(posting), FormatTfIdf(posting)),
string.Format("ExactWordMatchCount = {0}", posting.ExactWordMatchCount)
},
Snippet = string.Join(" … ", snippet.Skip(1).ToArray()),
Title = snippet.First(),
});
}
public void CouldCalculateRankOfArray()
{
var rng = new Random();
var values = new double[50];
for (int i = 0; i < values.Length; i++)
{
values[i] = rng.NextDouble();
}
var result = default(ArraySegment <KV <double, int> >);
var sw = new Stopwatch();
var gc0 = GC.CollectionCount(0);
var gc1 = GC.CollectionCount(0);
GC.Collect(3, GCCollectionMode.Forced, true);
sw.Start();
const int loopCount = 1000000;
for (int i = 0; i < loopCount; i++)
{
result = Ranker <double> .SortRank(new ArraySegment <double>(values));
}
sw.Stop();
GC.Collect(3, GCCollectionMode.Forced, true);
gc0 = GC.CollectionCount(0) - gc0;
gc1 = GC.CollectionCount(0) - gc1;
Console.WriteLine($"Elapsed {sw.ElapsedMilliseconds} msec");
Console.WriteLine($"GC0: {gc0}");
Console.WriteLine($"GC1: {gc1}");
Console.WriteLine($"{values.Length}-sized array per msec: {(double)loopCount / (sw.ElapsedMilliseconds) }");
for (int i = 0; i < values.Length; i++)
{
Console.WriteLine($"{result.Array[i].Key} - {result.Array[i].Value}");
}
// avoid GC-ing this objects
Console.WriteLine(rng.NextDouble());
Ranker <double> .SortRank(new ArraySegment <double>(values));
}
public void JustOneTest()
{
var soldier = new Ranker("Soldier", 20, 100, 100);
this.army.AddSoldier(soldier);
this.wareHouse.AddAmmunitions(nameof(AutomaticMachine), 1000);
this.wareHouse.AddAmmunitions(nameof(Gun), 1000);
this.wareHouse.AddAmmunitions(nameof(Helmet), 1000);
var mission1 = new Hard(soldier.OverallSkill * 1000);
var mission2 = new Easy(soldier.OverallSkill * 1000);
var mission3 = new Hard(soldier.OverallSkill * 1000);
var mission4 = new Medium(soldier.OverallSkill * 1000);
var mission5 = new Medium(soldier.OverallSkill);
this.missionController.Missions.Enqueue(mission1);
this.missionController.Missions.Enqueue(mission2);
this.missionController.Missions.Enqueue(mission3);
this.missionController.Missions.Enqueue(mission4);
// Act
var result = this.missionController.PerformMission(mission5).Trim();
var expectedResult = $"Mission declined - {mission1.Name}" +
Environment.NewLine +
$"Mission on hold - {mission2.Name}" +
Environment.NewLine +
$"Mission on hold - {mission3.Name}" +
Environment.NewLine +
$"Mission on hold - {mission4.Name}" +
Environment.NewLine +
$"Mission completed - {mission5.Name}";
// Assert
Assert.AreEqual(1, this.missionController.FailedMissionCounter, "Incorrect FailedMissionsCounter!");
Assert.AreEqual(1, this.missionController.SuccessMissionCounter, "Incorrect SuccessMissionCounter!");
Assert.AreEqual(3, this.missionController.Missions.Count, "Incorrect Mission count!");
Assert.AreEqual(true, result.Trim() == expectedResult);
}
static void Main(string[] args)
{
double brierScore = 0;
Console.Write("Enter SKU: ");
string sku = Console.ReadLine();
VexDbApi vexDbApi = new VexDbApi();
Ranker ranker = new Ranker();
IList <Match> matches = vexDbApi.GetMatchesBySku(sku);
foreach (var match in matches)
{
ranker.RankMatch(match);
var winChance = ranker.CalcWinChances(match.AllianceRed, match.AllianceBlue);
//calculate brier score
int actual = match.ScoreRed > match.ScoreBlue ? 1 : 0;
brierScore += Math.Pow((winChance.Item1 / 100) - actual, 2);
}
brierScore /= matches.Count;
Console.WriteLine("Brier score is: {0}", brierScore);
Console.ReadKey(true);
}
static void Main(string[] args)
{
Ranker ranker = new Ranker();
int numberOfPages = Int32.Parse(Console.ReadLine());
Crawler crawler = new Crawler();
int i = 1;
foreach (CrawlResult result in crawler.Crawl(numberOfPages))
{
Console.WriteLine("{0:00000}: {1}", i, result.URI.ToString());
i++;
ranker.Index.NewDocument(result.URI, result.Content);
}
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();
string query;
while ((query = Console.ReadLine()) != null)
{
foreach (Uri uri in ranker.ResolveQuery(query))
{
Console.WriteLine(uri);
}
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();
}
}
//public static T[] SubArray<T>(this T[] data, int index, int length)
//{
// T[] result = new T[length];
// Array.Copy(data, index, result, 0, length);
// return result;
//}
public static void useLowLevelInformationGainFeatureSelection(Instances data)
{
AttributeSelection attsel = new AttributeSelection();
InfoGainAttributeEval eval = new InfoGainAttributeEval();
Ranker search = new Ranker();
//1000 features >0, should be equal to -1 features and threshold 0.0. anyway i use 0.01.
//int numtoselect = 1000;// 1520;
//search.setThreshold(-1.7976931348623157E308d);
int numtoselect = 0;
float threshold = 0;
if (GuiPreferences.Instance.IgSelectionType == IGType.Threshold)
{
numtoselect = -1;
threshold = Convert.ToSingle(GuiPreferences.Instance.NudIGThreshold);
GuiPreferences.Instance.setLog("Filtering using IG threshold: " + GuiPreferences.Instance.NudIGThreshold.ToString());
}
else if (GuiPreferences.Instance.IgSelectionType == IGType.Voxels)
{
//num of vox plus above 0 cut off.
numtoselect = Convert.ToInt32(GuiPreferences.Instance.NudIGVoxelAmount);
//search.setThreshold(-1.7976931348623157E308d);
GuiPreferences.Instance.setLog("Filtering using IG Voxel Amount of: " + GuiPreferences.Instance.NudIGVoxelAmount.ToString());
}
else
{
GuiPreferences.Instance.setLog("error wrong IG type");
}
search.setNumToSelect(numtoselect);
search.setThreshold(threshold);
attsel.setEvaluator(eval);
attsel.setSearch(search);
attsel.SelectAttributes(data);
//reeturned back to the global instance
Preferences.Instance.attsel = attsel;
//hIstogram saving indices and ranked to preferences for easy access
SortedDictionary <double, int> Histogram = new SortedDictionary <double, int>();
double[][] blah = attsel.rankedAttributes();
for (double i = -0.05; i < 1.2; i += 0.05)
{
if (!Histogram.ContainsKey(i))
{
Histogram.Add(i, 0);
}
for (int j = 0; j < blah.Length; j++)
{
if (blah[j][1] > i - 0.05 && blah[j][1] <= i)
{
Histogram[i] += 1;
}
}
}
GuiPreferences.Instance.setLog("Histogram:");
for (double i = -0.05; i < 1.2; i += 0.05)
{
GuiPreferences.Instance.setLog("Threshold: " + i.ToString() + ": " + Histogram[i].ToString());
}
//--------------
if (GuiPreferences.Instance.IgSelectionType == IGType.Voxels)
{
//SELECT K BIGGER THAN ZERO.
int IgVoxAboveZero = 0;
for (int j = 0; j < GuiPreferences.Instance.NudIGVoxelAmount; j++)
{
if (blah[j][1] > 0)
{
IgVoxAboveZero++;
}
}
//this is a bit redundant, to replace this redundancy, create two vectors, selected() & ranked()
search.setNumToSelect(IgVoxAboveZero);
search.setThreshold(threshold);
attsel.setEvaluator(eval);
attsel.setSearch(search);
attsel.SelectAttributes(data);
Preferences.Instance.attsel = attsel;
GuiPreferences.Instance.NudIGVoxelAmount = IgVoxAboveZero;
GuiPreferences.Instance.setLog("Filtering using IG Voxel Amount of (above zero): " + GuiPreferences.Instance.NudIGVoxelAmount.ToString());
GuiPreferences.Instance.setLog("Changing NudIGVoxelAmount to the above figure!");
}
//////////////////////////////////////////////////////////////////////////////////////////////////////
///
//NOTE: uncommenting this proves that ranked attributes are 0-based. this sorts them out from 0 to 204799 (Also threshold above should be 0
/*double[][] blah = attsel.rankedAttributes();
* double[] blat = new double[blah.Length];
* for (int i = 0; i < blah.Length;i++ )
* {
* blat[i] = blah[i][0];
* }
* Array.Sort(blat);*/
//////////////////////////////////////////////////////////////////////////////////////////////////////
//if this code is used it will need to be ammended for numtoselect which is -1 (All) in this case
/*double[][] ranked = attsel.rankedAttributes();
* int indexBiggerThanZero = 0;
* int classColumn = indices[indices.Length - 1];
* while ((ranked[indexBiggerThanZero][1] > threshold) && (indexBiggerThanZero < numtoselect))
* indexBiggerThanZero++;
*
* //return K features whose IG value is bigger than zero
* int[] indicesFinal = new int[] {};
* //less than K features, we dynamically resize an array and return it + class
* if (indexBiggerThanZero < numtoselect)
* {
* Array.Resize(ref indicesFinal, indexBiggerThanZero+1);
* Array.Copy(indices, 0, indicesFinal, 0, indexBiggerThanZero);
*
* //Array.Copy(indices, 0, indices, 0, indices.Length+1);
* indicesFinal[indicesFinal.Length - 1] = classColumn;
*
* return indicesFinal;
* }
* else
* {
* //if indexBiggerThanZero is the same, all features ig values are aboce zero, we return the original int array
* return indices;
* }*/
}
static void Main(string[] args)
{
Crawler crawler = new Crawler("Ressources\\Seeds.txt");
Console.WriteLine("Starting...");
crawler.Start(1000);
Console.WriteLine("Finished crawling...");
Console.WriteLine("Press any key to start indexing...");
Console.ReadLine();
Store fileStore = new Store();
Indexer indexer = new Indexer();
Dictionary<string, string> filesMap = fileStore.LoadFileMap();
Console.WriteLine("Indexing started");
DateTime start = DateTime.Now;
int filesIndexed = 0;
foreach (var file in filesMap)
{
string fileContent = File.ReadAllText(Path.Combine(fileStore.OutputPath, file.Value));
indexer.Index(file.Key, fileContent);
Console.SetCursorPosition(0, 8);
Console.WriteLine("Pages indexed {0} / {1}", ++filesIndexed, filesMap.Count);
}
DateTime end = DateTime.Now;
TimeSpan timeElapsed = end.Subtract(start);
double pagesPerSec = filesMap.Count / timeElapsed.TotalSeconds;
Console.WriteLine("Indexed {0} pages per second", pagesPerSec);
Console.WriteLine("Indexing finished in {0}", timeElapsed.ToString(@"hh\:mm\:ss"));
Ranker ranker = new Ranker();
Console.Write("The search engine ready for your query: ");
string searchQuery = Console.ReadLine();
Tokenizer tokenizer = new Tokenizer();
List<string> queryTerms = tokenizer.TokenizeQuery(searchQuery).ToList();
for (int i = 0; i < queryTerms.Count(); i++)
{
Stemmer stemmer = new Stemmer();
stemmer.Stem(queryTerms[i]);
queryTerms[i] = stemmer.ToString();
}
IEnumerable<KeyValuePair<int, double>> scores = ranker.Rank(queryTerms, indexer.Terms, indexer.TotalDocuments);
int resultCount = 1;
foreach (var item in scores)
{
string url = indexer.GetUrlFromHash(item.Key);
if (url == null)
continue;
Console.WriteLine("{0}: {1}", resultCount++, url);
if (resultCount > 10)
break;
}
PageRank pageRank = new PageRank();
float[] vector = pageRank.DoRank();
for (int i = 0; i < vector.Length; i++)
{
Console.Write(vector[i]);
}
Console.WriteLine();
Console.WriteLine("Press any key to exit...");
Console.ReadLine();
}
/// <summary>
/// mail constructor
/// </summary>
public MainWindow()
{
ranker = new Ranker();
searcher = new Searcher(ranker);
searcher.SearcherChanged += vSearcherChanged;
}
public void Test2()
{
List<String> urls = new List<string>();
urls.Add("http://www.autonews.com/");
urls.Add("http://www.geonius.com/www/");
urls.Add("http://en.wikipedia.org/wiki/Main_Page");
urls.Add("http://www.computerworld.com/");
List<string> seeds = StorageSystem.StorageSystem.getInstance().getSeedList(taskId);
foreach (string seed in seeds)
{
urls.Add(seed);
}
List<Category> _categories;
Constraints _constraints;
_categories = StorageSystem.StorageSystem.getInstance().getCategories(taskId);
_constraints = StorageSystem.StorageSystem.getInstance().getRestrictions(taskId);
StorageSystem.StorageSystem.getInstance().getSeedList(taskId);
Filter filter = new Filter("http://", _constraints);
Categorizer categorizer = new Categorizer(_categories);
Ranker ranker = new Ranker(categorizer);
Extractor extractor = new Extractor();
HttpResourceFetcher httpfetcher = new HttpResourceFetcher();
foreach (String url in urls)
{
DateTime startTime = DateTime.Now;
ResourceContent resource = null;
if (httpfetcher.canFetch(url))
resource = httpfetcher.fetch(url, 10000, 100);
DateTime fetchEndTime = DateTime.Now;
if ((resource == null)||(resource.getResourceContent()==null))
continue;
/*** 0. fetching the link from the internet ***/
TimeSpan fetchingTime = fetchEndTime - startTime;
List<LinkItem> listOfLinks = new List<LinkItem>();
//extract all the links in page
listOfLinks = extractor.extractLinks(resource.getResourceUrl(), resource.getResourceContent());
RuntimeStatistics.addToExtractedUrls(listOfLinks.Count);
DateTime extEndTime = DateTime.Now;
/*** 1. Extracting the link from the request ***/
TimeSpan extRequest = extEndTime - fetchEndTime;
//reset the dictionary in filter that contains the urls from the same page
filter.resetDictionary();
int filteredUrlsCount = 0;
foreach (LinkItem item in listOfLinks)
{
//Filter the links and return only links that can be crawled
List<String> links = new List<String>();
links.Add(item.getLink());
List<String> filteredLinks = filter.filterLinks(links);
//If filteredLinks is not empty
if (filteredLinks.Count > 0)
{
filteredUrlsCount++;
Url url1 = new Url(filteredLinks[0], hashUrl(filteredLinks[0]), ranker.rankUrl(resource, item),
item.getDomainUrl(), hashUrl(item.getDomainUrl()));
deployLinksToFrontier(url1);
RuntimeStatistics.addToFeedUrls(1);
}
}
DateTime catStartTime = DateTime.Now;
/*** 2. Ranking and deployment to the frontier ***/
TimeSpan rankTotalRequest = catStartTime - extEndTime;
//Ascribe the url to all the categories it is belonged to.
List<Result> classifiedResults = categorizer.classifyContent(resource.getResourceContent(),
resource.getResourceUrl());
if (classifiedResults.Count != 0) RuntimeStatistics.addToCrawledUrls(1);
DateTime catEndTime = DateTime.Now;
/*** 3. Classification of the current request ***/
TimeSpan catTotalRequest = catEndTime - catStartTime;
foreach (Result classifiedResult in classifiedResults)
{
Result result = new Result("0", classifiedResult.getUrl(), classifiedResult.getCategoryID(),
resource.getRankOfUrl(), classifiedResult.getTrustMeter());
deployResourceToStorage(result);
}
DateTime endTime = DateTime.Now;
/*** 4. deployment to the database (result) ***/
TimeSpan deployRequest = endTime - catEndTime;
/*** 5. Total processing time ***/
TimeSpan totalRequest = endTime - startTime;
}
}
private int[] ReduceByInfoGain(weka.core.Instances insts)
{
weka.attributeSelection.InfoGainAttributeEval AttribEvaluator = new weka.attributeSelection.InfoGainAttributeEval();
AttribEvaluator.buildEvaluator(insts);
Ranker search2 = new Ranker();
int[] rang = search2.search(AttribEvaluator, insts);
return rang;
}
private int[] ReduceByPCA(weka.core.Instances insts)
{
int[] rang = null;
PrincipalComponents filter = new PrincipalComponents(); // package weka.filters.supervised.attribute!
filter.setTransformBackToOriginal(true);
filter.buildEvaluator(insts);
Ranker search2 = new Ranker();
// search2.setNumToSelect(2);
search2.setGenerateRanking(true);
rang = search2.search(filter, insts);
return rang;
}
