private List <int> GetDocumentsInRange(DateTime startDate, DateTime endDate)
{
List <int> docIDs = new List <int>();
//get the ids of all valid docs
string sql = "SELECT DocumentID FROM Document \n" +
"WHERE (DocumentDate >= " + DateFormater.ConvertTimeToString(startDate) + " AND DocumentDate <= " + DateFormater.ConvertTimeToString(endDate) + ") \n" +
"ORDER BY DocumentID ASC;";
SQLiteDataReader reader = dataBase.PerformQuery(sql);
//copy the doc ids to the list
while (reader.Read())
{
docIDs.Add((int)(long)reader["DocumentID"]);
}
reader.Close();
return(docIDs);
}
public static List <GlobalData> processGlobalData(DataBase db, DateTime startDate, DateTime endDate)
{
SQLiteDataReader reader;
string getPhraseData = "SELECT * FROM PhraseLocation INNER JOIN Document on Document.DocumentID = PhraseLocation.DocumentID\n" +
"INNER JOIN Phrase on Phrase.PhraseID = PhraseLocation.PhraseID\n";
// Filter by date, if not null
if (startDate != null && endDate != null)
{
getPhraseData += "WHERE (DocumentDate >= " + DateFormater.ConvertTimeToString(startDate) + " AND DocumentDate <= " + DateFormater.ConvertTimeToString(endDate) + ")\n";
}
reader = db.PerformQuery(getPhraseData);
//store all of the phrase data
Dictionary <int, PhraseData> phrases = new Dictionary <int, PhraseData>();
//process the data
while (reader.Read())
{
int id = (int)reader["PhraseID"];
if (!phrases.ContainsKey(id))
{
PhraseData temp = new PhraseData {
DocumentID = (int)reader["DocumentID"],
PhraseID = id,
ParagraphIndex = (int)reader["ParagraphIndex"],
SentenceIndex = (int)reader["SentenceIndex"],
WordIndex = (int)reader["WordIndex"],
Phrase = reader["PhraseText"].ToString(),
Frequency = 1,
NumWordsInPhrase = (int)reader["NumWordsInPhrase"]
};
phrases.Add(id, temp);
}
else
{
phrases[id].Frequency++;
}
}
reader.Close();
Random rand = new Random();
//move the data we need to a list for sorting
List <GlobalData> globalData = new List <GlobalData>();
foreach (KeyValuePair <int, PhraseData> item in phrases)
{
string term = item.Value.Phrase;
if (term.Length > 1 && !term.Equals("lrb") && !term.Equals("rrb"))
{
globalData.Add(new GlobalData(item.Value.Phrase, item.Value.Frequency, rand.Next(10, 1000)));
}
}
globalData.Sort();
List <GlobalData> top = new List <GlobalData>(500);
int index = 0;
foreach (GlobalData item in globalData)
{
if (index == 500)
{
break;
}
index++;
top.Add(item);
}
return(top);
}