DictionaryStruct ConvertDictionary(object obj)
{
DictionaryStruct dict = new DictionaryStruct();
Type t = obj.GetType();
dict.Type = GetTypeRef(t);
ICollection keys = ((ICollection)t.GetProperty("Keys").GetValue(obj));
PropertyInfo indexer = t.GetProperty("Item");
//sometimes the indexer is not named "Item", we need to find it manually in that case
if (indexer == null)
{
foreach (var property in t.GetProperties(BindingFlags.Public | BindingFlags.Instance))
{
if (property.GetIndexParameters().Length == 1)
{
indexer = property;
break;
}
}
}
dict.Keys = new ValueStruct[keys.Count];
dict.Values = new ValueStruct[keys.Count];
int i = 0;
foreach (var key in keys)
{
dict.Keys[i] = Convert(key);
dict.Values[i] = Convert(indexer.GetValue(obj, new[] { key }));
i++;
}
TypeStruct ts = types.GetValue(dict.Type);
dict.MemberValues = new ValueStruct[ts.Members.Length];
int slot = 0;
foreach (var member in ts.Members)
{
string name = strings.GetValue(member);
var p = t.GetProperty(name);
if (p != null)
{
dict.MemberValues[slot] = Convert(p.GetValue(obj));
}
else
{
dict.MemberValues[slot] = Convert(t.GetField(name).GetValue(obj));
}
slot++;
}
return(dict);
}
void WriteDictionary(DictionaryStruct dict)
{
WriteVInt(dict.Type);
foreach (var m in dict.MemberValues)
{
WriteValue(m);
}
WriteVInt((uint)dict.Keys.Length);
foreach (var m in dict.Keys)
{
WriteValue(m);
}
foreach (var m in dict.Values)
{
WriteValue(m);
}
}
public void ParseComments(dynamic commentsStruct, string story_id)
{
if (commentsStruct.data == null)
{
return;
}
foreach (dynamic comment in commentsStruct.data)
{
Dictionary<string, int> documentfrequency = new Dictionary<string, int>();
bool seen = false;
string toParse = comment.message;
string refined = toParse.Replace("\'", "");
refined = Regex.Replace(refined, @"[^\w\s]", " ");
refined = Regex.Replace(refined, @"\s+", " ");
refined = Regex.Replace(refined, @"\d", "");
while (refined != "" && refined[0] == ' ')
{
refined = refined.Substring(1);
}
if (refined != null && !referenceStoryDictionary.ContainsKey(comment.id.ToString()) && !referenceCommentDictionary.ContainsKey(comment.id.ToString()))
{
referenceCommentDictionary.Add(comment.id.ToString(), new CommentStruct() { comment = toParse.Replace("\'", "\'\'").Replace("\\", "\\\\").Replace(",", "\\,"), storyID = story_id, owner = comment.from.id });
}
if (referenceStoryDictionary.ContainsKey(comment.id.ToString()))
{
continue;
}
string currentWord = null;
int i = 0;
while (i < refined.Length)
{
if (refined.Contains(' '))
{
while (refined[i] != ' ')
{
currentWord += refined[i];
i++;
if (i >= refined.Length)
{
break;
}
}
}
else
{
currentWord = refined;
i += refined.Length;
}
string expansion;
if (refined != null && currentWord != null)
{
expansion = currentWord.ToLower();
expansion = new EnglishStemmer.EnglishWord(expansion).Stem;
}
else
{
continue;
}
if (expansion == null)
{
continue;
}
if (documentfrequency.ContainsKey(expansion))
{
seen = true;
}
if (!ReferenceDictionary.ContainsKey(expansion))
{
ReferenceDictionary.Add(expansion, new DictionaryStruct() { word_id = word_id, frequency = 1 });
Dictionary.Add(expansion, new DictionaryStruct() { word_id = word_id, frequency = 1 });
word_id++;
}
else if (!seen)
{
DictionaryStruct temp = ReferenceDictionary[expansion];
if (!Dictionary.ContainsKey(expansion))
{
Dictionary.Add(expansion, new DictionaryStruct() { frequency = 1, word_id = word_id });
word_id++;
}
else
{
Dictionary[expansion] = new DictionaryStruct() { frequency = temp.frequency + 1, word_id = Dictionary[expansion].word_id };
}
}
if (!documentfrequency.ContainsKey(expansion))
{
documentfrequency.Add(expansion, 1);
}
else
{
documentfrequency[expansion]++;
}
i++;
currentWord = "";
}
foreach (KeyValuePair<string, int> kvp in documentfrequency)
{
postings.Add(new KeyValuePair<int, WordStructure>(ReferenceDictionary[kvp.Key].word_id, new WordStructure(story_id, ReferenceDictionary[kvp.Key].word_id, kvp.Value)));
}
}
}
public void Parse(string toParse, string story_id, string owner, string likes)
{
Dictionary<string, int> documentfrequency = new Dictionary<string, int>();
string refined = toParse.Replace("\'", "");
refined = Regex.Replace(refined, @"[^\w\s]", " ");
refined = Regex.Replace(refined, @"\s+", " ");
refined = Regex.Replace(refined, @"\d", "");
while (refined != "" && refined[0] == ' ')
{
refined = refined.Substring(1);
}
if (refined != null && !referenceStoryDictionary.ContainsKey(story_id) && !storyDictionary.ContainsKey(story_id))
{
referenceStoryDictionary.Add(story_id, new StoryStruct() { story = toParse.Replace("\'","\'\'"), owner = owner, likes = Convert.ToInt32(likes) });
storyDictionary.Add(story_id, new StoryStruct() { story = toParse.Replace("\'", "\'\'"), owner = owner, likes = Convert.ToInt32(likes) });
}
string currentWord = null;
bool seen = false;
int i = 0;
while (i < refined.Length)
{
if (refined.Contains(' '))
{
while (refined[i] != ' ')
{
currentWord += refined[i];
i++;
if (i >= refined.Length)
{
break;
}
}
}
else
{
currentWord = refined;
i += refined.Length;
}
string expansion;
if (refined != null && currentWord != null)
{
expansion = currentWord.ToLower();
expansion = new EnglishStemmer.EnglishWord(expansion).Stem;
}
else
{
continue;
}
if (expansion == null)
{
continue;
}
if (stopwords.Contains<string>(expansion))
{
i += expansion.Length;
continue;
}
if(documentfrequency.ContainsKey(expansion))
{
seen = true;
}
if (!ReferenceDictionary.ContainsKey(expansion))
{
ReferenceDictionary.Add(expansion, new DictionaryStruct() { word_id = word_id, frequency = 1 });
Dictionary.Add(expansion, new DictionaryStruct() { word_id = word_id, frequency = 1 });
word_id++;
}
else if(!seen)
{
DictionaryStruct temp = ReferenceDictionary[expansion];
if (!Dictionary.ContainsKey(expansion))
{
Dictionary.Add(expansion, new DictionaryStruct() { frequency = 1, word_id = word_id });
word_id++;
}
else
{
Dictionary[expansion] = new DictionaryStruct() { frequency = temp.frequency + 1, word_id = Dictionary[expansion].word_id };
}
}
if (!documentfrequency.ContainsKey(expansion))
{
documentfrequency.Add(expansion, 1);
}
else
{
documentfrequency[expansion]++;
}
i++;
currentWord = "";
}
foreach (KeyValuePair<string, int> kvp in documentfrequency)
{
postings.Add(new KeyValuePair<int, WordStructure>(ReferenceDictionary[kvp.Key].word_id, new WordStructure(story_id, ReferenceDictionary[kvp.Key].word_id, kvp.Value)));
}
}
