public ThreadViewer(FaceLog log, ChatThread chat)
{
this.log = log;
this.chat = chat;
InitializeComponent();
for (int i = 0; i < chat.messages.Count; i++)
{
if (i >= 100)
{
break;
}
Message msg = chat.messages[i];
Label labelMsg = new Label();
labelMsg.Content = msg.author.ToString() + " : " + msg.text;
chatPanel.Children.Add(labelMsg);
}
TimelineDataController dataController = new TimelineDataController(chat.messages, log.beginScope, log.endScope, log.owner);
timeViewer = new TimelineViewer(chat.messages, log.beginScope, log.endScope, log.owner, dataController);
timelineControl.Content = timeViewer;
}
public LogViewer(FaceLog log)
{
this.log = log;
InitializeComponent();
selfLabel.Content = log.owner.name;
SolidColorBrush threadColor = new SolidColorBrush(Colors.LightCyan);
for (int i = 0; i < log.threads.Count; i++)
{
ChatThread thread = log.threads[i];
if (thread.messages.Count >= 100 || (thread.people.Count == 3 && thread.messages.Count >= 50))
{
Button threadBut = new Button();
string names = "";
for (int k = 0; k < thread.people.Count; k++)
{
if (thread.people[k] != log.owner)
{
names += thread.people[k] + "\n";
}
}
names = names.Substring(0, names.Length - 1);
threadBut.Content = names;
threadBut.DataContext = i;
threadBut.Background = threadColor;
threadBut.Click += ThreadBut_Click;
threadBut.Margin = new Thickness(1);
peoplePanel.Children.Add(threadBut);
}
}
dataController = new TimelineDataController(log.allMessages, log.beginScope, log.endScope, log.owner);
timeViewer = new TimelineViewer(log.allMessages, log.beginScope, log.endScope, log.owner, dataController);
timelineControl.Content = timeViewer;
Timer updateTicker = new Timer();
updateTicker.Interval = 50;
updateTicker.AutoReset = true;
updateTicker.Elapsed += UpdateTicker_Elapsed;;
updateTicker.Start();
}
private void FinalizeLoad()
{
//Create full message log
allMessages = new List <Message>();
for (int i = 0; i < threads.Count; i++)
{
ChatThread thread = threads[i];
for (int k = 0; k < thread.messages.Count; k++)
{
allMessages.Add(thread.messages[k]);
}
}
allMessages.Sort(Comparer <Message> .Create((m1, m2) => m1.time.Ticks.CompareTo(m2.time.Ticks)));
beginScope = allMessages[0].time;
endScope = allMessages[allMessages.Count - 1].time;
}
private void ComputeLexic()
{
ownerLexic = new Dictionary <string, int>();
otherLexic = new Dictionary <string, int>();
for (int i = 0; i < threads.Count; i++)
{
ChatThread chat = threads[i];
for (int k = 0; k < chat.messages.Count; k++)
{
Message msg = chat.messages[k];
string[] words = msg.text.Split(' ');
for (int w = 0; w < words.Length; w++)
{
string word = words[w];
if (msg.author == owner)
{
if (ownerLexic.ContainsKey(word))
{
ownerLexic[word]++;
}
else
{
ownerLexic.Add(word, 1);
}
}
else
{
if (otherLexic.ContainsKey(word))
{
otherLexic[word]++;
}
else
{
otherLexic.Add(word, 1);
}
}
}
}
}
}
private void ThreadBut_Click(object sender, RoutedEventArgs e)
{
ChatThread threadToView = log.threads[(int)(sender as Button).DataContext];
MainWindow.current.pushControl(new ThreadViewer(log, threadToView));
}
private void obfuscate()
{
if (obfuscatePeople)
{
for (int i = 0; i < people.Count; i++)
{
if (people[i] != owner)
{
people[i].name = "Person " + i;
}
}
}
if (obfuscateWords)
{
List <WordPresence> words = new List <WordPresence>();
foreach (KeyValuePair <string, int> node in ownerLexic)
{
words.Add(new WordPresence(node.Key, node.Value));
}
words.Sort((x, y) => x.count.CompareTo(y.count));
int removeMost = 0;
int pool = 140;
while (words.Count > pool && removeMost < 40)
{
words.RemoveAt(0);
removeMost++;
}
if (words.Count < pool)
{
pool = words.Count;
}
Random rdm = new Random(0);
Dictionary <string, string> obfuscator = new Dictionary <string, string>();
foreach (KeyValuePair <string, int> node in ownerLexic)
{
//obfuscator.Add(node.Key, words[rdm.Next(pool)].word);
obfuscator.Add(node.Key, "word " + rdm.Next(pool));
}
foreach (KeyValuePair <string, int> node in otherLexic)
{
if (!obfuscator.ContainsKey(node.Key))
{
//obfuscator.Add(node.Key, words[rdm.Next(pool)].word);
obfuscator.Add(node.Key, "word " + rdm.Next(pool));
}
}
for (int i = 0; i < threads.Count; i++)
{
ChatThread tt = threads[i];
for (int m = 0; m < tt.messages.Count; m++)
{
Message msg = tt.messages[m];
string[] tokens = msg.text.Split(' ');
msg.text = "";
for (int k = 0; k < tokens.Length; k++)
{
msg.text += obfuscator[tokens[k]] + " ";
}
}
}
}
}
private void GenerateMultivariateData()
{
List <DataRow> data = new List <DataRow>();
for (int i = 0; i < threads.Count; i++)
{
ChatThread thread = threads[i];
//Skip group threads
if (thread.people.Count != 2)
{
continue;
}
//get other person
Person other = (thread.people[0] == owner) ? thread.people[1] : thread.people[0];
DataRow row = new DataRow();
row.selfData = new PersonMetrics();
row.otherData = new PersonMetrics();
for (int k = 0; k < thread.messages.Count; k++)
{
Message msg = thread.messages[k];
//Check who's message it is
if (msg.author == owner)
{
row.selfData.charCount += (int)msg.measures[(int)Message.MeasureType.CharCount];
row.selfData.wordCount += (int)msg.measures[(int)Message.MeasureType.WordCount];
row.selfData.messageCount += 1;
row.selfData.interrogationCount += (int)msg.measures[(int)Message.MeasureType.InterrogationPresence];
row.selfData.exclamationCount += (int)msg.measures[(int)Message.MeasureType.ExclamationPresence];
row.selfData.reticencesCount += (int)msg.measures[(int)Message.MeasureType.ReticencePresence];
row.selfData.commaCount += (int)msg.measures[(int)Message.MeasureType.CommaPresence];
row.selfData.linkCount += (int)msg.measures[(int)Message.MeasureType.LinkPresence];
row.selfData.euCount += (msg.text.Contains("Eu ") || msg.text.Contains(" eu ")) ? 1 : 0;
row.selfData.tuCount += (msg.text.Contains("Tu ") || msg.text.Contains(" tu ")) ? 1 : 0;
}
else //other
{
row.otherData.charCount += (int)msg.measures[(int)Message.MeasureType.CharCount];
row.otherData.wordCount += (int)msg.measures[(int)Message.MeasureType.WordCount];
row.otherData.messageCount += 1;
row.otherData.interrogationCount += (int)msg.measures[(int)Message.MeasureType.InterrogationPresence];
row.otherData.exclamationCount += (int)msg.measures[(int)Message.MeasureType.ExclamationPresence];
row.otherData.reticencesCount += (int)msg.measures[(int)Message.MeasureType.ReticencePresence];
row.otherData.commaCount += (int)msg.measures[(int)Message.MeasureType.CommaPresence];
row.otherData.linkCount += (int)msg.measures[(int)Message.MeasureType.LinkPresence];
row.otherData.euCount += (msg.text.Contains("Eu ") || msg.text.Contains(" eu ")) ? 1 : 0;
row.otherData.tuCount += (msg.text.Contains("Tu ") || msg.text.Contains(" tu ")) ? 1 : 0;
}
}
data.Add(row);
}
data.Sort(Comparer <DataRow> .Create((d1, d2) => (d2.selfData.charCount + d2.otherData.charCount).CompareTo(d1.selfData.charCount + d1.otherData.charCount)));
using (StreamWriter file =
new StreamWriter("multivariateData.txt"))
{
//file.WriteLine("Rank " + DataRow.getHeader());
file.WriteLine(DataRow.getHeader());
for (int i = 0; i < data.Count; i++)
{
//file.WriteLine((i + 1) + " " + data[i].ToString());
file.WriteLine(data[i].ToString());
}
}
}
private void loadFromHtml()
{
people = new List <Person>();
threads = new List <ChatThread>();
//Parse HTML
Console.WriteLine("Parsing Html.");
HtmlDocument htmldoc = new HtmlDocument();
htmldoc.Load(location + "messages.htm");
Console.WriteLine("Html Parsed.");
//Find own name
Console.WriteLine("Finding owner name, #TODO, locked at Ruben Filipe.");
owner = new Person("Ruben Filipe");
people.Add(owner);
//Select threads
Console.WriteLine("Processing threads.");
HtmlNodeCollection htmlThreads = htmldoc.DocumentNode.SelectNodes("//*[@class='thread']");
//Process threads
int totalMessages = 0;
int unfiguredAuthorCount = 0;
for (int i = 0; i < htmlThreads.Count; i++)
{
//Console.WriteLine("Processing thread: " + i);
HtmlNode thread = htmlThreads[i];
//Get people in thread
String names = thread.FirstChild.OuterHtml;
String[] nameList = names.Split(',');
//Process other participants
List <Person> participants = new List <Person>();
for (int k = 0; k < nameList.Length; k++)
{
Person participant = null;
nameList[k] = removeOuterSpaces(nameList[k]);
if (!people.Exists(x => x.name == nameList[k]))
{
//Update People library
participant = new Person(nameList[k]);
people.Add(participant);
Console.WriteLine("Added to people Library: " + participant.name);
}
participants.Add(people.Find(x => x.name == nameList[k]));
}
if (!participants.Contains(owner))
{
//throw new FormatException("Invalid thread, owner not present.");
Console.Write("Thread found without owner, with participants:");
for (int k = 0; k < participants.Count; k++)
{
Console.Write(" '" + participants[k].name + "' ");
}
Console.WriteLine("");
continue;
}
//Create ChatThread
ChatThread chat = findThreadWith(participants);
if (chat == null)
{
chat = new ChatThread(participants);
threads.Add(chat);
}
//Console.WriteLine("Begin processing messages.");
int nodeCursor = 1; //index 0 is namelist
while (nodeCursor < thread.ChildNodes.Count)
{
//Console.WriteLine("Processing cursor at: " + nodeCursor);
//Process (Message Header, Text) node pair
//Check if there is a message header on cursor
if (!thread.ChildNodes[nodeCursor].HasAttributes)
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!thread.ChildNodes[nodeCursor].Attributes.Contains("class"))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!thread.ChildNodes[nodeCursor].HasClass("message"))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!(thread.ChildNodes[nodeCursor].FirstChild != null))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!(thread.ChildNodes[nodeCursor].FirstChild.HasAttributes))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!(thread.ChildNodes[nodeCursor].FirstChild.Attributes.Contains("class")))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!thread.ChildNodes[nodeCursor].FirstChild.HasClass("message_header"))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!(thread.ChildNodes[nodeCursor].FirstChild.ChildNodes.Count == 2))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
//Check if there is a paragraph after the header
if (nodeCursor + 1 >= thread.ChildNodes.Count)
{
throw new FormatException("Invalid node sructure in thread. Expected paragraph, found nothing.");
}
//Retrieve information
string author = thread.ChildNodes[nodeCursor].FirstChild.ChildNodes[0].InnerText;
DateTime time = processTimeFormat(thread.ChildNodes[nodeCursor].FirstChild.ChildNodes[1].InnerText);
string text = thread.ChildNodes[nodeCursor + 1].InnerText;
Person pAuthor = people.Find(x => x.name == author);
if (!participants.Contains(pAuthor))
{
//throw new FormatException("Invalid message found, on thread " + i + " : '" + pAuthor.name + "' shouldn't figure in this thread.");
unfiguredAuthorCount++;
nodeCursor += 2;
continue;
}
Message msg = new Message(pAuthor, text, time);
chat.messages.Add(msg);
totalMessages++;
nodeCursor += 2;
}
//Console.WriteLine("Messages: " + chat.messages.Count);
}
Console.WriteLine("-----------------");
Console.WriteLine("Log processed.");
Console.WriteLine("Total threads count: " + threads.Count);
Console.WriteLine("Total message count: " + totalMessages);
Console.WriteLine("Unfigured Authors message count: " + unfiguredAuthorCount);
threads.Sort(Comparer <ChatThread> .Create((c1, c2) => c2.messages.Count.CompareTo(c1.messages.Count)));
for (int k = 0; k < threads.Count; k++)
{
threads[k].people.Sort(Comparer <Person> .Create((p1, p2) => p1.name.CompareTo(p2.name)));
threads[k].messages.Sort(Comparer <Message> .Create((m1, m2) => m1.time.Ticks.CompareTo(m2.time.Ticks)));
if (LoadPrints)
{
Console.Write(("" + (k + 1)).PadRight(4) + " : " + ("" + threads[k].messages.Count).PadRight(10) + " :: ");
for (int j = 0; j < threads[k].people.Count; j++)
{
Console.Write(" '" + threads[k].people[j].name + "' ");
}
Console.WriteLine();
}
}
FinalizeLoad();
}
private void loadFromHtmlv2()
{
people = new List <Person>();
threads = new List <ChatThread>();
//Find owner name
Console.WriteLine("Finding owner name");
ParseOwner();
Console.WriteLine("Owner:" + owner.name);
//owner = new Person("Ruben Filipe");
//people.Add(owner);
//Get message files
if (!Directory.Exists(location + "\\messages"))
{
throw new Exception("Invalid Log directory");
}
string[] threadFiles = Directory.GetFiles(location + "\\messages");
threadFileCount = threadFiles.Length;
//Parse HTML
Console.WriteLine("Parsing Html.");
List <HtmlNodeCollection> allHtmlThreads = new List <HtmlNodeCollection>();
for (int i = 0; i < threadFiles.Length; i++)
{
HtmlDocument htmldoc = new HtmlDocument();
try
{
htmldoc.DetectEncodingAndLoad(threadFiles[i], true);
threadsProcessed++;
}
catch (Exception e)
{
Console.WriteLine("Failed to parse html thread file: '" + threadFiles[i] + "'");
continue;
}
allHtmlThreads.Add(htmldoc.DocumentNode.SelectNodes("//*[@class='thread']"));
}
//Process threads
int totalMessages = 0;
int unfiguredAuthorCount = 0;
int emptyParticipantCount = 0;
int untreatableThreadsCount = 0;
string startString = "Conversation with ";
for (int j = 0; j < allHtmlThreads.Count; j++)
{
HtmlNodeCollection htmlThreads = allHtmlThreads[j];
for (int i = 0; i < htmlThreads.Count; i++)
{
try
{
//Console.WriteLine("Processing thread: " + i);
HtmlNode thread = htmlThreads[i];
//Get people in thread
string header = thread.FirstChild.OuterHtml;
if (header[0] == '\n')
{
header = header.Substring(1);
}
//Remove startString from people
string names = header.Substring(startString.Length);
if (names == "")
{
Console.WriteLine("Found empty participant Thread.");
emptyParticipantCount++;
continue;
}
if (LoadPrints)
{
Console.WriteLine("Starting thread with: " + names);
}
String[] nameList = names.Split(',');
//Process other participants
List <Person> participants = new List <Person>();
for (int k = 0; k < nameList.Length; k++)
{
Person participant = null;
nameList[k] = removeOuterSpaces(nameList[k]);
if (!people.Exists(x => x.name == nameList[k]))
{
//Update People library
participant = new Person(nameList[k]);
people.Add(participant);
if (LoadPrints)
{
Console.WriteLine("Added to people Library: " + participant.name);
}
}
participants.Add(people.Find(x => x.name == nameList[k]));
}
//Deprecated, only happenned in older version
if (!participants.Contains(owner) && false)
{
//throw new FormatException("Invalid thread, owner not present.");
Console.Write("Thread found without owner, with participants:");
for (int k = 0; k < participants.Count; k++)
{
Console.Write(" '" + participants[k].name + "' ");
}
Console.WriteLine("");
continue;
}
//Force Add
participants.Add(owner);
//Create ChatThread
ChatThread chat = findThreadWith(participants);
if (chat == null)
{
chat = new ChatThread(participants);
threads.Add(chat);
}
//Console.WriteLine("Begin processing messages.");
//TODO better option find first message node
int nodeCursor = 1; //index 0 is namelist
while (nodeCursor < thread.ChildNodes.Count)
{
HtmlNode node = thread.ChildNodes[nodeCursor];
//Console.WriteLine("Processing cursor at: " + nodeCursor);
//Process (Message Header, Text) node pair
//Check if there is a message header on cursor
if (!node.HasAttributes)
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!node.Attributes.Contains("class"))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!node.HasClass("message"))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!(node.FirstChild != null))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!(node.FirstChild.HasAttributes))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!(node.FirstChild.Attributes.Contains("class")))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!node.FirstChild.HasClass("message_header"))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
if (!(node.FirstChild.ChildNodes.Count == 2))
{
throw new FormatException("Invalid node structure in threads. Expected message header , found: " + thread.ChildNodes[nodeCursor].OuterHtml);
}
//Check if there is a paragraph after the header
if (nodeCursor + 1 >= thread.ChildNodes.Count)
{
throw new FormatException("Invalid node sructure in thread. Expected paragraph, found nothing.");
}
//Retrieve information
string author = thread.ChildNodes[nodeCursor].FirstChild.ChildNodes[0].InnerText;
DateTime time = processTimeFormat(thread.ChildNodes[nodeCursor].FirstChild.ChildNodes[1].InnerText);
string text = thread.ChildNodes[nodeCursor + 1].InnerText;
Person pAuthor = people.Find(x => x.name == author);
if (!participants.Contains(pAuthor))
{
//throw new FormatException("Invalid message found, on thread " + i + " : '" + pAuthor.name + "' shouldn't figure in this thread.");
unfiguredAuthorCount++;
nodeCursor += 2;
continue;
}
Message msg = new Message(pAuthor, text, time);
chat.messages.Add(msg);
totalMessages++;
nodeCursor += 2;
}
//Console.WriteLine("Messages: " + chat.messages.Count);
}
catch (Exception e)
{
untreatableThreadsCount++;
}
}
}
Console.WriteLine("-----------------");
Console.WriteLine("Log processed.");
Console.WriteLine("Total threads count: " + threads.Count);
Console.WriteLine("Total message count: " + totalMessages);
Console.WriteLine("Unfigured Authors message count: " + unfiguredAuthorCount);
Console.WriteLine("Empty participant threads count: " + emptyParticipantCount);
threads.Sort(Comparer <ChatThread> .Create((c1, c2) => c2.messages.Count.CompareTo(c1.messages.Count)));
for (int k = 0; k < threads.Count; k++)
{
threads[k].people.Sort(Comparer <Person> .Create((p1, p2) => p1.name.CompareTo(p2.name)));
threads[k].messages.Sort(Comparer <Message> .Create((m1, m2) => m1.time.Ticks.CompareTo(m2.time.Ticks)));
if (LoadPrints)
{
Console.Write(("" + (k + 1)).PadRight(4) + " : " + ("" + threads[k].messages.Count).PadRight(10) + " :: ");
for (int j = 0; j < threads[k].people.Count; j++)
{
Console.Write(" '" + threads[k].people[j].name + "' ");
}
Console.WriteLine();
}
}
FinalizeLoad();
//GenerateMultivariateData();
AppData.data.AddLog(location);
ComputeLexic();
obfuscate();
loadFinished = true;
}
