/// <summary> /// Creates a matrix of cycle activity for each student /// </summary> /// <param name="vm"></param> private void OrderTransitionsByDate(TimelineAnalysisViewModel vm) { //step 1: get list of files to process List <string> filesToProcess = new List <string>(); string fileName = "a"; Console.WriteLine("Enter files to process (-1 to stop)"); while ((fileName = GetFile()).Length > 0) { filesToProcess.Add(fileName); } //step 2: setup grade-bands (e.g. A, B, C, etc.) Hard coded for now as this is just for a //single class double maxScore = 200; double[] gradeRanges = { 90, 78, 69, 60, 0 }; string[] gradeMap = { "A", "B", "C", "D", "F" }; //this produces a lot of files, so create a separate directory for the output string outputDirectory = "TransitionsByDate"; if (Directory.Exists(outputDirectory) == false) { Directory.CreateDirectory(outputDirectory); } //finally, begin processing //reset max score for A students maxScore = 200; //based on currently existing code, it is easier to reopen the file for //each grade range for (int i = 0; i < gradeRanges.Length; i++) { double bound = gradeRanges[i]; //reload the files LoadFile(filesToProcess[0]); for (int j = 1; j < filesToProcess.Count; j++) { vm.AppendTimeline(filesToProcess[j]); } //get grade data vm.AttachGrades(); //filter based on grade data vm.FilterByGrade("Assignment AVG", bound, maxScore); //get transitions for this grade level var result = vm.OrderTransitionsByDate(); var byDate = result.Item1; //update scores for next grade boundary maxScore = bound - 0.01; //hold all day keys for easier access int[] keys = byDate.Keys.OrderBy(k => k).ToArray(); //figure out all transitions Dictionary <string, string> transitionsDict = new Dictionary <string, string>(); foreach (int key in keys) { foreach (string transition in byDate[key].Keys) { transitionsDict[transition] = transition; } } string[] transitions = transitionsDict.Keys.ToArray(); //write aggregate information to a file CsvWriter writer = new CsvWriter(); //blank line for transitions writer.AddToCurrentLine("Transition"); //add in header row foreach (int key in keys) { writer.AddToCurrentLine(key); } writer.CreateNewRow(); //add in data foreach (string transition in transitions) { //data for given transition writer.AddToCurrentLine("T: " + transition); foreach (int key in keys) { if (byDate[key].ContainsKey(transition)) { //add in data for given transition writer.AddToCurrentLine(byDate[key][transition].Count); } else { //no data, add a 0 writer.AddToCurrentLine(0); } } writer.CreateNewRow(); } //aggregate class results using (TextWriter tw = File.CreateText(string.Format("{0}/aggregate_{1}.csv", outputDirectory, gradeMap[i]))) { tw.Write(writer.ToString()); Console.WriteLine("Created file aggregate_{0}.csv", gradeMap[i]); } //write individual student information to file writer = new CsvWriter(); writer.AddToCurrentLine("UserID"); //blank line for transitions writer.AddToCurrentLine("Transition"); //add in header row foreach (int key in keys) { writer.AddToCurrentLine(key); } writer.CreateNewRow(); //add in data var userData = result.Item2; foreach (int userId in userData.Keys) { foreach (string transition in transitions) { //user id writer.AddToCurrentLine(userId); //data for given transition writer.AddToCurrentLine("T: " + transition); foreach (int key in keys) { if (userData[userId][key].ContainsKey(transition)) { //add in data for given transition writer.AddToCurrentLine(userData[userId][key][transition].Count); } else { //no data, add a 0 writer.AddToCurrentLine(0); } } writer.CreateNewRow(); } } using (TextWriter tw = File.CreateText(string.Format("{0}/students_{1}.csv", outputDirectory, gradeMap[i]))) { tw.Write(writer.ToString()); Console.WriteLine("Created file students_{0}.csv", gradeMap[i]); } } }
/// <summary> /// Similar to <see cref="BuildTransitionFrequencyCounts"/>, but instead creates a single file /// per transition length for all students and all loaded files. /// </summary> /// <param name="vm"></param> private void AggregateTransitionFrequencyCounts(TimelineAnalysisViewModel vm) { //step 1: get list of files to process List <string> filesToProcess = new List <string>(); string fileName = "a"; Console.WriteLine("Enter files to process (-1 to stop)"); while ((fileName = GetFile()).Length > 0) { filesToProcess.Add(fileName); } //load all data into VM vm.LoadTimeline(filesToProcess[0]); for (int i = 1; i < filesToProcess.Count; i++) { vm.AppendTimeline(filesToProcess[i]); } //step2: get sequence boundaries. Again, hard coded for now int startingSequenceLength = 2; int endingSequenceLength = 25; //this produces a lot of files, so create a separate directory for the output string outputDirectory = "AggregateTransitionFrequencyCounts"; if (Directory.Exists(outputDirectory) == false) { Directory.CreateDirectory(outputDirectory); } /* * What I need to do: * Get all sequences. * For each sequence: * Determine if similar to other known sequences. If so, combine into same set. (disjoint set?) * */ Dictionary <int, Dictionary <string, int> > allTransitions = new Dictionary <int, Dictionary <string, int> >(); //begin file processing for (int sequenceLength = startingSequenceLength; sequenceLength <= endingSequenceLength; sequenceLength++) { //get grade data vm.AttachGrades(); //build markov transitions vm.BuildDefaultMarkovStates(); //figure out sequence distribution for entire data set and for individual students Dictionary <string, int> transitions = vm.GetAllTransitionCombinations(sequenceLength); //filter out singletons var smallKeys = transitions.Where(t => t.Value < 5).Select(t => t.Key).ToList(); foreach (string key in smallKeys) { transitions.Remove(key); } //save for future use allTransitions.Add(sequenceLength, transitions); Console.WriteLine("Loaded transitions of length {0}.", sequenceLength); } //use Needleman-Wunsch algorithm and disjoint sets to combine similar sequences DisjointSet <string> matches = new DisjointSet <string>(); int matchCount = 0; //start with large sequences as it will make it more likely that these will be the "top" of the disjoint set int startingNumber = (int)'a'; for (int sequenceLength = endingSequenceLength; sequenceLength >= startingSequenceLength; sequenceLength--) { Console.WriteLine("Matching sequences of length {0}", sequenceLength); //Needleman-Wunsch works on single characters, so we need to transform Markov-like numbers to letters Dictionary <string, int> originalSequences = allTransitions[sequenceLength]; Dictionary <string, int> modifiedSequences = new Dictionary <string, int>(); foreach (var kvp in originalSequences) { //convert into numbers int[] pieces = kvp.Key.Split('_').Select(k => Convert.ToInt32(k) + startingNumber).ToArray(); //then, convert back to characters char[] sequence = pieces.Select(p => Convert.ToChar(p)).ToArray(); //and finally into a string string charSequence = string.Join("_", sequence); //lastly, remember this sequence modifiedSequences.Add(charSequence, kvp.Value); } //prime the disjoint set foreach (string key in modifiedSequences.Keys) { matches.Find(key); } //having converted to character state representations, now run the Needleman-Wunsch algorithm List <string> sequences = modifiedSequences.Keys.ToList(); for (int i = 0; i < sequences.Count; i++) { for (int j = i + 1; j < sequences.Count; j++) { string first = matches.Find(sequences[i]); string second = matches.Find(sequences[j]); //automatically count sequences as the same when one sequence is a complete substring of another sequence string firstSequence = sequences[i]; string secondSequence = sequences[j]; if (firstSequence.Replace(secondSequence, "").Length == 0 || secondSequence.Replace(firstSequence, "").Length == 0 ) { matches.UnionWith(first, second); matchCount++; } else { //Use NW to check for alignment //align the two sequences var result = NeedlemanWunsch.Align(first, second); //if score is similar, then count the sequences as the same (union) if ((double)NeedlemanWunsch.ScoreNpsmSequence(result.Item1, result.Item2) < 3) { matches.UnionWith(first, second); matchCount++; } } } } } //now, get all sets and figure out popularity of each set Console.WriteLine("{0} unions performed.", matchCount); List <List <string> > allSets = matches.AllSets(); List <List <string> > smallerSets = allSets.Where(s => s.Count > 1).ToList(); Dictionary <string, int> popularityDict = new Dictionary <string, int>(); Console.WriteLine("Calculating popularity of {0} sets...", allSets.Count); foreach (List <string> set in allSets) { foreach (string item in set) { //convert back to Markov-style transitions int[] pieces = item.Split('_').Select(c => Convert.ToChar(c)).Select(c => (int)c - startingNumber).ToArray(); string key = string.Join("_", pieces); if (popularityDict.ContainsKey(key) == false) { popularityDict.Add(key, 0); } //add in counts to the popularity dictionary popularityDict[key] += allTransitions[pieces.Length][key]; } } //write this information to a file CsvWriter writer = new CsvWriter(); //aggregate class results Console.WriteLine("Writing most popular sequences to file."); foreach (KeyValuePair <string, int> kvp in popularityDict.OrderByDescending(p => p.Value)) { int[] pieces = kvp.Key.Split('_').Select(c => Convert.ToInt32(c)).ToArray(); string npsmKey = string.Join("_", pieces.Select(p => vm.StateNumberToNpsmString(p)).ToArray()); writer.AddToCurrentLine(npsmKey); writer.AddToCurrentLine(kvp.Value.ToString()); writer.CreateNewRow(); } using (TextWriter tw = File.CreateText(string.Format("popular_sequences.csv"))) { tw.Write(writer.ToString()); } }