} // *** end of constructor /// <summary> /// Tom 04/11/2015 using simulated file rather than actual file /// </summary> public Regroup(porterData input) { output = new ArrayList(); pd_infile = input; pd_infile.resetCursor(); }
} //end of constructor public AssignNumber(porterData pd)//s = stemmedFileName { //MessageBox.Show("s = " + s); pdinfile = pd; pdinfile.resetCursor(); stronggroupno = 0; weakgroupno = 0; samplesize = 0; } //end of constructor
} // *** end of method 'truncate' public static porterData pd_truncate(string o_name, int x) { //method variables, string inline, outline; int inlinelength; //create new output file and overwrite old files // FileStream outfile = new FileStream(o, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write); // StreamWriter dataout = new StreamWriter(outfile); porterData dataout = new porterData(o_name); //string fileName2 = System.Windows.Forms.Application.StartupPath + "\\GroupedWordList.txt";// + dataIn; //FileStream fs2 = new FileStream(dataIn, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); //StreamReader datain = new StreamReader(fs2); //read in first line inline = pd_dataIn.readLine(); //traverse file until a null line read in while (inline != null) { //find length of input string inlinelength = inline.Length; //if comment line is reached stop if (inline.Contains("//")) { break; } //when barrier read-in and write to outfile unchanged if (inline.Contains(strongbarrier) || inline.Contains(weakbarrier)) { outline = inline; } //only truncate word that are long enough else if (inlinelength > x) { outline = inline.Substring(0, ((x))); } else { outline = inline; } //output truncated word dataout.writeLine(outline); //dataout.Flush(); //read in next line inline = pd_dataIn.readLine(); } // *** end of while // *** close buffers used return(dataout); //fs0.Dispose(); }
} //end of constructor public OverCalc(porterData si, porterData wi, porterData sni, porterData wni, int ss) { pd_stronginput = si; pd_weakinput = wi; pd_strongnumberinput = sni; pd_weaknumberinput = wni; si.resetCursor(); wi.resetCursor(); sni.resetCursor(); wni.resetCursor(); samplesize = ss; strongGDNT = 0; strongGWMT = 0; strongGAMT = 0; strongOIG = 0; strongOIL = 0; strongDI = 0; weakGDNT = 0; weakGWMT = 0; weakGAMT = 0; weakOIG = 0; weakOIL = 0; weakDI = 0; } //end of constructor
public UnderCalc(porterData data) { //MessageBox.Show(s, "UnderCalc"); givenPData = data; //System.IO.StreamReader sr = System.IO.File.OpenText(infile) ; givenPData.resetCursor(); strongGDMT = 0.0; strongGUMT = 0.0; strongUI = 0.0; strongCI = 0.0; weakGDMT = 0.0; weakGUMT = 0.0; weakUI = 0.0; weakCI = 0.0; //calculateResults(); }
/// <summary> /// Takes path of initial input file, writes that file to memory for operating quickly on /// </summary> /// <param name="path"> path of file</param> /// <returns></returns> public static porterData fileToMemory(string path) { DirectoryInfo info = new DirectoryInfo(path); string fileName = info.Name; porterData newdata = new porterData(fileName); StreamReader fileReader = new StreamReader(path); string line; // read in file line by line and write to memory array while ((line = fileReader.ReadLine()) != null) { newdata.writeLine(line); } return(newdata); }
} // end of method 'strongAssignNumber' public static porterData pD_strongAssignNumber(string s) //s = output file name { pdinfile.resetCursor(); string stronginline = ""; string strongoutline = ""; //initialize reading and writing buffers // stronginput = new StreamReader(infile); // FileStream strongoutfile = new FileStream(s, FileMode.OpenOrCreate,FileAccess.ReadWrite); // StreamWriter strongprintout = new StreamWriter(strongoutfile); // stronginline = stronginput.ReadLine(); stronginline = pdinfile.readLine(); porterData strongoutfile = new porterData(s); //traverse file until null line read in while (stronginline != null) { //increment group number for strong & weak barriers if (stronginline.Contains(strongbarrier) || stronginline.Contains(weakbarrier)) { stronggroupno++; } else { //create and write output string strongoutline = stronggroupno + " " + stronginline; strongoutfile.writeLine(strongoutline); //count number of words in file samplesize++; } //read in next line stronginline = pdinfile.readLine(); } //end of while 'stronginline != null' //close buffers used // stronginput.Close(); // strongprintout.Close(); // strongoutfile.Close(); return(strongoutfile); } // end of method 'strongAssignNumber'
} //end of method 'weakAssignNumber' public static porterData pD_weakAssignNumber(string s)//s = output filename { pdinfile.resetCursor(); //MessageBox.Show("infile = "+ infile, "AssignNumber.weakAssignNumber"); string weakinline = ""; string weakoutline = ""; //initialize reading and writing buffers // FileStream fsinfile = new FileStream(infile, FileMode.Open,FileAccess.ReadWrite); // weakinput = new StreamReader(fsinfile); porterData pDataOut = new porterData(s); // FileStream weakoutfile = new FileStream(s, FileMode.OpenOrCreate,FileAccess.ReadWrite); // StreamWriter weakprintout = new StreamWriter(weakoutfile); weakinline = pdinfile.readLine(); //traverse file until null line read in while (weakinline != null) { //increment group number only for strong barriers if (weakinline.Contains(strongbarrier)) { weakgroupno++; } else if (!(weakinline.Contains(weakbarrier))) { //create and write output string weakoutline = weakgroupno + " " + weakinline; pDataOut.writeLine(weakoutline); } //read in next line weakinline = pdinfile.readLine(); } // *** end of while 'weakinline != null' // *** close buffers // weakinput.Close(); // weakprintout.Close(); // weakoutfile.Close(); return(pDataOut); } //end of method 'weakAssignNumber'
} // *** end of method 'regroupfile' public porterData pd_regroupfile(string name) //input should be the numbered-wordListFileName file { pd_infile.resetCursor(); string inputstring = ""; string test = ""; string outputstring = ""; string str1, str2; StringTokenizer st1, st2; // was Tokenizer bool added = false; int x = 0; int outputlength = 0; porterData numberedData = new porterData(name); //MessageBox.Show("s = " + s, "regroupfile(s)"); //s should be weakRegrouped_stemmedfilename //add first element to the output inputstring = pd_infile.readLine(); //System.Diagnostics.Debug.WriteLine("inputString" + inputstring); output.Add(inputstring); //19 June 2012 added below to filter out comment lines // 21/10/2015 Can't see a filter??? Tom inputstring = pd_infile.readLine(); while (inputstring != null) { //initialise variables st1 = new StringTokenizer(inputstring); str1 = st1.NextToken; str1 = st1.NextToken; //reset loop variables added = false; outputlength = output.Count; x = outputlength; while (x > 0) { test = (string)output[(x - 1)]; st2 = new StringTokenizer(test); str2 = st2.NextToken; str2 = st2.NextToken; // HACK Tom 21/10/2015 - Changed from CompareTo to Compare to remove cultural awareness // see Remarks @ https://msdn.microsoft.com/en-us/library/system.string.compareto%28v=vs.110%29.aspx //CultureInfo mInfo if (String.Compare(str1, str2, StringComparison.Ordinal) < 0) { x--; } else { output.Insert(x, inputstring); added = true; x = 0; } } // end of while 'x > 0' if (!added) { output.Insert(0, inputstring); } inputstring = pd_infile.readLine(); //inputstring = inputstring.TrimEnd(); } // end of while 'inputstring != null' outputlength = output.Count; //was .size // *** output sorted array into file for (int i = 0; i < outputlength; i++) { outputstring = (string)output[i]; //was .ElementAt //outputstring = outputstring.TrimEnd(); removed blank space between the 2 groups //System.Diagnostics.Debug.WriteLine("outputstring" + outputstring); numberedData.writeLine(outputstring); // swOut.WriteLine(outputstring); // swOut.Flush(); } // *** end of for // *** close buffers // datain.Close(); // // fsi.Close(); //added 5 July 2013 // swOut.Close(); // outfile.Close(); //was DeleteOnExit(); return(numberedData); }
} // *** end of constructor public Truncator(porterData givenData) // sortedFile { pd_dataIn = givenData; pd_dataIn.resetCursor(); }
void ListResults() { //added timer for diagnostics 14 Aug 2013 Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); try { string si = strPath + "\\StrongRegrouped" + FileA; //added 27 June 2013 string wni = strPath + "\\WeakNumbered" + FileA; string sni = strPath + "\\StrongNumbered" + FileA; string wi = strPath + "\\WeakRegrouped" + FileA; // Convert string[] to List<> List <string> listlA = new List <string>(ListA); List <string> listlB = new List <string>(ListB); //get unique words count var distinctWordsA = new List <string>(listlA.Distinct()); var distinctWordsB = new List <string>(listlB.Distinct()); decimal dmwc = w / s; //decimal truncatedMwc = decimal.Truncate((dmwc * 100m) / 100m); decimal dcf = ((w - s) / w); //decimal truncatedCf = decimal.Truncate((dcf * 100m) / 100m); //labelConflationClassSize.Text = "Mean Conflation Class size = " + String.Format("{0:0.###}", dmwc); //labelCompressionFactor.Text = "Compression Factor = " + String.Format("{0:0.###}", dcf); int editDistance = 0; int hammingDistance = 0; decimal relativeModifiedHammingDistance = 0; iSum = 0; iSumR = 0; double meanMHD = 0; double meanRelMHD = 0; double SSMLancs = 0; double meanLengthA = 0; double meanLengthB = 0; iSumLenA = 0; iSumLenB = 0; int j = 1; int i = 0; int count = 1; double[] numbers = new double[4]; if (lengthA == lengthB) { //added 13 Aug 2013 listViewResults.BeginUpdate(); for (i = 0; i < ListA.Length; i++) { editDistance = LevenshteinDistance.Compute(ListA[i], ListB[i]); if ((editDistance >= numericUpDownMin.Value) && (editDistance <= numericUpDownMax.Value)) { hammingDistance = ClassHammingDistanceSimple.ModifiedHammingDistance(ListA[i], ListB[i]); relativeModifiedHammingDistance = ClassHammingDistanceSimple.RelativeModifiedHammingDistance(ListA[i], ListB[i]); if ((ListA[i].Contains("====")) || (ListA[i].Contains("----")) || (ListA[i].Contains("//"))) { //do nothing } else { count = j++; //loop once for each word in the array ListA = string[] numbers = AddItemsToListViewByRange(count, i, hammingDistance, relativeModifiedHammingDistance, editDistance); } } backgroundWorker1.ReportProgress(count - 1); } //end of loop listViewResults.EndUpdate(); listViewResults.AutoResizeColumns(ColumnHeaderAutoResizeStyle.HeaderSize); iSumLenA = numbers[0]; iSumLenB = numbers[1]; iSum = numbers[2]; //sum of MHD column (MHD) iSumR = numbers[3]; //sum of RelMHD column meanLengthA = (double)iSumLenA / listViewResults.Items.Count; meanLengthB = (double)iSumLenB / listViewResults.Items.Count; meanMHD = (double)iSum / listViewResults.Items.Count; meanRelMHD = (double)(iSumR / listViewResults.Items.Count); //SSMLancs = (double)100 * (1 - (meanRelMHD/lengthA)); //TODO 13 AUG 2013 accuracy?? SSMLancs = (double)100 * (1 - meanRelMHD); //FIXED 18 Aug 2013 but meanRelMHD not accurate labelMeanLengthA.Text = "List A Mean Word Length = " + String.Format("{0:0.###}", meanLengthA.ToString("N3")); //13 Aug 2013 added ToString("N3") labelMeanLengthB.Text = "List B Mean Word Length = " + String.Format("{0:0.###}", meanLengthB.ToString("N3")); //13 Aug 2013 added ToString("N3") int UniqueWordsA = distinctWordsA.Count - GetWordsFromFile.IntBarrierCount; int UniqueWordsB = distinctWordsB.Count - GetWordsFromFile.IntBarrierCount; labelMeanCharsRemoved.Text = "Mean Characters Removed = " + String.Format("{0:0.###}", (meanLengthA - meanLengthB).ToString("N3")); //13 Aug 2013 added ToString("N3") //Similarity metrics this.labelInverseMeanMHD.Text = "Inverse Mean MHD = " + String.Format("{0:0.###}", (1 / meanMHD).ToString("N3")); //13 Aug 2013 add ToString("N3") this.labelSSM.Text = "SSM* = " + String.Format("{0:0.###}", SSMLancs.ToString("N3")) + "%"; //13 Aug 2013 added ToString("N3") //added 6 Aug 2013 to get % accurate to three decimal places int ItemsCount = listViewResults.Items.Count; double PercentageDiffs = (double)(100 * ItemsCount) / lengthA; labelDiffCount.Text = "Total = " + listViewResults.Items.Count + " from " + lengthA.ToString() + " [" + String.Format("{0:0.###}", PercentageDiffs.ToString("N3")) + "%]"; toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 85; // 85% progress.. //if both files have barriers then show OI and UI metrics if (barrierA && barrierB) { StemmingErrorGroupBox.Enabled = true; porterData sortedData = porterData.fileToMemory(FileBpath); UnderCalc uc = new UnderCalc(sortedData); UnderCalc.pd_calculateResults(); double strongUI = uc.getStrongUI(); double weakUI = uc.getWeakUI(); labelUnderStemSOnly.Text = String.Format("{0:0.######}", weakUI.ToString("N6")); labelUnderStemSW.Text = String.Format("{0:0.######}", strongUI.ToString("N6")); toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 90; // 90% progress.. this.Update(); // force update since ui thread is otherwise blocked AssignNumber assignNumber = new AssignNumber(sortedData); porterData weakNumberedData = AssignNumber.pD_weakAssignNumber("WeakNumbered"); porterData strongNumberedData = AssignNumber.pD_strongAssignNumber("StrongNumbered"); int samplesize = AssignNumber.getSampleSize(); Regroup regroup = new Regroup(weakNumberedData); porterData weakRegroupedData = regroup.pd_regroupfile("WeakRegrouped"); regroup = new Regroup(strongNumberedData); porterData strongRegroupedData = regroup.pd_regroupfile("StrongRegrouped"); OverCalc oc = new OverCalc(strongRegroupedData, weakRegroupedData, strongNumberedData, weakNumberedData, samplesize); OverCalc.pd_calculateResults(); string str1 = new FileInfo(FileApath).Directory.FullName; string str2 = new FileInfo(FileApath).Name; // Regroup regroup = new Regroup(str1 + "/WeakNumbered" + str2); // regroup.regroupfile(str1 + "/WeakRegrouped" + str2); // // regroup = new Regroup(str1 + "/StrongNumbered" + str2); // regroup.regroupfile(str1 + "/StrongRegrouped" + str2); // toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 92; // 92% progress.. this.Update(); // force update since ui thread is otherwise blocked toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 96; // 92% progress.. this.Update(); // force update since ui thread is otherwise blocked // HACK MemoryBarrier Tom 21/10/2015 see http://stackoverflow.com/questions/5996267/c-sharp-enforcing-order-of-statement-execution // Just added on suspicion that the below code somehow was getting executed before calculateResults finishes System.Threading.Thread.MemoryBarrier(); double strongOI = oc.getStrongOIG(); double SW = (double)strongOI / strongUI; double weakOIG = oc.getWeakOIG(); string OverStemmingIndex = strongOI.ToString("N8"); labelOverStemSWG.Text = String.Format("{0:0.########}", OverStemmingIndex); labelOverStemSOnlyG.Text = String.Format("{0:0.########}", weakOIG); double strongOIL = oc.getStrongOIL(); double weakOIL = oc.getWeakOIL(); labelOverStemSOnlyL.Text = String.Format("{0:0.########}", weakOIL); labelOverStenSWL.Text = String.Format("{0:0.########}", strongOIL); //Added 22 DEC 2013 for ERRT Calc double[] weakUIarray = new double[7]; double[] weakOILarray = new double[7]; double[] strongUIarray = new double[7]; double[] strongOILarray = new double[7]; //Tom 14/10/2015 - Adding logic for ERRT Calculation //Used SwingInterface.java lines 743 - 795 //from Lancaster Univ program as reference weakUIarray[0] = uc.getWeakUI(); weakOILarray[0] = oc.getWeakOIL(); strongUIarray[0] = uc.getStrongUI(); strongOILarray[0] = oc.getStrongOIL(); toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 97; // 92% progress.. this.Update(); // force update since ui thread is otherwise blocked //Java 743 sortedData = porterData.fileToMemory(FileApath); for (i = 3; i < 9; i++) { Truncator truncator = new Truncator(sortedData); porterData truncatedData = Truncator.pd_truncate("truncated", i); uc = new UnderCalc(truncatedData); UnderCalc.pd_calculateResults(); // AssignNumber asn = new AssignNumber(str1 + "truncated" + i + str2); AssignNumber asn = new AssignNumber(truncatedData); porterData weakNumberedTData = AssignNumber.pD_weakAssignNumber("WeakNumberedT"); // AssignNumber.strongAssignNumber(str1 + "StrongNumberedT" + i + str2); porterData strongNumberedTData = AssignNumber.pD_strongAssignNumber("StrongNumberedT"); samplesize = AssignNumber.getSampleSize(); // regroup = new Regroup(str1 + "WeakNumberedT" + i + str2); // regroup.regroupfile(str1 + "WeakRegroupedT" + i + str2); // regroup = new Regroup(str1 + "StrongNumberedT" + i + str2); // regroup.regroupfile(str1 + "StrongRegroupedT" + i +str2); regroup = new Regroup(weakNumberedTData); porterData weakRegroupedTData = regroup.pd_regroupfile("WeakRegroupedT"); regroup = new Regroup(strongNumberedTData); porterData strongRegroupedTData = regroup.pd_regroupfile("StrongRegroupedT"); // oc = new OverCalc(str1 + "StrongRegroupedT" + i + str2, // str1 + "WeakRegroupedT" + i + str2, // str1 + "StrongNumberedT" + i + str2, // str1 + "WeakNumberedT" + i + str2, samplesize); oc = new OverCalc(strongRegroupedTData, weakRegroupedTData, strongNumberedTData, weakNumberedTData, samplesize); OverCalc.pd_calculateResults(); weakUIarray[i - 2] = uc.getWeakUI(); weakOILarray[i - 2] = oc.getWeakOIL(); strongUIarray[i - 2] = uc.getStrongUI(); strongOILarray[i - 2] = oc.getStrongOIL(); toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 98; // 92% progress.. this.Update(); // force update since ui thread is otherwise blocked } //Changed last Parameter of ERRT Constructor to match Line 800 SwingInterface.java of original Lancs program.. //Although, parameter names do not seem to match to constructor (using weakOILarray in sOIL spot) ERRT errt; if (stemming_mode == MODE_DTS) { errt = new ERRT(weakUIarray, weakOILarray, strongUIarray, strongOILarray); } else { errt = new ERRT(weakUIarray, weakOILarray, strongUIarray, weakOILarray); } //double[] wUI, wOIL, sUI, sOIL // Tom 21/10/2015 - Changed last parameter from strongOILarray to weakOILarray to match Java 801 //ERRT errt = new ERRT(weakUIarray ,weakOILarray, strongUIarray, weakOILarray); errt.CalculateResults(); double derrt = errt.getStrongERRTL(); //MessageBox.Show("derrt" + derrt); //labelERRT.Text = "ERRT = " + String.Format("{0:0.#######}", derrt.ToString("N6")); // Tom 26/10/2015 see SwingInterface.java line 814 & // the answer @ http://stackoverflow.com/questions/2924242/replicating-javas-decimalformat-in-c-sharp string sws = "Test" + (errt.getWeakSWL() * 100.00).ToString(); sws = (errt.getWeakSWL() * 100).ToString("N2"); labelSWSOnly.Text = sws; string swlText = (errt.getStrongSWL() * 100.00).ToString(); swlText = (errt.getStrongSWL() * 100).ToString("N2"); labelSWSW.Text = swlText; string errtSOnlytext = (errt.getWeakERRTL() * 100).ToString("N2"); string errtText = (errt.getStrongERRTL() * 100.00).ToString("N2"); labelERRTSW.Text = errtText; labelERRTSOnly.Text = errtSOnlytext; System.Diagnostics.Debug.WriteLine("MainForm 821: UnderCalc.numDMTCalls= " + UnderCalc.numDMTCalls); System.Diagnostics.Debug.WriteLine("MainForm 822: OverCalc numStrongOILZero = " + OverCalc.numStrongOILZero + ", numStrongOILNonZero = " + OverCalc.numStrongOILNonZero); // tom 03/11/2015 writing out list of DMT(); results from underCalc to a file string DMTData = UnderCalc.sb.ToString(); //System.IO.StreamWriter file = new System.IO.StreamWriter("c:\\DMT_Test\\DMT_LA.txt"); //file.WriteLine(DMTData); //file.Close(); } else { StemmingErrorGroupBox.Enabled = false; } //MessageBox.Show(j.ToString()); } else { MessageBox.Show("Lengths of Lists are not the same", "ListResults"); } } catch (Exception ex) { MessageBox.Show(ex.Message + "\n \n" + ex.StackTrace, "ListResults"); //throw; } TimeSpan ts = stopWatch.Elapsed; // Format and display the TimeSpan value. string elapsedTime = String.Format("{0:00}:{1:00}.{2:00}", ts.Minutes, ts.Seconds, ts.Milliseconds / 10); toolStripStatusLabelElapsedTime.Text = elapsedTime; stopWatch.Stop(); toolStripProgressBar1.Value = toolStripProgressBar1.Maximum; // finished this.Update(); // force update since ui thread is otherwise blocked }