} // *** end of constructor
/// <summary>
/// Tom 04/11/2015 using simulated file rather than actual file
/// </summary>
public Regroup(porterData input)
{
output = new ArrayList();
pd_infile = input;
pd_infile.resetCursor();
}
} //end of constructor
public AssignNumber(porterData pd)//s = stemmedFileName
{
//MessageBox.Show("s = " + s);
pdinfile = pd;
pdinfile.resetCursor();
stronggroupno = 0;
weakgroupno = 0;
samplesize = 0;
} //end of constructor
} // *** end of method 'truncate'
public static porterData pd_truncate(string o_name, int x)
{
//method variables,
string inline, outline;
int inlinelength;
//create new output file and overwrite old files
// FileStream outfile = new FileStream(o, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write);
// StreamWriter dataout = new StreamWriter(outfile);
porterData dataout = new porterData(o_name);
//string fileName2 = System.Windows.Forms.Application.StartupPath + "\\GroupedWordList.txt";// + dataIn;
//FileStream fs2 = new FileStream(dataIn, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
//StreamReader datain = new StreamReader(fs2);
//read in first line
inline = pd_dataIn.readLine();
//traverse file until a null line read in
while (inline != null)
{
//find length of input string
inlinelength = inline.Length;
//if comment line is reached stop
if (inline.Contains("//"))
{
break;
}
//when barrier read-in and write to outfile unchanged
if (inline.Contains(strongbarrier) || inline.Contains(weakbarrier))
{
outline = inline;
}
//only truncate word that are long enough
else if (inlinelength > x)
{
outline = inline.Substring(0, ((x)));
}
else
{
outline = inline;
}
//output truncated word
dataout.writeLine(outline);
//dataout.Flush();
//read in next line
inline = pd_dataIn.readLine();
} // *** end of while
// *** close buffers used
return(dataout);
//fs0.Dispose();
}
} //end of constructor
public OverCalc(porterData si, porterData wi, porterData sni, porterData wni, int ss)
{
pd_stronginput = si;
pd_weakinput = wi;
pd_strongnumberinput = sni;
pd_weaknumberinput = wni;
si.resetCursor();
wi.resetCursor();
sni.resetCursor();
wni.resetCursor();
samplesize = ss;
strongGDNT = 0; strongGWMT = 0; strongGAMT = 0; strongOIG = 0; strongOIL = 0; strongDI = 0;
weakGDNT = 0; weakGWMT = 0; weakGAMT = 0; weakOIG = 0; weakOIL = 0; weakDI = 0;
} //end of constructor
public UnderCalc(porterData data)
{
//MessageBox.Show(s, "UnderCalc");
givenPData = data;
//System.IO.StreamReader sr = System.IO.File.OpenText(infile) ;
givenPData.resetCursor();
strongGDMT = 0.0;
strongGUMT = 0.0;
strongUI = 0.0;
strongCI = 0.0;
weakGDMT = 0.0;
weakGUMT = 0.0;
weakUI = 0.0;
weakCI = 0.0;
//calculateResults();
}
/// <summary>
/// Takes path of initial input file, writes that file to memory for operating quickly on
/// </summary>
/// <param name="path"> path of file</param>
/// <returns></returns>
public static porterData fileToMemory(string path)
{
DirectoryInfo info = new DirectoryInfo(path);
string fileName = info.Name;
porterData newdata = new porterData(fileName);
StreamReader fileReader = new StreamReader(path);
string line;
// read in file line by line and write to memory array
while ((line = fileReader.ReadLine()) != null)
{
newdata.writeLine(line);
}
return(newdata);
}
} // end of method 'strongAssignNumber'
public static porterData pD_strongAssignNumber(string s) //s = output file name
{
pdinfile.resetCursor();
string stronginline = "";
string strongoutline = "";
//initialize reading and writing buffers
// stronginput = new StreamReader(infile);
// FileStream strongoutfile = new FileStream(s, FileMode.OpenOrCreate,FileAccess.ReadWrite);
// StreamWriter strongprintout = new StreamWriter(strongoutfile);
// stronginline = stronginput.ReadLine();
stronginline = pdinfile.readLine();
porterData strongoutfile = new porterData(s);
//traverse file until null line read in
while (stronginline != null)
{
//increment group number for strong & weak barriers
if (stronginline.Contains(strongbarrier) || stronginline.Contains(weakbarrier))
{
stronggroupno++;
}
else
{
//create and write output string
strongoutline = stronggroupno + " " + stronginline;
strongoutfile.writeLine(strongoutline);
//count number of words in file
samplesize++;
}
//read in next line
stronginline = pdinfile.readLine();
} //end of while 'stronginline != null'
//close buffers used
// stronginput.Close();
// strongprintout.Close();
// strongoutfile.Close();
return(strongoutfile);
} // end of method 'strongAssignNumber'
} //end of method 'weakAssignNumber'
public static porterData pD_weakAssignNumber(string s)//s = output filename
{
pdinfile.resetCursor();
//MessageBox.Show("infile = "+ infile, "AssignNumber.weakAssignNumber");
string weakinline = "";
string weakoutline = "";
//initialize reading and writing buffers
// FileStream fsinfile = new FileStream(infile, FileMode.Open,FileAccess.ReadWrite);
// weakinput = new StreamReader(fsinfile);
porterData pDataOut = new porterData(s);
// FileStream weakoutfile = new FileStream(s, FileMode.OpenOrCreate,FileAccess.ReadWrite);
// StreamWriter weakprintout = new StreamWriter(weakoutfile);
weakinline = pdinfile.readLine();
//traverse file until null line read in
while (weakinline != null)
{
//increment group number only for strong barriers
if (weakinline.Contains(strongbarrier))
{
weakgroupno++;
}
else if (!(weakinline.Contains(weakbarrier)))
{
//create and write output string
weakoutline = weakgroupno + " " + weakinline;
pDataOut.writeLine(weakoutline);
}
//read in next line
weakinline = pdinfile.readLine();
} // *** end of while 'weakinline != null'
// *** close buffers
// weakinput.Close();
// weakprintout.Close();
// weakoutfile.Close();
return(pDataOut);
} //end of method 'weakAssignNumber'
} // *** end of method 'regroupfile'
public porterData pd_regroupfile(string name) //input should be the numbered-wordListFileName file
{
pd_infile.resetCursor();
string inputstring = "";
string test = "";
string outputstring = "";
string str1, str2;
StringTokenizer st1, st2; // was Tokenizer
bool added = false;
int x = 0;
int outputlength = 0;
porterData numberedData = new porterData(name);
//MessageBox.Show("s = " + s, "regroupfile(s)"); //s should be weakRegrouped_stemmedfilename
//add first element to the output
inputstring = pd_infile.readLine();
//System.Diagnostics.Debug.WriteLine("inputString" + inputstring);
output.Add(inputstring);
//19 June 2012 added below to filter out comment lines
// 21/10/2015 Can't see a filter??? Tom
inputstring = pd_infile.readLine();
while (inputstring != null)
{
//initialise variables
st1 = new StringTokenizer(inputstring);
str1 = st1.NextToken;
str1 = st1.NextToken;
//reset loop variables
added = false;
outputlength = output.Count;
x = outputlength;
while (x > 0)
{
test = (string)output[(x - 1)];
st2 = new StringTokenizer(test);
str2 = st2.NextToken;
str2 = st2.NextToken;
// HACK Tom 21/10/2015 - Changed from CompareTo to Compare to remove cultural awareness
// see Remarks @ https://msdn.microsoft.com/en-us/library/system.string.compareto%28v=vs.110%29.aspx
//CultureInfo mInfo
if (String.Compare(str1, str2, StringComparison.Ordinal) < 0)
{
x--;
}
else
{
output.Insert(x, inputstring);
added = true;
x = 0;
}
} // end of while 'x > 0'
if (!added)
{
output.Insert(0, inputstring);
}
inputstring = pd_infile.readLine();
//inputstring = inputstring.TrimEnd();
} // end of while 'inputstring != null'
outputlength = output.Count; //was .size
// *** output sorted array into file
for (int i = 0; i < outputlength; i++)
{
outputstring = (string)output[i]; //was .ElementAt
//outputstring = outputstring.TrimEnd(); removed blank space between the 2 groups
//System.Diagnostics.Debug.WriteLine("outputstring" + outputstring);
numberedData.writeLine(outputstring);
// swOut.WriteLine(outputstring);
// swOut.Flush();
} // *** end of for
// *** close buffers
// datain.Close();
//
// fsi.Close(); //added 5 July 2013
// swOut.Close();
// outfile.Close(); //was DeleteOnExit();
return(numberedData);
}
} // *** end of constructor
public Truncator(porterData givenData) // sortedFile
{
pd_dataIn = givenData;
pd_dataIn.resetCursor();
}
void ListResults()
{
//added timer for diagnostics 14 Aug 2013
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
try {
string si = strPath + "\\StrongRegrouped" + FileA; //added 27 June 2013
string wni = strPath + "\\WeakNumbered" + FileA;
string sni = strPath + "\\StrongNumbered" + FileA;
string wi = strPath + "\\WeakRegrouped" + FileA;
// Convert string[] to List<>
List <string> listlA = new List <string>(ListA);
List <string> listlB = new List <string>(ListB);
//get unique words count
var distinctWordsA = new List <string>(listlA.Distinct());
var distinctWordsB = new List <string>(listlB.Distinct());
decimal dmwc = w / s;
//decimal truncatedMwc = decimal.Truncate((dmwc * 100m) / 100m);
decimal dcf = ((w - s) / w);
//decimal truncatedCf = decimal.Truncate((dcf * 100m) / 100m);
//labelConflationClassSize.Text = "Mean Conflation Class size = " + String.Format("{0:0.###}", dmwc);
//labelCompressionFactor.Text = "Compression Factor = " + String.Format("{0:0.###}", dcf);
int editDistance = 0;
int hammingDistance = 0;
decimal relativeModifiedHammingDistance = 0;
iSum = 0;
iSumR = 0;
double meanMHD = 0;
double meanRelMHD = 0;
double SSMLancs = 0;
double meanLengthA = 0;
double meanLengthB = 0;
iSumLenA = 0;
iSumLenB = 0;
int j = 1;
int i = 0;
int count = 1;
double[] numbers = new double[4];
if (lengthA == lengthB)
{
//added 13 Aug 2013
listViewResults.BeginUpdate();
for (i = 0; i < ListA.Length; i++)
{
editDistance = LevenshteinDistance.Compute(ListA[i], ListB[i]);
if ((editDistance >= numericUpDownMin.Value) && (editDistance <= numericUpDownMax.Value))
{
hammingDistance = ClassHammingDistanceSimple.ModifiedHammingDistance(ListA[i], ListB[i]);
relativeModifiedHammingDistance = ClassHammingDistanceSimple.RelativeModifiedHammingDistance(ListA[i], ListB[i]);
if ((ListA[i].Contains("====")) || (ListA[i].Contains("----")) || (ListA[i].Contains("//")))
{
//do nothing
}
else
{
count = j++; //loop once for each word in the array ListA = string[]
numbers = AddItemsToListViewByRange(count, i, hammingDistance, relativeModifiedHammingDistance, editDistance);
}
}
backgroundWorker1.ReportProgress(count - 1);
} //end of loop
listViewResults.EndUpdate();
listViewResults.AutoResizeColumns(ColumnHeaderAutoResizeStyle.HeaderSize);
iSumLenA = numbers[0];
iSumLenB = numbers[1];
iSum = numbers[2]; //sum of MHD column (MHD)
iSumR = numbers[3]; //sum of RelMHD column
meanLengthA = (double)iSumLenA / listViewResults.Items.Count;
meanLengthB = (double)iSumLenB / listViewResults.Items.Count;
meanMHD = (double)iSum / listViewResults.Items.Count;
meanRelMHD = (double)(iSumR / listViewResults.Items.Count);
//SSMLancs = (double)100 * (1 - (meanRelMHD/lengthA)); //TODO 13 AUG 2013 accuracy??
SSMLancs = (double)100 * (1 - meanRelMHD); //FIXED 18 Aug 2013 but meanRelMHD not accurate
labelMeanLengthA.Text = "List A Mean Word Length = " + String.Format("{0:0.###}", meanLengthA.ToString("N3")); //13 Aug 2013 added ToString("N3")
labelMeanLengthB.Text = "List B Mean Word Length = " + String.Format("{0:0.###}", meanLengthB.ToString("N3")); //13 Aug 2013 added ToString("N3")
int UniqueWordsA = distinctWordsA.Count - GetWordsFromFile.IntBarrierCount;
int UniqueWordsB = distinctWordsB.Count - GetWordsFromFile.IntBarrierCount;
labelMeanCharsRemoved.Text = "Mean Characters Removed = " + String.Format("{0:0.###}", (meanLengthA - meanLengthB).ToString("N3")); //13 Aug 2013 added ToString("N3")
//Similarity metrics
this.labelInverseMeanMHD.Text = "Inverse Mean MHD = " + String.Format("{0:0.###}", (1 / meanMHD).ToString("N3")); //13 Aug 2013 add ToString("N3")
this.labelSSM.Text = "SSM* = " + String.Format("{0:0.###}", SSMLancs.ToString("N3")) + "%"; //13 Aug 2013 added ToString("N3")
//added 6 Aug 2013 to get % accurate to three decimal places
int ItemsCount = listViewResults.Items.Count;
double PercentageDiffs = (double)(100 * ItemsCount) / lengthA;
labelDiffCount.Text = "Total = " + listViewResults.Items.Count + " from " + lengthA.ToString() + " [" + String.Format("{0:0.###}", PercentageDiffs.ToString("N3")) + "%]";
toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 85; // 85% progress..
//if both files have barriers then show OI and UI metrics
if (barrierA && barrierB)
{
StemmingErrorGroupBox.Enabled = true;
porterData sortedData = porterData.fileToMemory(FileBpath);
UnderCalc uc = new UnderCalc(sortedData);
UnderCalc.pd_calculateResults();
double strongUI = uc.getStrongUI();
double weakUI = uc.getWeakUI();
labelUnderStemSOnly.Text = String.Format("{0:0.######}", weakUI.ToString("N6"));
labelUnderStemSW.Text = String.Format("{0:0.######}", strongUI.ToString("N6"));
toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 90; // 90% progress..
this.Update(); // force update since ui thread is otherwise blocked
AssignNumber assignNumber = new AssignNumber(sortedData);
porterData weakNumberedData = AssignNumber.pD_weakAssignNumber("WeakNumbered");
porterData strongNumberedData = AssignNumber.pD_strongAssignNumber("StrongNumbered");
int samplesize = AssignNumber.getSampleSize();
Regroup regroup = new Regroup(weakNumberedData);
porterData weakRegroupedData = regroup.pd_regroupfile("WeakRegrouped");
regroup = new Regroup(strongNumberedData);
porterData strongRegroupedData = regroup.pd_regroupfile("StrongRegrouped");
OverCalc oc = new OverCalc(strongRegroupedData, weakRegroupedData, strongNumberedData, weakNumberedData, samplesize);
OverCalc.pd_calculateResults();
string str1 = new FileInfo(FileApath).Directory.FullName;
string str2 = new FileInfo(FileApath).Name;
// Regroup regroup = new Regroup(str1 + "/WeakNumbered" + str2);
// regroup.regroupfile(str1 + "/WeakRegrouped" + str2);
//
// regroup = new Regroup(str1 + "/StrongNumbered" + str2);
// regroup.regroupfile(str1 + "/StrongRegrouped" + str2);
//
toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 92; // 92% progress..
this.Update(); // force update since ui thread is otherwise blocked
toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 96; // 92% progress..
this.Update(); // force update since ui thread is otherwise blocked
// HACK MemoryBarrier Tom 21/10/2015 see http://stackoverflow.com/questions/5996267/c-sharp-enforcing-order-of-statement-execution
// Just added on suspicion that the below code somehow was getting executed before calculateResults finishes
System.Threading.Thread.MemoryBarrier();
double strongOI = oc.getStrongOIG();
double SW = (double)strongOI / strongUI;
double weakOIG = oc.getWeakOIG();
string OverStemmingIndex = strongOI.ToString("N8");
labelOverStemSWG.Text = String.Format("{0:0.########}", OverStemmingIndex);
labelOverStemSOnlyG.Text = String.Format("{0:0.########}", weakOIG);
double strongOIL = oc.getStrongOIL();
double weakOIL = oc.getWeakOIL();
labelOverStemSOnlyL.Text = String.Format("{0:0.########}", weakOIL);
labelOverStenSWL.Text = String.Format("{0:0.########}", strongOIL);
//Added 22 DEC 2013 for ERRT Calc
double[] weakUIarray = new double[7];
double[] weakOILarray = new double[7];
double[] strongUIarray = new double[7];
double[] strongOILarray = new double[7];
//Tom 14/10/2015 - Adding logic for ERRT Calculation
//Used SwingInterface.java lines 743 - 795
//from Lancaster Univ program as reference
weakUIarray[0] = uc.getWeakUI();
weakOILarray[0] = oc.getWeakOIL();
strongUIarray[0] = uc.getStrongUI();
strongOILarray[0] = oc.getStrongOIL();
toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 97; // 92% progress..
this.Update(); // force update since ui thread is otherwise blocked
//Java 743
sortedData = porterData.fileToMemory(FileApath);
for (i = 3; i < 9; i++)
{
Truncator truncator = new Truncator(sortedData);
porterData truncatedData = Truncator.pd_truncate("truncated", i);
uc = new UnderCalc(truncatedData);
UnderCalc.pd_calculateResults();
// AssignNumber asn = new AssignNumber(str1 + "truncated" + i + str2);
AssignNumber asn = new AssignNumber(truncatedData);
porterData weakNumberedTData = AssignNumber.pD_weakAssignNumber("WeakNumberedT");
// AssignNumber.strongAssignNumber(str1 + "StrongNumberedT" + i + str2);
porterData strongNumberedTData = AssignNumber.pD_strongAssignNumber("StrongNumberedT");
samplesize = AssignNumber.getSampleSize();
// regroup = new Regroup(str1 + "WeakNumberedT" + i + str2);
// regroup.regroupfile(str1 + "WeakRegroupedT" + i + str2);
// regroup = new Regroup(str1 + "StrongNumberedT" + i + str2);
// regroup.regroupfile(str1 + "StrongRegroupedT" + i +str2);
regroup = new Regroup(weakNumberedTData);
porterData weakRegroupedTData = regroup.pd_regroupfile("WeakRegroupedT");
regroup = new Regroup(strongNumberedTData);
porterData strongRegroupedTData = regroup.pd_regroupfile("StrongRegroupedT");
// oc = new OverCalc(str1 + "StrongRegroupedT" + i + str2,
// str1 + "WeakRegroupedT" + i + str2,
// str1 + "StrongNumberedT" + i + str2,
// str1 + "WeakNumberedT" + i + str2, samplesize);
oc = new OverCalc(strongRegroupedTData,
weakRegroupedTData,
strongNumberedTData,
weakNumberedTData, samplesize);
OverCalc.pd_calculateResults();
weakUIarray[i - 2] = uc.getWeakUI();
weakOILarray[i - 2] = oc.getWeakOIL();
strongUIarray[i - 2] = uc.getStrongUI();
strongOILarray[i - 2] = oc.getStrongOIL();
toolStripProgressBar1.Value = (toolStripProgressBar1.Maximum / 100) * 98; // 92% progress..
this.Update(); // force update since ui thread is otherwise blocked
}
//Changed last Parameter of ERRT Constructor to match Line 800 SwingInterface.java of original Lancs program..
//Although, parameter names do not seem to match to constructor (using weakOILarray in sOIL spot)
ERRT errt;
if (stemming_mode == MODE_DTS)
{
errt = new ERRT(weakUIarray, weakOILarray, strongUIarray, strongOILarray);
}
else
{
errt = new ERRT(weakUIarray, weakOILarray, strongUIarray, weakOILarray);
}
//double[] wUI, wOIL, sUI, sOIL
// Tom 21/10/2015 - Changed last parameter from strongOILarray to weakOILarray to match Java 801
//ERRT errt = new ERRT(weakUIarray ,weakOILarray, strongUIarray, weakOILarray);
errt.CalculateResults();
double derrt = errt.getStrongERRTL();
//MessageBox.Show("derrt" + derrt);
//labelERRT.Text = "ERRT = " + String.Format("{0:0.#######}", derrt.ToString("N6"));
// Tom 26/10/2015 see SwingInterface.java line 814 &
// the answer @ http://stackoverflow.com/questions/2924242/replicating-javas-decimalformat-in-c-sharp
string sws = "Test" + (errt.getWeakSWL() * 100.00).ToString();
sws = (errt.getWeakSWL() * 100).ToString("N2");
labelSWSOnly.Text = sws;
string swlText = (errt.getStrongSWL() * 100.00).ToString();
swlText = (errt.getStrongSWL() * 100).ToString("N2");
labelSWSW.Text = swlText;
string errtSOnlytext = (errt.getWeakERRTL() * 100).ToString("N2");
string errtText = (errt.getStrongERRTL() * 100.00).ToString("N2");
labelERRTSW.Text = errtText;
labelERRTSOnly.Text = errtSOnlytext;
System.Diagnostics.Debug.WriteLine("MainForm 821: UnderCalc.numDMTCalls= " + UnderCalc.numDMTCalls);
System.Diagnostics.Debug.WriteLine("MainForm 822: OverCalc numStrongOILZero = " + OverCalc.numStrongOILZero + ", numStrongOILNonZero = " + OverCalc.numStrongOILNonZero);
// tom 03/11/2015 writing out list of DMT(); results from underCalc to a file
string DMTData = UnderCalc.sb.ToString();
//System.IO.StreamWriter file = new System.IO.StreamWriter("c:\\DMT_Test\\DMT_LA.txt");
//file.WriteLine(DMTData);
//file.Close();
}
else
{
StemmingErrorGroupBox.Enabled = false;
}
//MessageBox.Show(j.ToString());
}
else
{
MessageBox.Show("Lengths of Lists are not the same", "ListResults");
}
} catch (Exception ex) {
MessageBox.Show(ex.Message + "\n \n" + ex.StackTrace, "ListResults");
//throw;
}
TimeSpan ts = stopWatch.Elapsed;
// Format and display the TimeSpan value.
string elapsedTime = String.Format("{0:00}:{1:00}.{2:00}", ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
toolStripStatusLabelElapsedTime.Text = elapsedTime;
stopWatch.Stop();
toolStripProgressBar1.Value = toolStripProgressBar1.Maximum; // finished
this.Update(); // force update since ui thread is otherwise blocked
}