private void FrmFieldsToExtract_Load(object sender, EventArgs e)
{
Loading ld = new Loading();
ld.Show();
string fields = servRef.GetFieldNames(Globals.sqlCon, clientid, secret);
if (fields.Length == 0)
{
MessageBox.Show("No Fields to show..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Information);
ld.Close();
this.Close();
return;
}
string[] fieldArr = fields.Split(',');
if (fields.Length == 1)
{
if (fieldArr[0].ToString().Trim().Equals("0"))
{
MessageBox.Show("No Fields to show..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Information);
ld.Close();
this.Close();
return;
}
}
foreach (string field in fieldArr)
{
FieldsToExtract.Items.Add(field);
}
if (!string.IsNullOrEmpty(GetConfiguration.GetConfigurationValueFields()))
{
string[] split = GetConfiguration.GetConfigurationValueFields().Split(',');
foreach (string field in split)
{
for (int i = 0; i < FieldsToExtract.Items.Count; i++)
{
string strItem = FieldsToExtract.Items[i].ToString();
if (strItem.Equals(field))
{
FieldsToExtract.SetItemChecked(i, true);
}
}
}
}
ld.Close();
}
private void Main_Run_DoWork(object sender, DoWorkEventArgs e)
{
#region Set Up Run
DataTable finishedRun = new DataTable();
finishedRun.Columns.Add("Digitised Text");
finishedRun.Columns.Add("Translated Text");
finishedRun.Columns.Add("Fields Found");
finishedRun.Columns.Add("Clauses Found");
Globals.dpi = GetConfiguration.GetConfigurationValueDPI();
Globals.meta = GetConfiguration.GetConfigurationValueMeta();
Globals.ocrType = GetConfiguration.GetConfigurationValueOCR();
strLineRemoval = GetConfiguration.GetConfigurationValueGrayScale();
if (Globals.ocrType.Equals("Microsoft Cloud"))
{
Globals.ocrType = "1";
}
else if (Globals.ocrType.Equals("Google Cloud"))
{
Globals.ocrType = "2";
}
else if (Globals.ocrType.Equals("AWS Textract"))
{
Globals.ocrType = "3";
}
else
{
Globals.ocrType = "0";
}
//Make sure thread does not send PC to sleep
Kernel32.SetThreadExecutionState(Kernel32.EXECUTION_STATE.ES_CONTINUOUS |
Kernel32.EXECUTION_STATE.ES_SYSTEM_REQUIRED |
Kernel32.EXECUTION_STATE.ES_AWAYMODE_REQUIRED);
string[] docList = null;
if (fileType.Checked == false)
{
try
{
if (searchSubDirects == true)
{
docList = Directory.GetFiles(folderPath.Text.Trim(), "*.*", SearchOption.AllDirectories);
}
else
{
docList = Directory.GetFiles(folderPath.Text.Trim(), "*.*");
}
}
catch
{
MessageBox.Show("Unable to find Run directory..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
}
else
{
if (Globals.files != null)
{
docList = Globals.files;
}
else
{
MessageBox.Show("Please choose some files..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
}
docList = Array.FindAll(docList, isNotLockFile);
if (!Directory.Exists(savePath.Text.Trim()))
{
MessageBox.Show("Unable to find Save directory..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
if (docList.Length <= 0)
{
MessageBox.Show("Unable to find any files in the Run directory..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
//Prepare Run
string prepare = servRef.PrepareRun(clientID, secret, Globals.sqlCon);
if (!prepare.ToUpper().Equals("SUCCESS"))
{
MessageBox.Show("Oops. Something went wrong.", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
#endregion
string lastFile = docList[docList.Length - 1];
foreach (string file in docList)
{
UpdateStatusProgressDelegate UpdateProgress = Progress;
UpdateProgressTextDelegate UpdateText = ProgressLabel;
UpdateStatusTextDelegate UpdateStatus = StatusLabel;
UpdatePagesTextDelegate UpdatePages = PagesLabel;
bool convertedFile = false;
string runFile = file;
string tempFile = file;
bool corruptFile = false;
if (Path.GetExtension(file).ToUpper().Trim().Equals(".DOCX"))
{
Invoke(UpdateStatus, "Resaving Word Document as PDF");
try
{
//check for mixed media
var wordApplication = new Microsoft.Office.Interop.Word.Application();
wordApplication.Visible = false;
var document = wordApplication.Documents.Open(file, false, true);
Guid strGuid = Guid.NewGuid();
//convert to pdf
string tempPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
tempPath = Path.Combine(tempPath, "temps");
if (!Directory.Exists(tempPath))
{
Directory.CreateDirectory(tempPath);
}
string tempFileName = Path.GetFileNameWithoutExtension(file);
document.ExportAsFixedFormat(System.IO.Path.Combine(tempPath, strGuid + ".pdf"), WdExportFormat.wdExportFormatPDF);
runFile = System.IO.Path.Combine(tempPath, strGuid + ".pdf");
convertedFile = true;
// Close word
wordApplication.Quit();
document = null;
wordApplication = null;
GC.Collect();
}
catch (Exception exe)
{
if (wordWarning == false)
{
MessageBox.Show("Without Microsoft Word on this machine we will only be able to use the native text these types of document(.DOCX) document. To digitise any pictures in the document, please install Microsoft Word. Alternatively convert it to a .pdf and rerun Quill.", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Information);
wordWarning = true;
}
}
}
#region Loop Files
if (cancelMain == true)
{
break;
}
string fileName = Path.GetFileName(runFile);
Invoke(UpdateProgress, 0);
if (!string.IsNullOrEmpty(tempFile))
{
Invoke(UpdateText, "Working on file: " + Path.GetFileName(tempFile));
}
else
{
Invoke(UpdateText, "Working on file: " + fileName);
}
Invoke(UpdateStatus, "Transmitting File..");
#region Send File
//Convert to Bytes
FileInfo fi = new FileInfo(runFile);
long numBytes = fi.Length;
FileStream fs = new FileStream(runFile, FileMode.Open, FileAccess.Read);
BinaryReader br = new BinaryReader(fs);
byte[] fileArray = br.ReadBytes((int)numBytes);
br.Close();
fs.Close();
fs.Dispose();
Invoke(UpdateStatus, "Transmitting");
//Transmit File
Invoke(UpdateProgress, 10);
if (cancelMain == true)
{
return;
}
string transmit = servRef.SaveClientFile(fileArray, runFile, clientID, secret);
bool invalidTypeContinue = false;
if (!transmit.ToUpper().Equals("SUCCESS"))
{
DialogResult type = MessageBox.Show("Oops. Quill Can't convert: " + runFile + " Would you like to continue?", "Quill", MessageBoxButtons.YesNo, MessageBoxIcon.Error);
if (DialogResult.No == type)
{
cancelMain = true;
break;
}
else
{
invalidTypeContinue = true;
}
}
if (cancelMain == true)
{
break;
}
#endregion
if (invalidTypeContinue == false)
{
#region Get File ID
Invoke(UpdateStatus, "Get File ID");
string fileID = servRef.GetFileID(fileName, clientID, secret);
Invoke(UpdateProgress, 20);
#endregion
#region Check file type
Invoke(UpdateStatus, "Native Check..");
string native = servRef.NativeTextCheck(fileName, Globals.sqlCon, false, clientID, secret, fileID, Globals.meta, Globals.ignoreMeta);
if (native.Contains("QuillException: Document Limit Reached"))
{
MessageBox.Show("Document Limit Reached. You must purchase a license to continue, please visit www.QuillDigital.co.uk", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
Invoke(UpdateProgress, 30);
//Digitise
string fullText = string.Empty;
#endregion
#region Digitise
if (native.ToUpper().Equals("TRUE"))
{
if (cancelMain == true)
{
break;
}
Invoke(UpdateStatus, "Getting Text..");
fullText = servRef.GetFullTextByID(fileID, clientID, secret);
if (fullText.Contains("QuillException: Document Limit Reached"))
{
MessageBox.Show("Document Limit Reached. You must purchase a license to continue, please visit www.QuillDigital.co.uk", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
fullText = Regex.Replace(fullText, @"(\r\n){2,}", Environment.NewLine);
Invoke(UpdateProgress, 40);
}
else
{
Invoke(UpdateStatus, "Digitising..");
try
{
if (cancelMain == true)
{
break;
}
Invoke(UpdateProgress, 40);
string digitise = servRef.Digitise(fileName, fileID, clientID, secret, Globals.sqlCon, Globals.ocrType, strLineRemoval, Globals.dpi, "0");
if (digitise.Contains("QuillException: Document limit reached"))
{
MessageBox.Show("Document Limit Reached. You must purchase a license or more pages to continue, please visit www.QuillDigital.co.uk", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
if (digitise.Contains("File Corrupt- unable to convert"))
{
corruptFile = true;
}
}
catch (Exception ex)
{
//wait here for retort
}
Invoke(UpdateProgress, 50);
Invoke(UpdateStatus, "Getting Text..");
if (cancelMain == true)
{
break;
}
if (corruptFile == false)
{
fullText = servRef.GetFullTextByID(fileID, clientID, secret);
if (fullText.Contains("QuillException: Document limit reached"))
{
MessageBox.Show("Document Limit Reached. You must purchase a license to continue, please visit www.QuillDigital.co.uk", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
fullText = Regex.Replace(fullText, @"(\r\n){2,}", Environment.NewLine);
}
else
{
fullText = "File Corrupt - unable to convert";
}
}
string clausesFound = string.Empty;
string fields = string.Empty;
string translated = string.Empty;
#endregion
if (corruptFile == false)
{
Invoke(UpdateProgress, 50);
#region Translate
//Translate
if (!translationlang.ToUpper().Trim().Equals("NONE"))
{
Invoke(UpdateProgress, 60);
Invoke(UpdateStatus, "Translating..");
if (cancelMain == true)
{
break;
}
translated = servRef.TranslateByFileID(clientID, secret, Globals.sqlCon, fileID, translationlang);
if (translated.Contains("QuillException: Document Limit Reached"))
{
MessageBox.Show("Document Limit Reached. You must purchase a license to continue, please visit www.QuillDigital.co.uk", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
translated = Regex.Replace(translated, @"(\r\n){2,}", Environment.NewLine);
}
#endregion
//extract fields
#region Extract Fields
if (extractFields.Checked == true)
{
if (cancelMain == true)
{
break;
}
Invoke(UpdateProgress, 70);
Invoke(UpdateStatus, "Extracting Fields..");
if (cancelMain == true)
{
break;
}
fields = servRef.ExtractFieldsByFileID(fileID, fileName, clientID, secret, Globals.sqlCon, "0", GetConfiguration.GetConfigurationValueFields(), "0");
if (fields.Contains("QuillException: Document Limit Reached"))
{
MessageBox.Show("Document Limit Reached. You must purchase a license to continue, please visit www.QuillDigital.co.uk", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
fields = Regex.Replace(fields, @"(\r\n){1,}", Environment.NewLine);
}
#endregion
if (cancelMain == true)
{
break;
}
#region Check Clauses
if (clauses.Checked == true)
{
Invoke(UpdateProgress, 80);
Invoke(UpdateStatus, "Extracting Clauses..");
string clausesOUT = GetConfiguration.GetConfigurationValueClauses();
DataTable dtclausesFound = servRef.CheckForClausesByFileID(clientID, secret, Globals.sqlCon, fileID, fileName, clausesOUT);
if (clausesFound.Contains("QuillException: Document Limit Reached"))
{
MessageBox.Show("Document Limit Reached. You must purchase a license to continue, please visit www.QuillDigital.co.uk", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
if (dtclausesFound.Rows.Count <= 0)
{
clausesFound = "No Clauses Found.";
}
else
{
clausesFound = string.Empty;
foreach (DataRow row in dtclausesFound.Rows)
{
string tagOne = row["TagOne"].ToString();
if (string.IsNullOrEmpty(tagOne))
{
tagOne = "Tag One not found..";
}
string tagTwo = row["TagTwo"].ToString();
if (string.IsNullOrEmpty(tagTwo))
{
tagTwo = "Tag Two not found..";
}
string tagThree = row["TagThree"].ToString();
if (string.IsNullOrEmpty(tagThree))
{
tagThree = "Tag Three not found..";
}
string tagFour = row["TagFour"].ToString();
if (string.IsNullOrEmpty(tagFour))
{
tagFour = "Tag Four not found..";
}
string tagFive = row["TagFive"].ToString();
if (string.IsNullOrEmpty(tagFive))
{
tagFive = "Tag Five not found..";
}
string clauseFound = row["ClauseFound"].ToString();
string probablility = row["Probability"].ToString();
clausesFound = clausesFound + Environment.NewLine + tagOne + Environment.NewLine + tagTwo + Environment.NewLine + tagThree + Environment.NewLine + tagFour
+ Environment.NewLine + tagFive + Environment.NewLine + "Levenstein Distance: " + probablility + Environment.NewLine + Environment.NewLine;
}
}
clausesFound = Regex.Replace(clausesFound, @"(\r\n){2,}", Environment.NewLine);
//need to extract clauses
}
#endregion
}
#region NLP
if (Globals.nlpAnns != null)
{
Invoke(UpdateProgress, 90);
Invoke(UpdateStatus, "Running NLP..");
foreach (string anns in Globals.nlpAnns)
{
if (!string.IsNullOrEmpty(anns.Trim()))
{
Invoke(UpdateStatus, "Running " + anns + "..");
string nlpOUT = servRef.ProcessQuillNLP(fileID, anns.ToLower().Trim(), clientID, secret);
}
}
}
#endregion
Invoke(UpdateProgress, 90);
Invoke(UpdateStatus, "Writing Report..");
#region Write Report
string xmlDoc = string.Empty;
string xmlReport = servRef.GetReportByID(clientID, secret, fileID);
if (string.IsNullOrEmpty(tempFile))
{
xmlDoc = Path.GetFileNameWithoutExtension(file) + ".xml";
System.IO.File.WriteAllText(Path.Combine(savePath.Text, xmlDoc), xmlReport);
}
else
{
xmlDoc = Path.GetFileNameWithoutExtension(tempFile) + ".xml";
System.IO.File.WriteAllText(Path.Combine(savePath.Text, xmlDoc), xmlReport);
}
if (convertedFile == true)
{
File.Delete(runFile);
}
if (file.ToUpper().Trim().Equals(lastFile.ToUpper().Trim()) | tempFile.ToUpper().Trim().Equals(lastFile.ToUpper().Trim()))
{
break;
}
#endregion
#endregion
}
string pages = servRef.GetPagesLeft(clientID, secret);
Invoke(UpdatePages, "Pages left: " + pages);
}
return;
}
private void button2_Click(object sender, EventArgs e)
{
if (string.IsNullOrEmpty(folderPath.Text) && fileType.Checked == false)
{
MessageBox.Show("Please provide a folder path for the files you wish to run.", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
else
{
DialogResult run = MessageBox.Show("Run Quill?", "Quill", MessageBoxButtons.YesNo, MessageBoxIcon.Question);
if (run == DialogResult.Yes)
{
if (fileType.Checked == true && Globals.files == null)
{
MessageBox.Show("Please choose some files..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
try
{
File.WriteAllText(Path.Combine(savePath.Text.Trim(), "#QUILL#_OUTPUTS.txt"), "");
File.Delete(Path.Combine(savePath.Text.Trim(), "#QUILL#_OUTPUTS.txt"));
}
catch
{
MessageBox.Show("Unable to write to the Save directory.. Please choose a folder that you have write access to..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
if (string.IsNullOrEmpty(savePath.Text.Trim()))
{
MessageBox.Show("Please enter a Report directory..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
if (string.IsNullOrEmpty(folderPath.Text.Trim()) && fileType.Checked == false)
{
MessageBox.Show("Please enter a Run directory..", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
if (savePath.Text.Trim().ToUpper().Equals(folderPath.Text.ToUpper().Trim()))
{
MessageBox.Show("The Run Folder and Report Folder are the same. You need to create a Report Folder or browse to a different location", "Quill", MessageBoxButtons.OK, MessageBoxIcon.Warning);
button2.Enabled = true;
searchSubs.Enabled = true;
button5.Enabled = true;
button9.Enabled = true;
button7.Enabled = true;
folderPath.ReadOnly = false;
button11.Enabled = true;
button12.Enabled = true;
savePath.ReadOnly = false;
languages.Enabled = true;
button12.PerformClick();
return;
}
cancelMain = false;
translationlang = languages.Text.Trim();
languages.Enabled = false;
button5.Enabled = false;
button9.Enabled = false;
searchSubs.Enabled = false;
button7.Enabled = false;
button12.Enabled = false;
savePath.ReadOnly = true;
folderPath.ReadOnly = true;
fileType.Enabled = false;
button2.Enabled = false;
button11.Enabled = false;
progressBar.Visible = true;
if (extractFields.Checked == true)
{
if (GetConfiguration.GetConfigurationValueFields().Equals(string.Empty))
{
DialogResult extract = MessageBox.Show("You have not selected any fields to extract.. Click Fields To Extract to choose saved fields, or OK to continue with no extraction.", "Quill", MessageBoxButtons.OKCancel, MessageBoxIcon.Warning);
if (DialogResult.Cancel == extract)
{
searchSubs.Enabled = true;
button2.Enabled = true;
button7.Enabled = true;
button5.Enabled = true;
button9.Enabled = true;
if (fileType.Checked == false)
{
folderPath.ReadOnly = false;
}
button11.Enabled = true;
button12.Enabled = true;
savePath.ReadOnly = false;
languages.Enabled = true;
fileType.Enabled = true;
button6.PerformClick();
return;
}
else
{
extractFields.Checked = false;
button2.Enabled = false;
searchSubs.Enabled = false;
button5.Enabled = false;
button9.Enabled = false;
button12.Enabled = false;
languages.Enabled = false;
button7.Enabled = false;
savePath.ReadOnly = true;
fileType.Enabled = false;
}
}
}
if (clauses.Checked == true)
{
if (GetConfiguration.GetConfigurationValueClauses().Equals(string.Empty))
{
DialogResult extract = MessageBox.Show("You have not selected any clauses to extract.. Click Clauses To Extract to choose saved clauses, or OK to continue with no extraction.", "Quill", MessageBoxButtons.OKCancel, MessageBoxIcon.Warning);
if (DialogResult.Cancel == extract)
{
searchSubs.Enabled = true;
button5.Enabled = true;
button7.Enabled = true;
button9.Enabled = true;
button2.Enabled = true;
if (fileType.Checked == false)
{
folderPath.ReadOnly = false;
}
button11.Enabled = true;
button12.Enabled = true;
savePath.ReadOnly = false;
languages.Enabled = true;
fileType.Enabled = true;
button10.PerformClick();
return;
}
else
{
clauses.Checked = false;
button2.Enabled = false;
button7.Enabled = false;
searchSubs.Enabled = false;
button5.Enabled = false;
button9.Enabled = false;
button12.Enabled = false;
savePath.ReadOnly = true;
fileType.Enabled = false;
}
}
}
if (Globals.nlpAnns != null)
{
DialogResult anns = MessageBox.Show("You have chosed to extract the following NLP data: " + Globals.nlpAnns[0] + " This can add time to processing. Would you like to extract NLP data?", "Quill NLP", MessageBoxButtons.YesNo, MessageBoxIcon.Question);
if (anns == DialogResult.No)
{
Globals.nlpAnns = null;
}
}
main = new BackgroundWorker();
main.DoWork += Main_Run_DoWork;
main.RunWorkerCompleted += Main_Run_RunWorkerCompleted;
main.WorkerSupportsCancellation = true;
main.WorkerReportsProgress = true;
main.RunWorkerAsync();
}
else
{
return;
}
}
}