private void IterateFolders(DirectoryInfo dir, dtstResults dtst)
{
int i = DownloadOneFolderToDataset(dir, dtst, false, SearchType.cat_files);
foreach (var sDir in dir.GetDirectories())
{
IterateFolders(sDir, dtst);
}
}
private int QuerySecDB(dtstResults dtst)
{
var rows = 0;
var manager = new SECFormsManager();
var table = manager.GetFormsByFullTextSearchAndFormType
(this.textCriteria.Text, "(all)", LinkResponseType.NotClickableWithModifiedPath, "");
rows = table.Rows.Count;
var i = 0;
foreach (var row1 in table)
{
i--;
dtst.Result.AddResultRow(row1.FormDate, "SEC filing", row1.FormPartialURL, row1.FormPartialURL, "--No Preview--",
row1.CompanyName, string.Empty, string.Empty, i);
}
return(rows);
}
private void btnProcess_Click(object sender, EventArgs e)
{
if (!Validation())
{
return;
}
try
{
this.Cursor = Cursors.WaitCursor;
_summary.Clear();
_summary.Add("Searched for:" + HttpUtility.HtmlEncode(this.textCriteria.Text));
_summary.Add("Search time:" + DateTime.Now.ToString("yyyy-MM-dd HH:mm"));
var dir = new DirectoryInfo(this.textFolder.Text);
SetEstimates(dir);
var dtst = new dtstResults();
if (checkSEC.Checked)
{
try
{
var rows = QuerySecDB(dtst);
_summary.Add("Found SEC filings:" + rows.ToString());
this.progressBar1.Maximum += rows;
this.progressBar1.Value += rows;
this.textOutPut.AppendText("\r\nSEC Filings found: " + rows.ToString() + "\r\n");
}
catch (Exception ex)
{
this.textOutPut.AppendText("\r\nUnable to Query SecDB:\r\n" + ex.Message);
}
}
if (checkDocs.Checked)
{
if (this.checkIterate.Checked)
{
IterateFolders(dir, dtst);
}
else
{
DownloadOneFolderToDataset(dir, dtst, true, SearchType.cat_files);
}
}
if (checkEmail.Checked)
{
DownloadOneFolderToDataset(null, dtst, true, SearchType.email);
}
if (dtst.Result.Count > 0)
{
this.textOutPut.AppendText("\r\nPublishing to XLS\r\n");
PublishDatasetToExcel(dtst);
}
else
{
this.textOutPut.AppendText("\r\nNo matches found\r\n");
}
}
catch (Exception ex) { this.textOutPut.AppendText(ex.ToString() + "\r\n"); }
finally
{
this.progressBar1.Visible = false;
this.Cursor = Cursors.Default;
}
}
private int DownloadOneFolderToDataset(DirectoryInfo dir, dtstResults dtst, bool subFolders, SearchType searchType)
{
var position = 0;
while (true)
{
var res = QueryGoogoleDestop(dir, position, subFolders, searchType);
if (res == null || res.result == null)
{
break;
}
if (position == 0 && res.count > 0 && !subFolders)
{
try
{
this.textOutPut.AppendText("Path: " + dir.Name + " .. items: " + res.count + " ... ");
}
catch
{
this.textOutPut.AppendText("Path: " + this.GetFullDirNameSafe(dir) + " .. items: " + res.count + " ... ");
}
}
else if (subFolders)
{
this.textOutPut.AppendText("items: " + this.progressBar1.Value.ToString() + "\r\n");
}
foreach (var result1 in res.result)
{
DateTime d;
var snippet = result1.snippet;
var shortURL = searchType == SearchType.cat_files ? result1.url.ToLower().Replace(this.textFolder.Text, string.Empty) : result1.title;
try
{
var fi = new FileInfo(result1.url);
d = fi.LastWriteTime;
}
catch
{
d = DateTime.FromFileTime(result1.time);
}
if (string.IsNullOrEmpty(snippet))
{
snippet = " - no preview - ";
}
else
{
snippet = HttpUtility.HtmlDecode(snippet);
snippet = SECCrawler.BLL.SECFormsDocAnalyzer.StripHTML(snippet);
}
if (dtst.Result.FindByrsultId(result1.doc_id) == null)
{
dtst.Result.AddResultRow(d, result1.category, result1.url, shortURL, snippet,
result1.from, string.Empty, string.Empty, result1.doc_id);
}
position += 1;
}
if (this.progressBar1.Value + res.result.Length <= progressBar1.Maximum)
{
this.progressBar1.Value += res.result.Length;
if ((DateTime.Now - _lastRefresh).TotalSeconds >= 3)
{
this.Refresh();
System.Threading.Thread.Sleep(500);
_lastRefresh = DateTime.Now;
}
}
if (position >= (res.count))
{
break;
}
}
return(position);
}
