private void IterateFolders(DirectoryInfo dir, dtstResults dtst) { int i = DownloadOneFolderToDataset(dir, dtst, false, SearchType.cat_files); foreach (var sDir in dir.GetDirectories()) { IterateFolders(sDir, dtst); } }
private int QuerySecDB(dtstResults dtst) { var rows = 0; var manager = new SECFormsManager(); var table = manager.GetFormsByFullTextSearchAndFormType (this.textCriteria.Text, "(all)", LinkResponseType.NotClickableWithModifiedPath, ""); rows = table.Rows.Count; var i = 0; foreach (var row1 in table) { i--; dtst.Result.AddResultRow(row1.FormDate, "SEC filing", row1.FormPartialURL, row1.FormPartialURL, "--No Preview--", row1.CompanyName, string.Empty, string.Empty, i); } return(rows); }
private void btnProcess_Click(object sender, EventArgs e) { if (!Validation()) { return; } try { this.Cursor = Cursors.WaitCursor; _summary.Clear(); _summary.Add("Searched for:" + HttpUtility.HtmlEncode(this.textCriteria.Text)); _summary.Add("Search time:" + DateTime.Now.ToString("yyyy-MM-dd HH:mm")); var dir = new DirectoryInfo(this.textFolder.Text); SetEstimates(dir); var dtst = new dtstResults(); if (checkSEC.Checked) { try { var rows = QuerySecDB(dtst); _summary.Add("Found SEC filings:" + rows.ToString()); this.progressBar1.Maximum += rows; this.progressBar1.Value += rows; this.textOutPut.AppendText("\r\nSEC Filings found: " + rows.ToString() + "\r\n"); } catch (Exception ex) { this.textOutPut.AppendText("\r\nUnable to Query SecDB:\r\n" + ex.Message); } } if (checkDocs.Checked) { if (this.checkIterate.Checked) { IterateFolders(dir, dtst); } else { DownloadOneFolderToDataset(dir, dtst, true, SearchType.cat_files); } } if (checkEmail.Checked) { DownloadOneFolderToDataset(null, dtst, true, SearchType.email); } if (dtst.Result.Count > 0) { this.textOutPut.AppendText("\r\nPublishing to XLS\r\n"); PublishDatasetToExcel(dtst); } else { this.textOutPut.AppendText("\r\nNo matches found\r\n"); } } catch (Exception ex) { this.textOutPut.AppendText(ex.ToString() + "\r\n"); } finally { this.progressBar1.Visible = false; this.Cursor = Cursors.Default; } }
private int DownloadOneFolderToDataset(DirectoryInfo dir, dtstResults dtst, bool subFolders, SearchType searchType) { var position = 0; while (true) { var res = QueryGoogoleDestop(dir, position, subFolders, searchType); if (res == null || res.result == null) { break; } if (position == 0 && res.count > 0 && !subFolders) { try { this.textOutPut.AppendText("Path: " + dir.Name + " .. items: " + res.count + " ... "); } catch { this.textOutPut.AppendText("Path: " + this.GetFullDirNameSafe(dir) + " .. items: " + res.count + " ... "); } } else if (subFolders) { this.textOutPut.AppendText("items: " + this.progressBar1.Value.ToString() + "\r\n"); } foreach (var result1 in res.result) { DateTime d; var snippet = result1.snippet; var shortURL = searchType == SearchType.cat_files ? result1.url.ToLower().Replace(this.textFolder.Text, string.Empty) : result1.title; try { var fi = new FileInfo(result1.url); d = fi.LastWriteTime; } catch { d = DateTime.FromFileTime(result1.time); } if (string.IsNullOrEmpty(snippet)) { snippet = " - no preview - "; } else { snippet = HttpUtility.HtmlDecode(snippet); snippet = SECCrawler.BLL.SECFormsDocAnalyzer.StripHTML(snippet); } if (dtst.Result.FindByrsultId(result1.doc_id) == null) { dtst.Result.AddResultRow(d, result1.category, result1.url, shortURL, snippet, result1.from, string.Empty, string.Empty, result1.doc_id); } position += 1; } if (this.progressBar1.Value + res.result.Length <= progressBar1.Maximum) { this.progressBar1.Value += res.result.Length; if ((DateTime.Now - _lastRefresh).TotalSeconds >= 3) { this.Refresh(); System.Threading.Thread.Sleep(500); _lastRefresh = DateTime.Now; } } if (position >= (res.count)) { break; } } return(position); }