public void TestGetDownloadQItem()
{
var dal = new SECFormsManager();
var dtst = dal.GetFormDownloadQueue();
Assert.IsNotNull(dtst);
}
private void FillFormTypesAndSavedSearches()
{
dropFormTypes.Items.Clear();
DropSavedSearches.Items.Clear();
var dal = new SECFormsManager();
var dal2 = new SavedSearchManager();
var table = dal.GetFormTypes();
dropFormTypes.Items.Add("(all)");
foreach (var rowFt in table)
{
dropFormTypes.Items.Add(rowFt.formtype);
}
var table2 = dal2.GetSearches(SearchType);
DropSavedSearches.DataSource = table2;
DropSavedSearches.DataTextField = table2.descriptionColumn.ColumnName;
DropSavedSearches.DataValueField = table2.descriptionColumn.ColumnName;
DropSavedSearches.DataBind();
var table3 = dal2.GetSearches(AutoSaveSearchType);
foreach (var searchRow in table3)
{
searchRow.description = searchRow.description.ToLower();
}
DropSavedSearches2.DataSource = table3;
DropSavedSearches2.DataTextField = table3.descriptionColumn.ColumnName;
DropSavedSearches2.DataValueField = table3.descriptionColumn.ColumnName;
DropSavedSearches2.DataBind();
}
protected void ButtonSearch_Click(object sender, EventArgs e)
{
if (TextSearchForms.Text.Trim() == string.Empty)
{
Literal1.Text = "<span class=alarm>Please enter search parameters</span>";
return;
}
try
{
var manager = new SECFormsManager();
if (TextFormTypes.Text.Trim() == string.Empty)
{
TextFormTypes.Text = "%";
}
var table = manager.GetFormsByFullTextSearchAndFormType(TextSearchForms.Text, TextFormTypes.Text, LinkResponseType.NotChanged, string.Empty);
TextFormsList.Text = string.Empty;
var vw = new DataView(table, "", "companyname asc, formdate asc", DataViewRowState.CurrentRows);
var n = TextBoxCompany.Text.ToLower().Trim();
foreach (DataRowView rowV in vw)
{
var row = (secCrawlerData.tblSEC_FormsRow)rowV.Row;
if (!CheckCompany.Checked || row.CompanyName.ToLower().Contains(n))
{
TextFormsList.Text += row.FormID.ToString() + "\r\n";
}
}
}
catch (Exception ex)
{
Literal1.Text = "<span class=alarm>" + ex.Message + "</span>";
TextFormsList.Text = string.Empty;
}
}
protected void btnProcesXLS_Click(object sender, EventArgs e)
{
var criteria = this.textCriteria.Text.Trim();
if (criteria == string.Empty)
{
LabelStatus.Text = "Enter search criteria before searching";
return;
}
if (TextFormTypes.Text.Trim() == string.Empty)
{
TextFormTypes.Text = "%";
}
var dal = new SECFormsManager();
var table = dal.GetFormsByFullTextSearchAndFormTypeWithSnippet(criteria, dropFormTypes.SelectedItem.Text, LinkResponseType.NotChanged, "./ViewEditForm?FormID=", TextBoxCompanyName.Text + '%');
var dtst = new DataSet();
dtst.Tables.Add(table);
table.TableName = "Search_Results";
Response.ContentType = "text/xml";
Response.ClearContent();
Response.AppendHeader("Content-disposition", "Attachment; filename=SECfilingsSearchResult.xls");
//attachments
var summary = new List <string>
{
"Searched for:" + HttpUtility.HtmlEncode(this.textCriteria.Text),
"Documents found:" + table.Rows.Count.ToString(),
"Search Date:" + DateTime.Now.ToString("yyyy-MM-dd HH:mm")
};
COI.Util.ExcelEngine.Convert(dtst, Response.OutputStream, 2, 2, summary);
Response.End();
}
public void TestFullTextSearch()
{
var dal = new SECFormsManager();
var table = dal.GetFormsByFullTextSearchAndFormType("Chris", "(all)", LinkResponseType.NotChanged, string.Empty);
Assert.IsNotNull(table);
Assert.Greater(table.Rows.Count, 0);
}
public void TestGetIngestQItem()
{
var dal = new SECFormsManager();
var dtst = dal.GetFormIngestQueue();
Assert.IsNotNull(dtst);
Assert.IsTrue(dtst.tblSEC_Forms.IsInitialized);
}
public int ProcessQueue()
{
var itemsProcessed = 0;
try
{
var manager = new SECFormsManager();
var dtst = manager.GetFormDownloadQueue();
Log("Read" + dtst.tblSEC_Forms.Rows.Count);
var ftpClient = new FtpClient(_remotePath, _basePath);
//to prevent concurrency errors, mark as downloaded immediately:
foreach (var form in dtst.tblSEC_Forms)
{
form.LastDownloaded = DateTime.Now;
}
manager.Save(dtst);
Log("Blocked rows from parallel processes");
foreach (var form in dtst.tblSEC_Forms)
{
string error;
Log("form type: " + form.FormType + " company name: " + form.CompanyName);
ftpClient.Download(form.FormPartialURL, _useConsole, out error);
if (error == null)
{
error = string.Empty;
}
if (error == string.Empty)
{
form.LastDownloaded = DateTime.Now;
itemsProcessed++;
}
else
{
Log(error);
}
}
manager.Save(dtst);
Log("saved rows");
dtst.Clear();
}
catch (Exception ex)
{
Log(ex.Message);
}
return(itemsProcessed);
}
private int QuerySecDB(dtstResults dtst)
{
var rows = 0;
var manager = new SECFormsManager();
var table = manager.GetFormsByFullTextSearchAndFormType
(this.textCriteria.Text, "(all)", LinkResponseType.NotClickableWithModifiedPath, "");
rows = table.Rows.Count;
var i = 0;
foreach (var row1 in table)
{
i--;
dtst.Result.AddResultRow(row1.FormDate, "SEC filing", row1.FormPartialURL, row1.FormPartialURL, "--No Preview--",
row1.CompanyName, string.Empty, string.Empty, i);
}
return(rows);
}
private void FillFormTypes()
{
var dal = new SECFormsManager();
var table = dal.GetFormTypes();
dropFormTypes.Items.Add("(all)");
var li = new ListItem("(all 10-Qs)", "10%q%");
dropFormTypes.Items.Add(li);
li = new ListItem("(all 10-Ks)", "10%k%");
dropFormTypes.Items.Add(li);
li = new ListItem("(all 8-Ks)", "8%k%");
dropFormTypes.Items.Add(li);
foreach (var rowFt in table)
{
dropFormTypes.Items.Add(rowFt.formtype);
}
}
public int Snipe()
{
var generatedCount = 0;
var adapter = new SECFormsManager();
var manager = new SECFormsSnippetManager();
var analyzer = new SECFormsDocAnalyzer();
var formsTable = adapter.GetFormSniperQueue();
var snippertsTable = new secCrawlerData.tblSEC_Forms_snippetDataTable();
foreach (var form1 in formsTable)
{
var newSnippet = analyzer.GetFormSnippet(form1.FormID, 1);
snippertsTable.AddtblSEC_Forms_snippetRow(
form1.FormID, 1, newSnippet, "default snippet"
);
Log(newSnippet);
generatedCount++;
}
var snippetTypes = new[] { "former name" };//,"market information"};
var snippetCriteria = new[] { "FORMER CONFORMED NAME", "Market Information" };
var documentIndex = new[] { 0, 1 };
for (var i = 0; i < snippetTypes.Length; i++)
{
formsTable = adapter.GetFormSniperQueue(snippetTypes[i], snippetCriteria[i]);
foreach (var form1 in formsTable)
{
var newSnippet = analyzer.GetFormSnippet(form1.FormID, documentIndex[i], snippetCriteria[i]);
if (newSnippet.Length == 0)
{
continue;
}
snippertsTable.AddtblSEC_Forms_snippetRow(
form1.FormID, documentIndex[i], newSnippet, snippetTypes[i]);
Log(newSnippet);
generatedCount++;
}
manager.Save(snippertsTable);
}
return(generatedCount);
}
protected void btnProcess_Click(object sender, EventArgs e)
{
LabelStatus.Text = string.Empty;
var criteria = this.textCriteria.Text.Trim();
if (criteria == string.Empty)
{
LabelStatus.Text = "Enter search criteria before searching";
return;
}
try
{
if (TextFormTypes.Text.Trim() == string.Empty)
{
TextFormTypes.Text = "%";
}
var dal = new SECFormsManager();
var table = dal.GetFormsByFullTextSearchAndFormTypeWithSnippet(criteria, TextFormTypes.Text,
LinkResponseType.ClickableWithModifiedLink, "./ViewEditForm.aspx?formId=", TextBoxCompanyName.Text + "%");
GridView2.AutoGenerateColumns = false;
var v = new DataView(table, "", this.DropSortBy.SelectedValue, DataViewRowState.CurrentRows);
GridView2.DataSource = v;
GridView2.DataBind();
if (GridView2.Columns.Count == 9)
{
GridView2.Columns.RemoveAt(8);
GridView2.Columns.RemoveAt(7);
GridView2.Columns.RemoveAt(6);
GridView2.Columns.RemoveAt(1);
}
this.LabelStatus.Text = "Records found : " + table.Rows.Count;
}
catch (Exception ex)
{
this.LabelStatus.Text = ex.Message;
}
}
private DtstFilingSearchResult GetDtst()
{
Companies = new List <string>();
MinDates = new List <DateTime>();
MaxDates = new List <DateTime>();
KeywordsPerCo = new List <List <string> >();
var manager = new SECFormsManager();
var analyzer = new SECFormsDocAnalyzer();
var addPars = int.Parse(DropProximityAdd.SelectedValue);
var keywords = this.TextSearchPars.Text.Replace("\r", "").Split('\n');
var dtst = new DtstFilingSearchResult();
var table = dtst.MPSresults;
var i = 0;
foreach (var line in this.TextFormsList.Text.Replace("\r\n", "\n").Split('\n'))
{
try
{
if (string.IsNullOrEmpty(line))
{
continue;
}
var formID = new Guid(line);
var form = manager.GetFormsByFormID(formID)[0];
var companyIndex = Companies.IndexOf(form.CompanyName);
if (companyIndex == -1)
{
Companies.Add(form.CompanyName);
MinDates.Add(form.FormDate);
MaxDates.Add(form.FormDate);
KeywordsPerCo.Add(new List <string>());
companyIndex = Companies.IndexOf(form.CompanyName);
}
else //keep min and max date per company
{
if (form.FormDate < MinDates[companyIndex])
{
MinDates[companyIndex] = form.FormDate;
}
if (form.FormDate > MaxDates[companyIndex])
{
MaxDates[companyIndex] = form.FormDate;
}
}
var keywordsThisCo = KeywordsPerCo[companyIndex];
foreach (var keyword in keywords)
{
if (keyword.Trim() == string.Empty)
{
continue;
}
var pars = analyzer.GetFormSubDocParagraphsWithSearch(
formID, -1, keyword, addPars, CheckMultiple.Checked);
if (pars.Count == 0)
{
continue;
}
i++;
if (keywordsThisCo.IndexOf(keyword) == -1)
{
keywordsThisCo.Add(keyword);
}
foreach (var par in pars)
{
if (par.Trim() != string.Empty)
{
table.AddMPSresultsRow(i, form.FormDate, form.CompanyName,
form.FormType, keyword, par, form.FormPartialURL, formID);
}
}
}
}
catch (Exception ex)
{
Literal1.Text = ex.ToString();
}
}
return(dtst);
}
