/// <summary>
/// When the initial report on a bill is generated, it is in so-called "canonical" form, meaning that
/// no position is taken, there is no meaningful summary, and so on. In essence, a blank form is generated.
/// The user is then given the opportunity to edit the blank form, turning it into an actual report on the bill.
/// Once that editing is complete, the database BillRow table is updated with the position given in the
/// edited bill report.
/// </summary>
/// <param name="measure"></param>
public static void GenerateCanonicalReport(string measure)
{
// Generate the canonical bill
BillRow row = BillRow.Row(BillUtils.Ensure4DigitNumber(measure));
List <string> contents = BaseReportContents(row, string.Empty);
string path = $"{Path.Combine(Config.Instance.HtmlFolder, BillUtils.EnsureNoLeadingZerosBill(measure))}.html";
WriteTextFile(contents, path);
// Let the user edit the canonical bill, which has been written to disk.
var process = Process.Start("notepad.exe", path);
if (process != null)
{
process.WaitForExit();
}
else
{
LogAndShow($"CreateNewReports.GenerateCanonicalReport: Failed to start Notepad for {path}.");
}
// Update the database position
BillRow.UpdatePosition(BillUtils.Ensure4DigitNumber(measure), "");
GetPositionAndSummary(path, out List <string> summary, out List <string> position_list);
string first_line = position_list.FirstOrDefault();
string position = first_line != null?Regex.Replace(first_line, ".*?:(.*)", "$1") : "None Specified";
BillRow.UpdatePosition(measure, position.Trim());
LogAndShow($"Completed {path}");
}
public void Run(Form1 form1)
{
var start_time = DateTime.Now;
try {
LogAndDisplay(form1.txtImportProgress, "Determining most recent version of each bill.");
// Need to know which lob file describes the most recent version of each bill
GlobalData.MostRecentEachBill = MostRecentBills.Identify(Config.Instance.BillsFolder);
// Need to copy GlobalData.Profiles to a temporary, update the temporary, then copy the temporary back.
// BillRanker.Compute(ref GlobalData.Profiles) results in
// A property or indexer may not be passed as an out or ref parameter.
// Property "Profiles" access returns temporary value.
// "ref" argument must be an assignable value, field or array element.
LogAndDisplay(form1.txtImportProgress, "Computing positive and negative scores for all bills.");
List <BillProfile> mr_profiles = Profiles(GlobalData.MostRecentEachBill); // Prepare for ranking the bills
BillRanker.Compute(ref mr_profiles);
GlobalData.Profiles = mr_profiles;
LogAndDisplay(form1.txtImportProgress, "Initializing BillRows from BILL_TBL.dat and elsewhere.");
string path = $"{Path.Combine(Config.Instance.BillsFolder, "BILL_TBL.dat")}";
BuildBillRowsTable(form1, path, GlobalData.Profiles);
// Update the database BillRows table
LogAndDisplay(form1.txtImportProgress, "Clearing and rewriting BillRows in the database.");
BillRow.WriteRowset(GlobalData.BillRows);
} catch (Exception ex) {
LogAndThrow($"InitializeBillRows.Run: {ex.Message}.");
}
var elapsed = DateTime.Now - start_time;
LogAndDisplay(form1.txtImportProgress, $"BillRows initialization complete. Elapsed Time: {elapsed.ToString("c")} ");
}
private static MatchCollection SingleFile(BillRow row, Regex rx)
{
var result = string.Empty;
var contents = ContentsWithHTML(row);
var trimmed = RemoveHTML(contents);
//File.WriteAllText("C:/Scratch/temp.txt", contents);
return(rx.Matches(trimmed));
}
/// <summary>
/// Bills with no recorded positions are considered bills with no reports.
/// </summary>
/// <returns></returns>
private IOrderedEnumerable <BillRow> CollectNoPositionBills()
{
var all_bills = BillRow.RowSet(); // All bills for the current biennium.
var result =
from item in all_bills
where ((item.Position == string.Empty) && (item.NegativeScore > 0))
orderby item.NegativeScore descending select item;
return(result);
}
private DateTime DateFromHistoryTable(string path)
{
string bill = Path.GetFileNameWithoutExtension(path);
BillRow row = BillRow.Row(BillUtils.Ensure4DigitNumber(bill));
string name_ext = Path.GetFileName(row.Lob); // BillVersionTable bill_xml is unique
BillVersionRow bv_row = GlobalData.VersionTable.Scalar(name_ext);
List <BillHistoryRow> history = GlobalData.HistoryTable.RowSet(bv_row.BillID);
DateTime.TryParse(history.First().ActionDate, out DateTime date_result);
return(date_result);
}
private void Create(string bill, string path)
{
try {
BillRow row = BillRow.Row(BillUtils.Ensure4DigitNumber(bill));
List <string> contents = CreateIndividualReport.ReportContents(row, path);
WriteTextFile(contents, path);
var message = $"Regenerated {row.Bill} report.";
BaseController.LogThis(message);
} catch (Exception ex) {
LogAndThrow($"IndividualReport.Create: {ex.Message}.");
}
}
/// <summary>
/// This method reads a raw bill file, trims various introductory information, and returns the actual bill text.
/// It does not remove HTML control information embedded in that text.
/// </summary>
/// <param name="row"></param>
/// <returns></returns>
private static string ContentsWithHTML(BillRow row)
{
var contents = string.Empty;
if (File.Exists(row.Lob))
{
contents = FileUtils.FileContents(row.Lob);
var end_marker = "</caml:Preamble>";
var offset = contents.IndexOf(end_marker) + end_marker.Length;
contents = contents.Substring(offset);
}
return(contents);
}
/// <summary>
/// Sets contents of some database tables, using data from table files contained in the zipped download.
/// </summary>
/// <returns>True if all is well, False if unable to access table files expected to be present</returns>
public static bool EnsureGlobalData() // public so that XUnit TestNewOrChangePrefix can ensure table contents are available
{
bool result = true;
Config.Instance.ReadYourself(); // Start of configuration data lifetime
GlobalData.Profiles = new List <BillProfile>();
if (GlobalData.BillRows == null)
{
GlobalData.BillRows = BillRow.RowSet();
}
if (GlobalData.HistoryTable == null)
{
string path = Path.Combine(Config.Instance.BillsFolder, "BILL_HISTORY_TBL.dat");
if (File.Exists(path))
{
GlobalData.HistoryTable = new BillHistoryTable(path);
}
else
{
result = false;
}
}
if (GlobalData.VersionTable == null)
{
string path = Path.Combine(Config.Instance.BillsFolder, "BILL_VERSION_TBL.dat");
if (File.Exists(path))
{
GlobalData.VersionTable = new BillVersionTable(path);
}
else
{
result = false;
}
}
if (GlobalData.LocationTable == null)
{
string path = Path.Combine(Config.Instance.BillsFolder, "LOCATION_CODE_TBL.dat");
if (File.Exists(path))
{
GlobalData.LocationTable = new LocationCodeTable(path);
}
else
{
result = false;
}
}
GlobalData.MostRecentEachBill = new List <Bill_Identifier>();
return(result);
}
/// <summary>
/// Allow the user to update the current bill report.
/// If the position is changed, the database BillRow table is updated with the new position.
/// </summary>
/// <param name="measure"></param>
private void UpdateReport(string measure)
{
string path = $"{Path.Combine(Config.Instance.HtmlFolder, BillUtils.EnsureNoLeadingZerosBill(measure))}.html";
var process = Process.Start("notepad.exe", path);
if (process != null)
{
process.WaitForExit();
}
else
{
BaseController.LogAndShow($"CreateNewReports.GenerateCanonicalReport: Failed to start Notepad for {path}.");
}
// Update the database position
BillRow.UpdatePosition(BillUtils.Ensure4DigitNumber(measure), "");
BaseController.LogAndShow($"Update for {path} is complete.");
}
/// <summary>
/// Before a bill is chaptered, the last action is the .First() line in the history.
/// When a bill is chaptered, its history may not end with the "Chaptered by Secretary of State" because
/// usually multiple actions take place on the same day. Therefore, for a chaptered bill, report
/// the line containing "Chaptered by Secretary of State". It may not be the first line in the history.
/// </summary>
/// <param name="row">BillRow describing the bill being processed</param>
/// <returns></returns>
protected static string FindLastAction(BillRow row)
{
string name_ext = Path.GetFileName(row.Lob); // BillVersionTable bill_xml is unique
BillVersionRow bv_row = GlobalData.VersionTable.Scalar(name_ext);
List <BillHistoryRow> history = GlobalData.HistoryTable.RowSet(bv_row.BillID);
string result = "Could not find last action.";
if (row.MeasureState != "Chaptered")
{
result = history.First().Action;
}
else
{
var want_this = history.Find(x => x.Action.Contains("Chaptered by Secretary of State"));
result = want_this.Action;
}
return(result);
}
/// <summary>
/// Build GlobalData.BillRows from scratch.
/// The steps for this are
/// 1. Import the BILL_TBL.dat data into GlobalData.BillRows. This fills 9 of the 15 BillRow fields.
/// </summary>
/// <param name="form1">The main (Form1) display form. Display messages and progress here.</param>
/// <param name="path_bill_tbl">Fully-qualified path to BILL_TBL.dat</param>
/// <param name="mostRecentEachBill">For each bill, the Bill_Identifier identifying the most recent version</param>
private void BuildBillRowsTable(Form1 form1, string path_bill_tbl, List <BillProfile> profiles)
{
// Import BILL_TBL.dat, which is the legislative site's information on bills before the legislature.
// Trim all non-bill items during the import -- we want only type AB and SB (Assembly and Senate Bills)
var bill_table_wrapper = new Bill_Tbl_Wrapper();
bill_table_wrapper.ReadYourself(); // Import BILL_TBL.dat
List <BillRow> rows_with_positions = BillRow.RowsetByQuery("Select * from Billrows Where Position <> ''");
List <BillRow> all_rows = BillRow.RowSet();
List <string> reports = BillUtils.HtmlFolderContents();
// Re-create GlobalData.BillRows, using data from bill_table_wrapper and elsewhere.
GlobalData.BillRows.Clear();
List <string> SkipIf = new List <String>()
{
"Chaptered", "Died", "Enrolled", "Failed", "Failed Passage in Committee", "Vetoed"
};
foreach (var item in bill_table_wrapper)
{
// Don't process bills that will not progress further in the legislature.
//string result = SkipIf.FirstOrDefault(s => s == item.Current_status);
//if (result != null) continue;
// Use data from bill_table_wrapper. Some fields are left blank.
var bill_row = new BillRow();
bill_row.MeasureType = item.Measure_type; // e.g. AB
bill_row.MeasureNum = BillUtils.Ensure4DigitNumber(item.Measure_num); // e.g. 0010
bill_row.Bill = $"{bill_row.MeasureType}{BillUtils.EnsureNoLeadingZerosBill(bill_row.MeasureNum)}"; // e.g. AB10
//bill_row.Lob = item.;
//bill_row.NegativeScore = item;
//bill_row.PositiveScore = item;
//bill_row.Position = item; // Hand Entered, e.g. Monitor
bill_row.BillVersionID = VersionID(item); // e.g. 20190AB199INT
//bill_row.Author = item;
//bill_row.Title = item;
bill_row.Location = item.Current_location; // e.g. CX08
bill_row.Location2nd = item.Current_secondary_loc; // e.g. Committee
bill_row.MeasureState = item.Measure_state; // e.g. Amended Assembly
bill_row.CurrentHouse = item.Current_house; // e.g. Assembly
bill_row.CurrentStatus = item.Current_status; // e.g. In Committee Process
// Obtain the author, title, lob file path, and positive/negative scores from the profile for this bill
var four_digit_billid = BillUtils.Ensure4DigitNumber(bill_row.Bill);
var profile = (from x in profiles where x.Identifier.BillID == four_digit_billid select x).First();
if (profile != null)
{
bill_row.Author = profile.Identifier.Author;
bill_row.Title = profile.Identifier.Title;
bill_row.Lob = profile.Identifier.LobPath;
bill_row.NegativeScore = profile.NegScore;
bill_row.PositiveScore = profile.PosScore;
}
else
{
throw new ApplicationException($"InitializeBillRows.BuildBillRowsTable: Bill {bill_row.Bill} is present in bill_table_wrapper, but not in GlobalData.Profiles.");
}
// Fill in the Position data -- the position we are taking on this bill. If we have a position, it is
// in one of two places
// 1. The database BillRows table (not all bills have a report), or
var pos = (from x in rows_with_positions where x.Bill == bill_row.Bill select x).FirstOrDefault();
if (pos != null)
{
bill_row.Position = pos.Position;
}
// 2. If that table hasn't been updated, in the actual report
// If the two are in conflict, the bill report wins.
var short_id = BillUtils.EnsureNoLeadingZerosBill(bill_row.Bill);
var report_file = (from x in reports where x.Contains($"{short_id}.html") select x).FirstOrDefault();
if (report_file != null)
{
var report = new BillReport(report_file);
bill_row.Position = report.Position;
}
// Add this row to GlobalData.BillRows
GlobalData.BillRows.Add(bill_row);
}
// Sort the table before returning it. Ordered by bill ID, e.g. AB0001, communicates well
GlobalData.BillRows = GlobalData.BillRows.OrderBy(a => a.Bill).ToList();
}
/// <summary>
/// Generate a new individual report or re-generate one that already exists.
/// </summary>
/// <param name="row">Bill description from the BillRows table</param>
/// <param name="path">Path to the .lob file for the bill's current version</param>
/// <returns></returns>
protected static List <string> BaseReportContents(BillRow row, string path)
{
string name_ext = Path.GetFileName(row.Lob); // BillVersionTable bill_xml is unique
BillVersionRow bv_row = GlobalData.VersionTable.Scalar(name_ext);
List <BillHistoryRow> history = GlobalData.HistoryTable.RowSet(bv_row.BillID);
var location_code = history.First().TernaryLocation;
var location_code_row = GlobalData.LocationTable.Scalar(location_code);
string appropriation = bv_row.Appropriation;
string author = row.Author;
string bill_id = row.Bill;
string fiscal = bv_row.FiscalCommittee;
string house = history.First().PrimaryLocation;
string last_action = FindLastAction(row);
string location = location_code_row == null?BillUtils.WhenNullLocationCode(history) : location_code_row.Description;
string local_pgm = bv_row.LocalProgram;
string number = row.MeasureNum.TrimStart('0');
string title = row.Title;
string type_house = $"{bill_id.First()}B";
string vers_id = row.BillVersionID;
string vote = bv_row.VoteRequired;
// Position and Summary data come from the previous version of the bill report
// If the passed path is null or empty, then this method was called when no previous report exists.
// When regenerating a report, there is a previous report.
var summary = new List <string>();
var position = new List <string>();
var shortsummary = string.Empty;
var committees = string.Empty;
var likelihood = string.Empty;
if (CommonUtils.IsNullOrEmptyOrWhiteSpace(path))
{
// do nothing
}
else
{
summary = PreviousReport.Summary(path);
position = PreviousReport.Position(path);
shortsummary = PreviousReport.ShortSummary(path);
committees = PreviousReport.Committees(path);
likelihood = PreviousReport.Likelihood(path);
}
// With all necessary data obtained, generate the report file.
// Both initial creation (new bill) and re-generation are processed here.
List <string> result = BeginIndividualReport(type_house, number, author, title);
// Review
result.AddRange(ReportReview(summary));
// Position
result.AddRange(ReportPosition(position));
// Short Summary, Committees Prediction and Passage Likelihood
result.AddRange(ReportSummaryPredictLikelihood(shortsummary, committees, likelihood));
// Status, Location, etc
result.AddRange(ReportStatusLocationEtc(location, last_action, vote, appropriation, fiscal, local_pgm, history));
// Bill History
result.AddRange(ReportHistory(history));
return(result);
}
/// <summary>
/// This background worker performs the time-consuming task of searching all current bills for two words
/// that are within a specified distance of each other.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void bgw_SearchNearby(object sender, DoWorkEventArgs e)
{
BackgroundWorker worker = sender as BackgroundWorker;
if (worker == null)
{
throw new ApplicationException("bgw_SearchNearby failed to instantiate BackgroundWorker");
}
// Ensure passed strings are valid. If so take the minimum and maximum distance values.
string param_problem = "";
bgwSearchArguments args = (bgwSearchArguments)e.Argument;
if (NEWs(args.word1))
{
BuildErrorMessage(ref param_problem, "First word must be specified");
}
if (NEWs(args.word2))
{
BuildErrorMessage(ref param_problem, "Second word must be specified");
}
if (NEWs(args.text_min))
{
BuildErrorMessage(ref param_problem, "Minimum size must be specified");
}
if (NEWs(args.text_max))
{
BuildErrorMessage(ref param_problem, "Maximum must be specified");
}
args.word1 = args.word1.Trim(); args.word2 = args.word2.Trim();
args.text_min = args.text_min.Trim(); args.text_max = args.text_max.Trim();
if (!Int16.TryParse(args.text_min, out short min_dist))
{
BuildErrorMessage(ref param_problem, "Unable to parse Minimum distance");
}
if (min_dist < 0)
{
BuildErrorMessage(ref param_problem, "Minimum distance must not be negative");
}
if (!Int16.TryParse(args.text_max, out short max_dist))
{
BuildErrorMessage(ref param_problem, "Unable to parse Maximum distance");
}
if (max_dist < 0)
{
BuildErrorMessage(ref param_problem, "Maximum distance must not be negative");
}
if (param_problem.Length > 0)
{
throw new ApplicationException(param_problem);
}
GlobalData.BillRows = BillRow.RowSet();
var bills = GlobalData.BillRows.OrderBy(item => item.Bill).ToList();
int one_percent = bills.Count / 100;
Regex rx = CreateRegex(args.word1, args.word2, min_dist, max_dist);
int progress_bar_value = 0;
using (StreamWriter sw_matches = new StreamWriter("C:/Scratch/Scout2_Matches.txt")) {
using (StreamWriter sw_bills = new StreamWriter("C:/Scratch/Scout2_Bills.txt")) {
int count = 0;
foreach (var bill in bills)
{
MatchCollection found = SingleFile(bill, rx);
if (found.Count > 0)
{
sw_bills.WriteLine($"{bill.Bill}");
sw_matches.WriteLine($"{bill.Bill} {bill.Title} ({bill.Author})");
foreach (var match in found)
{
sw_matches.WriteLine($"\t{Cleanup(match.ToString())}");
}
}
if (++count % one_percent == 0)
{
worker.ReportProgress(++progress_bar_value);
}
}
}
}
}
// Create a Bill Report, given a bill identifier such as AB12
public static List <string> ReportContents(BillRow row, string path)
{
return(BaseReportContents(row, path));
}
