private static void writeModelText(borsaModel borsaModel) { StringBuilder sb = new StringBuilder(); PropertyInfo[] pis = borsaModel.GetType().GetProperties(); sb.AppendLine("<tr>"); foreach (PropertyInfo pi in pis) { string value = (pi.GetValue (borsaModel, null) ?? "").ToString(); sb.Append("<td style='border:1px solid black;'>"); if (pi.Name == "Link") sb.Append("<a href=\""+ value +"\">"); sb.Append(value); if (pi.Name == "Link") sb.Append("</a>"); sb.AppendLine("</td>"); } sb.AppendLine("</tr>"); lock (syncFileObj) { File.AppendAllText("output.html", sb.ToString()); } }
private static borsaModel getBorsaModel(borsaModel model) { string link = model.Link; WebClient client = new WebClient(); client.Headers.Add("User-Agent", ConfigurationManager.AppSettings["browserAgent"]); client.Encoding = Encoding.UTF8; string ret = ""; tryUntilSuccess(() => { ret = client.DownloadString(link);}); List<string> lstPage = getHtmlPageInList(ret); if (isWriteFile) { writeFileOfPage(lstPage); } PropertyInfo[] pis = model.GetType().GetProperties(); foreach (PropertyInfo pi in pis) { try { if (pi.GetValue(model, null) == null || pi.GetValue(model, null).ToString().Trim() == "" || Regex.IsMatch( pi.GetValue(model, null).ToString().Trim(),"^[0.]+$")) { object val = getProperty(pi.Name, lstPage, pi.PropertyType, analysisFolder); pi.SetValue(model, val, null); } } catch (Exception ex) { //Console.WriteLine (ex.ToString()); } } return model; }
public IEnumerable<borsaModel> getLinks(string url) { List<string> lstUrls = new List<string>(); List<borsaModel> lstLinks = new List<borsaModel>(); string allUrlsInConfig = ConfigurationManager.AppSettings["urls"]; lstUrls = allUrlsInConfig.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries).ToList(); string val = "href=\"" + ConfigurationManager.AppSettings["mainLink"]; val = Regex.Escape(val); foreach (string mainUrl in lstUrls) { string str = ""; if (mainUrl.StartsWith("local://", StringComparison.OrdinalIgnoreCase)) { str = File.ReadAllText(mainUrl.Substring(8)); } else { WebClient client = new WebClient(); client.Headers.Add("User-Agent", ConfigurationManager.AppSettings["browserAgent"]); client.Encoding = Encoding.UTF8; Program.tryUntilSuccess(() => { str = client.DownloadString(mainUrl); }); } Regex reg = new Regex(val + "[^\\\"]+\\\"", RegexOptions.IgnoreCase); Match msh = null; do { if (msh == null) { msh = reg.Match(str); } else { msh = msh.NextMatch(); } if (msh.Success) { string foundUrl = msh.Value.Substring(6, msh.Value.Length - 7); string fullUrl = Program.getFullUrl(mainUrl, foundUrl); borsaModel model = new borsaModel(); model.Link = fullUrl; if (lstLinks.Where(obj=>obj.Link == fullUrl).Count() == 0) lstLinks.Add(model); } } while (msh.Success); } return lstLinks; }
private static borsaModel getAnnounceOfModel(string annUrl, borsaModel model) { string link = model.Link; string annLink = link + annUrl; WebClient client = new WebClient(); client.Headers.Add("User-Agent", ConfigurationManager.AppSettings["browserAgent"]); client.Encoding = Encoding.UTF8; string ret = ""; tryUntilSuccess(() => { ret = client.DownloadString(annLink); }); string tableRowReg = @"\<tr[^/\>]*\>"; string endRowReg = @"\</tr\>"; Match mch = Regex.Match(ret, tableRowReg, RegexOptions.IgnoreCase); while (mch.Success) { Match end = Regex.Match(ret.Substring(mch.Index), endRowReg, RegexOptions.IgnoreCase); string inBetween = ret.Substring(mch.Index, end.Index + end.Length); if (inBetween.Contains("مدققة")) { List<string> cells = new List<string>(); string tableCellReg = @"\<td[^/\>]*\>"; string endCellReg = @"\</td\>"; Match cellMatch = Regex.Match(inBetween, tableCellReg, RegexOptions.IgnoreCase); while (cellMatch.Success) { Match endCellMatch = Regex.Match(inBetween.Substring(cellMatch.Index), endCellReg, RegexOptions.IgnoreCase); string cellString = inBetween.Substring(cellMatch.Index, endCellMatch.Index + endCellMatch.Length); cells.Add(cellString); cellMatch = cellMatch.NextMatch(); } string date = Regex.Replace(cells[0], "<[^>]+>", "", RegexOptions.IgnoreCase); string ann = Regex.Replace(cells[1], "<[^>]+>", "", RegexOptions.IgnoreCase); string annComp = Regex.Replace(cells[2], "<[^>]+>", "", RegexOptions.IgnoreCase); string note = Regex.Replace(cells[4], "<[^>]+>", "", RegexOptions.IgnoreCase); model.profitDate = date; model.profitAnn = getDoubleValue(ann); model.profitComp = getDoubleValue(annComp); model.profitType = note; return model; } mch = mch.NextMatch(); } return model; }
public IEnumerable<borsaModel> getLinks(string url) { List<string> lstUrls = new List<string>(); List<borsaModel> lstLinks = new List<borsaModel>(); string allUrlsInConfig = ConfigurationManager.AppSettings["urls"]; lstUrls = allUrlsInConfig.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries).ToList(); string mainLink = ConfigurationManager.AppSettings["mainLink"]; foreach (string mainUrl in lstUrls) { string str = ""; WebClient client = new WebClient(); client.Headers.Add("User-Agent", ConfigurationManager.AppSettings["browserAgent"]); client.Encoding = Encoding.UTF8; Program.tryUntilSuccess(() => { str = client.DownloadString(mainUrl); }); Regex reg = new Regex(@"<tr[^>]*RowStyle[^>]*>", RegexOptions.IgnoreCase); Match mch = reg.Match(str); while (mch.Success) { int lastIndex = getLastIndex(mch,str, "<tr[^>]*>", "</tr>"); string row = str.Substring(mch.Index, lastIndex-mch.Index); Regex regCol = new Regex(@"<td[^>]*>", RegexOptions.IgnoreCase); List<string> lstCols = new List<string>(); Match mchCol = regCol.Match(row); while (mchCol.Success) { int lastColIndex = getLastIndex(mchCol, row, "<td[^>]*>", "</td>"); string col = row.Substring(mchCol.Index, lastColIndex-mchCol.Index); col = Regex.Replace(col, "<[^>]+>", "").Trim(); lstCols.Add(col); mchCol = mchCol.NextMatch(); } borsaModel newModel = new borsaModel(); newModel.Link = string.Format(mainLink,lstCols[0]); newModel.name = lstCols[1]; newModel.typeOfComp = lstCols[2]; newModel.profitDate = lstCols[3]; double test = 0; double.TryParse(lstCols[5],out test); newModel.profitAnn = test*1000; test = 0; double.TryParse(lstCols[6], out test); newModel.profitComp = test*1000; newModel.change = lstCols[7]; newModel.currency = lstCols[8]; if (lstLinks.Where(obj => obj.Link == newModel.Link).Count() == 0) lstLinks.Add(newModel); mch = mch.NextMatch(); } } return lstLinks; }