private static void writeModelText(borsaModel borsaModel)
{
StringBuilder sb = new StringBuilder();
PropertyInfo[] pis = borsaModel.GetType().GetProperties();
sb.AppendLine("<tr>");
foreach (PropertyInfo pi in pis)
{
string value = (pi.GetValue (borsaModel, null) ?? "").ToString();
sb.Append("<td style='border:1px solid black;'>");
if (pi.Name == "Link")
sb.Append("<a href=\""+ value +"\">");
sb.Append(value);
if (pi.Name == "Link")
sb.Append("</a>");
sb.AppendLine("</td>");
}
sb.AppendLine("</tr>");
lock (syncFileObj)
{
File.AppendAllText("output.html", sb.ToString());
}
}
private static borsaModel getBorsaModel(borsaModel model)
{
string link = model.Link;
WebClient client = new WebClient();
client.Headers.Add("User-Agent", ConfigurationManager.AppSettings["browserAgent"]);
client.Encoding = Encoding.UTF8;
string ret = "";
tryUntilSuccess(() => { ret = client.DownloadString(link);});
List<string> lstPage = getHtmlPageInList(ret);
if (isWriteFile)
{
writeFileOfPage(lstPage);
}
PropertyInfo[] pis = model.GetType().GetProperties();
foreach (PropertyInfo pi in pis)
{
try
{
if (pi.GetValue(model, null) == null || pi.GetValue(model, null).ToString().Trim() == "" || Regex.IsMatch( pi.GetValue(model, null).ToString().Trim(),"^[0.]+$"))
{
object val = getProperty(pi.Name, lstPage, pi.PropertyType, analysisFolder);
pi.SetValue(model, val, null);
}
}
catch (Exception ex)
{
//Console.WriteLine (ex.ToString());
}
}
return model;
}
public IEnumerable<borsaModel> getLinks(string url)
{
List<string> lstUrls = new List<string>();
List<borsaModel> lstLinks = new List<borsaModel>();
string allUrlsInConfig = ConfigurationManager.AppSettings["urls"];
lstUrls = allUrlsInConfig.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries).ToList();
string val = "href=\"" + ConfigurationManager.AppSettings["mainLink"];
val = Regex.Escape(val);
foreach (string mainUrl in lstUrls)
{
string str = "";
if (mainUrl.StartsWith("local://", StringComparison.OrdinalIgnoreCase))
{
str = File.ReadAllText(mainUrl.Substring(8));
}
else
{
WebClient client = new WebClient();
client.Headers.Add("User-Agent", ConfigurationManager.AppSettings["browserAgent"]);
client.Encoding = Encoding.UTF8;
Program.tryUntilSuccess(() => { str = client.DownloadString(mainUrl); });
}
Regex reg = new Regex(val + "[^\\\"]+\\\"", RegexOptions.IgnoreCase);
Match msh = null;
do
{
if (msh == null)
{
msh = reg.Match(str);
}
else
{
msh = msh.NextMatch();
}
if (msh.Success)
{
string foundUrl = msh.Value.Substring(6, msh.Value.Length - 7);
string fullUrl = Program.getFullUrl(mainUrl, foundUrl);
borsaModel model = new borsaModel();
model.Link = fullUrl;
if (lstLinks.Where(obj=>obj.Link == fullUrl).Count() == 0)
lstLinks.Add(model);
}
} while (msh.Success);
}
return lstLinks;
}
private static borsaModel getAnnounceOfModel(string annUrl, borsaModel model)
{
string link = model.Link;
string annLink = link + annUrl;
WebClient client = new WebClient();
client.Headers.Add("User-Agent", ConfigurationManager.AppSettings["browserAgent"]);
client.Encoding = Encoding.UTF8;
string ret = "";
tryUntilSuccess(() => { ret = client.DownloadString(annLink); });
string tableRowReg = @"\<tr[^/\>]*\>";
string endRowReg = @"\</tr\>";
Match mch = Regex.Match(ret, tableRowReg, RegexOptions.IgnoreCase);
while (mch.Success)
{
Match end = Regex.Match(ret.Substring(mch.Index), endRowReg, RegexOptions.IgnoreCase);
string inBetween = ret.Substring(mch.Index, end.Index + end.Length);
if (inBetween.Contains("مدققة"))
{
List<string> cells = new List<string>();
string tableCellReg = @"\<td[^/\>]*\>";
string endCellReg = @"\</td\>";
Match cellMatch = Regex.Match(inBetween, tableCellReg, RegexOptions.IgnoreCase);
while (cellMatch.Success)
{
Match endCellMatch = Regex.Match(inBetween.Substring(cellMatch.Index), endCellReg, RegexOptions.IgnoreCase);
string cellString = inBetween.Substring(cellMatch.Index, endCellMatch.Index + endCellMatch.Length);
cells.Add(cellString);
cellMatch = cellMatch.NextMatch();
}
string date = Regex.Replace(cells[0], "<[^>]+>", "", RegexOptions.IgnoreCase);
string ann = Regex.Replace(cells[1], "<[^>]+>", "", RegexOptions.IgnoreCase);
string annComp = Regex.Replace(cells[2], "<[^>]+>", "", RegexOptions.IgnoreCase);
string note = Regex.Replace(cells[4], "<[^>]+>", "", RegexOptions.IgnoreCase);
model.profitDate = date;
model.profitAnn = getDoubleValue(ann);
model.profitComp = getDoubleValue(annComp);
model.profitType = note;
return model;
}
mch = mch.NextMatch();
}
return model;
}
public IEnumerable<borsaModel> getLinks(string url)
{
List<string> lstUrls = new List<string>();
List<borsaModel> lstLinks = new List<borsaModel>();
string allUrlsInConfig = ConfigurationManager.AppSettings["urls"];
lstUrls = allUrlsInConfig.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries).ToList();
string mainLink = ConfigurationManager.AppSettings["mainLink"];
foreach (string mainUrl in lstUrls)
{
string str = "";
WebClient client = new WebClient();
client.Headers.Add("User-Agent", ConfigurationManager.AppSettings["browserAgent"]);
client.Encoding = Encoding.UTF8;
Program.tryUntilSuccess(() => { str = client.DownloadString(mainUrl); });
Regex reg = new Regex(@"<tr[^>]*RowStyle[^>]*>", RegexOptions.IgnoreCase);
Match mch = reg.Match(str);
while (mch.Success)
{
int lastIndex = getLastIndex(mch,str, "<tr[^>]*>", "</tr>");
string row = str.Substring(mch.Index, lastIndex-mch.Index);
Regex regCol = new Regex(@"<td[^>]*>", RegexOptions.IgnoreCase);
List<string> lstCols = new List<string>();
Match mchCol = regCol.Match(row);
while (mchCol.Success)
{
int lastColIndex = getLastIndex(mchCol, row, "<td[^>]*>", "</td>");
string col = row.Substring(mchCol.Index, lastColIndex-mchCol.Index);
col = Regex.Replace(col, "<[^>]+>", "").Trim();
lstCols.Add(col);
mchCol = mchCol.NextMatch();
}
borsaModel newModel = new borsaModel();
newModel.Link = string.Format(mainLink,lstCols[0]);
newModel.name = lstCols[1];
newModel.typeOfComp = lstCols[2];
newModel.profitDate = lstCols[3];
double test = 0;
double.TryParse(lstCols[5],out test);
newModel.profitAnn = test*1000;
test = 0;
double.TryParse(lstCols[6], out test);
newModel.profitComp = test*1000;
newModel.change = lstCols[7];
newModel.currency = lstCols[8];
if (lstLinks.Where(obj => obj.Link == newModel.Link).Count() == 0)
lstLinks.Add(newModel);
mch = mch.NextMatch();
}
}
return lstLinks;
}