public async Task <SearchRet> WebSearch(string devId, string savePath, string pathForKvart)
{
//сделаем поисковый запрос
var retSearch = new SearchRet();
var webReqLog = (HttpWebRequest)WebRequest.Create(MainRef + "/buildings");
webReqLog.Method = Utility.Post;
webReqLog.SendChunked = false;
webReqLog.KeepAlive = true;
webReqLog.Referer = MainRef + "\\buildings";
webReqLog.AllowAutoRedirect = true;
webReqLog.CookieContainer = _cook;
webReqLog.UserAgent = Utility.UserAgent;
webReqLog.Accept = "*/*";
const string boundary = "---------------------------7dffc53902e0";
webReqLog.ContentType = Utility.ContentTypeMultipart + "; boundary=" + boundary;
webReqLog.ServicePoint.Expect100Continue = false;
var findString = "--" + boundary + "\r\n";
findString += "Content-Disposition: form-data; name=\"NewSearchFormFull[developer]\"\r\n\r\n";
findString += devId + "\r\n";
findString += "--" + boundary + "\r\n";
findString += "--" + boundary + "--\r\n";
var byteArr = Encoding.UTF8.GetBytes(findString);
webReqLog.ContentLength = byteArr.Length;
webReqLog.GetRequestStream().Write(byteArr, 0, byteArr.Length);
webReqLog.Timeout = 30000;
var document = new HtmlDocument();
bool isPaged = true;
int pages = 1;
using (var webResLog = await webReqLog.GetResponseAsync())
{
retSearch.RefString = webResLog.ResponseUri.ToString();
}
retSearch.SubDirectory = new Dictionary <string, string>();
var referer = MainRef + "buildings";
do
{
try
{
webReqLog = (HttpWebRequest)WebRequest.Create(retSearch.RefString + $"&page={pages}");
webReqLog.Method = Utility.Get;
webReqLog.Headers.Add("X-Requested-With", "XMLHttpRequest");
webReqLog.SendChunked = false;
webReqLog.KeepAlive = true;
webReqLog.Referer = referer;
webReqLog.AllowAutoRedirect = true;
webReqLog.CookieContainer = _cook;
webReqLog.UserAgent = Utility.UserAgent;
webReqLog.Accept = "*/*";
webReqLog.ContentType = Utility.ContentTypeUrlEncoded;
webReqLog.ServicePoint.Expect100Continue = false;
webReqLog.Timeout = 30000;
using (var webResLog1 = await webReqLog.GetResponseAsync())
{
using (var stream = webResLog1.GetResponseStream())
{
if (stream != null)
{
var sr = new StreamReader(stream);
document.Load(sr);
}
else
{
ErrWebResponse.Add("Ошибка при загрузке объекта " + devId + "\n" + "stream == null");
return(null);
}
//найдем объекты
var table_lines = document.DocumentNode.SelectNodes("//tr[@class = 'table-line']");
//var li = document.DocumentNode.SelectNodes("//li[@class = 'title']");
foreach (var tab in table_lines)
{
var a = tab.SelectSingleNode(".//td[@data-href]");//li1.ChildNodes.Where(c => c.Name == "a").ToList();
var title_td = tab.SelectSingleNode(".//td");
if (a == null)
{
continue;
}
var title = title_td.InnerText;
if (title_td.HasChildNodes)
{
title = title_td.FirstChild.InnerText;
}
title = Regex.Replace(title, "[<>\"*:/?\\|\n\t\r]", "");
var href = a.GetAttributeValue("data-href", "");
href = href.Substring(0, href.IndexOf("layout", StringComparison.InvariantCulture));
href += "materials";
retSearch.SubDirectory.Add(title, MainRef + href);
retSearch.DevSaveDirPath = savePath;
retSearch.KvartSaveDirPath = pathForKvart;
}
//теперь проверим на страницы
isPaged = false;
var lipage = document.DocumentNode.SelectNodes("//li[@class = 'next']");
if (lipage != null)
{
isPaged = true;
pages += 1;
referer = retSearch.RefString;
}
}
}
}
catch (WebException e)
{
ErrWebResponse.Add("Ошибка при загрузке объекта " + devId + "\n" + e.Message);
return(null);
}
catch (Exception ex)
{
ErrList.Add("Ошибка при загрузке объекта " + devId + "\n" + ex.Message);
return(null);
}
} while (isPaged);
return(retSearch);
}
public async Task <GetDirRet> GetSubDir(string objName, SearchRet shr)
{
var retDic = new GetDirRet();
string tempRef;
var document = new HtmlDocument();
try
{
var webReq = (HttpWebRequest)WebRequest.Create(shr.SubDirectory[objName]);
webReq.Method = Utility.Get;
webReq.KeepAlive = true;
webReq.AllowAutoRedirect = false;
webReq.CookieContainer = _cook;
webReq.UseDefaultCredentials = true;
webReq.UserAgent = Utility.UserAgent;
webReq.Referer = shr.RefString;
webReq.Timeout = 30000;
retDic.DevSaveDirPath = shr.DevSaveDirPath;
retDic.KvartSaveDirPath = shr.KvartSaveDirPath;
retDic.ObjName = objName;
retDic.SubDirectory = new Dictionary <string, string>();
using (var webRes = await webReq.GetResponseAsync())
{
using (var stream = webRes.GetResponseStream())
{
if (stream != null)
{
var sr = new StreamReader(stream);
document.Load(sr);
var lineNum = document.DocumentNode.SelectSingleNode("//div[@id='docsContainer']");
var javas = document.DocumentNode.SelectNodes("//script");
var tempJava = javas.Where(c => c.Line > lineNum.Line).ToList();
if (tempJava.Any())
{
var tempJava1 = tempJava.First().InnerText;
tempRef = Regex.Match(tempJava1, @"'[\d\D]+?'").Value;
tempRef = tempRef.Replace("'", "");
}
else
{
sr.Close();
return(null);
}
sr.Close();
}
else
{
ErrWebResponse.Add("Ошибка при загрузке объекта " + objName + "\n" + "stream == null");
return(null);
}
}
}
}
catch (WebException e)
{
ErrWebResponse.Add("Ошибка при загрузке объекта " + objName + "\n" + e.Message);
return(null);
}
catch (Exception ex)
{
ErrList.Add("Ошибка при загрузке объекта " + objName + "\n" + ex.Message);
return(null);
}
//найдем ссылку на папки
if (tempRef == "")
{
ErrWebResponse.Add("Ошибка при загрузке объекта " + objName + "\n" + "tempRef == ''");
return(null);
}
try
{
var webReq1 = (HttpWebRequest)WebRequest.Create(MainRef + tempRef);
webReq1.Method = Utility.Get;
webReq1.KeepAlive = true;
webReq1.AllowAutoRedirect = false;
webReq1.CookieContainer = _cook;
webReq1.UseDefaultCredentials = true;
webReq1.UserAgent = Utility.UserAgent;
webReq1.Referer = shr.RefString;
webReq1.Timeout = 30000;
using (var webRes = await webReq1.GetResponseAsync())
{
using (var stream = webRes.GetResponseStream())
{
if (stream == null)
{
ErrWebResponse.Add("Ошибка при загрузке объекта " + objName + "\n" + "stream == null");
return(null);
}
var sr = new StreamReader(stream);
document.Load(sr);
var allLi = document.DocumentNode.SelectNodes("//li[@class='directory collapsed']");
if (allLi == null)
{
return(null);
}
foreach (var li in allLi)
{
if (!li.HasChildNodes || li.ChildNodes.Count <= 1 || li.ChildNodes[1].Name != "div")
{
continue;
}
var div = li.ChildNodes[1];
if (!div.HasChildNodes || div.ChildNodes.Count <= 1 || div.ChildNodes[1].Name != "h3")
{
continue;
}
var dirName = div.ChildNodes[1].InnerText;
var refTemp = div.GetAttributeValue("data-path", "");
var dirRef = MainRef + refTemp;
if (refTemp.Length > 0)
{
retDic.SubDirectory.Add(dirName, dirRef);
}
}
return(retDic);
}
}
}
catch (WebException e)
{
ErrWebResponse.Add("Ошибка при загрузке объекта " + objName + "\n" + e.Message);
return(null);
}
catch (Exception ex)
{
ErrList.Add("Ошибка при загрузке объекта " + objName + "\n" + ex.Message);
return(null);
}
}
