public override void ReadAllMenuURL()
{
string url = "http://www.360kad.com/dymhh/allclass.shtml";
string content = request.HttpGet(url);
int startIndex = content.IndexOf("clearfix ksBoxs");
content = CommonFun.GetValue(content, "<ul class=\"clearfix ksBoxs\" ", "</ul>");
MatchCollection ms = CommonFun.GetValues(content, "<div class=\"right\">", "</div>");
List <string> menuUrl = new List <string>();
foreach (Match m in ms)
{
MatchCollection msUrl = CommonFun.GetValues(m.Value, "href=\"", "\"");
foreach (Match mUrl in msUrl)
{
if (!menuUrl.Contains(mUrl.Value))
{
menuUrl.Add(mUrl.Value);
}
}
}
AllMenuUrl = menuUrl;
}
public void ReadAllItem()
{
string url = "https://www.tkyfw.com/Ching_slist_{0}.html";
for (int i = 0; i < 30000; i++)
{
string content = request.HttpGet(string.Format(url, i), "");
content = CommonFun.GetValue(content, "<div class=\"buying\">", "</div>");
MatchCollection ms = CommonFun.GetValues(content, "<td class=\"td3\">", "</td>");
if (!string.IsNullOrEmpty(ms[0].Value))
{
try
{
ItemInfo item = new ItemInfo();
item.Name = ms[0].Value;
item.ID = ms.Count > 3 ? ms[2].Value : "";
item.Created = CommonFun.GetValue(content, "rel=\"theqiye\">", "</td>");
string priceStr = CommonFun.GetValue(content, "<strong class=\"value yahei\">", "</strong>");
try
{
if (string.IsNullOrEmpty(priceStr))
{
Console.WriteLine("Price is null ID;{0}, url:{1}", item.ID, string.Format(url, i));
continue;
}
item.ShopPrice = Convert.ToDecimal(priceStr);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString() + priceStr);
}
item.Format = CommonFun.GetValue(content, "<li class=\"cur\">", "</li>");
item.Format = CommonFun.GetValue(item.Format, "\">", "</a>");
string key = item.ID + "{" + item.Format + "}";
if (allItems.ContainsKey(key))
{
if (allItems[key].ShopPrice > item.ShopPrice)
{
allItems[key] = item;
}
}
else
{
allItems.Add(key, item);
}
CommonFun.WriteCSV("TK.csv", item);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
}
}
/// <summary>
/// 读取商城所有的菜单URL
/// </summary>
public void ReadShopALLMenuURL(string url, string data, string cookie)
{
/// string content = request.HttpGet("http://www.hyey.cn/Drug/DrugList.aspx", "fl=2&syz=14&ypmc=盐酸多巴酚丁胺注射液");
string content = request.HttpGet(url, data, cookie);
int startIndex = content.IndexOf("item bo");
int endIndex = content.LastIndexOf("</a></em>");
content = content.Substring(startIndex, endIndex - startIndex);
MatchCollection ms = CommonFun.GetValues(content, "<em><a", "</a></em>");
foreach (Match m in ms)
{
if (!m.Value.Contains("注射"))
{
string value = CommonFun.GetValue(m.Value, "href=\"", "\"");
shopAllMenuURL.Add(value);
// Console.WriteLine("value:{0}", value);
}
}
}
public List <ItemInfo> SeachInfoByID(string id)
{
List <ItemInfo> infos = new List <ItemInfo>();
string url = string.Format("http://www.yaofangwang.com/search.html?keyword={0}", id);
string content = request.HttpGetPlatform(url);
List <string> items = GetItemStr(content);
foreach (string item in items)
{
bool result = false;
do
{
try
{
ItemInfo info = new ItemInfo();
info.Format = CommonFun.GetValue(item, "规格：", "<");
info.Created = CommonFun.GetValue(item, "生产厂家：", "<");
//库存
info.ID = CommonFun.GetValue(item, "批准文号：", "<");
string priceStr = CommonFun.GetValue(item, "¥", "<");
info.ShopPrice = string.IsNullOrEmpty(priceStr) ? 0 : Convert.ToDecimal(priceStr);
content = request.HttpGetPlatform(CommonFun.GetValue(item, "<a target=\"_blank\" href=\"", "\""));
int startIndex = content.IndexOf("<div class=\"share clearfix\">");
int endIndx = content.IndexOf("id=\"priceA\">");
content = content.Substring(startIndex, endIndx - startIndex);
info.ViewCount = CommonFun.GetValue(content, "<dt>最近浏览</dt><dd class=\"w1\">", "次");
info.Name = CommonFun.GetValue(content, "<dd class=\"w2 l\"><strong>", "<");
MatchCollection ms = CommonFun.GetValues(content, "<dd class=\"w3 l\">", "</");
info.Type = ms[1].Value;
infos.Add(info);
result = true;
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
} while (!result);
}
return(infos);
}
public void ReadAllItem()
{
List <string> itemNames = new List <string>();
try
{
int count = 0;
foreach (string url in shopAllMenuURL)
{
count++;
string content = request.HttpGetPlatform("http://www.yaofangwang.com/" + url);
List <string> temp = GetItemName(content);
if (temp.Count > 0)
{
itemNames.AddRange(temp);
string pageStr = CommonFun.GetValue(content, "<span class=\"num\"><label>1</label> /", "<");
if (string.IsNullOrEmpty(pageStr))
{
Console.WriteLine("The url is null: {0}", url);
}
else
{
int pageCount = Convert.ToInt32(pageStr);
for (int i = 2; i <= pageCount; i++)
{
content = request.HttpGetPlatform("http://www.yaofangwang.com/" + url.Substring(0, url.IndexOf(".")) + "-p" + i + ".html");
temp = GetItemName(content);
itemNames.AddRange(temp);
}
}
}
else
{
Console.WriteLine("the url havn't content: {0}", url);
}
Console.WriteLine("Count:{0}, tatalCount:{1}, URL:{2}", count, shopAllMenuURL.Count, url);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
CreateItemInfo(itemNames);
}
private void SetMenuInfo(string content, ItemInfo info)
{
string menuStr = CommonFun.GetValue(content, "wrap-dtl-nav", "</div>");
MatchCollection menuMs = CommonFun.GetValues(menuStr, "\">", "</a>");
info.Menu1 = menuMs[1].Value;
info.Menu2 = menuMs[2].Value;
info.Menu3 = menuMs[3].Value;
}
public void ReadAllMenuURL()
{
string content = request.HttpGetPlatform("http://www.yaofangwang.com/Catalog-1.html");
content = CommonFun.GetValue(content, "<div id=\"wrap\">", "<div class=\"block clearfix lazyload\">");
MatchCollection ms = CommonFun.GetValues(content, "<a href=\"", "\"");
foreach (Match m in ms)
{
shopAllMenuURL.Add(m.Value);
}
}
/// <summary>
/// 获取药品名称
/// </summary>
/// <param name="content"></param>
/// <returns></returns>
private List <string> GetItemName(string content)
{
List <string> items = new List <string>();
string itemContent = CommonFun.GetValue(content, "<ul class=\"goodlist clearfix\">", "<div class=\"pager clearfix\">");
MatchCollection ms = CommonFun.GetValues(itemContent, "alt=\"", "\"");
foreach (Match m in ms)
{
items.Add(m.Value);
}
return(items);
}
/// <summary>
/// 提取药品信息
/// </summary>
/// <param name="content"></param>
/// <returns></returns>
private List <string> GetItemStr(string content)
{
List <string> items = new List <string>();
string itemContent = CommonFun.GetValue(content, "<ul class=\"goodlist clearfix\">", "</ul>");
MatchCollection ms = CommonFun.GetValues(itemContent, "<li>", " </li>");
foreach (Match m in ms)
{
items.Add(m.Value);
}
return(items);
}
private void GetItemUrl(string content)
{
string productContent = CommonFun.GetValue(content, "<ul class=\"Productlist\">", "</ul>");
MatchCollection msItem = CommonFun.GetValues(productContent, "<p class=\"t\">", "</p>");
foreach (Match m in msItem)
{
string url = CommonFun.GetValue(m.Value, "href=\"", "\"");
if (!AllItemUrl.Contains(url))
{
AllItemUrl.Add(url);
}
}
}
private int GetTotalPage(string content, bool isAspx)
{
if (isAspx)
{
string str = CommonFun.GetValue(content, "<strong>", "</strong>");
return(string.IsNullOrEmpty(str) ? 0 : Convert.ToInt32(str));
}
else
{
string pageContent = CommonFun.GetValue(content, "YPagebox", "</span>");
MatchCollection ms = CommonFun.GetValues(pageContent, ">", "</a>");
return(ms.Count - 2);
}
}
private void SetPicture(string content, ItemInfo info)
{
string prictureUrlStr = CommonFun.GetValue(content, "<div class=\"minPicScrolldiv\">", "</div>");
MatchCollection ms = CommonFun.GetValues(prictureUrlStr, "src=\"", "\"");
info.PicturePath = @"Picture/" + info.Name.Replace('.', '_').Replace('*', '_').Replace(':', '_').Replace(" ", "") + "/" + info.Format.Replace('.', '_').Replace('*', '_').Replace(':', '_').Replace(" ", "") + "/";
int pictureCount = 0;
//foreach (Match url in ms)
//{
// Image image = request.HttpGetPicture(url.Value);
// CommonFun.SavePicture(image, info.PicturePath + pictureCount++ + ".jpg");
//}
info.PicturePath = string.Format("=HYPERLINK(\"{0}\")", info.PicturePath);
}
public void ReadOneItem(string url)
{
try
{
string content = request.HttpGet(url);
int startIndex = content.IndexOf("<div class=\"prem-dtl-infomation\">");
int endIndex = content.IndexOf("<div class=\"inf-r-wxhb\">");
string mainStr = content.Substring(startIndex, endIndex - startIndex);
MatchCollection ms = CommonFun.GetValues(mainStr, "<div class=\"dtl-inf-r\">", "</div>");
if (!string.IsNullOrEmpty(ms[0].Value) && !ms[0].Value.Contains("注射"))
{
ItemInfo info = new ItemInfo();
info.Name = ms[0].Value;
string idStr = ms[1].Value;
info.ID = string.IsNullOrEmpty(idStr) ? "" : idStr.Substring(0, idStr.IndexOf('\n'));
info.Created = ms[2].Value;
info.Format = CommonFun.GetValue(content, "规格：", "<");
info.Type = CommonFun.GetValue(content, "剂型：", "<");
info.ItemName = CommonFun.GetValue(content, "商品名称：", "<");
info.BrandName = info.ItemName.Split(' ')[0];
info.DrugType = GetDrugType(content);
string priceStr = CommonFun.GetValue(content, "salePrice : ", ",");
info.ShopPrice = string.IsNullOrEmpty(priceStr) ? 0 : Convert.ToDecimal(priceStr);
string key = info.ID + "{" + info.Format + "}";
SetMenuInfo(content, info);
SetMainInfo(content, info);
SetPicture(content, info);
if (ShopAllItems.ContainsKey(key))
{
if (info.ShopPrice != 0 && ShopAllItems[key].ShopPrice > info.ShopPrice)
{
ShopAllItems[key] = info;
}
}
else
{
ShopAllItems.Add(key, info);
}
// CommonFun.WriteCSV("360Kad.csv", info);
}
}
catch (Exception ex)
{
Console.WriteLine(ex + string.Format(url));
}
}
/// <summary>
/// 获取所有商城数据
/// </summary>
public void ReadShopAllItem()
{
List <string> items = new List <string>();
try
{
int count = 0;
foreach (string url in AllMenuUrl)
{
count++;
string content = request.HttpGet("http://www.hyey.cn" + url);
List <string> temp = new List <string>();
temp = GetItmeStr(content);
if (temp.Count > 0)
{
items.AddRange(temp);
string pageStr = CommonFun.GetValue(content, "<span id=\"lblTotalPageV\" style=\"border-color: #00a1fd;\">", "</span>");
if (string.IsNullOrEmpty(pageStr))
{
Console.WriteLine(pageStr);
}
else
{
int pageIndex = Convert.ToInt32(pageStr);
for (int i = 1; i < pageIndex; i++)
{
string postString = null;
string otherParam = GetOtherParam(temp);
postString = string.Format("__EVENTTARGET=&__EVENTARGUMENT=&__VIEWSTATE={0}&__EVENTVALIDATION={1}&TextBox1=&txtSelect=&HiddenField1=&hidd=&hiddLV=1&{2}btnNextV={3}&txtPageSizeV=&hidPageV=1", System.Web.HttpUtility.UrlEncode(GetParamValue(content, "__VIEWSTATE")), System.Web.HttpUtility.UrlEncode(GetParamValue(content, "__EVENTVALIDATION")), otherParam, "下一页");
content = request.HttpPost("http://www.hyey.cn" + url, postString);
temp = GetItmeStr(content);
items.AddRange(temp);
}
}
// Console.WriteLine(content);
}
else
{
Console.WriteLine(url);
}
Console.WriteLine("Count:{0}, tatalCount:{1}, URL:{2}", count, AllMenuUrl.Count, url);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
CreateItemInfo(items);
}
private void CreateItemOtherInfo(object value)
{
string temp = null;
try
{
string name = (string)value;
if (name.Contains("注射"))
{
return;
}
HttpRequest tempRequest = new HttpRequest();
string url = string.Format("http://www.yaofangwang.com/search.html?keyword={0}", System.Web.HttpUtility.UrlEncode(name));
string content = tempRequest.HttpGetPlatform(url);
List <string> items = GetItemStr(content);
foreach (string item in items)
{
bool result = false;
do
{
try
{
temp = item;
ItemInfo info = new ItemInfo();
info.Format = CommonFun.GetValue(item, "规格：", "<");
info.Created = CommonFun.GetValue(item, "生产厂家：", "<");
//库存
info.ID = CommonFun.GetValue(item, "批准文号：", "<");
string priceStr = CommonFun.GetValue(item, "¥", "<");
info.ShopPrice = string.IsNullOrEmpty(priceStr) ? 0 : Convert.ToDecimal(priceStr);
content = tempRequest.HttpGetPlatform(CommonFun.GetValue(item, "<a target=\"_blank\" href=\"", "\""));
int startIndex = content.IndexOf("<div class=\"share clearfix\">");
int endIndx = content.IndexOf("id=\"priceA\">");
content = content.Substring(startIndex, endIndx - startIndex);
info.ViewCount = CommonFun.GetValue(content, "<dt>最近浏览</dt><dd class=\"w1\">", "次");
info.Name = CommonFun.GetValue(content, "<dd class=\"w2 l\"><strong>", "<");
MatchCollection ms = CommonFun.GetValues(content, "<dd class=\"w3 l\">", "</");
info.Type = ms[1].Value;
string key = info.ID + "{" + info.Format + "}";
if (platformItems.ContainsKey(key))
{
if (platformItems[key].ShopPrice > info.ShopPrice)
{
platformItems[key] = info;
}
}
else
{
platformItems.Add(key, info);
}
CommonFun.WriteCSV("平台信息.csv", info);
result = true;
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
} while (!result);
}
Console.WriteLine("finishName;{0}, finishCoutn:{1}, totalCount:{2}", name, ++finishCount, toltalCount);
if (finishCount == toltalCount)
{
Console.WriteLine("Finished !!!!!!!!!!!!!!!!!");
}
}
catch (Exception ex)
{
Console.WriteLine("content;{0}, error:{1}", temp, ex.ToString());
}
}
private void SetMainInfo(string content, ItemInfo info)
{
string mainInfoStr = CommonFun.GetValue(content, "<ul class=\"instructions-ul\">", "</ul>");
int index = mainInfoStr.IndexOf("见【");
List <string> replacelist = new List <string>();
int i = 0;
while (index > 0)
{
string tempStr = mainInfoStr.Substring(index, mainInfoStr.Length - index);
int end = tempStr.IndexOf("】");
string replaceStr = mainInfoStr.Substring(index, end + 1);
replacelist.Add(replaceStr);
mainInfoStr = mainInfoStr.Replace(replaceStr, string.Format("{{0}}", i++));
index = mainInfoStr.IndexOf("见【");
}
MatchCollection titles = CommonFun.GetValues(mainInfoStr, "【", "】");
MatchCollection ms = CommonFun.GetValues(mainInfoStr, "】", "【");
List <string> values = new List <string>();
foreach (Match m in ms)
{
values.Add(m.Value);
}
int startIndex = mainInfoStr.LastIndexOf("】");
string lastValue = mainInfoStr.Substring(startIndex + 1, mainInfoStr.Length - startIndex - 1);
int endIndex = lastValue.IndexOf('<');
lastValue = lastValue.Substring(0, endIndex);
values.Add(lastValue);
for (int j = 0; j < replacelist.Count; j++)
{
for (int m = 0; m < values.Count; m++)
{
string value = string.Format("{{0}}", j);
if (values[m].Contains(value))
{
values[m] = values[m].Replace(value, replacelist[j]);
continue;
}
}
}
if (ms.Count > 1)
{
int mIndex = GetIndex("成", titles);
if (mIndex != -1)
{
info.Basis = RemoveTag(values[mIndex]);
}
mIndex = GetIndex("性", titles);
if (mIndex != -1)
{
info.Character = RemoveTag(values[mIndex]);
}
mIndex = GetIndex("治", titles);
mIndex = mIndex == -1 ? mIndex = GetIndex("适", titles) : mIndex;
if (mIndex != -1)
{
info.Function = RemoveTag(values[mIndex]);
}
mIndex = GetIndex("法", titles);
if (mIndex != -1)
{
info.Use = RemoveTag(values[mIndex]);
}
mIndex = GetIndex("良", titles);
if (mIndex != -1)
{
info.AdverseReaction = RemoveTag(values[mIndex]);
}
mIndex = GetIndex("禁", titles);
if (mIndex != -1)
{
info.Contraindication = RemoveTag(values[mIndex]);
}
mIndex = GetIndex("意", titles);
if (mIndex != -1)
{
info.NoticMatters = RemoveTag(values[mIndex]);
}
mIndex = GetIndex("贮", titles);
if (mIndex != -1)
{
info.SaveType = RemoveTag(values[mIndex]);
}
}
}
/// <summary>
/// 生成商城信息
/// </summary>
/// <param name="items"></param>
private void CreateItemInfo(List <string> items)
{
string temp = null;
try
{
Console.WriteLine("ItemsCount:{0}", items.Count);
foreach (string item in items)
{
temp = item;
ItemInfo info = new ItemInfo();
MatchCollection ms = CommonFun.GetValues(item, "value=\"", "\"");
info.Name = ms[0].Value;
if (info.Name.Contains("注射"))
{
continue;
}
info.Format = ms[1].Value + "/" + ms[3];
info.Format = info.Format.Replace("*", "x");
info.Format = info.Format.Replace("s", "片");
info.Format = info.Format.Replace("代", "袋");
info.Created = ms[4].Value;
info.SellType = CommonFun.GetValue(item, "【", "】");
if (info.SellType.Count() > 1)
{
info.SellType = CommonFun.GetValue(info.SellType, ">", "<");
}
info.Inventory = CommonFun.GetValue(item, "数量：", "<");
info.ID = "国药准字" + CommonFun.GetValue(item, "批准文号：", "<");
info.ShopPrice = Convert.ToDecimal(CommonFun.GetValue(item, "价格：<span>", "元"));
string key = info.ID + "{" + info.Format + "}";
if (ShopAllItems.ContainsKey(key))
{
if (ShopAllItems[key].ShopPrice > info.ShopPrice)
{
ShopAllItems[key] = info;
}
}
else
{
ShopAllItems.Add(key, info);
}
// CommonFun.WriteCSV("商城信息.csv", info);
}
}
catch (Exception ex)
{
Console.WriteLine("content;{0}, error:{1}", temp, ex.ToString());
}
Console.WriteLine("ShopAllItemCount:{0}", ShopAllItems.Count);
}