/// <summary>
/// 具体数据类型转换为存储的String字符串
/// </summary>
/// <param name="value"></param>
/// <returns></returns>
public static string ObjectValue2TableString(object value, List <char> arraySplitFormat)
{
string res = "";
if (value == null)
{
return(res);
}
Type t = value.GetType();
if (t.IsArray)
{
// PropertyInfo pL= t.GetProperty("Length");
//int length= (int) pL.GetValue(value, null);
// MethodInfo mth= t.GetMethod("GetValue", new Type[] { typeof(int) });
// for (int i = 0; i < length; i++)
// {
// object v = mth.Invoke(value, new object[] { i });
// res += SingleData2String(v);
// if (i < length - 1)
// res += "|";
// }
res = ParseTool.ArrayObject2String(value, arraySplitFormat.ToArray());
}
else
{
res = SingleData2String(value);
}
return(res);
}
public void PaseArray1()
{
{
string arrayStringContent = "mm#123,haha|papa,haha|ko#888";
char[] arraySplitFormat = new char[] { ',', '#' };
Array array = ParseTool.String2Array(FieldType.StringArray, arrayStringContent, arraySplitFormat);
Debug.Assert(array != null);
Debug.Log(JsonUtils.ToJson(array));
string res = ParseTool.ArrayObject2String(array, arraySplitFormat);
Debug.Log(res);
Debug.Assert(res == arrayStringContent);
}
{
string arrayStringContent = "NULl";
char[] arraySplitFormat = new char[] { ',', '#' };
Array array = ParseTool.String2Array(FieldType.StringArray, arrayStringContent, arraySplitFormat);
Debug.Assert(array == null);
Debug.Log(JsonUtils.ToJson(array));
string res = ParseTool.ArrayObject2String(array, arraySplitFormat);
Debug.Log(res);
Debug.Assert(res == arrayStringContent.ToLower());
}
}
public string GetString(string key)
{
try
{
if (this.ContainsKey(key))
{
//String 读取null 的改进，兼容旧代码
#if Compatibility
return(this[key]);
#else
return(ParseTool.GetString(this[key]));
#endif
}
if (data.m_defaultValue.ContainsKey(key))
{
#if Compatibility
return(data.m_defaultValue[key]);
#else
return(ParseTool.GetString(data.m_defaultValue[key]));
#endif
}
}
catch (Exception e)
{
throw new Exception("SingleData GetString Error TableName is :->" + data.m_tableName + "<- key->" + key + "<- singleDataName : ->" + m_SingleDataKey + "<- \n" + e.ToString());
}
throw new Exception("Don't Exist Value or DefaultValue by ->" + key + "<- TableName is : ->" + data.m_tableName + "<- singleDataName : ->" + m_SingleDataKey + "<-");// throw
}
public int[] GetIntArray(string key)
{
string content = null;
try
{
if (this.ContainsKey(key))
{
content = this[key];
return(ParseTool.String2IntArray(content));
}
if (data.m_defaultValue.ContainsKey(key))
{
content = data.m_defaultValue[key];
return(ParseTool.String2IntArray(content));
}
}
catch (Exception e)
{
throw new Exception("SingleData GetIntArray Error TableName is :->" + data.m_tableName + "<- key : ->" + key + "<- singleDataName : ->" + m_SingleDataKey + "<- content: ->" + content + "<- \n" + e.ToString());
}
throw new Exception("Don't Exist Value or DefaultValue TableName is :->" + data.m_tableName + "<- key : ->" + key + "<- singleDataName : ->" + m_SingleDataKey + "<-");// throw
}
public async Task Crawl()
{
try
{
Rootobject obj = JsonConvert.DeserializeObject <Rootobject>(File.ReadAllText(AppDomain.CurrentDomain.BaseDirectory + "Area.txt"));
foreach (var item in obj.GetType().GetProperties())
{
Area rootArea = new Area()
{
Name = item.Name, Url = string.Empty, ParentId = ""
};
object Id = areaRepository.Add(rootArea);
foreach (var p in ParseTool.TransFerType(item.PropertyType.FullName).GetProperties())
{
var url = p.GetValue(item.GetValue(obj)).ToString().Substring(0, p.GetValue(item.GetValue(obj)).ToString().IndexOf("|", StringComparison.Ordinal));
Area areaa = new Area()
{
Name = p.Name,
Url = "http://" + url + ".58.com/",
ParentId = Id.ToString()
};
areaRepository.Add(areaa);
}
}
Console.WriteLine("区域全部抓取完成");
}
catch (Exception e)
{
log.Error(e.ToString());
}
}
public override void Execute(string[] args)
{
if (args.Length < 1)
{
App.Logger.Log($"[ParseCommand] - Use: {Name} {Syntax}");
}
else
{
string joined = string.Join(" ", args);
if (File.Exists(joined))
{
App.Logger.Log("[ParseCommand] - File found. Checking...");
ParseTool pTool = new ParseTool(joined);
NPC.ParseType type = pTool.GetParseType();
switch (type)
{
case NPC.ParseType.NPC:
App.Logger.Log("[ParseCommand] - Started parsing 'NPC'.");
MainWindow.CurrentProject.data.characters.Add(pTool.ParseCharacter());
App.Logger.Log("[ParseCommand] - 'NPC' parsed and imported into project.");
LastResult = true;
break;
case NPC.ParseType.Dialogue:
App.Logger.Log("[ParseCommand] - Started parsing 'Dialogue'.");
MainWindow.CurrentProject.data.dialogues.Add(pTool.ParseDialogue());
App.Logger.Log("[ParseCommand] - 'Dialogue' parsed and imported into project.");
LastResult = true;
break;
case NPC.ParseType.Vendor:
App.Logger.Log("[ParseCommand] - Started parsing 'Vendor'.");
MainWindow.CurrentProject.data.vendors.Add(pTool.ParseVendor());
App.Logger.Log("[ParseCommand] - 'Vendor' parsed and imported into project.");
LastResult = true;
break;
case NPC.ParseType.Quest:
App.Logger.Log("[ParseCommand] - Started parsing 'Quest'.");
MainWindow.CurrentProject.data.quests.Add(pTool.ParseQuest());
App.Logger.Log("[ParseCommand] - 'Quest' parsed and imported into project.");
LastResult = true;
break;
default:
LastResult = false;
App.Logger.Log("[ParseCommand] - Invalid file.");
break;
}
}
else
{
LastResult = false;
App.Logger.Log("[ParseCommand] - File not found.");
}
}
}
public void PaseArray2()
{
string arrayStringContent = "haha|papa|ko#888";
Array array = ParseTool.String2Array(FieldType.StringArray, arrayStringContent, new char[] { });
Debug.Assert(array != null);
Debug.Log(JsonUtils.ToJson(array));
}
public string[] GetStringArray(string key)
{
if (this.ContainsKey(key))
{
return(ParseTool.String2StringArray(this[key]));
}
if (data.m_defaultValue.ContainsKey(key))
{
return(ParseTool.String2StringArray(data.m_defaultValue[key]));
}
throw new Exception("Don't Exist Value or DefaultValue by ->" + key + "<- TableName is : ->" + data.m_tableName + "<- singleDataName : ->" + m_SingleDataName + "<-");// throw
}
public Color GetColor(string key)
{
if (this.ContainsKey(key))
{
return(ParseTool.String2Color(this[key]));
}
if (data.m_defaultValue.ContainsKey(key))
{
return(ParseTool.String2Color(data.m_defaultValue[key]));
}
throw new Exception("Don't Exist Value or DefaultValue by " + key); // throw
}
public Color GetColor(string key)
{
try
{
if (this.ContainsKey(key))
{
return(ParseTool.String2Color(this[key]));
}
if (data.m_defaultValue.ContainsKey(key))
{
return(ParseTool.String2Color(data.m_defaultValue[key]));
}
}
catch (Exception e)
{
throw new Exception("SingleData GetColor Error TableName is :->" + data.m_tableName + "<- key->" + key + "<- singleDataName : ->" + m_SingleDataKey + "<- \n" + e.ToString());
}
throw new Exception("Don't Exist Value or DefaultValue by ->" + key + "<- TableName is : ->" + data.m_tableName + "<- singleDataName : ->" + m_SingleDataKey + "<-"); // throw
}
public Array GetArray(string key)
{
try
{
if (this.ContainsKey(key))
{
return(ParseTool.String2Array(data.GetFieldType(key), this[key], data.GetArraySplitFormat(key)));
}
else if (data.m_defaultValue.ContainsKey(key))
{
return(ParseTool.String2Array(data.GetFieldType(key), data.m_defaultValue[key], data.GetArraySplitFormat(key)));
}
}
catch (Exception e)
{
throw new Exception("SingleData GetStringArray2 Error TableName is :->" + data.m_tableName + "<- key->" + key + "<- singleDataName : ->" + m_SingleDataKey + "<- \n" + e.ToString());
}
throw new Exception("Don't Exist Value or DefaultValue by ->" + key + "<- TableName is : ->" + data.m_tableName + "<- singleDataName : ->" + m_SingleDataKey + "<-");// throw
}
public Color GetColor()
{
return(ParseTool.String2Color(m_content));
}
public Vector3 GetVector3()
{
return(ParseTool.String2Vector3(m_content));
}
public new static bool TryParse(string notification, out AmazonSESNotification amazonSESNotification)
{
return(ParseTool.TryParseAmazonSESNotification(notification, out amazonSESNotification, typeof(AmazonSESComplaintNotification)));
}
public new static AmazonSESComplaintNotification Parse(string notification)
{
return((AmazonSESComplaintNotification)ParseTool.ParseAmazonSESNotification(notification, typeof(AmazonSESComplaintNotification)));
}
/// <summary>
/// 将字段类型（FieldType）转换为对应 数据类型（Type）
/// </summary>
/// <param name="fieldValueType"></param>
/// <param name="enumType"></param>
/// <returns></returns>
public static Type ConfigFieldValueType2Type(FieldType fieldValueType, string enumType, List <char> arraySplitFormat)
{
Type t = null;
switch (fieldValueType)
{
case FieldType.String:
t = typeof(string);
break;
case FieldType.Bool:
t = typeof(bool);
break;
case FieldType.Int:
t = typeof(int);
break;
case FieldType.Float:
t = typeof(float);
break;
case FieldType.Vector2:
t = typeof(Vector2);
break;
case FieldType.Vector3:
t = typeof(Vector3);
break;
case FieldType.Color:
t = typeof(Color);
break;
case FieldType.Enum:
t = ReflectionUtils.GetTypeByTypeFullName(enumType);
break;
case FieldType.StringArray:
t = ParseTool.CreateInterleavedType(typeof(string), arraySplitFormat.Count + 1);
// t = typeof(string[]);
break;
case FieldType.IntArray:
// t = typeof(int[]);
t = ParseTool.CreateInterleavedType(typeof(int), arraySplitFormat.Count + 1);
break;
case FieldType.FloatArray:
// t = typeof(float[]);
t = ParseTool.CreateInterleavedType(typeof(float), arraySplitFormat.Count + 1);
break;
case FieldType.BoolArray:
// t = typeof(bool[]);
t = ParseTool.CreateInterleavedType(typeof(bool), arraySplitFormat.Count + 1);
break;
case FieldType.Vector2Array:
// t = typeof(Vector2[]);
t = ParseTool.CreateInterleavedType(typeof(Vector2), arraySplitFormat.Count + 1);
break;
case FieldType.Vector3Array:
// t = typeof(Vector3[]);
t = ParseTool.CreateInterleavedType(typeof(Vector3), arraySplitFormat.Count + 1);
break;
default:
break;
}
return(t);
}
/// <summary>
/// 原始表格字符串类型转换为具体的数据
/// </summary>
/// <param name="v"></param>
/// <param name="fieldValueType"></param>
/// <param name="enumType"></param>
/// <returns></returns>
public static object TableString2ObjectValue(string v, FieldType fieldValueType, string enumType, List <char> m_ArraySplitFormat)
{
object t = null;
switch (fieldValueType)
{
case FieldType.String:
t = v;
break;
case FieldType.Bool:
t = bool.Parse(v);
break;
case FieldType.Int:
t = int.Parse(v);
break;
case FieldType.Float:
t = float.Parse(v);
break;
case FieldType.Vector2:
t = ParseTool.String2Vector2(v);
break;
case FieldType.Vector3:
t = ParseTool.String2Vector3(v);
break;
case FieldType.Color:
t = ParseTool.String2Color(v);
break;
case FieldType.Enum:
Type type = ConfigFieldValueType2Type(fieldValueType, enumType, m_ArraySplitFormat);
try
{
t = Enum.Parse(type, v);
}
catch (Exception e)
{
t = Enum.GetValues(type).GetValue(0);
// Debug.LogError("fieldValueType:" + fieldValueType + " enumType:" + enumType + " type:" + type + " v:" + v + "\n"+e);
throw e;
}
break;
case FieldType.StringArray:
//t = ParseTool.String2StringArray(v);
//break;
case FieldType.IntArray:
//t = ParseTool.String2IntArray(v);
//break;
case FieldType.FloatArray:
//t = ParseTool.String2FloatArray(v);
//break;
case FieldType.BoolArray:
//t = ParseTool.String2BoolArray(v);
//break;
case FieldType.Vector2Array:
//t = ParseTool.String2Vector2(v);
//break;
case FieldType.Vector3Array:
//t = ParseTool.String2Vector3Array(v);
t = ParseTool.String2Array(fieldValueType, v, m_ArraySplitFormat.ToArray());
break;
default:
break;
}
return(t);
}
/// <summary>
/// 抓取求租
/// </summary>
/// <param name="area"></param>
public void CrawlDataBegRent(Area area)
{
try
{
Crawler crawler = new Crawler();
string url = area.Url + ConstVar.求租 + "0/";
string html = crawler.Crawl(url, Encoding.UTF8);
var htmlParse = new HtmlParser();
IHtmlDocument docuement = htmlParse.Parse(html);
List <IElement> eles = docuement.QuerySelectorAll("div").ToList().Where(p => p.ClassName == "pager")
.ToList();
if (eles.Count > 0)
{
IHtmlDocument htmlA = htmlParse.Parse(eles[0].InnerHtml);
List <IElement> spanEles = htmlA.QuerySelectorAll("span").ToList();
int page = 0;
if (spanEles.Count > 2)
{
IElement el = spanEles[spanEles.Count - 2];
page = int.Parse(el.InnerHtml);
}
else
{
page = 1;
}
List <string> urls = new List <string>();//分页
for (int i = 1; i <= page; i++)
{
string str = url + "pn" + i + "/";
urls.Add(str);
}
foreach (var e in urls)
{
try
{
//抓取每页
string htmlB = crawler.Crawl(e, Encoding.UTF8);
IDocument docuemnt = htmlParse.Parse(htmlB);
IElement eleist = docuemnt.QuerySelectorAll("ul").Where(p => p.ClassName == "house-list-wrap")
.ToList().FirstOrDefault();
IDocument docuementC = htmlParse.Parse(eleist.InnerHtml);
List <IElement> eliss = docuementC.QuerySelectorAll("div").Where(p => p.ClassName == "list-info")
.ToList();
//抓取每条
Parallel.For(0, eliss.Count + 1, i =>
{
string itemUrl = string.Empty;
try
{
IDocument documentD = htmlParse.Parse(eliss[i].InnerHtml);
IElement eloo = documentD.QuerySelector("a");
itemUrl = eloo.GetAttribute("href");
var htmlE = crawler.Crawl(itemUrl, Encoding.UTF8);
//开始解析
IDocument documentE = htmlParse.Parse(htmlE);
IElement time = documentE.QuerySelectorAll("div")
.FirstOrDefault(o => o.ClassName == "other");
string update = time.InnerHtml.Substring(0, time.InnerHtml.IndexOf("<"))
.Replace("发布时间：", "").Trim();
DateTime updateime =
ParseTool.StringToDateTime(ParseTool.StringToDateTime(update)
.ToShortDateString());
if (updateime > DateTime.Now.AddMonths(-2))
{
//标题
string InfoTitle = documentE.QuerySelectorAll("h1").FirstOrDefault().InnerHtml
.Trim();
//详细内容
string InfoContent = documentE.QuerySelectorAll("div")
.FirstOrDefault(u => u.ClassName == "maincon").InnerHtml.Trim();
//电话
string Phone = documentE.QuerySelectorAll("span")
.FirstOrDefault(u => u.ClassName == "phone").InnerHtml.Trim();
//租金
string rentMoney = documentE.QuerySelectorAll("em")
.FirstOrDefault(u => u.ClassName == "redfont").InnerHtml.Trim();
//面积
string areasize =
htmlParse.Parse(documentE.QuerySelectorAll("ul")
.FirstOrDefault(u => u.ClassName == "info").InnerHtml)
.QuerySelectorAll("li").ToList()[2].InnerHtml.Replace("面积：", "")
.Replace("㎡", "").Trim();
//客户名
string customerName = documentE.QuerySelectorAll("a")
.Where(u => u.ClassName == "tx").ToList()[1].InnerHtml.Trim();
var infolilist = htmlParse
.Parse(documentE.QuerySelectorAll("ul")
.FirstOrDefault(u => u.ClassName == "info")
.InnerHtml).QuerySelectorAll("li");
//区域名字
string AreaName = string.Join(",",
htmlParse.Parse(infolilist[0].InnerHtml).QuerySelectorAll("a")
.Select(p => p.InnerHtml.Trim()).ToList()).Trim();
ShopBegRent shop = new ShopBegRent();
shop.AreaName = AreaName;
shop.AreaId = area.Id.ToString();
shop.InfoContent = InfoContent;
shop.InfoTitle = InfoTitle;
shop.Phone = Phone;
shop.MaxRentMoney = ParseTool.StringToDouble(rentMoney) + 1000;
shop.MinRentMoney = (ParseTool.StringToDouble(rentMoney) - 1000) > 0
? (ParseTool.StringToDouble(rentMoney) - 1000)
: 0;
shop.Customer = customerName;
shop.UpdateTime = updateime;
if (areasize.Contains("-"))
{
string[] areasizes = areasize.Split('-');
shop.MinArea = ParseTool.StringToDouble(areasizes[0]);
shop.MaxArea = ParseTool.StringToDouble(areasizes[1]);
}
else
{
shop.MinArea = ParseTool.StringToDouble(areasize) - 10 > 0
? double.Parse(areasize) - 10
: 0;
shop.MaxArea = ParseTool.StringToDouble(areasize) + 10;
}
shop.UpdateTime = updateime;
shop.Id = Guid.NewGuid();
shopbegrepo.Add(shop);
Console.WriteLine(area.Name + "添加了一条商铺求租");
}
}
catch (Exception exception)
{
errorUrlrepsitory.Add(new ErrorUrl()
{
Url = itemUrl, UrlType = UrlType.Item
});
log.Error(exception.ToString());
}
});
}
catch (Exception exception)
{
errorUrlrepsitory.Add(new ErrorUrl()
{
Url = e, UrlType = UrlType.Page
});
log.Error(exception.ToString());
}
}
}
Console.WriteLine("抓取" + area.Name + "求租信息完成");
}
catch (Exception e)
{
log.Error(e.ToString());
Console.WriteLine(e);
}
}
public string[] GetStringArray()
{
return(ParseTool.String2StringArray(m_content));
}
public static bool TryParse(string notification, out AmazonSESNotification amazonSESNotification)
{
return(ParseTool.TryParseAmazonSESNotification(notification, out amazonSESNotification));
}
public static AmazonSESNotification Parse(string notification)
{
return(ParseTool.ParseAmazonSESNotification(notification));
}
public static object TableString2ObjectValue(string v, FieldType fieldValueType, string enumType)
{
object t = null;
switch (fieldValueType)
{
case FieldType.String:
t = v;
break;
case FieldType.Bool:
t = bool.Parse(v);
break;
case FieldType.Int:
t = int.Parse(v);
break;
case FieldType.Float:
t = float.Parse(v);
break;
case FieldType.Vector2:
t = ParseTool.String2Vector2(v);
break;
case FieldType.Vector3:
t = ParseTool.String2Vector3(v);
break;
case FieldType.Color:
t = ParseTool.String2Color(v);
break;
case FieldType.Enum:
Type type = ConfigFieldValueType2Type(fieldValueType, enumType);
t = Enum.Parse(type, v);
break;
case FieldType.StringArray:
t = ParseTool.String2StringArray(v);
break;
case FieldType.IntArray:
t = ParseTool.String2IntArray(v);
break;
case FieldType.FloatArray:
t = ParseTool.String2FloatArray(v);
break;
case FieldType.BoolArray:
t = ParseTool.String2BoolArray(v);
break;
case FieldType.Vector2Array:
t = ParseTool.String2Vector2(v);
break;
case FieldType.Vector3Array:
t = ParseTool.String2Vector3Array(v);
break;
default:
break;
}
return(t);
}
/// <summary>
/// 抓取出租信息
/// </summary>
/// <param name="area"></param>
public void CrawlDataCz(Area area)
{
string url = string.Empty;
try
{
Crawler crawler = new Crawler();
url = area.Url + ConstVar.出租 + "0/";
string html = crawler.Crawl(url, Encoding.UTF8);
var htmlParse = new HtmlParser();
IHtmlDocument docuement = htmlParse.Parse(html);
List <IElement> eles = docuement.QuerySelectorAll("div").ToList().Where(p => p.ClassName == "pager")
.ToList();
if (eles.Count > 0)
{
IHtmlDocument htmlA = htmlParse.Parse(eles[0].InnerHtml);
List <IElement> spanEles = htmlA.QuerySelectorAll("span").ToList();
int page = 0;
if (spanEles.Count > 2)
{
IElement el = spanEles[spanEles.Count - 2];
page = int.Parse(el.InnerHtml);
}
else
{
page = 1;
}
for (int i = 1; i < page + 1; i++)
{
string str = string.Empty;
try
{
str = url + "pn" + i + "/";
Crawler crawlerA = new Crawler();
string htmlB = crawlerA.Crawl(str, Encoding.UTF8);
IDocument docuemnt = htmlParse.Parse(htmlB);
IElement eleist = docuemnt.QuerySelectorAll("ul")
.Where(p => p.ClassName == "house-list-wrap").ToList().FirstOrDefault();
IDocument docuementC = htmlParse.Parse(eleist.InnerHtml);
List <IElement> eliss = docuementC.QuerySelectorAll("div").Where(p => p.ClassName == "pic")
.ToList();
Parallel.ForEach(eliss, p =>
{
string urlA = string.Empty;
try
{
IDocument documentD = htmlParse.Parse(p.InnerHtml);
IElement eloo = documentD.QuerySelector("a");
urlA = eloo.GetAttribute("href").ToString();
var htmlE = crawler.Crawl(eloo.GetAttribute("href").ToString(), Encoding.UTF8);
IDocument documentE = htmlParse.Parse(htmlE);
IElement ele = documentE.QuerySelectorAll("span")
.Where(o => o.InnerHtml.StartsWith("更新于")).FirstOrDefault();
DateTime time = ParseTool.StringToDateTime(ele.InnerHtml.Replace("更新于", ""));
if (time > DateTime.Now.AddMonths(-2))
{
IElement InfoTitleElee = documentE.QuerySelectorAll("h1")
.FirstOrDefault(o => o.ClassName == "c_000 f20");
IElement money = documentE.QuerySelectorAll("span")
.FirstOrDefault(o => o.ClassName == "house_basic_title_money_num");
var InfoContent = documentE.QuerySelectorAll("div")
.Where(o => o.ClassName == "general-item-wrap").FirstOrDefault(u =>
u.ParentElement.ClassName == "general-item general-miaoshu");
var Customer = documentE.QuerySelectorAll("span")
.FirstOrDefault(o => o.ClassName == "f14 c_333 jjrsay");
var phone = documentE.QuerySelectorAll("p")
.FirstOrDefault(o => o.ClassName == "phone-num");
var InfoEles = htmlParse.Parse(documentE.QuerySelectorAll("ul")
.FirstOrDefault(o => o.ClassName == "house_basic_title_content")
?.InnerHtml).QuerySelectorAll("li").ToList();
//面积
IElement areasize = htmlParse.Parse(InfoEles[0].InnerHtml)
.QuerySelectorAll("span")
.FirstOrDefault(o => o.ClassName == "house_basic_title_content_item2");
//行业名字
IElement IndustryName = htmlParse.Parse(InfoEles[2].InnerHtml)
.QuerySelectorAll("span")
.FirstOrDefault(o => o.ClassName == "house_basic_title_content_item3");
IElement address = htmlParse.Parse(InfoEles[5].InnerHtml).QuerySelectorAll("a")
.FirstOrDefault(o =>
o.ClassName == "house_basic_title_content_item3 blue-link");
IElement addressDetail = htmlParse.Parse(InfoEles[5].InnerHtml)
.QuerySelectorAll("span")
.FirstOrDefault(o =>
o.ClassName == "house_basic_title_content_item3 xxdz-des");
var shoptransfer = new ShopRentOrTransfer()
{
Id = Guid.NewGuid(),
ShopArea = areasize == null ? "" : areasize.InnerHtml,
InfoTitle = InfoTitleElee == null ? "" : InfoTitleElee.InnerHtml,
TransFerMoney = money == null ? "" : money.InnerHtml,
Address = address == null ? "" : string.Join("", address.InnerHtml),
DetailAddress = addressDetail == null ? "" : addressDetail.InnerHtml,
InfoContent = InfoContent == null ? "" : InfoContent.InnerHtml,
InfoType = Model.BaseModel.InfoType.出租,
IndustryName = IndustryName == null ? "" : IndustryName.InnerHtml,
Customer = Customer == null ? "" : Customer.InnerHtml,
Phone = phone == null ? "" : phone.InnerHtml,
AreaId = area.Id.ToString(),
UpdateTime = time
};
var imgUl = documentE.QuerySelectorAll("ul")
.FirstOrDefault(o => o.ClassName == "general-pic-list");
object obj = shoprepo.Add(shoptransfer);
bool resultId = (bool)obj;
Console.WriteLine(area.Name + "添加一条出租信息");
if (imgUl != null && resultId)
{
IDocument documentf = htmlParse.Parse(imgUl.InnerHtml);
var tem = documentf.QuerySelectorAll("img")
.Select(o => o.GetAttribute("data-src"));
if (tem != null && tem.Count() > 0)
{
foreach (var o in tem)
{
if (o != null)
{
Bitmap img = crawler.CrawlPic(o);
if (img != null)
{
string path =
AppDomain.CurrentDomain.BaseDirectory + "Imgs/" +
shoptransfer.Id + "/";
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
string fullPath =
path + Guid.NewGuid().ToString().Replace("-", "") +
".png";
img.Save(fullPath);
string savePath = fullPath.Replace(
AppDomain.CurrentDomain.BaseDirectory,
"");
imgrepo.Add(new Model.Image()
{
FkId = shoptransfer.Id,
ImageUrl = savePath,
InfoType = TableType.ShopRentOrTransfer,
});
}
}
}
}
}
}
}
catch (Exception e)
{
errorUrlrepsitory.Add(new ErrorUrl()
{
UrlType = UrlType.Item, Url = urlA
});
log.Error(e.ToString());
}
});
}
catch (Exception e)
{
errorUrlrepsitory.Add(new ErrorUrl()
{
UrlType = UrlType.Page, Url = str
});
log.Error(e.ToString());
}
}
}
Console.WriteLine(area.Name + "出租信息抓取完成");
}
catch (Exception e)
{
errorUrlrepsitory.Add(new ErrorUrl()
{
UrlType = UrlType.All, Url = url
});
log.Error(e.ToString());
}
}