private string GetSelectorValue(Page page, BaseSelector selector)
{
string totalStr = string.Empty;
if (selector.Type == SelectorType.Enviroment)
{
if (SelectorUtils.Parse(TotalPageSelector) is EnviromentSelector enviromentSelector)
{
totalStr = EntityExtractor.GetEnviromentValue(enviromentSelector.Field, page, 0);
}
}
else
{
totalStr = page.Selectable.Select(SelectorUtils.Parse(TotalPageSelector)).GetValue();
}
if (!string.IsNullOrEmpty(totalStr) && TotalPageFormatters != null)
{
foreach (var formatter in TotalPageFormatters)
{
totalStr = formatter.Formate(totalStr);
}
}
if (string.IsNullOrEmpty(totalStr))
{
throw new SpiderException("The result of total selector is null.");
}
else
{
return(totalStr);
}
}
private string GetSelectorValue(Page page, BaseSelector selector)
{
string result = string.Empty;
if (selector.Type == SelectorType.Enviroment)
{
if (SelectorUtils.Parse(selector) is EnviromentSelector enviromentSelector)
{
result = SelectorUtils.GetEnviromentValue(enviromentSelector.Field, page, 0)?.ToString();
}
}
else
{
result = page.Selectable.Select(SelectorUtils.Parse(selector)).GetValue();
}
if (!string.IsNullOrEmpty(result) && TotalPageFormatters != null)
{
foreach (var formatter in TotalPageFormatters)
{
result = formatter.Formate(result)?.ToString();
}
}
if (string.IsNullOrWhiteSpace(result))
{
throw new SpiderException("The result of total selector is null.");
}
else
{
return(result);
}
}
public virtual List <DataObject> Extract(Page page)
{
List <DataObject> result = new List <DataObject>();
if (GlobalValues != null && GlobalValues.Count > 0)
{
foreach (var enviromentValue in GlobalValues)
{
string name = enviromentValue.Name;
var value = page.Selectable.Select(SelectorUtils.Parse(enviromentValue)).GetValue();
page.Request.PutExtra(name, value);
}
}
ISelector selector = SelectorUtils.Parse(EntityDefine.Selector);
if (selector != null && EntityDefine.Multi)
{
var list = page.Selectable.SelectList(selector).Nodes();
if (list == null || list.Count == 0)
{
result = null;
}
else
{
if (EntityDefine.Take > 0)
{
list = list.Take(EntityDefine.Take).ToList();
}
int index = 0;
foreach (var item in list)
{
var obj = ExtractSingle(page, item, index);
if (obj != null)
{
result.Add(obj);
}
index++;
}
}
}
else
{
ISelectable select = selector == null ? page.Selectable : page.Selectable.Select(selector);
if (select != null)
{
var singleResult = ExtractSingle(page, select, 0);
result = singleResult != null ? new List <DataObject> {
singleResult
} : null;
}
else
{
result = null;
}
}
return(result);
}
public List <T> Extract(Page page)
{
List <T> result = new List <T>();
if (EntityDefine.SharedValues != null && EntityDefine.SharedValues.Count > 0)
{
foreach (var enviromentValue in EntityDefine.SharedValues)
{
string name = enviromentValue.Name;
var value = page.Selectable.Select(SelectorUtils.Parse(enviromentValue)).GetValue();
page.Request.PutExtra(name, value);
}
}
ISelector selector = SelectorUtils.Parse(EntityDefine.Selector);
if (selector != null && EntityDefine.Multi)
{
var list = page.Selectable.SelectList(selector).Nodes();
if (list == null || list.Count == 0)
{
result = null;
}
else
{
if (EntityDefine.Take > 0)
{
list = list.Take(EntityDefine.Take).ToList();
}
for (int i = 0; i < list.Count; ++i)
{
var item = list[i];
var obj = ExtractSingle(page, item, i);
if (obj != null)
{
result.Add(obj);
}
}
}
}
else
{
ISelectable select = selector == null ? page.Selectable : page.Selectable.Select(selector);
if (select != null)
{
var item = ExtractSingle(page, select, 0);
result = item != null ? new List <T> {
item
} : null;
}
else
{
result = null;
}
}
return(result);
}
private object ExtractField(ISelectable item, Page page, Column field, int index)
{
if (field == null)
{
return(null);
}
ISelector selector = SelectorUtils.Parse(field.Selector);
if (selector == null)
{
return(null);
}
if (selector is EnviromentSelector)
{
var enviromentSelector = selector as EnviromentSelector;
var value = SelectorUtils.GetEnviromentValue(enviromentSelector.Field, page, index);
foreach (var formatter in field.Formatters)
{
#if DEBUG
try
{
#endif
value = formatter.Formate(value);
#if DEBUG
}
catch (Exception e)
{
Logger.Error(e.ToString());
}
#endif
}
return(TryConvert(value, field.DataType));
}
else
{
bool needCount = field.Option == PropertyDefine.Options.Count;
if (needCount)
{
var values = item.SelectList(selector).Nodes();
return(values.Count);
}
else
{
var value = (object)item.Select(selector)?.GetValue(field.Option == PropertyDefine.Options.PlainText);
foreach (var formatter in field.Formatters)
{
#if DEBUG
try
{
#endif
value = formatter.Formate(value);
#if DEBUG
}
catch (Exception e)
{
Logger.Error(e.ToString());
}
#endif
}
return(TryConvert(value, field.DataType));
}
}
}
private dynamic ExtractField(ISelectable item, Page page, Column field, int index)
{
if (field == null)
{
return(null);
}
ISelector selector = SelectorUtils.Parse(field.Selector);
if (selector == null)
{
return(null);
}
string tmpValue;
if (selector is EnviromentSelector)
{
var enviromentSelector = selector as EnviromentSelector;
tmpValue = GetEnviromentValue(enviromentSelector.Field, page, index);
foreach (var formatter in field.Formatters)
{
tmpValue = formatter.Formate(tmpValue);
}
return(tmpValue);
}
else
{
bool needPlainText = field.Option == PropertyDefine.Options.PlainText;
if (field.Multi)
{
var propertyValues = item.SelectList(selector).Nodes();
List <string> results = new List <string>();
foreach (var propertyValue in propertyValues)
{
results.Add(propertyValue.GetValue(needPlainText));
}
foreach (var formatter in field.Formatters)
{
results = formatter.Formate(results);
}
return(new JArray(results));
}
else
{
bool needCount = field.Option == PropertyDefine.Options.Count;
if (needCount)
{
var propertyValues = item.SelectList(selector).Nodes();
string count = propertyValues?.Count.ToString();
count = string.IsNullOrEmpty(count) ? "-1" : count;
return(count);
}
else
{
tmpValue = item.Select(selector)?.GetValue(needPlainText);
foreach (var formatter in field.Formatters)
{
tmpValue = formatter.Formate(tmpValue);
}
return(tmpValue);
}
}
}
}
private dynamic ExtractField(ISelectable item, Page page, Column field, int index)
{
if (field == null)
{
return(null);
}
ISelector selector = SelectorUtils.Parse(field.Selector);
if (selector == null)
{
return(null);
}
if (selector is EnviromentSelector)
{
var enviromentSelector = selector as EnviromentSelector;
var tmpValue = SelectorUtils.GetEnviromentValue(enviromentSelector.Field, page, index);
foreach (var formatter in field.Formatters)
{
#if DEBUG
try
{
#endif
tmpValue = formatter.Formate(tmpValue);
#if DEBUG
}
catch (Exception e)
{
}
#endif
}
return(tmpValue == null ? null : Convert.ChangeType(tmpValue, field.DataType));
}
else
{
bool needPlainText = field.Option == PropertyDefine.Options.PlainText;
if (field.Multi)
{
var propertyValues = item.SelectList(selector).Nodes();
List <dynamic> results = new List <dynamic>();
foreach (var propertyValue in propertyValues)
{
results.Add(Convert.ChangeType(propertyValue.GetValue(needPlainText), field.DataType));
}
foreach (var formatter in field.Formatters)
{
#if DEBUG
try
{
#endif
results = formatter.Formate(results);
#if DEBUG
}
catch (Exception e)
{
}
#endif
}
return(results);
}
else
{
bool needCount = field.Option == PropertyDefine.Options.Count;
if (needCount)
{
var values = item.SelectList(selector).Nodes();
return(values.Count);
}
else
{
var value = item.Select(selector)?.GetValue(needPlainText);
dynamic tmpValue = value;
foreach (var formatter in field.Formatters)
{
#if DEBUG
try
{
#endif
tmpValue = formatter.Formate(tmpValue);
#if DEBUG
}
catch (Exception e)
{
}
#endif
}
return(tmpValue == null ? null : Convert.ChangeType(tmpValue, field.DataType));
}
}
}
}