public static async Task <string> Readline(WorkItemManager manager, WorkItem wi, PropertyDescriptor propertyDescriptor, string currentValue)
{
ReadLine.ClearHistory();
ReadLine.AutoCompletionHandler = null;
if (propertyDescriptor.ValueProvider != null)
{
var valueProvider = manager.ValidationManager.CreateValueProvider(wi, propertyDescriptor.ValueProvider);
if (valueProvider.IsUserExpierenceEnumerable)
{
foreach (var providedValue in await valueProvider.ProvideAllValuesAsync())
{
ReadLine.AddHistory(providedValue.Value);
}
}
ReadLine.AutoCompletionHandler = new ValueProviderReadLineAutoCompletion(valueProvider);
}
string result;
if (string.IsNullOrWhiteSpace(currentValue))
{
result = ReadLine.Read($"{propertyDescriptor.Name}: ");
}
else
{
result = ReadLine.Read($"{propertyDescriptor.Name} [{currentValue}]: ");
}
return(result);
}
public ValidationContext(WorkItemManager workItemManager, WorkItem workItem, IEnumerable <PropertyChange> appliedChanges, bool internalEdit)
{
WorkItemManager = workItemManager;
WorkItem = workItem;
AppliedChanges = appliedChanges;
InternalEdit = internalEdit;
}
private static async Task <WorkItemManager> SetupManagerAsync(string?sourceName, string?configFile)
{
sourceName ??= "default";
configFile = FindFile(Environment.CurrentDirectory, configFile ?? "wi.config.json");
if (configFile is null)
{
throw new ArgumentException("configFile cannot be found", nameof(ArgumentException));
}
var configuration = new ConfigurationBuilder()
.SetBasePath(Environment.CurrentDirectory)
.AddJsonFile(configFile, true)
.Build();
var options = configuration.Get <CliOptions>(options =>
{
options.BindNonPublicProperties = true;
}) ?? throw new InvalidOperationException("No Configuration can be read.");
var selectedSource = options.Sources.FirstOrDefault(s => s.Name == sourceName) ?? throw new InvalidOperationException("No matching source configuration could be found.");
var provider = BuildProvider(selectedSource);
var descriptorProvider = new CommonSdlcDescriptorProvider();
var manager = new WorkItemManager(provider, new CommonSdlcDescriptorProvider());
return(manager);
}
public async Task ImmutableValidator_Validate_Error()
{
// arrange
WorkItemManager manager = BuildManager();
var properties = new Property[] {
new Property("A", "String", ""),
new Property("B", "String", "bb"),
};
// act
var result = await manager.CreateAsync("FOO", "BAR", properties);
// assert
Assert.NotNull(result);
Assert.False(result.Success);
Assert.NotNull(result.CreatedWorkItem);
Assert.Collection(result.Errors,
em =>
{
Assert.Equal(nameof(ImmutableValidator), em.Source);
Assert.Equal(string.Empty, em.ErrorCode);
Assert.Equal("FOO", em.ProjectCode);
Assert.Equal("1", em.Id);
Assert.Equal("B", em.Property);
}
);
}
public async Task WorkItemManager_ExecuteCommand_Invoke()
{
// arrange
var manager = new WorkItemManager(new InMemoryDataProvider(), new InMemoryDescriptorProvider(
WorkItemDescriptor.Create("BAR", new PropertyDescriptor[] {
PropertyDescriptor.Create("A", "String", initialValue: "a"),
PropertyDescriptor.Create("B", "String"),
},
stages: new StageDescriptor[] {
new StageDescriptor("stage-aa", new PropertyValueConditionDescriptor("A", "aa"),
Array.Empty <StagePropertyDescriptor>(),
new CommandDescriptor[] {
new ChangePropertyValueCommandDescriptor("make-bb", "MakeBBC", "B", "bbc")
}),
})
));
var issue = await manager.CreateAsync("FOO", "BAR", new Property[] {
new Property("A", "String", "aa"),
new Property("B", "String", "bb"),
});
// act
var result = await manager.ExecuteCommandAsync("FOO", issue.Id, "make-bb");
// assert
Assert.True(result.Success);
Assert.Collection(result.UpdatedWorkItem.Properties,
p => { Assert.Equal("aa", p.Value); Assert.Equal("A", p.Name); },
p => { Assert.Equal("bbc", p.Value); Assert.Equal("B", p.Name); }
);
}
public async Task WorkItemManager_CreateTemplate_WithDefaultValues()
{
// arrange
var workItemManager = new WorkItemManager(new InMemoryDataProvider(), new InMemoryDescriptorProvider(
WorkItemDescriptor.Create("BAR", new PropertyDescriptor[] {
PropertyDescriptor.Create("A", "String", initialValue: "a"),
PropertyDescriptor.Create("B", "String"),
})
));
// act
var template = await workItemManager.CreateTemplateAsync("FOO", "BAR");
// assert
Assert.NotNull(template);
Assert.Equal("BAR", template.WorkItemType);
Assert.Collection(template.Properties,
p =>
{
Assert.Equal("A", p.Name);
Assert.Equal("a", p.Value);
},
p =>
{
Assert.Equal("B", p.Name);
Assert.Equal("", p.Value);
}
);
}
public static async Task <int> ExecuteAsync(WorkItemManager workItemManager, string?project, string?type, TextWriter writer)
{
if (workItemManager is null)
{
throw new ArgumentNullException(nameof(workItemManager));
}
if (string.IsNullOrWhiteSpace(project))
{
throw new ArgumentException("message", nameof(project));
}
if (writer is null)
{
throw new ArgumentNullException(nameof(writer));
}
var items = await workItemManager.DataProvider.ListWorkItemsAsync(project, type);
var formatter = new WorkItemFormatter();
foreach (var item in items)
{
writer.WriteLine(formatter.FormatShortLine(item));
}
return(0);
}
public async Task WorkItemManager_Create_MissingWorkItemType()
{
// arrange
var manager = new WorkItemManager(null, new CommonSdlcDescriptorProvider());
// act
await Assert.ThrowsAsync <ArgumentException>(async() =>
{
await manager.CreateAsync("FOO", null, new Property[] { });
});
}
private static int RunWorkItems(WorkItemsCommandLineOptions options)
{
VssConnection connection = CreateConnection(options);
ISink sink = CreateSink("workitems", options);
WorkItemTrackingHttpClient witClient = connection.GetClient <WorkItemTrackingHttpClient>();
Task task = new WorkItemManager(witClient).WriteWorkItemDetails(sink);
task.Wait();
return(0);
}
void BuildQueryView()
{
queryStore.Clear();
foreach (var server in TFSVersionControlService.Instance.Servers)
{
var node = queryStore.AddNode().SetValue(titleField, server.Name);
foreach (var pc in server.ProjectCollections)
{
node.AddChild().SetValue(titleField, pc.Name);
var workItemManager = new WorkItemManager(pc);
foreach (var projectInfo in pc.Projects.OrderBy(x => x.Name))
{
var workItemProject = workItemManager.GetByGuid(projectInfo.Guid);
if (workItemProject == null)
{
continue;
}
node.AddChild().SetValue(titleField, projectInfo.Name);
var privateQueries = workItemManager.GetMyQueries(workItemProject);
if (privateQueries.Any())
{
node.AddChild().SetValue(titleField, "My Queries");
foreach (var query in privateQueries)
{
node.AddChild().SetValue(titleField, query.QueryName).SetValue(queryField, query);
node.MoveToParent();
}
node.MoveToParent();
}
var publicQueries = workItemManager.GetPublicQueries(workItemProject);
if (publicQueries.Any())
{
node.AddChild().SetValue(titleField, "Public");
foreach (var query in publicQueries)
{
node.AddChild().SetValue(titleField, query.QueryName).SetValue(queryField, query);
node.MoveToParent();
}
node.MoveToParent();
}
node.MoveToParent();
}
queryView.ExpandRow(node.CurrentPosition, true);
}
}
var cursor = queryStore.GetFirstNode();
if (cursor.MoveToChild()) //Move to Project Collections
{
queryView.ExpandToRow(cursor.CurrentPosition);
}
}
public void Run()
{
log.Debug("所有ip正在删除中...........");
work:
if (!WorkItemManager.CheckPassSpider())//****检查数据库是否有维护程序在执行******//
{
System.Threading.Thread.Sleep(60000);
goto work;
}
ProxyIpManager.DeleteNotEffectiveProxyIp();
log.Debug("所有ip删除完成...........");
}
private async Task WorkItemManager_Update_SimpleWithoutDescriptor(IDataProvider dataProvider)
{
// arrange
var manager = new WorkItemManager(dataProvider, new CommonSdlcDescriptorProvider());
var issue = await manager.CreateAsync("FOO", "BAR", new Property[] {
new Property("A", "String", "aa"),
new Property("B", "String", "bb"),
});
// act
var result = await manager.UpdateAsync("FOO", issue.Id, new Property[] {
new Property("A", "String", "aab"),
});
// assert
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.UpdatedWorkItem);
Assert.Equal("FOO", result.UpdatedWorkItem.ProjectCode);
Assert.Equal("BAR", result.UpdatedWorkItem.WorkItemType);
Assert.Collection(result.UpdatedWorkItem.Properties,
p =>
{
Assert.Equal("A", p.Name);
Assert.Equal("String", p.DataType);
Assert.Equal("aab", p.Value);
},
p =>
{
Assert.Equal("B", p.Name);
Assert.Equal("String", p.DataType);
Assert.Equal("bb", p.Value);
}
);
Assert.Collection(result.UpdatedWorkItem.Log,
l =>
{
Assert.Collection(l.Changes,
pc =>
{
Assert.Equal("A", pc.Name);
Assert.Equal("aa", pc.OldValue);
Assert.Equal("aab", pc.NewValue);
}
);
}
);
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.Object"/> class.
/// </summary>
public TfsContext(IServiceProvider serviceProvider)
{
AsyncOp = AsyncOperationManager.CreateOperation(null);
//WorkItemsListViewHelper.Instance
//TeamExplorerUtils.Instance
ServiceProvider = serviceProvider;
selectedProfile = WorkingProfile.DefaultProfile;
TfsContextManager = serviceProvider.Get <ITeamFoundationContextManager2>();
TfsContextManager.ContextChanged += ConnectionContextChanged;
IdentityManager = new TfsIdentityManager(this);
BuildDetailManager = new BuildDetailManager(this);
WorkItemManager = new WorkItemManager(this);
SelectedDirectory = GetService <SettingsService>().Get <string>(SettingsKeys.LastDirectoryKey);
}
public async Task WorkItemManager_Create_DoNotCreateWithoutProperties()
{
// arrange
var providerMock = new Mock <IDataProvider>();
providerMock.SetupGet(o => o.Write).Returns(true);
var manager = new WorkItemManager(providerMock.Object, new CommonSdlcDescriptorProvider());
// act
var result = await manager.CreateAsync("FOO", "BAR", new Property[] { });
// assert
Assert.False(result.Success);
providerMock.VerifyGet(o => o.Write);
providerMock.VerifyNoOtherCalls();
}
public async Task StringLengthValidator_Validate_NoSideEffect()
{
// arrange
WorkItemManager manager = BuildManager();
var properties = new Property[] {
new Property("A", "String", "ABC"),
new Property("B", "String", "ABCDEFGH"),
};
// act
var result = await manager.CreateAsync("FOO", "BAR", properties);
// assert
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.CreatedWorkItem);
Assert.Empty(result.Errors);
}
public async Task ImmutableValidator_Validate_Success()
{
// arrange
WorkItemManager manager = BuildManager();
var properties = new Property[] {
new Property("A", "String", "aa"),
new Property("B", "String", string.Empty),
};
// act
var result = await manager.CreateAsync("FOO", "BAR", properties);
// assert
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.CreatedWorkItem);
Assert.Empty(result.Errors);
}
public async Task EnumValueProvider_Validate_IgnoreNullValue()
{
// arrange
WorkItemManager manager = BuildManager();
var properties = new Property[] {
new Property("A", "String", "a"),
new Property("B", "String", "d"),
new Property("C", "String", ""),
};
// act
var result = await manager.CreateAsync("FOO", "BAR", properties);
// assert
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.CreatedWorkItem);
Assert.Collection(result.Errors);
}
public CheckInResult CheckIn(List <PendingChange> changes, string comment, Dictionary <int, WorkItemCheckinAction> workItems)
{
foreach (var change in changes)
{
this.VersionControlService.UploadFile(this, change);
}
var result = this.VersionControlService.CheckIn(this, changes, comment, workItems);
if (result.ChangeSet > 0)
{
WorkItemManager wm = new WorkItemManager(this.ProjectCollection);
wm.UpdateWorkItems(result.ChangeSet, workItems, comment);
}
this.RefreshPendingChanges();
ProcessGetOperations(result.LocalVersionUpdates, ProcessType.Get);
foreach (var file in changes.Where(ch => ch.ItemType == ItemType.File && !string.IsNullOrEmpty(ch.LocalItem)).Select(ch => ch.LocalItem).Distinct())
{
MakeFileReadOnly(file);
}
return(result);
}
public async System.Threading.Tasks.Task <string> ValidateWorkItemStateAsync()
{
string result = "";
string bodycontent = await Request.Content.ReadAsStringAsync();
JObject wit = JObject.Parse(bodycontent);
var resource = wit["resource"];
var workItemId = resource["workItemId"];
var fieldStatus = resource["fields"].Where(t => t.ToString().Contains("System.State")).SingleOrDefault();
var oldValue = fieldStatus.Values("oldValue").Select(t => (string)t).SingleOrDefault();
var newValue = fieldStatus.Values("newValue").Select(t => (string)t).SingleOrDefault();
if (!IsValidStated(oldValue, newValue))
{
WorkItemManager aux = new WorkItemManager();
aux.ReassignState(workItemId.ToString(), oldValue);
}
return(result);
}
public async Task WorkItemManager_ExecuteCommand_NotFound()
{
// arrange
var manager = new WorkItemManager(new InMemoryDataProvider(), new InMemoryDescriptorProvider(
WorkItemDescriptor.Create("BAR", new PropertyDescriptor[] {
PropertyDescriptor.Create("A", "String", initialValue: "a"),
PropertyDescriptor.Create("B", "String"),
})
));
var issue = await manager.CreateAsync("FOO", "BAR", new Property[] {
new Property("A", "String", "aa"),
new Property("B", "String", "bb"),
});
// act
var result = await manager.ExecuteCommandAsync("FOO", issue.Id, "Close");
// assert
Assert.False(result.Success);
}
public static async Task <int> ExecuteAsync(WorkItemManager manager, string?projectCode, string?id, string?command)
{
if (manager is null)
{
throw new ArgumentNullException(nameof(manager));
}
if (string.IsNullOrWhiteSpace(projectCode))
{
throw new ArgumentException("message", nameof(projectCode));
}
if (string.IsNullOrWhiteSpace(id))
{
throw new ArgumentException("message", nameof(id));
}
if (string.IsNullOrWhiteSpace(command))
{
throw new ArgumentNullException(nameof(command));
}
var result = await manager.ExecuteCommandAsync(projectCode, id, command);
if (result.Success)
{
Console.WriteLine($"Updated WorkItem and execute command {command} in project {result.UpdatedWorkItem?.ProjectCode} with id {result.UpdatedWorkItem?.Id}");
}
else
{
Console.WriteLine($"Failed to update WorkItem and execute command {command}");
foreach (var error in result.Errors)
{
Console.WriteLine($"{error.Property}: {error.Message}");
}
}
return(result.Success ? 0 : 1);
}
/// <summary>
/// 导入IP
/// </summary>
/// <param name="ip"></param>
/// <param name="ipArea"></param>
/// <param name="existsObj"></param>
/// <param name="message"></param>
/// <returns></returns>
public static int ImportProxyIp(string ip, string ipArea, out SysData_ProxyIp existsObj, out string message)
{
existsObj = null;
message = "";
if (string.IsNullOrEmpty(ip))
{
message = "ip不能为空";
return(0);
}
work:
if (!WorkItemManager.CheckPassSpider())//****检查数据库是否有维护程序在执行******//
{
System.Threading.Thread.Sleep(60000);
goto work;
}
existsObj = ProxyIpManager.GetProxyIpByIp(ip);
if (existsObj != null)
{
message = "ip已存在";
return(0);
}
if (!CheckProxyIp(ip))
{
message = "ip不可用";
return(0);
}
int result = ProxyIpManager.InsertProxyIp(ip, ipArea, out existsObj, out message, checkExists: false);
if (result != 1)
{
return(0);
}
else
{
message = "ip插入成功";
}
return(1);
}
public static async Task <int> ExecuteAsync(WorkItemManager manager, string?projectCode, string?id, TextWriter writer)
{
if (manager is null)
{
throw new ArgumentNullException(nameof(manager));
}
if (string.IsNullOrWhiteSpace(projectCode))
{
throw new ArgumentException("message", nameof(projectCode));
}
if (string.IsNullOrWhiteSpace(id))
{
throw new ArgumentException("message", nameof(id));
}
if (writer is null)
{
throw new ArgumentNullException(nameof(writer));
}
var workItem = await manager.GetAsync(projectCode, id);
if (workItem == null)
{
writer.WriteLine($"Could not find the specified workitem {id} in project {projectCode}");
}
else
{
var formatter = new WorkItemFormatter();
await formatter.FormatAsync(manager.DescriptorManager, workItem, writer);
}
return((workItem != null) ? 0 : 1);
}
public WorkItemOperation(ObjectPool <DevOpsConnectionPool> builderPool)
{
workItemManager = new WorkItemManager(builderPool);
}
public static async Task <int> ExecuteAsync(WorkItemManager manager, string?projectCode, string?id)
{
if (manager is null)
{
throw new ArgumentNullException(nameof(manager));
}
if (string.IsNullOrWhiteSpace(projectCode))
{
throw new ArgumentException("message", nameof(projectCode));
}
if (string.IsNullOrWhiteSpace(id))
{
throw new ArgumentException("message", nameof(id));
}
int resultCode = 1;
var workItem = await manager.GetAsync(projectCode, id);
if (workItem is null)
{
Console.WriteLine($"Could not find the specified workitem {id} in project {projectCode}");
}
else
{
var propertyDescriptors = manager.DescriptorManager.GetCurrentPropertyDescriptors(workItem);
var changedProperties = new List <Property>();
foreach (var property in workItem.Properties)
{
var propertyDescriptor = propertyDescriptors.FirstOrDefault(wi => wi.Name == property.Name);
string value;
if (propertyDescriptor is null)
{
Console.Write($"{property.Name} [{property.Value}]: ");
value = Console.ReadLine();
}
else
{
value = await ValueProviderReadLineAutoCompletion.Readline(manager, workItem, propertyDescriptor, property.Value);
}
if (!string.IsNullOrWhiteSpace(value))
{
changedProperties.Add(new Property(property.Name, property.DataType, value));
}
}
var result = await manager.UpdateAsync(workItem.ProjectCode, workItem.Id, changedProperties);
if (result.Success)
{
Console.WriteLine($"Updated WorkItem in project {result.UpdatedWorkItem?.ProjectCode} with id {result.UpdatedWorkItem?.Id}");
}
else
{
Console.WriteLine($"Failed to update WorkItem");
foreach (var error in result.Errors)
{
Console.WriteLine($"{error.Property}: {error.Message}");
}
}
resultCode = result.Success ? 0 : 1;
}
return(resultCode);
}
static void Main(string[] args)
{
string[] runType = args;
if (args == null || args.Length < 1)
{
string paraStr = Console.ReadLine();
runType = paraStr.Split(' ');
}
if (runType[0].Contains("1"))
{
WorkItemManager.SetAllStop(1);
Console.Write("爬取服务器暂停成功");
return;
}
else if (runType[0].Contains("0"))
{
WorkItemManager.SetAllStop(0);
Console.Write("爬取服务器启动成功");
return;
}
work:
if (!WorkItemManager.CheckPassSpider())//****检查数据库是否有维护程序在执行******//
{
System.Threading.Thread.Sleep(60000);
goto work;
}
if (!runType[0].Contains("IP代理"))
{
#region (设置日志名称)
Dictionary <string, List <string> > dic = new Dictionary <string, List <string> >();
//将参数添加到字典,用于生成日志文件名
foreach (string str in runType)
{
string strIndex1 = str.Split('.')[0];
string strIndex2 = str.Split('.')[1];
List <string> strList = new List <string>();
if (!dic.ContainsKey(strIndex1))
{
dic.Add(strIndex1, new List <string>());
}
strList = dic[strIndex1];
strList.Add(strIndex2);
dic[strIndex1] = strList;
}
StringBuilder sb = new StringBuilder();
foreach (KeyValuePair <string, List <string> > kvp in dic)
{
string key = kvp.Key;
List <string> list = kvp.Value;
sb.Append(key).Append("_");
for (int i = 0; i < list.Count; i++)
{
string str = list[i];
if (i < list.Count - 1)
{
sb.Append(str).Append(".");
}
else
{
sb.Append(str).Append(";");
}
}
}
InitLogger("(" + sb.ToString() + ")");
#endregion
foreach (string str in runType)
{
//运行操作
INewDataRum rum = null;
string directory = GetConfigDire();
string dllFill = directory + "\\FxtSpider.RunSource.dll";
Type type = Assembly.LoadFile(dllFill).GetType("FxtSpider.RunSource." + str);
if (type != null)
{
rum = Activator.CreateInstance(type) as INewDataRum;
}
else
{
type = Assembly.LoadFile(dllFill).GetType("FxtSpider.RunSource." + str.Split('.')[0] + ".其他城市");
rum = Activator.CreateInstance(type) as INewDataRum;
rum.CityName = str.Split('.')[1];
}
rum.start();
}
}
else
{
#region (设置日志名称)
Dictionary <string, List <string> > dic = new Dictionary <string, List <string> >();
StringBuilder sb = new StringBuilder();
//将参数添加到字典,用于生成日志文件名
foreach (string str in runType)
{
sb.Append(str).Append(";");
}
InitLogger("(" + sb.ToString() + ")");
#endregion
foreach (string str in runType)
{
string directory = GetConfigDire();
string dllFill = directory + "\\FxtSpider.RunSource.dll";
string className = str.Split('.')[0] + "." + str.Split('.')[1];
object objClass = Assembly.LoadFile(dllFill).CreateInstance("FxtSpider.RunSource." + className);
if (objClass != null)
{
MethodInfo method = objClass.GetType().GetMethod(str.Split('.')[2]);
ArrayList al = new ArrayList();
al.Add(method);
al.Add(objClass);
Thread m_thread = new Thread(new ParameterizedThreadStart(ExecIpSpider));
m_thread.Start(al);
}
}
}
}
public ValueProviderController(WorkItemManager workItemManager, ILogger <ValueProviderController> logger)
{
_workItemManager = workItemManager;
_logger = logger;
}
public WorkItemDescriptorController(WorkItemManager workItemManager, ILogger <WorkItemDescriptorController> logger)
{
_workItemManager = workItemManager;
_logger = logger;
}
public static async Task <int> ExecuteAsync(WorkItemManager manager, string?projectCode, string?workItemType)
{
if (manager is null)
{
throw new ArgumentNullException(nameof(manager));
}
if (string.IsNullOrWhiteSpace(projectCode))
{
throw new ArgumentException("message", nameof(projectCode));
}
if (string.IsNullOrWhiteSpace(workItemType))
{
throw new ArgumentException("message", nameof(workItemType));
}
var wi = await manager.CreateTemplateAsync(projectCode, workItemType);
var propertyDescriptors = manager.DescriptorManager.GetCurrentPropertyDescriptors(wi);
if (propertyDescriptors == null)
{
Console.WriteLine($"Cannot find type {workItemType} in project {projectCode}");
return(1);
}
var propertiesDictionary = new Dictionary <string, string>();
foreach (var propertyDescriptor in propertyDescriptors)
{
var value = await ValueProviderReadLineAutoCompletion.Readline(manager, wi, propertyDescriptor, string.Empty);
propertiesDictionary.Add(propertyDescriptor.Name, value);
}
var properties = wi.Properties
.Select(property =>
{
if (propertiesDictionary.TryGetValue(property.Name, out var value))
{
return(new Property(property.Name, property.DataType, value));
}
else
{
return(property);
}
})
.Where(p => p != null)
.ToArray();
var result = await manager.CreateAsync(projectCode, workItemType, properties);
if (result.Success)
{
Console.WriteLine($"Created WorkItem in project {result.CreatedWorkItem?.ProjectCode} with id {result.Id}");
}
else
{
Console.WriteLine($"Failed to create WorkItem");
foreach (var error in result.Errors)
{
Console.WriteLine($"{error.Property}: {error.Message}");
}
}
return(result.Success ? 0 : 1);
}
/// <summary>
/// 根据页面url,页面编码,正则表达式;获取相应字符串
/// </summary>
/// <param name="url">页面url</param>
/// <param name="encoding">页面编码</param>
/// <param name="dic">正则表达式规则</param>
/// <returns></returns>
public static Dictionary <string, List <string> > GetHtmlByRegex(string url, string encoding, Dictionary <string, RegexInfo> dic, 网站表 webObj, int cityId, string referer = null, bool keepAlive = false, int timeout = 60000)
{
string NowProxyIp = null;
int 网络异常重试次数 = 0;
int 网络异常时代理ip更换次数 = 0;
int 验证码异常时代理ip更换次数 = 0;
string resultHtml = "";
string ipStr = NowProxyIp == null ? "" : NowProxyIp + ",";
begin:
Dictionary <string, List <string> > resultDic = new Dictionary <string, List <string> >();
if (dic == null)
{
return(resultDic);
}
try
{
resultHtml = GetHtml(url, encoding, proxyIp: NowProxyIp, referer: referer, keepAlive: keepAlive, timeout: timeout);
}
catch (Exception ex)
{
if (网络异常重试次数 < 2)
{
System.Threading.Thread.Sleep(3000);
网络异常重试次数++;
goto begin;
}
if (webObj.BlockadeOfIP)
{
if (网络异常时代理ip更换次数 < 3)
{
System.Threading.Thread.Sleep(2000);
网络异常时代理ip更换次数++;
work:
if (!WorkItemManager.CheckPassSpider())//检查数据库是否有维护程序在执行
{
System.Threading.Thread.Sleep(60000);
goto work;
}
ProxyIpManager.SetNotEffectiveProxyIp(webObj.ID, NowProxyIp); //设置当前代理ip为不可用
NowProxyIp = ProxyIpManager.GetEffectiveProxyIp(webObj.ID); //获取新代理ip
ipStr = ipStr + NowProxyIp + ",";
goto begin;
}
}
log.Error(string.Format("SpiderHouse:(requestUrl:{0}--请求异常)", url), ex);
//记录爬取失败原因和信息(网络异常)
work2:
if (!WorkItemManager.CheckPassSpider())//检查数据库是否有维护程序在执行
{
System.Threading.Thread.Sleep(60000);
goto work2;
}
DatSpiderErrorLogManager.InsertError(cityId, webObj.ID, url, SysCodeManager.Code_1_1, "所用ip:" + ipStr);
goto end;
}
end:
resultHtml = Regex.Replace(resultHtml, @"(([\r\n])[\s]+|[\r\n]+|[\t]+)", "", RegexOptions.IgnoreCase);
//检测是否需要验证码
bool checkError = false;//记录是否为验证码错误
if (webObj.BlockadeOfIPType == WebsiteManager.BlockadeOfIPType1)
{
List <string> checkList = GetStrByRegexByIndex(resultHtml, regex_checkcode);
if (checkList != null && checkList.Count > 0)
{
网络异常时代理ip更换次数 = 0;
if (验证码异常时代理ip更换次数 < 3)
{
验证码异常时代理ip更换次数++;
//记录爬取失败原因和信息(网络异常)
work3:
if (!WorkItemManager.CheckPassSpider())//检查数据库是否有维护程序在执行
{
System.Threading.Thread.Sleep(60000);
goto work3;
}
ProxyIpManager.SetNotEffectiveProxyIp(webObj.ID, NowProxyIp); //设置当前代理ip为不可用
NowProxyIp = ProxyIpManager.GetEffectiveProxyIp(webObj.ID); //获取新代理ip
ipStr = ipStr + NowProxyIp + ",";
goto begin;
}
else
{
checkError = true;
//记录爬取时需要验证码
//记录爬取失败原因和信息(网络异常)
work4:
if (!WorkItemManager.CheckPassSpider())//检查数据库是否有维护程序在执行
{
System.Threading.Thread.Sleep(60000);
goto work4;
}
DatSpiderErrorLogManager.InsertError(cityId, webObj.ID, url, SysCodeManager.Code_1_3,
string.Format("所用ip:{0}", ipStr)
);
}
log.Debug(string.Format("SpiderHouse:(requestUrl:{0}--请求异常:需输入验证码)", url));
}
}
foreach (KeyValuePair <string, RegexInfo> kvp in dic)
{
string key = kvp.Key;
List <string> list = GetStrByRegexByIndex(resultHtml, kvp.Value);
resultDic.Add(key, list);
if (webObj != null && webObj.ID == WebsiteManager.搜房网_ID && resultHtml.Contains(">此房源已售出!<") && resultHtml.Contains(">您可以选择查看：<"))
{
continue;
}
//正则表达式为重要字段&&无网络异常&&无验证码异常&&正则表达式不为null
if (key.Contains("*") && !string.IsNullOrEmpty(resultHtml) && !checkError && kvp.Value != null && !string.IsNullOrEmpty(kvp.Value.RegexStr))
{
//通过规则未获取到信息&&不为验证码异常
if (list == null || list.Count < 1)
{
//记录爬取失败原因和信息(通过规则未获取到字符)
DatSpiderErrorLogManager.InsertError(cityId, webObj.ID, url, SysCodeManager.Code_1_2,
string.Format("描述:{0},规则:{1},索引:{2},其他规则个数:{3},所用ip:{4}",
key,
kvp.Value.RegexStr,
kvp.Value.RegexIndex,
kvp.Value.RegexInfoList == null ? 0 : kvp.Value.RegexInfoList.Count,
ipStr
)
);
}
}
}
if (webObj != null && webObj.ID == WebsiteManager.搜房网_ID && resultHtml.Contains(">此房源已售出!<") && resultHtml.Contains(">您可以选择查看：<"))
{
resultDic.Add("NotData", new List <string> {
"1"
});
}
return(resultDic);
}
