public void Execute(PerformContext context)
{
context.WriteLine("开始验证代理Ip的有效性");
var ips =
Task.Run(async() => await _proxyIpService.GetWaitValidateIps(20))
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
var total = ips.Count;
var successCount = 0;
var http = new HttpRequestProvider();
foreach (var proxyIpDto in ips)
{
var success = false;
try
{
var result = http.Get("http://www.jucheap.com/home/showip", null, proxyIpDto.Ip, proxyIpDto.Port);
success = "ok".Equals(result, StringComparison.OrdinalIgnoreCase);
}
catch (Exception ex)
{
context.WriteLine($"验证失败：{ex.Message}");
}
if (success)
{
successCount++;
}
Task.Run(async() => await _proxyIpService.Update(proxyIpDto.Id, success)).ConfigureAwait(false);
}
context.WriteLine($"验证完成，总共{total}条，成功{successCount}条");
context.WriteLine("验证执行完成");
}
/// <summary>
/// Initializes a new instance of <see cref="ODataContext"/> class.
/// </summary>
/// <param name="model">The Edm model.</param>
/// <param name="settings">The convert setting.</param>
public ODataContext(IEdmModel model, OpenApiConvertSettings settings)
{
Model = model ?? throw Error.ArgumentNull(nameof(model));
Settings = settings ?? throw Error.ArgumentNull(nameof(settings));
EdmSpatialTypeVisitor visitor = new EdmSpatialTypeVisitor();
visitor.Visit(model);
IsSpatialTypeUsed = visitor.IsSpatialTypeUsed;
OperationHanderProvider = new OperationHandlerProvider();
PathItemHanderProvider = new PathItemHandlerProvider();
_authorizationProvider = new AuthorizationProvider(model);
_httpRequestProvider = new HttpRequestProvider(model);
_pathProvider = new ODataPathProvider(settings);
if (settings.EnableKeyAsSegment != null)
{
// We have the global setting, use the global setting
KeyAsSegment = settings.EnableKeyAsSegment.Value;
}
else
{
KeyAsSegment = false;
if (model.EntityContainer != null)
{
var keyAsSegment = model.GetKeyAsSegmentSupported(model.EntityContainer);
if (keyAsSegment != null)
{
KeyAsSegment = keyAsSegment.IsSupported;
}
}
}
}
public async Task <bool> ProvideHttp1(StreamReader streamReader, Action <IHttpRequest> onRequest, ILogger logger, string request)
{
if (request == null)
{
return(false);
}
var firstSpace = request.IndexOf(' ');
var lastSpace = request.LastIndexOf(' ');
var tokens = new[]
{
request.Substring(0, firstSpace),
request.Substring(firstSpace + 1, lastSpace - firstSpace - 1),
request.Substring(lastSpace + 1)
};
if (tokens.Length != 3)
{
return(false);
}
var httpProtocol = tokens[2];
var url = tokens[1];
var queryString = GetQueryStringData(ref url);
var uri = new Uri(url, UriKind.Relative);
var headersRaw = new List <KeyValuePair <string, string> >();
// get the headers
string line;
while (!string.IsNullOrEmpty(line = await streamReader.ReadLineAsync().ConfigureAwait(false)))
{
var currentLine = line;
var headerKvp = SplitHeader(currentLine);
headersRaw.Add(headerKvp);
}
IHttpHeaders headers = new HttpHeaders
(headersRaw);
var post = await HttpRequestProvider.GetPostData(streamReader, headers).ConfigureAwait(false);
if (!headers.TryGetByName("_method", out var verb))
{
verb = tokens[0];
}
var httpMethod = HttpMethodProvider.Default.Provide(verb);
onRequest(new HttpRequest
(
headers, httpMethod, httpProtocol, uri,
uri.OriginalString.Split(Separators, StringSplitOptions.RemoveEmptyEntries), queryString, post
));
return(true);
}
public async Task <IActionResult> Index()
{
TokenInfo tokeninfo = await HttpRequestProvider.GetTokenClaims("appid", "appPwd");
LogHelpProvider.Debug(this, "default controller");
return(Content("11111"));
}
public Task <bool> ProvideHttp2(StreamReader streamReader, Action <IHttpRequest> onRequest, ILogger logger)
{
var wrap = streamReader.BaseStream.CreateStreams();
var connection = new Connection(new ConnectionConfigurationBuilder(true)
.UseStreamListener(stream =>
{
var headers = new H2Headers()
{
Headers = stream.ReadHeadersAsync().ConfigureAwait(false).GetAwaiter().GetResult()
};
var post = GetPostData(stream, headers).ConfigureAwait(false).GetAwaiter().GetResult();
var url = headers.GetByName(":path");
var uri = new Uri(url, UriKind.Relative);
var method = HttpMethodProvider.Default.Provide(headers.GetByName(":method"));
if (headers.TryGetByName(":authority", out var authority) && !headers.TryGetByName("host", out _))
{
headers.Headers = headers.Headers.Concat
(
new[]
{
new HeaderField()
{
Name = "Host", Value = authority.Replace("http://", "").Replace("https://", "")
}
}
);
}
headers.Headers = headers.Headers.Where(x => !x.Name.StartsWith(":"));
onRequest
(
new HttpRequest
(
headers, method, "HTTP/2.0", uri,
uri.OriginalString.Split(Separators, StringSplitOptions.RemoveEmptyEntries),
HttpRequestProvider.GetQueryStringData(ref url), post
)
);
return(true);
})
.UseSettings(Settings.Max)
.UseHuffmanStrategy(HuffmanStrategy.IfSmaller)
.Build(), wrap.ReadableStream, wrap.WriteableStream, new Connection.Options()
{
Logger = logger
});
var remoteGoAwayTask = connection.RemoteGoAwayReason;
var closeWhenRemoteGoAway = Task.Run(async() =>
{
await remoteGoAwayTask;
await connection.GoAwayAsync(ErrorCode.NoError, true);
});
return(Task.FromResult(true));
}
public void GetHttpRequestsReturnsNullForTargetWithoutAnnotations(string name)
{
// Arrange
IEdmModel model = GetEdmModel("", "");
var httpRequestProvider = new HttpRequestProvider(model);
IEdmNavigationSource navigationSource = model.FindDeclaredNavigationSource(name);
Assert.NotNull(navigationSource);
// Act & Assert
var requests = httpRequestProvider.GetHttpRequests(navigationSource as IEdmVocabularyAnnotatable);
// Assert
Assert.Null(requests);
}
public IPEndPoint Start(IPAddress ipAddress, int port)
{
var requestProvider = new HttpRequestProvider();
var httpServer = new HttpServer(requestProvider);
httpServer.Use(Serve);
var listener = new TcpListener(ipAddress, port);
httpServer.Use(new TcpListenerAdapter(listener));
httpServer.Start();
EndPoint = (IPEndPoint)listener.LocalEndpoint;
return(EndPoint);
}
private List <ProxyIpDto> GrabWest(PerformContext context)
{
var proxyIp = new List <ProxyIpDto>();
try
{
context.WriteLine("->西刺");
var http = new HttpRequestProvider();
var url = "http://www.xicidaili.com/nn/1"; // 西刺
var result = http.Get(url);
if (result.IsNotBlank())
{
var count = 0;
var regex = @"<td>(\d+\.\d+\.\d+\.\d+)</td>\s+<td>(\d+)</td>";
var mstr = Regex.Match(result, regex);
while (mstr.Success)
{
var ip = new ProxyIpDto
{
Ip = mstr.Groups[1].Value,
Port = mstr.Groups[2].Value.ToInt()
};
proxyIp.Add(ip);
mstr = mstr.NextMatch();
count++;
}
context.WriteLine($"->西刺，抓取到{count}条");
}
}
catch (Exception ex)
{
context.WriteLine($"->西刺，发生错误:{ex.Message}");
Log.Error(ex);
}
return(proxyIp);
}
private List <ProxyIpDto> Grab66(PerformContext context)
{
var http = new HttpRequestProvider();
var proxyIp = new List <ProxyIpDto>();
context.WriteLine("->66Ip");
try
{
var url = "http://www.66ip.cn/nmtq.php?getnum=100&isp=0&anonymoustype=3&start=&ports=&export=&ipaddress=&area=1&proxytype=2&api=66ip"; // 66ip
var result = http.Get(url);
if (result.IsNotBlank())
{
var count = 0;
var regex = @"(\d+\.\d+\.\d+\.\d+):(\d+)";
var mstr = Regex.Match(result, regex);
while (mstr.Success)
{
var ip = new ProxyIpDto
{
Ip = mstr.Groups[1].Value,
Port = mstr.Groups[2].Value.ToInt()
};
proxyIp.Add(ip);
mstr = mstr.NextMatch();
count++;
}
context.WriteLine($"->66Ip，抓取到{count}条");
}
}
catch (Exception ex)
{
context.WriteLine($"->66Ip，发生错误:{ex.Message}");
Log.Error(ex);
}
return(proxyIp);
}
public void GetHttpRequestsReturnsForEdmModelNavigationSourceWithAnnotations(string name)
{
// Arrange
string annotation = @"
<Annotation Term=""Org.OData.Core.V1.HttpRequests"">
<Collection>
<Record>
<PropertyValue Property=""Description"" />
<PropertyValue Property=""MethodDescription"" String=""Example"" />
<PropertyValue Property=""MethodType"" String=""DELETE"" />
<PropertyValue Property=""CustomHeaders"">
<Collection>
<Record>
<PropertyValue Property=""Name"" String=""Authorization"" />
<PropertyValue Property=""Description"" String=""String"" />
<PropertyValue Property=""Required"" Bool=""false"" />
</Record>
</Collection>
</PropertyValue>
<PropertyValue Property=""CustomQueryOptions"">
<Collection />
</PropertyValue>
<PropertyValue Property=""HttpResponses"">
<Collection>
<Record>
<PropertyValue Property=""ResponseCode"" String=""204"" />
<PropertyValue Property=""Examples"">
<Record Type=""Org.OData.Core.V1.InlineExample"">
<PropertyValue Property=""InlineExample"" />
<PropertyValue Property=""Description"" />
</Record>
</PropertyValue>
</Record>
</Collection>
</PropertyValue>
<PropertyValue Property=""SecuritySchemes"">
<Collection>
<Record>
<PropertyValue Property=""AuthorizationSchemeName"" String=""Delegated (work or school account)"" />
<PropertyValue Property=""RequiredScopes"">
<Collection>
<String>Directory.AccessAsUser.All</String>
</Collection>
</PropertyValue>
</Record>
</Collection>
</PropertyValue>
</Record>
<Record>
<PropertyValue Property=""Description"" />
<PropertyValue Property=""MethodDescription"" String=""Example"" />
<PropertyValue Property=""MethodType"" String=""GET"" />
<PropertyValue Property=""CustomHeaders"">
<Collection>
<Record>
<PropertyValue Property=""Name"" String=""Authorization"" />
<PropertyValue Property=""Description"" String=""String"" />
<PropertyValue Property=""Required"" Bool=""false"" />
</Record>
</Collection>
</PropertyValue>
<PropertyValue Property=""CustomQueryOptions"">
<Collection />
</PropertyValue>
<PropertyValue Property=""HttpResponses"">
<Collection>
<Record>
<PropertyValue Property=""ResponseCode"" String=""200"" />
<PropertyValue Property=""Examples"">
<Record Type=""Org.OData.Core.V1.InlineExample"">
<PropertyValue Property=""InlineExample"" String=""{
 "accountEnabled":false,
 "deviceId":"4c299165-6e8f-4b45-a5ba-c5d250a707ff",
 "displayName":"Test device",
 "id": "id-value",
 "operatingSystem":"linux",
 "operatingSystemVersion":"1"
}
"" />
<PropertyValue Property=""Description"" String=""application/json"" />
</Record>
</PropertyValue>
</Record>
</Collection>
</PropertyValue>
<PropertyValue Property=""SecuritySchemes"">
<Collection>
<Record>
<PropertyValue Property=""AuthorizationSchemeName"" String=""Delegated (work or school account)"" />
<PropertyValue Property=""RequiredScopes"">
<Collection>
<String>Directory.Read.All</String>
<String>Directory.ReadWrite.All</String>
<String>Directory.AccessAsUser.All</String>
</Collection>
</PropertyValue>
</Record>
<Record>
<PropertyValue Property=""AuthorizationSchemeName"" String=""Application"" />
<PropertyValue Property=""RequiredScopes"">
<Collection>
<String>Device.ReadWrite.All</String>
<String>Directory.Read.All</String>
<String>Directory.ReadWrite.All</String>
</Collection>
</PropertyValue>
</Record>
</Collection>
</PropertyValue>
</Record>
<Record>
<PropertyValue Property=""Description"" />
<PropertyValue Property=""MethodDescription"" String=""PATCH Example"" />
<PropertyValue Property=""MethodType"" String=""PATCH"" />
<PropertyValue Property=""CustomHeaders"">
<Collection>
<Record>
<PropertyValue Property=""Name"" String=""Authorization"" />
<PropertyValue Property=""Description"" String=""String"" />
<PropertyValue Property=""Required"" Bool=""false"" />
</Record>
</Collection>
</PropertyValue>
<PropertyValue Property=""CustomQueryOptions"">
<Collection />
</PropertyValue>
<PropertyValue Property=""HttpResponses"">
<Collection>
<Record>
<PropertyValue Property=""ResponseCode"" String=""204"" />
<PropertyValue Property=""Examples"">
<Record Type=""Org.OData.Core.V1.InlineExample"">
<PropertyValue Property=""InlineExample"" />
<PropertyValue Property=""Description"" />
</Record>
</PropertyValue>
</Record>
</Collection>
</PropertyValue>
<PropertyValue Property=""SecuritySchemes"">
<Collection>
<Record>
<PropertyValue Property=""AuthorizationSchemeName"" String=""Delegated (work or school account)"" />
<PropertyValue Property=""RequiredScopes"">
<Collection>
<String>Directory.ReadWrite.All</String>
<String>Directory.AccessAsUser.All</String>
</Collection>
</PropertyValue>
</Record>
</Collection>
</PropertyValue>
</Record>
</Collection>
</Annotation>";
IEdmModel model = GetEdmModel(annotation, annotation);
var httpRequestProvider = new HttpRequestProvider(model);
IEdmNavigationSource navigationSource = model.FindDeclaredNavigationSource(name);
Assert.NotNull(navigationSource);
// Act
var requests = httpRequestProvider.GetHttpRequests(navigationSource as IEdmVocabularyAnnotatable);
// Assert
Assert.NotEmpty(requests);
Assert.Equal(3, requests.Count());
// Act
var anotherRequests = httpRequestProvider.GetHttpRequests(navigationSource as IEdmVocabularyAnnotatable);
// Assert
Assert.True(ReferenceEquals(requests, anotherRequests));
// Act (PATCH)
var request = httpRequestProvider.GetHttpRequest(navigationSource as IEdmVocabularyAnnotatable, "PATCH");
// Assert
Assert.NotNull(request);
Assert.Equal("PATCH Example", request.MethodDescription);
Assert.NotNull(request.SecuritySchemes);
var securityScheme = Assert.Single(request.SecuritySchemes);
Assert.Equal("Delegated (work or school account)", securityScheme.AuthorizationSchemeName);
Assert.NotNull(securityScheme.RequiredScopes);
Assert.Equal(new[] { "Directory.ReadWrite.All", "Directory.AccessAsUser.All" }, securityScheme.RequiredScopes);
}