private void Stop()
{
try
{
lock (_internalLock)
{
CheckDisposed();
if (_state == State.Stopped)
{
return;
}
Log.ListenerStopping(Logger);
// If this instance registered URL prefixes then remove them before shutting down.
if (Options.RequestQueueMode == RequestQueueMode.Create || Options.RequestQueueMode == RequestQueueMode.CreateOrAttach)
{
Options.UrlPrefixes.UnregisterAllPrefixes();
UrlGroup.DetachFromQueue();
}
_state = State.Stopped;
}
}
catch (Exception exception)
{
Log.ListenerStopError(Logger, exception);
throw;
}
}
internal void RegisterAllPrefixes(UrlGroup urlGroup)
{
lock (_prefixes)
{
_urlGroup = urlGroup;
// go through the uri list and register for each one of them
// Call ToList to avoid modification when enumerating.
foreach (var pair in _prefixes.ToList())
{
var urlPrefix = pair.Value;
if (urlPrefix.PortValue == 0)
{
if (urlPrefix.IsHttps)
{
throw new InvalidOperationException("Cannot bind to port 0 with https.");
}
FindHttpPortUnsynchronized(pair.Key, urlPrefix);
}
else
{
// We'll get this index back on each request and use it to look up the prefix to calculate PathBase.
_urlGroup.RegisterPrefix(pair.Value.FullPrefix, pair.Key);
}
}
}
}
internal DelegationRule(UrlGroup sourceQueueUrlGroup, string queueName, string urlPrefix, ILogger logger)
{
_sourceQueueUrlGroup = sourceQueueUrlGroup;
_logger = logger;
QueueName = queueName;
UrlPrefix = urlPrefix;
Queue = new RequestQueue(queueName, _logger);
}
internal DelegationRule(UrlGroup sourceQueueUrlGroup, string queueName, string urlPrefix, ILogger logger)
{
_sourceQueueUrlGroup = sourceQueueUrlGroup;
_logger = logger;
QueueName = queueName;
UrlPrefix = urlPrefix;
Queue = new RequestQueue(queueName, UrlPrefix, _logger, receiver: true);
_urlGroup = Queue.UrlGroup;
}
public ServerDelegationPropertyFeature(RequestQueue queue, ILogger logger)
{
if (queue.UrlGroup == null)
{
throw new ArgumentException($"{nameof(queue)}.UrlGroup can't be null");
}
_urlGroup = queue.UrlGroup;
_logger = logger;
}
public HttpSysListener(HttpSysOptions options, ILoggerFactory loggerFactory)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
if (!HttpApi.Supported)
{
throw new PlatformNotSupportedException();
}
Debug.Assert(HttpApi.ApiVersion == HttpApiTypes.HTTP_API_VERSION.Version20, "Invalid Http api version");
Options = options;
Logger = loggerFactory.CreateLogger <HttpSysListener>();
_state = State.Stopped;
_internalLock = new object();
// V2 initialization sequence:
// 1. Create server session
// 2. Create url group
// 3. Create request queue
// 4. Add urls to url group - Done in Start()
// 5. Attach request queue to url group - Done in Start()
try
{
_serverSession = new ServerSession();
_requestQueue = new RequestQueue(options.RequestQueueName, options.RequestQueueMode, Logger);
_urlGroup = new UrlGroup(_serverSession, _requestQueue, Logger);
_disconnectListener = new DisconnectListener(_requestQueue, Logger);
}
catch (Exception exception)
{
// If Url group or request queue creation failed, close server session before throwing.
_requestQueue?.Dispose();
_urlGroup?.Dispose();
_serverSession?.Dispose();
Log.HttpSysListenerCtorError(Logger, exception);
throw;
}
}
/// <summary>
/// Start accepting incoming requests.
/// </summary>
public void Start()
{
CheckDisposed();
Log.ListenerStarting(Logger);
// Make sure there are no race conditions between Start/Stop/Abort/Close/Dispose.
// Start needs to setup all resources. Abort/Stop must not interfere while Start is
// allocating those resources.
lock (_internalLock)
{
try
{
CheckDisposed();
if (_state == State.Started)
{
return;
}
// Always configure the UrlGroup if the intent was to create, only configure the queue if we actually created it
if (Options.RequestQueueMode == RequestQueueMode.Create || Options.RequestQueueMode == RequestQueueMode.CreateOrAttach)
{
Options.Apply(UrlGroup, _requestQueue.Created ? RequestQueue : null);
UrlGroup.AttachToQueue();
// All resources are set up correctly. Now add all prefixes.
try
{
Options.UrlPrefixes.RegisterAllPrefixes(UrlGroup);
}
catch (HttpSysException)
{
// If an error occurred while adding prefixes, free all resources allocated by previous steps.
UrlGroup.DetachFromQueue();
throw;
}
}
_state = State.Started;
}
catch (Exception exception)
{
// Make sure the HttpListener instance can't be used if Start() failed.
_state = State.Disposed;
DisposeInternal();
Log.ListenerStartError(Logger, exception);
throw;
}
}
}
public void HandUrl(UrlGroup urlGroup, int depath)
{
if (DoneQueue.Contains(urlGroup))
{
return;
}
string html = HttpHelper.GetString(urlGroup.Url, Encoding.Default);
lock (DoneQueueLock)
{
DoneQueue.Enqueue(urlGroup);
}
//1、获取页面所有url
List <string> listAllUrl = MatchDomainURL(html);
//2、过滤：去掉外站、js、图片等url
string[] extArray = new string[] { ".jpg", ".png", ".gif", ".js" };
List <string> mlstUrl = new List <string>();
foreach (string url in listAllUrl)
{
var tmpurl = url.ToLower();
bool isfile = false;
foreach (string ext in extArray)
{
if (tmpurl.Contains(ext))
{
isfile = true;
break;
}
}
//匹配特殊url
SepcialMatch(url);
if (!isfile && !DoneQueue.Contains(url) && !UrlQueue.Contains(url))
{
mlstUrl.Add(url);
}
}
if (depath + 1 > MaxSearchDepth)
{
return; //达到最大搜索深度
}
lock (UrlQueueLock)
{
foreach (string url in mlstUrl)
{
UrlQueue.Enqueue(new UrlGroup()
{
Url = url, DepathNumber = depath + 1
});
if (UrlQueue.Count > 10000)
{
Thread.Sleep(3000);
}
else if (UrlQueue.Count > 50000)
{
Thread.Sleep(10000);
}
}
}
}
public void Search(string url)
{
if (string.IsNullOrEmpty(url) && string.IsNullOrEmpty(BaseURL))
{
return;
}
if (!string.IsNullOrEmpty(url))
{
BaseURL = url;
}
HandUrl(new UrlGroup()
{
Url = BaseURL, DepathNumber = 0
}, 0);
Thread.Sleep(1000);
//for (int i = 0; i < MaxThreadCount; i++)
//{
ThreadPool.QueueUserWorkItem(x =>
{
try
{
while (true)
{
if (IsStop)
{
break;
}
if (IsAllDone)
{
break;
}
if (IsPause)
{
continue;
}
if (UrlQueue.Count == 0 && SpecUrlQueue.Count == 0 && UrlQueue.Count == 0)
{
break;
}
if (UrlQueue.Count == 0)
{
Thread.Sleep(10000);
}
UrlGroup urlGroup = (UrlGroup)UrlQueue.Dequeue();
HandUrl(urlGroup, urlGroup.DepathNumber);
Thread.Sleep(1000);
}
}
catch (Exception ex)
{
throw ex;
}
});
//}
ThreadPool.QueueUserWorkItem(x =>
{
try
{
while (true)
{
if (IsStop)
{
break;
}
if (IsPause)
{
continue;
}
if (IsAllDone && SpecUrlQueue.Count == 0)
{
break;
}
if (SpecUrlQueue.Count == 0)
{
Thread.Sleep(10000);
}
HandSpecialUrl(SpecUrlQueue.Dequeue().ToString());
Thread.Sleep(1000);
}
}
catch (Exception ex)
{
throw ex;
}
});
}
internal void SetUrlGroupTimeouts(UrlGroup urlGroup)
{
_urlGroup = urlGroup;
SetUrlGroupTimeouts(_timeouts, _minSendBytesPerSecond);
}
internal RequestQueue(UrlGroup urlGroup, string?requestQueueName, RequestQueueMode mode, ILogger logger)
: this(urlGroup, requestQueueName, mode, logger, false)
{
}
internal void SetUrlGroupSecurity(UrlGroup urlGroup)
{
Debug.Assert(_urlGroup == null, "SetUrlGroupSecurity called more than once.");
_urlGroup = urlGroup;
SetUrlGroupSecurity();
}
