public static Load Create(int taskCount, int itemCount, int pages) { var channel = Channel.CreateBounded <DateBind>(itemCount); Func <Uri> getnexturi = () => CreteNextUri(Interlocked.Increment(ref pages)); MHttpClient htmlLoad = new MHttpClient(new MHttpClientHandler { StreamCallback = MHttpClientHandler.CreateNewConnectAsync( MHttpClientHandler.CreateCreateConnectAsyncFunc(DNS_HOST, 443), MHttpClientHandler.CreateCreateAuthenticateAsyncFunc(SNI_HOST, false) ) }); var load = new Load(); foreach (var item in Enumerable.Range(0, taskCount)) { Task.Run(() => load.LoadData(channel, htmlLoad, getnexturi)); } return(new Load() { Reader = channel, PageFunc = () => pages, Cannel = () => channel.Writer.TryComplete() }); }
static void GetHttpClient(WebSite webSite, int maxSize, int poolCount, out Uri host, out MHttpClient htmlClient, out MHttpClient imgClient) { var v = JsonSerializer.Deserialize <WebInfo[]>(InputData.WebInfo2); var wv = v.Where((item) => item.Key == webSite.ToString()).First(); var imgHandler = new MHttpClientHandler { MaxResponseContentSize = 1024 * 1024 * maxSize, MaxStreamPoolCount = poolCount, MaxStreamParallelRequestCount = 4, MaxStreamRequestCount = 30, }; SetImgWebInfo(wv, imgHandler); imgClient = new MHttpClient(imgHandler); var htmlHandle = new MHttpClientHandler(); SetWebInfo(wv, htmlHandle, out host); htmlClient = new MHttpClient(htmlHandle); }
async Task LoadData(ChannelWriter <DateBind> writer, MHttpClient load, Func <Uri> func) { async Task AddDate(string html) { foreach (var item in CreateHtmlPageData(html)) { try { byte[] buffer = await load.GetByteArrayAsync(item.ImgUri, item.Uri, default).ConfigureAwait(false); await writer.WriteAsync(new DateBind( ImageSource.FromStream(() => new MemoryStream(buffer)), item.Title, item.Uri)).ConfigureAwait(false); } catch (MHttpClientException) { } } } while (true) { try { string html = await load.GetStringAsync(func(), default).ConfigureAwait(false); await AddDate(html).ConfigureAwait(false); } catch (MHttpClientException) { } catch (ChannelClosedException) { return; } } }