public void TestRegexResolve() { Regex urlRegex = new Regex(@"(\{{0,1}([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}\}{0,1})"); var resolver = new RegexPathResolver(urlRegex, "thread_id_{1}"); var path = resolver.Resolve(new Uri("https://social.msdn.microsoft.com/Forums/windowsapps/en-US/2f2dd185-e7f7-4dcd-aaf9-f7fc09c8716d/uwpc-how-to-see-what-a-string-contains-with-certain-length?forum=wpdevelop")); }
public void TestIsCached() { var provider = DefaultLocalFileSystemHttpCacheProvider.Current; Regex urlRegex = new Regex(@"(\{{0,1}([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}\}{0,1})"); var resolver = new RegexPathResolver(urlRegex, "thread_id_{1}"); provider.Configure(@"D:\temp\cache", resolver); var isCached = provider.IsCached(new Uri("https://social.msdn.microsoft.com/Forums/windowsapps/en-US/2f2dd185-e7f7-4dcd-aaf9-f7fc09c8716d/uwpc-how-to-see-what-a-string-contains-with-certain-length?forum=wpdevelop")); Assert.IsTrue(isCached); }
public async Task RunAsync(WebCacheTask task) { var httpClient = new HttpClient(); Utility.SetGeneralHttpHeaders(httpClient); var frame = new IndexPageNavigation(task.Pagination.NavigationUriFormat); var encoding = Encoding.GetEncoding(task.Encoding); var cacheProvider = DefaultLocalFileSystemHttpCacheProvider.Current; Regex urlRegex = new Regex(task.Cache.UriToPathTransform.Pattern); var resolver = new RegexPathResolver(urlRegex, task.Cache.UriToPathTransform.TargetFormat); cacheProvider.Configure(task.Cache.RootFolder, resolver); var discovery = new XPathUriDiscovery(task.Pagination.Lookup.XPath, task.Pagination.Lookup.Attribute, string.IsNullOrEmpty(task.Pagination.UriFilter) ? null : new Regex(task.Pagination.UriFilter), task.Pagination.BasicUri ); for (int i = task.Pagination.StartPage; i < task.Pagination.StartPage + task.Pagination.PageLength; i++) { frame.NavigateTo(i); string text = string.Empty; using (var content = await frame.GetAsync()) using (var sr = new StreamReader(content, encoding)) { text = await sr.ReadToEndAsync(); } var uris = discovery.Discover(text); uris = TransformIfNeeded(task, uris); await CacheAll(httpClient, cacheProvider, uris); } }
public void TestCache() { var provider = DefaultLocalFileSystemHttpCacheProvider.Current; Regex urlRegex = new Regex(@"(\{{0,1}([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}\}{0,1})"); var resolver = new RegexPathResolver(urlRegex, "thread_id_{1}"); provider.Configure(@"D:\temp\cache", resolver); using (var mem = new MemoryStream()) using (var sw = new StreamWriter(mem)) { sw.WriteLine("HELLO Cache"); sw.Flush(); mem.Position = 0; provider.Cache(mem, new Uri("https://social.msdn.microsoft.com/Forums/windowsapps/en-US/2f2dd185-e7f7-4dcd-aaf9-f7fc09c8716d/uwpc-how-to-see-what-a-string-contains-with-certain-length?forum=wpdevelop")); } }
public async Task RunAsync(WebCacheServiceTask task) { var frame = new IndexPageNavigation(task.PageNavigationUriFormat); var encoding = Encoding.GetEncoding(task.Encoding); var cacheProvider = DefaultLocalFileSystemHttpCacheProvider.Current; Regex urlRegex = new Regex(task.CacheProviderUriPattern); var resolver = new RegexPathResolver(urlRegex, task.CacheProviderPathFormat); cacheProvider.Configure(task.CacheProviderRootFolder, resolver); var discovery = new XPathUriDiscovery(task.HtmlNodeXPath, task.HtmlNodeAttribute, string.IsNullOrEmpty(task.UriFilterPattern) ? null : new Regex(task.UriFilterPattern)); for (int i = task.StartPage; i < task.StartPage + task.PageLength; i++) { frame.NavigateTo(i); string text = string.Empty; using (var content = await frame.GetAsync()) using (var sr = new StreamReader(content, encoding)) { text = await sr.ReadToEndAsync(); } var uris = discovery.Discover(text); foreach(var uri in uris) { Console.WriteLine(uri); } } }
private static void FroumCacheDemo() { var provider = DefaultLocalFileSystemHttpCacheProvider.Current; Regex urlRegex = new Regex(@"(\{{0,1}([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}\}{0,1})"); var resolver = new RegexPathResolver(urlRegex, "thread_id_{1}.xml"); provider.Configure(@"D:\httpcache", resolver); var collection = new ThreadCollection(Community.MSDN, "wpdevelop"); var task = collection.NavigateToPageAsync(1); task.Wait(); var threads = task.Result; var client = new HttpClient(); foreach (var thread in threads) { var urlString = thread + "&outputAs=xml"; var uri = new Uri(urlString); if (provider.IsCached(uri)) { continue; } var httpTask = client.GetStreamAsync(uri); httpTask.Wait(); provider.Cache(httpTask.Result, uri); } }