/// <summary> /// Download a robots file /// </summary> /// <param name="robotsUri"></param> /// <param name="lastFetched"></param> /// <returns></returns> public static async Task <RobotsFile> DownloadRobots(Uri anyUri, DateTime?lastFetched) { RobotsFile robots = null; var robotsUri = MakeRobotsUri(anyUri); try { var ip = await Utility.GetHostIPAddress(robotsUri); /*while (!await FetchoConfiguration.Current.HostCache.WaitToFetch(ip, 60000)) * Utility.LogInfo("IP Congestion {0}", ip);*/ var bb = new BufferBlock <IWebResourceWriter>(); using (var ms = new MemoryStream()) { using (var packet = new WebDataPacketWriter(ms)) { // this is annoying, I shouldn't have to create a buffer block to get a robots file // or we should put robots into the standard flow of things await bb.SendAsync(packet); await(new HttpResourceFetcher()).Fetch(null, robotsUri, null, lastFetched, bb); } ms.Seek(0, SeekOrigin.Begin); using (var packet = new WebDataPacketReader(CreateXmlReader(ms))) { using (var stream = packet.GetResponseStream()) { if (stream == null) { robots = new RobotsFile(); } else { robots = new RobotsFile(robotsUri, stream); } } } } } catch (Exception ex) { Utility.LogInfo("Fetching {0}:", robotsUri); Utility.LogException(ex); } return(robots); }
public WorkspaceResourcePacketWriter(WebDataPacketWriter writer) { Writer = writer; }