static void Main(string[] args) { ///this test will try to download a file faster by getting the file size from the server ///and splitting the file into a series of ranges that are downloaded simultaneously /// //start test HttpGetter getter = new HttpGetter(); Stream resultsStream = getter.GetFast("http://tatts.com/pagedata/racing/racing.xml"); Console.WriteLine("Writing to testing file.."); //save result to file for testing FileStream testingFile = new FileStream("testfile", FileMode.Create); resultsStream.CopyTo(testingFile); testingFile.Close(); Console.WriteLine("Done."); Console.WriteLine("Parallel test.."); List<string> urlList = new List<string>(); urlList.Add("http://tatts.com/pagedata/racing/racing.xml"); urlList.Add("http://tatts.com/pagedata/sports/sports.xml"); urlList.Add("http://tatts.com/pagedata/racing/2011/10/5/BR.xml"); Dictionary<string, Stream> paraResults = MultiFileHttpGetter.GetMultipleFiles(urlList, 1024); Console.WriteLine("Done."); }
public static Dictionary<string, Stream> GetMultipleFiles(List<string> urlList, int chunkSizeInBytes = 51200) { Dictionary<string, Stream> paraResults = new Dictionary<string, Stream>(); Parallel.ForEach(urlList, url => { HttpGetter paraGetter = new HttpGetter(); Stream paraResult = paraGetter.GetFast(url); lock (paraResults) { paraResults.Add(url, paraResult); } }); return paraResults; }