public void ExecutesQueriesViaHttpClient()
 {
     var httpClient = new Mock<IHttpClient>(MockBehavior.Strict);
     var client = new WikimediaClient(httpClient.Object, "Foo", 5);
     var task = client.Query("foo=bar");
     httpClient.Verify(s => s.Get(client.BaseQueryUrl + "&foo=bar", "Foo"));
 }
Ejemplo n.º 2
0
        static async Task MainAsync(IMongoDatabase db)
        {
            var client = new WikimediaClient(USER_CLIENT, MAX_LAG);
            client.LaggedResponse += (sender, response) =>
                Console.WriteLine($"Response was lagged {response.Error.SecondsLagged} seconds. Waiting to issue next query.");

            var articles = db.GetCollection<BsonDocument>("Articles");
            await articles.Indexes.CreateOneAsync("{PageId: 1}");
            var start = await articles.CountAsync("{}");
            Console.WriteLine($"Starting download. {start.ToString("#,###,##0")} articles currently into the database. Press Ctrl+C to interrupt.");

            // Keep the same number of download tasks running at the same time
            int MAX_TASKS = 50;
            var queue = new List<Task>(MAX_TASKS);
            for(int i = 0; i < MAX_TASKS; i++) {
                queue.Add(InsertRandomDocumentsIntoDatabase(client, articles));
            }
            int completedTasks = 0;
            var sw = new Stopwatch();
            sw.Start();
            while (true) {
                // keep replacing finished tasks with new ones until the process is interrupted by the user.
                var finishedTask = await Task.WhenAny(queue);
                queue.Remove(finishedTask);
                queue.Add(InsertRandomDocumentsIntoDatabase(client, articles));
                completedTasks++;
                if(completedTasks % 100 == 0) {
                    var total = await articles.CountAsync("{}");
                    var soFar = total - start;
                    var totalSeconds = sw.Elapsed.TotalSeconds;
                    var speed = soFar / (double)totalSeconds;
                    Console.WriteLine($"Total of {total.ToString("#,###,##0")} articles inserted into the database ({speed.ToString("0.00")} articles/sec). Press Ctrl+C to interrupt.");
                }
            }
        }
Ejemplo n.º 3
0
 static async Task InsertRandomDocumentsIntoDatabase(WikimediaClient client, IMongoCollection<BsonDocument> collection)
 {
     var response = await client.GetMaxRandomArticles();
     var updates = response.Pages.Values.Select(page => {
         var doc = DocumentFromPage(page);
         return collection.ReplaceOneAsync(
             Builders<BsonDocument>.Filter.Eq(d => d["PageId"], page.PageId), doc,
             new UpdateOptions { IsUpsert = true });
     });
     await Task.WhenAll(updates);
 }
 public async Task GetsRandomArticles()
 {
     var client = new WikimediaClient(USER_CLIENT, 5);
     var response = await client.GetMaxRandomArticles();
     Assert.IsNotEmpty(response.Pages);
 }
 public async Task ReturnsQueryResponsesWithPages()
 {
     var httpClient = new Mock<IHttpClient>(MockBehavior.Strict);
     var client = new WikimediaClient(httpClient.Object, "Foo", 5);
     httpClient.Setup(c => c.Get(client.BaseQueryUrl + "&foo=bar", "Foo")).ReturnsAsync("{'query':{'pages':{}}}");
     var response = await client.Query("foo=bar");
     Assert.NotNull(response.Pages);
 }
 public void HasBaseQueryUrl()
 {
     var httpClient = Mock.Of<IHttpClient>();
     var client = new WikimediaClient(httpClient, "foo", 52);
     Assert.AreEqual("https://en.wikipedia.org/w/api.php?format=json&maxlag=52&action=query&prop=extracts|info&inprop=url&exlimit=max&exintro=1", client.BaseQueryUrl);
 }
 public void UsesDefaultClientWhenNotSupplied()
 {
     var client = new WikimediaClient("foo", 2);
     Assert.AreSame(HttpClient.Default, client.HttpClient);
 }
 public void HasMaxLag()
 {
     var httpClient = Mock.Of<IHttpClient>();
     var client = new WikimediaClient(httpClient, "foo", 5);
     Assert.AreEqual(5, client.MaxLag);
 }
 public void HasBaseUrl()
 {
     var httpClient = Mock.Of<IHttpClient>();
     var client = new WikimediaClient(httpClient, "foo", 25);
     Assert.AreEqual("https://en.wikipedia.org/w/api.php?format=json&maxlag=25", client.BaseUrl);
 }
 public void HasUserAgent()
 {
     var httpClient = Mock.Of<IHttpClient>();
     var client = new WikimediaClient(httpClient, "foo", 5);
     Assert.AreEqual("foo", client.UserAgent);
 }
 public void HasHttpClient()
 {
     var httpClient = Mock.Of<IHttpClient>();
     var client = new WikimediaClient(httpClient, "foo", 5);
     Assert.AreSame(httpClient, client.HttpClient);
 }
 public async Task RetriesHttpQueryWhileHasMaxLagError()
 {
     var httpClient = new Mock<IHttpClient>(MockBehavior.Strict);
     var client = new WikimediaClient(httpClient.Object, "FooAgent", 5, minimumDelaySeconds: 0); // To avoid delays during tests
     var errorResponse = "{'error':{'code':'maxlag','info':'0 seconds lagged'}}";
     var successResponse = "{'query':{}}";
     // Sets up mock responses in sequence to simulate a maxlag errors followed by a success response the third time around
     httpClient
         .SetupSequence(c => c.Get(client.BaseQueryUrl + "&foo=bar", "FooAgent"))
         .Returns(Task.FromResult(errorResponse))
         .Returns(Task.FromResult(errorResponse))
         .Returns(Task.FromResult(successResponse));
     var response = await client.Query("foo=bar");
     Assert.NotNull(response);
 }
 public async Task QueriesMaxNumberOfRandomArticles()
 {
     var httpClient = new Mock<IHttpClient>(MockBehavior.Strict);
     var client = new WikimediaClient(httpClient.Object, "Foo", 5);
     var expectedUrl = client.BaseQueryUrl + "&generator=random&grnnamespace=0&grnlimit=max";
     httpClient.Setup(c => c.Get(expectedUrl, "Foo")).ReturnsAsync("{'query':{}}");
     var response = await client.GetMaxRandomArticles();
     Assert.NotNull(response);
 }
 public async Task ReturnsQueryResponsesWithPageData()
 {
     var httpClient = new Mock<IHttpClient>(MockBehavior.Strict);
     var client = new WikimediaClient(httpClient.Object, "Foo", 5);
     httpClient.Setup(c => c.Get(client.BaseQueryUrl + "&foo=bar", "Foo")).ReturnsAsync("{'query':{'pages':{'a':{'title':'Hello world'}}}}");
     var response = await client.Query("foo=bar");
     Assert.AreEqual("Hello world", response.Pages["a"].Title);
 }