Example #1
0
        public async Task FetchData_WhenSinceIdisMax_ThenNoDataFetched()
        {
            var tweets = JsonConvert.DeserializeObject <List <Tweet> >(File.ReadAllText(@"FakeTweets.json"));
            var max    = tweets.Select(t => long.Parse(t.Tweetid)).ToList <long>().Max().ToString();

            downloader.Setup(x => x.GetWebContent <List <Tweet>, ErrorsTwitter>(It.Is <string>(s => s == $"https://api.twitter.com/1.1/statuses/user_timeline.json?include_entities=true&count=200&include_rts=true&sinceId={max}"), It.IsAny <AuthenticationHeaderValue>()))
            .ReturnsAsync(JsonConvert.DeserializeObject <List <Tweet> >("[]"));
            ConnectorTask connectorTask = new ConnectorTask
            {
                TenantId      = "tenantId",
                JobId         = "j1",
                TaskId        = "t1",
                StartTime     = DateTime.UtcNow.AddMonths(-2),
                EndTime       = DateTime.UtcNow,
                DirtyEntities = null,
                BlobSasUri    = "dummyUri"
            };

            SourceInfoTwitter sourceInfo = new SourceInfoTwitter()
            {
                SinceId = max,
            };

            jobProcessor = new JobProcessorTwitter(downloader.Object, uploader.Object, new TwitterSchemaToItemMapper());
            var listTweets = await jobProcessor.FetchData(connectorTask, JsonConvert.SerializeObject(sourceInfo));

            Assert.IsTrue(listTweets.Count == 0);
            downloader.Verify(m => m.GetWebContent <List <Tweet>, ErrorsTwitter>(It.IsAny <string>(), It.IsAny <AuthenticationHeaderValue>()), Times.Once);
            uploader.Verify(x => x.UploadItem(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <Item>()), Times.Never);
        }
        public async Task FetchData_WhenSinceIdisZero_ThenDataFetched()
        {
            var tweets = JsonConvert.DeserializeObject <List <Tweet> >(File.ReadAllText(@"FakeTweets.json"));

            downloader.Setup(x => x.GetWebContent <List <Tweet>, ErrorsTwitter>(It.Is <string>(s => s == "https://api.twitter.com/1.1/statuses/user_timeline.json?include_entities=true&count=200&include_rts=true&tweet_mode=extended"), It.IsAny <AuthenticationHeaderValue>()))
            .ReturnsAsync(tweets);
            downloader.Setup(x => x.GetWebContent <List <Tweet>, ErrorsTwitter>(It.Is <string>(s => s != "https://api.twitter.com/1.1/statuses/user_timeline.json?include_entities=true&count=200&include_rts=true&tweet_mode=extended"), It.IsAny <AuthenticationHeaderValue>()))
            .ReturnsAsync(JsonConvert.DeserializeObject <List <Tweet> >("[]"));
            ConnectorTask connectorTask = new ConnectorTask
            {
                TenantId      = "tenantId",
                JobId         = "j1",
                TaskId        = "t1",
                StartTime     = new DateTime(2018, 12, 01),
                EndTime       = new DateTime(2019, 05, 01),
                DirtyEntities = null,
                BlobSasUri    = "dummyUri"
            };

            SourceInfoTwitter sourceInfo = new SourceInfoTwitter()
            {
                SinceId = "0",
            };

            jobProcessor = new JobProcessorTwitter(downloader.Object, uploader.Object, new TwitterSchemaToItemMapper());
            var listTweets = await jobProcessor.FetchData(connectorTask, JsonConvert.SerializeObject(sourceInfo));

            tweets.RemoveAll(t => DateTime.Compare(DateTime.ParseExact(t.CreatedAt, "ddd MMM dd HH:mm:ss +ffff yyyy", new System.Globalization.CultureInfo("en-US")), connectorTask.StartTime) < 0);
            tweets.RemoveAll(t => DateTime.Compare(DateTime.ParseExact(t.CreatedAt, "ddd MMM dd HH:mm:ss +ffff yyyy", new System.Globalization.CultureInfo("en-US")), connectorTask.EndTime) > 0);
            Assert.AreEqual(listTweets.Count, tweets.Count);
            mockRepo.VerifyAll();
        }
        public void GetUserTokenForjobIdTest()
        {
            SourceInfoTwitter sourceinfo = new SourceInfoTwitter();

            sourceinfo.ClientSecret = "CLIENT_SECRET";
            sourceinfo.ClientToken  = "CLIENT_TOKEN";
            PageJobEntity pageJobEntity = new PageJobEntity()
            {
                PartitionKey = "123",
                RowKey       = "abc",
                SourceInfo   = JsonConvert.SerializeObject(sourceinfo),
            };
            var twitterAuthProvider = new TwitterAuthProvider(restApiRepositoryMock.Object, azureTableProvider);
            var token = twitterAuthProvider.GetUserTokenForjobId(pageJobEntity);

            Assert.IsTrue(token.Contains("CLIENT_TOKEN"));
        }
        public async Task Execute(string jobMessage)
        {
            ConnectorTask   taskInfo       = JsonConvert.DeserializeObject <ConnectorTask>(jobMessage);
            IEventApiClient eventApiClient = new EventApiClient(new Auth(Settings.AAdAppId, Settings.AAdAppSecret), Settings.EventAPIBaseUrl);
            IUploader       uploader       = new BlobUploader(taskInfo.BlobSasUri);
            string          sourceInfo     = await GetSourceInfoFromTable(taskInfo);

            Trace.TraceInformation($"Fetched job info from PageJobEntity Table for JobId: {taskInfo.JobId} and TaskId: {taskInfo.TaskId}");
            Status status;
            List <ItemMetadata>       itemMetadata = new List <ItemMetadata>();
            IDownloader               downloader   = new Downloader();
            TwitterSchemaToItemMapper itemMapper   = new TwitterSchemaToItemMapper();
            JobProcessorTwitter       jobProcessor = new JobProcessorTwitter(downloader, uploader, itemMapper);

            try
            {
                itemMetadata = await jobProcessor.FetchData(taskInfo, sourceInfo);

                SourceInfoTwitter twitterSourceInfo = JsonConvert.DeserializeObject <SourceInfoTwitter>(sourceInfo);
                var listId = itemMetadata.Select(t => long.Parse(t.id)).ToList();
                twitterSourceInfo.SinceId = listId.Count == 0 ? twitterSourceInfo.SinceId : listId.Max().ToString();
                await jobProcessor.UpdateSourceInfo(taskInfo, twitterSourceInfo);

                status = Status.Success;
                Trace.TraceInformation($"Successfully completed Job Execution, JobId:{taskInfo.JobId}, TaskId:{taskInfo.TaskId}");
            }
            catch (HttpRequestException e)
            {
                status = Status.TemporaryFailure;
                Trace.TraceError($"Connectivity Error, JobId:{taskInfo.JobId}, TaskId:{taskInfo.TaskId}, Error: {e.Message}, ErrorStackTrace: {e.StackTrace}");
            }
            catch (Exception e)
            {
                status = Status.PermanentFailure;
                Trace.TraceError($"Unknown Failure, Requires Attention, JobId:{taskInfo.JobId}, TaskId:{taskInfo.TaskId}, Error: {e.Message}, ErrorStackTrace: {e.StackTrace}");
            }
            itemMetadata.OrderBy(i => i.id);
            itemMetadata.Reverse();
            await eventApiClient.OnDownloadCompleteAsync(taskInfo.TenantId, taskInfo.JobId, taskInfo.TaskId, status, itemMetadata);
        }
Example #5
0
        public async Task FetchData_WhenErrorReturned_ThenExceptionThrown()
        {
            var tweets = JsonConvert.DeserializeObject <List <Tweet> >(File.ReadAllText(@"FakeTweets.json"));
            var max    = tweets.Select(t => long.Parse(t.Tweetid)).ToList <long>().Max().ToString();
            var error  = new ErrorsTwitter()
            {
                Errors = new List <ErrorTypeTwitter>
                {
                    new ErrorTypeTwitter()
                    {
                        Code         = 400,
                        ErrorMessage = "Bad Request"
                    }
                },
            };

            downloader.Setup(x => x.GetWebContent <List <Tweet>, ErrorsTwitter>(It.IsAny <string>(), It.IsAny <AuthenticationHeaderValue>()))
            .Throws(new HttpRequestException());
            ConnectorTask connectorTask = new ConnectorTask
            {
                TenantId      = "tenantId",
                JobId         = "j1",
                TaskId        = "t1",
                StartTime     = DateTime.UtcNow.AddMonths(-2),
                EndTime       = DateTime.UtcNow,
                DirtyEntities = null,
                BlobSasUri    = "dummyUri"
            };

            SourceInfoTwitter sourceInfo = new SourceInfoTwitter()
            {
                SinceId = max,
            };

            jobProcessor = new JobProcessorTwitter(downloader.Object, uploader.Object, new TwitterSchemaToItemMapper());
            var list = await jobProcessor.FetchData(connectorTask, JsonConvert.SerializeObject(sourceInfo));
        }