public TestingFileMetadataStorage( IOptions <TwitterMetadata> metadata, IOptions <FileJsonStorageOptions> optionsAccessor) { _options = optionsAccessor.Value; _metadata = metadata.Value; }
private static async Task StartJob( IJobManager jobManager, TwitterCredentialsOptions twitterCredentialsOptions, TwitterMetadata metadata ) { // var query = "snakebite;snakebites;\"morsure de serpent\";\"morsures de serpents\";\"لدغات الأفاعي\";\"لدغة الأفعى\";\"لدغات أفاعي\";\"لدغة أفعى\""; // TODO add NOT cocktail NOT music // var query = "snake bite NOT cocktail NOT darts NOT piercing"; var jobId = Guid.Parse("a43e8bb4-9c15-48a8-a0a3-7479b75eb6d0"); var jobConfig = new DataAcquirerJobConfig() { Attributes = new Dictionary <string, string> { { "TopicQuery", metadata.Query }, { "AccessToken", twitterCredentialsOptions.AccessToken }, { "AccessTokenSecret", twitterCredentialsOptions.AccessTokenSecret }, { "ApiKey", twitterCredentialsOptions.ApiKey }, { "ApiSecretKey", twitterCredentialsOptions.ApiSecretKey }, }, JobId = jobId, OutputMessageBrokerChannels = new string[] { "job_management.component_data_input.DataAnalyser_sentiment" } }; try { await jobManager.StartNewJobAsync(jobConfig); } catch { } }
private IEnumerable <string> AssertObject(TwitterMetadata a, TwitterMetadata b) { yield return(AssertProperty("BatchSize", r => r.BatchSize, a, b)); yield return(AssertProperty("Language", r => r.Language, a, b)); yield return(AssertProperty("MaxId", r => r.MaxId, a, b)); yield return(AssertProperty("Query", r => r.Query, a, b)); yield return(AssertProperty("SinceId", r => r.SinceId, a, b)); }
public async Task TestAsync(IDataAcquirerMetadataStorage storage) { var twitterMetadata = new TwitterMetadata() { BatchSize = 1, Language = "en", MaxId = 111, Query = "q1", SinceId = 2 }; var jobId = Guid.NewGuid(); await storage.SaveAsync(jobId, twitterMetadata); var retreived = await storage .GetAsync <TwitterMetadata>(jobId); var errors = AssertObject(twitterMetadata, retreived); AssertErrors("insert metadata", errors); }
public async Task TestStorageAsync() { var limit = (ulong)301; var maxTweetId = (ulong)300; var testTwitterContext = new TestTwitterContext(limit, maxTweetId); var starting = (ulong)100; var batchSize = 3; var query = "foo"; string language = null; var jobId = Guid.Parse("d2474631-3d4c-4b07-a992-9d9c1f269cd4"); var metadata = new TwitterMetadata { BatchSize = 3, MaxId = starting, SinceId = 0, Query = "foo", Language = null }; var metadataContextMock = new Mock <IDataAcquirerMetadataContext>(); metadataContextMock .Setup(r => r.GetOrCreateAsync(It.IsAny <TwitterMetadata>())) .Returns(Task.FromResult(metadata)); var metaDataContextProviderMock = new Mock <IDataAcquirerMetadataContextProvider>(); metaDataContextProviderMock.Setup(r => r.Get(It.IsAny <Guid>())) .Returns(metadataContextMock.Object); var batchLoaderOptions = Options.Create(new TwitterBatchLoaderOptions { NoPostWaitDelay = TimeSpan.Zero, ErrorEncounteredWaitDelay = TimeSpan.Zero, RateLimitExceededWaitDelay = TimeSpan.Zero }); var batchLoaderFactory = new TwitterBatchLoaderFactory( batchLoaderOptions, new Mock <IEventTracker <TwitterBatchLoader> >().Object, metaDataContextProviderMock.Object); var contextProviderMock = new Mock <ITwitterContextProvider>(); contextProviderMock .Setup(r => r.GetContextAsync(It.IsAny <TwitterCredentials>())) .Returns(Task.FromResult <ITwitterContext>(testTwitterContext)); var trackerMocq = new Mock <IEventTracker <TwitterDataAcquirer> >(); var twitterDataAcquirer = new TwitterDataAcquirer( batchLoaderFactory, contextProviderMock.Object, trackerMocq.Object); var input = new DataAcquirerInputModel( jobId, query, language, new DataAcquirerAttributes(new Dictionary <string, string>()), batchSize); var posts = twitterDataAcquirer.GetPostsAsync(input); var ids = new List <ulong>(); await foreach (var post in posts) { if ((ulong)ids.Count >= maxTweetId) { break; } var idStr = post.OriginalPostId; var id = ulong.Parse(idStr); ids.Add(id); } var actual = ids.OrderBy(r => r).ToList(); var expected = Enumerable.Range(1, (int)maxTweetId).Select(r => (ulong)r).ToList(); CollectionAssert.AreEqual( expected, actual); }