public TestingFileMetadataStorage(
IOptions <TwitterMetadata> metadata,
IOptions <FileJsonStorageOptions> optionsAccessor)
{
_options = optionsAccessor.Value;
_metadata = metadata.Value;
}
private static async Task StartJob(
IJobManager jobManager,
TwitterCredentialsOptions twitterCredentialsOptions,
TwitterMetadata metadata
)
{
// var query = "snakebite;snakebites;\"morsure de serpent\";\"morsures de serpents\";\"لدغات الأفاعي\";\"لدغة الأفعى\";\"لدغات أفاعي\";\"لدغة أفعى\"";
// TODO add NOT cocktail NOT music
// var query = "snake bite NOT cocktail NOT darts NOT piercing";
var jobId = Guid.Parse("a43e8bb4-9c15-48a8-a0a3-7479b75eb6d0");
var jobConfig = new DataAcquirerJobConfig()
{
Attributes = new Dictionary <string, string>
{
{ "TopicQuery", metadata.Query },
{ "AccessToken", twitterCredentialsOptions.AccessToken },
{ "AccessTokenSecret", twitterCredentialsOptions.AccessTokenSecret },
{ "ApiKey", twitterCredentialsOptions.ApiKey },
{ "ApiSecretKey", twitterCredentialsOptions.ApiSecretKey },
},
JobId = jobId,
OutputMessageBrokerChannels = new string[] { "job_management.component_data_input.DataAnalyser_sentiment" }
};
try
{
await jobManager.StartNewJobAsync(jobConfig);
}
catch
{
}
}
private IEnumerable <string> AssertObject(TwitterMetadata a, TwitterMetadata b)
{
yield return(AssertProperty("BatchSize", r => r.BatchSize, a, b));
yield return(AssertProperty("Language", r => r.Language, a, b));
yield return(AssertProperty("MaxId", r => r.MaxId, a, b));
yield return(AssertProperty("Query", r => r.Query, a, b));
yield return(AssertProperty("SinceId", r => r.SinceId, a, b));
}
public async Task TestAsync(IDataAcquirerMetadataStorage storage)
{
var twitterMetadata = new TwitterMetadata()
{
BatchSize = 1,
Language = "en",
MaxId = 111,
Query = "q1",
SinceId = 2
};
var jobId = Guid.NewGuid();
await storage.SaveAsync(jobId, twitterMetadata);
var retreived = await storage
.GetAsync <TwitterMetadata>(jobId);
var errors = AssertObject(twitterMetadata, retreived);
AssertErrors("insert metadata", errors);
}
public async Task TestStorageAsync()
{
var limit = (ulong)301;
var maxTweetId = (ulong)300;
var testTwitterContext = new TestTwitterContext(limit, maxTweetId);
var starting = (ulong)100;
var batchSize = 3;
var query = "foo";
string language = null;
var jobId = Guid.Parse("d2474631-3d4c-4b07-a992-9d9c1f269cd4");
var metadata = new TwitterMetadata
{
BatchSize = 3,
MaxId = starting,
SinceId = 0,
Query = "foo",
Language = null
};
var metadataContextMock = new Mock <IDataAcquirerMetadataContext>();
metadataContextMock
.Setup(r => r.GetOrCreateAsync(It.IsAny <TwitterMetadata>()))
.Returns(Task.FromResult(metadata));
var metaDataContextProviderMock = new Mock <IDataAcquirerMetadataContextProvider>();
metaDataContextProviderMock.Setup(r => r.Get(It.IsAny <Guid>()))
.Returns(metadataContextMock.Object);
var batchLoaderOptions = Options.Create(new TwitterBatchLoaderOptions
{
NoPostWaitDelay = TimeSpan.Zero,
ErrorEncounteredWaitDelay = TimeSpan.Zero,
RateLimitExceededWaitDelay = TimeSpan.Zero
});
var batchLoaderFactory = new TwitterBatchLoaderFactory(
batchLoaderOptions,
new Mock <IEventTracker <TwitterBatchLoader> >().Object,
metaDataContextProviderMock.Object);
var contextProviderMock = new Mock <ITwitterContextProvider>();
contextProviderMock
.Setup(r => r.GetContextAsync(It.IsAny <TwitterCredentials>()))
.Returns(Task.FromResult <ITwitterContext>(testTwitterContext));
var trackerMocq = new Mock <IEventTracker <TwitterDataAcquirer> >();
var twitterDataAcquirer = new TwitterDataAcquirer(
batchLoaderFactory,
contextProviderMock.Object,
trackerMocq.Object);
var input = new DataAcquirerInputModel(
jobId,
query,
language,
new DataAcquirerAttributes(new Dictionary <string, string>()),
batchSize);
var posts = twitterDataAcquirer.GetPostsAsync(input);
var ids = new List <ulong>();
await foreach (var post in posts)
{
if ((ulong)ids.Count >= maxTweetId)
{
break;
}
var idStr = post.OriginalPostId;
var id = ulong.Parse(idStr);
ids.Add(id);
}
var actual = ids.OrderBy(r => r).ToList();
var expected = Enumerable.Range(1, (int)maxTweetId).Select(r => (ulong)r).ToList();
CollectionAssert.AreEqual(
expected,
actual);
}
