public async void NormalGeneratorTest()
{
var finished = false;
async Task Generator(IAsyncEnumerableSink <int> sink)
{
// Yield some items.
await sink.YieldAndWait(10);
await sink.YieldAndWait(20);
// Do some work.
await Task.Delay(100);
// Yield some more items.
await sink.YieldAndWait(new[] { 30, 40 });
// Ditto.
await Task.Delay(100);
await sink.YieldAndWait(50);
finished = true;
}
var array = await AsyncEnumerableFactory.FromAsyncGenerator <int>(Generator).ToArrayAsync();
Assert.True(finished);
Assert.Equal(new[] { 10, 20, 30, 40, 50 }, array);
}
/// <summary>
/// Construct a sequence of <see cref="WikiPageStub"/> from the given page IDs.
/// </summary>
/// <param name="site">The site in which to query for the pages.</param>
/// <param name="ids">The page IDs to query.</param>
/// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="ids"/> is <c>null</c>.</exception>
/// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns>
/// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks>
public static IAsyncEnumerable <WikiPageStub> FromPageIds(WikiSite site, IEnumerable <int> ids)
{
return(AsyncEnumerableFactory.FromAsyncGenerator <WikiPageStub>(async(sink, ct) =>
{
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
? 500
: 50;
foreach (var partition in ids.Partition(titleLimit))
{
var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "query",
pageids = MediaWikiHelper.JoinValues(partition),
}), ct);
Debug.Assert(jresult["query"] != null);
var jpages = jresult["query"]["pages"];
foreach (var id in partition)
{
var jpage = jpages[id.ToString(CultureInfo.InvariantCulture)];
if (jpage["missing"] == null)
{
sink.Yield(new WikiPageStub(id, (string)jpage["title"], (int)jpage["ns"]));
}
else
{
sink.Yield(new WikiPageStub(id, MissingPageTitle, UnknownNamespaceId));
}
}
await sink.Wait();
}
}));
}
public void EmptyGeneratorTest()
{
async Task Generator(IAsyncEnumerableSink <int> sink)
{
await Task.Delay(100);
}
Assert.Empty(AsyncEnumerableFactory.FromAsyncGenerator <int>(Generator).ToEnumerable());
}
public static IAsyncEnumerable <TResult> SelectAsync <TSource, TResult>(this IEnumerable <TSource> source,
Func <TSource, Task <TResult> > selector)
{
return(AsyncEnumerableFactory.FromAsyncGenerator <TResult>(async sink =>
{
foreach (var item in source)
{
await sink.YieldAndWait(await selector(item));
}
}));
}
/// <summary>
/// Asynchronously enumerates the topics in this board.
/// </summary>
/// <param name="options">Enumeration options.</param>
/// <param name="pageSize">
/// How many topics should be fetched in batch per MediaWiki API request.
/// No more than 100 (100 for bots) is allowed.
/// </param>
public IAsyncEnumerable <Topic> EnumTopicsAsync(TopicListingOptions options, int pageSize)
{
if (pageSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(pageSize));
}
var sortParam = "user";
if ((options & TopicListingOptions.OrderByPosted) == TopicListingOptions.OrderByPosted)
{
sortParam = "newest";
}
if ((options & TopicListingOptions.OrderByUpdated) == TopicListingOptions.OrderByUpdated)
{
sortParam = "updated";
}
return(AsyncEnumerableFactory.FromAsyncGenerator <Topic>(async(sink, ct) =>
{
var queryParams = new Dictionary <string, object>
{
{ "action", "flow" },
{ "submodule", "view-topiclist" },
{ "page", Title },
{ "vtlsortby", sortParam },
{ "vtlsavesortby", (options & TopicListingOptions.SaveSortingPreference) == TopicListingOptions.SaveSortingPreference },
{ "vtllimit", pageSize },
{ "vtlformat", "wikitext" },
};
NEXT_PAGE:
var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), ct);
var jtopiclist = (JObject)jresult["flow"]["view-topiclist"]["result"]["topiclist"];
await sink.YieldAndWait(Topic.FromJsonTopicList(Site, jtopiclist));
// 2018-07-30 flow.view-topiclist.result.topiclist.links.pagination is [] instead of null for boards without pagination.
var jpagination = jtopiclist["links"]?["pagination"];
var nextPageUrl = jpagination == null || jpagination is JArray
? null
: (string)jpagination["fwd"]?["url"];
if (nextPageUrl != null)
{
var urlParams = FlowUtility.ParseUrlQueryParametrs(nextPageUrl);
foreach (var pa in urlParams)
{
if (pa.Key.StartsWith("topiclist_"))
{
queryParams["vtl" + pa.Key.Substring(10)] = pa.Value;
}
}
goto NEXT_PAGE;
}
}));
}
public async void GeneratorExceptionTest()
{
async Task Generator(IAsyncEnumerableSink <int> sink)
{
await Task.Delay(100);
await sink.YieldAndWait(10);
await sink.YieldAndWait(20);
throw new InvalidDataException();
}
Assert.Equal(new[] { 10, 20 }, await AsyncEnumerableFactory.FromAsyncGenerator <int>(Generator).Take(2).ToArrayAsync());
await Assert.ThrowsAsync <InvalidDataException>(async() =>
await AsyncEnumerableFactory.FromAsyncGenerator <int>(Generator).ToArrayAsync());
}
public async void CancellationTest()
{
async Task Generator(IAsyncEnumerableSink <int> sink, CancellationToken token)
{
int value = 1;
NEXT:
value *= 2;
await sink.YieldAndWait(value);
await Task.Delay(100, token);
goto NEXT;
}
var array = await AsyncEnumerableFactory.FromAsyncGenerator <int>(Generator).Take(5).ToArrayAsync();
Assert.Equal(new[] { 2, 4, 8, 16, 32 }, array);
}
/// <summary>
/// Asynchronously fetches the specified users' information.
/// </summary>
/// <param name="site">The site to issue the request.</param>
/// <param name="userNames">The user names to be fetched.</param>
/// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="userNames"/> is <c>null</c>.</exception>
/// <returns>
/// An asynchronous sequence containing the detailed user information.
/// The user names are normalized by the server. Inexistent user names are skipped.
/// </returns>
public static IAsyncEnumerable <UserInfo> FetchUsersAsync(this WikiaSite site, IEnumerable <string> userNames)
{
if (site == null)
{
throw new ArgumentNullException(nameof(site));
}
if (userNames == null)
{
throw new ArgumentNullException(nameof(userNames));
}
return(AsyncEnumerableFactory.FromAsyncGenerator <UserInfo>(async(sink, ct) =>
{
using (site.BeginActionScope(null, (object)(userNames as ICollection)))
{
foreach (var names in userNames.Partition(100))
{
JToken jresult;
try
{
jresult = await site.InvokeWikiaApiAsync("/User/Details",
new WikiaQueryRequestMessage(new { ids = string.Join(", ", names) }), ct);
}
catch (WikiaApiException ex) when(ex.ErrorType == "NotFoundApiException")
{
// All the usesers in this batch are not found.
// Pity.
continue;
}
var basePath = (string)jresult["basepath"];
var users = jresult["items"].ToObject <ICollection <UserInfo> >();
if (basePath != null)
{
foreach (var user in users)
{
user.ApplyBasePath(basePath);
}
}
await sink.YieldAndWait(users);
}
}
}));
}
/// <inheritdoc />
public IAsyncEnumerable <LocalWikiSearchResultItem> EnumItemsAsync()
{
return(AsyncEnumerableFactory.FromAsyncGenerator <LocalWikiSearchResultItem>(async(sink, ct) =>
{
int totalBatches = 1;
for (int currentBatch = 1; currentBatch <= totalBatches; currentBatch++)
{
var jresult = await Site.InvokeWikiaApiAsync("/Search/List", new WikiaQueryRequestMessage(new
{
query = Keyword, type = "articles", rank = SerializeRank(RankingType), limit = PaginationSize,
minArticleQuality = MinimumArticleQuality, namespaces = NamespaceIds == null ? null : string.Join(",", NamespaceIds),
batch = currentBatch,
}), ct);
totalBatches = (int)jresult["batches"];
var items = jresult["items"].ToObject <IEnumerable <LocalWikiSearchResultItem> >(
Utility.WikiaApiJsonSerializer);
await sink.YieldAndWait(items);
}
}));
}
public void Should_execute_stored_procedure_asynchronously_and_return_objects()
{
var command = _commandFactory.CreateStoredProcedure("proc1");
var asyncResult = _context.BeginExecuteEnumerable(command);
IAsyncEnumerableFactory asyncEnumerableFactory = new AsyncEnumerableFactory();
using (var asyncEnumerable = asyncEnumerableFactory.Create <TestDataContract>(_context, command, asyncResult))
{
asyncEnumerable.AsyncWaitHandle.WaitOne();
using (var dataContracts = asyncEnumerable.GetResults())
{
var dataContractList = dataContracts.ToList();
Assert.AreEqual(2, dataContractList.Count);
Assert.AreEqual(10, dataContractList[0].Id);
Assert.AreEqual("name1", dataContractList[0].Name);
Assert.AreEqual(20, dataContractList[1].Id);
Assert.AreEqual("name2", dataContractList[1].Name);
}
}
}
/// <inheritdoc />
/// <exception cref="OperationFailedException">
/// (When enumerating) There is any MediaWiki API failure during the operation.
/// </exception>
/// <exception cref="Exception">
/// (When enumerating) There can be other types of errors thrown.
/// See the respective <see cref="OnEnumItemsFailed"/> override documentations in the implementation classes.
/// </exception>
public IAsyncEnumerable <T> EnumItemsAsync()
{
return(AsyncEnumerableFactory.FromAsyncGenerator <T>(async(sink, ct) =>
{
var baseQueryParams = new Dictionary <string, object>
{
{ "action", "query" },
{ "maxlag", 5 },
{ "list", ListName },
};
foreach (var p in EnumListParameters())
{
baseQueryParams.Add(p.Key, p.Value);
}
ct.ThrowIfCancellationRequested();
var continuationParams = new Dictionary <string, object>();
using (Site.BeginActionScope(this))
{
// query parameters for this batch. The content/ref will be modified below.
var queryParams = new Dictionary <string, object>();
while (true)
{
queryParams.Clear();
queryParams.MergeFrom(baseQueryParams);
queryParams.MergeFrom(continuationParams);
try
{
var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), ct);
var listNode = RequestHelper.FindQueryResponseItemsRoot(jresult, ListName);
if (listNode != null)
{
await sink.YieldAndWait(listNode.Select(ItemFromJson));
}
// Check for continuation.
switch (RequestHelper.ParseContinuationParameters(jresult, queryParams, continuationParams))
{
case RequestHelper.CONTINUATION_DONE:
return;
case RequestHelper.CONTINUATION_AVAILABLE:
if (listNode == null)
{
Site.Logger.LogWarning("Empty query page with continuation received.");
}
break;
case RequestHelper.CONTINUATION_LOOP:
Site.Logger.LogWarning("Continuation information provided by server response leads to infinite loop. {RawData}",
RequestHelper.FindQueryContinuationParameterRoot(jresult));
// The following is just last effort.
var outOfLoop = false;
if (CompatibilityOptions != null)
{
if ((CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.FetchMore) ==
WikiListContinuationLoopBehaviors.FetchMore)
{
// xxlimit (length = 7)
var limitParamName =
queryParams.Keys.FirstOrDefault(k => k.Length == 7 && k.EndsWith("limit", StringComparison.Ordinal));
if (limitParamName == null)
{
Site.Logger.LogWarning("Failed to find the underlying parameter name for PaginationSize.");
}
else
{
var maxLimit = Site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 1000 : 500;
var currentLimit = Math.Max(PaginationSize, 50);
// Continuously expand PaginationSize, hopefully we can retrieve some different continuation param value.
while (currentLimit < maxLimit)
{
currentLimit = Math.Min(maxLimit, currentLimit * 2);
Site.Logger.LogDebug("Try to fetch more with {ParamName}={ParamValue}.", limitParamName, currentLimit);
queryParams.Clear();
queryParams.MergeFrom(baseQueryParams);
queryParams.MergeFrom(continuationParams);
queryParams[limitParamName] = currentLimit;
var jresult2 = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), ct);
var applyResult = RequestHelper.ParseContinuationParameters(jresult2, queryParams, continuationParams);
switch (applyResult)
{
case RequestHelper.CONTINUATION_AVAILABLE:
case RequestHelper.CONTINUATION_DONE:
var listNode2 = RequestHelper.FindQueryResponseItemsRoot(jresult2, ListName);
Site.Logger.LogInformation("Successfully got out of the continuation loop.");
if (listNode2 != null)
{
if (listNode != null)
{
// Eliminate items that we have already yielded.
var yieldedItems = new HashSet <JToken>(listNode, new JTokenEqualityComparer());
await sink.YieldAndWait(
listNode2.Where(n => !yieldedItems.Contains(n)).Select(ItemFromJson));
}
else
{
await sink.YieldAndWait(listNode2.Select(ItemFromJson));
}
}
outOfLoop = true;
if (applyResult == RequestHelper.CONTINUATION_DONE)
{
return;
}
break;
case RequestHelper.CONTINUATION_LOOP:
break;
}
}
}
}
//if (!outOfLoop && (CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.SkipItems) ==
// WikiListContinuationLoopBehaviors.SkipItems)
//{
//}
}
if (!outOfLoop)
{
throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop);
}
break;
}
}
catch (Exception ex)
{
OnEnumItemsFailed(ex);
throw;
}
}
}
}));
}
public static IAsyncEnumerable <JObject> QueryWithContinuation(WikiSite site,
IEnumerable <KeyValuePair <string, object> > parameters,
Func <IDisposable> beginActionScope,
bool distinctPages = false)
{
return(AsyncEnumerableFactory.FromAsyncGenerator <JObject>(async(sink, ct) =>
{
ct.ThrowIfCancellationRequested();
var retrivedPageIds = distinctPages ? new HashSet <int>() : null;
using (beginActionScope?.Invoke())
{
var baseQueryParams = parameters.ToDictionary(p => p.Key, p => p.Value);
Debug.Assert("query".Equals(baseQueryParams["action"]));
var continuationParams = new Dictionary <string, object>();
while (true)
{
var queryParams = new Dictionary <string, object>(baseQueryParams);
queryParams.MergeFrom(continuationParams);
var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), ct);
var jpages = (JObject)FindQueryResponseItemsRoot(jresult, "pages");
if (jpages != null)
{
if (retrivedPageIds != null)
{
// Remove duplicate results
var duplicateKeys = new List <string>(jpages.Count);
foreach (var jpage in jpages)
{
if (!retrivedPageIds.Add(Convert.ToInt32(jpage.Key)))
{
// The page has been retrieved before.
duplicateKeys.Add(jpage.Key);
}
}
var originalPageCount = jpages.Count;
foreach (var k in duplicateKeys)
{
jpages.Remove(k);
}
if (originalPageCount != jpages.Count)
{
site.Logger.LogWarning(
"Received {Count} results on {Site}, {DistinctCount} distinct results.",
originalPageCount, site, jpages.Count);
}
}
await sink.YieldAndWait(jpages);
}
switch (ParseContinuationParameters(jresult, queryParams, continuationParams))
{
case CONTINUATION_DONE:
return;
case CONTINUATION_AVAILABLE:
if (jpages == null)
{
site.Logger.LogWarning("Empty query page with continuation received on {Site}.", site);
}
// Continue the loop and fetch for the next page of query.
break;
case CONTINUATION_LOOP:
throw new UnexpectedDataException();
}
}
}
}));
}
public static IAsyncEnumerable <Post> EnumArticleCommentsAsync(Board board, PostQueryOptions options)
{
IList <Post> PostsFromJsonOutline(JObject commentList)
{
return(commentList.Properties().Select(p =>
{
var post = new Post(board.Site, board.Page, Convert.ToInt32(p.Name));
var level2 = p.Value["level2"];
if (level2 != null && level2.HasValues)
{
post.Replies = new ReadOnlyCollection <Post>(((JObject)level2).Properties().Select(p2 =>
new Post(board.Site, board.Page, Convert.ToInt32(p2.Name))).ToList());
}
return post;
}).ToList());
}
IEnumerable <Post> PostsAndDescendants(IEnumerable <Post> posts)
{
foreach (var p in posts)
{
yield return(p);
if (p.Replies.Count > 0)
{
foreach (var p2 in p.Replies)
{
yield return(p2);
// Wikia only supports level-2 comments for now.
Debug.Assert(p2.Replies.Count == 0);
}
}
}
}
return(AsyncEnumerableFactory.FromAsyncGenerator <Post>(async(sink, ct) =>
{
using (board.Site.BeginActionScope(board))
{
// Refresh to get the page id.
if (!board.Page.HasId)
{
await board.RefreshAsync(ct);
}
if (!board.Exists)
{
return;
}
var pagesCount = 1;
for (int page = 1; page <= pagesCount; page++)
{
var jroot = await board.Site.InvokeNirvanaAsync(new WikiaQueryRequestMessage(new
{
format = "json",
controller = "ArticleComments",
method = "Content",
articleId = board.Page.Id,
page = page
}), WikiaJsonResonseParser.Default, ct);
// Build comment structure.
var jcomments = jroot["commentListRaw"];
if (jcomments != null && jcomments.HasValues)
{
var comments = PostsFromJsonOutline((JObject)jcomments);
pagesCount = (int)jroot["pagesCount"];
await RefreshPostsAsync(PostsAndDescendants(comments), options, ct);
await sink.YieldAndWait(comments);
}
}
}
}));
}
public void Should_execute_stored_procedure_asynchronously_and_return_objects()
{
var command = _commandFactory.CreateStoredProcedure("proc1");
var asyncResult = _context.BeginExecuteEnumerable(command);
IAsyncEnumerableFactory asyncEnumerableFactory = new AsyncEnumerableFactory();
using (var asyncEnumerable = asyncEnumerableFactory.Create<TestDataContract>(_context, command, asyncResult))
{
asyncEnumerable.AsyncWaitHandle.WaitOne();
using (var dataContracts = asyncEnumerable.GetResults())
{
var dataContractList = dataContracts.ToList();
Assert.AreEqual(2, dataContractList.Count);
Assert.AreEqual(10, dataContractList[0].Id);
Assert.AreEqual("name1", dataContractList[0].Name);
Assert.AreEqual(20, dataContractList[1].Id);
Assert.AreEqual("name2", dataContractList[1].Name);
}
}
}
