EnumerateSearchStatuses
(
String searchTerm,
Int32 maximumStatuses,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert(!String.IsNullOrEmpty(searchTerm));
Debug.Assert(maximumStatuses > 0);
Debug.Assert(maximumStatuses != Int32.MaxValue);
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iPage = 1;
Int32 iStatusesEnumerated = 0;
String sQueryParametersForNextPage = null;
while (true)
{
if (iPage > 1)
{
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Getting page {0}."
,
iPage
);
}
Dictionary <String, Object> oResponseDictionary = null;
Object [] aoStatusesThisPage;
String sUrl = GetSearchUrl(searchTerm,
sQueryParametersForNextPage);
try
{
Object oDeserializedTwitterResponse =
(new JavaScriptSerializer()).DeserializeObject(
GetTwitterResponseAsString(
sUrl, requestStatistics, reportProgressHandler,
checkCancellationPendingHandler));
// The top level of the Json response contains a set of
// name/value pairs. The value for the "statuses" name is the
// array of objects this method will enumerate.
oResponseDictionary = (Dictionary <String, Object>)
oDeserializedTwitterResponse;
aoStatusesThisPage =
( Object [] )oResponseDictionary["statuses"];
}
catch (Exception oException)
{
// Rethrow the exception if appropriate.
OnExceptionWhileEnumeratingJsonValues(oException, iPage,
false);
// Otherwise, just halt the enumeration.
yield break;
}
Int32 iObjectsThisPage = aoStatusesThisPage.Length;
if (iObjectsThisPage == 0)
{
yield break;
}
for (Int32 i = 0; i < iObjectsThisPage; i++)
{
yield return(
(Dictionary <String, Object>)aoStatusesThisPage[i]);
iStatusesEnumerated++;
if (iStatusesEnumerated == maximumStatuses)
{
yield break;
}
}
iPage++;
if (!TryGetQueryParametersForNextSearchPage(oResponseDictionary,
out sQueryParametersForNextPage))
{
yield break;
}
// Get the next page...
}
}
EnumerateJsonValues
(
String url,
String jsonName,
Int32 maximumValues,
Boolean skipMostPage1Errors,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
// Note:
//
// The logic in this method is similar to the logic in
// EnumerateSearchStatuses(). In fact, at one time all enumeration was
// done through this EnumerateJsonValues() method.
// EnumerateSearchStatuses() was created only when version 1.1 of the
// Twitter API introduced yet another paging scheme, one that differs
// from the cursor scheme that this method handles.
//
// A possible work item is to recombine the two methods into one,
// possibly by using a delegate to handle the different paging schemes.
Debug.Assert(!String.IsNullOrEmpty(url));
Debug.Assert(maximumValues > 0);
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iPage = 1;
String sCursor = null;
Int32 iObjectsEnumerated = 0;
while (true)
{
if (iPage > 1)
{
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Getting page {0}."
,
iPage
);
}
String sUrlWithCursor = AppendCursorToUrl(url, sCursor);
Dictionary <String, Object> oValueDictionary = null;
Object [] aoObjectsThisPage;
try
{
Object oDeserializedTwitterResponse =
(new JavaScriptSerializer()).DeserializeObject(
GetTwitterResponseAsString(sUrlWithCursor,
requestStatistics, reportProgressHandler,
checkCancellationPendingHandler));
Object oObjectsThisPageAsObject;
if (jsonName == null)
{
// The top level of the Json response contains an array of
// objects this method will enumerate.
oObjectsThisPageAsObject = oDeserializedTwitterResponse;
}
else
{
// The top level of the Json response contains a set of
// name/value pairs. The value for the specified name is
// the array of objects this method will enumerate.
oValueDictionary = (Dictionary <String, Object>)
oDeserializedTwitterResponse;
oObjectsThisPageAsObject = oValueDictionary[jsonName];
}
aoObjectsThisPage = ( Object [] )oObjectsThisPageAsObject;
}
catch (Exception oException)
{
// Rethrow the exception if appropriate.
TwitterUtil.OnExceptionWhileEnumeratingJsonValues(
oException, iPage, skipMostPage1Errors);
// Otherwise, just halt the enumeration.
yield break;
}
Int32 iObjectsThisPage = aoObjectsThisPage.Length;
if (iObjectsThisPage == 0)
{
yield break;
}
for (Int32 i = 0; i < iObjectsThisPage; i++)
{
yield return(aoObjectsThisPage[i]);
iObjectsEnumerated++;
if (iObjectsEnumerated == maximumValues)
{
yield break;
}
}
iPage++;
// When the top level of the Json response contains a set of
// name/value pairs, a next_cursor_str value of "0" means "end of
// data."
if (
oValueDictionary == null
||
!TwitterJsonUtil.TryGetJsonValueFromDictionary(
oValueDictionary, "next_cursor_str", out sCursor)
||
sCursor == "0"
)
{
yield break;
}
// Get the next page...
}
}
GetTwitterResponseAsString
(
String url,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert(!String.IsNullOrEmpty(url));
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iRateLimitPauses = 0;
Int32 iInvalidJsonRepeats = 0;
const Int32 MaximumInvalidJsonRepeats = 3;
while (true)
{
// Add the required authorization information to the URL.
//
// Note: Don't do this outside the while (true) loop. The
// authorization information includes a timestamp that will
// probably expire if the code within the catch block pauses.
oAuthTwitter oAuthTwitter = new oAuthTwitter(
m_UserAgent, m_TimeoutMs);
oAuthTwitter.Token = m_TwitterAccessToken;
oAuthTwitter.TokenSecret = m_TwitterAccessTokenSecret;
String sAuthorizedUrl, sAuthorizedPostData;
oAuthTwitter.ConstructAuthWebRequest(oAuthTwitter.Method.GET,
url, String.Empty, out sAuthorizedUrl,
out sAuthorizedPostData);
url = sAuthorizedUrl;
Stream oStream = null;
try
{
oStream =
HttpSocialNetworkUtil.GetHttpWebResponseStreamWithRetries(
url, HttpStatusCodesToFailImmediately,
requestStatistics, m_UserAgent, m_TimeoutMs,
reportProgressHandler, checkCancellationPendingHandler);
return(GetTwitterResponseAsString(oStream));
}
catch (InvalidJsonException oInvalidJsonException)
{
iInvalidJsonRepeats++;
if (iInvalidJsonRepeats > MaximumInvalidJsonRepeats)
{
throw oInvalidJsonException;
}
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Received invalid JSON from Twitter. Trying again."
);
}
catch (WebException oWebException)
{
if (
!WebExceptionIsDueToRateLimit(oWebException)
||
iRateLimitPauses > 0
)
{
throw;
}
// Twitter rate limits have kicked in. Pause and try again.
iRateLimitPauses++;
Int32 iRateLimitPauseMs = GetRateLimitPauseMs(oWebException);
DateTime oWakeUpTime = DateTime.Now.AddMilliseconds(
iRateLimitPauseMs);
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Reached Twitter rate limits. Pausing until {0}."
,
oWakeUpTime.ToLongTimeString()
);
// Don't pause in one large interval, which would prevent
// cancellation.
const Int32 SleepCycleDurationMs = 1000;
Int32 iSleepCycles = (Int32)Math.Ceiling(
(Double)iRateLimitPauseMs / SleepCycleDurationMs);
for (Int32 i = 0; i < iSleepCycles; i++)
{
if (checkCancellationPendingHandler != null)
{
checkCancellationPendingHandler();
}
System.Threading.Thread.Sleep(SleepCycleDurationMs);
}
}
finally
{
if (oStream != null)
{
oStream.Close();
}
}
}
}
EnumerateUserValueDictionaries
(
String [] userIDsOrScreenNames,
Boolean userIDsSpecified,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert(userIDsOrScreenNames != null);
Debug.Assert(requestStatistics != null);
AssertValid();
// We'll use Twitter's users/lookup API, which gets extended
// information for up to 100 users in one call.
Int32 iUsers = userIDsOrScreenNames.Length;
Int32 iUsersProcessed = 0;
Int32 iCalls = 0;
while (iUsersProcessed < iUsers)
{
// For each call, ask for information about as many users as
// possible until either 100 is reached or the URL reaches an
// arbitrary maximum length. Twitter recommends using a POST here
// (without specifying why), but it would require revising the
// base-class HTTP calls and isn't worth the trouble.
Int32 iUsersProcessedThisCall = 0;
StringBuilder oUrl = new StringBuilder();
oUrl.AppendFormat(
"{0}users/lookup.json?{1}&{2}="
,
TwitterApiUrls.Rest,
TwitterApiUrlParameters.IncludeEntities,
userIDsSpecified ? "user_id" : "screen_name"
);
const Int32 MaxUsersPerCall = 100;
const Int32 MaxUrlLength = 2000;
// Construct the URL for this call.
while (
iUsersProcessed < iUsers
&&
iUsersProcessedThisCall < MaxUsersPerCall
&&
oUrl.Length < MaxUrlLength
)
{
if (iUsersProcessedThisCall > 0)
{
// Append an encoded comma. Using an unencoded comma
// causes Twitter to return a 401 "unauthorized" error.
//
// See this post for an explanation:
//
// https://dev.twitter.com/discussions/11399
oUrl.Append("%2C");
}
oUrl.Append(userIDsOrScreenNames[iUsersProcessed]);
iUsersProcessed++;
iUsersProcessedThisCall++;
}
iCalls++;
if (iCalls > 1)
{
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Getting page {0}."
,
iCalls
);
}
foreach (Object oResult in EnumerateJsonValues(oUrl.ToString(),
null, Int32.MaxValue, true, requestStatistics,
reportProgressHandler, checkCancellationPendingHandler))
{
yield return((Dictionary <String, Object>)oResult);
}
}
}