ReportProgress
(
ReportProgressHandler reportProgressHandler,
String progressFormat,
params Object [] progressArguments
)
{
Debug.Assert(progressFormat != null);
Debug.Assert(progressArguments != null);
if (reportProgressHandler != null)
{
reportProgressHandler(
String.Format(progressFormat, progressArguments)
);
}
}
/// <summary>
/// 支持断点续传的文件下载。要配合DownloadServer一起用！（在客户端）
/// </summary>
/// <param name="filename"></param>
/// <param name="savePath"></param>
/// <param name="hostIp"></param>
/// <param name="hostPort"></param>
/// <param name="breakpoint">大于零则是存在断点，也作为输出参数，作为下一次短点记录下来（单位：字节）</param>
/// <param name="fileLength">下载文件的总长度（单位：字节）</param>
/// <param name="timeout">接收文件的最大超时时间（单位：秒，默认为60秒）</param>
/// <returns></returns>
public bool DownloadWithContinuingBreakpoint(string filename, string savePath, string hostIp, int hostPort, ref int breakpoint, out int fileLength, ReportProgressHandler reportProgress, int timeout = 60)
{
fileLength = 0;
TcpClient client = new TcpClient();
Log("开始下载....", null);
try
{
client.ReceiveTimeout = 1000 * timeout;
if (string.IsNullOrEmpty(filename) || filename.Trim() == "")
{
return(false);
}
if (!TryConnect(client, hostIp, hostPort))
{
return(false); //尝试连接服务器
}
if (!client.Connected)
{
return(false); //连线失败,退出
}
GetFile(client, filename, breakpoint);
cancelDownload = false;
//开始接受数据....
Log("开始接受数据....", null);
using (NetworkStream ns = client.GetStream())
{
byte[] lenBytes = new byte[4];
int l = ns.Read(lenBytes, 0, 4);
fileLength = BitConverter.ToInt32(lenBytes, 0);
byte[] resBytes = new byte[downloadBufferSize];
int resSize;
//if (breakpoint > 0)//NetworkStream不支持查找操作！！！这里会引发异常，故屏蔽。采用GetFile方法在服务器端发送剩余的字节来解决此问题。
// ns.Seek(breakpoint, SeekOrigin.Current);
resSize = ns.Read(resBytes, 0, resBytes.Length);
if (resSize == 0)
{
return(false);
}
if (Byte2Str(resBytes) == FILE_NOT_FOUND)
{
return(false);
}
using (FileStream fs = new FileStream(savePath, FileMode.Append, FileAccess.Write, FileShare.Write))
{
do
{
if (cancelDownload)
{
break;
}
downloading = true;
using (MemoryStream ms = new System.IO.MemoryStream())
{
ms.Write(resBytes, 0, resSize);
if (ms.Length > 0)
{
if (breakpoint == 0)
{
using (FileStream f = new FileStream(savePath, FileMode.Create, FileAccess.Write, FileShare.Write))
{
f.Write(ms.ToArray(), 0, (int)ms.Length);
}
}
else
{
fs.Write(ms.ToArray(), 0, (int)ms.Length);
}
//else//这里会造出文件被其他进程占用无法访问的异常！故屏蔽。
//{
// using (FileStream fs = new FileStream(savePath, FileMode.Append, FileAccess.Write, FileShare.Write))
// {
// fs.Write(ms.ToArray(), 0, (int)ms.Length);
// }
//}
}
}
breakpoint += resSize;
reportProgress.Invoke(client, new ReportProgressArgs(breakpoint, fileLength));
} while (ns.DataAvailable);
downloading = false;
if (!ns.DataAvailable)
{
downloadFinished = true;
}
}
}
return(true);
}
catch (Exception e)
{
Log("DownloadWithContinuingBreakpoint", e);
return(false);
}
finally
{
client.Close();
OnDownloadExited(breakpoint);
Log("退出下载....", null);
}
}
EnumerateSearchStatuses
(
String searchTerm,
Int32 maximumStatuses,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert( !String.IsNullOrEmpty(searchTerm) );
Debug.Assert(maximumStatuses > 0);
Debug.Assert(maximumStatuses != Int32.MaxValue);
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iPage = 1;
Int32 iStatusesEnumerated = 0;
String sQueryParametersForNextPage = null;
while (true)
{
if (reportProgressHandler != null && iPage > 1)
{
reportProgressHandler("Getting page " + iPage + ".");
}
Dictionary<String, Object> oResponseDictionary = null;
Object [] aoStatusesThisPage;
String sUrl = GetSearchUrl(searchTerm,
sQueryParametersForNextPage);
try
{
Object oDeserializedTwitterResponse =
( new JavaScriptSerializer() ).DeserializeObject(
GetTwitterResponseAsString(
sUrl, requestStatistics, reportProgressHandler,
checkCancellationPendingHandler) );
// The top level of the Json response contains a set of
// name/value pairs. The value for the "statuses" name is the
// array of objects this method will enumerate.
oResponseDictionary = ( Dictionary<String, Object> )
oDeserializedTwitterResponse;
aoStatusesThisPage =
( Object [] )oResponseDictionary["statuses"];
}
catch (Exception oException)
{
// Rethrow the exception if appropriate.
OnExceptionWhileEnumeratingJsonValues(oException, iPage,
false);
// Otherwise, just halt the enumeration.
yield break;
}
Int32 iObjectsThisPage = aoStatusesThisPage.Length;
if (iObjectsThisPage == 0)
{
yield break;
}
for (Int32 i = 0; i < iObjectsThisPage; i++)
{
yield return (
( Dictionary<String, Object> )aoStatusesThisPage[i] );
iStatusesEnumerated++;
if (iStatusesEnumerated == maximumStatuses)
{
yield break;
}
}
iPage++;
if ( !TryGetQueryParametersForNextSearchPage(oResponseDictionary,
out sQueryParametersForNextPage) )
{
yield break;
}
// Get the next page...
}
}
GetTwitterResponseAsString
(
String url,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert( !String.IsNullOrEmpty(url) );
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iRateLimitPauses = 0;
while (true)
{
// Add the required authorization information to the URL.
//
// Note: Don't do this outside the while (true) loop. The
// authorization information includes a timestamp that will
// probably expire if the code within the catch block pauses.
oAuthTwitter oAuthTwitter = new oAuthTwitter(
m_UserAgent, m_TimeoutMs);
oAuthTwitter.Token = m_TwitterAccessToken;
oAuthTwitter.TokenSecret = m_TwitterAccessTokenSecret;
String sAuthorizedUrl, sAuthorizedPostData;
oAuthTwitter.ConstructAuthWebRequest(oAuthTwitter.Method.GET,
url, String.Empty, out sAuthorizedUrl,
out sAuthorizedPostData);
url = sAuthorizedUrl;
Stream oStream = null;
try
{
oStream =
HttpSocialNetworkUtil.GetHttpWebResponseStreamWithRetries(
url, HttpStatusCodesToFailImmediately,
requestStatistics, m_UserAgent, m_TimeoutMs,
reportProgressHandler, checkCancellationPendingHandler);
return ( new StreamReader(oStream).ReadToEnd() );
}
catch (WebException oWebException)
{
if (
!WebExceptionIsDueToRateLimit(oWebException)
||
iRateLimitPauses > 0
)
{
throw;
}
// Twitter rate limits have kicked in. Pause and try again.
iRateLimitPauses++;
Int32 iRateLimitPauseMs = GetRateLimitPauseMs(oWebException);
DateTime oWakeUpTime = DateTime.Now.AddMilliseconds(
iRateLimitPauseMs);
if (reportProgressHandler != null)
{
reportProgressHandler( String.Format(
"Reached Twitter rate limits. Pausing until {0}."
,
oWakeUpTime.ToLongTimeString()
) );
}
// Don't pause in one large interval, which would prevent
// cancellation.
const Int32 SleepCycleDurationMs = 1000;
Int32 iSleepCycles = (Int32)Math.Ceiling(
(Double)iRateLimitPauseMs / SleepCycleDurationMs) ;
for (Int32 i = 0; i < iSleepCycles; i++)
{
if (checkCancellationPendingHandler != null)
{
checkCancellationPendingHandler();
}
System.Threading.Thread.Sleep(SleepCycleDurationMs);
}
}
finally
{
if (oStream != null)
{
oStream.Close();
}
}
}
}
EnumerateJsonValues
(
String url,
String jsonName,
Int32 maximumValues,
Boolean skipMostPage1Errors,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
// Note:
//
// The logic in this method is similar to the logic in
// EnumerateSearchStatuses(). In fact, at one time all enumeration was
// done through this EnumerateJsonValues() method.
// EnumerateSearchStatuses() was created only when version 1.1 of the
// Twitter API introduced yet another paging scheme, one that differs
// from the cursor scheme that this method handles.
//
// A possible work item is to recombine the two methods into one,
// possibly by using a delegate to handle the different paging schemes.
Debug.Assert( !String.IsNullOrEmpty(url) );
Debug.Assert(maximumValues > 0);
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iPage = 1;
String sCursor = null;
Int32 iObjectsEnumerated = 0;
while (true)
{
if (iPage > 1 && reportProgressHandler != null)
{
reportProgressHandler("Getting page " + iPage + ".");
}
String sUrlWithCursor = AppendCursorToUrl(url, sCursor);
Dictionary<String, Object> oValueDictionary = null;
Object [] aoObjectsThisPage;
try
{
Object oDeserializedTwitterResponse =
( new JavaScriptSerializer() ).DeserializeObject(
GetTwitterResponseAsString(sUrlWithCursor,
requestStatistics, reportProgressHandler,
checkCancellationPendingHandler) );
Object oObjectsThisPageAsObject;
if (jsonName == null)
{
// The top level of the Json response contains an array of
// objects this method will enumerate.
oObjectsThisPageAsObject = oDeserializedTwitterResponse;
}
else
{
// The top level of the Json response contains a set of
// name/value pairs. The value for the specified name is
// the array of objects this method will enumerate.
oValueDictionary = ( Dictionary<String, Object> )
oDeserializedTwitterResponse;
oObjectsThisPageAsObject = oValueDictionary[jsonName];
}
aoObjectsThisPage = ( Object [] )oObjectsThisPageAsObject;
}
catch (Exception oException)
{
// Rethrow the exception if appropriate.
TwitterUtil.OnExceptionWhileEnumeratingJsonValues(
oException, iPage, skipMostPage1Errors);
// Otherwise, just halt the enumeration.
yield break;
}
Int32 iObjectsThisPage = aoObjectsThisPage.Length;
if (iObjectsThisPage == 0)
{
yield break;
}
for (Int32 i = 0; i < iObjectsThisPage; i++)
{
yield return ( aoObjectsThisPage[i] );
iObjectsEnumerated++;
if (iObjectsEnumerated == maximumValues)
{
yield break;
}
}
iPage++;
// When the top level of the Json response contains a set of
// name/value pairs, a next_cursor_str value of "0" means "end of
// data."
if (
oValueDictionary == null
||
!TwitterJsonUtil.TryGetJsonValueFromDictionary(
oValueDictionary, "next_cursor_str", out sCursor)
||
sCursor == "0"
)
{
yield break;
}
// Get the next page...
}
}
EnumerateUserValueDictionaries
(
String [] userIDsOrScreenNames,
Boolean userIDsSpecified,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert(userIDsOrScreenNames != null);
Debug.Assert(requestStatistics != null);
AssertValid();
// We'll use Twitter's users/lookup API, which gets extended
// information for up to 100 users in one call.
Int32 iUsers = userIDsOrScreenNames.Length;
Int32 iUsersProcessed = 0;
Int32 iCalls = 0;
while (iUsersProcessed < iUsers)
{
// For each call, ask for information about as many users as
// possible until either 100 is reached or the URL reaches an
// arbitrary maximum length. Twitter recommends using a POST here
// (without specifying why), but it would require revising the
// base-class HTTP calls and isn't worth the trouble.
Int32 iUsersProcessedThisCall = 0;
StringBuilder oUrl = new StringBuilder();
oUrl.AppendFormat(
"{0}users/lookup.json?{1}&{2}="
,
TwitterApiUrls.Rest,
TwitterApiUrlParameters.IncludeEntities,
userIDsSpecified ? "user_id" : "screen_name"
);
const Int32 MaxUsersPerCall = 100;
const Int32 MaxUrlLength = 2000;
// Construct the URL for this call.
while (
iUsersProcessed < iUsers
&&
iUsersProcessedThisCall < MaxUsersPerCall
&&
oUrl.Length < MaxUrlLength
)
{
if (iUsersProcessedThisCall > 0)
{
// Append an encoded comma. Using an unencoded comma
// causes Twitter to return a 401 "unauthorized" error.
//
// See this post for an explanation:
//
// https://dev.twitter.com/discussions/11399
oUrl.Append("%2C");
}
oUrl.Append( userIDsOrScreenNames[iUsersProcessed] );
iUsersProcessed++;
iUsersProcessedThisCall++;
}
iCalls++;
if (iCalls > 1 && reportProgressHandler != null)
{
reportProgressHandler("Getting page " + iCalls + ".");
}
foreach ( Object oResult in EnumerateJsonValues(oUrl.ToString(),
null, Int32.MaxValue, true, requestStatistics,
reportProgressHandler, checkCancellationPendingHandler) )
{
yield return ( ( Dictionary<String, Object> )oResult );
}
}
}
GetHttpWebResponseStreamWithRetries
(
String url,
HttpStatusCode [] httpStatusCodesToFailImmediately,
RequestStatistics requestStatistics,
String userAgent,
Int32 timeoutMs,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert(!String.IsNullOrEmpty(url));
Debug.Assert(requestStatistics != null);
Debug.Assert(!String.IsNullOrEmpty(userAgent));
Debug.Assert(timeoutMs > 0);
Int32 iMaximumRetries = HttpRetryDelaysSec.Length;
Int32 iRetriesSoFar = 0;
while (true)
{
if (reportProgressHandler != null && iRetriesSoFar > 0)
{
reportProgressHandler("Retrying request.");
}
// Important Note: You cannot use the same HttpWebRequest object
// for the retries. The object must be recreated each time.
HttpWebRequest oHttpWebRequest = CreateHttpWebRequest(
url, userAgent, timeoutMs);
Stream oStream = null;
try
{
if (checkCancellationPendingHandler != null)
{
checkCancellationPendingHandler();
}
oStream = GetHttpWebResponseStreamNoRetries(oHttpWebRequest);
if (reportProgressHandler != null && iRetriesSoFar > 0)
{
reportProgressHandler("Retry succeeded, continuing...");
}
requestStatistics.OnSuccessfulRequest();
return(oStream);
}
catch (Exception oException)
{
#if WriteRequestsToDebug
Debug.WriteLine("Exception: " + oException.Message);
#endif
if (oStream != null)
{
oStream.Close();
}
// The IOException "Unable to read data from the transport"
// connection: The connection was closed." can occur when
// requesting data from Twitter, for example.
if (!(
oException is WebException
||
oException is IOException
))
{
throw oException;
}
if (iRetriesSoFar == iMaximumRetries)
{
requestStatistics.OnUnexpectedException(oException);
throw (oException);
}
// If the status code is one of the ones specified in
// httpStatusCodesToFailImmediately, rethrow the exception
// without retrying the request.
if (httpStatusCodesToFailImmediately != null &&
oException is WebException)
{
if (WebExceptionHasHttpStatusCode(
(WebException)oException,
httpStatusCodesToFailImmediately))
{
throw (oException);
}
}
Int32 iSeconds = HttpRetryDelaysSec[iRetriesSoFar];
if (reportProgressHandler != null)
{
reportProgressHandler(String.Format(
"Request failed, pausing {0} {1} before retrying..."
,
iSeconds,
StringUtil.MakePlural("second", iSeconds)
));
}
System.Threading.Thread.Sleep(1000 * iSeconds);
iRetriesSoFar++;
}
}
}
EnumerateSearchStatuses
(
String searchTerm,
Int32 maximumStatuses,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert(!String.IsNullOrEmpty(searchTerm));
Debug.Assert(maximumStatuses > 0);
Debug.Assert(maximumStatuses != Int32.MaxValue);
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iPage = 1;
Int32 iStatusesEnumerated = 0;
String sQueryParametersForNextPage = null;
while (true)
{
if (iPage > 1)
{
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Getting page {0}."
,
iPage
);
}
Dictionary <String, Object> oResponseDictionary = null;
Object [] aoStatusesThisPage;
String sUrl = GetSearchUrl(searchTerm,
sQueryParametersForNextPage);
try
{
Object oDeserializedTwitterResponse =
(new JavaScriptSerializer()).DeserializeObject(
GetTwitterResponseAsString(
sUrl, requestStatistics, reportProgressHandler,
checkCancellationPendingHandler));
// The top level of the Json response contains a set of
// name/value pairs. The value for the "statuses" name is the
// array of objects this method will enumerate.
oResponseDictionary = (Dictionary <String, Object>)
oDeserializedTwitterResponse;
aoStatusesThisPage =
( Object [] )oResponseDictionary["statuses"];
}
catch (Exception oException)
{
// Rethrow the exception if appropriate.
OnExceptionWhileEnumeratingJsonValues(oException, iPage,
false);
// Otherwise, just halt the enumeration.
yield break;
}
Int32 iObjectsThisPage = aoStatusesThisPage.Length;
if (iObjectsThisPage == 0)
{
yield break;
}
for (Int32 i = 0; i < iObjectsThisPage; i++)
{
yield return(
(Dictionary <String, Object>)aoStatusesThisPage[i]);
iStatusesEnumerated++;
if (iStatusesEnumerated == maximumStatuses)
{
yield break;
}
}
iPage++;
if (!TryGetQueryParametersForNextSearchPage(oResponseDictionary,
out sQueryParametersForNextPage))
{
yield break;
}
// Get the next page...
}
}
GetTwitterResponseAsString
(
String url,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert(!String.IsNullOrEmpty(url));
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iRateLimitPauses = 0;
Int32 iInvalidJsonRepeats = 0;
const Int32 MaximumInvalidJsonRepeats = 3;
while (true)
{
// Add the required authorization information to the URL.
//
// Note: Don't do this outside the while (true) loop. The
// authorization information includes a timestamp that will
// probably expire if the code within the catch block pauses.
oAuthTwitter oAuthTwitter = new oAuthTwitter(
m_UserAgent, m_TimeoutMs);
oAuthTwitter.Token = m_TwitterAccessToken;
oAuthTwitter.TokenSecret = m_TwitterAccessTokenSecret;
String sAuthorizedUrl, sAuthorizedPostData;
oAuthTwitter.ConstructAuthWebRequest(oAuthTwitter.Method.GET,
url, String.Empty, out sAuthorizedUrl,
out sAuthorizedPostData);
url = sAuthorizedUrl;
Stream oStream = null;
try
{
oStream =
HttpSocialNetworkUtil.GetHttpWebResponseStreamWithRetries(
url, HttpStatusCodesToFailImmediately,
requestStatistics, m_UserAgent, m_TimeoutMs,
reportProgressHandler, checkCancellationPendingHandler);
return(GetTwitterResponseAsString(oStream));
}
catch (InvalidJsonException oInvalidJsonException)
{
iInvalidJsonRepeats++;
if (iInvalidJsonRepeats > MaximumInvalidJsonRepeats)
{
throw oInvalidJsonException;
}
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Received invalid JSON from Twitter. Trying again."
);
}
catch (WebException oWebException)
{
if (
!WebExceptionIsDueToRateLimit(oWebException)
||
iRateLimitPauses > 0
)
{
throw;
}
// Twitter rate limits have kicked in. Pause and try again.
iRateLimitPauses++;
Int32 iRateLimitPauseMs = GetRateLimitPauseMs(oWebException);
DateTime oWakeUpTime = DateTime.Now.AddMilliseconds(
iRateLimitPauseMs);
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Reached Twitter rate limits. Pausing until {0}."
,
oWakeUpTime.ToLongTimeString()
);
// Don't pause in one large interval, which would prevent
// cancellation.
const Int32 SleepCycleDurationMs = 1000;
Int32 iSleepCycles = (Int32)Math.Ceiling(
(Double)iRateLimitPauseMs / SleepCycleDurationMs);
for (Int32 i = 0; i < iSleepCycles; i++)
{
if (checkCancellationPendingHandler != null)
{
checkCancellationPendingHandler();
}
System.Threading.Thread.Sleep(SleepCycleDurationMs);
}
}
finally
{
if (oStream != null)
{
oStream.Close();
}
}
}
}
EnumerateJsonValues
(
String url,
String jsonName,
Int32 maximumValues,
Boolean skipMostPage1Errors,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
// Note:
//
// The logic in this method is similar to the logic in
// EnumerateSearchStatuses(). In fact, at one time all enumeration was
// done through this EnumerateJsonValues() method.
// EnumerateSearchStatuses() was created only when version 1.1 of the
// Twitter API introduced yet another paging scheme, one that differs
// from the cursor scheme that this method handles.
//
// A possible work item is to recombine the two methods into one,
// possibly by using a delegate to handle the different paging schemes.
Debug.Assert(!String.IsNullOrEmpty(url));
Debug.Assert(maximumValues > 0);
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iPage = 1;
String sCursor = null;
Int32 iObjectsEnumerated = 0;
while (true)
{
if (iPage > 1)
{
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Getting page {0}."
,
iPage
);
}
String sUrlWithCursor = AppendCursorToUrl(url, sCursor);
Dictionary <String, Object> oValueDictionary = null;
Object [] aoObjectsThisPage;
try
{
Object oDeserializedTwitterResponse =
(new JavaScriptSerializer()).DeserializeObject(
GetTwitterResponseAsString(sUrlWithCursor,
requestStatistics, reportProgressHandler,
checkCancellationPendingHandler));
Object oObjectsThisPageAsObject;
if (jsonName == null)
{
// The top level of the Json response contains an array of
// objects this method will enumerate.
oObjectsThisPageAsObject = oDeserializedTwitterResponse;
}
else
{
// The top level of the Json response contains a set of
// name/value pairs. The value for the specified name is
// the array of objects this method will enumerate.
oValueDictionary = (Dictionary <String, Object>)
oDeserializedTwitterResponse;
oObjectsThisPageAsObject = oValueDictionary[jsonName];
}
aoObjectsThisPage = ( Object [] )oObjectsThisPageAsObject;
}
catch (Exception oException)
{
// Rethrow the exception if appropriate.
TwitterUtil.OnExceptionWhileEnumeratingJsonValues(
oException, iPage, skipMostPage1Errors);
// Otherwise, just halt the enumeration.
yield break;
}
Int32 iObjectsThisPage = aoObjectsThisPage.Length;
if (iObjectsThisPage == 0)
{
yield break;
}
for (Int32 i = 0; i < iObjectsThisPage; i++)
{
yield return(aoObjectsThisPage[i]);
iObjectsEnumerated++;
if (iObjectsEnumerated == maximumValues)
{
yield break;
}
}
iPage++;
// When the top level of the Json response contains a set of
// name/value pairs, a next_cursor_str value of "0" means "end of
// data."
if (
oValueDictionary == null
||
!TwitterJsonUtil.TryGetJsonValueFromDictionary(
oValueDictionary, "next_cursor_str", out sCursor)
||
sCursor == "0"
)
{
yield break;
}
// Get the next page...
}
}
EnumerateUserValueDictionaries
(
String [] userIDsOrScreenNames,
Boolean userIDsSpecified,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert(userIDsOrScreenNames != null);
Debug.Assert(requestStatistics != null);
AssertValid();
// We'll use Twitter's users/lookup API, which gets extended
// information for up to 100 users in one call.
Int32 iUsers = userIDsOrScreenNames.Length;
Int32 iUsersProcessed = 0;
Int32 iCalls = 0;
while (iUsersProcessed < iUsers)
{
// For each call, ask for information about as many users as
// possible until either 100 is reached or the URL reaches an
// arbitrary maximum length. Twitter recommends using a POST here
// (without specifying why), but it would require revising the
// base-class HTTP calls and isn't worth the trouble.
Int32 iUsersProcessedThisCall = 0;
StringBuilder oUrl = new StringBuilder();
oUrl.AppendFormat(
"{0}users/lookup.json?{1}&{2}="
,
TwitterApiUrls.Rest,
TwitterApiUrlParameters.IncludeEntities,
userIDsSpecified ? "user_id" : "screen_name"
);
const Int32 MaxUsersPerCall = 100;
const Int32 MaxUrlLength = 2000;
// Construct the URL for this call.
while (
iUsersProcessed < iUsers
&&
iUsersProcessedThisCall < MaxUsersPerCall
&&
oUrl.Length < MaxUrlLength
)
{
if (iUsersProcessedThisCall > 0)
{
// Append an encoded comma. Using an unencoded comma
// causes Twitter to return a 401 "unauthorized" error.
//
// See this post for an explanation:
//
// https://dev.twitter.com/discussions/11399
oUrl.Append("%2C");
}
oUrl.Append(userIDsOrScreenNames[iUsersProcessed]);
iUsersProcessed++;
iUsersProcessedThisCall++;
}
iCalls++;
if (iCalls > 1)
{
HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,
"Getting page {0}."
,
iCalls
);
}
foreach (Object oResult in EnumerateJsonValues(oUrl.ToString(),
null, Int32.MaxValue, true, requestStatistics,
reportProgressHandler, checkCancellationPendingHandler))
{
yield return((Dictionary <String, Object>)oResult);
}
}
}
GetHttpWebResponseStreamWithRetries
(
String url,
HttpStatusCode [] httpStatusCodesToFailImmediately,
RequestStatistics requestStatistics,
String userAgent,
Int32 timeoutMs,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert( !String.IsNullOrEmpty(url) );
Debug.Assert(requestStatistics != null);
Debug.Assert( !String.IsNullOrEmpty(userAgent) );
Debug.Assert(timeoutMs > 0);
Int32 iMaximumRetries = HttpRetryDelaysSec.Length;
Int32 iRetriesSoFar = 0;
while (true)
{
if (reportProgressHandler != null && iRetriesSoFar > 0)
{
reportProgressHandler("Retrying request.");
}
// Important Note: You cannot use the same HttpWebRequest object
// for the retries. The object must be recreated each time.
HttpWebRequest oHttpWebRequest = CreateHttpWebRequest(
url, userAgent, timeoutMs);
Stream oStream = null;
try
{
if (checkCancellationPendingHandler != null)
{
checkCancellationPendingHandler();
}
oStream = GetHttpWebResponseStreamNoRetries(oHttpWebRequest);
if (reportProgressHandler != null && iRetriesSoFar > 0)
{
reportProgressHandler("Retry succeeded, continuing...");
}
requestStatistics.OnSuccessfulRequest();
return (oStream);
}
catch (Exception oException)
{
#if WriteRequestsToDebug
Debug.WriteLine("Exception: " + oException.Message);
#endif
if (oStream != null)
{
oStream.Close();
}
// The IOException "Unable to read data from the transport"
// connection: The connection was closed." can occur when
// requesting data from Twitter, for example.
if ( !(
oException is WebException
||
oException is IOException
) )
{
throw oException;
}
if (iRetriesSoFar == iMaximumRetries)
{
requestStatistics.OnUnexpectedException(oException);
throw (oException);
}
// If the status code is one of the ones specified in
// httpStatusCodesToFailImmediately, rethrow the exception
// without retrying the request.
if (httpStatusCodesToFailImmediately != null &&
oException is WebException)
{
if ( WebExceptionHasHttpStatusCode(
(WebException)oException,
httpStatusCodesToFailImmediately) )
{
throw (oException);
}
}
Int32 iSeconds = HttpRetryDelaysSec[iRetriesSoFar];
if (reportProgressHandler != null)
{
reportProgressHandler( String.Format(
"Request failed, pausing {0} {1} before retrying..."
,
iSeconds,
StringUtil.MakePlural("second", iSeconds)
) );
}
System.Threading.Thread.Sleep(1000 * iSeconds);
iRetriesSoFar++;
}
}
}
