TryGetXmlDocument ( String sUrl, RequestStatistics oRequestStatistics, out XmlDocument oXmlDocument, out XmlNamespaceManager oXmlNamespaceManager ) { Debug.Assert( !String.IsNullOrEmpty(sUrl) ); Debug.Assert(oRequestStatistics != null); AssertValid(); oXmlDocument = null; oXmlNamespaceManager = null; try { oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics, out oXmlNamespaceManager); return (true); } catch (Exception oException) { if ( !HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException)) { throw oException; } return (false); } }
TryGetXmlDocument ( String sUrl, RequestStatistics oRequestStatistics, out XmlDocument oXmlDocument ) { Debug.Assert(!String.IsNullOrEmpty(sUrl)); Debug.Assert(oRequestStatistics != null); AssertValid(); oXmlDocument = null; try { oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics); return(true); } catch (Exception oException) { // Note: Because a FlickrException is also a WebException, the // following logic will cause false to be returned for a // FlickrException. if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException)) { throw oException; } return(false); } }
WebExceptionIsDueToRateLimit ( WebException webException ) { Debug.Assert(webException != null); // Starting with version 1.1 of the Twitter API, a single HTTP status // code (429, "rate limit exceeded") is used for all rate-limit // responses. return(HttpSocialNetworkUtil.WebExceptionHasHttpStatusCode( webException, (HttpStatusCode)429)); }
TryGetUserValueDictionary ( String sScreenName, RequestStatistics oRequestStatistics, Boolean bIgnoreWebAndJsonExceptions, out Dictionary <String, Object> oUserValueDictionary ) { Debug.Assert(!String.IsNullOrEmpty(sScreenName)); Debug.Assert(oRequestStatistics != null); AssertValid(); oUserValueDictionary = null; String sUrl = String.Format( "{0}users/show.json?screen_name={1}&{2}" , TwitterApiUrls.Rest, TwitterUtil.EncodeUrlParameter(sScreenName), TwitterApiUrlParameters.IncludeEntities ); ReportProgress(String.Format( "Getting information about \"{0}\"." , sScreenName )); try { oUserValueDictionary = (Dictionary <String, Object>) (new JavaScriptSerializer()).DeserializeObject( GetTwitterResponseAsString(sUrl, oRequestStatistics)); return(true); } catch (Exception oException) { if (!HttpSocialNetworkUtil.ExceptionIsWebOrJson(oException) || !bIgnoreWebAndJsonExceptions) { throw oException; } return(false); } }
OnExceptionWhileEnumeratingJsonValues ( Exception exception, Int32 iPage, Boolean bSkipMostPage1Errors ) { Debug.Assert(exception != null); Debug.Assert(iPage > 0); if (!HttpSocialNetworkUtil.ExceptionIsWebOrJson(exception)) { // This is an unknown exception. throw (exception); } if ( exception is WebException && WebExceptionIsDueToRateLimit((WebException)exception) ) { throw (exception); } else if (iPage == 1) { if (bSkipMostPage1Errors) { return; } else { throw (exception); } } else { // Always skip non-rate-limit errors on page 2 and above. return; } }
/// <summary> /// Web Request Wrapper /// </summary> /// <param name="method">Http Method</param> /// <param name="url">Full url to the web resource</param> /// <param name="postData">Data to post in querystring format</param> /// <returns>The web server response.</returns> public string WebRequest(Method method, string url, string postData) { HttpWebRequest webRequest = null; StreamWriter requestWriter = null; string responseData = ""; webRequest = HttpSocialNetworkUtil.CreateHttpWebRequest( url, _userAgent, _timeoutMs); webRequest.Method = method.ToString(); webRequest.ServicePoint.Expect100Continue = false; if (method == Method.POST) { webRequest.ContentType = "application/x-www-form-urlencoded"; //POST the data. requestWriter = new StreamWriter(webRequest.GetRequestStream()); try { requestWriter.Write(postData); } catch { throw; } finally { requestWriter.Close(); requestWriter = null; } } responseData = WebResponseGet(webRequest); webRequest = null; return(responseData); }
GetXmlDocumentWithRetries ( String sUrl, HttpStatusCode [] aeHttpStatusCodesToFailImmediately, RequestStatistics oRequestStatistics ) { Debug.Assert(!String.IsNullOrEmpty(sUrl)); Debug.Assert(oRequestStatistics != null); AssertValid(); Stream oStream = null; try { oStream = HttpSocialNetworkUtil.GetHttpWebResponseStreamWithRetries( sUrl, aeHttpStatusCodesToFailImmediately, oRequestStatistics, UserAgent, HttpWebRequestTimeoutMs, new ReportProgressHandler(this.ReportProgress), new CheckCancellationPendingHandler( this.CheckCancellationPending) ); XmlDocument oXmlDocument = new XmlDocument(); oXmlDocument.Load(oStream); return(oXmlDocument); } finally { if (oStream != null) { oStream.Close(); } } }
GetRelatedTagsRecursive ( String sTag, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, String sApiKey, Int32 iRecursionLevel, GraphMLXmlDocument oGraphMLXmlDocument, Dictionary <String, XmlNode> oTagDictionary, RequestStatistics oRequestStatistics ) { Debug.Assert(!String.IsNullOrEmpty(sTag)); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(!String.IsNullOrEmpty(sApiKey)); Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oTagDictionary != null); Debug.Assert(oRequestStatistics != null); AssertValid(); /* * Here is what this method should do, based on the eNetworkLevel and * iRecursionLevel parameters. * * eNetworkLevel * |One | OnePointFive | Two * ---|------------------| ------------------| ----------------- * i 1 |Add all vertices. | Add all vertices. | Add all vertices. * R | | | * e |Add all edges. | Add all edges. | Add all edges. * c | | | * u |Do not recurse. | Recurse. | Recurse. * r | | | * s ---|------------------|-------------------|------------------ * i 2 |Impossible. | Do not add | Add all vertices. * o | | vertices. | * n | | | * L | | Add edges only if | Add all edges. * e | | vertices are | * v | | already included. | * e | | | * l | | Do not recurse. | Do not recurse. | | | | ---|------------------|-------------------|------------------ */ Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel, iRecursionLevel); Boolean bNeedToAppendVertices = GetNeedToAppendVertices( eNetworkLevel, iRecursionLevel); ReportProgress("Getting tags related to \"" + sTag + "\"."); String sUrl = GetFlickrMethodUrl("flickr.tags.getRelated", sApiKey, "&tag=" + UrlUtil.EncodeUrlParameter(sTag)); XmlDocument oXmlDocument; try { oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics); } catch (Exception oException) { // If the exception is not a WebException or XmlException, or if // none of the network has been obtained yet, throw the exception. if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException) || !oGraphMLXmlDocument.HasVertexXmlNode) { throw oException; } return; } // The document consists of a single "tags" node with zero or more // "tag" child nodes. String sOtherTag = null; XmlNodeList oTagNodes = oXmlDocument.DocumentElement.SelectNodes( "tags/tag"); if (oTagNodes.Count > 0) { AppendVertexXmlNode(sTag, oGraphMLXmlDocument, oTagDictionary); } foreach (XmlNode oTagNode in oTagNodes) { sOtherTag = XmlUtil2.SelectRequiredSingleNodeAsString(oTagNode, "text()", null); if (bNeedToAppendVertices) { AppendVertexXmlNode(sOtherTag, oGraphMLXmlDocument, oTagDictionary); } if (bNeedToAppendVertices || oTagDictionary.ContainsKey(sOtherTag)) { oGraphMLXmlDocument.AppendEdgeXmlNode(sTag, sOtherTag); } } if (bNeedToRecurse) { foreach (XmlNode oTagNode in oTagNodes) { sOtherTag = XmlUtil2.SelectRequiredSingleNodeAsString(oTagNode, "text()", null); GetRelatedTagsRecursive(sOtherTag, eWhatToInclude, eNetworkLevel, sApiKey, 2, oGraphMLXmlDocument, oTagDictionary, oRequestStatistics); } } }
EnumerateXmlNodes ( String sUrl, String sXPath, Int32 iMaximumXmlNodes, Boolean bSkipMostPage1Errors, RequestStatistics oRequestStatistics ) { Debug.Assert( !String.IsNullOrEmpty(sUrl) ); Debug.Assert( !String.IsNullOrEmpty(sXPath) ); Debug.Assert(iMaximumXmlNodes > 0); Debug.Assert(oRequestStatistics != null); AssertValid(); Int32 iStartIndex = 1; Int32 iMaxResults = 50; Int32 iXmlNodesEnumerated = 0; while (true) { if (iStartIndex > 1) { ReportProgress("Getting page starting with item " + iStartIndex + "."); } String sUrlWithPagination = String.Format( "{0}{1}start-index={2}&max-results={3}" , sUrl, sUrl.IndexOf('?') == -1 ? '?' : '&', iStartIndex, iMaxResults ); XmlDocument oXmlDocument; XmlNamespaceManager oXmlNamespaceManager; try { oXmlDocument = GetXmlDocument(sUrlWithPagination, oRequestStatistics, out oXmlNamespaceManager); } catch (Exception oException) { if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException)) { throw oException; } if (iStartIndex > 1 || bSkipMostPage1Errors) { // Always skip errors on page 2 and above. yield break; } throw (oException); } XmlNodeList oXmlNodesThisPage = oXmlDocument.SelectNodes(sXPath, oXmlNamespaceManager); Int32 iXmlNodesThisPage = oXmlNodesThisPage.Count; if (iXmlNodesThisPage == 0) { yield break; } for (Int32 i = 0; i < iXmlNodesThisPage; i++) { yield return ( oXmlNodesThisPage[i] ); iXmlNodesEnumerated++; if (iXmlNodesEnumerated == iMaximumXmlNodes) { yield break; } } // The next page, if there is one, is obtained from a link tag that // looks something like this: // // <link rel="next" ... href="http://gdata.youtube.com/feeds/...? // start-index=26&max-results=25"/> String sHRef; if ( !XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "a:feed/a:link[@rel='next']/@href", oXmlNamespaceManager, out sHRef) ) { yield break; } const String Pattern = @"start-index=(?<NextStartIndex>\d+)"; Regex oRegex = new Regex(Pattern); Match oMatch = oRegex.Match(sHRef); if (!oMatch.Success || !MathUtil.TryParseCultureInvariantInt32( oMatch.Groups["NextStartIndex"].Value, out iStartIndex) ) { yield break; } // Get the next page... } }
EnumerateSearchStatuses ( String searchTerm, Int32 maximumStatuses, RequestStatistics requestStatistics, ReportProgressHandler reportProgressHandler, CheckCancellationPendingHandler checkCancellationPendingHandler ) { Debug.Assert(!String.IsNullOrEmpty(searchTerm)); Debug.Assert(maximumStatuses > 0); Debug.Assert(maximumStatuses != Int32.MaxValue); Debug.Assert(requestStatistics != null); AssertValid(); Int32 iPage = 1; Int32 iStatusesEnumerated = 0; String sQueryParametersForNextPage = null; while (true) { if (iPage > 1) { HttpSocialNetworkUtil.ReportProgress(reportProgressHandler, "Getting page {0}." , iPage ); } Dictionary <String, Object> oResponseDictionary = null; Object [] aoStatusesThisPage; String sUrl = GetSearchUrl(searchTerm, sQueryParametersForNextPage); try { Object oDeserializedTwitterResponse = (new JavaScriptSerializer()).DeserializeObject( GetTwitterResponseAsString( sUrl, requestStatistics, reportProgressHandler, checkCancellationPendingHandler)); // The top level of the Json response contains a set of // name/value pairs. The value for the "statuses" name is the // array of objects this method will enumerate. oResponseDictionary = (Dictionary <String, Object>) oDeserializedTwitterResponse; aoStatusesThisPage = ( Object [] )oResponseDictionary["statuses"]; } catch (Exception oException) { // Rethrow the exception if appropriate. OnExceptionWhileEnumeratingJsonValues(oException, iPage, false); // Otherwise, just halt the enumeration. yield break; } Int32 iObjectsThisPage = aoStatusesThisPage.Length; if (iObjectsThisPage == 0) { yield break; } for (Int32 i = 0; i < iObjectsThisPage; i++) { yield return( (Dictionary <String, Object>)aoStatusesThisPage[i]); iStatusesEnumerated++; if (iStatusesEnumerated == maximumStatuses) { yield break; } } iPage++; if (!TryGetQueryParametersForNextSearchPage(oResponseDictionary, out sQueryParametersForNextPage)) { yield break; } // Get the next page... } }
GetTwitterResponseAsString ( String url, RequestStatistics requestStatistics, ReportProgressHandler reportProgressHandler, CheckCancellationPendingHandler checkCancellationPendingHandler ) { Debug.Assert(!String.IsNullOrEmpty(url)); Debug.Assert(requestStatistics != null); AssertValid(); Int32 iRateLimitPauses = 0; Int32 iInvalidJsonRepeats = 0; const Int32 MaximumInvalidJsonRepeats = 3; while (true) { // Add the required authorization information to the URL. // // Note: Don't do this outside the while (true) loop. The // authorization information includes a timestamp that will // probably expire if the code within the catch block pauses. oAuthTwitter oAuthTwitter = new oAuthTwitter( m_UserAgent, m_TimeoutMs); oAuthTwitter.Token = m_TwitterAccessToken; oAuthTwitter.TokenSecret = m_TwitterAccessTokenSecret; String sAuthorizedUrl, sAuthorizedPostData; oAuthTwitter.ConstructAuthWebRequest(oAuthTwitter.Method.GET, url, String.Empty, out sAuthorizedUrl, out sAuthorizedPostData); url = sAuthorizedUrl; Stream oStream = null; try { oStream = HttpSocialNetworkUtil.GetHttpWebResponseStreamWithRetries( url, HttpStatusCodesToFailImmediately, requestStatistics, m_UserAgent, m_TimeoutMs, reportProgressHandler, checkCancellationPendingHandler); return(GetTwitterResponseAsString(oStream)); } catch (InvalidJsonException oInvalidJsonException) { iInvalidJsonRepeats++; if (iInvalidJsonRepeats > MaximumInvalidJsonRepeats) { throw oInvalidJsonException; } HttpSocialNetworkUtil.ReportProgress(reportProgressHandler, "Received invalid JSON from Twitter. Trying again." ); } catch (WebException oWebException) { if ( !WebExceptionIsDueToRateLimit(oWebException) || iRateLimitPauses > 0 ) { throw; } // Twitter rate limits have kicked in. Pause and try again. iRateLimitPauses++; Int32 iRateLimitPauseMs = GetRateLimitPauseMs(oWebException); DateTime oWakeUpTime = DateTime.Now.AddMilliseconds( iRateLimitPauseMs); HttpSocialNetworkUtil.ReportProgress(reportProgressHandler, "Reached Twitter rate limits. Pausing until {0}." , oWakeUpTime.ToLongTimeString() ); // Don't pause in one large interval, which would prevent // cancellation. const Int32 SleepCycleDurationMs = 1000; Int32 iSleepCycles = (Int32)Math.Ceiling( (Double)iRateLimitPauseMs / SleepCycleDurationMs); for (Int32 i = 0; i < iSleepCycles; i++) { if (checkCancellationPendingHandler != null) { checkCancellationPendingHandler(); } System.Threading.Thread.Sleep(SleepCycleDurationMs); } } finally { if (oStream != null) { oStream.Close(); } } } }
EnumerateJsonValues ( String url, String jsonName, Int32 maximumValues, Boolean skipMostPage1Errors, RequestStatistics requestStatistics, ReportProgressHandler reportProgressHandler, CheckCancellationPendingHandler checkCancellationPendingHandler ) { // Note: // // The logic in this method is similar to the logic in // EnumerateSearchStatuses(). In fact, at one time all enumeration was // done through this EnumerateJsonValues() method. // EnumerateSearchStatuses() was created only when version 1.1 of the // Twitter API introduced yet another paging scheme, one that differs // from the cursor scheme that this method handles. // // A possible work item is to recombine the two methods into one, // possibly by using a delegate to handle the different paging schemes. Debug.Assert(!String.IsNullOrEmpty(url)); Debug.Assert(maximumValues > 0); Debug.Assert(requestStatistics != null); AssertValid(); Int32 iPage = 1; String sCursor = null; Int32 iObjectsEnumerated = 0; while (true) { if (iPage > 1) { HttpSocialNetworkUtil.ReportProgress(reportProgressHandler, "Getting page {0}." , iPage ); } String sUrlWithCursor = AppendCursorToUrl(url, sCursor); Dictionary <String, Object> oValueDictionary = null; Object [] aoObjectsThisPage; try { Object oDeserializedTwitterResponse = (new JavaScriptSerializer()).DeserializeObject( GetTwitterResponseAsString(sUrlWithCursor, requestStatistics, reportProgressHandler, checkCancellationPendingHandler)); Object oObjectsThisPageAsObject; if (jsonName == null) { // The top level of the Json response contains an array of // objects this method will enumerate. oObjectsThisPageAsObject = oDeserializedTwitterResponse; } else { // The top level of the Json response contains a set of // name/value pairs. The value for the specified name is // the array of objects this method will enumerate. oValueDictionary = (Dictionary <String, Object>) oDeserializedTwitterResponse; oObjectsThisPageAsObject = oValueDictionary[jsonName]; } aoObjectsThisPage = ( Object [] )oObjectsThisPageAsObject; } catch (Exception oException) { // Rethrow the exception if appropriate. TwitterUtil.OnExceptionWhileEnumeratingJsonValues( oException, iPage, skipMostPage1Errors); // Otherwise, just halt the enumeration. yield break; } Int32 iObjectsThisPage = aoObjectsThisPage.Length; if (iObjectsThisPage == 0) { yield break; } for (Int32 i = 0; i < iObjectsThisPage; i++) { yield return(aoObjectsThisPage[i]); iObjectsEnumerated++; if (iObjectsEnumerated == maximumValues) { yield break; } } iPage++; // When the top level of the Json response contains a set of // name/value pairs, a next_cursor_str value of "0" means "end of // data." if ( oValueDictionary == null || !TwitterJsonUtil.TryGetJsonValueFromDictionary( oValueDictionary, "next_cursor_str", out sCursor) || sCursor == "0" ) { yield break; } // Get the next page... } }
EnumerateUserValueDictionaries ( String [] userIDsOrScreenNames, Boolean userIDsSpecified, RequestStatistics requestStatistics, ReportProgressHandler reportProgressHandler, CheckCancellationPendingHandler checkCancellationPendingHandler ) { Debug.Assert(userIDsOrScreenNames != null); Debug.Assert(requestStatistics != null); AssertValid(); // We'll use Twitter's users/lookup API, which gets extended // information for up to 100 users in one call. Int32 iUsers = userIDsOrScreenNames.Length; Int32 iUsersProcessed = 0; Int32 iCalls = 0; while (iUsersProcessed < iUsers) { // For each call, ask for information about as many users as // possible until either 100 is reached or the URL reaches an // arbitrary maximum length. Twitter recommends using a POST here // (without specifying why), but it would require revising the // base-class HTTP calls and isn't worth the trouble. Int32 iUsersProcessedThisCall = 0; StringBuilder oUrl = new StringBuilder(); oUrl.AppendFormat( "{0}users/lookup.json?{1}&{2}=" , TwitterApiUrls.Rest, TwitterApiUrlParameters.IncludeEntities, userIDsSpecified ? "user_id" : "screen_name" ); const Int32 MaxUsersPerCall = 100; const Int32 MaxUrlLength = 2000; // Construct the URL for this call. while ( iUsersProcessed < iUsers && iUsersProcessedThisCall < MaxUsersPerCall && oUrl.Length < MaxUrlLength ) { if (iUsersProcessedThisCall > 0) { // Append an encoded comma. Using an unencoded comma // causes Twitter to return a 401 "unauthorized" error. // // See this post for an explanation: // // https://dev.twitter.com/discussions/11399 oUrl.Append("%2C"); } oUrl.Append(userIDsOrScreenNames[iUsersProcessed]); iUsersProcessed++; iUsersProcessedThisCall++; } iCalls++; if (iCalls > 1) { HttpSocialNetworkUtil.ReportProgress(reportProgressHandler, "Getting page {0}." , iCalls ); } foreach (Object oResult in EnumerateJsonValues(oUrl.ToString(), null, Int32.MaxValue, true, requestStatistics, reportProgressHandler, checkCancellationPendingHandler)) { yield return((Dictionary <String, Object>)oResult); } } }
EnumerateXmlNodes ( String sUrl, String sXPath, Int32 iMaximumXmlNodes, Boolean bSkipMostPage1Errors, RequestStatistics oRequestStatistics ) { Debug.Assert(!String.IsNullOrEmpty(sUrl)); Debug.Assert(!String.IsNullOrEmpty(sXPath)); Debug.Assert(iMaximumXmlNodes > 0); Debug.Assert(oRequestStatistics != null); AssertValid(); // A maximum per-page value of 500 works for // flickr.contacts.getPublicList (maximum per_page = 1000) and // flickr.people.getPublicPhotos (maximum per_page = 500). Debug.Assert(sUrl.IndexOf("flickr.contacts.getPublicList") >= 0 || sUrl.IndexOf("flickr.people.getPublicPhotos") >= 0); Int32 iMaximumPerPage = Math.Min(500, iMaximumXmlNodes); Int32 iPage = 1; Int32 iXmlNodesEnumerated = 0; while (true) { String sUrlWithPagination = String.Format( "{0}{1}per_page={2}&page={3}" , sUrl, sUrl.IndexOf('?') == -1 ? '?' : '&', iMaximumPerPage, iPage ); XmlDocument oXmlDocument; try { oXmlDocument = GetXmlDocument(sUrlWithPagination, oRequestStatistics); } catch (Exception oException) { if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException)) { throw oException; } if (iPage > 1 || bSkipMostPage1Errors) { // Always skip errors on page 2 and above. yield break; } throw (oException); } XmlNodeList oXmlNodesThisPage = oXmlDocument.SelectNodes(sXPath, null); Int32 iXmlNodesThisPage = oXmlNodesThisPage.Count; if (iXmlNodesThisPage == 0) { yield break; } for (Int32 i = 0; i < iXmlNodesThisPage; i++) { yield return(oXmlNodesThisPage[i]); iXmlNodesEnumerated++; if (iXmlNodesEnumerated == iMaximumXmlNodes) { yield break; } } iPage++; // Get the next page... } }