TryGetXmlDocument
    (
        String sUrl,
        RequestStatistics oRequestStatistics,
        out XmlDocument oXmlDocument,
        out XmlNamespaceManager oXmlNamespaceManager
    )
    {
        Debug.Assert( !String.IsNullOrEmpty(sUrl) );
        Debug.Assert(oRequestStatistics != null);
        AssertValid();

        oXmlDocument = null;
        oXmlNamespaceManager = null;

        try
        {
            oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics,
                out oXmlNamespaceManager);

            return (true);
        }
        catch (Exception oException)
        {
            if ( !HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException))
            {
                throw oException;
            }

            return (false);
        }
    }
Example #2
0
        TryGetXmlDocument
        (
            String sUrl,
            RequestStatistics oRequestStatistics,
            out XmlDocument oXmlDocument
        )
        {
            Debug.Assert(!String.IsNullOrEmpty(sUrl));
            Debug.Assert(oRequestStatistics != null);
            AssertValid();

            oXmlDocument = null;

            try
            {
                oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics);

                return(true);
            }
            catch (Exception oException)
            {
                // Note: Because a FlickrException is also a WebException, the
                // following logic will cause false to be returned for a
                // FlickrException.

                if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException))
                {
                    throw oException;
                }

                return(false);
            }
        }
Example #3
0
        WebExceptionIsDueToRateLimit
        (
            WebException webException
        )
        {
            Debug.Assert(webException != null);

            // Starting with version 1.1 of the Twitter API, a single HTTP status
            // code (429, "rate limit exceeded") is used for all rate-limit
            // responses.

            return(HttpSocialNetworkUtil.WebExceptionHasHttpStatusCode(
                       webException, (HttpStatusCode)429));
        }
        TryGetUserValueDictionary
        (
            String sScreenName,
            RequestStatistics oRequestStatistics,
            Boolean bIgnoreWebAndJsonExceptions,
            out Dictionary <String, Object> oUserValueDictionary
        )
        {
            Debug.Assert(!String.IsNullOrEmpty(sScreenName));
            Debug.Assert(oRequestStatistics != null);
            AssertValid();

            oUserValueDictionary = null;

            String sUrl = String.Format(

                "{0}users/show.json?screen_name={1}&{2}"
                ,
                TwitterApiUrls.Rest,
                TwitterUtil.EncodeUrlParameter(sScreenName),
                TwitterApiUrlParameters.IncludeEntities
                );

            ReportProgress(String.Format(

                               "Getting information about \"{0}\"."
                               ,
                               sScreenName
                               ));

            try
            {
                oUserValueDictionary = (Dictionary <String, Object>)
                                           (new JavaScriptSerializer()).DeserializeObject(
                    GetTwitterResponseAsString(sUrl, oRequestStatistics));

                return(true);
            }
            catch (Exception oException)
            {
                if (!HttpSocialNetworkUtil.ExceptionIsWebOrJson(oException) ||
                    !bIgnoreWebAndJsonExceptions)
                {
                    throw oException;
                }

                return(false);
            }
        }
Example #5
0
        OnExceptionWhileEnumeratingJsonValues
        (
            Exception exception,
            Int32 iPage,
            Boolean bSkipMostPage1Errors
        )
        {
            Debug.Assert(exception != null);
            Debug.Assert(iPage > 0);

            if (!HttpSocialNetworkUtil.ExceptionIsWebOrJson(exception))
            {
                // This is an unknown exception.

                throw (exception);
            }

            if (
                exception is WebException
                &&
                WebExceptionIsDueToRateLimit((WebException)exception)
                )
            {
                throw (exception);
            }
            else if (iPage == 1)
            {
                if (bSkipMostPage1Errors)
                {
                    return;
                }
                else
                {
                    throw (exception);
                }
            }
            else
            {
                // Always skip non-rate-limit errors on page 2 and above.

                return;
            }
        }
Example #6
0
        /// <summary>
        /// Web Request Wrapper
        /// </summary>
        /// <param name="method">Http Method</param>
        /// <param name="url">Full url to the web resource</param>
        /// <param name="postData">Data to post in querystring format</param>
        /// <returns>The web server response.</returns>
        public string WebRequest(Method method, string url, string postData)
        {
            HttpWebRequest webRequest    = null;
            StreamWriter   requestWriter = null;
            string         responseData  = "";

            webRequest = HttpSocialNetworkUtil.CreateHttpWebRequest(
                url, _userAgent, _timeoutMs);

            webRequest.Method = method.ToString();
            webRequest.ServicePoint.Expect100Continue = false;

            if (method == Method.POST)
            {
                webRequest.ContentType = "application/x-www-form-urlencoded";

                //POST the data.
                requestWriter = new StreamWriter(webRequest.GetRequestStream());
                try
                {
                    requestWriter.Write(postData);
                }
                catch
                {
                    throw;
                }
                finally
                {
                    requestWriter.Close();
                    requestWriter = null;
                }
            }

            responseData = WebResponseGet(webRequest);

            webRequest = null;

            return(responseData);
        }
Example #7
0
        GetXmlDocumentWithRetries
        (
            String sUrl,
            HttpStatusCode [] aeHttpStatusCodesToFailImmediately,
            RequestStatistics oRequestStatistics
        )
        {
            Debug.Assert(!String.IsNullOrEmpty(sUrl));
            Debug.Assert(oRequestStatistics != null);

            AssertValid();

            Stream oStream = null;

            try
            {
                oStream =
                    HttpSocialNetworkUtil.GetHttpWebResponseStreamWithRetries(
                        sUrl, aeHttpStatusCodesToFailImmediately,
                        oRequestStatistics, UserAgent, HttpWebRequestTimeoutMs,
                        new ReportProgressHandler(this.ReportProgress),

                        new CheckCancellationPendingHandler(
                            this.CheckCancellationPending)
                        );

                XmlDocument oXmlDocument = new XmlDocument();
                oXmlDocument.Load(oStream);

                return(oXmlDocument);
            }
            finally
            {
                if (oStream != null)
                {
                    oStream.Close();
                }
            }
        }
        GetRelatedTagsRecursive
        (
            String sTag,
            WhatToInclude eWhatToInclude,
            NetworkLevel eNetworkLevel,
            String sApiKey,
            Int32 iRecursionLevel,
            GraphMLXmlDocument oGraphMLXmlDocument,
            Dictionary <String, XmlNode> oTagDictionary,
            RequestStatistics oRequestStatistics
        )
        {
            Debug.Assert(!String.IsNullOrEmpty(sTag));

            Debug.Assert(eNetworkLevel == NetworkLevel.One ||
                         eNetworkLevel == NetworkLevel.OnePointFive ||
                         eNetworkLevel == NetworkLevel.Two);

            Debug.Assert(!String.IsNullOrEmpty(sApiKey));
            Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2);
            Debug.Assert(oGraphMLXmlDocument != null);
            Debug.Assert(oTagDictionary != null);
            Debug.Assert(oRequestStatistics != null);
            AssertValid();

            /*
             * Here is what this method should do, based on the eNetworkLevel and
             * iRecursionLevel parameters.
             *
             *      eNetworkLevel
             *
             |One               | OnePointFive      | Two
             *  ---|------------------| ------------------| -----------------
             * i   1  |Add all vertices. | Add all vertices. | Add all vertices.
             * R      |                  |                   |
             * e      |Add all edges.    | Add all edges.    | Add all edges.
             * c      |                  |                   |
             * u      |Do not recurse.   | Recurse.          | Recurse.
             * r      |                  |                   |
             * s   ---|------------------|-------------------|------------------
             * i   2  |Impossible.       | Do not add        | Add all vertices.
             * o      |                  | vertices.         |
             * n      |                  |                   |
             * L      |                  | Add edges only if | Add all edges.
             * e      |                  | vertices are      |
             * v      |                  | already included. |
             * e      |                  |                   |
             * l      |                  | Do not recurse.   | Do not recurse.
             |                  |                   |
             |  ---|------------------|-------------------|------------------
             */

            Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel,
                                                      iRecursionLevel);

            Boolean bNeedToAppendVertices = GetNeedToAppendVertices(
                eNetworkLevel, iRecursionLevel);

            ReportProgress("Getting tags related to \"" + sTag + "\".");

            String sUrl = GetFlickrMethodUrl("flickr.tags.getRelated", sApiKey,
                                             "&tag=" + UrlUtil.EncodeUrlParameter(sTag));

            XmlDocument oXmlDocument;

            try
            {
                oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics);
            }
            catch (Exception oException)
            {
                // If the exception is not a WebException or XmlException, or if
                // none of the network has been obtained yet, throw the exception.

                if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException) ||
                    !oGraphMLXmlDocument.HasVertexXmlNode)
                {
                    throw oException;
                }

                return;
            }

            // The document consists of a single "tags" node with zero or more
            // "tag" child nodes.

            String sOtherTag = null;

            XmlNodeList oTagNodes = oXmlDocument.DocumentElement.SelectNodes(
                "tags/tag");

            if (oTagNodes.Count > 0)
            {
                AppendVertexXmlNode(sTag, oGraphMLXmlDocument, oTagDictionary);
            }

            foreach (XmlNode oTagNode in oTagNodes)
            {
                sOtherTag = XmlUtil2.SelectRequiredSingleNodeAsString(oTagNode,
                                                                      "text()", null);

                if (bNeedToAppendVertices)
                {
                    AppendVertexXmlNode(sOtherTag, oGraphMLXmlDocument,
                                        oTagDictionary);
                }

                if (bNeedToAppendVertices ||
                    oTagDictionary.ContainsKey(sOtherTag))
                {
                    oGraphMLXmlDocument.AppendEdgeXmlNode(sTag, sOtherTag);
                }
            }

            if (bNeedToRecurse)
            {
                foreach (XmlNode oTagNode in oTagNodes)
                {
                    sOtherTag = XmlUtil2.SelectRequiredSingleNodeAsString(oTagNode,
                                                                          "text()", null);

                    GetRelatedTagsRecursive(sOtherTag, eWhatToInclude,
                                            eNetworkLevel, sApiKey, 2, oGraphMLXmlDocument,
                                            oTagDictionary, oRequestStatistics);
                }
            }
        }
    EnumerateXmlNodes
    (
        String sUrl,
        String sXPath,
        Int32 iMaximumXmlNodes,
        Boolean bSkipMostPage1Errors,
        RequestStatistics oRequestStatistics
    )
    {
        Debug.Assert( !String.IsNullOrEmpty(sUrl) );
        Debug.Assert( !String.IsNullOrEmpty(sXPath) );
        Debug.Assert(iMaximumXmlNodes > 0);
        Debug.Assert(oRequestStatistics != null);

        AssertValid();

        Int32 iStartIndex = 1;
        Int32 iMaxResults = 50;
        Int32 iXmlNodesEnumerated = 0;

        while (true)
        {
            if (iStartIndex > 1)
            {
                ReportProgress("Getting page starting with item "
                    + iStartIndex + ".");
            }

            String sUrlWithPagination = String.Format(
            
                "{0}{1}start-index={2}&max-results={3}"
                ,
                sUrl,
                sUrl.IndexOf('?') == -1 ? '?' : '&',
                iStartIndex,
                iMaxResults
                );

            XmlDocument oXmlDocument;
            XmlNamespaceManager oXmlNamespaceManager;

            try
            {
                oXmlDocument = GetXmlDocument(sUrlWithPagination,
                    oRequestStatistics, out oXmlNamespaceManager);
            }
            catch (Exception oException)
            {
                if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException))
                {
                    throw oException;
                }

                if (iStartIndex > 1 || bSkipMostPage1Errors)
                {
                    // Always skip errors on page 2 and above.

                    yield break;
                }

                throw (oException);
            }

            XmlNodeList oXmlNodesThisPage = oXmlDocument.SelectNodes(sXPath,
                oXmlNamespaceManager);

            Int32 iXmlNodesThisPage = oXmlNodesThisPage.Count;

            if (iXmlNodesThisPage == 0)
            {
                yield break;
            }

            for (Int32 i = 0; i < iXmlNodesThisPage; i++)
            {
                yield return ( oXmlNodesThisPage[i] );

                iXmlNodesEnumerated++;

                if (iXmlNodesEnumerated == iMaximumXmlNodes)
                {
                    yield break;
                }
            }

            // The next page, if there is one, is obtained from a link tag that
            // looks something like this:
            //
            // <link rel="next" ... href="http://gdata.youtube.com/feeds/...?
            // start-index=26&max-results=25"/>

            String sHRef;

            if ( !XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
                "a:feed/a:link[@rel='next']/@href", oXmlNamespaceManager,
                out sHRef) )
            {
                yield break;
            }

            const String Pattern = @"start-index=(?<NextStartIndex>\d+)";
            Regex oRegex = new Regex(Pattern);
            Match oMatch = oRegex.Match(sHRef);

            if (!oMatch.Success
                ||
                !MathUtil.TryParseCultureInvariantInt32(
                    oMatch.Groups["NextStartIndex"].Value, out iStartIndex)
                )
            {
                yield break;
            }

            // Get the next page...
        }
    }
Example #10
0
        EnumerateSearchStatuses
        (
            String searchTerm,
            Int32 maximumStatuses,
            RequestStatistics requestStatistics,
            ReportProgressHandler reportProgressHandler,
            CheckCancellationPendingHandler checkCancellationPendingHandler
        )
        {
            Debug.Assert(!String.IsNullOrEmpty(searchTerm));
            Debug.Assert(maximumStatuses > 0);
            Debug.Assert(maximumStatuses != Int32.MaxValue);
            Debug.Assert(requestStatistics != null);
            AssertValid();

            Int32  iPage = 1;
            Int32  iStatusesEnumerated         = 0;
            String sQueryParametersForNextPage = null;

            while (true)
            {
                if (iPage > 1)
                {
                    HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,

                                                         "Getting page {0}."
                                                         ,
                                                         iPage
                                                         );
                }

                Dictionary <String, Object> oResponseDictionary = null;
                Object [] aoStatusesThisPage;

                String sUrl = GetSearchUrl(searchTerm,
                                           sQueryParametersForNextPage);

                try
                {
                    Object oDeserializedTwitterResponse =
                        (new JavaScriptSerializer()).DeserializeObject(
                            GetTwitterResponseAsString(
                                sUrl, requestStatistics, reportProgressHandler,
                                checkCancellationPendingHandler));

                    // The top level of the Json response contains a set of
                    // name/value pairs.  The value for the "statuses" name is the
                    // array of objects this method will enumerate.

                    oResponseDictionary = (Dictionary <String, Object>)
                                          oDeserializedTwitterResponse;

                    aoStatusesThisPage =
                        ( Object [] )oResponseDictionary["statuses"];
                }
                catch (Exception oException)
                {
                    // Rethrow the exception if appropriate.

                    OnExceptionWhileEnumeratingJsonValues(oException, iPage,
                                                          false);

                    // Otherwise, just halt the enumeration.

                    yield break;
                }

                Int32 iObjectsThisPage = aoStatusesThisPage.Length;

                if (iObjectsThisPage == 0)
                {
                    yield break;
                }

                for (Int32 i = 0; i < iObjectsThisPage; i++)
                {
                    yield return(
                        (Dictionary <String, Object>)aoStatusesThisPage[i]);

                    iStatusesEnumerated++;

                    if (iStatusesEnumerated == maximumStatuses)
                    {
                        yield break;
                    }
                }

                iPage++;

                if (!TryGetQueryParametersForNextSearchPage(oResponseDictionary,
                                                            out sQueryParametersForNextPage))
                {
                    yield break;
                }

                // Get the next page...
            }
        }
Example #11
0
        GetTwitterResponseAsString
        (
            String url,
            RequestStatistics requestStatistics,
            ReportProgressHandler reportProgressHandler,
            CheckCancellationPendingHandler checkCancellationPendingHandler
        )
        {
            Debug.Assert(!String.IsNullOrEmpty(url));
            Debug.Assert(requestStatistics != null);
            AssertValid();

            Int32       iRateLimitPauses          = 0;
            Int32       iInvalidJsonRepeats       = 0;
            const Int32 MaximumInvalidJsonRepeats = 3;

            while (true)
            {
                // Add the required authorization information to the URL.
                //
                // Note: Don't do this outside the while (true) loop.  The
                // authorization information includes a timestamp that will
                // probably expire if the code within the catch block pauses.

                oAuthTwitter oAuthTwitter = new oAuthTwitter(
                    m_UserAgent, m_TimeoutMs);

                oAuthTwitter.Token       = m_TwitterAccessToken;
                oAuthTwitter.TokenSecret = m_TwitterAccessTokenSecret;

                String sAuthorizedUrl, sAuthorizedPostData;

                oAuthTwitter.ConstructAuthWebRequest(oAuthTwitter.Method.GET,
                                                     url, String.Empty, out sAuthorizedUrl,
                                                     out sAuthorizedPostData);

                url = sAuthorizedUrl;

                Stream oStream = null;

                try
                {
                    oStream =
                        HttpSocialNetworkUtil.GetHttpWebResponseStreamWithRetries(
                            url, HttpStatusCodesToFailImmediately,
                            requestStatistics, m_UserAgent, m_TimeoutMs,
                            reportProgressHandler, checkCancellationPendingHandler);

                    return(GetTwitterResponseAsString(oStream));
                }
                catch (InvalidJsonException oInvalidJsonException)
                {
                    iInvalidJsonRepeats++;

                    if (iInvalidJsonRepeats > MaximumInvalidJsonRepeats)
                    {
                        throw oInvalidJsonException;
                    }

                    HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,

                                                         "Received invalid JSON from Twitter.  Trying again."
                                                         );
                }
                catch (WebException oWebException)
                {
                    if (
                        !WebExceptionIsDueToRateLimit(oWebException)
                        ||
                        iRateLimitPauses > 0
                        )
                    {
                        throw;
                    }

                    // Twitter rate limits have kicked in.  Pause and try again.

                    iRateLimitPauses++;
                    Int32 iRateLimitPauseMs = GetRateLimitPauseMs(oWebException);

                    DateTime oWakeUpTime = DateTime.Now.AddMilliseconds(
                        iRateLimitPauseMs);

                    HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,

                                                         "Reached Twitter rate limits.  Pausing until {0}."
                                                         ,
                                                         oWakeUpTime.ToLongTimeString()
                                                         );

                    // Don't pause in one large interval, which would prevent
                    // cancellation.

                    const Int32 SleepCycleDurationMs = 1000;

                    Int32 iSleepCycles = (Int32)Math.Ceiling(
                        (Double)iRateLimitPauseMs / SleepCycleDurationMs);

                    for (Int32 i = 0; i < iSleepCycles; i++)
                    {
                        if (checkCancellationPendingHandler != null)
                        {
                            checkCancellationPendingHandler();
                        }

                        System.Threading.Thread.Sleep(SleepCycleDurationMs);
                    }
                }
                finally
                {
                    if (oStream != null)
                    {
                        oStream.Close();
                    }
                }
            }
        }
Example #12
0
        EnumerateJsonValues
        (
            String url,
            String jsonName,
            Int32 maximumValues,
            Boolean skipMostPage1Errors,
            RequestStatistics requestStatistics,
            ReportProgressHandler reportProgressHandler,
            CheckCancellationPendingHandler checkCancellationPendingHandler
        )
        {
            // Note:
            //
            // The logic in this method is similar to the logic in
            // EnumerateSearchStatuses().  In fact, at one time all enumeration was
            // done through this EnumerateJsonValues() method.
            // EnumerateSearchStatuses() was created only when version 1.1 of the
            // Twitter API introduced yet another paging scheme, one that differs
            // from the cursor scheme that this method handles.
            //
            // A possible work item is to recombine the two methods into one,
            // possibly by using a delegate to handle the different paging schemes.

            Debug.Assert(!String.IsNullOrEmpty(url));
            Debug.Assert(maximumValues > 0);
            Debug.Assert(requestStatistics != null);
            AssertValid();

            Int32  iPage              = 1;
            String sCursor            = null;
            Int32  iObjectsEnumerated = 0;

            while (true)
            {
                if (iPage > 1)
                {
                    HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,

                                                         "Getting page {0}."
                                                         ,
                                                         iPage
                                                         );
                }

                String sUrlWithCursor = AppendCursorToUrl(url, sCursor);

                Dictionary <String, Object> oValueDictionary = null;
                Object [] aoObjectsThisPage;

                try
                {
                    Object oDeserializedTwitterResponse =
                        (new JavaScriptSerializer()).DeserializeObject(
                            GetTwitterResponseAsString(sUrlWithCursor,
                                                       requestStatistics, reportProgressHandler,
                                                       checkCancellationPendingHandler));

                    Object oObjectsThisPageAsObject;

                    if (jsonName == null)
                    {
                        // The top level of the Json response contains an array of
                        // objects this method will enumerate.

                        oObjectsThisPageAsObject = oDeserializedTwitterResponse;
                    }
                    else
                    {
                        // The top level of the Json response contains a set of
                        // name/value pairs.  The value for the specified name is
                        // the array of objects this method will enumerate.

                        oValueDictionary = (Dictionary <String, Object>)
                                           oDeserializedTwitterResponse;

                        oObjectsThisPageAsObject = oValueDictionary[jsonName];
                    }

                    aoObjectsThisPage = ( Object [] )oObjectsThisPageAsObject;
                }
                catch (Exception oException)
                {
                    // Rethrow the exception if appropriate.

                    TwitterUtil.OnExceptionWhileEnumeratingJsonValues(
                        oException, iPage, skipMostPage1Errors);

                    // Otherwise, just halt the enumeration.

                    yield break;
                }

                Int32 iObjectsThisPage = aoObjectsThisPage.Length;

                if (iObjectsThisPage == 0)
                {
                    yield break;
                }

                for (Int32 i = 0; i < iObjectsThisPage; i++)
                {
                    yield return(aoObjectsThisPage[i]);

                    iObjectsEnumerated++;

                    if (iObjectsEnumerated == maximumValues)
                    {
                        yield break;
                    }
                }

                iPage++;

                // When the top level of the Json response contains a set of
                // name/value pairs, a next_cursor_str value of "0" means "end of
                // data."

                if (
                    oValueDictionary == null
                    ||
                    !TwitterJsonUtil.TryGetJsonValueFromDictionary(
                        oValueDictionary, "next_cursor_str", out sCursor)
                    ||
                    sCursor == "0"
                    )
                {
                    yield break;
                }

                // Get the next page...
            }
        }
Example #13
0
        EnumerateUserValueDictionaries
        (
            String [] userIDsOrScreenNames,
            Boolean userIDsSpecified,
            RequestStatistics requestStatistics,
            ReportProgressHandler reportProgressHandler,
            CheckCancellationPendingHandler checkCancellationPendingHandler
        )
        {
            Debug.Assert(userIDsOrScreenNames != null);
            Debug.Assert(requestStatistics != null);
            AssertValid();

            // We'll use Twitter's users/lookup API, which gets extended
            // information for up to 100 users in one call.

            Int32 iUsers          = userIDsOrScreenNames.Length;
            Int32 iUsersProcessed = 0;
            Int32 iCalls          = 0;

            while (iUsersProcessed < iUsers)
            {
                // For each call, ask for information about as many users as
                // possible until either 100 is reached or the URL reaches an
                // arbitrary maximum length.  Twitter recommends using a POST here
                // (without specifying why), but it would require revising the
                // base-class HTTP calls and isn't worth the trouble.

                Int32 iUsersProcessedThisCall = 0;

                StringBuilder oUrl = new StringBuilder();

                oUrl.AppendFormat(
                    "{0}users/lookup.json?{1}&{2}="
                    ,
                    TwitterApiUrls.Rest,
                    TwitterApiUrlParameters.IncludeEntities,
                    userIDsSpecified ? "user_id" : "screen_name"
                    );

                const Int32 MaxUsersPerCall = 100;
                const Int32 MaxUrlLength    = 2000;

                // Construct the URL for this call.

                while (
                    iUsersProcessed < iUsers
                    &&
                    iUsersProcessedThisCall < MaxUsersPerCall
                    &&
                    oUrl.Length < MaxUrlLength
                    )
                {
                    if (iUsersProcessedThisCall > 0)
                    {
                        // Append an encoded comma.  Using an unencoded comma
                        // causes Twitter to return a 401 "unauthorized" error.
                        //
                        // See this post for an explanation:
                        //
                        // https://dev.twitter.com/discussions/11399

                        oUrl.Append("%2C");
                    }

                    oUrl.Append(userIDsOrScreenNames[iUsersProcessed]);
                    iUsersProcessed++;
                    iUsersProcessedThisCall++;
                }

                iCalls++;

                if (iCalls > 1)
                {
                    HttpSocialNetworkUtil.ReportProgress(reportProgressHandler,

                                                         "Getting page {0}."
                                                         ,
                                                         iCalls
                                                         );
                }

                foreach (Object oResult in EnumerateJsonValues(oUrl.ToString(),
                                                               null, Int32.MaxValue, true, requestStatistics,
                                                               reportProgressHandler, checkCancellationPendingHandler))
                {
                    yield return((Dictionary <String, Object>)oResult);
                }
            }
        }
Example #14
0
        EnumerateXmlNodes
        (
            String sUrl,
            String sXPath,
            Int32 iMaximumXmlNodes,
            Boolean bSkipMostPage1Errors,
            RequestStatistics oRequestStatistics
        )
        {
            Debug.Assert(!String.IsNullOrEmpty(sUrl));
            Debug.Assert(!String.IsNullOrEmpty(sXPath));
            Debug.Assert(iMaximumXmlNodes > 0);
            Debug.Assert(oRequestStatistics != null);

            AssertValid();

            // A maximum per-page value of 500 works for
            // flickr.contacts.getPublicList (maximum per_page = 1000) and
            // flickr.people.getPublicPhotos (maximum per_page = 500).

            Debug.Assert(sUrl.IndexOf("flickr.contacts.getPublicList") >= 0 ||
                         sUrl.IndexOf("flickr.people.getPublicPhotos") >= 0);

            Int32 iMaximumPerPage = Math.Min(500, iMaximumXmlNodes);

            Int32 iPage = 1;
            Int32 iXmlNodesEnumerated = 0;

            while (true)
            {
                String sUrlWithPagination = String.Format(

                    "{0}{1}per_page={2}&page={3}"
                    ,
                    sUrl,
                    sUrl.IndexOf('?') == -1 ? '?' : '&',
                    iMaximumPerPage,
                    iPage
                    );

                XmlDocument oXmlDocument;

                try
                {
                    oXmlDocument = GetXmlDocument(sUrlWithPagination,
                                                  oRequestStatistics);
                }
                catch (Exception oException)
                {
                    if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException))
                    {
                        throw oException;
                    }

                    if (iPage > 1 || bSkipMostPage1Errors)
                    {
                        // Always skip errors on page 2 and above.

                        yield break;
                    }

                    throw (oException);
                }

                XmlNodeList oXmlNodesThisPage = oXmlDocument.SelectNodes(sXPath,
                                                                         null);

                Int32 iXmlNodesThisPage = oXmlNodesThisPage.Count;

                if (iXmlNodesThisPage == 0)
                {
                    yield break;
                }

                for (Int32 i = 0; i < iXmlNodesThisPage; i++)
                {
                    yield return(oXmlNodesThisPage[i]);

                    iXmlNodesEnumerated++;

                    if (iXmlNodesEnumerated == iMaximumXmlNodes)
                    {
                        yield break;
                    }
                }

                iPage++;

                // Get the next page...
            }
        }