CreateSnaTitle ( String searchTerm, RequestStatistics requestStatistics ) { Debug.Assert(!String.IsNullOrEmpty(searchTerm)); Debug.Assert(requestStatistics != null); // Sample title, where searchTerm is "usfca": // // "usfca Twitter NodeXL SNA Map and Report for Saturday, // 05 April 2014 at 18:47 UTC" DateTime oStartTimeUtc = requestStatistics.StartTimeUtc; return(String.Format( "{0} Twitter NodeXL SNA Map and Report for {1} at {2} UTC" , searchTerm, oStartTimeUtc.ToString("dddd, dd MMMM yyyy"), oStartTimeUtc.ToString("HH:mm") )); }
TryGetXmlDocument ( String sUrl, RequestStatistics oRequestStatistics, out XmlDocument oXmlDocument, out XmlNamespaceManager oXmlNamespaceManager ) { Debug.Assert( !String.IsNullOrEmpty(sUrl) ); Debug.Assert(oRequestStatistics != null); AssertValid(); oXmlDocument = null; oXmlNamespaceManager = null; try { oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics, out oXmlNamespaceManager); return (true); } catch (Exception oException) { if ( !HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException)) { throw oException; } return (false); } }
GetVideoNetworkInternal ( String sSearchTerm, WhatToInclude eWhatToInclude, Int32 iMaximumVideos ) { Debug.Assert(!String.IsNullOrEmpty(sSearchTerm)); Debug.Assert(iMaximumVideos > 0); AssertValid(); GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument(); RequestStatistics oRequestStatistics = new RequestStatistics(); try { GetVideoNetworkInternal(sSearchTerm, eWhatToInclude, iMaximumVideos, oRequestStatistics, oGraphMLXmlDocument); } catch (Exception oException) { OnTerminatingException(oException); } OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument, oRequestStatistics); return(oGraphMLXmlDocument); }
AppendAllStatisticGraphMLAttributeValues ( GraphMLXmlDocument oGraphMLXmlDocument, Dictionary <String, XmlNode> oUserNameDictionary, RequestStatistics oRequestStatistics ) { Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oUserNameDictionary != null); Debug.Assert(oRequestStatistics != null); AssertValid(); foreach (KeyValuePair <String, XmlNode> oKeyValuePair in oUserNameDictionary) { String sUserName = oKeyValuePair.Key; ReportProgress(String.Format( "Getting statistics for \"{0}\"." , sUserName )); AppendAllStatisticGraphMLAttributeValues(sUserName, oKeyValuePair.Value, oGraphMLXmlDocument, oRequestStatistics); } }
AppendSampleThumbnails ( Dictionary <String, XmlNode> oTagDictionary, GraphMLXmlDocument oGraphMLXmlDocument, String sApiKey, RequestStatistics oRequestStatistics ) { Debug.Assert(oTagDictionary != null); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(!String.IsNullOrEmpty(sApiKey)); Debug.Assert(oRequestStatistics != null); AssertValid(); foreach (KeyValuePair <String, XmlNode> oKeyValuePair in oTagDictionary) { String sTag = oKeyValuePair.Key; ReportProgress("Getting sample image file for \"" + sTag + "\"."); String sSampleImageUrl; if (TryGetSampleImageUrl(sTag, sApiKey, oRequestStatistics, out sSampleImageUrl)) { oGraphMLXmlDocument.AppendGraphMLAttributeValue( oKeyValuePair.Value, NodeXLGraphMLUtil.VertexImageFileID, sSampleImageUrl); } } }
AppendUserInformationGraphMLAttributeValues ( GraphMLXmlDocument oGraphMLXmlDocument, Dictionary <String, XmlNode> oUserIDDictionary, String sApiKey, RequestStatistics oRequestStatistics ) { Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oUserIDDictionary != null); Debug.Assert(!String.IsNullOrEmpty(sApiKey)); Debug.Assert(oRequestStatistics != null); AssertValid(); foreach (KeyValuePair <String, XmlNode> oKeyValuePair in oUserIDDictionary) { String sUserID = oKeyValuePair.Key; XmlNode oVertexXmlNode = oKeyValuePair.Value; ReportProgress(String.Format( "Getting information about \"{0}\"." , GetScreenNameFromVertexXmlNode(oVertexXmlNode) )); AppendUserInformationGraphMLAttributeValues(sUserID, oVertexXmlNode, oGraphMLXmlDocument, sApiKey, oRequestStatistics); } }
FlickrScreenNameToUserID ( String sScreenNameAnyCase, String sApiKey, RequestStatistics oRequestStatistics, out String sUserID, out String sScreenNameCorrectCase ) { Debug.Assert(!String.IsNullOrEmpty(sScreenNameAnyCase)); Debug.Assert(!String.IsNullOrEmpty(sApiKey)); Debug.Assert(oRequestStatistics != null); AssertValid(); XmlDocument oXmlDocument = GetXmlDocument( GetFlickrMethodUrl("flickr.people.findByUsername", sApiKey, "&username="******"rsp/user/@nsid", null); sScreenNameCorrectCase = XmlUtil2.SelectRequiredSingleNodeAsString( oXmlDocument, "rsp/user/username/text()", null); }
TryGetXmlDocument ( String sUrl, RequestStatistics oRequestStatistics, out XmlDocument oXmlDocument ) { Debug.Assert(!String.IsNullOrEmpty(sUrl)); Debug.Assert(oRequestStatistics != null); AssertValid(); oXmlDocument = null; try { oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics); return(true); } catch (Exception oException) { // Note: Because a FlickrException is also a WebException, the // following logic will cause false to be returned for a // FlickrException. if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException)) { throw oException; } return(false); } }
GetXmlDocument ( String sUrl, RequestStatistics oRequestStatistics, out XmlNamespaceManager oXmlNamespaceManager ) { Debug.Assert( !String.IsNullOrEmpty(sUrl) ); Debug.Assert(oRequestStatistics != null); AssertValid(); // Always request API version 2. String sUrlWithVersion = String.Format( "{0}{1}v=2" , sUrl, sUrl.IndexOf('?') == -1 ? '?' : '&' ); XmlDocument oXmlDocument = GetXmlDocumentWithRetries(sUrlWithVersion, HttpStatusCodesToFailImmediately, oRequestStatistics); oXmlNamespaceManager = CreateXmlNamespaceManager(oXmlDocument); return (oXmlDocument); }
AppendFriendOrFollowerEdgeXmlNode ( String sScreenName1, String sScreenName2, Boolean bAppendFriendEdgeXmlNode, GraphMLXmlDocument oGraphMLXmlDocument, RequestStatistics oRequestStatistics ) { Debug.Assert(!String.IsNullOrEmpty(sScreenName1)); Debug.Assert(!String.IsNullOrEmpty(sScreenName2)); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oRequestStatistics != null); AssertValid(); // Don't use "friend" or "follower" terminology here, which can be // confusing. Instead, simply indicate who follows whom. XmlNode oEdgeXmlNode = NodeXLGraphMLUtil.AppendEdgeXmlNode( oGraphMLXmlDocument, bAppendFriendEdgeXmlNode ? sScreenName1 : sScreenName2, bAppendFriendEdgeXmlNode ? sScreenName2 : sScreenName1, "Follows" ); AppendStartTimeRelationshipDateUtcGraphMLAttributeValue( oGraphMLXmlDocument, oEdgeXmlNode, oRequestStatistics); }
AppendInt32GraphMLAttributeValue ( String sUrl, String sXPath, GraphMLXmlDocument oGraphMLXmlDocument, RequestStatistics oRequestStatistics, XmlNode oVertexXmlNode, String sGraphMLAttributeID ) { Debug.Assert(!String.IsNullOrEmpty(sUrl)); Debug.Assert(!String.IsNullOrEmpty(sXPath)); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oRequestStatistics != null); Debug.Assert(oVertexXmlNode != null); Debug.Assert(!String.IsNullOrEmpty(sGraphMLAttributeID)); AssertValid(); XmlDocument oXmlDocument; XmlNamespaceManager oXmlNamespaceManager; if (!TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument, out oXmlNamespaceManager)) { return; } AppendInt32GraphMLAttributeValue(oXmlDocument, sXPath, oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, sGraphMLAttributeID); }
GetNetworkInternal ( StatusCriteria oStatusCriteria, Boolean bExpandStatusUrls, String sGraphServerUserName, String sGraphServerPassword, RequestStatistics oRequestStatistics ) { Debug.Assert(oStatusCriteria != null); Debug.Assert(!String.IsNullOrEmpty(sGraphServerUserName)); Debug.Assert(!String.IsNullOrEmpty(sGraphServerPassword)); Debug.Assert(oRequestStatistics != null); AssertValid(); GraphServiceClient oClient = new GraphServiceClient( GetWcfServiceBinding(), new EndpointAddress(GraphServiceUrl)); Byte [] abtZippedGraphML; // There are two ways to get the network: With a maximum status date, // and with a maximum number of statuses. if (oStatusCriteria.HasDateRange) { abtZippedGraphML = oClient.GetTwitterSearchNetworkAsZippedGraphML( oStatusCriteria.SearchTerm, oStatusCriteria.MinimumStatusDateUtc, oStatusCriteria.MaximumStatusDateUtc, bExpandStatusUrls, sGraphServerUserName, sGraphServerPassword ); } else { abtZippedGraphML = oClient.GetTwitterSearchNetworkAsZippedGraphML2( oStatusCriteria.SearchTerm, oStatusCriteria.MaximumStatusDateUtc, oStatusCriteria.MaximumStatusesGoingBackward, bExpandStatusUrls, sGraphServerUserName, sGraphServerPassword ); } String sGraphML = ZipUtil.UnzipOneTextFile(abtZippedGraphML); abtZippedGraphML = null; XmlDocument oXmlDocument = new XmlDocument(); // Note: When the DotNetZip library used by ZipUtil unzips the GraphML, // it includes a BOM as the first character. Remove that character. oXmlDocument.LoadXml(sGraphML.Substring(1)); return(oXmlDocument); }
GetCommentersEnumerator ( String sUserID, Int32 iMaximumPerRequest, Boolean bSkipMostPage1Errors, String sApiKey, RequestStatistics oRequestStatistics ) { Debug.Assert(!String.IsNullOrEmpty(sUserID)); Debug.Assert(iMaximumPerRequest > 0); Debug.Assert(!String.IsNullOrEmpty(sApiKey)); Debug.Assert(oRequestStatistics != null); AssertValid(); // Get the user's public photos, which are paged. String sUrl = GetFlickrMethodUrl("flickr.people.getPublicPhotos", sApiKey, GetUserIDUrlParameter(sUserID)); foreach (XmlNode oPhotoXmlNode in EnumerateXmlNodes(sUrl, "rsp/photos/photo", iMaximumPerRequest, bSkipMostPage1Errors, oRequestStatistics)) { String sPhotoID; if (!XmlUtil2.TrySelectSingleNodeAsString(oPhotoXmlNode, "@id", null, out sPhotoID)) { continue; } // Get the photo's comments, which are not paged. ReportProgress(String.Format( "Getting comments for the photo \"{0}\"." , sPhotoID )); sUrl = GetFlickrMethodUrl("flickr.photos.comments.getList", sApiKey, "&photo_id=" + UrlUtil.EncodeUrlParameter(sPhotoID) ); XmlDocument oXmlDocument; if (TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument)) { foreach (XmlNode oCommentXmlNode in oXmlDocument.SelectNodes( "rsp/comments/comment")) { yield return(oCommentXmlNode); } } } }
GetVideoNetworkInternal ( String sSearchTerm, WhatToInclude eWhatToInclude, Int32 iMaximumVideos, RequestStatistics oRequestStatistics, GraphMLXmlDocument oGraphMLXmlDocument ) { Debug.Assert(!String.IsNullOrEmpty(sSearchTerm)); Debug.Assert(iMaximumVideos > 0); Debug.Assert(oRequestStatistics != null); Debug.Assert(oGraphMLXmlDocument != null); AssertValid(); // First, add a vertex for each video matching the search term. HashSet <String> oVideoIDs; Dictionary <String, LinkedList <String> > oCategoryDictionary; AppendVertexXmlNodes(sSearchTerm, eWhatToInclude, iMaximumVideos, oGraphMLXmlDocument, oRequestStatistics, out oVideoIDs, out oCategoryDictionary); // Now add whatever edges were requested. if (WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SharedCategoryEdges)) { Debug.Assert(oCategoryDictionary != null); ReportProgress("Adding edges for shared categories."); AppendEdgesFromDictionary(oCategoryDictionary, oGraphMLXmlDocument, "Shared category", SharedCategoryID); } oCategoryDictionary = null; if (WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SharedCommenterEdges)) { AppendSharedResponderEdges(oGraphMLXmlDocument, oVideoIDs, MaximumCommentsPerVideo, "http://gdata.youtube.com/feeds/api/videos/{0}/comments", "commenter", SharedCommenterID, oRequestStatistics); } if (WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SharedVideoResponderEdges)) { AppendSharedResponderEdges(oGraphMLXmlDocument, oVideoIDs, iMaximumVideos, "http://gdata.youtube.com/feeds/api/videos/{0}/responses", "video responder", SharedVideoResponderID, oRequestStatistics); } }
GetUserNetworkInternal ( String sUserNameToAnalyze, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, Int32 iMaximumPeoplePerRequest, RequestStatistics oRequestStatistics, GraphMLXmlDocument oGraphMLXmlDocument ) { Debug.Assert(!String.IsNullOrEmpty(sUserNameToAnalyze)); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(iMaximumPeoplePerRequest > 0); Debug.Assert(oRequestStatistics != null); Debug.Assert(oGraphMLXmlDocument != null); AssertValid(); // The key is the user name and the value is the corresponding GraphML // XML node that represents the user. This is used to prevent the same // user name from being added to the XmlDocument twice. Dictionary <String, XmlNode> oUserNameDictionary = new Dictionary <String, XmlNode>(); // Include friends, subscriptions, both, or neither. Boolean [] abIncludes = new Boolean[] { WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.FriendVertices), WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SubscriptionVertices), }; for (Int32 i = 0; i < 2; i++) { if (abIncludes[i]) { GetUserNetworkRecursive(sUserNameToAnalyze, eWhatToInclude, (i == 0), eNetworkLevel, iMaximumPeoplePerRequest, 1, oGraphMLXmlDocument, oUserNameDictionary, oRequestStatistics); } } if (WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.AllStatistics)) { AppendAllStatisticGraphMLAttributeValues(oGraphMLXmlDocument, oUserNameDictionary, oRequestStatistics); } }
GetNetworkInternal ( String sSearchTerm, WhatToInclude eWhatToInclude, Int32 iMaximumStatuses, RequestStatistics oRequestStatistics, GraphMLXmlDocument oGraphMLXmlDocument ) { Debug.Assert(!String.IsNullOrEmpty(sSearchTerm)); Debug.Assert(iMaximumStatuses > 0); Debug.Assert(iMaximumStatuses != Int32.MaxValue); Debug.Assert(oRequestStatistics != null); Debug.Assert(oGraphMLXmlDocument != null); AssertValid(); // The key is the Twitter user ID and the value is the corresponding // TwitterUser. Dictionary <String, TwitterUser> oUserIDDictionary = new Dictionary <String, TwitterUser>(); // First, append a vertex for each person who has tweeted the search // term. AppendVertexXmlNodesForSearchTerm(sSearchTerm, eWhatToInclude, iMaximumStatuses, oGraphMLXmlDocument, oUserIDDictionary, oRequestStatistics); // Now append a vertex for each person who was mentioned or replied to // by the first set of people, but who didn't tweet the search term // himself. AppendVertexXmlNodesForMentionsAndRepliesTo(eWhatToInclude, oGraphMLXmlDocument, oUserIDDictionary, oRequestStatistics); TwitterSearchNetworkGraphMLUtil.AppendVertexTooltipXmlNodes( oGraphMLXmlDocument, oUserIDDictionary); if (WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.FollowedEdges)) { // Look at each author's friends, and if a friend has also tweeted // the search term, add an edge between the author and the friend. AppendFriendEdgeXmlNodes(oUserIDDictionary, MaximumFriends, oGraphMLXmlDocument, oRequestStatistics); } AppendRepliesToAndMentionsEdgeXmlNodes(oGraphMLXmlDocument, oUserIDDictionary.Values, TwitterGraphMLUtil.TwitterUsersToUniqueScreenNames( oUserIDDictionary.Values) ); }
//************************************************************************* // Constructor: PartialNetworkException() // /// <summary> /// Initializes a new instance of the <see /// cref="PartialNetworkException" /> class. /// </summary> /// /// <param name="partialNetwork"> /// The partial network that was obtained. /// </param> /// /// <param name="requestStatistics"> /// Information about the requests that were made while getting the /// network. /// </param> //************************************************************************* public PartialNetworkException( XmlDocument partialNetwork, RequestStatistics requestStatistics ) { m_oPartialNetwork = partialNetwork; m_oRequestStatistics = requestStatistics; AssertValid(); }
//************************************************************************* // Constructor: PartialNetworkException() // /// <summary> /// Initializes a new instance of the <see /// cref="PartialNetworkException" /> class. /// </summary> /// /// <param name="partialNetwork"> /// The partial network that was obtained. /// </param> /// /// <param name="requestStatistics"> /// Information about the requests that were made while getting the /// network. /// </param> //************************************************************************* public PartialNetworkException ( XmlDocument partialNetwork, RequestStatistics requestStatistics ) { m_oPartialNetwork = partialNetwork; m_oRequestStatistics = requestStatistics; AssertValid(); }
GetXmlDocument ( String sUrl, RequestStatistics oRequestStatistics ) { Debug.Assert(!String.IsNullOrEmpty(sUrl)); Debug.Assert(oRequestStatistics != null); AssertValid(); XmlDocument oXmlDocument = GetXmlDocumentWithRetries(sUrl, HttpStatusCodesToFailImmediately, oRequestStatistics); String sStatus; if ( XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/@stat", null, out sStatus) && sStatus == "ok" ) { return(oXmlDocument); } // Flickr indicates errors by returning an XML document containing an // rsp/err node. The following code turns such an error document into // a custom FlickrException. String sErrorMessage; if (XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/err/@msg", null, out sErrorMessage)) { if (sErrorMessage.ToLower().IndexOf("user not found") >= 0) { sErrorMessage = "The user wasn't found. Either there is no such user, or" + " she has hidden herself from public searches." ; } } else { sErrorMessage = "Flickr provided information in an unrecognized format."; } throw new FlickrException(sErrorMessage); }
OnNetworkObtained ( XmlDocument oGraphMLXmlDocument, RequestStatistics oRequestStatistics, String sNetworkDescription, String sNetworkTitle, String sPartialFileName ) { Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oRequestStatistics != null); Debug.Assert(!String.IsNullOrEmpty(sNetworkDescription)); Debug.Assert(!String.IsNullOrEmpty(sNetworkTitle)); Debug.Assert(!String.IsNullOrEmpty(sPartialFileName)); AssertValid(); XmlNode oGraphXmlNode = XmlUtil2.SelectRequiredSingleNode( oGraphMLXmlDocument, "g:graphml/g:graph", GraphMLXmlDocument.CreateXmlNamespaceManager( oGraphMLXmlDocument, "g") ); XmlUtil2.SetAttributes(oGraphXmlNode, "description", sNetworkDescription); XmlUtil2.SetAttributes(oGraphXmlNode, "suggestedTitle", sNetworkTitle); String sSuggestedFileNameNoExtension = String.Format( "{0} NodeXL {1}" , DateTimeUtil2.ToCultureInvariantFileName( oRequestStatistics.StartTimeUtc), sPartialFileName ); XmlUtil2.SetAttributes(oGraphXmlNode, "suggestedFileNameNoExtension", sSuggestedFileNameNoExtension); if (oRequestStatistics.UnexpectedExceptions > 0) { // The network is partial. throw new PartialNetworkException(oGraphMLXmlDocument, oRequestStatistics); } }
TryGetUserValueDictionary ( String sScreenName, RequestStatistics oRequestStatistics, Boolean bIgnoreWebAndJsonExceptions, out Dictionary <String, Object> oUserValueDictionary ) { Debug.Assert(!String.IsNullOrEmpty(sScreenName)); Debug.Assert(oRequestStatistics != null); AssertValid(); oUserValueDictionary = null; String sUrl = String.Format( "{0}users/show.json?screen_name={1}&{2}" , TwitterApiUrls.Rest, TwitterUtil.EncodeUrlParameter(sScreenName), TwitterApiUrlParameters.IncludeEntities ); ReportProgress(String.Format( "Getting information about \"{0}\"." , sScreenName )); try { oUserValueDictionary = (Dictionary <String, Object>) (new JavaScriptSerializer()).DeserializeObject( GetTwitterResponseAsString(sUrl, oRequestStatistics)); return(true); } catch (Exception oException) { if (!HttpSocialNetworkUtil.ExceptionIsWebOrJson(oException) || !bIgnoreWebAndJsonExceptions) { throw oException; } return(false); } }
GetUserNetworkInternal ( String sScreenName, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, Int32 iMaximumPerRequest, String sApiKey ) { Debug.Assert(!String.IsNullOrEmpty(sScreenName)); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(iMaximumPerRequest > 0); Debug.Assert(!String.IsNullOrEmpty(sApiKey)); AssertValid(); GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.CommenterVertices), WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.UserInformation) ); RequestStatistics oRequestStatistics = new RequestStatistics(); try { GetUserNetworkInternal(sScreenName, eWhatToInclude, eNetworkLevel, iMaximumPerRequest, sApiKey, oRequestStatistics, oGraphMLXmlDocument); } catch (Exception oException) { OnTerminatingException(oException); } OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument, oRequestStatistics); return(oGraphMLXmlDocument); }
GetUserNetworkInternal ( String sUserNameToAnalyze, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, Int32 iMaximumPeoplePerRequest ) { Debug.Assert(!String.IsNullOrEmpty(sUserNameToAnalyze)); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(iMaximumPeoplePerRequest > 0); AssertValid(); GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument(WhatToIncludeFlagIsSet( eWhatToInclude, WhatToInclude.AllStatistics)); RequestStatistics oRequestStatistics = new RequestStatistics(); try { GetUserNetworkInternal(sUserNameToAnalyze, eWhatToInclude, eNetworkLevel, iMaximumPeoplePerRequest, oRequestStatistics, oGraphMLXmlDocument); } catch (Exception oException) { OnUnexpectedException(oException, oGraphMLXmlDocument, oRequestStatistics); } String sNetworkTitle = "YouTube User " + sUserNameToAnalyze; OnNetworkObtained(oGraphMLXmlDocument, oRequestStatistics, GetNetworkDescription(sUserNameToAnalyze, eWhatToInclude, eNetworkLevel, iMaximumPeoplePerRequest), sNetworkTitle, sNetworkTitle ); return(oGraphMLXmlDocument); }
TryGetSampleImageUrl ( String sTag, String sApiKey, RequestStatistics oRequestStatistics, out String sSampleImageUrl ) { Debug.Assert(!String.IsNullOrEmpty(sTag)); Debug.Assert(!String.IsNullOrEmpty(sApiKey)); Debug.Assert(oRequestStatistics != null); AssertValid(); sSampleImageUrl = null; String sUrl = GetFlickrMethodUrl("flickr.tags.getClusterPhotos", sApiKey, "&tag=" + UrlUtil.EncodeUrlParameter(sTag)); XmlDocument oXmlDocument; String sPhotoID; if ( !TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument) || !XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/photos/photo/@id", null, out sPhotoID) ) { return(false); } sUrl = GetFlickrMethodUrl("flickr.photos.getSizes", sApiKey, "&photo_id=" + UrlUtil.EncodeUrlParameter(sPhotoID)); if ( !TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument) || !XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/sizes/size[@label='Thumbnail']/@source", null, out sSampleImageUrl) ) { return(false); } return(true); }
public void DoesNotOverflow() { var elapsedTicks = 0L; var sut = new RequestStatistics(() => elapsedTicks, 1000); for (var i = 0; i < 1000; i++) { var id = Guid.NewGuid(); sut.StartOperation(id); elapsedTicks += TimeSpan.FromMinutes(36).Ticks; sut.EndOperation(id); } sut.GetMeasurementDetails(); }
EnumerateUserValueDictionaries ( String [] asUserIDsOrScreenNames, Boolean bUserIDsSpecified, RequestStatistics oRequestStatistics ) { Debug.Assert(asUserIDsOrScreenNames != null); Debug.Assert(oRequestStatistics != null); AssertValid(); return(m_oTwitterUtil.EnumerateUserValueDictionaries( asUserIDsOrScreenNames, bUserIDsSpecified, oRequestStatistics, new ReportProgressHandler(this.ReportProgress), new CheckCancellationPendingHandler(this.CheckCancellationPending) )); }
/// <summary> /// The API returns the detected language and a numeric score between 0 and 1. /// </summary> /// <remarks> /// Scores close to 1 indicate 100% certainty that the identified language is /// true. A total of 120 languages are supported. /// </remarks> /// <param name='operations'> /// The operations group for this extension method. /// </param> /// <param name='showStats'> /// (optional) if set to true, response will contain input and document level /// statistics. /// </param> /// <param name='inputText'> /// Input text of one document. /// </param> /// <param name='countryHint'> /// Contry hint. /// </param> /// <param name='cancellationToken'> /// The cancellation token. /// </param> public static LanguageResult DetectLanguage( this ITextAnalyticsClient operations, string inputText = default, string countryHint = "US", bool?showStats = default, CancellationToken cancellationToken = default) { var languageBatchInput = new LanguageBatchInput(new List <LanguageInput> { new LanguageInput("1", inputText, countryHint) }); var _result = operations.DetectLanguageWithHttpMessagesAsync(showStats, languageBatchInput, null, cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult(); IList <DetectedLanguage> languages = _result.Body.Documents.Count > 0 ? _result.Body.Documents[0].DetectedLanguages : null; string errorMessage = _result.Body.Errors.Count > 0 ? _result.Body.Errors[0].Message : null; RequestStatistics stats = _result.Body.Statistics; return(new LanguageResult(languages, errorMessage, stats)); }
/// <summary> /// The API returns a numeric score between 0 and 1. /// </summary> /// <remarks> /// Scores close to 1 indicate positive sentiment, while scores close to 0 /// indicate negative sentiment. A score of 0.5 indicates the lack of sentiment /// (e.g. a factoid statement). See the <a /// href="https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/overview#supported-languages">Text /// Analytics Documentation</a> for details about the languages that are /// supported by sentiment analysis. /// </remarks> /// <param name='operations'> /// The operations group for this extension method. /// </param> /// <param name='showStats'> /// (optional) if set to true, response will contain input and document level /// statistics. /// </param> /// <param name='inputText'> /// Input text of one document. /// </param> /// <param name='language'> /// Language code. /// </param> /// <param name='cancellationToken'> /// The cancellation token. /// </param> public static SentimentResult Sentiment( this ITextAnalyticsClient operations, string inputText = default, string language = "en", bool?showStats = default, CancellationToken cancellationToken = default) { var multiLanguageBatchInput = new MultiLanguageBatchInput(new List <MultiLanguageInput> { new MultiLanguageInput("1", inputText, language) }); var _result = operations.SentimentWithHttpMessagesAsync(showStats, multiLanguageBatchInput, null, cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult(); double? score = _result.Body.Documents.Count > 0 ? _result.Body.Documents[0].Score : null; string errorMessage = _result.Body.Errors.Count > 0 ? _result.Body.Errors[0].Message : null; RequestStatistics stats = _result.Body.Statistics; return(new SentimentResult(score, errorMessage, stats)); }
GetTwitterResponseAsString ( String sUrl, RequestStatistics oRequestStatistics ) { Debug.Assert(!String.IsNullOrEmpty(sUrl)); Debug.Assert(oRequestStatistics != null); AssertValid(); Debug.Assert(m_oTwitterUtil != null); return(m_oTwitterUtil.GetTwitterResponseAsString(sUrl, oRequestStatistics, new ReportProgressHandler(this.ReportProgress), new CheckCancellationPendingHandler(this.CheckCancellationPending) )); }
GetUserNetworkInternal ( String sScreenNameToAnalyze, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, Int32 iMaximumPeoplePerRequest ) { Debug.Assert(!String.IsNullOrEmpty(sScreenNameToAnalyze)); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(iMaximumPeoplePerRequest > 0); AssertValid(); BeforeGetNetwork(); Boolean bIncludeLatestStatus = WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.LatestStatuses); GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument(true, bIncludeLatestStatus); RequestStatistics oRequestStatistics = new RequestStatistics(); try { GetUserNetworkInternal(sScreenNameToAnalyze, eWhatToInclude, eNetworkLevel, iMaximumPeoplePerRequest, oRequestStatistics, oGraphMLXmlDocument); } catch (Exception oException) { OnTerminatingException(oException); } OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument, oRequestStatistics); return(oGraphMLXmlDocument); }
GetRelatedTagsInternal ( String sTag, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, String sApiKey ) { Debug.Assert(!String.IsNullOrEmpty(sTag)); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(!String.IsNullOrEmpty(sApiKey)); AssertValid(); GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SampleThumbnails)); RequestStatistics oRequestStatistics = new RequestStatistics(); try { GetRelatedTagsInternal(sTag, eWhatToInclude, eNetworkLevel, sApiKey, oRequestStatistics, oGraphMLXmlDocument); } catch (Exception oException) { OnUnexpectedException(oException, oGraphMLXmlDocument, oRequestStatistics); } String sNetworkTitle = "Flickr Tag " + sTag; OnNetworkObtained(oGraphMLXmlDocument, oRequestStatistics, GetNetworkDescription(sTag, eWhatToInclude, eNetworkLevel), sNetworkTitle, sNetworkTitle ); return(oGraphMLXmlDocument); }
//************************************************************************* // Method: AppendAllStatisticGraphMLAttributeValues() // /// <overloads> /// Appends statistic GraphML attribute values to the GraphML document. /// </overloads> /// /// <summary> /// Appends statistic GraphML attribute values to the GraphML document for /// all users in the network. /// </summary> /// /// <param name="oGraphMLXmlDocument"> /// GraphMLXmlDocument being populated. /// </param> /// /// <param name="oUserNameDictionary"> /// The key is the user name and the value is the corresponding GraphML XML /// node that represents the user. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> //************************************************************************* protected void AppendAllStatisticGraphMLAttributeValues( GraphMLXmlDocument oGraphMLXmlDocument, Dictionary<String, XmlNode> oUserNameDictionary, RequestStatistics oRequestStatistics ) { Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oUserNameDictionary != null); Debug.Assert(oRequestStatistics != null); AssertValid(); foreach (KeyValuePair<String, XmlNode> oKeyValuePair in oUserNameDictionary) { String sUserName = oKeyValuePair.Key; ReportProgress( String.Format( "Getting statistics for \"{0}\"." , sUserName ) ); AppendAllStatisticGraphMLAttributeValues(sUserName, oKeyValuePair.Value, oGraphMLXmlDocument, oRequestStatistics); } }
BackgroundWorker_DoWork(object sender, DoWorkEventArgs e) { Debug.Assert(sender is BackgroundWorker); BackgroundWorker bw = sender as BackgroundWorker; Debug.Assert(e.Argument is NetworkAsyncArgs); NetworkAsyncArgs args = e.Argument as NetworkAsyncArgs; this.requestStatistics = new RequestStatistics(); try { CheckCancellationPending(); ReportProgress("Starting"); // shell include ego node in the graph this.includeEgo = args.includeMe; // prepare rest contexts var context = new VkRestApi.VkRestContext(args.userId, args.accessToken); // get ego node vkRestApi.CallVkFunction(VkFunction.GetProfiles, context); // wait for the user data readyEvent.WaitOne(); CheckCancellationPending(); ReportProgress("Retrieving friends"); // prepare fields context parameter StringBuilder sb = new StringBuilder("fields="); sb.Append(args.fields); context.Parameters = sb.ToString(); // get friends node vkRestApi.CallVkFunction(VkFunction.LoadFriends, context); // wait for the friends data readyEvent.WaitOne(); int total = this.friendIds.Count; int current = 0; foreach (string targetId in this.friendIds) { CheckCancellationPending(); current++; ReportProgress("Retrieving friends mutual " + current + " out of " + total); // Append target friend ids sb.Length = 0; sb.Append("target_uid="); sb.Append(targetId); context.Parameters = sb.ToString(); context.Cookie = targetId; // pass target id in the context's cookie field // get mutual friends vkRestApi.CallVkFunction(VkFunction.FriendsGetMutual, context); // wait for the mutual data readyEvent.WaitOne(); // play it nice, sleep for 1/3 sec to stay within 3 requests/second limit // TODO: account for time spent in processing Thread.Sleep(333); } if (includeEgo) { CreateIncludeMeEdges(); } CheckCancellationPending(); ReportProgress("Building network graph document"); // create default attributes (values will be empty) AttributesDictionary<String> attributes = new AttributesDictionary<String>(VKAttributes); // build the file XmlDocument graph = GenerateNetworkDocument(vertices, edges, attributes); if (this.requestStatistics.UnexpectedExceptions > 0) { // there was errors - pop up partial network dialog throw new PartialNetworkException(graph, this.requestStatistics); } else { // all good e.Result = graph; } } catch (CancellationPendingException) { e.Cancel = true; } }
//************************************************************************* // Method: GetUserNetworkInternal() // /// <summary> /// Gets a network of Twitter users, given a GraphMLXmlDocument. /// </summary> /// /// <param name="sScreenNameToAnalyze"> /// The screen name of the Twitter user whose network should be analyzed. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="eNetworkLevel"> /// Network level to include. Must be NetworkLevel.One, OnePointFive, or /// Two. /// </param> /// /// <param name="iMaximumPeoplePerRequest"> /// Maximum number of people to request for each query, or Int32.MaxValue /// for no limit. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// The GraphMLXmlDocument to populate with the requested network. /// </param> //************************************************************************* protected void GetUserNetworkInternal( String sScreenNameToAnalyze, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, Int32 iMaximumPeoplePerRequest, RequestStatistics oRequestStatistics, GraphMLXmlDocument oGraphMLXmlDocument ) { Debug.Assert( !String.IsNullOrEmpty(sScreenNameToAnalyze) ); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(iMaximumPeoplePerRequest > 0); Debug.Assert(oRequestStatistics != null); Debug.Assert(oGraphMLXmlDocument != null); AssertValid(); // The key is the screen name and the value is the corresponding // TwitterVertex. This is used to prevent the same screen name from // being added to the XmlDocument twice. Dictionary<String, TwitterVertex> oScreenNameDictionary = new Dictionary<String, TwitterVertex>(); // Include followed, followers, both, or neither. Boolean [] abIncludes = new Boolean[] { WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.FollowedVertices), WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.FollowerVertices), }; for (Int32 i = 0; i < 2; i++) { if ( abIncludes[i] ) { GetUserNetworkRecursive(sScreenNameToAnalyze, eWhatToInclude, (i == 0), eNetworkLevel, iMaximumPeoplePerRequest, 1, oGraphMLXmlDocument, oScreenNameDictionary, oRequestStatistics); } } Boolean bIncludeLatestStatus = WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.LatestStatuses); AppendMissingGraphMLAttributeValues(oGraphMLXmlDocument, oScreenNameDictionary, true, bIncludeLatestStatus, oRequestStatistics); AppendRepliesToAndMentionsXmlNodes(oGraphMLXmlDocument, oScreenNameDictionary, WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.RepliesToEdges), WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.MentionsEdges) ); }
//************************************************************************* // Method: GetVideoNetworkInternal() // /// <overloads> /// Gets a network of related YouTube videos. /// </overloads> /// /// <summary> /// Gets a network of related YouTube videos. /// </summary> /// /// <param name="sSearchTerm"> /// The term to search for. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="iMaximumVideos"> /// Maximum number of videos to request, or Int32.MaxValue for no limit. /// </param> /// /// <returns> /// An XmlDocument containing the network as GraphML. /// </returns> //************************************************************************* protected XmlDocument GetVideoNetworkInternal( String sSearchTerm, WhatToInclude eWhatToInclude, Int32 iMaximumVideos ) { Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) ); Debug.Assert(iMaximumVideos > 0); AssertValid(); GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument(); RequestStatistics oRequestStatistics = new RequestStatistics(); try { GetVideoNetworkInternal(sSearchTerm, eWhatToInclude, iMaximumVideos, oRequestStatistics, oGraphMLXmlDocument); } catch (Exception oException) { OnTerminatingException(oException); } OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument, oRequestStatistics); return (oGraphMLXmlDocument); }
//************************************************************************* // Method: AppendSharedResponderEdges() // /// <summary> /// Appends an edge XML node for each pair of videos for which the same /// person responded. /// </summary> /// /// <param name="oGraphMLXmlDocument"> /// The GraphMLXmlDocument being populated. /// </param> /// /// <param name="oVideoIDs"> /// The set of unique video IDs. /// </param> /// /// <param name="iMaximumResponses"> /// Maximum number of responses to request, or Int32.MaxValue for no limit. /// </param> /// /// <param name="sUrlPattern"> /// URL to get the responses for one video, with "{0}" where the video ID /// goes. /// </param> /// /// <param name="sResponderTitle"> /// Title describing the person who has responded. Samples: "commenter", /// "video responder". /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> //************************************************************************* protected void AppendSharedResponderEdges( GraphMLXmlDocument oGraphMLXmlDocument, HashSet<String> oVideoIDs, Int32 iMaximumResponses, String sUrlPattern, String sResponderTitle, RequestStatistics oRequestStatistics ) { Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oVideoIDs != null); Debug.Assert(iMaximumResponses > 0); Debug.Assert( !String.IsNullOrEmpty(sUrlPattern) ); Debug.Assert(sUrlPattern.IndexOf("{0}") >= 0); Debug.Assert( !String.IsNullOrEmpty(sResponderTitle) ); Debug.Assert(oRequestStatistics != null); AssertValid(); // The key is the name of an author and the value is a LinkedList of // the video IDs to which the author has responded. Dictionary< String, LinkedList<String> > oAuthorUserNameDictionary = new Dictionary< String, LinkedList<String> >(); foreach (String sVideoID in oVideoIDs) { ReportProgress(String.Format( "Getting {0}s for the video with the ID \"{1}\"." , sResponderTitle, sVideoID ) ); // This is to prevent self-loop edges that would result when the // same author responds to the same video more than once. HashSet<String> oAuthorUserNames = new HashSet<String>(); String sUrl = String.Format(sUrlPattern, sVideoID); // The document consists of an "entry" XML node for each response. foreach ( XmlNode oEntryXmlNode in EnumerateXmlNodes(sUrl, "a:feed/a:entry", iMaximumResponses, true, oRequestStatistics) ) { XmlNamespaceManager oXmlNamespaceManager = CreateXmlNamespaceManager(oEntryXmlNode.OwnerDocument); String sAuthorUserName; if ( XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode, "a:author/a:name/text()", oXmlNamespaceManager, out sAuthorUserName) && !oAuthorUserNames.Contains(sAuthorUserName) ) { AddVideoIDToDictionary(sAuthorUserName, sVideoID, oAuthorUserNameDictionary); oAuthorUserNames.Add(sAuthorUserName); } } } ReportProgress("Adding edges for shared " + sResponderTitle + "s."); AppendEdgesFromDictionary(oAuthorUserNameDictionary, oGraphMLXmlDocument, "Shared " + sResponderTitle); }
//************************************************************************* // Method: GetXmlDocumentWithRetries() // /// <summary> /// Gets an XML document given an URL. Retries after an error. /// </summary> /// /// <param name="sUrl"> /// URL to use. /// </param> /// /// <param name="aeHttpStatusCodesToFailImmediately"> /// An array of status codes that should be failed immediately, or null to /// retry all failures. An example is HttpStatusCode.Unauthorized (401), /// which Twitter returns when information about a user who has "protected" /// status is requested. This should not be retried, because the retries /// would produce exactly the same error response. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <param name="asOptionalHeaderNameValuePairs"> /// Array of name/value pairs for HTTP headers to add to the request, or /// null to not add any pairs. Sample: {"Authorization", "Basic 36A4E798"} /// </param> /// /// <returns> /// The XmlDocument. /// </returns> /// /// <remarks> /// If the request fails and the HTTP status code is not one of the codes /// specified in <paramref name="aeHttpStatusCodesToFailImmediately" />, /// the request is retried. If the retries also fail, an exception is /// thrown. /// /// <para> /// If the request fails with one of the HTTP status code contained in /// <paramref name="aeHttpStatusCodesToFailImmediately" />, an exception is /// thrown immediately. /// </para> /// /// <para> /// In either case, it is always up to the caller to handle the exceptions. /// This method never ignores an exception; it either retries it and throws /// it if all retries fail, or throws it immediately. /// </para> /// /// </remarks> //************************************************************************* protected XmlDocument GetXmlDocumentWithRetries( String sUrl, HttpStatusCode [] aeHttpStatusCodesToFailImmediately, RequestStatistics oRequestStatistics, params String[] asOptionalHeaderNameValuePairs ) { Debug.Assert( !String.IsNullOrEmpty(sUrl) ); Debug.Assert(oRequestStatistics != null); Debug.Assert(asOptionalHeaderNameValuePairs == null || asOptionalHeaderNameValuePairs.Length % 2 == 0); AssertValid(); Int32 iMaximumRetries = HttpRetryDelaysSec.Length; Int32 iRetriesSoFar = 0; while (true) { if (iRetriesSoFar > 0) { ReportProgress("Retrying request."); } // Important Note: You cannot use the same HttpWebRequest object // for the retries. The object must be recreated each time. HttpWebRequest oHttpWebRequest = (HttpWebRequest)WebRequest.Create(sUrl); Int32 iHeaderNamesAndValues = (asOptionalHeaderNameValuePairs == null) ? 0 : asOptionalHeaderNameValuePairs.Length; for (Int32 i = 0; i < iHeaderNamesAndValues; i += 2) { String sHeaderName = asOptionalHeaderNameValuePairs[i + 0]; String sHeaderValue = asOptionalHeaderNameValuePairs[i + 1]; Debug.Assert( !String.IsNullOrEmpty(sHeaderName) ); Debug.Assert( !String.IsNullOrEmpty(sHeaderValue) ); oHttpWebRequest.Headers[sHeaderName] = sHeaderValue; } try { XmlDocument oXmlDocument = GetXmlDocumentNoRetries(oHttpWebRequest); if (iRetriesSoFar > 0) { ReportProgress("Retry succeeded, continuing..."); } oRequestStatistics.OnSuccessfulRequest(); return (oXmlDocument); } catch (Exception oException) { if ( !ExceptionIsWebOrXml(oException) ) { throw oException; } // A WebException or XmlException has occurred. if (iRetriesSoFar == iMaximumRetries) { oRequestStatistics.OnUnexpectedException(oException); throw (oException); } // If the status code is one of the ones specified in // aeHttpStatusCodesToFailImmediately, rethrow the exception // without retrying the request. if (aeHttpStatusCodesToFailImmediately != null && oException is WebException) { if ( WebExceptionHasHttpStatusCode( (WebException)oException, aeHttpStatusCodesToFailImmediately) ) { throw (oException); } } Int32 iSeconds = HttpRetryDelaysSec[iRetriesSoFar]; ReportProgress( String.Format( "Request failed, pausing {0} {1} before retrying..." , iSeconds, StringUtil.MakePlural("second", iSeconds) ) ); System.Threading.Thread.Sleep(1000 * iSeconds); iRetriesSoFar++; } } }
//************************************************************************* // Method: TryGetSampleImageUrl() // /// <summary> /// Attempts to get an URL to a sample image for a tag. /// </summary> /// /// <param name="sTag"> /// Tag to get a sample image URL for. /// </param> /// /// <param name="sApiKey"> /// Flickr API key. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <param name="sSampleImageUrl"> /// Where the URL gets stored if true is returned. /// </param> /// /// <returns> /// true if the sample image URL was obtained, false if some sort of error /// occurred while attempting to get it. /// </returns> //************************************************************************* protected Boolean TryGetSampleImageUrl( String sTag, String sApiKey, RequestStatistics oRequestStatistics, out String sSampleImageUrl ) { Debug.Assert( !String.IsNullOrEmpty(sTag) ); Debug.Assert( !String.IsNullOrEmpty(sApiKey) ); Debug.Assert(oRequestStatistics != null); AssertValid(); sSampleImageUrl = null; String sUrl = GetFlickrMethodUrl( "flickr.tags.getClusterPhotos", sApiKey, "&tag=" + UrlUtil.EncodeUrlParameter(sTag) ); XmlDocument oXmlDocument; String sPhotoID; if ( !TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument) || ! XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/photos/photo/@id", null, out sPhotoID) ) { return (false); } sUrl = GetFlickrMethodUrl( "flickr.photos.getSizes", sApiKey, "&photo_id=" + UrlUtil.EncodeUrlParameter(sPhotoID) ); if ( !TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument) || ! XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/sizes/size[@label='Thumbnail']/@source", null, out sSampleImageUrl) ) { return (false); } return (true); }
EnumerateUserValueDictionaries ( String [] userIDsOrScreenNames, Boolean userIDsSpecified, RequestStatistics requestStatistics, ReportProgressHandler reportProgressHandler, CheckCancellationPendingHandler checkCancellationPendingHandler ) { Debug.Assert(userIDsOrScreenNames != null); Debug.Assert(requestStatistics != null); AssertValid(); // We'll use Twitter's users/lookup API, which gets extended // information for up to 100 users in one call. Int32 iUsers = userIDsOrScreenNames.Length; Int32 iUsersProcessed = 0; Int32 iCalls = 0; while (iUsersProcessed < iUsers) { // For each call, ask for information about as many users as // possible until either 100 is reached or the URL reaches an // arbitrary maximum length. Twitter recommends using a POST here // (without specifying why), but it would require revising the // base-class HTTP calls and isn't worth the trouble. Int32 iUsersProcessedThisCall = 0; StringBuilder oUrl = new StringBuilder(); oUrl.AppendFormat( "{0}users/lookup.json?{1}&{2}=" , TwitterApiUrls.Rest, TwitterApiUrlParameters.IncludeEntities, userIDsSpecified ? "user_id" : "screen_name" ); const Int32 MaxUsersPerCall = 100; const Int32 MaxUrlLength = 2000; // Construct the URL for this call. while ( iUsersProcessed < iUsers && iUsersProcessedThisCall < MaxUsersPerCall && oUrl.Length < MaxUrlLength ) { if (iUsersProcessedThisCall > 0) { // Append an encoded comma. Using an unencoded comma // causes Twitter to return a 401 "unauthorized" error. // // See this post for an explanation: // // https://dev.twitter.com/discussions/11399 oUrl.Append("%2C"); } oUrl.Append( userIDsOrScreenNames[iUsersProcessed] ); iUsersProcessed++; iUsersProcessedThisCall++; } iCalls++; if (iCalls > 1 && reportProgressHandler != null) { reportProgressHandler("Getting page " + iCalls + "."); } foreach ( Object oResult in EnumerateJsonValues(oUrl.ToString(), null, Int32.MaxValue, true, requestStatistics, reportProgressHandler, checkCancellationPendingHandler) ) { yield return ( ( Dictionary<String, Object> )oResult ); } } }
EnumerateJsonValues ( String url, String jsonName, Int32 maximumValues, Boolean skipMostPage1Errors, RequestStatistics requestStatistics, ReportProgressHandler reportProgressHandler, CheckCancellationPendingHandler checkCancellationPendingHandler ) { // Note: // // The logic in this method is similar to the logic in // EnumerateSearchStatuses(). In fact, at one time all enumeration was // done through this EnumerateJsonValues() method. // EnumerateSearchStatuses() was created only when version 1.1 of the // Twitter API introduced yet another paging scheme, one that differs // from the cursor scheme that this method handles. // // A possible work item is to recombine the two methods into one, // possibly by using a delegate to handle the different paging schemes. Debug.Assert( !String.IsNullOrEmpty(url) ); Debug.Assert(maximumValues > 0); Debug.Assert(requestStatistics != null); AssertValid(); Int32 iPage = 1; String sCursor = null; Int32 iObjectsEnumerated = 0; while (true) { if (iPage > 1 && reportProgressHandler != null) { reportProgressHandler("Getting page " + iPage + "."); } String sUrlWithCursor = AppendCursorToUrl(url, sCursor); Dictionary<String, Object> oValueDictionary = null; Object [] aoObjectsThisPage; try { Object oDeserializedTwitterResponse = ( new JavaScriptSerializer() ).DeserializeObject( GetTwitterResponseAsString(sUrlWithCursor, requestStatistics, reportProgressHandler, checkCancellationPendingHandler) ); Object oObjectsThisPageAsObject; if (jsonName == null) { // The top level of the Json response contains an array of // objects this method will enumerate. oObjectsThisPageAsObject = oDeserializedTwitterResponse; } else { // The top level of the Json response contains a set of // name/value pairs. The value for the specified name is // the array of objects this method will enumerate. oValueDictionary = ( Dictionary<String, Object> ) oDeserializedTwitterResponse; oObjectsThisPageAsObject = oValueDictionary[jsonName]; } aoObjectsThisPage = ( Object [] )oObjectsThisPageAsObject; } catch (Exception oException) { // Rethrow the exception if appropriate. TwitterUtil.OnExceptionWhileEnumeratingJsonValues( oException, iPage, skipMostPage1Errors); // Otherwise, just halt the enumeration. yield break; } Int32 iObjectsThisPage = aoObjectsThisPage.Length; if (iObjectsThisPage == 0) { yield break; } for (Int32 i = 0; i < iObjectsThisPage; i++) { yield return ( aoObjectsThisPage[i] ); iObjectsEnumerated++; if (iObjectsEnumerated == maximumValues) { yield break; } } iPage++; // When the top level of the Json response contains a set of // name/value pairs, a next_cursor_str value of "0" means "end of // data." if ( oValueDictionary == null || !TwitterJsonUtil.TryGetJsonValueFromDictionary( oValueDictionary, "next_cursor_str", out sCursor) || sCursor == "0" ) { yield break; } // Get the next page... } }
//************************************************************************* // Method: GetUserNetworkRecursive() // /// <summary> /// Recursively gets a network of YouTube users. /// </summary> /// /// <param name="sUserName"> /// The user name to use in this call. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="bIncludeFriendsThisCall"> /// true to include the user's friends, false to include the people /// subscribed to by the user. /// </param> /// /// <param name="eNetworkLevel"> /// Network level to include. Must be NetworkLevel.One, OnePointFive, or /// Two. /// </param> /// /// <param name="iMaximumPeoplePerRequest"> /// Maximum number of people to request for each query, or Int32.MaxValue /// for no limit. /// </param> /// /// <param name="iRecursionLevel"> /// Recursion level for this call. Must be 1 or 2. Gets incremented when /// recursing. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// GraphMLXmlDocument being populated. /// </param> /// /// <param name="oUserNameDictionary"> /// The key is the user name and the value is the corresponding GraphML XML /// node that represents the user. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> //************************************************************************* protected void GetUserNetworkRecursive( String sUserName, WhatToInclude eWhatToInclude, Boolean bIncludeFriendsThisCall, NetworkLevel eNetworkLevel, Int32 iMaximumPeoplePerRequest, Int32 iRecursionLevel, GraphMLXmlDocument oGraphMLXmlDocument, Dictionary<String, XmlNode> oUserNameDictionary, RequestStatistics oRequestStatistics ) { Debug.Assert( !String.IsNullOrEmpty(sUserName) ); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(iMaximumPeoplePerRequest > 0); Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oUserNameDictionary != null); Debug.Assert(oRequestStatistics != null); AssertValid(); /* Here is what this method should do, based on the eNetworkLevel and iRecursionLevel parameters. eNetworkLevel |One | OnePointFive | Two ---|------------------| ------------------| ----------------- i 1 |Add all vertices. | Add all vertices. | Add all vertices. R | | | e |Add all edges. | Add all edges. | Add all edges. c | | | u |Do not recurse. | Recurse. | Recurse. r | | | s ---|------------------|-------------------|------------------ i 2 |Impossible. | Do not add | Add all vertices. o | | vertices. | n | | | L | | Add edges only if | Add all edges. e | | vertices are | v | | already included. | e | | | l | | Do not recurse. | Do not recurse. | | | ---|------------------|-------------------|------------------ */ Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel, iRecursionLevel); List<String> oUserNamesToRecurse = new List<String>(); ReportProgressForFriendsOrSubscriptions(sUserName, bIncludeFriendsThisCall); Boolean bThisUserAppended = false; // If the GraphMLXmlDocument already contains at least one vertex node, // then this is the second time that this method has been called, a // partial network has already been obtained, and most errors should // now be skipped. However, if none of the network has been obtained // yet, errors on page 1 should throw an immediate exception. Boolean bSkipMostPage1Errors = oGraphMLXmlDocument.HasVertexXmlNode; // The document consists of a single "feed" node with zero or more // "entry" child nodes. foreach ( XmlNode oEntryXmlNode in EnumerateXmlNodes( GetFriendsOrSubscriptionsUrl(sUserName, bIncludeFriendsThisCall), "a:feed/a:entry", iMaximumPeoplePerRequest, bSkipMostPage1Errors, oRequestStatistics) ) { XmlNamespaceManager oXmlNamespaceManager = CreateXmlNamespaceManager(oEntryXmlNode.OwnerDocument); String sOtherUserName; if ( !XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode, "yt:username/text()", oXmlNamespaceManager, out sOtherUserName) ) { continue; } if (!bThisUserAppended) { // Append a vertex node for this request's user. // // This used to be done after the foreach loop, which avoided // the need for a "bThisUserAppended" flag. That caused the // following bug: If a YouTube error occurred within // EnumerateXmlNodes() after some edges had been added, and the // user decided to import the resulting partial network, the // GraphML might contain edges that referenced "this user" // without containing a vertex for "this user." That is an // illegal state for GraphML, which the ExcelTemplate project // caught and reported as an error. TryAppendVertexXmlNode(sUserName, null, oGraphMLXmlDocument, oUserNameDictionary); bThisUserAppended = true; } Boolean bNeedToAppendVertices = GetNeedToAppendVertices( eNetworkLevel, iRecursionLevel); if (bNeedToAppendVertices) { if ( TryAppendVertexXmlNode(sOtherUserName, oEntryXmlNode, oGraphMLXmlDocument, oUserNameDictionary) && bNeedToRecurse ) { oUserNamesToRecurse.Add(sOtherUserName); } } if ( bNeedToAppendVertices || oUserNameDictionary.ContainsKey(sOtherUserName) ) { String sRelationship; if (bIncludeFriendsThisCall) { sRelationship = "Friend of"; } else { sRelationship = "Subscribes to"; String sSubscriptionType = null; if ( XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode, "a:category[@scheme='http://gdata.youtube.com/schemas/" + "2007/subscriptiontypes.cat']/@term", oXmlNamespaceManager, out sSubscriptionType) ) { sRelationship += " " + sSubscriptionType; } } AppendEdgeXmlNode(oGraphMLXmlDocument, sUserName, sOtherUserName, sRelationship); } } if (bNeedToRecurse) { foreach (String sUserNameToRecurse in oUserNamesToRecurse) { GetUserNetworkRecursive(sUserNameToRecurse, eWhatToInclude, bIncludeFriendsThisCall, eNetworkLevel, iMaximumPeoplePerRequest, 2, oGraphMLXmlDocument, oUserNameDictionary, oRequestStatistics); } } }
//************************************************************************* // Method: GetUserNetworkInternal() // /// <summary> /// Gets a network of YouTube users, given a GraphMLXmlDocument. /// </summary> /// /// <param name="sUserNameToAnalyze"> /// The user name of the YouTube user whose network should be analyzed. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="eNetworkLevel"> /// Network level to include. Must be NetworkLevel.One, OnePointFive, or /// Two. /// </param> /// /// <param name="iMaximumPeoplePerRequest"> /// Maximum number of people to request for each query, or Int32.MaxValue /// for no limit. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// The GraphMLXmlDocument to populate with the requested network. /// </param> //************************************************************************* protected void GetUserNetworkInternal( String sUserNameToAnalyze, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, Int32 iMaximumPeoplePerRequest, RequestStatistics oRequestStatistics, GraphMLXmlDocument oGraphMLXmlDocument ) { Debug.Assert( !String.IsNullOrEmpty(sUserNameToAnalyze) ); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(iMaximumPeoplePerRequest > 0); Debug.Assert(oRequestStatistics != null); Debug.Assert(oGraphMLXmlDocument != null); AssertValid(); // The key is the user name and the value is the corresponding GraphML // XML node that represents the user. This is used to prevent the same // user name from being added to the XmlDocument twice. Dictionary<String, XmlNode> oUserNameDictionary = new Dictionary<String, XmlNode>(); // Include friends, subscriptions, both, or neither. Boolean [] abIncludes = new Boolean[] { WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.FriendVertices), WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SubscriptionVertices), }; for (Int32 i = 0; i < 2; i++) { if ( abIncludes[i] ) { GetUserNetworkRecursive(sUserNameToAnalyze, eWhatToInclude, (i == 0), eNetworkLevel, iMaximumPeoplePerRequest, 1, oGraphMLXmlDocument, oUserNameDictionary, oRequestStatistics); } } if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.AllStatistics) ) { AppendAllStatisticGraphMLAttributeValues(oGraphMLXmlDocument, oUserNameDictionary, oRequestStatistics); } }
//************************************************************************* // Method: GetUserNetworkInternal() // /// <overloads> /// Gets a network of YouTube users. /// </overloads> /// /// <summary> /// Gets a network of YouTube users. /// </summary> /// /// <param name="sUserNameToAnalyze"> /// The user name of the YouTube user whose network should be analyzed. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="eNetworkLevel"> /// Network level to include. Must be NetworkLevel.One, OnePointFive, or /// Two. /// </param> /// /// <param name="iMaximumPeoplePerRequest"> /// Maximum number of people to request for each query, or Int32.MaxValue /// for no limit. /// </param> /// /// <returns> /// An XmlDocument containing the network as GraphML. /// </returns> //************************************************************************* protected XmlDocument GetUserNetworkInternal( String sUserNameToAnalyze, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, Int32 iMaximumPeoplePerRequest ) { Debug.Assert( !String.IsNullOrEmpty(sUserNameToAnalyze) ); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(iMaximumPeoplePerRequest > 0); AssertValid(); GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument( WhatToIncludeFlagIsSet( eWhatToInclude, WhatToInclude.AllStatistics) ); RequestStatistics oRequestStatistics = new RequestStatistics(); try { GetUserNetworkInternal(sUserNameToAnalyze, eWhatToInclude, eNetworkLevel, iMaximumPeoplePerRequest, oRequestStatistics, oGraphMLXmlDocument); } catch (Exception oException) { OnTerminatingException(oException); } OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument, oRequestStatistics); return (oGraphMLXmlDocument); }
//************************************************************************* // Method: AppendInt32GraphMLAttributeValue() // /// <summary> /// Appends an Int32 GraphML-Attribute value from an XmlDocument to a /// vertex XML node. /// </summary> /// /// <param name="sUrl"> /// The URL to get the XmlDocument from. /// </param> /// /// <param name="sXPath"> /// Path to the value within the XmlDocument. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// GraphMLXmlDocument being populated. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <param name="oVertexXmlNode"> /// The vertex XML node from <paramref name="oGraphMLXmlDocument" /> to add /// the GraphML attribute value to. /// </param> /// /// <param name="sGraphMLAttributeID"> /// GraphML ID of the attribute. /// </param> /// /// <remarks> /// This method looks for an Int32 value within an XmlDocument returned by /// YouTube and stores the value on a vertex XML node in the GraphML /// document. /// </remarks> //************************************************************************* protected void AppendInt32GraphMLAttributeValue( String sUrl, String sXPath, GraphMLXmlDocument oGraphMLXmlDocument, RequestStatistics oRequestStatistics, XmlNode oVertexXmlNode, String sGraphMLAttributeID ) { Debug.Assert( !String.IsNullOrEmpty(sUrl) ); Debug.Assert( !String.IsNullOrEmpty(sXPath) ); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oRequestStatistics != null); Debug.Assert(oVertexXmlNode != null); Debug.Assert( !String.IsNullOrEmpty(sGraphMLAttributeID) ); AssertValid(); XmlDocument oXmlDocument; XmlNamespaceManager oXmlNamespaceManager; if ( !TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument, out oXmlNamespaceManager) ) { return; } AppendInt32GraphMLAttributeValue(oXmlDocument, sXPath, oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, sGraphMLAttributeID); }
//************************************************************************* // Method: AppendAllStatisticGraphMLAttributeValues() // /// <summary> /// Appends statistic GraphML attribute values to the GraphML document for /// one user in the network. /// </summary> /// /// <param name="sUserName"> /// The user name. /// </param> /// /// <param name="oVertexXmlNode"> /// The GraphML XML node corresponding to the user. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// GraphMLXmlDocument being populated. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> //************************************************************************* protected void AppendAllStatisticGraphMLAttributeValues( String sUserName, XmlNode oVertexXmlNode, GraphMLXmlDocument oGraphMLXmlDocument, RequestStatistics oRequestStatistics ) { Debug.Assert( !String.IsNullOrEmpty(sUserName) ); Debug.Assert(oVertexXmlNode != null); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oRequestStatistics != null); AssertValid(); XmlDocument oXmlDocument; XmlNamespaceManager oXmlNamespaceManager; // Some of the statistics are available in the user's profile. String sUrl = "http://gdata.youtube.com/feeds/api/users/" + EncodeUrlParameter(sUserName); if ( TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument, out oXmlNamespaceManager) ) { AppendYouTubeDateGraphMLAttributeValue(oXmlDocument, "a:entry/a:published/text()", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, JoinedDateUtcID); AppendStringGraphMLAttributeValue(oXmlDocument, "a:entry/media:thumbnail/@url", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, ImageFileID); XmlNode oStatisticsXmlNode; if ( XmlUtil2.TrySelectSingleNode(oXmlDocument, "a:entry/yt:statistics", oXmlNamespaceManager, out oStatisticsXmlNode) ) { AppendInt32GraphMLAttributeValue(oStatisticsXmlNode, "@videoWatchCount", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, VideosWatchedID); AppendInt32GraphMLAttributeValue(oStatisticsXmlNode, "@subscriberCount", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, SubscribersID); } } // The remaining statistics must be obtained from the totalResults // XML node within documents obtained from other URLs. const String TotalResultsXPath = "a:feed/openSearch:totalResults/text()"; sUrl = GetFriendsOrSubscriptionsUrl(sUserName, true) + "?max-results=1"; AppendInt32GraphMLAttributeValue(sUrl, TotalResultsXPath, oGraphMLXmlDocument, oRequestStatistics, oVertexXmlNode, FriendsID); sUrl = GetFriendsOrSubscriptionsUrl(sUserName, false) + "?max-results=1"; AppendInt32GraphMLAttributeValue(sUrl, TotalResultsXPath, oGraphMLXmlDocument, oRequestStatistics, oVertexXmlNode, SubscriptionsID); sUrl = "http://gdata.youtube.com/feeds/api/users/" + EncodeUrlParameter(sUserName) + "/uploads?max-results=1"; AppendInt32GraphMLAttributeValue(sUrl, TotalResultsXPath, oGraphMLXmlDocument, oRequestStatistics, oVertexXmlNode, VideosUploadedID); }
//************************************************************************* // Method: GetRelatedTagsInternal() // /// <summary> /// Gets the Flickr tags related to a specified tag, given a /// GraphXMLXmlDocument. /// </summary> /// /// <param name="sTag"> /// Tag to get related tags for. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="eNetworkLevel"> /// Network level to include. Must be NetworkLevel.One, OnePointFive, or /// Two. /// </param> /// /// <param name="sApiKey"> /// Flickr API key. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// The GraphMLXmlDocument to populate with the requested network. /// </param> //************************************************************************* protected void GetRelatedTagsInternal( String sTag, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, String sApiKey, RequestStatistics oRequestStatistics, GraphMLXmlDocument oGraphMLXmlDocument ) { Debug.Assert( !String.IsNullOrEmpty(sTag) ); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert( !String.IsNullOrEmpty(sApiKey) ); Debug.Assert(oRequestStatistics != null); Debug.Assert(oGraphMLXmlDocument != null); AssertValid(); // The key is the tag name and the value is the corresponding GraphML // XML node that represents the tag. This is used to prevent the same // tag from being added to the XmlDocument twice. Dictionary<String, XmlNode> oTagDictionary = new Dictionary<String, XmlNode>(); GetRelatedTagsRecursive(sTag, eWhatToInclude, eNetworkLevel, sApiKey, 1, oGraphMLXmlDocument, oTagDictionary, oRequestStatistics); if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SampleThumbnails) ) { AppendSampleThumbnails(oTagDictionary, oGraphMLXmlDocument, sApiKey, oRequestStatistics); } }
//************************************************************************* // Method: GetRelatedTagsRecursive() // /// <summary> /// Recursively gets a tag's related tags. /// </summary> /// /// <param name="sTag"> /// Tag to get related tags for. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="eNetworkLevel"> /// Network level to include. Must be NetworkLevel.One, OnePointFive, or /// Two. /// </param> /// /// <param name="sApiKey"> /// Flickr API key. /// </param> /// /// <param name="iRecursionLevel"> /// Recursion level for this call. Must be 1 or 2. Gets incremented when /// recursing. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// GraphMLXmlDocument being populated. /// </param> /// /// <param name="oTagDictionary"> /// The key is the tag name and the value is the corresponding GraphML XML /// node that represents the tag. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> //************************************************************************* protected void GetRelatedTagsRecursive( String sTag, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, String sApiKey, Int32 iRecursionLevel, GraphMLXmlDocument oGraphMLXmlDocument, Dictionary<String, XmlNode> oTagDictionary, RequestStatistics oRequestStatistics ) { Debug.Assert( !String.IsNullOrEmpty(sTag) ); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert( !String.IsNullOrEmpty(sApiKey) ); Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oTagDictionary != null); Debug.Assert(oRequestStatistics != null); AssertValid(); /* Here is what this method should do, based on the eNetworkLevel and iRecursionLevel parameters. eNetworkLevel |One | OnePointFive | Two ---|------------------| ------------------| ----------------- i 1 |Add all vertices. | Add all vertices. | Add all vertices. R | | | e |Add all edges. | Add all edges. | Add all edges. c | | | u |Do not recurse. | Recurse. | Recurse. r | | | s ---|------------------|-------------------|------------------ i 2 |Impossible. | Do not add | Add all vertices. o | | vertices. | n | | | L | | Add edges only if | Add all edges. e | | vertices are | v | | already included. | e | | | l | | Do not recurse. | Do not recurse. | | | ---|------------------|-------------------|------------------ */ Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel, iRecursionLevel); Boolean bNeedToAppendVertices = GetNeedToAppendVertices( eNetworkLevel, iRecursionLevel); ReportProgress("Getting tags related to \"" + sTag + "\"."); String sUrl = GetFlickrMethodUrl( "flickr.tags.getRelated", sApiKey, "&tag=" + UrlUtil.EncodeUrlParameter(sTag) ); XmlDocument oXmlDocument; try { oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics); } catch (Exception oException) { // If the exception is not a WebException or XmlException, or if // none of the network has been obtained yet, throw the exception. if (!ExceptionIsWebOrXml(oException) || !oGraphMLXmlDocument.HasVertexXmlNode) { throw oException; } return; } // The document consists of a single "tags" node with zero or more // "tag" child nodes. String sOtherTag = null; XmlNodeList oTagNodes = oXmlDocument.DocumentElement.SelectNodes( "tags/tag"); if (oTagNodes.Count > 0) { AppendVertexXmlNode(sTag, oGraphMLXmlDocument, oTagDictionary); } foreach (XmlNode oTagNode in oTagNodes) { sOtherTag = XmlUtil2.SelectRequiredSingleNodeAsString(oTagNode, "text()", null); if (bNeedToAppendVertices) { AppendVertexXmlNode(sOtherTag, oGraphMLXmlDocument, oTagDictionary); } if ( bNeedToAppendVertices || oTagDictionary.ContainsKey(sOtherTag) ) { oGraphMLXmlDocument.AppendEdgeXmlNode(sTag, sOtherTag); } } if (bNeedToRecurse) { foreach (XmlNode oTagNode in oTagNodes) { sOtherTag = XmlUtil2.SelectRequiredSingleNodeAsString(oTagNode, "text()", null); GetRelatedTagsRecursive(sOtherTag, eWhatToInclude, eNetworkLevel, sApiKey, 2, oGraphMLXmlDocument, oTagDictionary, oRequestStatistics); } } }
//************************************************************************* // Method: OnNetworkObtainedWithoutTerminatingException() // /// <summary> /// Call this when part or all of the network has been obtained without a /// terminating exception occurring. /// </summary> /// /// <param name="oGraphMLXmlDocument"> /// GraphMLXmlDocument being populated. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <remarks> /// If the entire network has been obtained, this method does nothing. /// Otherwise, a PartialNetworkException is thrown. /// </remarks> //************************************************************************* protected void OnNetworkObtainedWithoutTerminatingException( GraphMLXmlDocument oGraphMLXmlDocument, RequestStatistics oRequestStatistics ) { Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oRequestStatistics != null); AssertValid(); if (oRequestStatistics.UnexpectedExceptions > 0) { // The network is partial. throw new PartialNetworkException(oGraphMLXmlDocument, oRequestStatistics); } }
//************************************************************************* // Method: AppendFollowedEdgeXmlNodes() // /// <summary> /// Appends an edge XML node for each pair of people who have tweeted a /// specified search term and one follows the other. /// </summary> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="iMaximumPeoplePerRequest"> /// Maximum number of people to request for each query, or Int32.MaxValue /// for no limit. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// GraphMLXmlDocument being populated. /// </param> /// /// <param name="oScreenNameDictionary"> /// The key is the screen name in lower case and the value is the /// corresponding TwitterVertex. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> //************************************************************************* protected void AppendFollowedEdgeXmlNodes( WhatToInclude eWhatToInclude, Int32 iMaximumPeoplePerRequest, GraphMLXmlDocument oGraphMLXmlDocument, Dictionary<String, TwitterVertex> oScreenNameDictionary, RequestStatistics oRequestStatistics ) { Debug.Assert(iMaximumPeoplePerRequest > 0); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oScreenNameDictionary != null); Debug.Assert(oRequestStatistics != null); AssertValid(); Boolean bIncludeStatistics = WhatToIncludeFlagIsSet( eWhatToInclude, WhatToInclude.Statistics); // Look at the people followed by each author, and if a followed has // also tweeted the search term, add an edge between the author and the // followed. foreach (String sScreenName in oScreenNameDictionary.Keys) { ReportProgressForFollowedOrFollowing(sScreenName, true); foreach ( XmlNode oOtherUserXmlNode in EnumerateXmlNodes( GetFollowedOrFollowingUrl(sScreenName, true), "users_list/users/user", null, null, Int32.MaxValue, iMaximumPeoplePerRequest, false, true, oRequestStatistics) ) { String sOtherScreenName; if ( !TryGetScreenName(oOtherUserXmlNode, out sOtherScreenName) ) { continue; } TwitterVertex oOtherTwitterVertex; if ( oScreenNameDictionary.TryGetValue(sOtherScreenName, out oOtherTwitterVertex) ) { XmlNode oOtherVertexXmlNode = oOtherTwitterVertex.VertexXmlNode; XmlNode oEdgeXmlNode = AppendEdgeXmlNode( oGraphMLXmlDocument, sScreenName, sOtherScreenName, "Followed"); AppendRelationshipDateUtcGraphMLAttributeValue( oGraphMLXmlDocument, oEdgeXmlNode, oRequestStatistics); if (bIncludeStatistics && !AppendFromUserXmlNodeCalled(oGraphMLXmlDocument, oOtherVertexXmlNode) ) { // The other vertex node has no statistics. Add them. AppendFromUserXmlNode(oOtherUserXmlNode, oGraphMLXmlDocument, oOtherTwitterVertex, bIncludeStatistics, false); } } } } }
//************************************************************************* // Method: AppendVertexXmlNodes() // /// <summary> /// Appends a vertex XML node for each person who has tweeted a specified /// search term. /// </summary> /// /// <param name="sSearchTerm"> /// The term to search for. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="iMaximumPeoplePerRequest"> /// Maximum number of people to request for each query, or Int32.MaxValue /// for no limit. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// The GraphMLXmlDocument being populated. /// </param> /// /// <param name="oScreenNameDictionary"> /// The key is the screen name in lower case and the value is the /// corresponding TwitterVertex. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> //************************************************************************* protected void AppendVertexXmlNodes( String sSearchTerm, WhatToInclude eWhatToInclude, Int32 iMaximumPeoplePerRequest, GraphMLXmlDocument oGraphMLXmlDocument, Dictionary<String, TwitterVertex> oScreenNameDictionary, RequestStatistics oRequestStatistics ) { Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) ); Debug.Assert(iMaximumPeoplePerRequest > 0); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oScreenNameDictionary != null); Debug.Assert(oRequestStatistics != null); AssertValid(); // Convert spaces in the search term to a plus sign. // // (Note: Don't try to use Twitter's "show_user" URL parameter in an // attempt to get a "user" XML node for each author. That's not what // the URL parameter does.) String sUrl = String.Format( "http://search.twitter.com/search.atom?q={0}&rpp=100" , UrlUtil.EncodeUrlParameter(sSearchTerm).Replace("%20", "+") ); ReportProgress("Getting a list of tweets."); Boolean bIncludeStatistics = WhatToIncludeFlagIsSet( eWhatToInclude, WhatToInclude.Statistics); Boolean bIncludeStatuses = WhatToIncludeFlagIsSet( eWhatToInclude, WhatToInclude.Statuses); // The document consists of an "entry" XML node for each tweet that // contains the search term. Multiple tweets may have the same author, // so we have to enumerate until the requested maximum number of unique // authors is reached. foreach ( XmlNode oAuthorNameXmlNode in EnumerateXmlNodes( sUrl, "a:feed/a:entry/a:author/a:name", "a", AtomNamespaceUri, 15, Int32.MaxValue, true, false, oRequestStatistics) ) { if (oScreenNameDictionary.Count == iMaximumPeoplePerRequest) { break; } // The author name is in this format: // // james_laker (James Laker) String sAuthorName; if ( !XmlUtil2.TrySelectSingleNodeAsString(oAuthorNameXmlNode, "text()", null, out sAuthorName) ) { continue; } Int32 iIndexOfSpace = sAuthorName.IndexOf(' '); String sScreenName = sAuthorName; if (iIndexOfSpace != -1) { sScreenName = sAuthorName.Substring(0, iIndexOfSpace); } sScreenName = sScreenName.ToLower(); TwitterVertex oTwitterVertex; if ( !TryAppendVertexXmlNode(sScreenName, null, oGraphMLXmlDocument, oScreenNameDictionary, bIncludeStatistics, false, out oTwitterVertex) ) { // A tweet for the author has already been found. continue; } XmlNode oVertexXmlNode = oTwitterVertex.VertexXmlNode; XmlNode oEntryXmlNode = oAuthorNameXmlNode.ParentNode.ParentNode; XmlNamespaceManager oXmlNamespaceManager = new XmlNamespaceManager( oEntryXmlNode.OwnerDocument.NameTable); oXmlNamespaceManager.AddNamespace("a", AtomNamespaceUri); // Get the image URL and status for the tweet's author. AppendStringGraphMLAttributeValue(oEntryXmlNode, "a:link[@rel='image']/@href", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, ImageFileID); String sStatus; if ( XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode, "a:title/text()", oXmlNamespaceManager, out sStatus) ) { String sStatusDateUtc; if ( XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode, "a:published/text()", oXmlNamespaceManager, out sStatusDateUtc) ) { sStatusDateUtc = TwitterDateParser.ParseTwitterDate( sStatusDateUtc); } oTwitterVertex.StatusForAnalysis = sStatus; oTwitterVertex.StatusForAnalysisDateUtc = sStatusDateUtc; if (bIncludeStatuses) { oGraphMLXmlDocument.AppendGraphMLAttributeValue( oVertexXmlNode, StatusID, sStatus); if ( !String.IsNullOrEmpty(sStatusDateUtc) ) { oGraphMLXmlDocument.AppendGraphMLAttributeValue( oVertexXmlNode, StatusDateUtcID, sStatusDateUtc); } } } } }
//************************************************************************* // Method: GetUserNetworkRecursive() // /// <summary> /// Recursively gets a network of Twitter users. /// </summary> /// /// <param name="sScreenName"> /// The screen name to use in this call. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="bIncludeFollowedThisCall"> /// true to include the people followed by the user, false to include the /// people following the user. /// </param> /// /// <param name="eNetworkLevel"> /// Network level to include. Must be NetworkLevel.One, OnePointFive, or /// Two. /// </param> /// /// <param name="iMaximumPeoplePerRequest"> /// Maximum number of people to request for each query, or Int32.MaxValue /// for no limit. /// </param> /// /// <param name="iRecursionLevel"> /// Recursion level for this call. Must be 1 or 2. Gets incremented when /// recursing. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// GraphMLXmlDocument being populated. /// </param> /// /// <param name="oScreenNameDictionary"> /// The key is the screen name in lower case and the value is the /// corresponding TwitterVertex. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> //************************************************************************* protected void GetUserNetworkRecursive( String sScreenName, WhatToInclude eWhatToInclude, Boolean bIncludeFollowedThisCall, NetworkLevel eNetworkLevel, Int32 iMaximumPeoplePerRequest, Int32 iRecursionLevel, GraphMLXmlDocument oGraphMLXmlDocument, Dictionary<String, TwitterVertex> oScreenNameDictionary, RequestStatistics oRequestStatistics ) { Debug.Assert( !String.IsNullOrEmpty(sScreenName) ); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert(iMaximumPeoplePerRequest > 0); Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oScreenNameDictionary != null); Debug.Assert(oRequestStatistics != null); AssertValid(); /* Here is what this method should do, based on the eNetworkLevel and iRecursionLevel parameters. It's assumed that eWhatToInclude.FollowedFollowerEdge is set. eNetworkLevel |One | OnePointFive | Two ---|------------------| ------------------| ----------------- i 1 |Add all vertices. | Add all vertices. | Add all vertices. R | | | e |Add all edges. | Add all edges. | Add all edges. c | | | u |Do not recurse. | Recurse. | Recurse. r | | | s ---|------------------|-------------------|------------------ i 2 |Impossible. | Do not add | Add all vertices. o | | vertices. | n | | | L | | Add edges only if | Add all edges. e | | vertices are | v | | already included. | e | | | l | | Do not recurse. | Do not recurse. | | | ---|------------------|-------------------|------------------ */ Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel, iRecursionLevel); List<String> oScreenNamesToRecurse = new List<String>(); Boolean bIncludeLatestStatuses = WhatToIncludeFlagIsSet( eWhatToInclude, WhatToInclude.LatestStatuses); ReportProgressForFollowedOrFollowing(sScreenName, bIncludeFollowedThisCall); Boolean bThisUserAppended = false; // If the GraphMLXmlDocument already contains at least one vertex node, // then this is the second time that this method has been called, a // partial network has already been obtained, and most errors should // now be skipped. However, if none of the network has been obtained // yet, errors on page 1 should throw an immediate exception. Boolean bSkipMostPage1Errors = oGraphMLXmlDocument.HasVertexXmlNode; // The document consists of a single "users" node with zero or more // "user" child nodes. foreach ( XmlNode oUserXmlNode in EnumerateXmlNodes( GetFollowedOrFollowingUrl(sScreenName, bIncludeFollowedThisCall), "users_list/users/user", null, null, Int32.MaxValue, iMaximumPeoplePerRequest, false, bSkipMostPage1Errors, oRequestStatistics) ) { String sOtherScreenName; if ( !TryGetScreenName(oUserXmlNode, out sOtherScreenName) ) { // Nothing can be done without a screen name. continue; } if (!bThisUserAppended) { // Append a vertex node for this request's user. // // This used to be done after the foreach loop, which avoided // the need for a "bThisUserAppended" flag. That caused the // following bug: If a Twitter error occurred within // EnumerateXmlNodes() after some edges had been added, and the // user decided to import the resulting partial network, the // GraphML might contain edges that referenced "this user" // without containing a vertex for "this user." That is an // illegal state for GraphML, which the ExcelTemplate project // caught and reported as an error. TryAppendVertexXmlNode(sScreenName, null, oGraphMLXmlDocument, oScreenNameDictionary, true, bIncludeLatestStatuses); bThisUserAppended = true; } Boolean bNeedToAppendVertices = GetNeedToAppendVertices( eNetworkLevel, iRecursionLevel); if (bNeedToAppendVertices) { if ( TryAppendVertexXmlNode(sOtherScreenName, oUserXmlNode, oGraphMLXmlDocument, oScreenNameDictionary, true, bIncludeLatestStatuses) && bNeedToRecurse ) { oScreenNamesToRecurse.Add(sOtherScreenName); } } if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.FollowedFollowerEdges) ) { if ( bNeedToAppendVertices || oScreenNameDictionary.ContainsKey(sOtherScreenName) ) { XmlNode oEdgeXmlNode; if (bIncludeFollowedThisCall) { oEdgeXmlNode = AppendEdgeXmlNode(oGraphMLXmlDocument, sScreenName, sOtherScreenName, "Followed"); } else { oEdgeXmlNode = AppendEdgeXmlNode(oGraphMLXmlDocument, sOtherScreenName, sScreenName, "Follower"); } AppendRelationshipDateUtcGraphMLAttributeValue( oGraphMLXmlDocument, oEdgeXmlNode, oRequestStatistics); } } } if (bNeedToRecurse) { foreach (String sScreenNameToRecurse in oScreenNamesToRecurse) { GetUserNetworkRecursive(sScreenNameToRecurse, eWhatToInclude, bIncludeFollowedThisCall, eNetworkLevel, iMaximumPeoplePerRequest, 2, oGraphMLXmlDocument, oScreenNameDictionary, oRequestStatistics); } } }
//************************************************************************* // Method: GetSearchNetworkInternal() // /// <overloads> /// Gets the network of people who have tweeted a specified search term. /// </overloads> /// /// <summary> /// Gets the network of people who have tweeted a specified search term. /// </summary> /// /// <param name="sSearchTerm"> /// The term to search for. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="iMaximumPeoplePerRequest"> /// Maximum number of people to request for each query, or Int32.MaxValue /// for no limit. /// </param> /// /// <returns> /// An XmlDocument containing the network as GraphML. /// </returns> //************************************************************************* protected XmlDocument GetSearchNetworkInternal( String sSearchTerm, WhatToInclude eWhatToInclude, Int32 iMaximumPeoplePerRequest ) { Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) ); Debug.Assert(iMaximumPeoplePerRequest > 0); AssertValid(); BeforeGetNetwork(); Boolean bIncludeStatistics = WhatToIncludeFlagIsSet( eWhatToInclude, WhatToInclude.Statistics); GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument( bIncludeStatistics, false); RequestStatistics oRequestStatistics = new RequestStatistics(); try { GetSearchNetworkInternal(sSearchTerm, eWhatToInclude, iMaximumPeoplePerRequest, oRequestStatistics, oGraphMLXmlDocument); } catch (Exception oException) { OnTerminatingException(oException); } OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument, oRequestStatistics); return (oGraphMLXmlDocument); }
//************************************************************************* // Method: AppendVertexXmlNodes() // /// <summary> /// Appends a vertex XML node for each video that matches a specified /// search term. /// </summary> /// /// <param name="sSearchTerm"> /// The term to search for. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="iMaximumVideos"> /// Maximum number of videos to request, or Int32.MaxValue for no limit. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// The GraphMLXmlDocument being populated. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <param name="oVideoIDs"> /// Where the set of unique video IDs gets stored. /// </param> /// /// <param name="oTagDictionary"> /// If an edge should be included for each pair of videos that share the /// same tag, this gets set to a Dictionary for which the key is a /// lower-case tag and the value is a LinkedList of the video IDs that have /// the tag. Otherwise, it gets set to null. /// </param> //************************************************************************* protected void AppendVertexXmlNodes( String sSearchTerm, WhatToInclude eWhatToInclude, Int32 iMaximumVideos, GraphMLXmlDocument oGraphMLXmlDocument, RequestStatistics oRequestStatistics, out HashSet<String> oVideoIDs, out Dictionary< String, LinkedList<String> > oTagDictionary ) { Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) ); Debug.Assert(iMaximumVideos> 0); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oRequestStatistics != null); AssertValid(); ReportProgress("Getting a list of videos."); // This is used to skip duplicate videos in the results returned by // YouTube. (I'm not sure why YouTube sometimes returns duplicates, // but it does.) oVideoIDs = new HashSet<String>(); // If an edge should be included for each pair of videos that share the // same tag, the key is a lower-case tag and the value is a LinkedList // of the video IDs that have the tag. if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SharedTagEdges) ) { oTagDictionary = new Dictionary< String, LinkedList<String> >(); } else { oTagDictionary = null; } String sUrl = String.Format( "http://gdata.youtube.com/feeds/api/videos?q={0}" , EncodeUrlParameter(sSearchTerm) ); // The document consists of an "entry" XML node for each video. foreach ( XmlNode oEntryXmlNode in EnumerateXmlNodes(sUrl, "a:feed/a:entry", iMaximumVideos, false, oRequestStatistics) ) { XmlNamespaceManager oXmlNamespaceManager = CreateXmlNamespaceManager(oEntryXmlNode.OwnerDocument); // Use the video ID as the GraphML vertex name. The video title // can't be used because it is not unique. String sVideoID; if ( !XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode, "media:group/yt:videoid/text()", oXmlNamespaceManager, out sVideoID) || oVideoIDs.Contains(sVideoID) ) { continue; } oVideoIDs.Add(sVideoID); XmlNode oVertexXmlNode = oGraphMLXmlDocument.AppendVertexXmlNode( sVideoID); AppendStringGraphMLAttributeValue(oEntryXmlNode, "a:title/text()", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, TitleID); AppendDoubleGraphMLAttributeValue(oEntryXmlNode, "gd:rating/@average", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, RatingID); AppendInt32GraphMLAttributeValue(oEntryXmlNode, "yt:statistics/@viewCount", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, ViewsID); AppendInt32GraphMLAttributeValue(oEntryXmlNode, "yt:statistics/@favoriteCount", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, FavoritedID); AppendInt32GraphMLAttributeValue(oEntryXmlNode, "gd:comments/gd:feedLink/@countHint", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, CommentsID); AppendYouTubeDateGraphMLAttributeValue(oEntryXmlNode, "a:published/text()", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, CreatedDateUtcID); AppendStringGraphMLAttributeValue(oEntryXmlNode, "media:group/media:thumbnail/@url", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, ImageFileID); if ( AppendStringGraphMLAttributeValue(oEntryXmlNode, "media:group/media:player/@url", oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode, MenuActionID) ) { oGraphMLXmlDocument.AppendGraphMLAttributeValue(oVertexXmlNode, MenuTextID, "Play Video in Browser"); } if (oTagDictionary != null) { CollectTags(oEntryXmlNode, sVideoID, oXmlNamespaceManager, oTagDictionary); } } }
//************************************************************************* // Method: GetSearchNetworkInternal() // /// <summary> /// Gets the network of people who have tweeted a specified search term, /// given a GraphMLXmlDocument. /// </summary> /// /// <param name="sSearchTerm"> /// The term to search for. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="iMaximumPeoplePerRequest"> /// Maximum number of people to request for each query, or Int32.MaxValue /// for no limit. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// The GraphMLXmlDocument to populate with the requested network. /// </param> //************************************************************************* protected void GetSearchNetworkInternal( String sSearchTerm, WhatToInclude eWhatToInclude, Int32 iMaximumPeoplePerRequest, RequestStatistics oRequestStatistics, GraphMLXmlDocument oGraphMLXmlDocument ) { Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) ); Debug.Assert(iMaximumPeoplePerRequest > 0); Debug.Assert(oRequestStatistics != null); Debug.Assert(oGraphMLXmlDocument != null); AssertValid(); if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.Statuses) ) { oGraphMLXmlDocument.DefineGraphMLAttribute(false, StatusID, "Tweet", "string", null); oGraphMLXmlDocument.DefineGraphMLAttribute(false, StatusDateUtcID, "Tweet Date (UTC)", "string", null); } // The key is the screen name and the value is the corresponding // TwitterVertex. This is used to prevent the same screen name from // being added to the XmlDocument twice. Dictionary<String, TwitterVertex> oScreenNameDictionary = new Dictionary<String, TwitterVertex>(); // First, add a vertex for each person who has tweeted the search term. AppendVertexXmlNodes(sSearchTerm, eWhatToInclude, iMaximumPeoplePerRequest, oGraphMLXmlDocument, oScreenNameDictionary, oRequestStatistics); if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.FollowedEdges) ) { // Look at the people followed by each author, and if a followed // has also tweeted the search term, add an edge between the author // and the followed. AppendFollowedEdgeXmlNodes(eWhatToInclude, iMaximumPeoplePerRequest, oGraphMLXmlDocument, oScreenNameDictionary, oRequestStatistics); } Boolean bIncludeStatistics = WhatToIncludeFlagIsSet( eWhatToInclude, WhatToInclude.Statistics); AppendMissingGraphMLAttributeValues(oGraphMLXmlDocument, oScreenNameDictionary, bIncludeStatistics, false, oRequestStatistics); AppendRepliesToAndMentionsXmlNodes(oGraphMLXmlDocument, oScreenNameDictionary, WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.RepliesToEdges), WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.MentionsEdges) ); }
//************************************************************************* // Method: GetVideoNetworkInternal() // /// <summary> /// Gets a network of related YouTube videos, given a GraphMLXmlDocument. /// </summary> /// /// <param name="sSearchTerm"> /// The term to search for. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="iMaximumVideos"> /// Maximum number of videos to request, or Int32.MaxValue for no limit. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// The GraphMLXmlDocument to populate with the requested network. /// </param> //************************************************************************* protected void GetVideoNetworkInternal( String sSearchTerm, WhatToInclude eWhatToInclude, Int32 iMaximumVideos, RequestStatistics oRequestStatistics, GraphMLXmlDocument oGraphMLXmlDocument ) { Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) ); Debug.Assert(iMaximumVideos > 0); Debug.Assert(oRequestStatistics != null); Debug.Assert(oGraphMLXmlDocument != null); AssertValid(); // First, add a vertex for each video matching the search term. HashSet<String> oVideoIDs; Dictionary< String, LinkedList<String> > oTagDictionary; AppendVertexXmlNodes(sSearchTerm, eWhatToInclude, iMaximumVideos, oGraphMLXmlDocument, oRequestStatistics, out oVideoIDs, out oTagDictionary); // Now add whatever edges were requested. if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SharedTagEdges) ) { Debug.Assert(oTagDictionary != null); ReportProgress("Adding edges for shared tags."); AppendEdgesFromDictionary(oTagDictionary, oGraphMLXmlDocument, "Shared tag"); } oTagDictionary = null; if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SharedCommenterEdges) ) { AppendSharedResponderEdges(oGraphMLXmlDocument, oVideoIDs, MaximumCommentsPerVideo, "http://gdata.youtube.com/feeds/api/videos/{0}/comments", "commenter", oRequestStatistics); } if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SharedVideoResponderEdges) ) { AppendSharedResponderEdges(oGraphMLXmlDocument, oVideoIDs, iMaximumVideos, "http://gdata.youtube.com/feeds/api/videos/{0}/responses", "video responder", oRequestStatistics); } }
//************************************************************************* // Method: AppendSampleThumbnails() // /// <summary> /// Appends sample thumbnails to the GraphMLXmlDocument being populated. /// </summary> /// /// <param name="oTagDictionary"> /// The key is the tag name and the value is the corresponding GraphML XML /// node that represents the tag. /// </param> /// /// <param name="oGraphMLXmlDocument"> /// GraphMLXmlDocument being populated. /// </param> /// /// <param name="sApiKey"> /// Flickr API key. /// </param> /// /// <param name="oRequestStatistics"> /// A <see cref="RequestStatistics" /> object that is keeping track of /// requests made while getting the network. /// </param> //************************************************************************* protected void AppendSampleThumbnails( Dictionary<String, XmlNode> oTagDictionary, GraphMLXmlDocument oGraphMLXmlDocument, String sApiKey, RequestStatistics oRequestStatistics ) { Debug.Assert(oTagDictionary != null); Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert( !String.IsNullOrEmpty(sApiKey) ); Debug.Assert(oRequestStatistics != null); AssertValid(); foreach (KeyValuePair<String, XmlNode> oKeyValuePair in oTagDictionary) { String sTag = oKeyValuePair.Key; ReportProgress("Getting sample image file for \"" + sTag + "\"."); String sSampleImageUrl; if ( TryGetSampleImageUrl(sTag, sApiKey, oRequestStatistics, out sSampleImageUrl) ) { oGraphMLXmlDocument.AppendGraphMLAttributeValue( oKeyValuePair.Value, ImageFileID, sSampleImageUrl); } } }
OnNetworkObtainedWithoutTerminatingException ( GraphMLXmlDocument oGraphMLXmlDocument, RequestStatistics oRequestStatistics, String sNetworkDescription ) { Debug.Assert(oGraphMLXmlDocument != null); Debug.Assert(oRequestStatistics != null); Debug.Assert( !String.IsNullOrEmpty(sNetworkDescription) ); AssertValid(); XmlUtil2.SetAttributes(oGraphMLXmlDocument.GraphXmlNode, "description", sNetworkDescription); if (oRequestStatistics.UnexpectedExceptions > 0) { // The network is partial. throw new PartialNetworkException(oGraphMLXmlDocument, oRequestStatistics); } }
//************************************************************************* // Method: GetRelatedTagsInternal() // /// <overloads> /// Gets the Flickr tags related to a specified tag. /// </overloads> /// /// <summary> /// Gets the Flickr tags related to a specified tag. /// </summary> /// /// <param name="sTag"> /// Tag to get related tags for. /// </param> /// /// <param name="eWhatToInclude"> /// Specifies what should be included in the network. /// </param> /// /// <param name="eNetworkLevel"> /// Network level to include. Must be NetworkLevel.One, OnePointFive, or /// Two. /// </param> /// /// <param name="sApiKey"> /// Flickr API key. /// </param> /// /// <returns> /// An XmlDocument containing the network as GraphML. /// </returns> //************************************************************************* protected XmlDocument GetRelatedTagsInternal( String sTag, WhatToInclude eWhatToInclude, NetworkLevel eNetworkLevel, String sApiKey ) { Debug.Assert( !String.IsNullOrEmpty(sTag) ); Debug.Assert(eNetworkLevel == NetworkLevel.One || eNetworkLevel == NetworkLevel.OnePointFive || eNetworkLevel == NetworkLevel.Two); Debug.Assert( !String.IsNullOrEmpty(sApiKey) ); AssertValid(); GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.SampleThumbnails) ); RequestStatistics oRequestStatistics = new RequestStatistics(); try { GetRelatedTagsInternal(sTag, eWhatToInclude, eNetworkLevel, sApiKey, oRequestStatistics, oGraphMLXmlDocument); } catch (Exception oException) { OnTerminatingException(oException); } OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument, oRequestStatistics); return (oGraphMLXmlDocument); }
EnumerateSearchStatuses ( String searchTerm, Int32 maximumStatuses, RequestStatistics requestStatistics, ReportProgressHandler reportProgressHandler, CheckCancellationPendingHandler checkCancellationPendingHandler ) { Debug.Assert( !String.IsNullOrEmpty(searchTerm) ); Debug.Assert(maximumStatuses > 0); Debug.Assert(maximumStatuses != Int32.MaxValue); Debug.Assert(requestStatistics != null); AssertValid(); Int32 iPage = 1; Int32 iStatusesEnumerated = 0; String sQueryParametersForNextPage = null; while (true) { if (reportProgressHandler != null && iPage > 1) { reportProgressHandler("Getting page " + iPage + "."); } Dictionary<String, Object> oResponseDictionary = null; Object [] aoStatusesThisPage; String sUrl = GetSearchUrl(searchTerm, sQueryParametersForNextPage); try { Object oDeserializedTwitterResponse = ( new JavaScriptSerializer() ).DeserializeObject( GetTwitterResponseAsString( sUrl, requestStatistics, reportProgressHandler, checkCancellationPendingHandler) ); // The top level of the Json response contains a set of // name/value pairs. The value for the "statuses" name is the // array of objects this method will enumerate. oResponseDictionary = ( Dictionary<String, Object> ) oDeserializedTwitterResponse; aoStatusesThisPage = ( Object [] )oResponseDictionary["statuses"]; } catch (Exception oException) { // Rethrow the exception if appropriate. OnExceptionWhileEnumeratingJsonValues(oException, iPage, false); // Otherwise, just halt the enumeration. yield break; } Int32 iObjectsThisPage = aoStatusesThisPage.Length; if (iObjectsThisPage == 0) { yield break; } for (Int32 i = 0; i < iObjectsThisPage; i++) { yield return ( ( Dictionary<String, Object> )aoStatusesThisPage[i] ); iStatusesEnumerated++; if (iStatusesEnumerated == maximumStatuses) { yield break; } } iPage++; if ( !TryGetQueryParametersForNextSearchPage(oResponseDictionary, out sQueryParametersForNextPage) ) { yield break; } // Get the next page... } }
GetTwitterResponseAsString ( String url, RequestStatistics requestStatistics, ReportProgressHandler reportProgressHandler, CheckCancellationPendingHandler checkCancellationPendingHandler ) { Debug.Assert( !String.IsNullOrEmpty(url) ); Debug.Assert(requestStatistics != null); AssertValid(); Int32 iRateLimitPauses = 0; while (true) { // Add the required authorization information to the URL. // // Note: Don't do this outside the while (true) loop. The // authorization information includes a timestamp that will // probably expire if the code within the catch block pauses. oAuthTwitter oAuthTwitter = new oAuthTwitter( m_UserAgent, m_TimeoutMs); oAuthTwitter.Token = m_TwitterAccessToken; oAuthTwitter.TokenSecret = m_TwitterAccessTokenSecret; String sAuthorizedUrl, sAuthorizedPostData; oAuthTwitter.ConstructAuthWebRequest(oAuthTwitter.Method.GET, url, String.Empty, out sAuthorizedUrl, out sAuthorizedPostData); url = sAuthorizedUrl; Stream oStream = null; try { oStream = HttpSocialNetworkUtil.GetHttpWebResponseStreamWithRetries( url, HttpStatusCodesToFailImmediately, requestStatistics, m_UserAgent, m_TimeoutMs, reportProgressHandler, checkCancellationPendingHandler); return ( new StreamReader(oStream).ReadToEnd() ); } catch (WebException oWebException) { if ( !WebExceptionIsDueToRateLimit(oWebException) || iRateLimitPauses > 0 ) { throw; } // Twitter rate limits have kicked in. Pause and try again. iRateLimitPauses++; Int32 iRateLimitPauseMs = GetRateLimitPauseMs(oWebException); DateTime oWakeUpTime = DateTime.Now.AddMilliseconds( iRateLimitPauseMs); if (reportProgressHandler != null) { reportProgressHandler( String.Format( "Reached Twitter rate limits. Pausing until {0}." , oWakeUpTime.ToLongTimeString() ) ); } // Don't pause in one large interval, which would prevent // cancellation. const Int32 SleepCycleDurationMs = 1000; Int32 iSleepCycles = (Int32)Math.Ceiling( (Double)iRateLimitPauseMs / SleepCycleDurationMs) ; for (Int32 i = 0; i < iSleepCycles; i++) { if (checkCancellationPendingHandler != null) { checkCancellationPendingHandler(); } System.Threading.Thread.Sleep(SleepCycleDurationMs); } } finally { if (oStream != null) { oStream.Close(); } } } }