CreateSnaTitle
(
String searchTerm,
RequestStatistics requestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(searchTerm));
Debug.Assert(requestStatistics != null);
// Sample title, where searchTerm is "usfca":
//
// "usfca Twitter NodeXL SNA Map and Report for Saturday,
// 05 April 2014 at 18:47 UTC"
DateTime oStartTimeUtc = requestStatistics.StartTimeUtc;
return(String.Format(
"{0} Twitter NodeXL SNA Map and Report for {1} at {2} UTC"
,
searchTerm,
oStartTimeUtc.ToString("dddd, dd MMMM yyyy"),
oStartTimeUtc.ToString("HH:mm")
));
}
TryGetXmlDocument
(
String sUrl,
RequestStatistics oRequestStatistics,
out XmlDocument oXmlDocument,
out XmlNamespaceManager oXmlNamespaceManager
)
{
Debug.Assert( !String.IsNullOrEmpty(sUrl) );
Debug.Assert(oRequestStatistics != null);
AssertValid();
oXmlDocument = null;
oXmlNamespaceManager = null;
try
{
oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics,
out oXmlNamespaceManager);
return (true);
}
catch (Exception oException)
{
if ( !HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException))
{
throw oException;
}
return (false);
}
}
GetVideoNetworkInternal
(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumVideos
)
{
Debug.Assert(!String.IsNullOrEmpty(sSearchTerm));
Debug.Assert(iMaximumVideos > 0);
AssertValid();
GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument();
RequestStatistics oRequestStatistics = new RequestStatistics();
try
{
GetVideoNetworkInternal(sSearchTerm, eWhatToInclude,
iMaximumVideos, oRequestStatistics, oGraphMLXmlDocument);
}
catch (Exception oException)
{
OnTerminatingException(oException);
}
OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument,
oRequestStatistics);
return(oGraphMLXmlDocument);
}
AppendAllStatisticGraphMLAttributeValues
(
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary <String, XmlNode> oUserNameDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oUserNameDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
foreach (KeyValuePair <String, XmlNode> oKeyValuePair in
oUserNameDictionary)
{
String sUserName = oKeyValuePair.Key;
ReportProgress(String.Format(
"Getting statistics for \"{0}\"."
,
sUserName
));
AppendAllStatisticGraphMLAttributeValues(sUserName,
oKeyValuePair.Value, oGraphMLXmlDocument, oRequestStatistics);
}
}
AppendSampleThumbnails
(
Dictionary <String, XmlNode> oTagDictionary,
GraphMLXmlDocument oGraphMLXmlDocument,
String sApiKey,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(oTagDictionary != null);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
Debug.Assert(oRequestStatistics != null);
AssertValid();
foreach (KeyValuePair <String, XmlNode> oKeyValuePair in oTagDictionary)
{
String sTag = oKeyValuePair.Key;
ReportProgress("Getting sample image file for \"" + sTag + "\".");
String sSampleImageUrl;
if (TryGetSampleImageUrl(sTag, sApiKey, oRequestStatistics,
out sSampleImageUrl))
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(
oKeyValuePair.Value, NodeXLGraphMLUtil.VertexImageFileID,
sSampleImageUrl);
}
}
}
AppendUserInformationGraphMLAttributeValues
(
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary <String, XmlNode> oUserIDDictionary,
String sApiKey,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oUserIDDictionary != null);
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
Debug.Assert(oRequestStatistics != null);
AssertValid();
foreach (KeyValuePair <String, XmlNode> oKeyValuePair in
oUserIDDictionary)
{
String sUserID = oKeyValuePair.Key;
XmlNode oVertexXmlNode = oKeyValuePair.Value;
ReportProgress(String.Format(
"Getting information about \"{0}\"."
,
GetScreenNameFromVertexXmlNode(oVertexXmlNode)
));
AppendUserInformationGraphMLAttributeValues(sUserID,
oVertexXmlNode, oGraphMLXmlDocument, sApiKey,
oRequestStatistics);
}
}
FlickrScreenNameToUserID
(
String sScreenNameAnyCase,
String sApiKey,
RequestStatistics oRequestStatistics,
out String sUserID,
out String sScreenNameCorrectCase
)
{
Debug.Assert(!String.IsNullOrEmpty(sScreenNameAnyCase));
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
Debug.Assert(oRequestStatistics != null);
AssertValid();
XmlDocument oXmlDocument = GetXmlDocument(
GetFlickrMethodUrl("flickr.people.findByUsername", sApiKey,
"&username="******"rsp/user/@nsid", null);
sScreenNameCorrectCase = XmlUtil2.SelectRequiredSingleNodeAsString(
oXmlDocument, "rsp/user/username/text()", null);
}
TryGetXmlDocument
(
String sUrl,
RequestStatistics oRequestStatistics,
out XmlDocument oXmlDocument
)
{
Debug.Assert(!String.IsNullOrEmpty(sUrl));
Debug.Assert(oRequestStatistics != null);
AssertValid();
oXmlDocument = null;
try
{
oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics);
return(true);
}
catch (Exception oException)
{
// Note: Because a FlickrException is also a WebException, the
// following logic will cause false to be returned for a
// FlickrException.
if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException))
{
throw oException;
}
return(false);
}
}
GetXmlDocument
(
String sUrl,
RequestStatistics oRequestStatistics,
out XmlNamespaceManager oXmlNamespaceManager
)
{
Debug.Assert( !String.IsNullOrEmpty(sUrl) );
Debug.Assert(oRequestStatistics != null);
AssertValid();
// Always request API version 2.
String sUrlWithVersion = String.Format(
"{0}{1}v=2"
,
sUrl,
sUrl.IndexOf('?') == -1 ? '?' : '&'
);
XmlDocument oXmlDocument = GetXmlDocumentWithRetries(sUrlWithVersion,
HttpStatusCodesToFailImmediately, oRequestStatistics);
oXmlNamespaceManager = CreateXmlNamespaceManager(oXmlDocument);
return (oXmlDocument);
}
AppendFriendOrFollowerEdgeXmlNode
(
String sScreenName1,
String sScreenName2,
Boolean bAppendFriendEdgeXmlNode,
GraphMLXmlDocument oGraphMLXmlDocument,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sScreenName1));
Debug.Assert(!String.IsNullOrEmpty(sScreenName2));
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
// Don't use "friend" or "follower" terminology here, which can be
// confusing. Instead, simply indicate who follows whom.
XmlNode oEdgeXmlNode = NodeXLGraphMLUtil.AppendEdgeXmlNode(
oGraphMLXmlDocument,
bAppendFriendEdgeXmlNode ? sScreenName1 : sScreenName2,
bAppendFriendEdgeXmlNode ? sScreenName2 : sScreenName1,
"Follows"
);
AppendStartTimeRelationshipDateUtcGraphMLAttributeValue(
oGraphMLXmlDocument, oEdgeXmlNode, oRequestStatistics);
}
AppendInt32GraphMLAttributeValue
(
String sUrl,
String sXPath,
GraphMLXmlDocument oGraphMLXmlDocument,
RequestStatistics oRequestStatistics,
XmlNode oVertexXmlNode,
String sGraphMLAttributeID
)
{
Debug.Assert(!String.IsNullOrEmpty(sUrl));
Debug.Assert(!String.IsNullOrEmpty(sXPath));
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oVertexXmlNode != null);
Debug.Assert(!String.IsNullOrEmpty(sGraphMLAttributeID));
AssertValid();
XmlDocument oXmlDocument;
XmlNamespaceManager oXmlNamespaceManager;
if (!TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument,
out oXmlNamespaceManager))
{
return;
}
AppendInt32GraphMLAttributeValue(oXmlDocument, sXPath,
oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode,
sGraphMLAttributeID);
}
GetNetworkInternal
(
StatusCriteria oStatusCriteria,
Boolean bExpandStatusUrls,
String sGraphServerUserName,
String sGraphServerPassword,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(oStatusCriteria != null);
Debug.Assert(!String.IsNullOrEmpty(sGraphServerUserName));
Debug.Assert(!String.IsNullOrEmpty(sGraphServerPassword));
Debug.Assert(oRequestStatistics != null);
AssertValid();
GraphServiceClient oClient = new GraphServiceClient(
GetWcfServiceBinding(), new EndpointAddress(GraphServiceUrl));
Byte [] abtZippedGraphML;
// There are two ways to get the network: With a maximum status date,
// and with a maximum number of statuses.
if (oStatusCriteria.HasDateRange)
{
abtZippedGraphML = oClient.GetTwitterSearchNetworkAsZippedGraphML(
oStatusCriteria.SearchTerm,
oStatusCriteria.MinimumStatusDateUtc,
oStatusCriteria.MaximumStatusDateUtc,
bExpandStatusUrls,
sGraphServerUserName,
sGraphServerPassword
);
}
else
{
abtZippedGraphML = oClient.GetTwitterSearchNetworkAsZippedGraphML2(
oStatusCriteria.SearchTerm,
oStatusCriteria.MaximumStatusDateUtc,
oStatusCriteria.MaximumStatusesGoingBackward,
bExpandStatusUrls,
sGraphServerUserName,
sGraphServerPassword
);
}
String sGraphML = ZipUtil.UnzipOneTextFile(abtZippedGraphML);
abtZippedGraphML = null;
XmlDocument oXmlDocument = new XmlDocument();
// Note: When the DotNetZip library used by ZipUtil unzips the GraphML,
// it includes a BOM as the first character. Remove that character.
oXmlDocument.LoadXml(sGraphML.Substring(1));
return(oXmlDocument);
}
GetCommentersEnumerator
(
String sUserID,
Int32 iMaximumPerRequest,
Boolean bSkipMostPage1Errors,
String sApiKey,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sUserID));
Debug.Assert(iMaximumPerRequest > 0);
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
Debug.Assert(oRequestStatistics != null);
AssertValid();
// Get the user's public photos, which are paged.
String sUrl = GetFlickrMethodUrl("flickr.people.getPublicPhotos",
sApiKey, GetUserIDUrlParameter(sUserID));
foreach (XmlNode oPhotoXmlNode in EnumerateXmlNodes(sUrl,
"rsp/photos/photo", iMaximumPerRequest, bSkipMostPage1Errors,
oRequestStatistics))
{
String sPhotoID;
if (!XmlUtil2.TrySelectSingleNodeAsString(oPhotoXmlNode, "@id",
null, out sPhotoID))
{
continue;
}
// Get the photo's comments, which are not paged.
ReportProgress(String.Format(
"Getting comments for the photo \"{0}\"."
,
sPhotoID
));
sUrl = GetFlickrMethodUrl("flickr.photos.comments.getList",
sApiKey, "&photo_id=" + UrlUtil.EncodeUrlParameter(sPhotoID)
);
XmlDocument oXmlDocument;
if (TryGetXmlDocument(sUrl, oRequestStatistics,
out oXmlDocument))
{
foreach (XmlNode oCommentXmlNode in oXmlDocument.SelectNodes(
"rsp/comments/comment"))
{
yield return(oCommentXmlNode);
}
}
}
}
GetVideoNetworkInternal
(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumVideos,
RequestStatistics oRequestStatistics,
GraphMLXmlDocument oGraphMLXmlDocument
)
{
Debug.Assert(!String.IsNullOrEmpty(sSearchTerm));
Debug.Assert(iMaximumVideos > 0);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oGraphMLXmlDocument != null);
AssertValid();
// First, add a vertex for each video matching the search term.
HashSet <String> oVideoIDs;
Dictionary <String, LinkedList <String> > oCategoryDictionary;
AppendVertexXmlNodes(sSearchTerm, eWhatToInclude, iMaximumVideos,
oGraphMLXmlDocument, oRequestStatistics, out oVideoIDs,
out oCategoryDictionary);
// Now add whatever edges were requested.
if (WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SharedCategoryEdges))
{
Debug.Assert(oCategoryDictionary != null);
ReportProgress("Adding edges for shared categories.");
AppendEdgesFromDictionary(oCategoryDictionary, oGraphMLXmlDocument,
"Shared category", SharedCategoryID);
}
oCategoryDictionary = null;
if (WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SharedCommenterEdges))
{
AppendSharedResponderEdges(oGraphMLXmlDocument, oVideoIDs,
MaximumCommentsPerVideo,
"http://gdata.youtube.com/feeds/api/videos/{0}/comments",
"commenter", SharedCommenterID, oRequestStatistics);
}
if (WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SharedVideoResponderEdges))
{
AppendSharedResponderEdges(oGraphMLXmlDocument, oVideoIDs,
iMaximumVideos,
"http://gdata.youtube.com/feeds/api/videos/{0}/responses",
"video responder", SharedVideoResponderID, oRequestStatistics);
}
}
GetUserNetworkInternal
(
String sUserNameToAnalyze,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
Int32 iMaximumPeoplePerRequest,
RequestStatistics oRequestStatistics,
GraphMLXmlDocument oGraphMLXmlDocument
)
{
Debug.Assert(!String.IsNullOrEmpty(sUserNameToAnalyze));
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oGraphMLXmlDocument != null);
AssertValid();
// The key is the user name and the value is the corresponding GraphML
// XML node that represents the user. This is used to prevent the same
// user name from being added to the XmlDocument twice.
Dictionary <String, XmlNode> oUserNameDictionary =
new Dictionary <String, XmlNode>();
// Include friends, subscriptions, both, or neither.
Boolean [] abIncludes = new Boolean[] {
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.FriendVertices),
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SubscriptionVertices),
};
for (Int32 i = 0; i < 2; i++)
{
if (abIncludes[i])
{
GetUserNetworkRecursive(sUserNameToAnalyze, eWhatToInclude,
(i == 0), eNetworkLevel, iMaximumPeoplePerRequest, 1,
oGraphMLXmlDocument, oUserNameDictionary,
oRequestStatistics);
}
}
if (WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.AllStatistics))
{
AppendAllStatisticGraphMLAttributeValues(oGraphMLXmlDocument,
oUserNameDictionary, oRequestStatistics);
}
}
GetNetworkInternal
(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumStatuses,
RequestStatistics oRequestStatistics,
GraphMLXmlDocument oGraphMLXmlDocument
)
{
Debug.Assert(!String.IsNullOrEmpty(sSearchTerm));
Debug.Assert(iMaximumStatuses > 0);
Debug.Assert(iMaximumStatuses != Int32.MaxValue);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oGraphMLXmlDocument != null);
AssertValid();
// The key is the Twitter user ID and the value is the corresponding
// TwitterUser.
Dictionary <String, TwitterUser> oUserIDDictionary =
new Dictionary <String, TwitterUser>();
// First, append a vertex for each person who has tweeted the search
// term.
AppendVertexXmlNodesForSearchTerm(sSearchTerm, eWhatToInclude,
iMaximumStatuses, oGraphMLXmlDocument, oUserIDDictionary,
oRequestStatistics);
// Now append a vertex for each person who was mentioned or replied to
// by the first set of people, but who didn't tweet the search term
// himself.
AppendVertexXmlNodesForMentionsAndRepliesTo(eWhatToInclude,
oGraphMLXmlDocument, oUserIDDictionary, oRequestStatistics);
TwitterSearchNetworkGraphMLUtil.AppendVertexTooltipXmlNodes(
oGraphMLXmlDocument, oUserIDDictionary);
if (WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.FollowedEdges))
{
// Look at each author's friends, and if a friend has also tweeted
// the search term, add an edge between the author and the friend.
AppendFriendEdgeXmlNodes(oUserIDDictionary, MaximumFriends,
oGraphMLXmlDocument, oRequestStatistics);
}
AppendRepliesToAndMentionsEdgeXmlNodes(oGraphMLXmlDocument,
oUserIDDictionary.Values,
TwitterGraphMLUtil.TwitterUsersToUniqueScreenNames(
oUserIDDictionary.Values)
);
}
//*************************************************************************
// Constructor: PartialNetworkException()
//
/// <summary>
/// Initializes a new instance of the <see
/// cref="PartialNetworkException" /> class.
/// </summary>
///
/// <param name="partialNetwork">
/// The partial network that was obtained.
/// </param>
///
/// <param name="requestStatistics">
/// Information about the requests that were made while getting the
/// network.
/// </param>
//*************************************************************************
public PartialNetworkException(
XmlDocument partialNetwork,
RequestStatistics requestStatistics
)
{
m_oPartialNetwork = partialNetwork;
m_oRequestStatistics = requestStatistics;
AssertValid();
}
//*************************************************************************
// Constructor: PartialNetworkException()
//
/// <summary>
/// Initializes a new instance of the <see
/// cref="PartialNetworkException" /> class.
/// </summary>
///
/// <param name="partialNetwork">
/// The partial network that was obtained.
/// </param>
///
/// <param name="requestStatistics">
/// Information about the requests that were made while getting the
/// network.
/// </param>
//*************************************************************************
public PartialNetworkException
(
XmlDocument partialNetwork,
RequestStatistics requestStatistics
)
{
m_oPartialNetwork = partialNetwork;
m_oRequestStatistics = requestStatistics;
AssertValid();
}
GetXmlDocument
(
String sUrl,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sUrl));
Debug.Assert(oRequestStatistics != null);
AssertValid();
XmlDocument oXmlDocument = GetXmlDocumentWithRetries(sUrl,
HttpStatusCodesToFailImmediately, oRequestStatistics);
String sStatus;
if (
XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/@stat",
null, out sStatus)
&&
sStatus == "ok"
)
{
return(oXmlDocument);
}
// Flickr indicates errors by returning an XML document containing an
// rsp/err node. The following code turns such an error document into
// a custom FlickrException.
String sErrorMessage;
if (XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/err/@msg",
null, out sErrorMessage))
{
if (sErrorMessage.ToLower().IndexOf("user not found") >= 0)
{
sErrorMessage =
"The user wasn't found. Either there is no such user, or"
+ " she has hidden herself from public searches."
;
}
}
else
{
sErrorMessage =
"Flickr provided information in an unrecognized format.";
}
throw new FlickrException(sErrorMessage);
}
OnNetworkObtained
(
XmlDocument oGraphMLXmlDocument,
RequestStatistics oRequestStatistics,
String sNetworkDescription,
String sNetworkTitle,
String sPartialFileName
)
{
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(!String.IsNullOrEmpty(sNetworkDescription));
Debug.Assert(!String.IsNullOrEmpty(sNetworkTitle));
Debug.Assert(!String.IsNullOrEmpty(sPartialFileName));
AssertValid();
XmlNode oGraphXmlNode = XmlUtil2.SelectRequiredSingleNode(
oGraphMLXmlDocument, "g:graphml/g:graph",
GraphMLXmlDocument.CreateXmlNamespaceManager(
oGraphMLXmlDocument, "g")
);
XmlUtil2.SetAttributes(oGraphXmlNode,
"description", sNetworkDescription);
XmlUtil2.SetAttributes(oGraphXmlNode,
"suggestedTitle", sNetworkTitle);
String sSuggestedFileNameNoExtension = String.Format(
"{0} NodeXL {1}"
,
DateTimeUtil2.ToCultureInvariantFileName(
oRequestStatistics.StartTimeUtc),
sPartialFileName
);
XmlUtil2.SetAttributes(oGraphXmlNode,
"suggestedFileNameNoExtension", sSuggestedFileNameNoExtension);
if (oRequestStatistics.UnexpectedExceptions > 0)
{
// The network is partial.
throw new PartialNetworkException(oGraphMLXmlDocument,
oRequestStatistics);
}
}
TryGetUserValueDictionary
(
String sScreenName,
RequestStatistics oRequestStatistics,
Boolean bIgnoreWebAndJsonExceptions,
out Dictionary <String, Object> oUserValueDictionary
)
{
Debug.Assert(!String.IsNullOrEmpty(sScreenName));
Debug.Assert(oRequestStatistics != null);
AssertValid();
oUserValueDictionary = null;
String sUrl = String.Format(
"{0}users/show.json?screen_name={1}&{2}"
,
TwitterApiUrls.Rest,
TwitterUtil.EncodeUrlParameter(sScreenName),
TwitterApiUrlParameters.IncludeEntities
);
ReportProgress(String.Format(
"Getting information about \"{0}\"."
,
sScreenName
));
try
{
oUserValueDictionary = (Dictionary <String, Object>)
(new JavaScriptSerializer()).DeserializeObject(
GetTwitterResponseAsString(sUrl, oRequestStatistics));
return(true);
}
catch (Exception oException)
{
if (!HttpSocialNetworkUtil.ExceptionIsWebOrJson(oException) ||
!bIgnoreWebAndJsonExceptions)
{
throw oException;
}
return(false);
}
}
GetUserNetworkInternal
(
String sScreenName,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
Int32 iMaximumPerRequest,
String sApiKey
)
{
Debug.Assert(!String.IsNullOrEmpty(sScreenName));
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPerRequest > 0);
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
AssertValid();
GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument(
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.CommenterVertices),
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.UserInformation)
);
RequestStatistics oRequestStatistics = new RequestStatistics();
try
{
GetUserNetworkInternal(sScreenName, eWhatToInclude, eNetworkLevel,
iMaximumPerRequest, sApiKey, oRequestStatistics,
oGraphMLXmlDocument);
}
catch (Exception oException)
{
OnTerminatingException(oException);
}
OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument,
oRequestStatistics);
return(oGraphMLXmlDocument);
}
GetUserNetworkInternal
(
String sUserNameToAnalyze,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
Int32 iMaximumPeoplePerRequest
)
{
Debug.Assert(!String.IsNullOrEmpty(sUserNameToAnalyze));
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPeoplePerRequest > 0);
AssertValid();
GraphMLXmlDocument oGraphMLXmlDocument =
CreateGraphMLXmlDocument(WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.AllStatistics));
RequestStatistics oRequestStatistics = new RequestStatistics();
try
{
GetUserNetworkInternal(sUserNameToAnalyze, eWhatToInclude,
eNetworkLevel, iMaximumPeoplePerRequest, oRequestStatistics,
oGraphMLXmlDocument);
}
catch (Exception oException)
{
OnUnexpectedException(oException, oGraphMLXmlDocument,
oRequestStatistics);
}
String sNetworkTitle = "YouTube User " + sUserNameToAnalyze;
OnNetworkObtained(oGraphMLXmlDocument, oRequestStatistics,
GetNetworkDescription(sUserNameToAnalyze, eWhatToInclude,
eNetworkLevel, iMaximumPeoplePerRequest),
sNetworkTitle, sNetworkTitle
);
return(oGraphMLXmlDocument);
}
TryGetSampleImageUrl
(
String sTag,
String sApiKey,
RequestStatistics oRequestStatistics,
out String sSampleImageUrl
)
{
Debug.Assert(!String.IsNullOrEmpty(sTag));
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
Debug.Assert(oRequestStatistics != null);
AssertValid();
sSampleImageUrl = null;
String sUrl = GetFlickrMethodUrl("flickr.tags.getClusterPhotos",
sApiKey, "&tag=" + UrlUtil.EncodeUrlParameter(sTag));
XmlDocument oXmlDocument;
String sPhotoID;
if (
!TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument)
||
!XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
"rsp/photos/photo/@id", null, out sPhotoID)
)
{
return(false);
}
sUrl = GetFlickrMethodUrl("flickr.photos.getSizes", sApiKey,
"&photo_id=" + UrlUtil.EncodeUrlParameter(sPhotoID));
if (
!TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument)
||
!XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
"rsp/sizes/size[@label='Thumbnail']/@source", null,
out sSampleImageUrl)
)
{
return(false);
}
return(true);
}
public void DoesNotOverflow()
{
var elapsedTicks = 0L;
var sut = new RequestStatistics(() => elapsedTicks, 1000);
for (var i = 0; i < 1000; i++)
{
var id = Guid.NewGuid();
sut.StartOperation(id);
elapsedTicks += TimeSpan.FromMinutes(36).Ticks;
sut.EndOperation(id);
}
sut.GetMeasurementDetails();
}
EnumerateUserValueDictionaries
(
String [] asUserIDsOrScreenNames,
Boolean bUserIDsSpecified,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(asUserIDsOrScreenNames != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
return(m_oTwitterUtil.EnumerateUserValueDictionaries(
asUserIDsOrScreenNames, bUserIDsSpecified, oRequestStatistics,
new ReportProgressHandler(this.ReportProgress),
new CheckCancellationPendingHandler(this.CheckCancellationPending)
));
}
/// <summary>
/// The API returns the detected language and a numeric score between 0 and 1.
/// </summary>
/// <remarks>
/// Scores close to 1 indicate 100% certainty that the identified language is
/// true. A total of 120 languages are supported.
/// </remarks>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='showStats'>
/// (optional) if set to true, response will contain input and document level
/// statistics.
/// </param>
/// <param name='inputText'>
/// Input text of one document.
/// </param>
/// <param name='countryHint'>
/// Contry hint.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
public static LanguageResult DetectLanguage(
this ITextAnalyticsClient operations,
string inputText = default,
string countryHint = "US",
bool?showStats = default,
CancellationToken cancellationToken = default)
{
var languageBatchInput = new LanguageBatchInput(new List <LanguageInput> {
new LanguageInput("1", inputText, countryHint)
});
var _result = operations.DetectLanguageWithHttpMessagesAsync(showStats, languageBatchInput, null, cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult();
IList <DetectedLanguage> languages = _result.Body.Documents.Count > 0 ? _result.Body.Documents[0].DetectedLanguages : null;
string errorMessage = _result.Body.Errors.Count > 0 ? _result.Body.Errors[0].Message : null;
RequestStatistics stats = _result.Body.Statistics;
return(new LanguageResult(languages, errorMessage, stats));
}
/// <summary>
/// The API returns a numeric score between 0 and 1.
/// </summary>
/// <remarks>
/// Scores close to 1 indicate positive sentiment, while scores close to 0
/// indicate negative sentiment. A score of 0.5 indicates the lack of sentiment
/// (e.g. a factoid statement). See the <a
/// href="https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/overview#supported-languages">Text
/// Analytics Documentation</a> for details about the languages that are
/// supported by sentiment analysis.
/// </remarks>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='showStats'>
/// (optional) if set to true, response will contain input and document level
/// statistics.
/// </param>
/// <param name='inputText'>
/// Input text of one document.
/// </param>
/// <param name='language'>
/// Language code.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
public static SentimentResult Sentiment(
this ITextAnalyticsClient operations,
string inputText = default,
string language = "en",
bool?showStats = default,
CancellationToken cancellationToken = default)
{
var multiLanguageBatchInput = new MultiLanguageBatchInput(new List <MultiLanguageInput> {
new MultiLanguageInput("1", inputText, language)
});
var _result = operations.SentimentWithHttpMessagesAsync(showStats, multiLanguageBatchInput, null, cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult();
double? score = _result.Body.Documents.Count > 0 ? _result.Body.Documents[0].Score : null;
string errorMessage = _result.Body.Errors.Count > 0 ? _result.Body.Errors[0].Message : null;
RequestStatistics stats = _result.Body.Statistics;
return(new SentimentResult(score, errorMessage, stats));
}
GetTwitterResponseAsString
(
String sUrl,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sUrl));
Debug.Assert(oRequestStatistics != null);
AssertValid();
Debug.Assert(m_oTwitterUtil != null);
return(m_oTwitterUtil.GetTwitterResponseAsString(sUrl,
oRequestStatistics,
new ReportProgressHandler(this.ReportProgress),
new CheckCancellationPendingHandler(this.CheckCancellationPending)
));
}
GetUserNetworkInternal
(
String sScreenNameToAnalyze,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
Int32 iMaximumPeoplePerRequest
)
{
Debug.Assert(!String.IsNullOrEmpty(sScreenNameToAnalyze));
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPeoplePerRequest > 0);
AssertValid();
BeforeGetNetwork();
Boolean bIncludeLatestStatus = WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.LatestStatuses);
GraphMLXmlDocument oGraphMLXmlDocument =
CreateGraphMLXmlDocument(true, bIncludeLatestStatus);
RequestStatistics oRequestStatistics = new RequestStatistics();
try
{
GetUserNetworkInternal(sScreenNameToAnalyze, eWhatToInclude,
eNetworkLevel, iMaximumPeoplePerRequest, oRequestStatistics,
oGraphMLXmlDocument);
}
catch (Exception oException)
{
OnTerminatingException(oException);
}
OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument,
oRequestStatistics);
return(oGraphMLXmlDocument);
}
GetRelatedTagsInternal
(
String sTag,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
String sApiKey
)
{
Debug.Assert(!String.IsNullOrEmpty(sTag));
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
AssertValid();
GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument(
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SampleThumbnails));
RequestStatistics oRequestStatistics = new RequestStatistics();
try
{
GetRelatedTagsInternal(sTag, eWhatToInclude, eNetworkLevel,
sApiKey, oRequestStatistics, oGraphMLXmlDocument);
}
catch (Exception oException)
{
OnUnexpectedException(oException, oGraphMLXmlDocument,
oRequestStatistics);
}
String sNetworkTitle = "Flickr Tag " + sTag;
OnNetworkObtained(oGraphMLXmlDocument, oRequestStatistics,
GetNetworkDescription(sTag, eWhatToInclude, eNetworkLevel),
sNetworkTitle, sNetworkTitle
);
return(oGraphMLXmlDocument);
}
//*************************************************************************
// Method: AppendAllStatisticGraphMLAttributeValues()
//
/// <overloads>
/// Appends statistic GraphML attribute values to the GraphML document.
/// </overloads>
///
/// <summary>
/// Appends statistic GraphML attribute values to the GraphML document for
/// all users in the network.
/// </summary>
///
/// <param name="oGraphMLXmlDocument">
/// GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oUserNameDictionary">
/// The key is the user name and the value is the corresponding GraphML XML
/// node that represents the user.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
//*************************************************************************
protected void AppendAllStatisticGraphMLAttributeValues(
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary<String, XmlNode> oUserNameDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oUserNameDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
foreach (KeyValuePair<String, XmlNode> oKeyValuePair in
oUserNameDictionary)
{
String sUserName = oKeyValuePair.Key;
ReportProgress( String.Format(
"Getting statistics for \"{0}\"."
,
sUserName
) );
AppendAllStatisticGraphMLAttributeValues(sUserName,
oKeyValuePair.Value, oGraphMLXmlDocument, oRequestStatistics);
}
}
BackgroundWorker_DoWork(object sender, DoWorkEventArgs e)
{
Debug.Assert(sender is BackgroundWorker);
BackgroundWorker bw = sender as BackgroundWorker;
Debug.Assert(e.Argument is NetworkAsyncArgs);
NetworkAsyncArgs args = e.Argument as NetworkAsyncArgs;
this.requestStatistics = new RequestStatistics();
try
{
CheckCancellationPending();
ReportProgress("Starting");
// shell include ego node in the graph
this.includeEgo = args.includeMe;
// prepare rest contexts
var context = new VkRestApi.VkRestContext(args.userId, args.accessToken);
// get ego node
vkRestApi.CallVkFunction(VkFunction.GetProfiles, context);
// wait for the user data
readyEvent.WaitOne();
CheckCancellationPending();
ReportProgress("Retrieving friends");
// prepare fields context parameter
StringBuilder sb = new StringBuilder("fields=");
sb.Append(args.fields);
context.Parameters = sb.ToString();
// get friends node
vkRestApi.CallVkFunction(VkFunction.LoadFriends, context);
// wait for the friends data
readyEvent.WaitOne();
int total = this.friendIds.Count;
int current = 0;
foreach (string targetId in this.friendIds)
{
CheckCancellationPending();
current++;
ReportProgress("Retrieving friends mutual " + current + " out of " + total);
// Append target friend ids
sb.Length = 0;
sb.Append("target_uid=");
sb.Append(targetId);
context.Parameters = sb.ToString();
context.Cookie = targetId; // pass target id in the context's cookie field
// get mutual friends
vkRestApi.CallVkFunction(VkFunction.FriendsGetMutual, context);
// wait for the mutual data
readyEvent.WaitOne();
// play it nice, sleep for 1/3 sec to stay within 3 requests/second limit
// TODO: account for time spent in processing
Thread.Sleep(333);
}
if (includeEgo)
{
CreateIncludeMeEdges();
}
CheckCancellationPending();
ReportProgress("Building network graph document");
// create default attributes (values will be empty)
AttributesDictionary<String> attributes = new AttributesDictionary<String>(VKAttributes);
// build the file
XmlDocument graph = GenerateNetworkDocument(vertices, edges, attributes);
if (this.requestStatistics.UnexpectedExceptions > 0)
{
// there was errors - pop up partial network dialog
throw new PartialNetworkException(graph, this.requestStatistics);
}
else
{
// all good
e.Result = graph;
}
}
catch (CancellationPendingException)
{
e.Cancel = true;
}
}
//*************************************************************************
// Method: GetUserNetworkInternal()
//
/// <summary>
/// Gets a network of Twitter users, given a GraphMLXmlDocument.
/// </summary>
///
/// <param name="sScreenNameToAnalyze">
/// The screen name of the Twitter user whose network should be analyzed.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="eNetworkLevel">
/// Network level to include. Must be NetworkLevel.One, OnePointFive, or
/// Two.
/// </param>
///
/// <param name="iMaximumPeoplePerRequest">
/// Maximum number of people to request for each query, or Int32.MaxValue
/// for no limit.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// The GraphMLXmlDocument to populate with the requested network.
/// </param>
//*************************************************************************
protected void GetUserNetworkInternal(
String sScreenNameToAnalyze,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
Int32 iMaximumPeoplePerRequest,
RequestStatistics oRequestStatistics,
GraphMLXmlDocument oGraphMLXmlDocument
)
{
Debug.Assert( !String.IsNullOrEmpty(sScreenNameToAnalyze) );
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oGraphMLXmlDocument != null);
AssertValid();
// The key is the screen name and the value is the corresponding
// TwitterVertex. This is used to prevent the same screen name from
// being added to the XmlDocument twice.
Dictionary<String, TwitterVertex> oScreenNameDictionary =
new Dictionary<String, TwitterVertex>();
// Include followed, followers, both, or neither.
Boolean [] abIncludes = new Boolean[] {
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.FollowedVertices),
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.FollowerVertices),
};
for (Int32 i = 0; i < 2; i++)
{
if ( abIncludes[i] )
{
GetUserNetworkRecursive(sScreenNameToAnalyze, eWhatToInclude,
(i == 0), eNetworkLevel, iMaximumPeoplePerRequest, 1,
oGraphMLXmlDocument, oScreenNameDictionary,
oRequestStatistics);
}
}
Boolean bIncludeLatestStatus = WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.LatestStatuses);
AppendMissingGraphMLAttributeValues(oGraphMLXmlDocument,
oScreenNameDictionary, true, bIncludeLatestStatus,
oRequestStatistics);
AppendRepliesToAndMentionsXmlNodes(oGraphMLXmlDocument,
oScreenNameDictionary,
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.RepliesToEdges),
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.MentionsEdges)
);
}
//*************************************************************************
// Method: GetVideoNetworkInternal()
//
/// <overloads>
/// Gets a network of related YouTube videos.
/// </overloads>
///
/// <summary>
/// Gets a network of related YouTube videos.
/// </summary>
///
/// <param name="sSearchTerm">
/// The term to search for.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="iMaximumVideos">
/// Maximum number of videos to request, or Int32.MaxValue for no limit.
/// </param>
///
/// <returns>
/// An XmlDocument containing the network as GraphML.
/// </returns>
//*************************************************************************
protected XmlDocument GetVideoNetworkInternal(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumVideos
)
{
Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) );
Debug.Assert(iMaximumVideos > 0);
AssertValid();
GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument();
RequestStatistics oRequestStatistics = new RequestStatistics();
try
{
GetVideoNetworkInternal(sSearchTerm, eWhatToInclude,
iMaximumVideos, oRequestStatistics, oGraphMLXmlDocument);
}
catch (Exception oException)
{
OnTerminatingException(oException);
}
OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument,
oRequestStatistics);
return (oGraphMLXmlDocument);
}
//*************************************************************************
// Method: AppendSharedResponderEdges()
//
/// <summary>
/// Appends an edge XML node for each pair of videos for which the same
/// person responded.
/// </summary>
///
/// <param name="oGraphMLXmlDocument">
/// The GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oVideoIDs">
/// The set of unique video IDs.
/// </param>
///
/// <param name="iMaximumResponses">
/// Maximum number of responses to request, or Int32.MaxValue for no limit.
/// </param>
///
/// <param name="sUrlPattern">
/// URL to get the responses for one video, with "{0}" where the video ID
/// goes.
/// </param>
///
/// <param name="sResponderTitle">
/// Title describing the person who has responded. Samples: "commenter",
/// "video responder".
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
//*************************************************************************
protected void AppendSharedResponderEdges(
GraphMLXmlDocument oGraphMLXmlDocument,
HashSet<String> oVideoIDs,
Int32 iMaximumResponses,
String sUrlPattern,
String sResponderTitle,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oVideoIDs != null);
Debug.Assert(iMaximumResponses > 0);
Debug.Assert( !String.IsNullOrEmpty(sUrlPattern) );
Debug.Assert(sUrlPattern.IndexOf("{0}") >= 0);
Debug.Assert( !String.IsNullOrEmpty(sResponderTitle) );
Debug.Assert(oRequestStatistics != null);
AssertValid();
// The key is the name of an author and the value is a LinkedList of
// the video IDs to which the author has responded.
Dictionary< String, LinkedList<String> > oAuthorUserNameDictionary =
new Dictionary< String, LinkedList<String> >();
foreach (String sVideoID in oVideoIDs)
{
ReportProgress(String.Format(
"Getting {0}s for the video with the ID \"{1}\"."
,
sResponderTitle,
sVideoID
) );
// This is to prevent self-loop edges that would result when the
// same author responds to the same video more than once.
HashSet<String> oAuthorUserNames = new HashSet<String>();
String sUrl = String.Format(sUrlPattern, sVideoID);
// The document consists of an "entry" XML node for each response.
foreach ( XmlNode oEntryXmlNode in EnumerateXmlNodes(sUrl,
"a:feed/a:entry", iMaximumResponses, true,
oRequestStatistics) )
{
XmlNamespaceManager oXmlNamespaceManager =
CreateXmlNamespaceManager(oEntryXmlNode.OwnerDocument);
String sAuthorUserName;
if (
XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"a:author/a:name/text()", oXmlNamespaceManager,
out sAuthorUserName)
&&
!oAuthorUserNames.Contains(sAuthorUserName)
)
{
AddVideoIDToDictionary(sAuthorUserName, sVideoID,
oAuthorUserNameDictionary);
oAuthorUserNames.Add(sAuthorUserName);
}
}
}
ReportProgress("Adding edges for shared " + sResponderTitle + "s.");
AppendEdgesFromDictionary(oAuthorUserNameDictionary,
oGraphMLXmlDocument, "Shared " + sResponderTitle);
}
//*************************************************************************
// Method: GetXmlDocumentWithRetries()
//
/// <summary>
/// Gets an XML document given an URL. Retries after an error.
/// </summary>
///
/// <param name="sUrl">
/// URL to use.
/// </param>
///
/// <param name="aeHttpStatusCodesToFailImmediately">
/// An array of status codes that should be failed immediately, or null to
/// retry all failures. An example is HttpStatusCode.Unauthorized (401),
/// which Twitter returns when information about a user who has "protected"
/// status is requested. This should not be retried, because the retries
/// would produce exactly the same error response.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <param name="asOptionalHeaderNameValuePairs">
/// Array of name/value pairs for HTTP headers to add to the request, or
/// null to not add any pairs. Sample: {"Authorization", "Basic 36A4E798"}
/// </param>
///
/// <returns>
/// The XmlDocument.
/// </returns>
///
/// <remarks>
/// If the request fails and the HTTP status code is not one of the codes
/// specified in <paramref name="aeHttpStatusCodesToFailImmediately" />,
/// the request is retried. If the retries also fail, an exception is
/// thrown.
///
/// <para>
/// If the request fails with one of the HTTP status code contained in
/// <paramref name="aeHttpStatusCodesToFailImmediately" />, an exception is
/// thrown immediately.
/// </para>
///
/// <para>
/// In either case, it is always up to the caller to handle the exceptions.
/// This method never ignores an exception; it either retries it and throws
/// it if all retries fail, or throws it immediately.
/// </para>
///
/// </remarks>
//*************************************************************************
protected XmlDocument GetXmlDocumentWithRetries(
String sUrl,
HttpStatusCode [] aeHttpStatusCodesToFailImmediately,
RequestStatistics oRequestStatistics,
params String[] asOptionalHeaderNameValuePairs
)
{
Debug.Assert( !String.IsNullOrEmpty(sUrl) );
Debug.Assert(oRequestStatistics != null);
Debug.Assert(asOptionalHeaderNameValuePairs == null ||
asOptionalHeaderNameValuePairs.Length % 2 == 0);
AssertValid();
Int32 iMaximumRetries = HttpRetryDelaysSec.Length;
Int32 iRetriesSoFar = 0;
while (true)
{
if (iRetriesSoFar > 0)
{
ReportProgress("Retrying request.");
}
// Important Note: You cannot use the same HttpWebRequest object
// for the retries. The object must be recreated each time.
HttpWebRequest oHttpWebRequest =
(HttpWebRequest)WebRequest.Create(sUrl);
Int32 iHeaderNamesAndValues =
(asOptionalHeaderNameValuePairs == null) ?
0 : asOptionalHeaderNameValuePairs.Length;
for (Int32 i = 0; i < iHeaderNamesAndValues; i += 2)
{
String sHeaderName = asOptionalHeaderNameValuePairs[i + 0];
String sHeaderValue = asOptionalHeaderNameValuePairs[i + 1];
Debug.Assert( !String.IsNullOrEmpty(sHeaderName) );
Debug.Assert( !String.IsNullOrEmpty(sHeaderValue) );
oHttpWebRequest.Headers[sHeaderName] = sHeaderValue;
}
try
{
XmlDocument oXmlDocument =
GetXmlDocumentNoRetries(oHttpWebRequest);
if (iRetriesSoFar > 0)
{
ReportProgress("Retry succeeded, continuing...");
}
oRequestStatistics.OnSuccessfulRequest();
return (oXmlDocument);
}
catch (Exception oException)
{
if ( !ExceptionIsWebOrXml(oException) )
{
throw oException;
}
// A WebException or XmlException has occurred.
if (iRetriesSoFar == iMaximumRetries)
{
oRequestStatistics.OnUnexpectedException(oException);
throw (oException);
}
// If the status code is one of the ones specified in
// aeHttpStatusCodesToFailImmediately, rethrow the exception
// without retrying the request.
if (aeHttpStatusCodesToFailImmediately != null &&
oException is WebException)
{
if ( WebExceptionHasHttpStatusCode(
(WebException)oException,
aeHttpStatusCodesToFailImmediately) )
{
throw (oException);
}
}
Int32 iSeconds = HttpRetryDelaysSec[iRetriesSoFar];
ReportProgress( String.Format(
"Request failed, pausing {0} {1} before retrying..."
,
iSeconds,
StringUtil.MakePlural("second", iSeconds)
) );
System.Threading.Thread.Sleep(1000 * iSeconds);
iRetriesSoFar++;
}
}
}
//*************************************************************************
// Method: TryGetSampleImageUrl()
//
/// <summary>
/// Attempts to get an URL to a sample image for a tag.
/// </summary>
///
/// <param name="sTag">
/// Tag to get a sample image URL for.
/// </param>
///
/// <param name="sApiKey">
/// Flickr API key.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <param name="sSampleImageUrl">
/// Where the URL gets stored if true is returned.
/// </param>
///
/// <returns>
/// true if the sample image URL was obtained, false if some sort of error
/// occurred while attempting to get it.
/// </returns>
//*************************************************************************
protected Boolean TryGetSampleImageUrl(
String sTag,
String sApiKey,
RequestStatistics oRequestStatistics,
out String sSampleImageUrl
)
{
Debug.Assert( !String.IsNullOrEmpty(sTag) );
Debug.Assert( !String.IsNullOrEmpty(sApiKey) );
Debug.Assert(oRequestStatistics != null);
AssertValid();
sSampleImageUrl = null;
String sUrl = GetFlickrMethodUrl( "flickr.tags.getClusterPhotos",
sApiKey, "&tag=" + UrlUtil.EncodeUrlParameter(sTag) );
XmlDocument oXmlDocument;
String sPhotoID;
if (
!TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument)
||
! XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
"rsp/photos/photo/@id", null, out sPhotoID)
)
{
return (false);
}
sUrl = GetFlickrMethodUrl( "flickr.photos.getSizes", sApiKey,
"&photo_id=" + UrlUtil.EncodeUrlParameter(sPhotoID) );
if (
!TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument)
||
! XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
"rsp/sizes/size[@label='Thumbnail']/@source", null,
out sSampleImageUrl)
)
{
return (false);
}
return (true);
}
EnumerateUserValueDictionaries
(
String [] userIDsOrScreenNames,
Boolean userIDsSpecified,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert(userIDsOrScreenNames != null);
Debug.Assert(requestStatistics != null);
AssertValid();
// We'll use Twitter's users/lookup API, which gets extended
// information for up to 100 users in one call.
Int32 iUsers = userIDsOrScreenNames.Length;
Int32 iUsersProcessed = 0;
Int32 iCalls = 0;
while (iUsersProcessed < iUsers)
{
// For each call, ask for information about as many users as
// possible until either 100 is reached or the URL reaches an
// arbitrary maximum length. Twitter recommends using a POST here
// (without specifying why), but it would require revising the
// base-class HTTP calls and isn't worth the trouble.
Int32 iUsersProcessedThisCall = 0;
StringBuilder oUrl = new StringBuilder();
oUrl.AppendFormat(
"{0}users/lookup.json?{1}&{2}="
,
TwitterApiUrls.Rest,
TwitterApiUrlParameters.IncludeEntities,
userIDsSpecified ? "user_id" : "screen_name"
);
const Int32 MaxUsersPerCall = 100;
const Int32 MaxUrlLength = 2000;
// Construct the URL for this call.
while (
iUsersProcessed < iUsers
&&
iUsersProcessedThisCall < MaxUsersPerCall
&&
oUrl.Length < MaxUrlLength
)
{
if (iUsersProcessedThisCall > 0)
{
// Append an encoded comma. Using an unencoded comma
// causes Twitter to return a 401 "unauthorized" error.
//
// See this post for an explanation:
//
// https://dev.twitter.com/discussions/11399
oUrl.Append("%2C");
}
oUrl.Append( userIDsOrScreenNames[iUsersProcessed] );
iUsersProcessed++;
iUsersProcessedThisCall++;
}
iCalls++;
if (iCalls > 1 && reportProgressHandler != null)
{
reportProgressHandler("Getting page " + iCalls + ".");
}
foreach ( Object oResult in EnumerateJsonValues(oUrl.ToString(),
null, Int32.MaxValue, true, requestStatistics,
reportProgressHandler, checkCancellationPendingHandler) )
{
yield return ( ( Dictionary<String, Object> )oResult );
}
}
}
EnumerateJsonValues
(
String url,
String jsonName,
Int32 maximumValues,
Boolean skipMostPage1Errors,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
// Note:
//
// The logic in this method is similar to the logic in
// EnumerateSearchStatuses(). In fact, at one time all enumeration was
// done through this EnumerateJsonValues() method.
// EnumerateSearchStatuses() was created only when version 1.1 of the
// Twitter API introduced yet another paging scheme, one that differs
// from the cursor scheme that this method handles.
//
// A possible work item is to recombine the two methods into one,
// possibly by using a delegate to handle the different paging schemes.
Debug.Assert( !String.IsNullOrEmpty(url) );
Debug.Assert(maximumValues > 0);
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iPage = 1;
String sCursor = null;
Int32 iObjectsEnumerated = 0;
while (true)
{
if (iPage > 1 && reportProgressHandler != null)
{
reportProgressHandler("Getting page " + iPage + ".");
}
String sUrlWithCursor = AppendCursorToUrl(url, sCursor);
Dictionary<String, Object> oValueDictionary = null;
Object [] aoObjectsThisPage;
try
{
Object oDeserializedTwitterResponse =
( new JavaScriptSerializer() ).DeserializeObject(
GetTwitterResponseAsString(sUrlWithCursor,
requestStatistics, reportProgressHandler,
checkCancellationPendingHandler) );
Object oObjectsThisPageAsObject;
if (jsonName == null)
{
// The top level of the Json response contains an array of
// objects this method will enumerate.
oObjectsThisPageAsObject = oDeserializedTwitterResponse;
}
else
{
// The top level of the Json response contains a set of
// name/value pairs. The value for the specified name is
// the array of objects this method will enumerate.
oValueDictionary = ( Dictionary<String, Object> )
oDeserializedTwitterResponse;
oObjectsThisPageAsObject = oValueDictionary[jsonName];
}
aoObjectsThisPage = ( Object [] )oObjectsThisPageAsObject;
}
catch (Exception oException)
{
// Rethrow the exception if appropriate.
TwitterUtil.OnExceptionWhileEnumeratingJsonValues(
oException, iPage, skipMostPage1Errors);
// Otherwise, just halt the enumeration.
yield break;
}
Int32 iObjectsThisPage = aoObjectsThisPage.Length;
if (iObjectsThisPage == 0)
{
yield break;
}
for (Int32 i = 0; i < iObjectsThisPage; i++)
{
yield return ( aoObjectsThisPage[i] );
iObjectsEnumerated++;
if (iObjectsEnumerated == maximumValues)
{
yield break;
}
}
iPage++;
// When the top level of the Json response contains a set of
// name/value pairs, a next_cursor_str value of "0" means "end of
// data."
if (
oValueDictionary == null
||
!TwitterJsonUtil.TryGetJsonValueFromDictionary(
oValueDictionary, "next_cursor_str", out sCursor)
||
sCursor == "0"
)
{
yield break;
}
// Get the next page...
}
}
//*************************************************************************
// Method: GetUserNetworkRecursive()
//
/// <summary>
/// Recursively gets a network of YouTube users.
/// </summary>
///
/// <param name="sUserName">
/// The user name to use in this call.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="bIncludeFriendsThisCall">
/// true to include the user's friends, false to include the people
/// subscribed to by the user.
/// </param>
///
/// <param name="eNetworkLevel">
/// Network level to include. Must be NetworkLevel.One, OnePointFive, or
/// Two.
/// </param>
///
/// <param name="iMaximumPeoplePerRequest">
/// Maximum number of people to request for each query, or Int32.MaxValue
/// for no limit.
/// </param>
///
/// <param name="iRecursionLevel">
/// Recursion level for this call. Must be 1 or 2. Gets incremented when
/// recursing.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oUserNameDictionary">
/// The key is the user name and the value is the corresponding GraphML XML
/// node that represents the user.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
//*************************************************************************
protected void GetUserNetworkRecursive(
String sUserName,
WhatToInclude eWhatToInclude,
Boolean bIncludeFriendsThisCall,
NetworkLevel eNetworkLevel,
Int32 iMaximumPeoplePerRequest,
Int32 iRecursionLevel,
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary<String, XmlNode> oUserNameDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert( !String.IsNullOrEmpty(sUserName) );
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oUserNameDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
/*
Here is what this method should do, based on the eNetworkLevel and
iRecursionLevel parameters.
eNetworkLevel
|One | OnePointFive | Two
---|------------------| ------------------| -----------------
i 1 |Add all vertices. | Add all vertices. | Add all vertices.
R | | |
e |Add all edges. | Add all edges. | Add all edges.
c | | |
u |Do not recurse. | Recurse. | Recurse.
r | | |
s ---|------------------|-------------------|------------------
i 2 |Impossible. | Do not add | Add all vertices.
o | | vertices. |
n | | |
L | | Add edges only if | Add all edges.
e | | vertices are |
v | | already included. |
e | | |
l | | Do not recurse. | Do not recurse.
| | |
---|------------------|-------------------|------------------
*/
Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel,
iRecursionLevel);
List<String> oUserNamesToRecurse = new List<String>();
ReportProgressForFriendsOrSubscriptions(sUserName,
bIncludeFriendsThisCall);
Boolean bThisUserAppended = false;
// If the GraphMLXmlDocument already contains at least one vertex node,
// then this is the second time that this method has been called, a
// partial network has already been obtained, and most errors should
// now be skipped. However, if none of the network has been obtained
// yet, errors on page 1 should throw an immediate exception.
Boolean bSkipMostPage1Errors = oGraphMLXmlDocument.HasVertexXmlNode;
// The document consists of a single "feed" node with zero or more
// "entry" child nodes.
foreach ( XmlNode oEntryXmlNode in EnumerateXmlNodes(
GetFriendsOrSubscriptionsUrl(sUserName, bIncludeFriendsThisCall),
"a:feed/a:entry", iMaximumPeoplePerRequest, bSkipMostPage1Errors,
oRequestStatistics) )
{
XmlNamespaceManager oXmlNamespaceManager =
CreateXmlNamespaceManager(oEntryXmlNode.OwnerDocument);
String sOtherUserName;
if ( !XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"yt:username/text()", oXmlNamespaceManager,
out sOtherUserName) )
{
continue;
}
if (!bThisUserAppended)
{
// Append a vertex node for this request's user.
//
// This used to be done after the foreach loop, which avoided
// the need for a "bThisUserAppended" flag. That caused the
// following bug: If a YouTube error occurred within
// EnumerateXmlNodes() after some edges had been added, and the
// user decided to import the resulting partial network, the
// GraphML might contain edges that referenced "this user"
// without containing a vertex for "this user." That is an
// illegal state for GraphML, which the ExcelTemplate project
// caught and reported as an error.
TryAppendVertexXmlNode(sUserName, null, oGraphMLXmlDocument,
oUserNameDictionary);
bThisUserAppended = true;
}
Boolean bNeedToAppendVertices = GetNeedToAppendVertices(
eNetworkLevel, iRecursionLevel);
if (bNeedToAppendVertices)
{
if (
TryAppendVertexXmlNode(sOtherUserName, oEntryXmlNode,
oGraphMLXmlDocument, oUserNameDictionary)
&&
bNeedToRecurse
)
{
oUserNamesToRecurse.Add(sOtherUserName);
}
}
if ( bNeedToAppendVertices ||
oUserNameDictionary.ContainsKey(sOtherUserName) )
{
String sRelationship;
if (bIncludeFriendsThisCall)
{
sRelationship = "Friend of";
}
else
{
sRelationship = "Subscribes to";
String sSubscriptionType = null;
if ( XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"a:category[@scheme='http://gdata.youtube.com/schemas/"
+ "2007/subscriptiontypes.cat']/@term",
oXmlNamespaceManager, out sSubscriptionType) )
{
sRelationship += " " + sSubscriptionType;
}
}
AppendEdgeXmlNode(oGraphMLXmlDocument, sUserName,
sOtherUserName, sRelationship);
}
}
if (bNeedToRecurse)
{
foreach (String sUserNameToRecurse in oUserNamesToRecurse)
{
GetUserNetworkRecursive(sUserNameToRecurse,
eWhatToInclude, bIncludeFriendsThisCall, eNetworkLevel,
iMaximumPeoplePerRequest, 2, oGraphMLXmlDocument,
oUserNameDictionary, oRequestStatistics);
}
}
}
//*************************************************************************
// Method: GetUserNetworkInternal()
//
/// <summary>
/// Gets a network of YouTube users, given a GraphMLXmlDocument.
/// </summary>
///
/// <param name="sUserNameToAnalyze">
/// The user name of the YouTube user whose network should be analyzed.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="eNetworkLevel">
/// Network level to include. Must be NetworkLevel.One, OnePointFive, or
/// Two.
/// </param>
///
/// <param name="iMaximumPeoplePerRequest">
/// Maximum number of people to request for each query, or Int32.MaxValue
/// for no limit.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// The GraphMLXmlDocument to populate with the requested network.
/// </param>
//*************************************************************************
protected void GetUserNetworkInternal(
String sUserNameToAnalyze,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
Int32 iMaximumPeoplePerRequest,
RequestStatistics oRequestStatistics,
GraphMLXmlDocument oGraphMLXmlDocument
)
{
Debug.Assert( !String.IsNullOrEmpty(sUserNameToAnalyze) );
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oGraphMLXmlDocument != null);
AssertValid();
// The key is the user name and the value is the corresponding GraphML
// XML node that represents the user. This is used to prevent the same
// user name from being added to the XmlDocument twice.
Dictionary<String, XmlNode> oUserNameDictionary =
new Dictionary<String, XmlNode>();
// Include friends, subscriptions, both, or neither.
Boolean [] abIncludes = new Boolean[] {
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.FriendVertices),
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SubscriptionVertices),
};
for (Int32 i = 0; i < 2; i++)
{
if ( abIncludes[i] )
{
GetUserNetworkRecursive(sUserNameToAnalyze, eWhatToInclude,
(i == 0), eNetworkLevel, iMaximumPeoplePerRequest, 1,
oGraphMLXmlDocument, oUserNameDictionary,
oRequestStatistics);
}
}
if ( WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.AllStatistics) )
{
AppendAllStatisticGraphMLAttributeValues(oGraphMLXmlDocument,
oUserNameDictionary, oRequestStatistics);
}
}
//*************************************************************************
// Method: GetUserNetworkInternal()
//
/// <overloads>
/// Gets a network of YouTube users.
/// </overloads>
///
/// <summary>
/// Gets a network of YouTube users.
/// </summary>
///
/// <param name="sUserNameToAnalyze">
/// The user name of the YouTube user whose network should be analyzed.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="eNetworkLevel">
/// Network level to include. Must be NetworkLevel.One, OnePointFive, or
/// Two.
/// </param>
///
/// <param name="iMaximumPeoplePerRequest">
/// Maximum number of people to request for each query, or Int32.MaxValue
/// for no limit.
/// </param>
///
/// <returns>
/// An XmlDocument containing the network as GraphML.
/// </returns>
//*************************************************************************
protected XmlDocument GetUserNetworkInternal(
String sUserNameToAnalyze,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
Int32 iMaximumPeoplePerRequest
)
{
Debug.Assert( !String.IsNullOrEmpty(sUserNameToAnalyze) );
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPeoplePerRequest > 0);
AssertValid();
GraphMLXmlDocument oGraphMLXmlDocument =
CreateGraphMLXmlDocument( WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.AllStatistics) );
RequestStatistics oRequestStatistics = new RequestStatistics();
try
{
GetUserNetworkInternal(sUserNameToAnalyze, eWhatToInclude,
eNetworkLevel, iMaximumPeoplePerRequest, oRequestStatistics,
oGraphMLXmlDocument);
}
catch (Exception oException)
{
OnTerminatingException(oException);
}
OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument,
oRequestStatistics);
return (oGraphMLXmlDocument);
}
//*************************************************************************
// Method: AppendInt32GraphMLAttributeValue()
//
/// <summary>
/// Appends an Int32 GraphML-Attribute value from an XmlDocument to a
/// vertex XML node.
/// </summary>
///
/// <param name="sUrl">
/// The URL to get the XmlDocument from.
/// </param>
///
/// <param name="sXPath">
/// Path to the value within the XmlDocument.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <param name="oVertexXmlNode">
/// The vertex XML node from <paramref name="oGraphMLXmlDocument" /> to add
/// the GraphML attribute value to.
/// </param>
///
/// <param name="sGraphMLAttributeID">
/// GraphML ID of the attribute.
/// </param>
///
/// <remarks>
/// This method looks for an Int32 value within an XmlDocument returned by
/// YouTube and stores the value on a vertex XML node in the GraphML
/// document.
/// </remarks>
//*************************************************************************
protected void AppendInt32GraphMLAttributeValue(
String sUrl,
String sXPath,
GraphMLXmlDocument oGraphMLXmlDocument,
RequestStatistics oRequestStatistics,
XmlNode oVertexXmlNode,
String sGraphMLAttributeID
)
{
Debug.Assert( !String.IsNullOrEmpty(sUrl) );
Debug.Assert( !String.IsNullOrEmpty(sXPath) );
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oVertexXmlNode != null);
Debug.Assert( !String.IsNullOrEmpty(sGraphMLAttributeID) );
AssertValid();
XmlDocument oXmlDocument;
XmlNamespaceManager oXmlNamespaceManager;
if ( !TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument,
out oXmlNamespaceManager) )
{
return;
}
AppendInt32GraphMLAttributeValue(oXmlDocument, sXPath,
oXmlNamespaceManager, oGraphMLXmlDocument, oVertexXmlNode,
sGraphMLAttributeID);
}
//*************************************************************************
// Method: AppendAllStatisticGraphMLAttributeValues()
//
/// <summary>
/// Appends statistic GraphML attribute values to the GraphML document for
/// one user in the network.
/// </summary>
///
/// <param name="sUserName">
/// The user name.
/// </param>
///
/// <param name="oVertexXmlNode">
/// The GraphML XML node corresponding to the user.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
//*************************************************************************
protected void AppendAllStatisticGraphMLAttributeValues(
String sUserName,
XmlNode oVertexXmlNode,
GraphMLXmlDocument oGraphMLXmlDocument,
RequestStatistics oRequestStatistics
)
{
Debug.Assert( !String.IsNullOrEmpty(sUserName) );
Debug.Assert(oVertexXmlNode != null);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
XmlDocument oXmlDocument;
XmlNamespaceManager oXmlNamespaceManager;
// Some of the statistics are available in the user's profile.
String sUrl = "http://gdata.youtube.com/feeds/api/users/"
+ EncodeUrlParameter(sUserName);
if ( TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument,
out oXmlNamespaceManager) )
{
AppendYouTubeDateGraphMLAttributeValue(oXmlDocument,
"a:entry/a:published/text()", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, JoinedDateUtcID);
AppendStringGraphMLAttributeValue(oXmlDocument,
"a:entry/media:thumbnail/@url", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, ImageFileID);
XmlNode oStatisticsXmlNode;
if ( XmlUtil2.TrySelectSingleNode(oXmlDocument,
"a:entry/yt:statistics", oXmlNamespaceManager,
out oStatisticsXmlNode) )
{
AppendInt32GraphMLAttributeValue(oStatisticsXmlNode,
"@videoWatchCount", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, VideosWatchedID);
AppendInt32GraphMLAttributeValue(oStatisticsXmlNode,
"@subscriberCount", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, SubscribersID);
}
}
// The remaining statistics must be obtained from the totalResults
// XML node within documents obtained from other URLs.
const String TotalResultsXPath =
"a:feed/openSearch:totalResults/text()";
sUrl = GetFriendsOrSubscriptionsUrl(sUserName, true)
+ "?max-results=1";
AppendInt32GraphMLAttributeValue(sUrl, TotalResultsXPath,
oGraphMLXmlDocument, oRequestStatistics, oVertexXmlNode,
FriendsID);
sUrl = GetFriendsOrSubscriptionsUrl(sUserName, false)
+ "?max-results=1";
AppendInt32GraphMLAttributeValue(sUrl, TotalResultsXPath,
oGraphMLXmlDocument, oRequestStatistics, oVertexXmlNode,
SubscriptionsID);
sUrl = "http://gdata.youtube.com/feeds/api/users/"
+ EncodeUrlParameter(sUserName) + "/uploads?max-results=1";
AppendInt32GraphMLAttributeValue(sUrl, TotalResultsXPath,
oGraphMLXmlDocument, oRequestStatistics, oVertexXmlNode,
VideosUploadedID);
}
//*************************************************************************
// Method: GetRelatedTagsInternal()
//
/// <summary>
/// Gets the Flickr tags related to a specified tag, given a
/// GraphXMLXmlDocument.
/// </summary>
///
/// <param name="sTag">
/// Tag to get related tags for.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="eNetworkLevel">
/// Network level to include. Must be NetworkLevel.One, OnePointFive, or
/// Two.
/// </param>
///
/// <param name="sApiKey">
/// Flickr API key.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// The GraphMLXmlDocument to populate with the requested network.
/// </param>
//*************************************************************************
protected void GetRelatedTagsInternal(
String sTag,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
String sApiKey,
RequestStatistics oRequestStatistics,
GraphMLXmlDocument oGraphMLXmlDocument
)
{
Debug.Assert( !String.IsNullOrEmpty(sTag) );
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert( !String.IsNullOrEmpty(sApiKey) );
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oGraphMLXmlDocument != null);
AssertValid();
// The key is the tag name and the value is the corresponding GraphML
// XML node that represents the tag. This is used to prevent the same
// tag from being added to the XmlDocument twice.
Dictionary<String, XmlNode> oTagDictionary =
new Dictionary<String, XmlNode>();
GetRelatedTagsRecursive(sTag, eWhatToInclude, eNetworkLevel, sApiKey,
1, oGraphMLXmlDocument, oTagDictionary, oRequestStatistics);
if ( WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SampleThumbnails) )
{
AppendSampleThumbnails(oTagDictionary, oGraphMLXmlDocument,
sApiKey, oRequestStatistics);
}
}
//*************************************************************************
// Method: GetRelatedTagsRecursive()
//
/// <summary>
/// Recursively gets a tag's related tags.
/// </summary>
///
/// <param name="sTag">
/// Tag to get related tags for.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="eNetworkLevel">
/// Network level to include. Must be NetworkLevel.One, OnePointFive, or
/// Two.
/// </param>
///
/// <param name="sApiKey">
/// Flickr API key.
/// </param>
///
/// <param name="iRecursionLevel">
/// Recursion level for this call. Must be 1 or 2. Gets incremented when
/// recursing.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oTagDictionary">
/// The key is the tag name and the value is the corresponding GraphML XML
/// node that represents the tag.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
//*************************************************************************
protected void GetRelatedTagsRecursive(
String sTag,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
String sApiKey,
Int32 iRecursionLevel,
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary<String, XmlNode> oTagDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert( !String.IsNullOrEmpty(sTag) );
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert( !String.IsNullOrEmpty(sApiKey) );
Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oTagDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
/*
Here is what this method should do, based on the eNetworkLevel and
iRecursionLevel parameters.
eNetworkLevel
|One | OnePointFive | Two
---|------------------| ------------------| -----------------
i 1 |Add all vertices. | Add all vertices. | Add all vertices.
R | | |
e |Add all edges. | Add all edges. | Add all edges.
c | | |
u |Do not recurse. | Recurse. | Recurse.
r | | |
s ---|------------------|-------------------|------------------
i 2 |Impossible. | Do not add | Add all vertices.
o | | vertices. |
n | | |
L | | Add edges only if | Add all edges.
e | | vertices are |
v | | already included. |
e | | |
l | | Do not recurse. | Do not recurse.
| | |
---|------------------|-------------------|------------------
*/
Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel,
iRecursionLevel);
Boolean bNeedToAppendVertices = GetNeedToAppendVertices(
eNetworkLevel, iRecursionLevel);
ReportProgress("Getting tags related to \"" + sTag + "\".");
String sUrl = GetFlickrMethodUrl( "flickr.tags.getRelated", sApiKey,
"&tag=" + UrlUtil.EncodeUrlParameter(sTag) );
XmlDocument oXmlDocument;
try
{
oXmlDocument = GetXmlDocument(sUrl, oRequestStatistics);
}
catch (Exception oException)
{
// If the exception is not a WebException or XmlException, or if
// none of the network has been obtained yet, throw the exception.
if (!ExceptionIsWebOrXml(oException) ||
!oGraphMLXmlDocument.HasVertexXmlNode)
{
throw oException;
}
return;
}
// The document consists of a single "tags" node with zero or more
// "tag" child nodes.
String sOtherTag = null;
XmlNodeList oTagNodes = oXmlDocument.DocumentElement.SelectNodes(
"tags/tag");
if (oTagNodes.Count > 0)
{
AppendVertexXmlNode(sTag, oGraphMLXmlDocument, oTagDictionary);
}
foreach (XmlNode oTagNode in oTagNodes)
{
sOtherTag = XmlUtil2.SelectRequiredSingleNodeAsString(oTagNode,
"text()", null);
if (bNeedToAppendVertices)
{
AppendVertexXmlNode(sOtherTag, oGraphMLXmlDocument,
oTagDictionary);
}
if ( bNeedToAppendVertices ||
oTagDictionary.ContainsKey(sOtherTag) )
{
oGraphMLXmlDocument.AppendEdgeXmlNode(sTag, sOtherTag);
}
}
if (bNeedToRecurse)
{
foreach (XmlNode oTagNode in oTagNodes)
{
sOtherTag = XmlUtil2.SelectRequiredSingleNodeAsString(oTagNode,
"text()", null);
GetRelatedTagsRecursive(sOtherTag, eWhatToInclude,
eNetworkLevel, sApiKey, 2, oGraphMLXmlDocument,
oTagDictionary, oRequestStatistics);
}
}
}
//*************************************************************************
// Method: OnNetworkObtainedWithoutTerminatingException()
//
/// <summary>
/// Call this when part or all of the network has been obtained without a
/// terminating exception occurring.
/// </summary>
///
/// <param name="oGraphMLXmlDocument">
/// GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <remarks>
/// If the entire network has been obtained, this method does nothing.
/// Otherwise, a PartialNetworkException is thrown.
/// </remarks>
//*************************************************************************
protected void OnNetworkObtainedWithoutTerminatingException(
GraphMLXmlDocument oGraphMLXmlDocument,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
if (oRequestStatistics.UnexpectedExceptions > 0)
{
// The network is partial.
throw new PartialNetworkException(oGraphMLXmlDocument,
oRequestStatistics);
}
}
//*************************************************************************
// Method: AppendFollowedEdgeXmlNodes()
//
/// <summary>
/// Appends an edge XML node for each pair of people who have tweeted a
/// specified search term and one follows the other.
/// </summary>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="iMaximumPeoplePerRequest">
/// Maximum number of people to request for each query, or Int32.MaxValue
/// for no limit.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oScreenNameDictionary">
/// The key is the screen name in lower case and the value is the
/// corresponding TwitterVertex.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
//*************************************************************************
protected void AppendFollowedEdgeXmlNodes(
WhatToInclude eWhatToInclude,
Int32 iMaximumPeoplePerRequest,
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary<String, TwitterVertex> oScreenNameDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oScreenNameDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
Boolean bIncludeStatistics = WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.Statistics);
// Look at the people followed by each author, and if a followed has
// also tweeted the search term, add an edge between the author and the
// followed.
foreach (String sScreenName in oScreenNameDictionary.Keys)
{
ReportProgressForFollowedOrFollowing(sScreenName, true);
foreach ( XmlNode oOtherUserXmlNode in EnumerateXmlNodes(
GetFollowedOrFollowingUrl(sScreenName, true),
"users_list/users/user", null, null, Int32.MaxValue,
iMaximumPeoplePerRequest, false, true, oRequestStatistics) )
{
String sOtherScreenName;
if ( !TryGetScreenName(oOtherUserXmlNode,
out sOtherScreenName) )
{
continue;
}
TwitterVertex oOtherTwitterVertex;
if ( oScreenNameDictionary.TryGetValue(sOtherScreenName,
out oOtherTwitterVertex) )
{
XmlNode oOtherVertexXmlNode =
oOtherTwitterVertex.VertexXmlNode;
XmlNode oEdgeXmlNode = AppendEdgeXmlNode(
oGraphMLXmlDocument, sScreenName, sOtherScreenName,
"Followed");
AppendRelationshipDateUtcGraphMLAttributeValue(
oGraphMLXmlDocument, oEdgeXmlNode, oRequestStatistics);
if (bIncludeStatistics &&
!AppendFromUserXmlNodeCalled(oGraphMLXmlDocument,
oOtherVertexXmlNode) )
{
// The other vertex node has no statistics. Add them.
AppendFromUserXmlNode(oOtherUserXmlNode,
oGraphMLXmlDocument, oOtherTwitterVertex,
bIncludeStatistics, false);
}
}
}
}
}
//*************************************************************************
// Method: AppendVertexXmlNodes()
//
/// <summary>
/// Appends a vertex XML node for each person who has tweeted a specified
/// search term.
/// </summary>
///
/// <param name="sSearchTerm">
/// The term to search for.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="iMaximumPeoplePerRequest">
/// Maximum number of people to request for each query, or Int32.MaxValue
/// for no limit.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// The GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oScreenNameDictionary">
/// The key is the screen name in lower case and the value is the
/// corresponding TwitterVertex.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
//*************************************************************************
protected void AppendVertexXmlNodes(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumPeoplePerRequest,
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary<String, TwitterVertex> oScreenNameDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) );
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oScreenNameDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
// Convert spaces in the search term to a plus sign.
//
// (Note: Don't try to use Twitter's "show_user" URL parameter in an
// attempt to get a "user" XML node for each author. That's not what
// the URL parameter does.)
String sUrl = String.Format(
"http://search.twitter.com/search.atom?q={0}&rpp=100"
,
UrlUtil.EncodeUrlParameter(sSearchTerm).Replace("%20", "+")
);
ReportProgress("Getting a list of tweets.");
Boolean bIncludeStatistics = WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.Statistics);
Boolean bIncludeStatuses = WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.Statuses);
// The document consists of an "entry" XML node for each tweet that
// contains the search term. Multiple tweets may have the same author,
// so we have to enumerate until the requested maximum number of unique
// authors is reached.
foreach ( XmlNode oAuthorNameXmlNode in EnumerateXmlNodes(
sUrl, "a:feed/a:entry/a:author/a:name", "a", AtomNamespaceUri,
15, Int32.MaxValue, true, false, oRequestStatistics) )
{
if (oScreenNameDictionary.Count == iMaximumPeoplePerRequest)
{
break;
}
// The author name is in this format:
//
// james_laker (James Laker)
String sAuthorName;
if ( !XmlUtil2.TrySelectSingleNodeAsString(oAuthorNameXmlNode,
"text()", null, out sAuthorName) )
{
continue;
}
Int32 iIndexOfSpace = sAuthorName.IndexOf(' ');
String sScreenName = sAuthorName;
if (iIndexOfSpace != -1)
{
sScreenName = sAuthorName.Substring(0, iIndexOfSpace);
}
sScreenName = sScreenName.ToLower();
TwitterVertex oTwitterVertex;
if ( !TryAppendVertexXmlNode(sScreenName, null,
oGraphMLXmlDocument, oScreenNameDictionary, bIncludeStatistics,
false, out oTwitterVertex) )
{
// A tweet for the author has already been found.
continue;
}
XmlNode oVertexXmlNode = oTwitterVertex.VertexXmlNode;
XmlNode oEntryXmlNode = oAuthorNameXmlNode.ParentNode.ParentNode;
XmlNamespaceManager oXmlNamespaceManager = new XmlNamespaceManager(
oEntryXmlNode.OwnerDocument.NameTable);
oXmlNamespaceManager.AddNamespace("a", AtomNamespaceUri);
// Get the image URL and status for the tweet's author.
AppendStringGraphMLAttributeValue(oEntryXmlNode,
"a:link[@rel='image']/@href", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, ImageFileID);
String sStatus;
if ( XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"a:title/text()", oXmlNamespaceManager, out sStatus) )
{
String sStatusDateUtc;
if ( XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"a:published/text()", oXmlNamespaceManager,
out sStatusDateUtc) )
{
sStatusDateUtc = TwitterDateParser.ParseTwitterDate(
sStatusDateUtc);
}
oTwitterVertex.StatusForAnalysis = sStatus;
oTwitterVertex.StatusForAnalysisDateUtc = sStatusDateUtc;
if (bIncludeStatuses)
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(
oVertexXmlNode, StatusID, sStatus);
if ( !String.IsNullOrEmpty(sStatusDateUtc) )
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(
oVertexXmlNode, StatusDateUtcID, sStatusDateUtc);
}
}
}
}
}
//*************************************************************************
// Method: GetUserNetworkRecursive()
//
/// <summary>
/// Recursively gets a network of Twitter users.
/// </summary>
///
/// <param name="sScreenName">
/// The screen name to use in this call.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="bIncludeFollowedThisCall">
/// true to include the people followed by the user, false to include the
/// people following the user.
/// </param>
///
/// <param name="eNetworkLevel">
/// Network level to include. Must be NetworkLevel.One, OnePointFive, or
/// Two.
/// </param>
///
/// <param name="iMaximumPeoplePerRequest">
/// Maximum number of people to request for each query, or Int32.MaxValue
/// for no limit.
/// </param>
///
/// <param name="iRecursionLevel">
/// Recursion level for this call. Must be 1 or 2. Gets incremented when
/// recursing.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oScreenNameDictionary">
/// The key is the screen name in lower case and the value is the
/// corresponding TwitterVertex.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
//*************************************************************************
protected void GetUserNetworkRecursive(
String sScreenName,
WhatToInclude eWhatToInclude,
Boolean bIncludeFollowedThisCall,
NetworkLevel eNetworkLevel,
Int32 iMaximumPeoplePerRequest,
Int32 iRecursionLevel,
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary<String, TwitterVertex> oScreenNameDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert( !String.IsNullOrEmpty(sScreenName) );
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oScreenNameDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
/*
Here is what this method should do, based on the eNetworkLevel and
iRecursionLevel parameters. It's assumed that
eWhatToInclude.FollowedFollowerEdge is set.
eNetworkLevel
|One | OnePointFive | Two
---|------------------| ------------------| -----------------
i 1 |Add all vertices. | Add all vertices. | Add all vertices.
R | | |
e |Add all edges. | Add all edges. | Add all edges.
c | | |
u |Do not recurse. | Recurse. | Recurse.
r | | |
s ---|------------------|-------------------|------------------
i 2 |Impossible. | Do not add | Add all vertices.
o | | vertices. |
n | | |
L | | Add edges only if | Add all edges.
e | | vertices are |
v | | already included. |
e | | |
l | | Do not recurse. | Do not recurse.
| | |
---|------------------|-------------------|------------------
*/
Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel,
iRecursionLevel);
List<String> oScreenNamesToRecurse = new List<String>();
Boolean bIncludeLatestStatuses = WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.LatestStatuses);
ReportProgressForFollowedOrFollowing(sScreenName,
bIncludeFollowedThisCall);
Boolean bThisUserAppended = false;
// If the GraphMLXmlDocument already contains at least one vertex node,
// then this is the second time that this method has been called, a
// partial network has already been obtained, and most errors should
// now be skipped. However, if none of the network has been obtained
// yet, errors on page 1 should throw an immediate exception.
Boolean bSkipMostPage1Errors = oGraphMLXmlDocument.HasVertexXmlNode;
// The document consists of a single "users" node with zero or more
// "user" child nodes.
foreach ( XmlNode oUserXmlNode in EnumerateXmlNodes(
GetFollowedOrFollowingUrl(sScreenName, bIncludeFollowedThisCall),
"users_list/users/user", null, null, Int32.MaxValue,
iMaximumPeoplePerRequest, false, bSkipMostPage1Errors,
oRequestStatistics) )
{
String sOtherScreenName;
if ( !TryGetScreenName(oUserXmlNode, out sOtherScreenName) )
{
// Nothing can be done without a screen name.
continue;
}
if (!bThisUserAppended)
{
// Append a vertex node for this request's user.
//
// This used to be done after the foreach loop, which avoided
// the need for a "bThisUserAppended" flag. That caused the
// following bug: If a Twitter error occurred within
// EnumerateXmlNodes() after some edges had been added, and the
// user decided to import the resulting partial network, the
// GraphML might contain edges that referenced "this user"
// without containing a vertex for "this user." That is an
// illegal state for GraphML, which the ExcelTemplate project
// caught and reported as an error.
TryAppendVertexXmlNode(sScreenName, null, oGraphMLXmlDocument,
oScreenNameDictionary, true, bIncludeLatestStatuses);
bThisUserAppended = true;
}
Boolean bNeedToAppendVertices = GetNeedToAppendVertices(
eNetworkLevel, iRecursionLevel);
if (bNeedToAppendVertices)
{
if (
TryAppendVertexXmlNode(sOtherScreenName, oUserXmlNode,
oGraphMLXmlDocument, oScreenNameDictionary, true,
bIncludeLatestStatuses)
&&
bNeedToRecurse
)
{
oScreenNamesToRecurse.Add(sOtherScreenName);
}
}
if ( WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.FollowedFollowerEdges) )
{
if ( bNeedToAppendVertices ||
oScreenNameDictionary.ContainsKey(sOtherScreenName) )
{
XmlNode oEdgeXmlNode;
if (bIncludeFollowedThisCall)
{
oEdgeXmlNode = AppendEdgeXmlNode(oGraphMLXmlDocument,
sScreenName, sOtherScreenName, "Followed");
}
else
{
oEdgeXmlNode = AppendEdgeXmlNode(oGraphMLXmlDocument,
sOtherScreenName, sScreenName, "Follower");
}
AppendRelationshipDateUtcGraphMLAttributeValue(
oGraphMLXmlDocument, oEdgeXmlNode, oRequestStatistics);
}
}
}
if (bNeedToRecurse)
{
foreach (String sScreenNameToRecurse in oScreenNamesToRecurse)
{
GetUserNetworkRecursive(sScreenNameToRecurse, eWhatToInclude,
bIncludeFollowedThisCall, eNetworkLevel,
iMaximumPeoplePerRequest, 2, oGraphMLXmlDocument,
oScreenNameDictionary, oRequestStatistics);
}
}
}
//*************************************************************************
// Method: GetSearchNetworkInternal()
//
/// <overloads>
/// Gets the network of people who have tweeted a specified search term.
/// </overloads>
///
/// <summary>
/// Gets the network of people who have tweeted a specified search term.
/// </summary>
///
/// <param name="sSearchTerm">
/// The term to search for.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="iMaximumPeoplePerRequest">
/// Maximum number of people to request for each query, or Int32.MaxValue
/// for no limit.
/// </param>
///
/// <returns>
/// An XmlDocument containing the network as GraphML.
/// </returns>
//*************************************************************************
protected XmlDocument GetSearchNetworkInternal(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumPeoplePerRequest
)
{
Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) );
Debug.Assert(iMaximumPeoplePerRequest > 0);
AssertValid();
BeforeGetNetwork();
Boolean bIncludeStatistics = WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.Statistics);
GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument(
bIncludeStatistics, false);
RequestStatistics oRequestStatistics = new RequestStatistics();
try
{
GetSearchNetworkInternal(sSearchTerm, eWhatToInclude,
iMaximumPeoplePerRequest, oRequestStatistics,
oGraphMLXmlDocument);
}
catch (Exception oException)
{
OnTerminatingException(oException);
}
OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument,
oRequestStatistics);
return (oGraphMLXmlDocument);
}
//*************************************************************************
// Method: AppendVertexXmlNodes()
//
/// <summary>
/// Appends a vertex XML node for each video that matches a specified
/// search term.
/// </summary>
///
/// <param name="sSearchTerm">
/// The term to search for.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="iMaximumVideos">
/// Maximum number of videos to request, or Int32.MaxValue for no limit.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// The GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <param name="oVideoIDs">
/// Where the set of unique video IDs gets stored.
/// </param>
///
/// <param name="oTagDictionary">
/// If an edge should be included for each pair of videos that share the
/// same tag, this gets set to a Dictionary for which the key is a
/// lower-case tag and the value is a LinkedList of the video IDs that have
/// the tag. Otherwise, it gets set to null.
/// </param>
//*************************************************************************
protected void AppendVertexXmlNodes(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumVideos,
GraphMLXmlDocument oGraphMLXmlDocument,
RequestStatistics oRequestStatistics,
out HashSet<String> oVideoIDs,
out Dictionary< String, LinkedList<String> > oTagDictionary
)
{
Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) );
Debug.Assert(iMaximumVideos> 0);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
ReportProgress("Getting a list of videos.");
// This is used to skip duplicate videos in the results returned by
// YouTube. (I'm not sure why YouTube sometimes returns duplicates,
// but it does.)
oVideoIDs = new HashSet<String>();
// If an edge should be included for each pair of videos that share the
// same tag, the key is a lower-case tag and the value is a LinkedList
// of the video IDs that have the tag.
if ( WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SharedTagEdges) )
{
oTagDictionary = new Dictionary< String, LinkedList<String> >();
}
else
{
oTagDictionary = null;
}
String sUrl = String.Format(
"http://gdata.youtube.com/feeds/api/videos?q={0}"
,
EncodeUrlParameter(sSearchTerm)
);
// The document consists of an "entry" XML node for each video.
foreach ( XmlNode oEntryXmlNode in EnumerateXmlNodes(sUrl,
"a:feed/a:entry", iMaximumVideos, false, oRequestStatistics) )
{
XmlNamespaceManager oXmlNamespaceManager =
CreateXmlNamespaceManager(oEntryXmlNode.OwnerDocument);
// Use the video ID as the GraphML vertex name. The video title
// can't be used because it is not unique.
String sVideoID;
if (
!XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"media:group/yt:videoid/text()", oXmlNamespaceManager,
out sVideoID)
||
oVideoIDs.Contains(sVideoID)
)
{
continue;
}
oVideoIDs.Add(sVideoID);
XmlNode oVertexXmlNode = oGraphMLXmlDocument.AppendVertexXmlNode(
sVideoID);
AppendStringGraphMLAttributeValue(oEntryXmlNode,
"a:title/text()", oXmlNamespaceManager, oGraphMLXmlDocument,
oVertexXmlNode, TitleID);
AppendDoubleGraphMLAttributeValue(oEntryXmlNode,
"gd:rating/@average", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, RatingID);
AppendInt32GraphMLAttributeValue(oEntryXmlNode,
"yt:statistics/@viewCount", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, ViewsID);
AppendInt32GraphMLAttributeValue(oEntryXmlNode,
"yt:statistics/@favoriteCount", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, FavoritedID);
AppendInt32GraphMLAttributeValue(oEntryXmlNode,
"gd:comments/gd:feedLink/@countHint", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, CommentsID);
AppendYouTubeDateGraphMLAttributeValue(oEntryXmlNode,
"a:published/text()", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, CreatedDateUtcID);
AppendStringGraphMLAttributeValue(oEntryXmlNode,
"media:group/media:thumbnail/@url", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, ImageFileID);
if ( AppendStringGraphMLAttributeValue(oEntryXmlNode,
"media:group/media:player/@url", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, MenuActionID) )
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(oVertexXmlNode,
MenuTextID, "Play Video in Browser");
}
if (oTagDictionary != null)
{
CollectTags(oEntryXmlNode, sVideoID, oXmlNamespaceManager,
oTagDictionary);
}
}
}
//*************************************************************************
// Method: GetSearchNetworkInternal()
//
/// <summary>
/// Gets the network of people who have tweeted a specified search term,
/// given a GraphMLXmlDocument.
/// </summary>
///
/// <param name="sSearchTerm">
/// The term to search for.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="iMaximumPeoplePerRequest">
/// Maximum number of people to request for each query, or Int32.MaxValue
/// for no limit.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// The GraphMLXmlDocument to populate with the requested network.
/// </param>
//*************************************************************************
protected void GetSearchNetworkInternal(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumPeoplePerRequest,
RequestStatistics oRequestStatistics,
GraphMLXmlDocument oGraphMLXmlDocument
)
{
Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) );
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oGraphMLXmlDocument != null);
AssertValid();
if ( WhatToIncludeFlagIsSet(eWhatToInclude, WhatToInclude.Statuses) )
{
oGraphMLXmlDocument.DefineGraphMLAttribute(false, StatusID,
"Tweet", "string", null);
oGraphMLXmlDocument.DefineGraphMLAttribute(false, StatusDateUtcID,
"Tweet Date (UTC)", "string", null);
}
// The key is the screen name and the value is the corresponding
// TwitterVertex. This is used to prevent the same screen name from
// being added to the XmlDocument twice.
Dictionary<String, TwitterVertex> oScreenNameDictionary =
new Dictionary<String, TwitterVertex>();
// First, add a vertex for each person who has tweeted the search term.
AppendVertexXmlNodes(sSearchTerm, eWhatToInclude,
iMaximumPeoplePerRequest, oGraphMLXmlDocument,
oScreenNameDictionary, oRequestStatistics);
if ( WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.FollowedEdges) )
{
// Look at the people followed by each author, and if a followed
// has also tweeted the search term, add an edge between the author
// and the followed.
AppendFollowedEdgeXmlNodes(eWhatToInclude,
iMaximumPeoplePerRequest, oGraphMLXmlDocument,
oScreenNameDictionary, oRequestStatistics);
}
Boolean bIncludeStatistics = WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.Statistics);
AppendMissingGraphMLAttributeValues(oGraphMLXmlDocument,
oScreenNameDictionary, bIncludeStatistics, false,
oRequestStatistics);
AppendRepliesToAndMentionsXmlNodes(oGraphMLXmlDocument,
oScreenNameDictionary,
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.RepliesToEdges),
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.MentionsEdges)
);
}
//*************************************************************************
// Method: GetVideoNetworkInternal()
//
/// <summary>
/// Gets a network of related YouTube videos, given a GraphMLXmlDocument.
/// </summary>
///
/// <param name="sSearchTerm">
/// The term to search for.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="iMaximumVideos">
/// Maximum number of videos to request, or Int32.MaxValue for no limit.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// The GraphMLXmlDocument to populate with the requested network.
/// </param>
//*************************************************************************
protected void GetVideoNetworkInternal(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumVideos,
RequestStatistics oRequestStatistics,
GraphMLXmlDocument oGraphMLXmlDocument
)
{
Debug.Assert( !String.IsNullOrEmpty(sSearchTerm) );
Debug.Assert(iMaximumVideos > 0);
Debug.Assert(oRequestStatistics != null);
Debug.Assert(oGraphMLXmlDocument != null);
AssertValid();
// First, add a vertex for each video matching the search term.
HashSet<String> oVideoIDs;
Dictionary< String, LinkedList<String> > oTagDictionary;
AppendVertexXmlNodes(sSearchTerm, eWhatToInclude, iMaximumVideos,
oGraphMLXmlDocument, oRequestStatistics, out oVideoIDs,
out oTagDictionary);
// Now add whatever edges were requested.
if ( WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SharedTagEdges) )
{
Debug.Assert(oTagDictionary != null);
ReportProgress("Adding edges for shared tags.");
AppendEdgesFromDictionary(oTagDictionary, oGraphMLXmlDocument,
"Shared tag");
}
oTagDictionary = null;
if ( WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SharedCommenterEdges) )
{
AppendSharedResponderEdges(oGraphMLXmlDocument, oVideoIDs,
MaximumCommentsPerVideo,
"http://gdata.youtube.com/feeds/api/videos/{0}/comments",
"commenter", oRequestStatistics);
}
if ( WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SharedVideoResponderEdges) )
{
AppendSharedResponderEdges(oGraphMLXmlDocument, oVideoIDs,
iMaximumVideos,
"http://gdata.youtube.com/feeds/api/videos/{0}/responses",
"video responder", oRequestStatistics);
}
}
//*************************************************************************
// Method: AppendSampleThumbnails()
//
/// <summary>
/// Appends sample thumbnails to the GraphMLXmlDocument being populated.
/// </summary>
///
/// <param name="oTagDictionary">
/// The key is the tag name and the value is the corresponding GraphML XML
/// node that represents the tag.
/// </param>
///
/// <param name="oGraphMLXmlDocument">
/// GraphMLXmlDocument being populated.
/// </param>
///
/// <param name="sApiKey">
/// Flickr API key.
/// </param>
///
/// <param name="oRequestStatistics">
/// A <see cref="RequestStatistics" /> object that is keeping track of
/// requests made while getting the network.
/// </param>
//*************************************************************************
protected void AppendSampleThumbnails(
Dictionary<String, XmlNode> oTagDictionary,
GraphMLXmlDocument oGraphMLXmlDocument,
String sApiKey,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(oTagDictionary != null);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert( !String.IsNullOrEmpty(sApiKey) );
Debug.Assert(oRequestStatistics != null);
AssertValid();
foreach (KeyValuePair<String, XmlNode> oKeyValuePair in oTagDictionary)
{
String sTag = oKeyValuePair.Key;
ReportProgress("Getting sample image file for \"" + sTag + "\".");
String sSampleImageUrl;
if ( TryGetSampleImageUrl(sTag, sApiKey, oRequestStatistics,
out sSampleImageUrl) )
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(
oKeyValuePair.Value, ImageFileID, sSampleImageUrl);
}
}
}
OnNetworkObtainedWithoutTerminatingException
(
GraphMLXmlDocument oGraphMLXmlDocument,
RequestStatistics oRequestStatistics,
String sNetworkDescription
)
{
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oRequestStatistics != null);
Debug.Assert( !String.IsNullOrEmpty(sNetworkDescription) );
AssertValid();
XmlUtil2.SetAttributes(oGraphMLXmlDocument.GraphXmlNode, "description",
sNetworkDescription);
if (oRequestStatistics.UnexpectedExceptions > 0)
{
// The network is partial.
throw new PartialNetworkException(oGraphMLXmlDocument,
oRequestStatistics);
}
}
//*************************************************************************
// Method: GetRelatedTagsInternal()
//
/// <overloads>
/// Gets the Flickr tags related to a specified tag.
/// </overloads>
///
/// <summary>
/// Gets the Flickr tags related to a specified tag.
/// </summary>
///
/// <param name="sTag">
/// Tag to get related tags for.
/// </param>
///
/// <param name="eWhatToInclude">
/// Specifies what should be included in the network.
/// </param>
///
/// <param name="eNetworkLevel">
/// Network level to include. Must be NetworkLevel.One, OnePointFive, or
/// Two.
/// </param>
///
/// <param name="sApiKey">
/// Flickr API key.
/// </param>
///
/// <returns>
/// An XmlDocument containing the network as GraphML.
/// </returns>
//*************************************************************************
protected XmlDocument GetRelatedTagsInternal(
String sTag,
WhatToInclude eWhatToInclude,
NetworkLevel eNetworkLevel,
String sApiKey
)
{
Debug.Assert( !String.IsNullOrEmpty(sTag) );
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert( !String.IsNullOrEmpty(sApiKey) );
AssertValid();
GraphMLXmlDocument oGraphMLXmlDocument = CreateGraphMLXmlDocument(
WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SampleThumbnails) );
RequestStatistics oRequestStatistics = new RequestStatistics();
try
{
GetRelatedTagsInternal(sTag, eWhatToInclude, eNetworkLevel,
sApiKey, oRequestStatistics, oGraphMLXmlDocument);
}
catch (Exception oException)
{
OnTerminatingException(oException);
}
OnNetworkObtainedWithoutTerminatingException(oGraphMLXmlDocument,
oRequestStatistics);
return (oGraphMLXmlDocument);
}
EnumerateSearchStatuses
(
String searchTerm,
Int32 maximumStatuses,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert( !String.IsNullOrEmpty(searchTerm) );
Debug.Assert(maximumStatuses > 0);
Debug.Assert(maximumStatuses != Int32.MaxValue);
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iPage = 1;
Int32 iStatusesEnumerated = 0;
String sQueryParametersForNextPage = null;
while (true)
{
if (reportProgressHandler != null && iPage > 1)
{
reportProgressHandler("Getting page " + iPage + ".");
}
Dictionary<String, Object> oResponseDictionary = null;
Object [] aoStatusesThisPage;
String sUrl = GetSearchUrl(searchTerm,
sQueryParametersForNextPage);
try
{
Object oDeserializedTwitterResponse =
( new JavaScriptSerializer() ).DeserializeObject(
GetTwitterResponseAsString(
sUrl, requestStatistics, reportProgressHandler,
checkCancellationPendingHandler) );
// The top level of the Json response contains a set of
// name/value pairs. The value for the "statuses" name is the
// array of objects this method will enumerate.
oResponseDictionary = ( Dictionary<String, Object> )
oDeserializedTwitterResponse;
aoStatusesThisPage =
( Object [] )oResponseDictionary["statuses"];
}
catch (Exception oException)
{
// Rethrow the exception if appropriate.
OnExceptionWhileEnumeratingJsonValues(oException, iPage,
false);
// Otherwise, just halt the enumeration.
yield break;
}
Int32 iObjectsThisPage = aoStatusesThisPage.Length;
if (iObjectsThisPage == 0)
{
yield break;
}
for (Int32 i = 0; i < iObjectsThisPage; i++)
{
yield return (
( Dictionary<String, Object> )aoStatusesThisPage[i] );
iStatusesEnumerated++;
if (iStatusesEnumerated == maximumStatuses)
{
yield break;
}
}
iPage++;
if ( !TryGetQueryParametersForNextSearchPage(oResponseDictionary,
out sQueryParametersForNextPage) )
{
yield break;
}
// Get the next page...
}
}
GetTwitterResponseAsString
(
String url,
RequestStatistics requestStatistics,
ReportProgressHandler reportProgressHandler,
CheckCancellationPendingHandler checkCancellationPendingHandler
)
{
Debug.Assert( !String.IsNullOrEmpty(url) );
Debug.Assert(requestStatistics != null);
AssertValid();
Int32 iRateLimitPauses = 0;
while (true)
{
// Add the required authorization information to the URL.
//
// Note: Don't do this outside the while (true) loop. The
// authorization information includes a timestamp that will
// probably expire if the code within the catch block pauses.
oAuthTwitter oAuthTwitter = new oAuthTwitter(
m_UserAgent, m_TimeoutMs);
oAuthTwitter.Token = m_TwitterAccessToken;
oAuthTwitter.TokenSecret = m_TwitterAccessTokenSecret;
String sAuthorizedUrl, sAuthorizedPostData;
oAuthTwitter.ConstructAuthWebRequest(oAuthTwitter.Method.GET,
url, String.Empty, out sAuthorizedUrl,
out sAuthorizedPostData);
url = sAuthorizedUrl;
Stream oStream = null;
try
{
oStream =
HttpSocialNetworkUtil.GetHttpWebResponseStreamWithRetries(
url, HttpStatusCodesToFailImmediately,
requestStatistics, m_UserAgent, m_TimeoutMs,
reportProgressHandler, checkCancellationPendingHandler);
return ( new StreamReader(oStream).ReadToEnd() );
}
catch (WebException oWebException)
{
if (
!WebExceptionIsDueToRateLimit(oWebException)
||
iRateLimitPauses > 0
)
{
throw;
}
// Twitter rate limits have kicked in. Pause and try again.
iRateLimitPauses++;
Int32 iRateLimitPauseMs = GetRateLimitPauseMs(oWebException);
DateTime oWakeUpTime = DateTime.Now.AddMilliseconds(
iRateLimitPauseMs);
if (reportProgressHandler != null)
{
reportProgressHandler( String.Format(
"Reached Twitter rate limits. Pausing until {0}."
,
oWakeUpTime.ToLongTimeString()
) );
}
// Don't pause in one large interval, which would prevent
// cancellation.
const Int32 SleepCycleDurationMs = 1000;
Int32 iSleepCycles = (Int32)Math.Ceiling(
(Double)iRateLimitPauseMs / SleepCycleDurationMs) ;
for (Int32 i = 0; i < iSleepCycles; i++)
{
if (checkCancellationPendingHandler != null)
{
checkCancellationPendingHandler();
}
System.Threading.Thread.Sleep(SleepCycleDurationMs);
}
}
finally
{
if (oStream != null)
{
oStream.Close();
}
}
}
}
