AppendStringGraphMLAttributeValue
(
XmlNode oXmlNodeToSelectFrom,
String sXPath,
XmlNamespaceManager oXmlNamespaceManager,
GraphMLXmlDocument oGraphMLXmlDocument,
XmlNode oEdgeOrVertexXmlNode,
String sGraphMLAttributeID
)
{
Debug.Assert(oXmlNodeToSelectFrom != null);
Debug.Assert(!String.IsNullOrEmpty(sXPath));
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oEdgeOrVertexXmlNode != null);
Debug.Assert(!String.IsNullOrEmpty(sGraphMLAttributeID));
AssertValid();
String sAttributeValue;
if (XmlUtil2.TrySelectSingleNodeAsString(oXmlNodeToSelectFrom, sXPath,
oXmlNamespaceManager, out sAttributeValue))
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(
oEdgeOrVertexXmlNode, sGraphMLAttributeID, sAttributeValue);
return(true);
}
return(false);
}
GetCommentersEnumerator
(
String sUserID,
Int32 iMaximumPerRequest,
Boolean bSkipMostPage1Errors,
String sApiKey,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sUserID));
Debug.Assert(iMaximumPerRequest > 0);
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
Debug.Assert(oRequestStatistics != null);
AssertValid();
// Get the user's public photos, which are paged.
String sUrl = GetFlickrMethodUrl("flickr.people.getPublicPhotos",
sApiKey, GetUserIDUrlParameter(sUserID));
foreach (XmlNode oPhotoXmlNode in EnumerateXmlNodes(sUrl,
"rsp/photos/photo", iMaximumPerRequest, bSkipMostPage1Errors,
oRequestStatistics))
{
String sPhotoID;
if (!XmlUtil2.TrySelectSingleNodeAsString(oPhotoXmlNode, "@id",
null, out sPhotoID))
{
continue;
}
// Get the photo's comments, which are not paged.
ReportProgress(String.Format(
"Getting comments for the photo \"{0}\"."
,
sPhotoID
));
sUrl = GetFlickrMethodUrl("flickr.photos.comments.getList",
sApiKey, "&photo_id=" + UrlUtil.EncodeUrlParameter(sPhotoID)
);
XmlDocument oXmlDocument;
if (TryGetXmlDocument(sUrl, oRequestStatistics,
out oXmlDocument))
{
foreach (XmlNode oCommentXmlNode in oXmlDocument.SelectNodes(
"rsp/comments/comment"))
{
yield return(oCommentXmlNode);
}
}
}
}
ParseEdges
(
IGraph oGraph,
XmlNode oGraphXmlNode,
XmlNamespaceManager oXmlNamespaceManager,
Dictionary <String, IVertex> oVertexDictionary,
Dictionary <String, GraphMLAttribute> oGraphMLAttributeDictionary
)
{
Debug.Assert(oGraph != null);
Debug.Assert(oGraphXmlNode != null);
Debug.Assert(oXmlNamespaceManager != null);
Debug.Assert(oVertexDictionary != null);
Debug.Assert(oGraphMLAttributeDictionary != null);
AssertValid();
IEdgeCollection oEdges = oGraph.Edges;
Boolean bGraphIsDirected =
(oGraph.Directedness == GraphDirectedness.Directed);
foreach (XmlNode oEdgeXmlNode in oGraphXmlNode.SelectNodes(
GraphMLPrefix + ":edge", oXmlNamespaceManager))
{
IVertex oVertex1, oVertex2;
if (
!TryGraphMLNodeIDToVertex(oEdgeXmlNode, "source",
oVertexDictionary, out oVertex1)
||
!TryGraphMLNodeIDToVertex(oEdgeXmlNode, "target",
oVertexDictionary, out oVertex2)
)
{
// From the GraphML Primer:
//
// For applications which can not handle nested graphs the
// fall-back behaviour is to ignore nodes which are not
// contained in the top-level graph and to ignore edges which
// have do not have both endpoints in the top-level graph.
continue;
}
IEdge oEdge = oEdges.Add(oVertex1, oVertex2, bGraphIsDirected);
String sID;
if (XmlUtil2.TrySelectSingleNodeAsString(oEdgeXmlNode, "@id",
null, out sID))
{
oEdge.Name = sID;
}
ParseGraphMLAttributeValues(oEdgeXmlNode, oXmlNamespaceManager,
oEdge, false, oGraphMLAttributeDictionary);
}
}
YouTubeErrorResponseToMessage
(
HttpWebResponse oHttpWebResponse
)
{
Debug.Assert(oHttpWebResponse != null);
AssertValid();
// YouTube provides error information as an XmlDocument.
String sMessage = null;
XmlDocument oXmlDocument = new XmlDocument();
Stream oStream = null;
try
{
oStream = oHttpWebResponse.GetResponseStream();
oXmlDocument.Load(oStream);
}
catch (XmlException)
{
return (null);
}
finally
{
if (oStream != null)
{
oStream.Close();
}
}
XmlNamespaceManager oXmlNamespaceManager =
YouTubeUserNetworkAnalyzer.CreateXmlNamespaceManager(oXmlDocument);
// Although the document may contain multiple error nodes, just look at
// the first one for now.
if ( XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
"gd:errors/gd:error/gd:code/text()", oXmlNamespaceManager,
out sMessage) )
{
if (sMessage.IndexOf("too_many_recent_calls") >= 0)
{
sMessage =
"You have made too many YouTube network requests recently."
+ " Wait a few minutes and try again."
;
}
}
else
{
sMessage = null;
}
return (sMessage);
}
GetXmlDocument
(
String sUrl,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sUrl));
Debug.Assert(oRequestStatistics != null);
AssertValid();
XmlDocument oXmlDocument = GetXmlDocumentWithRetries(sUrl,
HttpStatusCodesToFailImmediately, oRequestStatistics);
String sStatus;
if (
XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/@stat",
null, out sStatus)
&&
sStatus == "ok"
)
{
return(oXmlDocument);
}
// Flickr indicates errors by returning an XML document containing an
// rsp/err node. The following code turns such an error document into
// a custom FlickrException.
String sErrorMessage;
if (XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument, "rsp/err/@msg",
null, out sErrorMessage))
{
if (sErrorMessage.ToLower().IndexOf("user not found") >= 0)
{
sErrorMessage =
"The user wasn't found. Either there is no such user, or"
+ " she has hidden herself from public searches."
;
}
}
else
{
sErrorMessage =
"Flickr provided information in an unrecognized format.";
}
throw new FlickrException(sErrorMessage);
}
TryGetSampleImageUrl
(
String sTag,
String sApiKey,
RequestStatistics oRequestStatistics,
out String sSampleImageUrl
)
{
Debug.Assert(!String.IsNullOrEmpty(sTag));
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
Debug.Assert(oRequestStatistics != null);
AssertValid();
sSampleImageUrl = null;
String sUrl = GetFlickrMethodUrl("flickr.tags.getClusterPhotos",
sApiKey, "&tag=" + UrlUtil.EncodeUrlParameter(sTag));
XmlDocument oXmlDocument;
String sPhotoID;
if (
!TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument)
||
!XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
"rsp/photos/photo/@id", null, out sPhotoID)
)
{
return(false);
}
sUrl = GetFlickrMethodUrl("flickr.photos.getSizes", sApiKey,
"&photo_id=" + UrlUtil.EncodeUrlParameter(sPhotoID));
if (
!TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument)
||
!XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
"rsp/sizes/size[@label='Thumbnail']/@source", null,
out sSampleImageUrl)
)
{
return(false);
}
return(true);
}
GetTwitterCommonConfiguration
(
XmlNode oParentNode,
out Int32 iMaximumPeoplePerRequest,
out String sNetworkFileFolderPath,
out NetworkFileFormats eNetworkFileFormats,
out Boolean bAutomateNodeXLWorkbook
)
{
Debug.Assert(oParentNode != null);
AssertValid();
String sMaximumPeoplePerRequest;
iMaximumPeoplePerRequest = Int32.MaxValue;
if (XmlUtil2.TrySelectSingleNodeAsString(oParentNode,
"MaximumPeoplePerRequest/text()", null,
out sMaximumPeoplePerRequest))
{
if (!Int32.TryParse(sMaximumPeoplePerRequest,
out iMaximumPeoplePerRequest))
{
throw new XmlException(
"The MaximumPeoplePerRequest value is not valid."
);
}
}
sNetworkFileFolderPath = XmlUtil2.SelectRequiredSingleNodeAsString(
oParentNode, "NetworkFileFolder/text()", null);
eNetworkFileFormats = GetRequiredEnumValue <NetworkFileFormats>(
oParentNode, "NetworkFileFormats/text()", "NetworkFileFormats");
// The AutomateNodeXLWorkbook node was added in a later version of the
// program, so it is not required.
if (!XmlUtil2.TrySelectSingleNodeAsBoolean(
oParentNode, "AutomateNodeXLWorkbook/text()", null,
out bAutomateNodeXLWorkbook))
{
bAutomateNodeXLWorkbook = false;
}
}
ParseGraphAttribute
(
IGraph oGraph,
XmlNode oGraphXmlNode,
String sXmlAttributeName,
String sKey
)
{
Debug.Assert(oGraph != null);
Debug.Assert(oGraphXmlNode != null);
Debug.Assert(!String.IsNullOrEmpty(sXmlAttributeName));
Debug.Assert(!String.IsNullOrEmpty(sKey));
AssertValid();
String sValue;
if (XmlUtil2.TrySelectSingleNodeAsString(oGraphXmlNode,
"@" + sXmlAttributeName, null, out sValue))
{
oGraph.SetValue(sKey, sValue);
}
}
TryGetEdgeGraphMLAttributeValue
(
XmlNode oEdgeXmlNode,
String sAttributeID,
XmlNamespaceManager oXmlNamespaceManager,
out String sAttributeValue
)
{
Debug.Assert(oEdgeXmlNode != null);
Debug.Assert(!String.IsNullOrEmpty(sAttributeID));
Debug.Assert(oXmlNamespaceManager != null);
String sXPath = String.Format(
"g:data[@key=\"{0}\"]/text()"
,
sAttributeID
);
return(XmlUtil2.TrySelectSingleNodeAsString(
oEdgeXmlNode, sXPath, oXmlNamespaceManager, out sAttributeValue));
}
GetAttributeValue
(
XmlNode dataXmlNode
)
{
Debug.Assert(dataXmlNode != null);
Debug.Assert(dataXmlNode.Name == "data");
AssertValid();
String sKey = XmlUtil2.SelectRequiredSingleNodeAsString(dataXmlNode,
"@key", null);
Debug.Assert(sKey == m_sID);
String sAttributeValue;
if (!XmlUtil2.TrySelectSingleNodeAsString(dataXmlNode, "text()",
null, out sAttributeValue))
{
// Allow missing inner text for GraphML-attributes of type string.
// This was found in a GraphML file created by the yED program.
sAttributeValue = String.Empty;
}
try
{
return(ConvertAttributeValue(sAttributeValue));
}
catch (FormatException)
{
throw new XmlException(
"The GraphML-attribute value specified for a \"data\" XML node"
+ " with the key \"" + sKey + "\" is not of the specified"
+ " type."
);
}
}
AppendVertexXmlNodes
(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumVideos,
GraphMLXmlDocument oGraphMLXmlDocument,
RequestStatistics oRequestStatistics,
out HashSet <String> oVideoIDs,
out Dictionary <String, LinkedList <String> > oCategoryDictionary
)
{
Debug.Assert(!String.IsNullOrEmpty(sSearchTerm));
Debug.Assert(iMaximumVideos > 0);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
ReportProgress("Getting a list of videos.");
// This is used to skip duplicate videos in the results returned by
// YouTube. (I'm not sure why YouTube sometimes returns duplicates,
// but it does.)
oVideoIDs = new HashSet <String>();
// If an edge should be included for each pair of videos that share the
// same category, the key is a lower-case category and the value is a
// LinkedList of the video IDs that have the category.
if (WhatToIncludeFlagIsSet(eWhatToInclude,
WhatToInclude.SharedCategoryEdges))
{
oCategoryDictionary =
new Dictionary <String, LinkedList <String> >();
}
else
{
oCategoryDictionary = null;
}
String sUrl = String.Format(
"http://gdata.youtube.com/feeds/api/videos?q={0}"
,
EncodeUrlParameter(sSearchTerm)
);
// The document consists of an "entry" XML node for each video.
foreach (XmlNode oEntryXmlNode in EnumerateXmlNodes(sUrl,
"a:feed/a:entry", iMaximumVideos, false, oRequestStatistics))
{
XmlNamespaceManager oXmlNamespaceManager =
CreateXmlNamespaceManager(oEntryXmlNode.OwnerDocument);
// Use the video ID as the GraphML vertex name. The video title
// can't be used because it is not unique.
String sVideoID;
if (
!XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"media:group/yt:videoid/text()", oXmlNamespaceManager,
out sVideoID)
||
oVideoIDs.Contains(sVideoID)
)
{
continue;
}
oVideoIDs.Add(sVideoID);
XmlNode oVertexXmlNode = oGraphMLXmlDocument.AppendVertexXmlNode(
sVideoID);
AppendStringGraphMLAttributeValue(oEntryXmlNode,
"a:title/text()", oXmlNamespaceManager, oGraphMLXmlDocument,
oVertexXmlNode, TitleID);
AppendStringGraphMLAttributeValue(oEntryXmlNode,
"a:author/a:name/text()", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, AuthorID);
AppendDoubleGraphMLAttributeValue(oEntryXmlNode,
"gd:rating/@average", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, RatingID);
AppendInt32GraphMLAttributeValue(oEntryXmlNode,
"yt:statistics/@viewCount", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, ViewsID);
AppendInt32GraphMLAttributeValue(oEntryXmlNode,
"yt:statistics/@favoriteCount", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, FavoritedID);
AppendInt32GraphMLAttributeValue(oEntryXmlNode,
"gd:comments/gd:feedLink/@countHint", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, CommentsID);
AppendYouTubeDateGraphMLAttributeValue(oEntryXmlNode,
"a:published/text()", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, CreatedDateUtcID);
AppendStringGraphMLAttributeValue(oEntryXmlNode,
"media:group/media:thumbnail/@url", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode,
NodeXLGraphMLUtil.VertexImageFileID);
if (AppendStringGraphMLAttributeValue(oEntryXmlNode,
"media:group/media:player/@url", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode,
NodeXLGraphMLUtil.VertexMenuActionID))
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(oVertexXmlNode,
NodeXLGraphMLUtil.VertexMenuTextID,
"Play Video in Browser");
}
if (oCategoryDictionary != null)
{
CollectCategories(oEntryXmlNode, sVideoID,
oXmlNamespaceManager, oCategoryDictionary);
}
}
}
GetUserNetworkRecursive
(
String sUserName,
WhatToInclude eWhatToInclude,
Boolean bIncludeFriendsThisCall,
NetworkLevel eNetworkLevel,
Int32 iMaximumPeoplePerRequest,
Int32 iRecursionLevel,
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary <String, XmlNode> oUserNameDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sUserName));
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oUserNameDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
/*
* Here is what this method should do, based on the eNetworkLevel and
* iRecursionLevel parameters.
*
* eNetworkLevel
*
|One | OnePointFive | Two
* ---|------------------| ------------------| -----------------
* i 1 |Add all vertices. | Add all vertices. | Add all vertices.
* R | | |
* e |Add all edges. | Add all edges. | Add all edges.
* c | | |
* u |Do not recurse. | Recurse. | Recurse.
* r | | |
* s ---|------------------|-------------------|------------------
* i 2 |Impossible. | Do not add | Add all vertices.
* o | | vertices. |
* n | | |
* L | | Add edges only if | Add all edges.
* e | | vertices are |
* v | | already included. |
* e | | |
* l | | Do not recurse. | Do not recurse.
| | |
| ---|------------------|-------------------|------------------
*/
Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel,
iRecursionLevel);
List <String> oUserNamesToRecurse = new List <String>();
ReportProgressForFriendsOrSubscriptions(sUserName,
bIncludeFriendsThisCall);
Boolean bThisUserAppended = false;
// If the GraphMLXmlDocument already contains at least one vertex node,
// then this is the second time that this method has been called, a
// partial network has already been obtained, and most errors should
// now be skipped. However, if none of the network has been obtained
// yet, errors on page 1 should throw an immediate exception.
Boolean bSkipMostPage1Errors = oGraphMLXmlDocument.HasVertexXmlNode;
// The document consists of a single "feed" node with zero or more
// "entry" child nodes.
foreach (XmlNode oEntryXmlNode in EnumerateXmlNodes(
GetFriendsOrSubscriptionsUrl(sUserName, bIncludeFriendsThisCall),
"a:feed/a:entry", iMaximumPeoplePerRequest, bSkipMostPage1Errors,
oRequestStatistics))
{
XmlNamespaceManager oXmlNamespaceManager =
CreateXmlNamespaceManager(oEntryXmlNode.OwnerDocument);
String sOtherUserName;
if (!XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"yt:username/text()", oXmlNamespaceManager,
out sOtherUserName))
{
continue;
}
if (!bThisUserAppended)
{
// Append a vertex node for this request's user.
//
// This used to be done after the foreach loop, which avoided
// the need for a "bThisUserAppended" flag. That caused the
// following bug: If a YouTube error occurred within
// EnumerateXmlNodes() after some edges had been added, and the
// user decided to import the resulting partial network, the
// GraphML might contain edges that referenced "this user"
// without containing a vertex for "this user." That is an
// illegal state for GraphML, which the ExcelTemplate project
// caught and reported as an error.
TryAppendVertexXmlNode(sUserName, null, oGraphMLXmlDocument,
oUserNameDictionary);
bThisUserAppended = true;
}
Boolean bNeedToAppendVertices = GetNeedToAppendVertices(
eNetworkLevel, iRecursionLevel);
if (bNeedToAppendVertices)
{
if (
TryAppendVertexXmlNode(sOtherUserName, oEntryXmlNode,
oGraphMLXmlDocument, oUserNameDictionary)
&&
bNeedToRecurse
)
{
oUserNamesToRecurse.Add(sOtherUserName);
}
}
if (bNeedToAppendVertices ||
oUserNameDictionary.ContainsKey(sOtherUserName))
{
String sRelationship;
if (bIncludeFriendsThisCall)
{
sRelationship = "Friend of";
}
else
{
sRelationship = "Subscribes to";
String sSubscriptionType = null;
if (XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"a:category[@scheme='http://gdata.youtube.com/schemas/"
+ "2007/subscriptiontypes.cat']/@term",
oXmlNamespaceManager, out sSubscriptionType))
{
sRelationship += " " + sSubscriptionType;
}
}
NodeXLGraphMLUtil.AppendEdgeXmlNode(oGraphMLXmlDocument,
sUserName, sOtherUserName, sRelationship);
}
}
if (bNeedToRecurse)
{
foreach (String sUserNameToRecurse in oUserNamesToRecurse)
{
GetUserNetworkRecursive(sUserNameToRecurse,
eWhatToInclude, bIncludeFriendsThisCall, eNetworkLevel,
iMaximumPeoplePerRequest, 2, oGraphMLXmlDocument,
oUserNameDictionary, oRequestStatistics);
}
}
}
AppendVertexXmlNodes
(
String sSearchTerm,
WhatToInclude eWhatToInclude,
Int32 iMaximumPeoplePerRequest,
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary <String, TwitterVertex> oScreenNameDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sSearchTerm));
Debug.Assert(iMaximumPeoplePerRequest > 0);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oScreenNameDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
// Convert spaces in the search term to a plus sign.
//
// (Note: Don't try to use Twitter's "show_user" URL parameter in an
// attempt to get a "user" XML node for each author. That's not what
// the URL parameter does.)
String sUrl = String.Format(
"http://search.twitter.com/search.atom?q={0}&rpp=100"
,
UrlUtil.EncodeUrlParameter(sSearchTerm).Replace("%20", "+")
);
ReportProgress("Getting a list of tweets.");
Boolean bIncludeStatistics = WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.Statistics);
Boolean bIncludeStatuses = WhatToIncludeFlagIsSet(
eWhatToInclude, WhatToInclude.Statuses);
// The document consists of an "entry" XML node for each tweet that
// contains the search term. Multiple tweets may have the same author,
// so we have to enumerate until the requested maximum number of unique
// authors is reached.
foreach (XmlNode oAuthorNameXmlNode in EnumerateXmlNodes(
sUrl, "a:feed/a:entry/a:author/a:name", "a", AtomNamespaceUri,
15, Int32.MaxValue, true, false, oRequestStatistics))
{
if (oScreenNameDictionary.Count == iMaximumPeoplePerRequest)
{
break;
}
// The author name is in this format:
//
// james_laker (James Laker)
String sAuthorName;
if (!XmlUtil2.TrySelectSingleNodeAsString(oAuthorNameXmlNode,
"text()", null, out sAuthorName))
{
continue;
}
Int32 iIndexOfSpace = sAuthorName.IndexOf(' ');
String sScreenName = sAuthorName;
if (iIndexOfSpace != -1)
{
sScreenName = sAuthorName.Substring(0, iIndexOfSpace);
}
sScreenName = sScreenName.ToLower();
TwitterVertex oTwitterVertex;
if (!TryAppendVertexXmlNode(sScreenName, null,
oGraphMLXmlDocument, oScreenNameDictionary, bIncludeStatistics,
false, out oTwitterVertex))
{
// A tweet for the author has already been found.
continue;
}
XmlNode oVertexXmlNode = oTwitterVertex.VertexXmlNode;
XmlNode oEntryXmlNode = oAuthorNameXmlNode.ParentNode.ParentNode;
XmlNamespaceManager oXmlNamespaceManager = new XmlNamespaceManager(
oEntryXmlNode.OwnerDocument.NameTable);
oXmlNamespaceManager.AddNamespace("a", AtomNamespaceUri);
// Get the image URL and status for the tweet's author.
AppendStringGraphMLAttributeValue(oEntryXmlNode,
"a:link[@rel='image']/@href", oXmlNamespaceManager,
oGraphMLXmlDocument, oVertexXmlNode, ImageFileID);
String sStatus;
if (XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"a:title/text()", oXmlNamespaceManager, out sStatus))
{
String sStatusDateUtc;
if (XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"a:published/text()", oXmlNamespaceManager,
out sStatusDateUtc))
{
sStatusDateUtc = TwitterDateParser.ParseTwitterDate(
sStatusDateUtc);
}
oTwitterVertex.StatusForAnalysis = sStatus;
oTwitterVertex.StatusForAnalysisDateUtc = sStatusDateUtc;
if (bIncludeStatuses)
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(
oVertexXmlNode, StatusID, sStatus);
if (!String.IsNullOrEmpty(sStatusDateUtc))
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(
oVertexXmlNode, StatusDateUtcID, sStatusDateUtc);
}
}
}
}
}
AppendUserInformationGraphMLAttributeValues
(
String sUserID,
XmlNode oVertexXmlNode,
GraphMLXmlDocument oGraphMLXmlDocument,
String sApiKey,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sUserID));
Debug.Assert(oVertexXmlNode != null);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
Debug.Assert(oRequestStatistics != null);
AssertValid();
String sUrl = GetFlickrMethodUrl("flickr.people.getInfo",
sApiKey, GetUserIDUrlParameter(sUserID));
XmlDocument oXmlDocument;
if (!TryGetXmlDocument(sUrl, oRequestStatistics, out oXmlDocument))
{
// Ignore errors. The user information isn't critical.
return;
}
const String XPathRoot = "rsp/person/";
AppendStringGraphMLAttributeValue(oXmlDocument,
XPathRoot + "realname/text()", null, oGraphMLXmlDocument,
oVertexXmlNode, RealNameID);
AppendInt32GraphMLAttributeValue(oXmlDocument,
XPathRoot + "photos/count/text()", null, oGraphMLXmlDocument,
oVertexXmlNode, TotalPhotosID);
String sIsProfessional;
if (XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
XPathRoot + "@ispro", null, out sIsProfessional))
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(oVertexXmlNode,
IsProfessionalID,
(sIsProfessional == "0") ? "No" : "Yes"
);
}
// Get the URL to the user's buddy icon, which is explained here:
//
// http://www.flickr.com/services/api/misc.buddyicons.html
Int32 iIconServer, iIconFarm;
String sBuddyIconUrl;
if (
XmlUtil2.TrySelectSingleNodeAsInt32(oXmlDocument,
XPathRoot + "@iconserver", null, out iIconServer)
&&
XmlUtil2.TrySelectSingleNodeAsInt32(oXmlDocument,
XPathRoot + "@iconfarm", null, out iIconFarm)
&&
iIconServer > 0
)
{
sBuddyIconUrl = String.Format(
"http://farm{0}.static.flickr.com/{1}/buddyicons/{2}.jpg"
,
iIconFarm,
iIconServer,
sUserID
);
}
else
{
sBuddyIconUrl = "http://www.flickr.com/images/buddyicon.jpg";
}
oGraphMLXmlDocument.AppendGraphMLAttributeValue(
oVertexXmlNode, ImageFileID, sBuddyIconUrl);
if (AppendStringGraphMLAttributeValue(oXmlDocument,
XPathRoot + "photosurl/text()", null, oGraphMLXmlDocument,
oVertexXmlNode, MenuActionID))
{
oGraphMLXmlDocument.AppendGraphMLAttributeValue(oVertexXmlNode,
MenuTextID, "Open Flickr Page for This Person");
}
}
ParseKeyXmlNode
(
XmlNode oKeyXmlNode,
XmlNamespaceManager oXmlNamespaceManager,
String sGraphMLPrefix
)
{
Debug.Assert(oKeyXmlNode != null);
Debug.Assert(oXmlNamespaceManager != null);
Debug.Assert(!String.IsNullOrEmpty(sGraphMLPrefix));
m_sID = XmlUtil2.SelectRequiredSingleNodeAsString(oKeyXmlNode,
"@id", null);
String sFor = XmlUtil2.SelectRequiredSingleNodeAsString(oKeyXmlNode,
"@for", null);
// Note that the @attr.name and @attr.type attributes are optional.
// Default to sensible values if they are missing.
if (!XmlUtil2.TrySelectSingleNodeAsString(oKeyXmlNode,
"@attr.name", null, out m_sName))
{
m_sName = m_sID;
}
String sType;
if (!XmlUtil2.TrySelectSingleNodeAsString(oKeyXmlNode, "@attr.type",
null, out sType))
{
sType = "string";
}
switch (sFor)
{
case "node":
m_bIsForVertex = true;
break;
case "edge":
m_bIsForVertex = false;
break;
// NodeXL doesn't support the "graph" or "all" values allowed by
// the GraphML specification. The caller should filter out such
// "key" XML nodes before using this class.
default:
throw new XmlException(
"The \"for\" attribute on the \"key\" XML node with the"
+ " id \"" + m_sID + "\" must be either \"node\" or"
+ " \"edge\"."
);
}
switch (sType)
{
case "boolean":
m_eType = AttributeType.Boolean;
break;
case "int":
m_eType = AttributeType.Int;
break;
case "long":
m_eType = AttributeType.Long;
break;
case "float":
m_eType = AttributeType.Float;
break;
case "double":
m_eType = AttributeType.Double;
break;
case "string":
m_eType = AttributeType.String;
break;
default:
throw new XmlException(
"The \"attr.type\" attribute on the \"key\" XML node with"
+ " id \"" + m_sID + "\" does not have a valid value."
);
}
m_oDefaultAttributeValue = null;
String sDefaultAttributeValue;
XmlNode oDefaultXmlNode = oKeyXmlNode.SelectSingleNode(
sGraphMLPrefix + ":default", oXmlNamespaceManager);
if (
oDefaultXmlNode != null
&&
XmlUtil2.TrySelectSingleNodeAsString(oDefaultXmlNode, "text()",
null, out sDefaultAttributeValue)
)
{
try
{
m_oDefaultAttributeValue =
ConvertAttributeValue(sDefaultAttributeValue);
}
catch (FormatException)
{
throw new XmlException(
"The default value specified for the \"key\" XML node with"
+ " the id \"" + m_sID + "\" is not of the specified type."
);
}
}
AssertValid();
}
GetUserNetworkRecursive
(
String sUserID,
String sScreenName,
WhatToInclude eWhatToInclude,
Boolean bIncludeContactsThisCall,
NetworkLevel eNetworkLevel,
Int32 iMaximumPerRequest,
String sApiKey,
Int32 iRecursionLevel,
GraphMLXmlDocument oGraphMLXmlDocument,
Dictionary <String, XmlNode> oUserIDDictionary,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(!String.IsNullOrEmpty(sUserID));
Debug.Assert(!String.IsNullOrEmpty(sScreenName));
Debug.Assert(eNetworkLevel == NetworkLevel.One ||
eNetworkLevel == NetworkLevel.OnePointFive ||
eNetworkLevel == NetworkLevel.Two);
Debug.Assert(iMaximumPerRequest > 0);
Debug.Assert(!String.IsNullOrEmpty(sApiKey));
Debug.Assert(iRecursionLevel == 1 || iRecursionLevel == 2);
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oUserIDDictionary != null);
Debug.Assert(oRequestStatistics != null);
AssertValid();
/*
* Here is what this method should do, based on the eNetworkLevel and
* iRecursionLevel parameters.
*
* eNetworkLevel
*
|One | OnePointFive | Two
* ---|------------------| ------------------| -----------------
* i 1 |Add all vertices. | Add all vertices. | Add all vertices.
* R | | |
* e |Add all edges. | Add all edges. | Add all edges.
* c | | |
* u |Do not recurse. | Recurse. | Recurse.
* r | | |
* s ---|------------------|-------------------|------------------
* i 2 |Impossible. | Do not add | Add all vertices.
* o | | vertices. |
* n | | |
* L | | Add edges only if | Add all edges.
* e | | vertices are |
* v | | already included. |
* e | | |
* l | | Do not recurse. | Do not recurse.
| | |
| ---|------------------|-------------------|------------------
*/
Boolean bNeedToRecurse = GetNeedToRecurse(eNetworkLevel,
iRecursionLevel);
Boolean bNeedToAppendVertices = GetNeedToAppendVertices(eNetworkLevel,
iRecursionLevel);
List <String> oUserIDsToRecurse = new List <String>();
ReportProgressForContactsOrCommenters(sScreenName,
bIncludeContactsThisCall);
Boolean bThisUserAppended = false;
foreach (XmlNode oChildXmlNode in GetContactsOrCommentersEnumerator(
sUserID, bIncludeContactsThisCall, iMaximumPerRequest,
oGraphMLXmlDocument, sApiKey, oRequestStatistics))
{
String sOtherScreenName, sOtherUserID;
if (
!XmlUtil2.TrySelectSingleNodeAsString(oChildXmlNode,
bIncludeContactsThisCall ? "@username" : "@authorname",
null, out sOtherScreenName)
||
!XmlUtil2.TrySelectSingleNodeAsString(oChildXmlNode,
bIncludeContactsThisCall ? "@nsid" : "@author",
null, out sOtherUserID)
)
{
continue;
}
if (!bThisUserAppended)
{
// Append a vertex node for this request's user.
//
// This used to be done after the foreach loop, which avoided
// the need for a "bThisUserAppended" flag. That caused the
// following bug: If a YouTube error occurred within
// EnumerateXmlNodes() after some edges had been added, and the
// user decided to import the resulting partial network, the
// GraphML might contain edges that referenced "this user"
// without containing a vertex for "this user." That is an
// illegal state for GraphML, which the ExcelTemplate project
// caught and reported as an error.
TryAppendVertexXmlNode(sUserID, sScreenName,
oGraphMLXmlDocument, oUserIDDictionary);
bThisUserAppended = true;
}
if (bNeedToAppendVertices)
{
if (
TryAppendVertexXmlNode(sOtherUserID, sOtherScreenName,
oGraphMLXmlDocument, oUserIDDictionary)
&&
bNeedToRecurse
)
{
oUserIDsToRecurse.Add(sOtherUserID);
}
}
if (bNeedToAppendVertices ||
oUserIDDictionary.ContainsKey(sOtherUserID))
{
// Append an edge node and optional attributes.
XmlNode oEdgeXmlNode;
if (bIncludeContactsThisCall)
{
oEdgeXmlNode = AppendEdgeXmlNode(oGraphMLXmlDocument,
sScreenName, sOtherScreenName, "Contact");
}
else
{
// (Note the swapping of screen names in the commenter
// case.)
oEdgeXmlNode = AppendEdgeXmlNode(oGraphMLXmlDocument,
sOtherScreenName, sScreenName, "Commenter");
UInt32 uCommentDateUtc;
if (XmlUtil2.TrySelectSingleNodeAsUInt32(oChildXmlNode,
"@datecreate", null, out uCommentDateUtc))
{
DateTime oCommentDateUtc =
DateTimeUtil2.UnixTimestampToDateTimeUtc(
uCommentDateUtc);
oGraphMLXmlDocument.AppendGraphMLAttributeValue(
oEdgeXmlNode, CommentDateUtcID,
ExcelDateTimeUtil.DateTimeToStringLocale1033(
oCommentDateUtc, ExcelColumnFormat.DateAndTime)
);
}
AppendStringGraphMLAttributeValue(oChildXmlNode,
"@permalink", null, oGraphMLXmlDocument, oEdgeXmlNode,
CommentUrlID);
}
}
}
if (bNeedToRecurse)
{
foreach (String sUserIDToRecurse in oUserIDsToRecurse)
{
XmlNode oVertexXmlNode = oUserIDDictionary[sUserIDToRecurse];
String sScreenNameToRecurse = GetScreenNameFromVertexXmlNode(
oVertexXmlNode);
GetUserNetworkRecursive(sUserIDToRecurse, sScreenNameToRecurse,
eWhatToInclude, bIncludeContactsThisCall, eNetworkLevel,
iMaximumPerRequest, sApiKey, 2, oGraphMLXmlDocument,
oUserIDDictionary, oRequestStatistics);
}
}
}
AppendSharedResponderEdges
(
GraphMLXmlDocument oGraphMLXmlDocument,
HashSet <String> oVideoIDs,
Int32 iMaximumResponses,
String sUrlPattern,
String sResponderTitle,
String sKeyAttributeID,
RequestStatistics oRequestStatistics
)
{
Debug.Assert(oGraphMLXmlDocument != null);
Debug.Assert(oVideoIDs != null);
Debug.Assert(iMaximumResponses > 0);
Debug.Assert(!String.IsNullOrEmpty(sUrlPattern));
Debug.Assert(sUrlPattern.IndexOf("{0}") >= 0);
Debug.Assert(!String.IsNullOrEmpty(sResponderTitle));
Debug.Assert(!String.IsNullOrEmpty(sKeyAttributeID));
Debug.Assert(oRequestStatistics != null);
AssertValid();
// The key is the name of an author and the value is a LinkedList of
// the video IDs to which the author has responded.
Dictionary <String, LinkedList <String> > oAuthorUserNameDictionary =
new Dictionary <String, LinkedList <String> >();
foreach (String sVideoID in oVideoIDs)
{
ReportProgress(String.Format(
"Getting {0}s for the video with the ID \"{1}\"."
,
sResponderTitle,
sVideoID
));
// This is to prevent self-loop edges that would result when the
// same author responds to the same video more than once.
HashSet <String> oAuthorUserNames = new HashSet <String>();
String sUrl = String.Format(sUrlPattern, sVideoID);
// The document consists of an "entry" XML node for each response.
foreach (XmlNode oEntryXmlNode in EnumerateXmlNodes(sUrl,
"a:feed/a:entry", iMaximumResponses, true,
oRequestStatistics))
{
XmlNamespaceManager oXmlNamespaceManager =
CreateXmlNamespaceManager(oEntryXmlNode.OwnerDocument);
String sAuthorUserName;
if (
XmlUtil2.TrySelectSingleNodeAsString(oEntryXmlNode,
"a:author/a:name/text()", oXmlNamespaceManager,
out sAuthorUserName)
&&
!oAuthorUserNames.Contains(sAuthorUserName)
)
{
AddVideoIDToDictionary(sAuthorUserName, sVideoID,
oAuthorUserNameDictionary);
oAuthorUserNames.Add(sAuthorUserName);
}
}
}
ReportProgress("Adding edges for shared " + sResponderTitle + "s.");
AppendEdgesFromDictionary(oAuthorUserNameDictionary,
oGraphMLXmlDocument, "Shared " + sResponderTitle, sKeyAttributeID);
}
EnumerateXmlNodes
(
String sUrl,
String sXPath,
Int32 iMaximumXmlNodes,
Boolean bSkipMostPage1Errors,
RequestStatistics oRequestStatistics
)
{
Debug.Assert( !String.IsNullOrEmpty(sUrl) );
Debug.Assert( !String.IsNullOrEmpty(sXPath) );
Debug.Assert(iMaximumXmlNodes > 0);
Debug.Assert(oRequestStatistics != null);
AssertValid();
Int32 iStartIndex = 1;
Int32 iMaxResults = 50;
Int32 iXmlNodesEnumerated = 0;
while (true)
{
if (iStartIndex > 1)
{
ReportProgress("Getting page starting with item "
+ iStartIndex + ".");
}
String sUrlWithPagination = String.Format(
"{0}{1}start-index={2}&max-results={3}"
,
sUrl,
sUrl.IndexOf('?') == -1 ? '?' : '&',
iStartIndex,
iMaxResults
);
XmlDocument oXmlDocument;
XmlNamespaceManager oXmlNamespaceManager;
try
{
oXmlDocument = GetXmlDocument(sUrlWithPagination,
oRequestStatistics, out oXmlNamespaceManager);
}
catch (Exception oException)
{
if (!HttpSocialNetworkUtil.ExceptionIsWebOrXml(oException))
{
throw oException;
}
if (iStartIndex > 1 || bSkipMostPage1Errors)
{
// Always skip errors on page 2 and above.
yield break;
}
throw (oException);
}
XmlNodeList oXmlNodesThisPage = oXmlDocument.SelectNodes(sXPath,
oXmlNamespaceManager);
Int32 iXmlNodesThisPage = oXmlNodesThisPage.Count;
if (iXmlNodesThisPage == 0)
{
yield break;
}
for (Int32 i = 0; i < iXmlNodesThisPage; i++)
{
yield return ( oXmlNodesThisPage[i] );
iXmlNodesEnumerated++;
if (iXmlNodesEnumerated == iMaximumXmlNodes)
{
yield break;
}
}
// The next page, if there is one, is obtained from a link tag that
// looks something like this:
//
// <link rel="next" ... href="http://gdata.youtube.com/feeds/...?
// start-index=26&max-results=25"/>
String sHRef;
if ( !XmlUtil2.TrySelectSingleNodeAsString(oXmlDocument,
"a:feed/a:link[@rel='next']/@href", oXmlNamespaceManager,
out sHRef) )
{
yield break;
}
const String Pattern = @"start-index=(?<NextStartIndex>\d+)";
Regex oRegex = new Regex(Pattern);
Match oMatch = oRegex.Match(sHRef);
if (!oMatch.Success
||
!MathUtil.TryParseCultureInvariantInt32(
oMatch.Groups["NextStartIndex"].Value, out iStartIndex)
)
{
yield break;
}
// Get the next page...
}
}
