/// <summary>
/// Authenticate to Google Using Oauth2
/// Documentation https://developers.google.com/accounts/docs/OAuth2
/// </summary>
/// <param name="clientId">From Google Developer console https://console.developers.google.com</param>
/// <param name="clientSecret">From Google Developer console https://console.developers.google.com</param>
/// <param name="userName">A string used to identify a user.</param>
/// <returns></returns>
public static WebmastersService AuthenticateOauth(string clientId, string clientSecret, string userName)
{
string[] scopes = new string[] { WebmastersService.Scope.WebmastersReadonly }; // View analytics data
try
{
// here is where we Request the user to give us access, or use the Refresh Token that was previously stored in %AppData%
UserCredential credential = GoogleWebAuthorizationBroker.AuthorizeAsync(new ClientSecrets {
ClientId = clientId, ClientSecret = clientSecret
}
, scopes
, userName
, CancellationToken.None
, new FileDataStore("Daimto.GoogleWebMasters.Auth.Store")).Result;
WebmastersService service = new WebmastersService(new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = "WebMasters API Sample",
});
return(service);
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return(null);
}
}
/// <summary>
/// Authenticating to Google using a Service account
/// Documentation: https://developers.google.com/accounts/docs/OAuth2#serviceaccount
/// </summary>
/// <param name="serviceAccountEmail">From Google Developer console https://console.developers.google.com</param>
/// <param name="keyFilePath">Location of the Service account key file downloaded from Google Developer console https://console.developers.google.com</param>
/// <returns></returns>
public static WebmastersService AuthenticateServiceAccount(string serviceAccountEmail, string keyFilePath)
{
// check the file exists
if (!File.Exists(keyFilePath))
{
Console.WriteLine("An Error occurred - Key file does not exist");
return(null);
}
string[] scopes = new string[] { WebmastersService.Scope.Webmasters }; // View WebMasters data
var certificate = new X509Certificate2(keyFilePath, "notasecret", X509KeyStorageFlags.Exportable);
try
{
ServiceAccountCredential credential = new ServiceAccountCredential(
new ServiceAccountCredential.Initializer(serviceAccountEmail)
{
Scopes = scopes
}.FromCertificate(certificate));
// Create the service.
WebmastersService service = new WebmastersService(new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = "WebMasters API Sample",
});
return(service);
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return(null);
}
}
/// <summary>
/// Query your data with filters and parameters that you define. Returns zero or more rows grouped by the row keys that you define. You must define a date range of one or more days.When date is one of the group by values, any days without data are omitted from the result list. If you need to know which days have data, issue a broad date range query grouped by date for any metric, and see which day rows are returned.
/// Documentation https://developers.google.com/webmasters/v3/reference/searchanalytics/query
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Webmasters service.</param>
/// <param name="siteUrl">The site's URL, including protocol. For example: http://www.example.com/</param>
/// <param name="body">A valid Webmasters v3 body.</param>
/// <returns>SearchAnalyticsQueryResponseResponse</returns>
public static SearchAnalyticsQueryResponse Query(WebmastersService service, string siteUrl, SearchAnalyticsQueryRequest body)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (body == null)
{
throw new ArgumentNullException("body");
}
if (siteUrl == null)
{
throw new ArgumentNullException(siteUrl);
}
// Make the request.
return(service.Searchanalytics.Query(body, siteUrl).Execute());
}
catch (Exception ex)
{
throw new Exception("Request Searchanalytics.Query failed.", ex);
}
}
/// <summary>
/// Retrieves information about a specific sitemap.
/// Documentation https://developers.google.com/webmasters/v3/reference/sitemaps/get
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Webmasters service.</param>
/// <param name="siteUrl">The site's URL, including protocol. For example: http://www.example.com/</param>
/// <param name="feedpath">The URL of the actual sitemap. For example: http://www.example.com/sitemap.xml</param>
/// <returns>WmxSitemapResponse</returns>
public static WmxSitemap Get(WebmastersService service, string siteUrl, string feedpath)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (siteUrl == null)
{
throw new ArgumentNullException(siteUrl);
}
if (feedpath == null)
{
throw new ArgumentNullException(feedpath);
}
// Make the request.
return(service.Sitemaps.Get(siteUrl, feedpath).Execute());
}
catch (Exception ex)
{
throw new Exception("Request Sitemaps.Get failed.", ex);
}
}
/// <summary>
/// Retrieves details about crawl errors for a site's sample URL.
/// Documentation https://developers.google.com/webmasters/v3/reference/urlcrawlerrorssamples/get
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Webmasters service.</param>
/// <param name="siteUrl">The site's URL, including protocol. For example: http://www.example.com/</param>
/// <param name="url">The relative path (without the site) of the sample URL. It must be one of the URLs returned by list(). For example, for the URL https://www.example.com/pagename on the site https://www.example.com/, the url value is pagename</param>
/// <param name="category">The crawl error category. For example: authPermissions</param>
/// <param name="platform">The user agent type (platform) that made the request. For example: web</param>
/// <returns>UrlCrawlErrorsSampleResponse</returns>
public static UrlCrawlErrorsSample Get(WebmastersService service, string siteUrl, string url, string category, string platform)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (siteUrl == null)
{
throw new ArgumentNullException(siteUrl);
}
if (url == null)
{
throw new ArgumentNullException(url);
}
if (category == null)
{
throw new ArgumentNullException(category);
}
if (platform == null)
{
throw new ArgumentNullException(platform);
}
// Make the request.
return(service.Urlcrawlerrorssamples.Get(siteUrl, url, category, platform).Execute());
}
catch (Exception ex)
{
throw new Exception("Request Urlcrawlerrorssamples.Get failed.", ex);
}
}
/// <summary>
/// Retrieves a time series of the number of URL crawl errors per error category and platform.
/// Documentation https://developers.google.com/webmasters/v3/reference/urlcrawlerrorscounts/query
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Webmasters service.</param>
/// <param name="siteUrl">The site's URL, including protocol. For example: http://www.example.com/</param>
/// <param name="optional">Optional paramaters.</param>
/// <returns>UrlCrawlErrorsCountsQueryResponseResponse</returns>
public static UrlCrawlErrorsCountsQueryResponse Query(WebmastersService service, string siteUrl, UrlcrawlerrorscountsQueryOptionalParms optional = null)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (siteUrl == null)
{
throw new ArgumentNullException(siteUrl);
}
// Building the initial request.
var request = service.Urlcrawlerrorscounts.Query(siteUrl);
// Applying optional parameters to the request.
request = (UrlcrawlerrorscountsResource.QueryRequest)SampleHelpers.ApplyOptionalParms(request, optional);
// Requesting data.
return(request.Execute());
}
catch (Exception ex)
{
throw new Exception("Request Urlcrawlerrorscounts.Query failed.", ex);
}
}
/// <summary>
/// Authenticate to Google Using Oauth2
/// Documentation https://developers.google.com/accounts/docs/OAuth2
/// </summary>
/// <param name="clientId">From Google Developer console https://console.developers.google.com</param>
/// <param name="clientSecret">From Google Developer console https://console.developers.google.com</param>
/// <param name="userName">A string used to identify a user.</param>
/// <returns></returns>
public static WebmastersService AuthenticateOauth(string clientId, string clientSecret, string userName)
{
string[] scopes = new string[] { WebmastersService.Scope.WebmastersReadonly}; // View analytics data
try
{
// here is where we Request the user to give us access, or use the Refresh Token that was previously stored in %AppData%
UserCredential credential = GoogleWebAuthorizationBroker.AuthorizeAsync(new ClientSecrets { ClientId = clientId, ClientSecret = clientSecret }
, scopes
, userName
, CancellationToken.None
, new FileDataStore("Daimto.GoogleWebMasters.Auth.Store")).Result;
WebmastersService service = new WebmastersService(new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = "WebMasters API Sample",
});
return service;
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return null;
}
}
/// <summary>
/// Lists the sitemaps-entries submitted for this site, or included in the sitemap index file (if sitemapIndex is specified in the request).
/// Documentation https://developers.google.com/webmasters/v3/reference/sitemaps/list
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Webmasters service.</param>
/// <param name="siteUrl">The site's URL, including protocol. For example: http://www.example.com/</param>
/// <param name="optional">Optional paramaters.</param>
/// <returns>SitemapsListResponseResponse</returns>
public static SitemapsListResponse List(WebmastersService service, string siteUrl, SitemapsListOptionalParms optional = null)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (siteUrl == null)
{
throw new ArgumentNullException(siteUrl);
}
// Building the initial request.
var request = service.Sitemaps.List(siteUrl);
// Applying optional parameters to the request.
request = (SitemapsResource.ListRequest)SampleHelpers.ApplyOptionalParms(request, optional);
// Requesting data.
return(request.Execute());
}
catch (Exception ex)
{
throw new Exception("Request Sitemaps.List failed.", ex);
}
}
/// <summary>
/// Submits a sitemap for a site.
/// Documentation: https://developers.google.com/webmaster-tools/v3/sitemaps/submit
/// </summary>
/// <param name="service"></param>
/// <param name="site"> The site's URL, including protocol. For example: http://www.example.com/ </param>
/// <param name="feedPath"> The URL of the actual sitemap. For example: http://www.example.com/sitemap.xml </param>
/// <returns>If successful, this method returns an empty response body. </returns>
public static string add(WebmastersService service, string site, string feedPath)
{
try
{
return(service.Sitemaps.Submit(site, feedPath).Execute());
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return(null);
}
}
/// <summary>
/// Adds a site to the set of the user's sites in Webmaster Tools.
/// Documentation: https://developers.google.com/webmaster-tools/v3/sites/add
/// </summary>
/// <param name="service"></param>
/// <param name="site"> The site's URL, including protocol. For example: http://www.example.com/ </param>
/// <returns>If successful, this method returns an empty response body. </returns>
public static string delete(WebmastersService service, string site)
{
try
{
return(service.Sites.Delete(site).Execute());
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return(null);
}
}
/// <summary>
/// Lists the user's Webmaster Tools sites.
/// Documentation: https://developers.google.com/webmaster-tools/v3/sites/list
/// </summary>
/// <param name="service"></param>
/// <returns></returns>
public static SitesListResponse list(WebmastersService service)
{
try
{
var x = service.Sites.List().Execute();
return(x);
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return(null);
}
}
/// <summary>
///Retrieves a time series of the number of URL crawl errors per error category and platform.
/// Documentation: https://developers.google.com/webmaster-tools/v3/urlcrawlerrorscounts/query
/// </summary>
/// <param name="service"></param>
/// <param name="site"> The site's URL, including protocol. For example: http://www.example.com/ </param>
/// <returns> </returns>
public static UrlCrawlErrorsCountsQueryResponse query(WebmastersService service, string site)
{
try
{
var x = service.Urlcrawlerrorscounts.Query(site).Execute();
return(x);
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return(null);
}
}
/// <summary>
/// Deletes a sitemap from this site
/// Documentation: https://developers.google.com/webmaster-tools/v3/sitemaps/delete
/// </summary>
/// <param name="service"></param>
/// <param name="site"> The site's URL, including protocol. For example: http://www.example.com/ </param>
/// <param name="feedPath"> The URL of the actual sitemap. For example: http://www.example.com/sitemap.xml </param>
/// <returns>If successful, this method returns an empty response body. </returns>
public static string delete(WebmastersService service, string site, string feedPath)
{
try
{
return service.Sitemaps.Delete(site,feedPath).Execute();
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return null;
}
}
/// <summary>
/// Removes a site from the set of the user's Webmaster Tools sites.
/// Documentation: https://developers.google.com/webmaster-tools/v3/sites/delete
/// </summary>
/// <param name="service"></param>
/// <param name="site"> The site's URL, including protocol. For example: http://www.example.com/ </param>
/// <returns>If successful, this method returns an empty response body. </returns>
public static string add(WebmastersService service, string site)
{
try
{
return service.Sites.Add(site).Execute();
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return null;
}
}
/// <summary>
/// Creates new UserCredential and WebmastersService
/// </summary>
private async Task InitGoogleServices()
{
string[] scopes = new string[] { WebmastersService.Scope.WebmastersReadonly };
Credential = GoogleWebAuthorizationBroker.AuthorizeAsync(new ClientSecrets
{
ClientId = Json.ClientId,
ClientSecret = Json.ClientSecret
}, scopes, UserName, CancellationToken.None, new FileDataStore(DataStoreName)).Result;
Service = new WebmastersService(new BaseClientService.Initializer()
{
HttpClientInitializer = Credential,
ApplicationName = AppName
});
}
/// <summary>
/// Lists the sitemaps-entries submitted for this site, or included in the sitemap index file (if sitemapIndex is specified in the request).
/// Documentation: https://developers.google.com/webmaster-tools/v3/sitemaps/list
/// </summary>
/// <param name="service"></param>
/// <param name="site"> The site's URL, including protocol. For example: http://www.example.com/ </param>
/// <returns></returns>
public static SitemapsListResponse list(WebmastersService service, string site)
{
try
{
var x = service.Sitemaps.List(site).Execute();
return x;
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return null;
}
}
/// <summary>
///Retrieves a time series of the number of URL crawl errors per error category and platform.
/// Documentation: https://developers.google.com/webmaster-tools/v3/urlcrawlerrorscounts/query
/// </summary>
/// <param name="service"></param>
/// <param name="site"> The site's URL, including protocol. For example: http://www.example.com/ </param>
/// <returns> </returns>
public static UrlCrawlErrorsCountsQueryResponse query(WebmastersService service, string site)
{
try
{
var x = service.Urlcrawlerrorscounts.Query(site).Execute();
return x;
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return null;
}
}
/// <summary>
/// Lists the user's Search Console sites.
/// Documentation https://developers.google.com/webmasters/v3/reference/sites/list
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Webmasters service.</param>
/// <returns>SitesListResponseResponse</returns>
public static SitesListResponse List(WebmastersService service)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
// Make the request.
return(service.Sites.List().Execute());
}
catch (Exception ex)
{
throw new Exception("Request Sites.List failed.", ex);
}
}
/// <summary>
/// Removes a site from the set of the user's Search Console sites.
/// Documentation https://developers.google.com/webmasters/v3/reference/sites/delete
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Webmasters service.</param>
/// <param name="siteUrl">The URI of the property as defined in Search Console. Examples: http://www.example.com/ or android-app://com.example/</param>
public static void Delete(WebmastersService service, string siteUrl)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (siteUrl == null)
{
throw new ArgumentNullException(siteUrl);
}
// Make the request.
service.Sites.Delete(siteUrl).Execute();
}
catch (Exception ex)
{
throw new Exception("Request Sites.Delete failed.", ex);
}
}
static void Main(string[] args)
{
string clientId = ConfigurationManager.AppSettings.Get("clientId");
string clientSecret = ConfigurationManager.AppSettings.Get("clientSecret");
string userName = ConfigurationManager.AppSettings.Get("userName");
SearchConsoleContext db = new SearchConsoleContext();
WebmastersService service = AuthenticateOauth(clientId, clientSecret, userName);
if (db != null && service != null)
{
foreach (Domain domain in db.Domains.ToList())
{
if (domain.Id == 2)
{
continue;
}
IList <string> dimensions = new List <string>();
db.FilterTypes.ToList().ForEach(ft => dimensions.Add(ft.Type));
//SearchConsoleDataKey dataKey = null;
SearchConsoleDataSet dataKey = null;
FilterType dateFilterType = db.FilterTypes.FirstOrDefault(ft => ft.Type == "date");
if (dateFilterType != null && dateFilterType.Id != 0)
{
//dataKey = db.SearchConsoleDataKeys.Where(d => d.FilterTypeId == dateFilterType.Id).OrderByDescending(k => k.Key).FirstOrDefault();
dataKey = db.SearchConsoleDataSets.Where(d => d.DomainId == domain.Id).OrderByDescending(k => k.Date).FirstOrDefault();
}
SearchAnalyticsQueryRequest request = new SearchAnalyticsQueryRequest();
request.RowLimit = 5000;
request.Dimensions = dimensions;
request.AggregationType = "bypage";
DateTime startDate = DateTime.Now.AddDays(-78).Date;
DateTime endDate = startDate;
if (dataKey != null)
{
//startDate = DateTime.Parse(dataKey.Key).AddDays(1);
startDate = dataKey.Date.AddDays(1);
endDate = startDate;
}
//while (startDate < endDate)
//{
request.StartDate = startDate.ToString("yyyy-MM-dd");
request.EndDate = endDate.ToString("yyyy-MM-dd");
var result = service.Searchanalytics.Query(request, domain.Name).Execute();
if (result != null &&
result.Rows != null &&
result.Rows.Count > 0)
{
for (int i = 0; i < result.Rows.Count; i++)
{
SearchConsoleData data = new SearchConsoleData();
data.Clicks = result.Rows[i].Clicks;
data.CTR = result.Rows[i].Ctr;
data.Impressions = result.Rows[i].Impressions;
data.Position = result.Rows[i].Position;
data.DomainId = domain.Id;
data.SearchConsoleDataKeys = new List <SearchConsoleDataKey>();
SearchConsoleDataSet dataInOneRow = new SearchConsoleDataSet();
dataInOneRow.Clicks = result.Rows[i].Clicks;
dataInOneRow.CTR = result.Rows[i].Ctr;
dataInOneRow.Impressions = result.Rows[i].Impressions;
dataInOneRow.Position = result.Rows[i].Position;
dataInOneRow.DomainId = domain.Id;
dataInOneRow.Query = result.Rows[i].Keys[0];
dataInOneRow.Page = result.Rows[i].Keys[1];
dataInOneRow.country = result.Rows[i].Keys[2];
string device = result.Rows[i].Keys[3];
DeviceType deviceType = db.DeviceTypes.FirstOrDefault(dt => dt.Type == device);
dataInOneRow.DeviceType = deviceType.Id;
dataInOneRow.Date = DateTime.Parse(result.Rows[i].Keys[4]);
//NOTE: Keys are arranged according to how the dimensions was arranged in the request.
for (int key = 0; key < dimensions.Count; key++)
{
string filter = dimensions[key];
FilterType filterType = db.FilterTypes.FirstOrDefault(ft => ft.Type == filter);
data.SearchConsoleDataKeys.Add(
new SearchConsoleDataKey
{
FilterTypeId = filterType.Id,
Key = result.Rows[i].Keys[key]
});
}
db.SearchConsoleDataCollection.Add(data);
db.SearchConsoleDataSets.Add(dataInOneRow);
db.SaveChanges();
}
}
// startDate = startDate.AddDays(1);
//}
}
}
}
/// <summary>
/// Authenticating to Google using a Service account
/// Documentation: https://developers.google.com/accounts/docs/OAuth2#serviceaccount
/// </summary>
/// <param name="serviceAccountEmail">From Google Developer console https://console.developers.google.com</param>
/// <param name="keyFilePath">Location of the Service account key file downloaded from Google Developer console https://console.developers.google.com</param>
/// <returns></returns>
public static WebmastersService AuthenticateServiceAccount(string serviceAccountEmail, string keyFilePath)
{
// check the file exists
if (!File.Exists(keyFilePath))
{
Console.WriteLine("An Error occurred - Key file does not exist");
return null;
}
string[] scopes = new string[] { WebmastersService.Scope.Webmasters}; // View WebMasters data
var certificate = new X509Certificate2(keyFilePath, "notasecret", X509KeyStorageFlags.Exportable);
try
{
ServiceAccountCredential credential = new ServiceAccountCredential(
new ServiceAccountCredential.Initializer(serviceAccountEmail)
{
Scopes = scopes
}.FromCertificate(certificate));
// Create the service.
WebmastersService service = new WebmastersService(new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = "WebMasters API Sample",
});
return service;
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
return null;
}
}
