private void ClearCrawler()
{
// Retrieve storage account from connection string
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the queue and table client
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
// Retrieve a reference to CrawlingQueue and CrawlingTable
CloudQueue crawlingQueue = queueClient.GetQueueReference(WebRole.AZURE_CRAWLING_QUEUE);
CloudTable crawlingTable = tableClient.GetTableReference(WebRole.AZURE_CRAWLING_TABLE);
// Create the queue and table if it doesn't already exist
crawlingQueue.CreateIfNotExists();
crawlingTable.CreateIfNotExists();
crawledUrls.Clear();
crawlingQueue.Clear();
disallowedDirs.Clear();
crawlingTable.Delete();
crawlingTable.CreateIfNotExists();
statsEntity = new StatsEntity();
}
public void deleteTable()
{
getReference g = new getReference();
CloudTable table = g.getTable();
table.Delete();
}
public StorageAccountTable()
{
// Sample run
CreateAccount();
CloudTable table = GetTable($"customers{DateTime.Now.ToLongTimeString().Replace(":", string.Empty)}");
CreateCustomer(table, new CustomerUK("David", "*****@*****.**"));
CreateCustomer(table, new CustomerUK("Davidco", "*****@*****.**"));
PrintAllCustomers(table);
CustomerUK customerToUpdate = GetCustomer(table, "UK", "*****@*****.**");
customerToUpdate.Name = "Dave";
UpdateCustomer(table, customerToUpdate);
PrintAllCustomers(table);
DeleteCustomer(table, customerToUpdate);
PrintAllCustomers(table);
BatchInsert(table);
PrintAllCustomers(table);
table.Delete();
}
public void TableSasInvalidOperations()
{
CloudTableClient tableClient = GenerateCloudTableClient();
CloudTable table = tableClient.GetTableReference("T" + Guid.NewGuid().ToString("N"));
try
{
table.Create();
// Prepare SAS authentication with full permissions
string sasString = table.GetSharedAccessSignature(
new SharedAccessTablePolicy
{
Permissions = SharedAccessTablePermissions.Delete,
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddMinutes(30)
},
null,
null,
null,
null,
null);
CloudTableClient sasClient = new CloudTableClient(tableClient.BaseUri, new StorageCredentials(sasString));
// Construct a valid set of service properties to upload.
ServiceProperties properties = new ServiceProperties();
properties.Logging.Version = "1.0";
properties.Metrics.Version = "1.0";
properties.Logging.RetentionDays = 9;
sasClient.GetServiceProperties();
sasClient.SetServiceProperties(properties);
// Test invalid client operations
// BUGBUG: ListTables hides the exception. We should fix this
// TestHelpers.ExpectedException(() => sasClient.ListTablesSegmented(), "List tables with SAS", HttpStatusCode.NotFound);
TestHelper.ExpectedException(() => sasClient.GetServiceProperties(), "Get service properties with SAS", HttpStatusCode.NotFound);
TestHelper.ExpectedException(() => sasClient.SetServiceProperties(properties), "Set service properties with SAS", HttpStatusCode.NotFound);
CloudTable sasTable = sasClient.GetTableReference(table.Name);
// Verify that creation fails with SAS
TestHelper.ExpectedException(() => sasTable.Create(), "Create a table with SAS", HttpStatusCode.NotFound);
// Create the table.
table.Create();
// Test invalid table operations
TestHelper.ExpectedException(() => sasTable.Delete(), "Delete a table with SAS", HttpStatusCode.NotFound);
TestHelper.ExpectedException(() => sasTable.GetPermissions(), "Get ACL with SAS", HttpStatusCode.NotFound);
TestHelper.ExpectedException(() => sasTable.SetPermissions(new TablePermissions()), "Set ACL with SAS", HttpStatusCode.NotFound);
}
finally
{
table.DeleteIfExists();
}
}
static CustomerManager()
{
storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]);
tableClient = storageAccount.CreateCloudTableClient();
customersTable = tableClient.GetTableReference("Customers");
if (customersTable.Exists())
{
customersTable.Delete();
}
customersTable.DeleteIfExists();
customersTable.Create();
}
/* public string AddToTable(Entity urlEntity)
* {
* CloudTable table = tableClient.GetTableReference("hw3table");
*
* TableOperation insertOperation = TableOperation.Insert(urlEntity);
* var result = table.Execute(insertOperation);
* if (result != null)
* {
* return "Success";
* }
* else
* {
* return "Failed";
* }
* }
*/
public void DeleteAll()
{
CloudQueue queue = queueClient.GetQueueReference("hw3urlqueue");
queue.Delete();
queue = queueClient.GetQueueReference("hw3commandqueue");
queue.Delete();
CloudTable table = tableClient.GetTableReference("hw3table");
table.Delete();
}
public static void go()
{
const String partitionName = "Samples_Partition_1";
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.ConnectionStrings["AzureWebJobsStorage"].ToString());
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("Samples");
// If table doesn't already exist in storage then create and populate it with some initial values, otherwise do nothing
if (!table.Exists())
{
// Create table if it doesn't exist already
table.CreateIfNotExists();
// Create the batch operation.
TableBatchOperation batchOperation = new TableBatchOperation();
// Create a sample entity and add it to the table.
SampleEntity sample1 = new SampleEntity(partitionName, "1");
sample1.Title = "randomTitle";
sample1.CreatedDate = null;
sample1.SampleDate = null;
SampleEntity sample2 = new SampleEntity(partitionName, "2");
sample2.Title = "randomTitle2";
sample2.CreatedDate = null;
sample2.SampleDate = null;
// Add sample entities to the batch insert operation.
batchOperation.Insert(sample1);
batchOperation.Insert(sample2);
// Execute the batch operation.
try
{
table.ExecuteBatch(batchOperation);
}
catch (StorageException e)
{
Debug.WriteLine(e.RequestInformation.ExtendedErrorInformation.ErrorMessage);
table.Delete();
}
}
}
public void stopCrawl()
{
getReference g = new getReference();
CloudQueue queue = g.commandQueue();
queue.CreateIfNotExists();
CloudQueueMessage message = new CloudQueueMessage("stop");
queue.AddMessage(message);
CloudQueue storage = g.getQueue();
storage.Clear();
CloudTable table = g.getTable();
table.Delete();
}
public TableStorageProvider(string tableName, bool deleteOldData)
{
account = CloudStorageAccount.Parse("UseDevelopmentStorage=true");
CloudTableClient tableClient = new CloudTableClient(new Uri(account.TableEndpoint.AbsoluteUri), account.Credentials);
table = tableClient.GetTableReference(tableName);
if (deleteOldData)
{
if (table.Exists())
{
table.Delete();
}
}
table.CreateIfNotExists();
}
static void Main(string[] args)
{
string ConnectionString =
@"DefaultEndpointsProtocol=https;AccountName=table-api-audit;AccountKey=RL2GOiD6lvPLmrHmkmtgPNG9211G1jFd7wyfGEXlyNb3GJHJaON5MIIoWWwzGobyRCsPy3BaYauwn2AVfRo6Lw==;TableEndpoint=https://table-api-audit.table.cosmosdb.azure.com:443/;";
try
{
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(ConnectionString);
CloudTableClient tableClient = cloudStorageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("Audit");
// delete table if it exists
table.Delete();
table.CreateIfNotExists();
List <AuditEvent> auditEvents = AuditEventFactory.GeneratAuditEvents(20, 20, 10);
TableBatchOperation tableBatchOperation = new TableBatchOperation();
auditEvents.ForEach(a => tableBatchOperation.Insert(a));
table.ExecuteBatch(tableBatchOperation);
// read all records
TableQuery <AuditEvent> query = new TableQuery <AuditEvent>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "BOSHost"));
var results = table.ExecuteQuery(query);
foreach (AuditEvent audit in results)
{
Console.WriteLine(audit.ToString());
}
Console.WriteLine("Table API Finished!");
Console.ReadLine();
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public static void deleteTables()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("htmltable");
if (table.Exists())
{
table.Delete();
}
CloudTable table2 = tableClient.GetTableReference("exceptiontable");
if (table2.Exists())
{
table2.Delete();
}
}
public string ClearTables()
{
clearingEverything = true;
//
statsTable.CreateIfNotExists();
urlTable.CreateIfNotExists();
// Delete all tables
statsTable.Delete();
urlTable.Delete();
// Reset statistics to the default (When it was first made)
state = "idling";
tableTotal = "0";
urlTotal = "0";
lastTenStatistic = "";
// Loop to ensure that the tables are created correctly
while (true)
{
try
{
statsTable.CreateIfNotExists();
urlTable.CreateIfNotExists();
break;
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode == 409)
{
Thread.Sleep(1000);
}
else
{
throw;
}
}
}
clearingEverything = false;
return("The tables have been cleared");
}
private void InitializeCrawlingTable()
{
// Retrieve storage account from connection string
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the table client
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
// Retrieve a reference to CrawlingTable
CloudTable crawlingTable = tableClient.GetTableReference(WebRole.AZURE_CRAWLING_TABLE);
// Create the table if it doesn't exist
crawlingTable.CreateIfNotExists();
// Clear the table
crawlingTable.Delete();
crawlingTable.Create();
}
public string clearTables()
{
stop();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable resultsTable = tableClient.GetTableReference("crawltable");
CloudTable statsTable = tableClient.GetTableReference("stattable");
CloudTable errorTable = tableClient.GetTableReference("errortable");
try
{
resultsTable.Delete();
statsTable.Delete();
errorTable.Delete();
}
catch (Exception e)
{
return("Tables already deleted: " + e.Message);
}
return("Crawl stopped, Clearing tables...");
}
public AzureStorage(string tableName, bool deleteOldData)
{
account = CloudStorageAccount.Parse("UseDevelopmentStorage=true");
CloudTableClient tableClient = new CloudTableClient(new Uri(account.TableEndpoint.AbsoluteUri), account.Credentials);
table = tableClient.GetTableReference(tableName);
CloudBlobClient blobClient = account.CreateCloudBlobClient();
blob = blobClient.GetContainerReference(tableName.ToLower());
if (deleteOldData)
{
if (table.Exists())
{
table.Delete();
}
if (blob.Exists())
{
blob.Delete();
}
}
table.CreateIfNotExists();
blob.CreateIfNotExists();
}
public void Delete()
{
table.Delete();
}
public void DeletePageTable()
{
pageTable.Delete();
}
public void Dispose()
{
_table.Delete();
}
public void Delete(TableRequestOptions requestOptions = null, OperationContext operationContext = null)
{
_cloudTable.Delete(requestOptions, operationContext);
}
public void ClearTable()
{
_table.Delete();
_table.Create();
}
/// <summary>
/// Method deletes multiple specified tables
/// The tables to be deleted are specified using the following extended properties
/// Extended Properties
/// tablesToDelete - Name of the tables (comma separated) to be deleted.
/// At least one table needs to be specified.
/// Extended Properties Example
/// "tablesToDelete": "Following",
/// Activity Operation
/// The activity iterates through all the tables from the tablesToDelete extended property,
/// checks for the table and deletes it if found.
/// </summary>
/// <param name="linkedServices">Linked services referenced by activity definition.</param>
/// <param name="datasets">Datasets referenced by activity definition.</param>
/// <param name="activity">Activity definition.</param>
/// <param name="logger">Used to log messages during activity execution.</param>
/// <returns>Activity state at the end of execution</returns>
public IDictionary <string, string> Execute(
IEnumerable <LinkedService> linkedServices,
IEnumerable <Dataset> datasets,
Activity activity,
IActivityLogger logger)
{
DotNetActivity dotNetActivity = (DotNetActivity)activity.TypeProperties;
IDictionary <string, string> extendedProperties = dotNetActivity.ExtendedProperties;
logger.Write("Logging extended properties if any...");
foreach (KeyValuePair <string, string> entry in extendedProperties)
{
logger.Write("<key:{0}> <value:{1}>", entry.Key, entry.Value);
}
string[] tablesToDelete = null;
if (extendedProperties.ContainsKey("tablesToDelete"))
{
tablesToDelete = extendedProperties["tablesToDelete"].Split(',');
}
if (tablesToDelete == null || tablesToDelete.Length <= 0)
{
logger.Write("No tables to delete");
return(new Dictionary <string, string>());
}
AzureStorageLinkedService inputLinkedService;
AzureTableDataset sourceTable;
// Use the input dataset to get the storage connection string
// For activities working on a single dataset, the first entry is the input dataset.
// The activity.Inputs can have multiple datasets for building pipeline workflow dependencies. We can ignore the rest of the datasets
Dataset inputDataset = datasets.Single(dataset => dataset.Name == activity.Inputs.First().Name);
sourceTable = inputDataset.Properties.TypeProperties as AzureTableDataset;
logger.Write("input table:{0}", sourceTable.TableName);
inputLinkedService = linkedServices.First(
ls =>
ls.Name ==
inputDataset.Properties.LinkedServiceName).Properties.TypeProperties
as AzureStorageLinkedService;
string inputConnectionString = inputLinkedService.ConnectionString;
// create storage client for input. Pass the connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(inputConnectionString);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
foreach (string tableName in tablesToDelete)
{
CloudTable table = tableClient.GetTableReference(tableName);
if (!table.Exists())
{
logger.Write("Table {0} does not exist.", tableName);
}
else
{
table.Delete();
logger.Write("Table {0} deleted.", tableName);
}
}
return(new Dictionary <string, string>());
}
public ActionResult DeleteEntity(string id, string partKey = PartitionKey)
{
try
{
var entities = new CloudTable<Analysis>(Providers.TableStorage, TableName);
entities.Delete(partKey, id);
return RedirectToAction("Index");
}
catch
{
return View();
}
}
public async Task DeleteAsync <T>(string key)
{
await _table.Delete(key, key);
}
/// <summary>
/// Delete the specified azure storage table
/// </summary>
/// <param name="table">Cloud table object</param>
/// <param name="requestOptions">Table request options</param>
/// <param name="operationContext">Operation context</param>
public void Delete(CloudTable table, TableRequestOptions requestOptions, OperationContext operationContext)
{
table.Delete(requestOptions, operationContext);
}
public static void DeleteAndWaitForCompletion(this CloudTable table)
{
RepeatUntilNotFound(() => table.Delete());
}
public override void Run()
{
while (true)
{
Thread.Sleep(500);
Trace.TraceInformation("Working");
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue queue = queueClient.GetQueueReference("urls");
queue.CreateIfNotExists();
CloudQueue rootQueue = queueClient.GetQueueReference("rooturls");
rootQueue.CreateIfNotExists();
CloudQueue commandQueue = queueClient.GetQueueReference("commands");
commandQueue.CreateIfNotExists();
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("results");
table.CreateIfNotExists();
CloudTable outputTable = tableClient.GetTableReference("output");
outputTable.CreateIfNotExists();
CloudQueueMessage currentUrl = queue.GetMessage();
CloudQueueMessage command = commandQueue.GetMessage();
CloudQueueMessage baseSite = rootQueue.GetMessage();
if (command != null)
{
if (command.AsString == "update")
{
OutputEntity update = new OutputEntity(status, urlCrawled, queueCount, tableCount, lastTen);
TableOperation insertOp = TableOperation.InsertOrMerge(update);
outputTable.Execute(insertOp);
}
else if (command.AsString == "stop")
{
queue.Clear();
rootQueue.Clear();
commandQueue.Clear();
table.Delete();
visitedLinks.Clear();
disallowList.Clear();
siteMaps.Clear();
lastTen.Clear();
queueCount = 0;
tableCount = 0;
urlCrawled = 0;
currentUrl = null;
baseSite = null;
}
}
if (baseSite != null)
{
if (baseSite.AsString.EndsWith("robots.txt"))
{
string rootUrl = baseSite.AsString;
WebRequest getDomain = WebRequest.Create(rootUrl);
Stream domainStream = getDomain.GetResponse().GetResponseStream();
StreamReader robots = new StreamReader(domainStream);
String line;
while ((line = robots.ReadLine()) != null)
{
if (line.StartsWith("Disallow:"))
{
disallowList.Add(line.Replace("Disallow: ", ""));
}
else if (line.StartsWith("Sitemap:"))
{
siteMaps.Add(line.Replace("Sitemap: ", ""));
}
}
foreach (string map in siteMaps)
{
WebRequest crawlMap = WebRequest.Create(map);
Stream urlStream = crawlMap.GetResponse().GetResponseStream();
StreamReader readUrl = new StreamReader(urlStream);
XmlDocument urlContent = new XmlDocument();
urlContent.Load(readUrl);
XmlNodeList nodes = urlContent.DocumentElement.ChildNodes;
foreach (XmlNode node in nodes)
{
Boolean valid = true;
if (node.Name == "sitemap")
{
foreach (XmlNode item in node.ChildNodes)
{
if (item.Name == "loc")
{
foreach (string rule in disallowList)
{
if (node.Name.Contains(rule))
{
valid = false;
break;
}
}
if (valid != false)
{
string itemDate = item.NextSibling.InnerText;
DateTime date = XmlConvert.ToDateTime(itemDate);
if (DateTime.Now.Date <= date.AddMonths(3))
{
CloudQueueMessage message = new CloudQueueMessage(item.InnerText.Trim());
queue.AddMessage(message);
queueCount++;
}
}
}
}
}
}
}
}
rootQueue.DeleteMessage(baseSite);
}
else if (currentUrl != null)
{
if (currentUrl.AsString.EndsWith(".xml"))
{
WebRequest crawlXml = WebRequest.Create(currentUrl.AsString);
Stream urlStream = crawlXml.GetResponse().GetResponseStream();
StreamReader readUrl = new StreamReader(urlStream);
XmlDocument urlContent = new XmlDocument();
urlContent.Load(readUrl);
XmlNodeList nodes = urlContent.DocumentElement.ChildNodes;
foreach (XmlNode node in nodes)
{
Boolean valid = true;
if (node.Name == "url")
{
foreach (XmlNode item in node.ChildNodes)
{
if (item.Name == "loc")
{
foreach (string rule in disallowList)
{
if (node.Name.Contains(rule))
{
valid = false;
break;
}
}
if (valid != false)
{
if (item.NextSibling.Name.Equals("lastmod"))
{
string itemDate = item.NextSibling.InnerText;
CloudQueueMessage message = new CloudQueueMessage(item.InnerText);
queue.AddMessage(message);
queueCount++;
}
}
}
}
}
}
}
else
{
if (!visitedLinks.Contains(currentUrl.AsString))
{
HttpWebRequest htmlPage = (HttpWebRequest)HttpWebRequest.Create(currentUrl.AsString);
Stream html = htmlPage.GetResponse().GetResponseStream();
StreamReader getHtml = new StreamReader(html);
urlCrawled++;
string data = "";
string line;
while ((line = getHtml.ReadLine()) != null)
{
data += line;
}
HtmlDocument page = new HtmlDocument();
page.LoadHtml(data);
HtmlNodeCollection links = page.DocumentNode.SelectNodes("//a[@href]");
List <String> newLinks = new List <String>();
if (links != null)
{
foreach (HtmlNode link in links)
{
string item = link.GetAttributeValue("href", "");
if (item.StartsWith("http://www.cnn.com") || (item.StartsWith("http://www.sportsillustrated.cnn.com") && item.Contains("/basketball/nba")))
{
newLinks.Add(item);
}
}
bool valid = true;
foreach (string potential in newLinks)
{
foreach (string rule in disallowList)
{
if (potential.Contains(rule))
{
valid = false;
break;
}
}
if (valid == true && !potential.Contains("disqus_thread"))
{
CloudQueueMessage toCrawl = new CloudQueueMessage(potential);
queue.AddMessage(toCrawl);
queueCount++;
}
valid = true;
}
}
string title = page.DocumentNode.SelectSingleNode("//head/title").InnerText;
Regex r = new Regex("[^a-zA-Z.-]");
string tempUrl = currentUrl.AsString;
string[] titleWords = title.Split(' ');
foreach (string word in titleWords)
{
string cleanWord = r.Replace(word, "");
CrawlEntity addToTable = new CrawlEntity(cleanWord.ToLower(), title, tempUrl);
if (addToTable.Title.Length > 0)
{
TableOperation insert = TableOperation.InsertOrReplace(addToTable);
table.Execute(insert);
tableCount++;
lastTen.Add(currentUrl.AsString);
if (lastTen.Count > 0)
{
lastTen.RemoveAt(0);
}
}
visitedLinks.Add(currentUrl.AsString);
}
}
}
queue.DeleteMessage(currentUrl);
queueCount--;
}
}
}
