public SmartCrawler(string name, string url, string pathOutput, string[] urlAcceptWords, string[] ignoreKeywords, int dept)
{
Name = name;
Url = url;
CountRun = 0;
OutputFile = pathOutput;
UrlKeyWords = urlAcceptWords;
Depth = dept;
Step = 0;
IgnoreKeywords = ignoreKeywords;
InitSchedules();
UrlSmart = new SmartUrl(Url);
Page = new WebPage(Url);
if (urlAcceptWords.Length <= 0)
{
UrlKeyWords.ToList().Add(Url);
}
if (pathOutput == null || pathOutput == string.Empty)
{
pathOutput = @"C:\TmpC#\Crawlers\" + Name + ".txt";
}
AddResult();
GetNextUrls();
}
protected void InitFormByUrl()
{
var input1 = SmartUrl.ExtractInput(Request.Params, 1);
var input2 = SmartUrl.ExtractInput(Request.Params, 2);
// Sync form controls
if (!string.IsNullOrWhiteSpace(input1.DatabaseName))
{
if (ddlDatabase1.Items.FindByValue(input1.DatabaseName) != null)
{
ddlDatabase1.SelectedValue = input1.DatabaseName;
}
}
if (!string.IsNullOrWhiteSpace(input2.DatabaseName))
{
if (ddlDatabase2.Items.FindByValue(input2.DatabaseName) != null)
{
ddlDatabase2.SelectedValue = input2.DatabaseName;
}
}
// Item
txtItemId.Text = input1.ItemId?.ToString();
// Solr
if (Configs.IsUsingSolr)
{
if (!string.IsNullOrWhiteSpace(input1.SolrIndexes?[0]))
{
ddlIndex1.SelectedValue = input1.SolrIndexes?[0];
}
if (!string.IsNullOrWhiteSpace(input2.SolrIndexes?[0]))
{
ddlIndex2.SelectedValue = input2.SolrIndexes?[0];
}
if (!string.IsNullOrWhiteSpace(input1.SolrIndexes?[1]))
{
ddlIndex3.SelectedValue = input1.SolrIndexes?[1];
}
if (!string.IsNullOrWhiteSpace(input2.SolrIndexes?[1]))
{
ddlIndex4.SelectedValue = input2.SolrIndexes?[1];
}
}
if (input1.ItemId != (ID)null)
{
// Field
LoadFields(input1.ItemId, input1.Database ?? input2.Database);
ddlField.SelectedValue = input1.FieldId.ToGuid().ToString();
// and load item into viewer
LoadItem(input1, input2);
}
}
protected void InitComponent()
{
GrouperService = new ValueGrouper();
SmartUrl = new SmartUrl();
ApiViewer = new ApiViewer(GrouperService, SmartUrl);
SqlViewer = new SqlViewer(GrouperService, SmartUrl);
if (Configs.IsUsingSolr)
{
SolrViewer = new SolrViewer();
}
InitDropDowns();
}
public SmartCrawler(string url)
{
if (url.IsUrl())
{
UrlSmart = new SmartUrl(url);
Page = new WebPage(url);
Depth = 0;
Name = url;
CountRun = 0;
InitSchedules();
}
else
{
throw new Exception("url is not valid");
}
}
public async Task <IHttpActionResult> PostLink([FromUri] SmartUrl url)
{
if (url.IsEmpty)
{
return(BadRequest("Emtpy/Bad Url"));
}
Uri PunyUrl = await shortener.ProcessAsync(url);
Domain.Entities.PunyUrl punyUrl = await storage.AddOrUpdateAsync(url, PunyUrl);
return(Ok(new
{
topDomain = punyUrl.PartitionKey.FromBase64(),
puny = punyUrl.RowKey.FromBase64(),
original = punyUrl.Original
}));
}
protected void LoadItem(params LensInput[] input)
{
// Init values comparison
GrouperService.Reset();
// Populate API/Cached results
ResultApiMaster.Text = ApiViewer.GetHtml(input?[0]);
ResultApiWeb.Text = ApiViewer.GetHtml(input?[1]);
// Populate SQL results
ResultSqlMaster.Text = SqlViewer.GetHtml(input?[0]);
ResultSqlWeb.Text = SqlViewer.GetHtml(input?[1]);
// Populate Solr results
if (Configs.IsUsingSolr)
{
litIndexesLeft.Text = SolrViewer.GetHtml(input?[0]);
litIndexesRight.Text = SolrViewer.GetHtml(input?[1]);
}
// Set Share
litShare.Text = $"<a href=\"?{SmartUrl.ToQueryString(input)}\" class=\"btn btn-secondary\">Share Link</a>";
}
public SmartCrawler(string crawlerTemplate, string crawlerOutput)
{
SmartFile _file = new SmartFile(crawlerTemplate);
string[] _items = _file.Lines[0].Split(new string[] { "|" }, StringSplitOptions.None);
Name = _items[0]; //CrawlerName
Url = _items[1]; //Url
OutputFile = _items[2];
Depth = int.Parse(_items[3]);
UrlKeyWords = _items[4].Split(new string[] { ";" }, StringSplitOptions.None).Where(t => t.Length > 0).ToArray();
InitSchedules();
CountRun = 0;
Step = 0;
if (!string.IsNullOrWhiteSpace(_items[5]))
{
IgnoreKeywords = _items[5].Split(new string[] { ";" }, StringSplitOptions.None).Where(t => t.Length > 0).ToArray();
}
if (OutputFile == null || OutputFile == string.Empty)
{
OutputFile = @"C:\TmpC#\Crawlers\" + Name + ".txt";
}
UrlSmart = new SmartUrl(Url);
Page = new WebPage(Url);
OutputFile = crawlerOutput + OutputFile;
if (!File.Exists(OutputFile))
{
File.Create(OutputFile);
File.WriteAllText(OutputFile, "Crawler " + Name);
}
AddResult();
NextUrls = new List <string>();
NextUrls = Page.HtmlSmart.InternalLinks.Where(t => t.IsUrl() && IsAccept(t)).Distinct().ToList();
}
public override object ConvertFrom(ITypeDescriptorContext context, CultureInfo culture, object value)
{
return(SmartUrl.Parse(value as string));
}
