public void ParseUnknownTest()
{
string text = "}";
ITextProvider tp = new StringTextProvider(text);
{
ParseItem pi = UnknownItem.ParseUnknown(
null, new ItemFactory(tp, null), tp,
Helpers.MakeTokenStream(tp),
ParseErrorType.OpenCurlyBraceMissingForRule);
Assert.IsNotNull(pi);
Assert.IsTrue(pi.HasParseErrors);
Assert.AreEqual(ParseErrorType.OpenCurlyBraceMissingForRule, pi.ParseErrors[0].ErrorType);
Assert.AreEqual(ParseErrorLocation.WholeItem, pi.ParseErrors[0].Location);
}
// Try a URL
{
text = "url('foo.jpg')";
tp = new StringTextProvider(text);
UrlItem pi = UnknownItem.ParseUnknown(
null, new ItemFactory(tp, null), tp,
Helpers.MakeTokenStream(tp)) as UrlItem;
Assert.IsNotNull(pi);
}
}
public ItemCheckResult CheckItem(ParseItem item, ICssCheckerContext context)
{
UrlItem url = (UrlItem)item;
if (!WESettings.GetBoolean(WESettings.Keys.ValidateEmbedImages) || !url.IsValid || url.UrlString.Text.Contains("base64,") || context == null)
{
return(ItemCheckResult.Continue);
}
string fileName = ImageQuickInfo.GetFileName(url.UrlString.Text);
if (fileName.Contains("://"))
{
return(ItemCheckResult.Continue);
}
FileInfo file = new FileInfo(fileName);
if (file.Exists && file.Length < (1024 * 3))
{
Declaration dec = url.FindType <Declaration>();
if (dec != null && dec.PropertyName != null && dec.PropertyName.Text[0] != '*' && dec.PropertyName.Text[0] != '_')
{
string error = string.Format(Resources.PerformanceEmbedImageAsDataUri, file.Length);
context.AddError(new SimpleErrorTag(url.UrlString, error));
}
}
return(ItemCheckResult.Continue);
}
public void Should_mark_job_finished_when_it_is_stopped()
{
const string testUrl = "http://test.com";
const string testHost = "test.com";
using (_db.CreateTransaction())
{
#region action
var urlItem = new UrlItem
{
Url = testUrl,
Host = testHost
};
var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>());
var job = jobRep.Start(urlItem); // adds the 1st log message
// wait a little bit to force difference in time between two log messages and sort them later
Thread.Sleep(100);
jobRep.Stop(job); // marks the job finished
#endregion
#region assertion
// make sure the job is created in DB
using (var ctx = _db.CreateDbContext())
{
var finishedJob = ctx.JobItems.Single(j => j.Id == job.Id);
Assert.True(finishedJob.DateFinish.HasValue);
}
#endregion
}
}
public IActionResult Create([FromBody] UrlItem item)
{
if (item == null)
{
return(BadRequest());
}
//count HTML tags
using (WebClient client1 = new WebClient())
{
HtmlWeb web = new HtmlWeb();
HtmlDocument doc = web.Load(item.Url);
item.DivCounter = doc.DocumentNode.Descendants("div").Count();
item.ACounter = doc.DocumentNode.Descendants("a").Count();
item.SpanCounter = doc.DocumentNode.Descendants("span").Count();
}
var connectionString = "mongodb://localhost:27017";
MongoClient client = new MongoClient(connectionString);
MongoServer server = client.GetServer();
MongoDatabase database = server.GetDatabase("ScannedURL");
MongoCollection collection = database.GetCollection <UrlItem>("URLScanningResults");
item.Id = collection.Count() + 1;
collection.Insert(item);
string res = "Scanning number " + item.Id + "\nURL: " + item.Url
+ "\nNumber of div element: " + item.DivCounter +
"\nNumber of A element: " + item.ACounter + "\nNumber of span element: " + item.SpanCounter;
return(Content(res));
}
public void Should_create_new_job_for_new_url()
{
const string testUrl = "http://test.com";
const string testHost = "test.com";
using (_db.CreateTransaction())
{
#region action
var urlItem = new UrlItem
{
Url = testUrl,
Host = testHost
};
var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>());
var job = jobRep.Start(urlItem);
#endregion
#region assertion
// make sure the job is created in DB
using (var ctx = _db.CreateDbContext())
{
// try to find unfinished job for the url
var actualJob = ctx.JobItems.Include(j => j.Url).SingleOrDefault(j => j.DateFinish.HasValue == false);
Assert.NotNull(actualJob);
Assert.Equal(job.Id, actualJob.Id);
Assert.Equal(testUrl, actualJob.Url.Url);
Assert.False(actualJob.DateFinish.HasValue);
}
#endregion
}
}
public void Should_add_item_to_history_when_job_is_stopped()
{
var mockLogger = new Mock <IActivityLogRepository>();
const string testUrl = "http://test.com";
const string testHost = "test.com";
using (_db.CreateTransaction())
{
#region action
var urlItem = new UrlItem
{
Url = testUrl,
Host = testHost
};
var jobRep = new JobRepository(mockLogger.Object);
var job = jobRep.Start(urlItem); // adds the 1st log message
// wait a little bit to force difference in time between two log messages and sort them later
Thread.Sleep(100);
jobRep.Stop(job); // adds the 2nd log message
#endregion
#region assertion
mockLogger.Verify(m => m.JobStarted(It.IsAny <JobItem>()), Times.Once);
mockLogger.Verify(m => m.JobStopped(It.IsAny <JobItem>()), Times.Once);
#endregion
}
}
public void Should_return_null_if_job_is_already_running_for_url()
{
const string testUrl = "http://test.com";
const string testHost = "test.com";
using (_db.CreateTransaction())
{
var urlItem = new UrlItem
{
Url = testUrl,
Host = testHost
};
// add not finished job
using (var ctx = _db.CreateDbContext())
{
ctx.JobItems.Add(new JobItem
{
Id = Guid.NewGuid(),
DateStart = DateTime.Now.AddDays(-2),
DateFinish = null, // make sure the job is NOT finished
Url = urlItem
});
ctx.Commit();
}
var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>());
Assert.Throws <JobIsAlreadyRunningException>(() => jobRep.Start(urlItem));
}
}
public void Should_add_log_message_when_job_is_started()
{
var mockLogger = new Mock <IActivityLogRepository>();
const string testUrl = "http://test.com";
const string testHost = "test.com";
using (_db.CreateTransaction())
{
#region action
var urlItem = new UrlItem
{
Url = testUrl,
Host = testHost
};
var jobRep = new JobRepository(mockLogger.Object);
jobRep.Start(urlItem);
#endregion
#region assertion
mockLogger.Verify(m => m.JobStarted(It.IsAny <JobItem>()), Times.Once);
#endregion
}
}
public void Should_return_default_settings_when_host_dont_have()
{
#region get default settings
HostSetting defaultSettings;
using (var ctx = _db.CreateDbContext())
{
defaultSettings = ctx.HostSettings.Single(s => s.Host == string.Empty);
}
#endregion
const string testUrl = "http://sub.testhost.com/page?param=1¶m=2";
const string testHost = "testhost.com";
#region get settings for host
var urlItem = new UrlItem
{
Url = testUrl, Host = testHost
};
var settingsRep = new CrawlerSettingsRepository(Mock.Of <IActivityLogRepository>());
var hostSettings = settingsRep.GetSettings(urlItem);
#endregion
Assert.NotNull(hostSettings);
Assert.Equal(defaultSettings.Host, hostSettings.Host);
Assert.Equal(defaultSettings.CrawlDelay, hostSettings.CrawlDelay);
Assert.Equal(defaultSettings.RobotsTxt, hostSettings.RobotsTxt);
Assert.True(defaultSettings.Disallow.SequenceEqual(hostSettings.Disallow));
}
public IEnumerable <ISmartTagAction> GetSmartTagActions(ParseItem item, int position, ITrackingSpan itemTrackingSpan, ITextView view)
{
UrlItem url = (UrlItem)item;
if (!url.IsValid || url.UrlString == null || string.IsNullOrEmpty(url.UrlString.Text))
{
yield break;
}
string text = url.UrlString.Text.Trim('"', '\'');
string testSupportFileName = text;
if (url.IsDataUri())
{
string mime = FileHelpers.GetMimeTypeFromBase64(text);
testSupportFileName = "file." + FileHelpers.GetExtension(mime);
}
if (!ImageCompressor.IsFileSupported(testSupportFileName))
{
yield break;
}
yield return(new OptimizeImageSmartTagAction(itemTrackingSpan, url));
}
public ItemCheckResult CheckItem(ParseItem item, ICssCheckerContext context)
{
UrlItem url = (UrlItem)item;
if (!WESettings.Instance.Css.ValidateEmbedImages || !url.IsValid || url.UrlString == null || url.UrlString.Text.Contains("base64,") || context == null)
{
return(ItemCheckResult.Continue);
}
string fileName = ImageQuickInfo.GetFullUrl(url.UrlString.Text, EditorExtensionsPackage.DTE.ActiveDocument.FullName);
if (string.IsNullOrEmpty(fileName) || fileName.Contains("://"))
{
return(ItemCheckResult.Continue);
}
FileInfo file = new FileInfo(fileName);
if (file.Exists && file.Length < (1024 * 3))
{
Declaration dec = url.FindType <Declaration>();
if (dec != null && dec.PropertyName != null && dec.PropertyName.Text[0] != '*' && dec.PropertyName.Text[0] != '_')
{
string error = string.Format(CultureInfo.CurrentCulture, Resources.PerformanceEmbedImageAsDataUri, file.Length);
context.AddError(new SimpleErrorTag(url.UrlString, error));
}
}
return(ItemCheckResult.Continue);
}
public ActionResult Index(string shortenUrl)
{
UrlItem item = _urlRepo.GetById(UrlGenerator.Decode(shortenUrl)) ?? _urlRepo.All().FirstOrDefault(i => i.CustomUrl.Equals(shortenUrl));
if (item == null)
{
return(View("NotFound"));
}
if (IsExpired(item))
{
_urlRepo.Delete(item);
return(View("Expired"));
}
var urlHit = new UrlHit {
UrlItemId = item.Id,
ClientIp = Request.UserHostAddress
};
_urlRepo.Update(item);
_hitRepo.Add(urlHit);
return(Redirect(item.OriginUrl));
}
public void SetSettings(UrlItem urlItem, HostSetting settings)
{
try
{
using (var ctx = CreateContext())
{
var existing = ctx.HostSettings.SingleOrDefault(s => s.Host == urlItem.Host);
var newSettings = existing ?? settings;
newSettings.Host = urlItem.Host; // ensure the host is correct
// update if exists
if (existing != null)
{
newSettings.CrawlDelay = settings.CrawlDelay;
newSettings.RobotsTxt = settings.RobotsTxt;
newSettings.Disallow = settings.Disallow;
}
ctx.HostSettings.Add(newSettings);
ctx.Commit();
}
_logger.SettingsStored(urlItem);
}
catch (Exception err)
{
_logger.LogError(urlItem, err);
throw;
}
}
public async Task <ActionResult> DeleteConfirmed(string id)
{
UrlItem urlItem = await db.UrlItems.FindAsync(id);
db.UrlItems.Remove(urlItem);
await db.SaveChangesAsync();
return(RedirectToAction("Index"));
}
public static bool IsDataUri(this UrlItem item)
{
if (item.UrlString == null || string.IsNullOrEmpty(item.UrlString.Text))
{
return(false);
}
return(item.UrlString.Text.Contains(";base64,"));
}
public static string apply_taoke_url_item(CmsForm cmsForm, ContentItem contentItem
, UrlItem urlItem, string id, string pid)
{
string click_url = UserUtil.query_click_url(cmsForm, pid, id, urlItem.url);
urlItem.click_url = click_url;
return(click_url);
}
public EmbedSmartTagAction(ITrackingSpan span, UrlItem url)
{
_span = span;
_url = url;
if (Icon == null)
{
Icon = BitmapFrame.Create(new Uri("pack://application:,,,/WebEssentials2015;component/Resources/Images/embed.png", UriKind.RelativeOrAbsolute));
}
}
public EmbedSmartTagAction(ITrackingSpan span, UrlItem url)
{
_span = span;
_url = url;
if (Icon == null)
{
Icon = BitmapFrame.Create(new Uri("pack://application:,,,/WebEssentials2013;component/Resources/Images/embed.png", UriKind.RelativeOrAbsolute));
}
}
public async Task <ActionResult> Edit([Bind(Include = "Id,Link")] UrlItem urlItem)
{
if (ModelState.IsValid)
{
db.Entry(urlItem).State = EntityState.Modified;
await db.SaveChangesAsync();
return(RedirectToAction("Index"));
}
return(View(urlItem));
}
public void SettingsStored(UrlItem url)
{
using (var ctx = CreateContext())
{
ctx.ActivityMessages.Add(new ActivityMessage
{
Message = string.Format(LogMessages.SettingsStored, url)
});
ctx.Commit();
}
}
public async Task <ActionResult> Create([Bind(Include = "Id,Link")] UrlItem urlItem)
{
if (ModelState.IsValid)
{
db.UrlItems.Add(urlItem);
await db.SaveChangesAsync();
return(RedirectToAction("Index"));
}
return(View(urlItem));
}
public void LogError(UrlItem urlItem, Exception err)
{
using (var ctx = CreateContext())
{
ctx.ActivityMessages.Add(new ActivityMessage
{
Type = ActivityMessageType.Error,
Message = string.Format(LogMessages.LogErrorUrlItem, urlItem.Id, err)
});
ctx.Commit();
}
}
public static UrlItem Create(string url, DateTime nextAvailableTime)
{
var uri = new Uri(url);
var urlItem = new UrlItem
{
Url = url,
Host = uri.Host,
EvaliableFromDate = nextAvailableTime
};
return(urlItem);
}
public void Add()
{
var item = new UrlItem {
ExpireMode = (short)ExpireMode.Never,
OriginUrl = "http://www.google.com/"
};
_repo.Add(item);
UrlItem storedItem = _repo.GetById(UrlGenerator.Decode(item.CustomUrl));
Assert.That(storedItem, Is.Not.Null);
}
internal async Task CreateAsync(UrlItem urlItem)
{
// Add item to database
_dbContext.Urls.Add(urlItem);
var success = false;
Exception?lastException = null;
// Try max 10 times before giving up
string key = null;
for (var i = 0; i < 10; i++)
{
// Create random id
key = Helpers.GetRandomKey(_config.Url);
// Try to add item to database
urlItem.Key = key;
try
{
await _dbContext.SaveChangesAsync();
success = true;
}
catch (DbUpdateException dbUpdateException) when((dbUpdateException.InnerException as SqlException)?.Number == 2601)
{
// Already exists: We need to retry
_logger.LogInformation(dbUpdateException, $"Collision creating ShortUrl: {urlItem.Key}");
lastException = dbUpdateException;
continue;
}
catch (Exception exception)
{
// Not sure what is wrong, log it and try again
_logger.LogError(exception, $"Creating new ShortUrl: {JsonSerializer.Serialize(urlItem)}");
lastException = exception;
continue;
}
}
// Not success: Throw last error
if (!success)
{
throw lastException !;
}
// Success: Add item to recent used cache since it will probably be used immediately
var cacheEntryOptions = new MemoryCacheEntryOptions();
cacheEntryOptions.Size = 1;
_cache.Set <UrlItem>(key, urlItem, cacheEntryOptions);
}
/// <summary>
/// 获取 UrlItem
/// </summary>
/// <param name="xmlNode"></param>
/// <returns></returns>
private UrlItem CreateUrlItem(XmlNode xmlNode)
{
if (xmlNode == null)
{
return(null);
}
UrlItem urlItem = new UrlItem();
urlItem.RegexString = xmlNode.SelectSingleNode("RegexString").InnerText;
urlItem.UrlFomart = xmlNode.SelectSingleNode("UrlFomart").InnerText;
urlItem.TitleGroupIndex = int.Parse(xmlNode.SelectSingleNode("TitleGroupIndex").InnerText);
urlItem.UrlGroupIndex = int.Parse(xmlNode.SelectSingleNode("UrlGroupIndex").InnerText);
return(urlItem);
}
// GET: Url/Delete/5
public async Task <ActionResult> Delete(string id)
{
if (id == null)
{
return(new HttpStatusCodeResult(HttpStatusCode.BadRequest));
}
UrlItem urlItem = await db.UrlItems.FindAsync(id);
if (urlItem == null)
{
return(HttpNotFound());
}
return(View(urlItem));
}
public async Task <CreateResponse> Create([FromBody] CreateRequest request)
{
// Security: Request will only be allowed if "RequireTokenForGet" is false OR request.AuthToken is in list or in database
if (_config.Security.RequireTokenForGet
// Not in allowed list in config
&& _config.Security.AuthenticationTokens != null &&
!_config.Security.AuthenticationTokens.Contains(request.AuthToken)
// And not in database
&& !_dbContext.AuthTokens.Any(w => w.AuthToken == request.AuthToken && w.CanCreate)
)
{
// Not accepted
return(new CreateResponse()
{
Success = false,
ErrorMessage = "Invalid authentication token"
});
}
// Create item
var urlItem = new UrlItem()
{
Key = Helpers.GetRandomKey(_config.Url),
Expires = request.Expires,
Url = request.Url,
Metadata = request.Metadata
};
try
{
await _shortUrlService.CreateAsync(urlItem);
}
catch (Exception ex)
{
_logger.LogError(ex, $"API: Creating short url for: {request.Url}");
return(new CreateResponse()
{
Success = false,
ErrorMessage = "Error: Have admin check logs for more information."
});
}
return(new CreateResponse()
{
Success = true,
Key = urlItem.Key,
ShortUrl = Helpers.GetShortUrl(_config.Url.OverrideUrl, Request, urlItem.Key),
Expires = request.Expires
});
}
public static void parseContentUrlList(CouponContent contentItem, CmsForm cmsForm, string content)
{
try
{
ArrayList arrayLists1 = new ArrayList();
MatchCollection matchCollections = (new Regex(Constants.regex_url)).Matches(content);
int num = 1;
foreach (Match match in matchCollections)
{
LogUtil.log_call(cmsForm, string.Concat("正在处理第【", num, "】【", match.Value.ToString(), "】个链接！"));
UrlItem urlItem = new UrlItem()
{
ori_url = match.Value.ToString()
};
if ((urlItem.ori_url.Contains("activityId") && urlItem.ori_url.Contains("uland.taobao.com") && !urlItem.ori_url.Contains("uland.taobao.com/quan/detail")) ||
(((urlItem.ori_url.Contains("taobao.com") ||
urlItem.ori_url.Contains("tmall.com") ||
urlItem.ori_url.Contains("yao.95095.com")) &&
urlItem.ori_url.Contains("item.htm") && urlItem.ori_url.Contains("id="))))
{
string num_iid = TaobaoUtil.get_num_iid(urlItem.ori_url);
if (!string.IsNullOrEmpty(num_iid) && string.IsNullOrEmpty(contentItem.num_iid))
{
contentItem.num_iid = num_iid;
}
}
else
{
string str = TaobaoUtil.get_redirect_url(urlItem.ori_url, urlItem.ori_url);
if ((urlItem.ori_url.Contains("activityId") && urlItem.ori_url.Contains("uland.taobao.com") && !urlItem.ori_url.Contains("uland.taobao.com/quan/detail")) ||
(((urlItem.ori_url.Contains("taobao.com") ||
urlItem.ori_url.Contains("tmall.com") ||
urlItem.ori_url.Contains("yao.95095.com")) &&
urlItem.ori_url.Contains("item.htm") && urlItem.ori_url.Contains("id="))))
{
string num_iid = TaobaoUtil.get_num_iid(urlItem.ori_url);
if (!string.IsNullOrEmpty(num_iid) && string.IsNullOrEmpty(contentItem.num_iid))
{
contentItem.num_iid = num_iid;
}
}
}
num++;
}
}
catch (Exception exception)
{
LogUtil.log_call(cmsForm, string.Concat("[parseUrl]出错！", exception.ToString()));
}
}
public IEnumerable <ISmartTagAction> GetSmartTagActions(ParseItem item, int position, ITrackingSpan itemTrackingSpan, ITextView view)
{
UrlItem url = (UrlItem)item;
if (!url.IsValid || url.UrlString == null)
{
yield break;
}
if (url.UrlString.Text.Contains(";base64,"))
{
yield return(new ReverseEmbedSmartTagAction(itemTrackingSpan, url));
}
}
public void Should_create_new_job_for_url_which_has_inactive_job()
{
const string testUrl = "http://test.com";
const string testHost = "test.com";
using (_db.CreateTransaction())
{
var urlItem = new UrlItem
{
Url = testUrl,
Host = testHost
};
// add finished job
using (var ctx = _db.CreateDbContext())
{
ctx.JobItems.Add(new JobItem
{
Id = Guid.NewGuid(),
DateStart = DateTime.Now.AddDays(-2),
DateFinish = DateTime.Now.AddDays(-2), // make sure the job is finished
Url = urlItem
});
ctx.Commit();
}
#region action
var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>());
var job = jobRep.Start(urlItem);
#endregion
#region assertion
// make sure the job is created in DB
using (var ctx = _db.CreateDbContext())
{
// try to find unfinished job for the url
var actualJob = ctx.JobItems.Include(j => j.Url).SingleOrDefault(j => j.DateFinish.HasValue == false);
Assert.NotNull(actualJob);
Assert.Equal(job.Id, actualJob.Id);
Assert.Equal(testUrl, actualJob.Url.Url);
Assert.False(actualJob.DateFinish.HasValue);
}
#endregion
}
}
