public void ParseUnknownTest() { string text = "}"; ITextProvider tp = new StringTextProvider(text); { ParseItem pi = UnknownItem.ParseUnknown( null, new ItemFactory(tp, null), tp, Helpers.MakeTokenStream(tp), ParseErrorType.OpenCurlyBraceMissingForRule); Assert.IsNotNull(pi); Assert.IsTrue(pi.HasParseErrors); Assert.AreEqual(ParseErrorType.OpenCurlyBraceMissingForRule, pi.ParseErrors[0].ErrorType); Assert.AreEqual(ParseErrorLocation.WholeItem, pi.ParseErrors[0].Location); } // Try a URL { text = "url('foo.jpg')"; tp = new StringTextProvider(text); UrlItem pi = UnknownItem.ParseUnknown( null, new ItemFactory(tp, null), tp, Helpers.MakeTokenStream(tp)) as UrlItem; Assert.IsNotNull(pi); } }
public ItemCheckResult CheckItem(ParseItem item, ICssCheckerContext context) { UrlItem url = (UrlItem)item; if (!WESettings.GetBoolean(WESettings.Keys.ValidateEmbedImages) || !url.IsValid || url.UrlString.Text.Contains("base64,") || context == null) { return(ItemCheckResult.Continue); } string fileName = ImageQuickInfo.GetFileName(url.UrlString.Text); if (fileName.Contains("://")) { return(ItemCheckResult.Continue); } FileInfo file = new FileInfo(fileName); if (file.Exists && file.Length < (1024 * 3)) { Declaration dec = url.FindType <Declaration>(); if (dec != null && dec.PropertyName != null && dec.PropertyName.Text[0] != '*' && dec.PropertyName.Text[0] != '_') { string error = string.Format(Resources.PerformanceEmbedImageAsDataUri, file.Length); context.AddError(new SimpleErrorTag(url.UrlString, error)); } } return(ItemCheckResult.Continue); }
public void Should_mark_job_finished_when_it_is_stopped() { const string testUrl = "http://test.com"; const string testHost = "test.com"; using (_db.CreateTransaction()) { #region action var urlItem = new UrlItem { Url = testUrl, Host = testHost }; var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>()); var job = jobRep.Start(urlItem); // adds the 1st log message // wait a little bit to force difference in time between two log messages and sort them later Thread.Sleep(100); jobRep.Stop(job); // marks the job finished #endregion #region assertion // make sure the job is created in DB using (var ctx = _db.CreateDbContext()) { var finishedJob = ctx.JobItems.Single(j => j.Id == job.Id); Assert.True(finishedJob.DateFinish.HasValue); } #endregion } }
public IActionResult Create([FromBody] UrlItem item) { if (item == null) { return(BadRequest()); } //count HTML tags using (WebClient client1 = new WebClient()) { HtmlWeb web = new HtmlWeb(); HtmlDocument doc = web.Load(item.Url); item.DivCounter = doc.DocumentNode.Descendants("div").Count(); item.ACounter = doc.DocumentNode.Descendants("a").Count(); item.SpanCounter = doc.DocumentNode.Descendants("span").Count(); } var connectionString = "mongodb://localhost:27017"; MongoClient client = new MongoClient(connectionString); MongoServer server = client.GetServer(); MongoDatabase database = server.GetDatabase("ScannedURL"); MongoCollection collection = database.GetCollection <UrlItem>("URLScanningResults"); item.Id = collection.Count() + 1; collection.Insert(item); string res = "Scanning number " + item.Id + "\nURL: " + item.Url + "\nNumber of div element: " + item.DivCounter + "\nNumber of A element: " + item.ACounter + "\nNumber of span element: " + item.SpanCounter; return(Content(res)); }
public void Should_create_new_job_for_new_url() { const string testUrl = "http://test.com"; const string testHost = "test.com"; using (_db.CreateTransaction()) { #region action var urlItem = new UrlItem { Url = testUrl, Host = testHost }; var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>()); var job = jobRep.Start(urlItem); #endregion #region assertion // make sure the job is created in DB using (var ctx = _db.CreateDbContext()) { // try to find unfinished job for the url var actualJob = ctx.JobItems.Include(j => j.Url).SingleOrDefault(j => j.DateFinish.HasValue == false); Assert.NotNull(actualJob); Assert.Equal(job.Id, actualJob.Id); Assert.Equal(testUrl, actualJob.Url.Url); Assert.False(actualJob.DateFinish.HasValue); } #endregion } }
public void Should_add_item_to_history_when_job_is_stopped() { var mockLogger = new Mock <IActivityLogRepository>(); const string testUrl = "http://test.com"; const string testHost = "test.com"; using (_db.CreateTransaction()) { #region action var urlItem = new UrlItem { Url = testUrl, Host = testHost }; var jobRep = new JobRepository(mockLogger.Object); var job = jobRep.Start(urlItem); // adds the 1st log message // wait a little bit to force difference in time between two log messages and sort them later Thread.Sleep(100); jobRep.Stop(job); // adds the 2nd log message #endregion #region assertion mockLogger.Verify(m => m.JobStarted(It.IsAny <JobItem>()), Times.Once); mockLogger.Verify(m => m.JobStopped(It.IsAny <JobItem>()), Times.Once); #endregion } }
public void Should_return_null_if_job_is_already_running_for_url() { const string testUrl = "http://test.com"; const string testHost = "test.com"; using (_db.CreateTransaction()) { var urlItem = new UrlItem { Url = testUrl, Host = testHost }; // add not finished job using (var ctx = _db.CreateDbContext()) { ctx.JobItems.Add(new JobItem { Id = Guid.NewGuid(), DateStart = DateTime.Now.AddDays(-2), DateFinish = null, // make sure the job is NOT finished Url = urlItem }); ctx.Commit(); } var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>()); Assert.Throws <JobIsAlreadyRunningException>(() => jobRep.Start(urlItem)); } }
public void Should_add_log_message_when_job_is_started() { var mockLogger = new Mock <IActivityLogRepository>(); const string testUrl = "http://test.com"; const string testHost = "test.com"; using (_db.CreateTransaction()) { #region action var urlItem = new UrlItem { Url = testUrl, Host = testHost }; var jobRep = new JobRepository(mockLogger.Object); jobRep.Start(urlItem); #endregion #region assertion mockLogger.Verify(m => m.JobStarted(It.IsAny <JobItem>()), Times.Once); #endregion } }
public void Should_return_default_settings_when_host_dont_have() { #region get default settings HostSetting defaultSettings; using (var ctx = _db.CreateDbContext()) { defaultSettings = ctx.HostSettings.Single(s => s.Host == string.Empty); } #endregion const string testUrl = "http://sub.testhost.com/page?param=1¶m=2"; const string testHost = "testhost.com"; #region get settings for host var urlItem = new UrlItem { Url = testUrl, Host = testHost }; var settingsRep = new CrawlerSettingsRepository(Mock.Of <IActivityLogRepository>()); var hostSettings = settingsRep.GetSettings(urlItem); #endregion Assert.NotNull(hostSettings); Assert.Equal(defaultSettings.Host, hostSettings.Host); Assert.Equal(defaultSettings.CrawlDelay, hostSettings.CrawlDelay); Assert.Equal(defaultSettings.RobotsTxt, hostSettings.RobotsTxt); Assert.True(defaultSettings.Disallow.SequenceEqual(hostSettings.Disallow)); }
public IEnumerable <ISmartTagAction> GetSmartTagActions(ParseItem item, int position, ITrackingSpan itemTrackingSpan, ITextView view) { UrlItem url = (UrlItem)item; if (!url.IsValid || url.UrlString == null || string.IsNullOrEmpty(url.UrlString.Text)) { yield break; } string text = url.UrlString.Text.Trim('"', '\''); string testSupportFileName = text; if (url.IsDataUri()) { string mime = FileHelpers.GetMimeTypeFromBase64(text); testSupportFileName = "file." + FileHelpers.GetExtension(mime); } if (!ImageCompressor.IsFileSupported(testSupportFileName)) { yield break; } yield return(new OptimizeImageSmartTagAction(itemTrackingSpan, url)); }
public ItemCheckResult CheckItem(ParseItem item, ICssCheckerContext context) { UrlItem url = (UrlItem)item; if (!WESettings.Instance.Css.ValidateEmbedImages || !url.IsValid || url.UrlString == null || url.UrlString.Text.Contains("base64,") || context == null) { return(ItemCheckResult.Continue); } string fileName = ImageQuickInfo.GetFullUrl(url.UrlString.Text, EditorExtensionsPackage.DTE.ActiveDocument.FullName); if (string.IsNullOrEmpty(fileName) || fileName.Contains("://")) { return(ItemCheckResult.Continue); } FileInfo file = new FileInfo(fileName); if (file.Exists && file.Length < (1024 * 3)) { Declaration dec = url.FindType <Declaration>(); if (dec != null && dec.PropertyName != null && dec.PropertyName.Text[0] != '*' && dec.PropertyName.Text[0] != '_') { string error = string.Format(CultureInfo.CurrentCulture, Resources.PerformanceEmbedImageAsDataUri, file.Length); context.AddError(new SimpleErrorTag(url.UrlString, error)); } } return(ItemCheckResult.Continue); }
public ActionResult Index(string shortenUrl) { UrlItem item = _urlRepo.GetById(UrlGenerator.Decode(shortenUrl)) ?? _urlRepo.All().FirstOrDefault(i => i.CustomUrl.Equals(shortenUrl)); if (item == null) { return(View("NotFound")); } if (IsExpired(item)) { _urlRepo.Delete(item); return(View("Expired")); } var urlHit = new UrlHit { UrlItemId = item.Id, ClientIp = Request.UserHostAddress }; _urlRepo.Update(item); _hitRepo.Add(urlHit); return(Redirect(item.OriginUrl)); }
public void SetSettings(UrlItem urlItem, HostSetting settings) { try { using (var ctx = CreateContext()) { var existing = ctx.HostSettings.SingleOrDefault(s => s.Host == urlItem.Host); var newSettings = existing ?? settings; newSettings.Host = urlItem.Host; // ensure the host is correct // update if exists if (existing != null) { newSettings.CrawlDelay = settings.CrawlDelay; newSettings.RobotsTxt = settings.RobotsTxt; newSettings.Disallow = settings.Disallow; } ctx.HostSettings.Add(newSettings); ctx.Commit(); } _logger.SettingsStored(urlItem); } catch (Exception err) { _logger.LogError(urlItem, err); throw; } }
public async Task <ActionResult> DeleteConfirmed(string id) { UrlItem urlItem = await db.UrlItems.FindAsync(id); db.UrlItems.Remove(urlItem); await db.SaveChangesAsync(); return(RedirectToAction("Index")); }
public static bool IsDataUri(this UrlItem item) { if (item.UrlString == null || string.IsNullOrEmpty(item.UrlString.Text)) { return(false); } return(item.UrlString.Text.Contains(";base64,")); }
public static string apply_taoke_url_item(CmsForm cmsForm, ContentItem contentItem , UrlItem urlItem, string id, string pid) { string click_url = UserUtil.query_click_url(cmsForm, pid, id, urlItem.url); urlItem.click_url = click_url; return(click_url); }
public EmbedSmartTagAction(ITrackingSpan span, UrlItem url) { _span = span; _url = url; if (Icon == null) { Icon = BitmapFrame.Create(new Uri("pack://application:,,,/WebEssentials2015;component/Resources/Images/embed.png", UriKind.RelativeOrAbsolute)); } }
public EmbedSmartTagAction(ITrackingSpan span, UrlItem url) { _span = span; _url = url; if (Icon == null) { Icon = BitmapFrame.Create(new Uri("pack://application:,,,/WebEssentials2013;component/Resources/Images/embed.png", UriKind.RelativeOrAbsolute)); } }
public async Task <ActionResult> Edit([Bind(Include = "Id,Link")] UrlItem urlItem) { if (ModelState.IsValid) { db.Entry(urlItem).State = EntityState.Modified; await db.SaveChangesAsync(); return(RedirectToAction("Index")); } return(View(urlItem)); }
public void SettingsStored(UrlItem url) { using (var ctx = CreateContext()) { ctx.ActivityMessages.Add(new ActivityMessage { Message = string.Format(LogMessages.SettingsStored, url) }); ctx.Commit(); } }
public async Task <ActionResult> Create([Bind(Include = "Id,Link")] UrlItem urlItem) { if (ModelState.IsValid) { db.UrlItems.Add(urlItem); await db.SaveChangesAsync(); return(RedirectToAction("Index")); } return(View(urlItem)); }
public void LogError(UrlItem urlItem, Exception err) { using (var ctx = CreateContext()) { ctx.ActivityMessages.Add(new ActivityMessage { Type = ActivityMessageType.Error, Message = string.Format(LogMessages.LogErrorUrlItem, urlItem.Id, err) }); ctx.Commit(); } }
public static UrlItem Create(string url, DateTime nextAvailableTime) { var uri = new Uri(url); var urlItem = new UrlItem { Url = url, Host = uri.Host, EvaliableFromDate = nextAvailableTime }; return(urlItem); }
public void Add() { var item = new UrlItem { ExpireMode = (short)ExpireMode.Never, OriginUrl = "http://www.google.com/" }; _repo.Add(item); UrlItem storedItem = _repo.GetById(UrlGenerator.Decode(item.CustomUrl)); Assert.That(storedItem, Is.Not.Null); }
internal async Task CreateAsync(UrlItem urlItem) { // Add item to database _dbContext.Urls.Add(urlItem); var success = false; Exception?lastException = null; // Try max 10 times before giving up string key = null; for (var i = 0; i < 10; i++) { // Create random id key = Helpers.GetRandomKey(_config.Url); // Try to add item to database urlItem.Key = key; try { await _dbContext.SaveChangesAsync(); success = true; } catch (DbUpdateException dbUpdateException) when((dbUpdateException.InnerException as SqlException)?.Number == 2601) { // Already exists: We need to retry _logger.LogInformation(dbUpdateException, $"Collision creating ShortUrl: {urlItem.Key}"); lastException = dbUpdateException; continue; } catch (Exception exception) { // Not sure what is wrong, log it and try again _logger.LogError(exception, $"Creating new ShortUrl: {JsonSerializer.Serialize(urlItem)}"); lastException = exception; continue; } } // Not success: Throw last error if (!success) { throw lastException !; } // Success: Add item to recent used cache since it will probably be used immediately var cacheEntryOptions = new MemoryCacheEntryOptions(); cacheEntryOptions.Size = 1; _cache.Set <UrlItem>(key, urlItem, cacheEntryOptions); }
/// <summary> /// 获取 UrlItem /// </summary> /// <param name="xmlNode"></param> /// <returns></returns> private UrlItem CreateUrlItem(XmlNode xmlNode) { if (xmlNode == null) { return(null); } UrlItem urlItem = new UrlItem(); urlItem.RegexString = xmlNode.SelectSingleNode("RegexString").InnerText; urlItem.UrlFomart = xmlNode.SelectSingleNode("UrlFomart").InnerText; urlItem.TitleGroupIndex = int.Parse(xmlNode.SelectSingleNode("TitleGroupIndex").InnerText); urlItem.UrlGroupIndex = int.Parse(xmlNode.SelectSingleNode("UrlGroupIndex").InnerText); return(urlItem); }
// GET: Url/Delete/5 public async Task <ActionResult> Delete(string id) { if (id == null) { return(new HttpStatusCodeResult(HttpStatusCode.BadRequest)); } UrlItem urlItem = await db.UrlItems.FindAsync(id); if (urlItem == null) { return(HttpNotFound()); } return(View(urlItem)); }
public async Task <CreateResponse> Create([FromBody] CreateRequest request) { // Security: Request will only be allowed if "RequireTokenForGet" is false OR request.AuthToken is in list or in database if (_config.Security.RequireTokenForGet // Not in allowed list in config && _config.Security.AuthenticationTokens != null && !_config.Security.AuthenticationTokens.Contains(request.AuthToken) // And not in database && !_dbContext.AuthTokens.Any(w => w.AuthToken == request.AuthToken && w.CanCreate) ) { // Not accepted return(new CreateResponse() { Success = false, ErrorMessage = "Invalid authentication token" }); } // Create item var urlItem = new UrlItem() { Key = Helpers.GetRandomKey(_config.Url), Expires = request.Expires, Url = request.Url, Metadata = request.Metadata }; try { await _shortUrlService.CreateAsync(urlItem); } catch (Exception ex) { _logger.LogError(ex, $"API: Creating short url for: {request.Url}"); return(new CreateResponse() { Success = false, ErrorMessage = "Error: Have admin check logs for more information." }); } return(new CreateResponse() { Success = true, Key = urlItem.Key, ShortUrl = Helpers.GetShortUrl(_config.Url.OverrideUrl, Request, urlItem.Key), Expires = request.Expires }); }
public static void parseContentUrlList(CouponContent contentItem, CmsForm cmsForm, string content) { try { ArrayList arrayLists1 = new ArrayList(); MatchCollection matchCollections = (new Regex(Constants.regex_url)).Matches(content); int num = 1; foreach (Match match in matchCollections) { LogUtil.log_call(cmsForm, string.Concat("正在处理第【", num, "】【", match.Value.ToString(), "】个链接!")); UrlItem urlItem = new UrlItem() { ori_url = match.Value.ToString() }; if ((urlItem.ori_url.Contains("activityId") && urlItem.ori_url.Contains("uland.taobao.com") && !urlItem.ori_url.Contains("uland.taobao.com/quan/detail")) || (((urlItem.ori_url.Contains("taobao.com") || urlItem.ori_url.Contains("tmall.com") || urlItem.ori_url.Contains("yao.95095.com")) && urlItem.ori_url.Contains("item.htm") && urlItem.ori_url.Contains("id=")))) { string num_iid = TaobaoUtil.get_num_iid(urlItem.ori_url); if (!string.IsNullOrEmpty(num_iid) && string.IsNullOrEmpty(contentItem.num_iid)) { contentItem.num_iid = num_iid; } } else { string str = TaobaoUtil.get_redirect_url(urlItem.ori_url, urlItem.ori_url); if ((urlItem.ori_url.Contains("activityId") && urlItem.ori_url.Contains("uland.taobao.com") && !urlItem.ori_url.Contains("uland.taobao.com/quan/detail")) || (((urlItem.ori_url.Contains("taobao.com") || urlItem.ori_url.Contains("tmall.com") || urlItem.ori_url.Contains("yao.95095.com")) && urlItem.ori_url.Contains("item.htm") && urlItem.ori_url.Contains("id=")))) { string num_iid = TaobaoUtil.get_num_iid(urlItem.ori_url); if (!string.IsNullOrEmpty(num_iid) && string.IsNullOrEmpty(contentItem.num_iid)) { contentItem.num_iid = num_iid; } } } num++; } } catch (Exception exception) { LogUtil.log_call(cmsForm, string.Concat("[parseUrl]出错!", exception.ToString())); } }
public IEnumerable <ISmartTagAction> GetSmartTagActions(ParseItem item, int position, ITrackingSpan itemTrackingSpan, ITextView view) { UrlItem url = (UrlItem)item; if (!url.IsValid || url.UrlString == null) { yield break; } if (url.UrlString.Text.Contains(";base64,")) { yield return(new ReverseEmbedSmartTagAction(itemTrackingSpan, url)); } }
public void Should_create_new_job_for_url_which_has_inactive_job() { const string testUrl = "http://test.com"; const string testHost = "test.com"; using (_db.CreateTransaction()) { var urlItem = new UrlItem { Url = testUrl, Host = testHost }; // add finished job using (var ctx = _db.CreateDbContext()) { ctx.JobItems.Add(new JobItem { Id = Guid.NewGuid(), DateStart = DateTime.Now.AddDays(-2), DateFinish = DateTime.Now.AddDays(-2), // make sure the job is finished Url = urlItem }); ctx.Commit(); } #region action var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>()); var job = jobRep.Start(urlItem); #endregion #region assertion // make sure the job is created in DB using (var ctx = _db.CreateDbContext()) { // try to find unfinished job for the url var actualJob = ctx.JobItems.Include(j => j.Url).SingleOrDefault(j => j.DateFinish.HasValue == false); Assert.NotNull(actualJob); Assert.Equal(job.Id, actualJob.Id); Assert.Equal(testUrl, actualJob.Url.Url); Assert.False(actualJob.DateFinish.HasValue); } #endregion } }