public CommentController(CommentService commentService, PersonService personService, HtmlService htmlService, PostService postService)
{
this.commentService = commentService ?? throw new ArgumentNullException(nameof(commentService));
this.personService = personService ?? throw new ArgumentNullException(nameof(personService));
this.htmlService = htmlService ?? throw new ArgumentNullException(nameof(htmlService));
this.postService = postService ?? throw new ArgumentNullException(nameof(postService));
}
public void _01_ETF50Tests()
{
var htmlService = new HtmlService();
var instrumentService = new InstrumentService(htmlService);
var etf50Insts = instrumentService.GetInstruments(SinaInstrumentCategory.Index.Index_Sh_50).Take(10);
// Category
var categService = new CategorizationService(htmlService);
var categorizes = new HashSet <string>();
foreach (var inst in etf50Insts)
{
var cates = categService.GetCateogories(inst.SymbolShort);
foreach (var cate in cates)
{
categorizes.Add(cate);
}
inst.AsDynamic().Categories = cates;
}
// Equity structure
var structureService = new EquityStructureService(htmlService);
foreach (var inst in etf50Insts)
{
var structure = structureService.GetEquityStructure(inst.SymbolShort);
inst.AsDynamic().EquityStructure = structure;
}
var str = JsonUtil.Serialize(etf50Insts);
}
public string BookmarkSelector()
{
var request = _context.Request;
var pageId = request["pageId"];
if (string.IsNullOrEmpty(pageId)) return string.Empty;
var selectedBookmark = request["bookmark"];
var generator = new HtmlService();
var html = generator.HtmlFor(new PageReference(pageId));
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(html);
var bookmarksLinks = htmlDocument.DocumentNode
.Descendants("a")
.Where(n => n.InnerText == string.Empty && n.Attributes.Contains("name"))
.ToArray();
var bookmarks = bookmarksLinks.Select(lnk => lnk.Attributes.First(a => a.Name == "name").Value).ToArray();
const string optionFormat = "<option value='{0}'{1}>{2}</option>";
const string selectedAttribute = " selected='selected'";
var builder = new StringBuilder();
builder.AppendLine("<div id='ddlBookmarkSelectorContainer'>");
builder.AppendLine("<label class='episize100 epiindent bookmark-selector-label' for='ddlBookmarkSelector'>Bookmark:</label>");
builder.AppendLine("<select id='ddlBookmarkSelector' class='episize240' name='ddlBookmarkSelector'>");
builder.AppendFormat(optionFormat, DefaultBookmark, bookmarks.All(b => b != selectedBookmark) ? selectedAttribute : string.Empty, DefaultBookmark);
foreach(var bookmark in bookmarks)
{
builder.AppendFormat(optionFormat, bookmark, bookmark == selectedBookmark ? selectedAttribute : string.Empty, bookmark);
}
builder.AppendLine("</select>");
builder.AppendLine("</div>");
return builder.ToString();
}
public void ParseChildUrl(int id, string url, bool isCritical, string pattern, int depth, int maxDepth)
{
var job = ParserJobsRepository.GetById(id);
if (maxDepth <= depth)
{
return;
}
var document = HtmlService.Load(url);
var count = HtmlService.CountText(document, pattern);
ParserJobsRepository.UpdateCount(job, count);
var childUrls = HtmlService.GetUrls(document);
var jobBatchId = StartBatchJobs(id, childUrls, isCritical, pattern, depth + 1, maxDepth);
if (depth + 1 > maxDepth)
{
BatchJobClient.AwaitBatch(jobBatchId, x =>
{
if (isCritical)
{
x.Enqueue(() => SetCriticalJobStatus(id, ParserJobStatus.Completed));
}
else
{
x.Enqueue(() => SetJobStatus(id, ParserJobStatus.Completed));
}
});
}
}
public void GetTitleTest()
{
IHtmlService htmlService = new HtmlService();
const string htmlDocument = "<!DOCTYPE html><html><head><title>Title of HTML Document</title></head><body><h1>Heading</h1><p>Paragraph.</p></body></html>";
Assert.AreEqual("Title of HTML Document", htmlService.GetTitle(htmlDocument));
}
internal static void EnsureInitialized()
{
if (_instances == null)
{
lock (_lockObject)
if (_instances == null)
{
var localInstances = new Dictionary <Type, object>();
var pageService = new PageService();
#region Service Bindings
var mobileConfiguration = new MobileConfiguration();
var mobileRedirectionService = new MobileRedirectionService(mobileConfiguration);
localInstances[typeof(IMobileConfiguration)] = mobileConfiguration;
localInstances[typeof(IPageRouteRegistrar)] = new PageRouteRegistrar(pageService, mobileConfiguration);
localInstances[typeof(IPageService)] = pageService;
localInstances[typeof(IConfiguredContentFragmentService)] = pageService;
localInstances[typeof(IMobileRedirectionService)] = mobileRedirectionService;
localInstances[typeof(IHtmlService)] = new HtmlService(mobileConfiguration);
#endregion
_instances = localInstances;
}
}
}
public void ParseUrl(int id)
{
var job = ParserJobsRepository.GetById(id);
var document = HtmlService.Load(job.Url);
job.Count = HtmlService.CountText(document, job.Pattern);
job.Status = ParserJobStatus.Processing;
ParserJobsRepository.SaveChanges();
var childUrls = HtmlService.GetUrls(document);
var jobBatchId = StartBatchJobs(id, childUrls, job.IsCritical, job.Pattern, 1, job.Depth);
if (job.Depth == 1)
{
BatchJobClient.AwaitBatch(jobBatchId, x =>
{
if (job.IsCritical)
{
x.Enqueue(() => SetCriticalJobStatus(job.Id, ParserJobStatus.Completed));
}
else
{
x.Enqueue(() => SetJobStatus(job.Id, ParserJobStatus.Completed));
}
});
}
}
public void _02_HistoricalPriceServiceTest_ByDate()
{
var begin = new DateTime(2017, 5, 9);
var end = new DateTime(2017, 5, 9);
var html = new HtmlService();
var service = new PriceHistoryService(html);
// test Begin
var prices = service.GetDailyTadingInfo("600030", begin).ToArray();
Assert.IsTrue(prices.Length > 0);
Assert.IsTrue(prices.Min(p => p.Time) == begin);
// test end
//prices = service.GetDailyTadingInfo("600030", null, end).ToArray();
//Assert.IsTrue(prices.Length > 0);
//Assert.IsTrue(prices.Max(p => p.Time) == begin);
// test begin + end
prices = service.GetDailyTadingInfo("600030", begin, begin).ToArray();
Assert.IsTrue(prices.Length == 1);
Assert.IsTrue(prices.Min(p => p.Time) == begin);
Assert.IsTrue(prices.Max(p => p.Time) == begin);
prices = service.GetDailyTadingInfo("600030", begin, end.AddMonths(7)).ToArray();
Assert.IsTrue(prices.Length > 1);
Assert.IsTrue(prices.Min(p => p.Time) == begin);
Assert.IsTrue(prices.Max(p => p.Time) == end.AddMonths(7));
}
public UrlImportMap(Database db, string ConnectionString, Item importItem, ILogger l)
: base(db, ConnectionString, importItem, l)
{
ImportItem = importItem;
ProcessorService = new ProcessorService(l);
HtmlService = new HtmlService(l);
}
public void _01_InstrumentServiceTest_GetInstruments()
{
var html = new HtmlService();
var service = new InstrumentService(html);
var instruments = service.GetInstruments();
Assert.IsTrue(instruments.Length > 2000);
}
public void _02_PnlTableServiceTest()
{
var html = new HtmlService();
var service = new PnlService(html);
var years = service.GetYears("600030");
Assert.IsTrue(years.Count() > 0);
}
public void _01_EquityStructureServiceTest()
{
var html = new HtmlService();
var service = new EquityStructureService(html);
var structures = service.GetEquityStructure("600030");
Assert.IsTrue(structures.Count() > 0);
}
public void _01_PnlTableServiceTest()
{
var html = new HtmlService();
var service = new PnlService(html);
var pnl = service.GetPnlTable("600030", 2018);
Assert.IsTrue(pnl.Count() > 0);
}
public void _02_CategorizationServiceTest()
{
var html = new HtmlService();
var service = new CategorizationService(html);
var categories = service.GetCateogories("600519");
Assert.IsTrue(categories.Count() > 0);
}
public void _01_HistoricalPriceServiceTest()
{
var html = new HtmlService();
var service = new PriceHistoryService(html);
var prices = service.GetDailyTadingInfo("600030", 2018, 3).ToArray();
Assert.IsTrue(prices.Length > 0);
}
static async Task Main(string[] args)
{
FileReader handler = new FileReader();
HtmlService htmlService = new HtmlService();
Console.WriteLine("Enter full path to file...");
string path = Console.ReadLine();
IEnumerable<string> companies = handler.ReadAllRows(path);
decimal counter = 0;
var itemList = new List<ProfileListItem>();
foreach (var companyName in companies)
{
Thread.Sleep(2000);
var html = await htmlService.GetDocument($"https://www.allabolag.se/what/{companyName}");
try
{
Result<ProfileListItem> item = new ProfileBuilder().Build(new ProfileListItemStrategy(html));
counter++;
Console.Clear();
Console.WriteLine($"{(counter / companies.Count()):P2}");
}
catch (AggregateException)
{
itemList.Add(new ProfileListItem { Header = companyName });
}
}
using (StreamWriter outputFile = new StreamWriter("C:\\Users\\F.Boethius-Fjarem\\source\\repos\\AllaBolag\\AllaBolag.Infrastructure.Tests\\Companies_Output.csv"))
{
outputFile.WriteLine($"Bolagsnamn\tOrg.Nr\tCEO\tTelefon\tAntal Anställda\tOmsättning(tkr)\tBRUTTOVINSTMARGINAL\tVINSTMARGINAL\tKASSALIKVIDITET\tSOLIDITET");
counter = 0;
foreach (ProfileListItem item in itemList.Where(i => !string.IsNullOrEmpty(i.ProfileLink)))
{
counter++;
Console.Clear();
Console.WriteLine($"{counter / itemList.Count:P4}");
try
{
Task<HtmlResult> document = htmlService.GetDocument(item.ProfileLink);
string result = new ProfileBuilder().Build(new ProfileItemStrategy(document.Result)).ToString();
outputFile.WriteLine(result);
}
catch (Exception)
{
outputFile.WriteLine(new ProfileItem { CompanyName = item.Header }.ToString());
}
Thread.Sleep(2000);
}
}
}
public MagnetScrapeTests(ITestOutputHelper output)
{
_output = output;
_logger = new LoggerBuilder <MagnetScrapeService>().Build();
_magnetSettings = new MagnetSettingBuilder().Build();
_commonSettings = new CommonSettingBuilder().Build();
_htmlService = new HtmlService(null, _commonSettings);
_movieService = new MovieServiceBuilder().Build();
_movieMagService = new MovieMagServiceBuilder().Build();
_magnetScrapeService = new MagnetScrapeService(_logger, _magnetSettings, _movieService, _movieMagService, _htmlService);
}
public ToRichText(Item i, ILogger l) : base(i, l)
{
Assert.IsNotNull(i, "i");
//store fields
UnwantedTags = GetItemField(i, "Unwanted Tags").Split(comSplitr, StringSplitOptions.RemoveEmptyEntries);
UnwantedAttributes = GetItemField(i, "Unwanted Attributes")
.Split(comSplitr, StringSplitOptions.RemoveEmptyEntries);
MediaParentItem = i.Database.GetItem(GetItemField(i, "Media Parent Item"));
FromDB = Factory.GetDatabase("master");
HtmlService = new HtmlService(l);
MediaService = new MediaService(l);
}
public ActivationsController(MyContext context
, HtmlService htmlService
, IOptions <EmailSettings> emailSettings
, EmailService emailService
, LicenceService licenceService
) : base(context)
{
_emailService = emailService;
_htmlService = htmlService;
_emailSettings = emailSettings.Value;
_licenceService = licenceService;
}
public List <string> getPrizeNumber(string year, string month)
{
month = _regularization.regularizeMonth(month);
string urlAddress = "https://www.etax.nat.gov.tw/etw-main/web/ETW183W2_" +
year + month + "/";
WebService webService = new WebService();
string html = webService.getHTML(urlAddress, Encoding.UTF8);
if (html != "")
{
HtmlService htmlService = new HtmlService(html);
result = htmlService.getPrizeNumber();
}
return(result);
}
public AccountsController(MyContext context
, EmailService emailService
, AccountService accountService
, TokkenHandler tokkenHandler
, HtmlService htmlService
, IOptions <EmailSettings> emailSettings
)
: base(context)
{
_emailService = emailService;
_accountService = accountService;
_tokkenHandler = tokkenHandler;
_htmlService = htmlService;
_emailSettings = emailSettings.Value;
}
public JavScrapeTests(ITestOutputHelper output)
{
_output = output;
_logger = new LoggerBuilder <JavScrapeService>().Build();
_javlibSettings = new JavlibSettingBuilder().Build();
_commonSettings = new CommonSettingBuilder().Build();
_htmlService = new HtmlService(null, _commonSettings);
_movieService = new MovieServiceBuilder().Build();
_categoryService = new CategoryServiceBuilder().Build();
_companyService = new CompanyServiceBuilder().Build();
_directorService = new DirectorServiceBuilder().Build();
_actorService = new ActorServiceBuilder().Build();
_publisherService = new PublisherServiceBuilder().Build();
_javScrapeService = new JavScrapeService(_logger, _javlibSettings, _movieService, _actorService, _categoryService, _companyService, _directorService, _publisherService, _htmlService);
}
static void Main()
{
Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false);
MainForm form = new MainForm();
IJsonService jsonService = new JsonService();
IHtmlService htmlService = new HtmlService();
IHtmlParser htmlParser = new HtmlParser();
ILoggerFactory loggerFactory = new LoggerFactory();
var presenter = new MainPresenter(form, jsonService, htmlService, htmlParser, loggerFactory);
Application.Run(form);
}
public static CacheResponse IsCached(List <string> urls)
{
var queryString = "?";
bool isFirst = true;
foreach (var url in urls)
{
if (!isFirst)
{
queryString += "&";
}
else
{
isFirst = !isFirst;
}
queryString += "items[]=" + url;
}
return(HtmlService.PostJson <CacheResponse>(new Uri("https://www.premiumize.me/api/cache/check"), queryString));
}
public string CleanHtml(IDataMap map, string itemPath, string html)
{
if (String.IsNullOrEmpty(html))
{
return(html);
}
var document = new HtmlDocument();
document.LoadHtml(html);
HtmlNodeCollection tryGetNodes = document.DocumentNode.SelectNodes("./*|./text()");
if (tryGetNodes == null || !tryGetNodes.Any())
{
return(html);
}
var nodes = new Queue <HtmlNode>(tryGetNodes);
while (nodes.Any())
{
HandleNextNode(nodes, map, itemPath);
}
var cleanedHtml = document.DocumentNode.InnerHtml;
bool modified = false;
string fixedHtml = HtmlService.FixOrphanedText(cleanedHtml, out modified);
if (modified)
{
map.Logger.Log("Fixed Orphaned Text in Rich Text.", itemPath);
}
return(fixedHtml);
}
public void Sanitize(string htmlToEscape)
{
// Arrange
var htmlService = new HtmlService();
var rawHtml =
@"<script>alert('xss')</script><div onload=""alert('xss')"""
+ @"style=""background-color: test"">Test<img src=""test.gif"""
+ @"style=""background-image: url(javascript:alert('xss'));"
+ @"margin: 10px""></div>";
var expectedContent =
@"<div style=""background-color: test"">"
+ @"Test<img src="""
+ @"test.gif"""
+ @" style=""margin: 10px""></div>";
// Act
var resultContent = htmlService.Sanitize(rawHtml);
// Assert
Assert.DoesNotContain(htmlToEscape, resultContent);
Assert.Equal(expectedContent, resultContent);
}
public static CacheResponse IsCached(string url)
{
var queryString = "?" + "items[]=" + url;
return(HtmlService.PostJson <CacheResponse>(new Uri("https://www.premiumize.me/api/cache/check"), queryString));
}
public PnlService(HtmlService service) : base(service)
{
}
public void _01_TestInfoPrice()
{
var htmlService = new HtmlService();
var infoPriceService = new PriceService(htmlService);
var ret = infoPriceService.GetInfoPrices("sh600030", "sh600001").ToArray();
}
public CommentService(DbFactory dbFactory, HtmlService htmlService)
{
this.dbFactory = dbFactory ?? throw new ArgumentNullException(nameof(dbFactory));
this.htmlService = htmlService ?? throw new ArgumentNullException(nameof(htmlService));
}
