static async Task Main(params string[] args) { string broker = args[0]; string topic = args[1]; int partitionsCount = int.Parse(args[2]); var sections = args.Skip(3); var logger = new ConsoleLogger(); s_producer = new KafkaProducerBase <string, Product>(logger, broker, topic, partitionsCount); LoaderOptions lo = new LoaderOptions() { IsNavigationDisabled = false, IsResourceLoadingEnabled = false }; //Use the default configuration for AngleSharp var config = Configuration.Default.WithDefaultLoader(lo); foreach (var category in sections) { await ProcessSection(config, category, s_sections[category]); } }
//public readonly MyHttpClient HttpClient; //public Browser(MyHttpClient httpClient) public Browser() { //HttpClient = httpClient; //We require a custom configuration var config = Configuration.Default.WithJs(); //Create a new context for evaluating webpages with the given config var context = BrowsingContext.New(config); LoaderOptions lo = new LoaderOptions(); lo.IsNavigationDisabled = false; lo.IsResourceLoadingEnabled = true; //var requester = new HttpClientRequester(httpClient); var configuration = Configuration.Default .WithDefaultLoader(lo) .WithJs() .WithPersistentCookies("d:\\logs") .WithRequesters(new MyClientHandler()) //.WithCss() //.WithCookies() ; Context = BrowsingContext.New(configuration); }
private IEnumerable <IContent> GetChildren( ContentReference contentReference, LoaderOptions loaderOptions, int startIndex, int maxRows, out bool hasNextPage) { hasNextPage = false; var items = _contentLoader .GetChildren <IContent>( contentReference, loaderOptions, startIndex, maxRows ) .FilterHiddenGraphTypes(); if (items.Count() < maxRows) { return(items); } hasNextPage = _contentLoader .GetChildren <IContent>( contentReference, loaderOptions, startIndex, 1 ) .FilterHiddenGraphTypes() .Any(); return(items); }
private LoaderOptions CreateDefaultListOption() { LoaderOptions loaderOptions = new LoaderOptions(); loaderOptions.Add <LanguageLoaderOption>(LanguageLoaderOption.Fallback(_languageAccessor.Language)); return(loaderOptions); }
public virtual IContent GetLocalizationPage(string[] normalizedKey, LoaderOptions loaderOptions) { ContentReference cachedLocalizationPageReference; var basePath = normalizedKey[1]; if (CachedLocalizationPages.TryGetValue(basePath, out cachedLocalizationPageReference)) { try { return(_contentRepository.Service.Get <IContent>(cachedLocalizationPageReference, loaderOptions)); } catch (PageNotFoundException) { // this can happen when we cached the localization pagereference but the page has been removed mean while. ContentReference removedReference; CachedLocalizationPages.TryRemove(basePath, out removedReference); } } var localizationPages = GetAllLocalizationPages(loaderOptions); var localizationPage = localizationPages.SingleOrDefault( x => x.Property.GetPropertyValue <string>("BasePath").Equals(basePath, StringComparison.OrdinalIgnoreCase)); if (localizationPage != null) { CachedLocalizationPages.TryAdd(basePath, localizationPage.ContentLink); } return(localizationPage); }
public void TestReadingEncodedFile(string fileName, System.Text.Encoding encoding, System.Text.Encoding defaultEncoding) { // Set some options, so that we can know if things are working... LoaderOptions loaderOptions = new LoaderOptions(); loaderOptions.DetectEncoding = true; loaderOptions.DefaultEncoding = defaultEncoding; loaderOptions.ParserOptions.IncludeMetaData = true; // Load multi-byte html file into memory XmlDocument doc = XHtmlLoader.LoadHtml(_sampleMultiByteHtml); // Ensure Sample directory exists string sampleDir = (new DirectoryInfo(AssemblyDirectory)).Parent.Parent.Parent.FullName + "\\SampleData\\"; if (!Directory.Exists(sampleDir)) { Directory.CreateDirectory(sampleDir); } // Create Encoded file string fullName = sampleDir + fileName; using (TextWriter sw = new StreamWriter(File.Create(fullName), encoding)) { doc.Save(sw); } // Re-load into memory XmlDocument doc2 = XHtmlLoader.LoadWebPageAsync("file://" + fullName, loaderOptions).Result; Console.WriteLine("Reading file: " + fileName); Console.WriteLine(doc2.OuterXml); Assert.AreEqual(doc.SelectSingleNode("//body").OuterXml, doc2.SelectSingleNode("//body").OuterXml); }
public TTarget GetContent <TTarget>(DocumentIndexModel indexItem, bool filterOnCulture = true) where TTarget : ContentData { if (indexItem == null || string.IsNullOrEmpty(indexItem.Id)) { return(default(TTarget)); } Guid result; if (Guid.TryParse(((IEnumerable <string>)indexItem.Id.Split('|')).FirstOrDefault <string>(), out result)) { LoaderOptions loaderOptions; if (filterOnCulture) { loaderOptions = this.GetLoaderOptions(indexItem.Language); } else { loaderOptions = new LoaderOptions(); loaderOptions.Add <LanguageLoaderOption>(LanguageLoaderOption.Fallback((CultureInfo)null)); } LoaderOptions settings = loaderOptions; TTarget content = null; this._contentRepository.TryGet <TTarget>(result, settings, out content); return(content); } return(default(TTarget)); }
public virtual IEnumerable <T> GetReferencesToCategories <T>(IEnumerable <ContentReference> categories, CultureInfo culture) where T : ICategorizableContent, IContentData { var loaderOptions = new LoaderOptions { LanguageLoaderOption.Specific(culture) }; return(GetReferencesToCategories <T>(categories, loaderOptions)); }
protected Unifiable ProcessLoadProc(LoaderOptions loaderOptions) { if (templateNode.Name.ToLower() == "category") { request.Loader.processCategory(templateNode, loaderOptions.CurrentFilename); } return(Unifiable.Empty); }
public T GetFirstBySegment <T>(string urlSegment, CultureInfo culture) where T : CategoryData { var loaderOptions = new LoaderOptions { LanguageLoaderOption.Specific(culture) }; return(GetFirstBySegment <T>(urlSegment, loaderOptions)); }
private LoaderOptions CreateDefaultLoadOption() { LoaderOptions loaderOptions = new LoaderOptions { LanguageLoaderOption.FallbackWithMaster(_languageAccessor.Language) }; return(loaderOptions); }
public LoaderOptionsAttribute(string requiredVersion, string minimalMirandaVersion, LoaderOptions options) { if (!String.IsNullOrEmpty(requiredVersion)) this.requiredVersion = new Version(requiredVersion); if (!String.IsNullOrEmpty(minimalMirandaVersion)) this.minimalMirandaVersion = new Version(minimalMirandaVersion); this.options = options; }
public FetchService(Database database) { var config = new LoaderOptions(); IConfiguration configuration = Configuration.Default .With(new DefaultHttpRequester()) .With <IDocumentLoader>(ctx => new CrabtopusDocumentLoader(ctx, config.Filter)) .With <IResourceLoader>(ctx => new DefaultResourceLoader(ctx, config.Filter)); _context = BrowsingContext.New(configuration); _database = database; }
public T GetFirstBySegment <T>(string urlSegment, LoaderOptions loaderOptions) where T : CategoryData { if (SiteDefinition.Current.SiteAssetsRoot != SiteDefinition.Current.GlobalAssetsRoot) { var firstSiteCategory = GetFirstBySegment <T>(ContentRepository.GetOrCreateSiteCategoriesRoot(), urlSegment, loaderOptions); if (firstSiteCategory != null) { return(firstSiteCategory); } } return(GetFirstBySegment <T>(ContentRepository.GetOrCreateGlobalCategoriesRoot(), urlSegment, loaderOptions)); }
private LoaderOptions GetLoaderOptions(string languageCode) { if (string.IsNullOrEmpty(languageCode)) { LoaderOptions loaderOptions = new LoaderOptions(); loaderOptions.Add <LanguageLoaderOption>(LanguageLoaderOption.FallbackWithMaster((CultureInfo)null)); return(loaderOptions); } CultureInfo language = languageCode == "iv" ? CultureInfo.InvariantCulture : CultureInfo.GetCultureInfo(languageCode); LoaderOptions loaderOptions1 = new LoaderOptions(); loaderOptions1.Add <LanguageLoaderOption>(LanguageLoaderOption.Specific(language)); return(loaderOptions1); }
public KeyProcessor(LoaderOptions options, GenericLoader g) { parent = g; Options = options; TruncExt = new SqlCommand("truncate table [ext].[" + Options.TableName + "Keys]"); KeyValues.Columns.Add("FileSourceId", typeof(string)); InsertFileSource = new SqlCommand("InsertFileSource"); InsertFileSource.Parameters.Add("@filename", SqlDbType.NVarChar); InsertFileSource.Parameters.Add("@filesize", SqlDbType.Int); InsertFileSource.CommandType = CommandType.StoredProcedure; }
public CryptoInfo(LoaderOptions options) { this.options = options; Log.Info("Loading encryption keys."); var stopwatch = new Stopwatch(); stopwatch.Start(); var parsedD20Engine = new ParsedD20RulesEngine(Path.Combine(options.CBPath, "D20RulesEngine.dll")); this.expectedDemoHash = parsedD20Engine.expectedDemoHash; this.expectedNormalHash = parsedD20Engine.expectedNormalHash; // This is intentionally loaded first, so there's a chance to override it from other sources. if (options.KeyFile != null) { ProcessUpdateFile(CB_APP_ID, options.KeyFile); } string heroicDemoPath = Path.Combine(options.CBPath, "HeroicDemo.update"); if (File.Exists(heroicDemoPath)) { ProcessUpdateFile(CB_APP_ID, heroicDemoPath, true); } LoadRegistryKeys(CB_APP_ID); var regPatcherPath = Path.Combine(options.CBPath, "RegPatcher.dat"); if (File.Exists(regPatcherPath)) { Log.Debug($" - Loading default key from {regPatcherPath}"); keyStore.FallbackKey = Convert.FromBase64String(File.ReadAllText(regPatcherPath)); } if (options.WriteKeyFile) { SaveKeyFile(); } stopwatch.Stop(); Log.Debug($"Finished in {stopwatch.ElapsedMilliseconds} ms"); Log.Debug(); }
private static ContentReference GetOrCreateCategoriesRoot(this IContentRepository contentRepository, ContentReference parentLink, string name, string routeSegment) { var loaderOptions = new LoaderOptions { LanguageLoaderOption.FallbackWithMaster() }; var rootCategory = contentRepository.GetChildren <CategoryRoot>(parentLink, loaderOptions).FirstOrDefault(); if (rootCategory != null) { return(rootCategory.ContentLink); } rootCategory = contentRepository.GetDefault <CategoryRoot>(parentLink); rootCategory.Name = name; rootCategory.RouteSegment = routeSegment; return(contentRepository.Save(rootCategory, SaveAction.Publish, AccessLevel.NoAccess)); }
public virtual ChangeTaskViewModel Map(ApprovalCommandBase approvalCommand, IPrincipal principal) { var commandViewModel1 = _mapper.Map(approvalCommand) as ChangeTaskViewModel; var name1 = approvalCommand is ICultureSpecificApprovalCommand specificApprovalCommand ? specificApprovalCommand.AppliedOnLanguageBranch : null; var cultureInfo = string.IsNullOrEmpty(name1) ? null : new CultureInfo(name1); var commandViewModel2 = commandViewModel1; var contentRepository = _contentRepository; var appliedOnContentLink = approvalCommand.AppliedOnContentLink; var settings = new LoaderOptions { new LanguageLoaderOption() { Language = cultureInfo, FallbackBehaviour = LanguageBehaviour.FallbackWithMaster } }; var name2 = contentRepository.Get <IContent>(appliedOnContentLink, settings)?.Name; if (commandViewModel2 != null) { commandViewModel2.Name = name2; } if (commandViewModel1 != null) { commandViewModel1.Status = (int)approvalCommand.CommandStatus; commandViewModel1.CanExecute = approvalCommand.CommandStatus == CommandMetaData.ChangeTaskApprovalStatus.InReview; var fullName = approvalCommand.GetType().FullName; if (fullName != null) { commandViewModel1.TypeIdentifier = fullName.ToLower(); } commandViewModel1.Id = approvalCommand.Id.ExternalId.ToString(); commandViewModel1.IsCommandDataValid = approvalCommand.IsValid(); commandViewModel1.CreatedBy = _uiHelper.GetDisplayNameForUser(commandViewModel1.CreatedBy); commandViewModel1.ChangedBy = _uiHelper.GetDisplayNameForUser(commandViewModel1.ChangedBy); commandViewModel1.CanUserActOnHisOwnChanges = !string.Equals(approvalCommand.CreatedBy, principal.Identity.Name, StringComparison.OrdinalIgnoreCase); return(commandViewModel1); } return(null); }
private TaxonomyData GetTaxonomyDataRecursively(SegmentContext segmentContext) { var contentReference = this.taxonomyRoot; var loaderOptions = new LoaderOptions { LanguageLoaderOption.FallbackWithMaster(CultureInfo.GetCultureInfo(segmentContext.Language)) }; TaxonomyData taxonomyData = null; while (true) { var segment = segmentContext.GetNextValue(segmentContext.RemainingPath); if (string.IsNullOrEmpty(segment.Next)) { break; } var content = this.contentLoader.GetBySegment(contentReference, segment.Next, loaderOptions); if (content == null) { break; } contentReference = content.ContentLink; if (content is TaxonomyData) { taxonomyData = content as TaxonomyData; // Only set remaining path if we found what we were looking for, // otherwise let others try to parse the URL. segmentContext.RemainingPath = segment.Remaining; } } return(taxonomyData); }
public static IEnumerable <IContent> GetAllLanguageVersions(this IContentLoader contentLoader, ContentReference contentReference) { var content = contentLoader.Get <IContent>(contentReference); if (content is ILocalizable rootContent) { foreach (var cultureInfo in rootContent.ExistingLanguages) { var loaderOptions = new LoaderOptions { LanguageLoaderOption.Specific(cultureInfo) }; var contentInSpecificLanguage = contentLoader.Get <IContent>(contentReference.ToReferenceWithoutVersion(), loaderOptions); yield return(contentInSpecificLanguage); } } else { yield return(content); } }
public async Task DownloadResources() { var urls = new List <Url>(); var loaderOptions = new LoaderOptions { IsResourceLoadingEnabled = true, Filter = (req) => { urls.Add(req.Address); return(true); }, }; var config = Configuration.Default .WithDefaultLoader(loaderOptions) .WithCss(); var document = "<style>div { background: url('https://avatars1.githubusercontent.com/u/10828168?s=200&v=4'); }</style><div></div>".ToHtmlDocument(config); var tree = document.DefaultView.Render(); var node = tree.Find(document.QuerySelector("div")); await node.DownloadResources(); Assert.AreEqual(1, urls.Count); Assert.AreEqual("https://avatars1.githubusercontent.com/u/10828168?s=200&v=4", urls[0].Href); }
private TaxonomyData GetTaxonomyDataRecursively(SegmentContext segmentContext) { var contentReference = this.taxonomyRoot; var loaderOptions = new LoaderOptions { LanguageLoaderOption.FallbackWithMaster(CultureInfo.GetCultureInfo(segmentContext.Language)) }; TaxonomyData taxonomyData = null; while (true) { var segment = this.ReadNextSegment(segmentContext); if (string.IsNullOrEmpty(segment.Next)) { break; } var content = this.contentLoader.GetBySegment(contentReference, segment.Next, loaderOptions); if (content == null) { break; } contentReference = content.ContentLink; if (content is TaxonomyData) { taxonomyData = content as TaxonomyData; } } return(taxonomyData); }
static async Task Main(params string[] args) { string broker = args[0]; var sections = args.Skip(1); var logger = new ConsoleLogger(); s_producer = new KafkaProducerBase <string, Product>(logger, broker, MessagingSchema.RawProductsTopic, MessagingSchema.RawProductsPartitions); LoaderOptions lo = new LoaderOptions() { IsNavigationDisabled = false, IsResourceLoadingEnabled = true }; //Use the default configuration for AngleSharp var config = Configuration.Default.WithDefaultLoader(lo); foreach (var category in s_sections) { await ProcessSection(config, category.Key, category.Value); } }
public Connection <IContent> CreateIContentConnection( ResolveConnectionContext <IContent> context, LoaderOptions loaderOptions, int defaultPageSize) { int first = context.PageSize ?? defaultPageSize; int.TryParse(context.After, out int after); var items = GetChildren( context.Source.ContentLink, loaderOptions, after, first, out bool hasNextPage ); var pageInfo = CreatePageInfo(first, after, hasNextPage); int endCursor = int.Parse(pageInfo.EndCursor); var edges = new List <Edge <IContent> >(); for (int i = 0; i < items.Count(); i++) { edges.Add(new Edge <IContent> { Node = items.ElementAt(i), Cursor = ((endCursor - first) + i + 1).ToString() }); } return(new Connection <IContent> { TotalCount = items.Count(), Edges = edges, PageInfo = pageInfo }); }
public LoaderOptionsAttribute(LoaderOptions options) : this(null, null, options) { }
public virtual T Get <T>(ContentReference contentLink, LoaderOptions settings) where T : IContentData { throw new NotImplementedException(); }
public IEnumerable <T> GetChildrenWithReviews <T>( ContentReference contentLink, LoaderOptions loaderOptions, int startIndex, int maxRows) where T : IContentData { if (ContentReference.IsNullOrEmpty(contentLink)) { throw new ArgumentNullException(nameof(contentLink), "Parameter has no value set"); } if (!ExternalReview.IsInExternalReviewContext) { return(_contentLoader.GetChildren <T>(contentLink)); } ContentReference referenceWithoutVersion = contentLink.ToReferenceWithoutVersion(); if (referenceWithoutVersion == ContentReference.WasteBasket) { return(_contentLoader.GetChildren <T>(contentLink)); } var provider = _contentProviderManager.ProviderMap.GetProvider(referenceWithoutVersion); var parentContent = _contentLoader.Get <IContent>(referenceWithoutVersion, loaderOptions); var localizable = parentContent as ILocalizable; var languageID = localizable != null ? localizable.Language.Name : (string)null; var childrenReferences = provider.GetChildrenReferences <T>(referenceWithoutVersion, languageID, startIndex, maxRows); var result = new List <ContentReference>(); foreach (var childReference in childrenReferences) { var referenceToLoad = LoadUnpublishedVersion(childReference.ContentLink); if (referenceToLoad == null) { result.Add(childReference.ContentLink); } else { var content = _contentLoader.Get <T>(referenceToLoad); if (HasExpired(content as IVersionable)) { continue; } if ((content as IContent).IsPublished()) { // for published version return the original method result result.Add(childReference.ContentLink); continue; } result.Add((content as IContent).ContentLink); } } var childrenWithReviews = result.Select(_contentLoader.Get <T>).ToList(); if (childrenWithReviews.Count > 0) { var pageData = parentContent as PageData; if (pageData != null && pageData.ChildSortOrder == FilterSortOrder.Alphabetical && (startIndex == -1 && maxRows == -1 || childrenWithReviews.Count < maxRows && startIndex == 0)) { var collection = _childrenSorter.Sort((IEnumerable <IContent>)childrenWithReviews); childrenWithReviews = collection.Cast <T>().ToList(); } } return(childrenWithReviews); }
public LoaderOptionsAttribute(string requiredVersion, LoaderOptions options) : this(requiredVersion, null, options) { }
public IEnumerable <T> GetChildrenWithReviews <T>(ContentReference contentLink, LoaderOptions loaderOptions) where T : IContentData { return(GetChildrenWithReviews <T>(contentLink, loaderOptions, -1, -1)); }
public virtual IEnumerable<IContent> GetAllLocalizationPages(LoaderOptions loaderOptions) { return _contentRepository.Service.GetChildren<IContent>(GetOrAddLocalizationContainer().ContentLink, loaderOptions); }
public virtual IEnumerable <T> GetChildren <T>(ContentReference contentLink, LoaderOptions settings, int startIndex, int maxRows) where T : IContentData { return(GetChildren <T>(contentLink)); }
public Loader CreateLoader(LoaderOptions defaults = null, Application compilationTarget = null) { defaults = defaults ?? new LoaderOptions(null, null); var planOptions = _plan.Options; if(planOptions != null) defaults.InheritFrom(planOptions); compilationTarget = compilationTarget ?? InstantiateForBuild(); Debug.Assert(compilationTarget.Module.Name == _module.Name); var lowPrioritySymbols = defaults.Symbols; SymbolStore predef; if(lowPrioritySymbols.IsEmpty) { predef = SymbolStore.Create(ExternalSymbols); } else { // First create an intermediate symbol store as a copy // of ExternalSymbols, inheriting from lowPrioritySymbols predef = SymbolStore.Create(lowPrioritySymbols); foreach (var externalSymbol in ExternalSymbols) predef.Declare(externalSymbol.Key, externalSymbol.Value); // Then create the final SymbolStore inheriting from the intermediate one. // that way we can later extract the declarations made by this module. predef = SymbolStore.Create(predef); } var finalOptions = new LoaderOptions(_compilationEngine, compilationTarget, predef) {ReconstructSymbols = false, RegisterCommands = false}; finalOptions.InheritFrom(defaults); return new Loader(finalOptions); }
/// <summary> /// Registers the default loader service, if no other loader has been registered yet. /// </summary> /// <param name="configuration">The configuration to extend.</param> /// <param name="setup">Configuration for the loader service.</param> /// <returns>The new configuration with the service.</returns> public static IConfiguration WithDefaultLoader(this IConfiguration configuration, LoaderOptions setup = null) { var config = setup ?? new LoaderOptions(); if (!configuration.Has <IRequester>()) { configuration = configuration .With(new DefaultHttpRequester()); } if (!config.IsNavigationDisabled) { configuration = configuration .With <IDocumentLoader>(ctx => new DefaultDocumentLoader(ctx, config.Filter)); } if (config.IsResourceLoadingEnabled) { configuration = configuration .With <IResourceLoader>(ctx => new DefaultResourceLoader(ctx, config.Filter)); } return(configuration); }
public virtual T Get <T>(Guid contentGuid, LoaderOptions settings) where T : IContentData { throw new NotImplementedException(); }
public virtual IContent GetLocalizationPage(string[] normalizedKey, LoaderOptions loaderOptions) { ContentReference cachedLocalizationPageReference; var basePath = normalizedKey[1]; if (CachedLocalizationPages.TryGetValue(basePath, out cachedLocalizationPageReference)) { try { return _contentRepository.Service.Get<IContent>(cachedLocalizationPageReference, loaderOptions); } catch (PageNotFoundException) { // this can happen when we cached the localization pagereference but the page has been removed mean while. ContentReference removedReference; CachedLocalizationPages.TryRemove(basePath, out removedReference); } } var localizationPages = GetAllLocalizationPages(loaderOptions); var localizationPage = localizationPages.SingleOrDefault( x => x.Property.GetPropertyValue<string>("BasePath").Equals(basePath, StringComparison.OrdinalIgnoreCase)); if (localizationPage != null) CachedLocalizationPages.TryAdd(basePath, localizationPage.ContentLink); return localizationPage; }