static async Task Main(params string[] args)
{
string broker = args[0];
string topic = args[1];
int partitionsCount = int.Parse(args[2]);
var sections = args.Skip(3);
var logger = new ConsoleLogger();
s_producer = new KafkaProducerBase <string, Product>(logger, broker, topic, partitionsCount);
LoaderOptions lo = new LoaderOptions()
{
IsNavigationDisabled = false,
IsResourceLoadingEnabled = false
};
//Use the default configuration for AngleSharp
var config = Configuration.Default.WithDefaultLoader(lo);
foreach (var category in sections)
{
await ProcessSection(config, category, s_sections[category]);
}
}
//public readonly MyHttpClient HttpClient;
//public Browser(MyHttpClient httpClient)
public Browser()
{
//HttpClient = httpClient;
//We require a custom configuration
var config = Configuration.Default.WithJs();
//Create a new context for evaluating webpages with the given config
var context = BrowsingContext.New(config);
LoaderOptions lo = new LoaderOptions();
lo.IsNavigationDisabled = false;
lo.IsResourceLoadingEnabled = true;
//var requester = new HttpClientRequester(httpClient);
var configuration = Configuration.Default
.WithDefaultLoader(lo)
.WithJs()
.WithPersistentCookies("d:\\logs")
.WithRequesters(new MyClientHandler())
//.WithCss()
//.WithCookies()
;
Context = BrowsingContext.New(configuration);
}
private IEnumerable <IContent> GetChildren(
ContentReference contentReference,
LoaderOptions loaderOptions,
int startIndex,
int maxRows,
out bool hasNextPage)
{
hasNextPage = false;
var items = _contentLoader
.GetChildren <IContent>(
contentReference,
loaderOptions,
startIndex,
maxRows
)
.FilterHiddenGraphTypes();
if (items.Count() < maxRows)
{
return(items);
}
hasNextPage = _contentLoader
.GetChildren <IContent>(
contentReference,
loaderOptions,
startIndex,
1
)
.FilterHiddenGraphTypes()
.Any();
return(items);
}
private LoaderOptions CreateDefaultListOption()
{
LoaderOptions loaderOptions = new LoaderOptions();
loaderOptions.Add <LanguageLoaderOption>(LanguageLoaderOption.Fallback(_languageAccessor.Language));
return(loaderOptions);
}
public virtual IContent GetLocalizationPage(string[] normalizedKey, LoaderOptions loaderOptions)
{
ContentReference cachedLocalizationPageReference;
var basePath = normalizedKey[1];
if (CachedLocalizationPages.TryGetValue(basePath, out cachedLocalizationPageReference))
{
try
{
return(_contentRepository.Service.Get <IContent>(cachedLocalizationPageReference, loaderOptions));
}
catch (PageNotFoundException)
{
// this can happen when we cached the localization pagereference but the page has been removed mean while.
ContentReference removedReference;
CachedLocalizationPages.TryRemove(basePath, out removedReference);
}
}
var localizationPages = GetAllLocalizationPages(loaderOptions);
var localizationPage = localizationPages.SingleOrDefault(
x => x.Property.GetPropertyValue <string>("BasePath").Equals(basePath, StringComparison.OrdinalIgnoreCase));
if (localizationPage != null)
{
CachedLocalizationPages.TryAdd(basePath, localizationPage.ContentLink);
}
return(localizationPage);
}
public void TestReadingEncodedFile(string fileName, System.Text.Encoding encoding, System.Text.Encoding defaultEncoding)
{
// Set some options, so that we can know if things are working...
LoaderOptions loaderOptions = new LoaderOptions();
loaderOptions.DetectEncoding = true;
loaderOptions.DefaultEncoding = defaultEncoding;
loaderOptions.ParserOptions.IncludeMetaData = true;
// Load multi-byte html file into memory
XmlDocument doc = XHtmlLoader.LoadHtml(_sampleMultiByteHtml);
// Ensure Sample directory exists
string sampleDir = (new DirectoryInfo(AssemblyDirectory)).Parent.Parent.Parent.FullName + "\\SampleData\\";
if (!Directory.Exists(sampleDir))
{
Directory.CreateDirectory(sampleDir);
}
// Create Encoded file
string fullName = sampleDir + fileName;
using (TextWriter sw = new StreamWriter(File.Create(fullName), encoding))
{
doc.Save(sw);
}
// Re-load into memory
XmlDocument doc2 = XHtmlLoader.LoadWebPageAsync("file://" + fullName, loaderOptions).Result;
Console.WriteLine("Reading file: " + fileName);
Console.WriteLine(doc2.OuterXml);
Assert.AreEqual(doc.SelectSingleNode("//body").OuterXml, doc2.SelectSingleNode("//body").OuterXml);
}
public TTarget GetContent <TTarget>(DocumentIndexModel indexItem, bool filterOnCulture = true) where TTarget : ContentData
{
if (indexItem == null || string.IsNullOrEmpty(indexItem.Id))
{
return(default(TTarget));
}
Guid result;
if (Guid.TryParse(((IEnumerable <string>)indexItem.Id.Split('|')).FirstOrDefault <string>(), out result))
{
LoaderOptions loaderOptions;
if (filterOnCulture)
{
loaderOptions = this.GetLoaderOptions(indexItem.Language);
}
else
{
loaderOptions = new LoaderOptions();
loaderOptions.Add <LanguageLoaderOption>(LanguageLoaderOption.Fallback((CultureInfo)null));
}
LoaderOptions settings = loaderOptions;
TTarget content = null;
this._contentRepository.TryGet <TTarget>(result, settings, out content);
return(content);
}
return(default(TTarget));
}
public virtual IEnumerable <T> GetReferencesToCategories <T>(IEnumerable <ContentReference> categories, CultureInfo culture) where T : ICategorizableContent, IContentData
{
var loaderOptions = new LoaderOptions {
LanguageLoaderOption.Specific(culture)
};
return(GetReferencesToCategories <T>(categories, loaderOptions));
}
protected Unifiable ProcessLoadProc(LoaderOptions loaderOptions)
{
if (templateNode.Name.ToLower() == "category")
{
request.Loader.processCategory(templateNode, loaderOptions.CurrentFilename);
}
return(Unifiable.Empty);
}
public T GetFirstBySegment <T>(string urlSegment, CultureInfo culture) where T : CategoryData
{
var loaderOptions = new LoaderOptions
{
LanguageLoaderOption.Specific(culture)
};
return(GetFirstBySegment <T>(urlSegment, loaderOptions));
}
private LoaderOptions CreateDefaultLoadOption()
{
LoaderOptions loaderOptions = new LoaderOptions
{
LanguageLoaderOption.FallbackWithMaster(_languageAccessor.Language)
};
return(loaderOptions);
}
public LoaderOptionsAttribute(string requiredVersion, string minimalMirandaVersion, LoaderOptions options)
{
if (!String.IsNullOrEmpty(requiredVersion))
this.requiredVersion = new Version(requiredVersion);
if (!String.IsNullOrEmpty(minimalMirandaVersion))
this.minimalMirandaVersion = new Version(minimalMirandaVersion);
this.options = options;
}
public FetchService(Database database)
{
var config = new LoaderOptions();
IConfiguration configuration = Configuration.Default
.With(new DefaultHttpRequester())
.With <IDocumentLoader>(ctx => new CrabtopusDocumentLoader(ctx, config.Filter))
.With <IResourceLoader>(ctx => new DefaultResourceLoader(ctx, config.Filter));
_context = BrowsingContext.New(configuration);
_database = database;
}
public T GetFirstBySegment <T>(string urlSegment, LoaderOptions loaderOptions) where T : CategoryData
{
if (SiteDefinition.Current.SiteAssetsRoot != SiteDefinition.Current.GlobalAssetsRoot)
{
var firstSiteCategory = GetFirstBySegment <T>(ContentRepository.GetOrCreateSiteCategoriesRoot(), urlSegment, loaderOptions);
if (firstSiteCategory != null)
{
return(firstSiteCategory);
}
}
return(GetFirstBySegment <T>(ContentRepository.GetOrCreateGlobalCategoriesRoot(), urlSegment, loaderOptions));
}
private LoaderOptions GetLoaderOptions(string languageCode)
{
if (string.IsNullOrEmpty(languageCode))
{
LoaderOptions loaderOptions = new LoaderOptions();
loaderOptions.Add <LanguageLoaderOption>(LanguageLoaderOption.FallbackWithMaster((CultureInfo)null));
return(loaderOptions);
}
CultureInfo language = languageCode == "iv" ? CultureInfo.InvariantCulture : CultureInfo.GetCultureInfo(languageCode);
LoaderOptions loaderOptions1 = new LoaderOptions();
loaderOptions1.Add <LanguageLoaderOption>(LanguageLoaderOption.Specific(language));
return(loaderOptions1);
}
public KeyProcessor(LoaderOptions options, GenericLoader g)
{
parent = g;
Options = options;
TruncExt = new SqlCommand("truncate table [ext].[" + Options.TableName + "Keys]");
KeyValues.Columns.Add("FileSourceId", typeof(string));
InsertFileSource = new SqlCommand("InsertFileSource");
InsertFileSource.Parameters.Add("@filename", SqlDbType.NVarChar);
InsertFileSource.Parameters.Add("@filesize", SqlDbType.Int);
InsertFileSource.CommandType = CommandType.StoredProcedure;
}
public CryptoInfo(LoaderOptions options)
{
this.options = options;
Log.Info("Loading encryption keys.");
var stopwatch = new Stopwatch();
stopwatch.Start();
var parsedD20Engine = new ParsedD20RulesEngine(Path.Combine(options.CBPath, "D20RulesEngine.dll"));
this.expectedDemoHash = parsedD20Engine.expectedDemoHash;
this.expectedNormalHash = parsedD20Engine.expectedNormalHash;
// This is intentionally loaded first, so there's a chance to override it from other sources.
if (options.KeyFile != null)
{
ProcessUpdateFile(CB_APP_ID, options.KeyFile);
}
string heroicDemoPath = Path.Combine(options.CBPath, "HeroicDemo.update");
if (File.Exists(heroicDemoPath))
{
ProcessUpdateFile(CB_APP_ID, heroicDemoPath, true);
}
LoadRegistryKeys(CB_APP_ID);
var regPatcherPath = Path.Combine(options.CBPath, "RegPatcher.dat");
if (File.Exists(regPatcherPath))
{
Log.Debug($" - Loading default key from {regPatcherPath}");
keyStore.FallbackKey = Convert.FromBase64String(File.ReadAllText(regPatcherPath));
}
if (options.WriteKeyFile)
{
SaveKeyFile();
}
stopwatch.Stop();
Log.Debug($"Finished in {stopwatch.ElapsedMilliseconds} ms");
Log.Debug();
}
private static ContentReference GetOrCreateCategoriesRoot(this IContentRepository contentRepository, ContentReference parentLink, string name, string routeSegment)
{
var loaderOptions = new LoaderOptions
{
LanguageLoaderOption.FallbackWithMaster()
};
var rootCategory = contentRepository.GetChildren <CategoryRoot>(parentLink, loaderOptions).FirstOrDefault();
if (rootCategory != null)
{
return(rootCategory.ContentLink);
}
rootCategory = contentRepository.GetDefault <CategoryRoot>(parentLink);
rootCategory.Name = name;
rootCategory.RouteSegment = routeSegment;
return(contentRepository.Save(rootCategory, SaveAction.Publish, AccessLevel.NoAccess));
}
public virtual ChangeTaskViewModel Map(ApprovalCommandBase approvalCommand, IPrincipal principal)
{
var commandViewModel1 = _mapper.Map(approvalCommand) as ChangeTaskViewModel;
var name1 = approvalCommand is ICultureSpecificApprovalCommand specificApprovalCommand ? specificApprovalCommand.AppliedOnLanguageBranch : null;
var cultureInfo = string.IsNullOrEmpty(name1) ? null : new CultureInfo(name1);
var commandViewModel2 = commandViewModel1;
var contentRepository = _contentRepository;
var appliedOnContentLink = approvalCommand.AppliedOnContentLink;
var settings = new LoaderOptions
{
new LanguageLoaderOption()
{
Language = cultureInfo, FallbackBehaviour = LanguageBehaviour.FallbackWithMaster
}
};
var name2 = contentRepository.Get <IContent>(appliedOnContentLink, settings)?.Name;
if (commandViewModel2 != null)
{
commandViewModel2.Name = name2;
}
if (commandViewModel1 != null)
{
commandViewModel1.Status = (int)approvalCommand.CommandStatus;
commandViewModel1.CanExecute = approvalCommand.CommandStatus == CommandMetaData.ChangeTaskApprovalStatus.InReview;
var fullName = approvalCommand.GetType().FullName;
if (fullName != null)
{
commandViewModel1.TypeIdentifier = fullName.ToLower();
}
commandViewModel1.Id = approvalCommand.Id.ExternalId.ToString();
commandViewModel1.IsCommandDataValid = approvalCommand.IsValid();
commandViewModel1.CreatedBy = _uiHelper.GetDisplayNameForUser(commandViewModel1.CreatedBy);
commandViewModel1.ChangedBy = _uiHelper.GetDisplayNameForUser(commandViewModel1.ChangedBy);
commandViewModel1.CanUserActOnHisOwnChanges = !string.Equals(approvalCommand.CreatedBy, principal.Identity.Name, StringComparison.OrdinalIgnoreCase);
return(commandViewModel1);
}
return(null);
}
private TaxonomyData GetTaxonomyDataRecursively(SegmentContext segmentContext)
{
var contentReference = this.taxonomyRoot;
var loaderOptions = new LoaderOptions
{
LanguageLoaderOption.FallbackWithMaster(CultureInfo.GetCultureInfo(segmentContext.Language))
};
TaxonomyData taxonomyData = null;
while (true)
{
var segment = segmentContext.GetNextValue(segmentContext.RemainingPath);
if (string.IsNullOrEmpty(segment.Next))
{
break;
}
var content = this.contentLoader.GetBySegment(contentReference, segment.Next, loaderOptions);
if (content == null)
{
break;
}
contentReference = content.ContentLink;
if (content is TaxonomyData)
{
taxonomyData = content as TaxonomyData;
// Only set remaining path if we found what we were looking for,
// otherwise let others try to parse the URL.
segmentContext.RemainingPath = segment.Remaining;
}
}
return(taxonomyData);
}
public static IEnumerable <IContent> GetAllLanguageVersions(this IContentLoader contentLoader, ContentReference contentReference)
{
var content = contentLoader.Get <IContent>(contentReference);
if (content is ILocalizable rootContent)
{
foreach (var cultureInfo in rootContent.ExistingLanguages)
{
var loaderOptions = new LoaderOptions {
LanguageLoaderOption.Specific(cultureInfo)
};
var contentInSpecificLanguage = contentLoader.Get <IContent>(contentReference.ToReferenceWithoutVersion(), loaderOptions);
yield return(contentInSpecificLanguage);
}
}
else
{
yield return(content);
}
}
public async Task DownloadResources()
{
var urls = new List <Url>();
var loaderOptions = new LoaderOptions
{
IsResourceLoadingEnabled = true,
Filter = (req) =>
{
urls.Add(req.Address);
return(true);
},
};
var config = Configuration.Default
.WithDefaultLoader(loaderOptions)
.WithCss();
var document = "<style>div { background: url('https://avatars1.githubusercontent.com/u/10828168?s=200&v=4'); }</style><div></div>".ToHtmlDocument(config);
var tree = document.DefaultView.Render();
var node = tree.Find(document.QuerySelector("div"));
await node.DownloadResources();
Assert.AreEqual(1, urls.Count);
Assert.AreEqual("https://avatars1.githubusercontent.com/u/10828168?s=200&v=4", urls[0].Href);
}
private TaxonomyData GetTaxonomyDataRecursively(SegmentContext segmentContext)
{
var contentReference = this.taxonomyRoot;
var loaderOptions = new LoaderOptions
{
LanguageLoaderOption.FallbackWithMaster(CultureInfo.GetCultureInfo(segmentContext.Language))
};
TaxonomyData taxonomyData = null;
while (true)
{
var segment = this.ReadNextSegment(segmentContext);
if (string.IsNullOrEmpty(segment.Next))
{
break;
}
var content = this.contentLoader.GetBySegment(contentReference, segment.Next, loaderOptions);
if (content == null)
{
break;
}
contentReference = content.ContentLink;
if (content is TaxonomyData)
{
taxonomyData = content as TaxonomyData;
}
}
return(taxonomyData);
}
static async Task Main(params string[] args)
{
string broker = args[0];
var sections = args.Skip(1);
var logger = new ConsoleLogger();
s_producer = new KafkaProducerBase <string, Product>(logger, broker,
MessagingSchema.RawProductsTopic, MessagingSchema.RawProductsPartitions);
LoaderOptions lo = new LoaderOptions()
{
IsNavigationDisabled = false,
IsResourceLoadingEnabled = true
};
//Use the default configuration for AngleSharp
var config = Configuration.Default.WithDefaultLoader(lo);
foreach (var category in s_sections)
{
await ProcessSection(config, category.Key, category.Value);
}
}
public Connection <IContent> CreateIContentConnection(
ResolveConnectionContext <IContent> context,
LoaderOptions loaderOptions,
int defaultPageSize)
{
int first = context.PageSize ?? defaultPageSize;
int.TryParse(context.After, out int after);
var items = GetChildren(
context.Source.ContentLink,
loaderOptions,
after,
first,
out bool hasNextPage
);
var pageInfo = CreatePageInfo(first, after, hasNextPage);
int endCursor = int.Parse(pageInfo.EndCursor);
var edges = new List <Edge <IContent> >();
for (int i = 0; i < items.Count(); i++)
{
edges.Add(new Edge <IContent> {
Node = items.ElementAt(i), Cursor = ((endCursor - first) + i + 1).ToString()
});
}
return(new Connection <IContent>
{
TotalCount = items.Count(),
Edges = edges,
PageInfo = pageInfo
});
}
public LoaderOptionsAttribute(LoaderOptions options)
: this(null, null, options) { }
public virtual T Get <T>(ContentReference contentLink, LoaderOptions settings) where T : IContentData
{
throw new NotImplementedException();
}
public IEnumerable <T> GetChildrenWithReviews <T>(
ContentReference contentLink, LoaderOptions loaderOptions, int startIndex, int maxRows) where T : IContentData
{
if (ContentReference.IsNullOrEmpty(contentLink))
{
throw new ArgumentNullException(nameof(contentLink), "Parameter has no value set");
}
if (!ExternalReview.IsInExternalReviewContext)
{
return(_contentLoader.GetChildren <T>(contentLink));
}
ContentReference referenceWithoutVersion = contentLink.ToReferenceWithoutVersion();
if (referenceWithoutVersion == ContentReference.WasteBasket)
{
return(_contentLoader.GetChildren <T>(contentLink));
}
var provider = _contentProviderManager.ProviderMap.GetProvider(referenceWithoutVersion);
var parentContent = _contentLoader.Get <IContent>(referenceWithoutVersion, loaderOptions);
var localizable = parentContent as ILocalizable;
var languageID = localizable != null ? localizable.Language.Name : (string)null;
var childrenReferences =
provider.GetChildrenReferences <T>(referenceWithoutVersion, languageID, startIndex, maxRows);
var result = new List <ContentReference>();
foreach (var childReference in childrenReferences)
{
var referenceToLoad = LoadUnpublishedVersion(childReference.ContentLink);
if (referenceToLoad == null)
{
result.Add(childReference.ContentLink);
}
else
{
var content = _contentLoader.Get <T>(referenceToLoad);
if (HasExpired(content as IVersionable))
{
continue;
}
if ((content as IContent).IsPublished())
{
// for published version return the original method result
result.Add(childReference.ContentLink);
continue;
}
result.Add((content as IContent).ContentLink);
}
}
var childrenWithReviews = result.Select(_contentLoader.Get <T>).ToList();
if (childrenWithReviews.Count > 0)
{
var pageData = parentContent as PageData;
if (pageData != null && pageData.ChildSortOrder == FilterSortOrder.Alphabetical && (startIndex == -1 && maxRows == -1 ||
childrenWithReviews.Count < maxRows && startIndex == 0))
{
var collection = _childrenSorter.Sort((IEnumerable <IContent>)childrenWithReviews);
childrenWithReviews = collection.Cast <T>().ToList();
}
}
return(childrenWithReviews);
}
public LoaderOptionsAttribute(string requiredVersion, LoaderOptions options)
: this(requiredVersion, null, options) { }
public IEnumerable <T> GetChildrenWithReviews <T>(ContentReference contentLink, LoaderOptions loaderOptions) where T : IContentData
{
return(GetChildrenWithReviews <T>(contentLink, loaderOptions, -1, -1));
}
public virtual IEnumerable<IContent> GetAllLocalizationPages(LoaderOptions loaderOptions)
{
return _contentRepository.Service.GetChildren<IContent>(GetOrAddLocalizationContainer().ContentLink, loaderOptions);
}
public virtual IEnumerable <T> GetChildren <T>(ContentReference contentLink, LoaderOptions settings, int startIndex, int maxRows) where T : IContentData
{
return(GetChildren <T>(contentLink));
}
public Loader CreateLoader(LoaderOptions defaults = null, Application compilationTarget = null)
{
defaults = defaults ?? new LoaderOptions(null, null);
var planOptions = _plan.Options;
if(planOptions != null)
defaults.InheritFrom(planOptions);
compilationTarget = compilationTarget ?? InstantiateForBuild();
Debug.Assert(compilationTarget.Module.Name == _module.Name);
var lowPrioritySymbols = defaults.Symbols;
SymbolStore predef;
if(lowPrioritySymbols.IsEmpty)
{
predef = SymbolStore.Create(ExternalSymbols);
}
else
{
// First create an intermediate symbol store as a copy
// of ExternalSymbols, inheriting from lowPrioritySymbols
predef = SymbolStore.Create(lowPrioritySymbols);
foreach (var externalSymbol in ExternalSymbols)
predef.Declare(externalSymbol.Key, externalSymbol.Value);
// Then create the final SymbolStore inheriting from the intermediate one.
// that way we can later extract the declarations made by this module.
predef = SymbolStore.Create(predef);
}
var finalOptions = new LoaderOptions(_compilationEngine, compilationTarget, predef)
{ReconstructSymbols = false, RegisterCommands = false};
finalOptions.InheritFrom(defaults);
return new Loader(finalOptions);
}
/// <summary>
/// Registers the default loader service, if no other loader has been registered yet.
/// </summary>
/// <param name="configuration">The configuration to extend.</param>
/// <param name="setup">Configuration for the loader service.</param>
/// <returns>The new configuration with the service.</returns>
public static IConfiguration WithDefaultLoader(this IConfiguration configuration, LoaderOptions setup = null)
{
var config = setup ?? new LoaderOptions();
if (!configuration.Has <IRequester>())
{
configuration = configuration
.With(new DefaultHttpRequester());
}
if (!config.IsNavigationDisabled)
{
configuration = configuration
.With <IDocumentLoader>(ctx => new DefaultDocumentLoader(ctx, config.Filter));
}
if (config.IsResourceLoadingEnabled)
{
configuration = configuration
.With <IResourceLoader>(ctx => new DefaultResourceLoader(ctx, config.Filter));
}
return(configuration);
}
public virtual T Get <T>(Guid contentGuid, LoaderOptions settings) where T : IContentData
{
throw new NotImplementedException();
}
public virtual IContent GetLocalizationPage(string[] normalizedKey, LoaderOptions loaderOptions)
{
ContentReference cachedLocalizationPageReference;
var basePath = normalizedKey[1];
if (CachedLocalizationPages.TryGetValue(basePath, out cachedLocalizationPageReference))
{
try
{
return _contentRepository.Service.Get<IContent>(cachedLocalizationPageReference, loaderOptions);
}
catch (PageNotFoundException)
{
// this can happen when we cached the localization pagereference but the page has been removed mean while.
ContentReference removedReference;
CachedLocalizationPages.TryRemove(basePath, out removedReference);
}
}
var localizationPages = GetAllLocalizationPages(loaderOptions);
var localizationPage = localizationPages.SingleOrDefault(
x => x.Property.GetPropertyValue<string>("BasePath").Equals(basePath, StringComparison.OrdinalIgnoreCase));
if (localizationPage != null)
CachedLocalizationPages.TryAdd(basePath, localizationPage.ContentLink);
return localizationPage;
}
