public override void Setup()
{
base.Setup();
DocMaker docMaker = RunData.DocMaker;
doc = docMaker.MakeDocument();
}
public static void Main(string[] args)
{
FileInfo wikipedia = null;
DirectoryInfo outputDir = new DirectoryInfo("./enwiki");
bool keepImageOnlyDocs = true;
for (int i = 0; i < args.Length; i++)
{
string arg = args[i];
if (arg.Equals("--input", StringComparison.Ordinal) || arg.Equals("-i", StringComparison.Ordinal))
{
wikipedia = new FileInfo(args[i + 1]);
i++;
}
else if (arg.Equals("--output", StringComparison.Ordinal) || arg.Equals("-o", StringComparison.Ordinal))
{
outputDir = new DirectoryInfo(args[i + 1]);
i++;
}
else if (arg.Equals("--discardImageOnlyDocs", StringComparison.Ordinal) || arg.Equals("-d", StringComparison.Ordinal))
{
keepImageOnlyDocs = false;
}
}
IDictionary <string, string> properties = new Dictionary <string, string>
{
["docs.file"] = wikipedia.FullName,
["content.source.forever"] = "false",
["keep.image.only.docs"] = keepImageOnlyDocs.ToString()
};
Config config = new Config(properties);
ContentSource source = new EnwikiContentSource();
source.SetConfig(config);
DocMaker docMaker = new DocMaker();
docMaker.SetConfig(config, source);
docMaker.ResetInputs();
if (wikipedia.Exists)
{
Console.WriteLine("Extracting Wikipedia to: " + outputDir + " using EnwikiContentSource");
outputDir.Create();
ExtractWikipedia extractor = new ExtractWikipedia(docMaker, outputDir);
extractor.Extract();
}
else
{
// LUCENENET specific - our wrapper console shows correct usage
throw new ArgumentException();
//PrintUsage();
}
}
public ExtractWikipedia(DocMaker docMaker, DirectoryInfo outputDir)
{
this.outputDir = outputDir;
this.m_docMaker = docMaker;
Console.WriteLine("Deleting all files in " + outputDir);
FileInfo[] files = outputDir.GetFiles();
for (int i = 0; i < files.Length; i++)
{
files[i].Delete();
}
}
private readonly object lineFileLock = new object(); // LUCENENET specific - lock to ensure writes don't collide for this instance
public WriteLineDocTask(PerfRunData runData)
: base(runData)
{
Config config = runData.Config;
m_fname = config.Get("line.file.out", null);
if (m_fname == null)
{
throw new ArgumentException("line.file.out must be set");
}
Stream @out = StreamUtils.GetOutputStream(new FileInfo(m_fname));
lineFileOut = new StreamWriter(@out, Encoding.UTF8);
docMaker = runData.DocMaker;
// init fields
string f2r = config.Get("line.fields", null);
if (f2r == null)
{
fieldsToWrite = DEFAULT_FIELDS;
}
else
{
if (f2r.IndexOf(SEP) >= 0)
{
throw new ArgumentException("line.fields " + f2r + " should not contain the separator char: " + SEP);
}
fieldsToWrite = f2r.Split(',').TrimEnd();
}
// init sufficient fields
sufficientFields = new bool[fieldsToWrite.Length];
string suff = config.Get("sufficient.fields", DEFAULT_SUFFICIENT_FIELDS);
if (",".Equals(suff, StringComparison.Ordinal))
{
checkSufficientFields = false;
}
else
{
checkSufficientFields = true;
ISet <string> sf = new JCG.HashSet <string>(suff.Split(',').TrimEnd());
for (int i = 0; i < fieldsToWrite.Length; i++)
{
if (sf.Contains(fieldsToWrite[i]))
{
sufficientFields[i] = true;
}
}
}
WriteHeader(lineFileOut);
}
public override void Setup()
{
base.Setup();
DocMaker docMaker = RunData.DocMaker;
if (docSize > 0)
{
m_doc = docMaker.MakeDocument(docSize);
}
else
{
m_doc = docMaker.MakeDocument();
}
}
static void Main(string[] args)
{
string dllFile = "";
string xmlFile = "";
string outputDir = "";
bool isNoPath = true;
string linkroot = "";
if (args.Length == 1 && (args[0].ToLower() == "-h" || args[0] == " ?"))
{
Console.WriteLine("Welcome MaoMD! 'MaoMD -h' for help.");
Console.WriteLine("Usage:");
Console.WriteLine("MaoMD [-h||?] || [dllFile [xmlFile [outputDir [isNoPath [linkRoot [logger]]]]]]");
Console.WriteLine("parameters:");
Console.WriteLine("dllFile: The source dll file.");
Console.WriteLine("xmlFile: The xml documentation file generated by Visual Studio.");
Console.WriteLine("outputDir: The destination directory to put all .md files.");
Console.WriteLine("isNoPath: Whether the destination system support path or not.");
Console.WriteLine("linkRoot: The root path of all .md file. Use on systems supported path.");
Console.WriteLine("no parameters: make documents with myself as a demo.");
Console.WriteLine(@"github:https://github.com/foomow/MaoMD");
}
if (args.Length > 0)
{
dllFile = args[0];
}
if (args.Length > 1)
{
xmlFile = args[1];
}
if (args.Length > 2)
{
outputDir = args[2];
}
if (args.Length > 3)
{
isNoPath = args[3].ToLower() != "false";
}
if (args.Length > 4)
{
linkroot = args[4];
}
using (DocMaker docMaker = new DocMaker(dllFile, xmlFile, outputDir, isNoPath, linkroot))
{
docMaker.Make();
};
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// string array of URLs *without* any trailing slash
string[] articleUrls =
{
"https://wakeupandcode.com/authentication-authorization-in-asp-net-core-razor-pages",
"https://wakeupandcode.com/blazor-full-stack-web-dev-in-asp-net-core",
"https://wakeupandcode.com/cookies-and-consent-in-asp-net-core",
"https://wakeupandcode.com/deploying-asp-net-core-to-azure-app-service",
"https://wakeupandcode.com/ef-core-relationships-in-asp-net-core",
"https://wakeupandcode.com/forms-and-fields-in-asp-net-core",
"https://wakeupandcode.com/generic-host-builder-in-asp-net-core",
"https://wakeupandcode.com/handling-errors-in-asp-net-core",
"https://wakeupandcode.com/iis-hosting-for-asp-net-core-web-apps",
"https://wakeupandcode.com/javascript-css-html-static-files-in-asp-net-core",
"https://wakeupandcode.com/key-vault-for-asp-net-core-web-apps",
"https://wakeupandcode.com/logging-in-asp-net-core",
"https://wakeupandcode.com/middleware-in-asp-net-core",
"https://wakeupandcode.com/net-core-3-vs2019-and-csharp-8",
"https://wakeupandcode.com/organizational-accounts-for-asp-net-core",
"https://wakeupandcode.com/production-tips-for-asp-net-core-web-apps",
"https://wakeupandcode.com/query-tags-in-ef-core-for-asp-net-core",
"https://wakeupandcode.com/razor-pages-in-asp-net-core",
"https://wakeupandcode.com/summarizing-build-2019-signalr-service",
"https://wakeupandcode.com/tag-helper-authoring-in-asp-net-core",
"https://wakeupandcode.com/unit-testing-in-asp-net-core",
"https://wakeupandcode.com/validation-in-asp-net-core",
"https://wakeupandcode.com/worker-service-in-asp-net-core",
"https://wakeupandcode.com/xml-json-serialization-in-asp-net-core",
"https://wakeupandcode.com/yaml-defined-cicd-for-asp-net-core",
"https://wakeupandcode.com/zero-downtime-web-apps-for-asp-net-core"
};
_logger.LogInformation($"Processing {articleUrls.Length} docs at: {DateTimeOffset.Now}");
for (var articleCounter = 0; articleCounter < articleUrls.Length; articleCounter++)
{
_logger.LogInformation($"Making doc {articleCounter + 1} at: {DateTimeOffset.Now}");
DocMaker.MakeDoc(articleUrls[articleCounter]);
}
_logger.LogInformation($"Completed {articleUrls.Length} docs at: {DateTimeOffset.Now}");
}
// constructor
public PerfRunData(Config config)
{
this.config = config;
// analyzer (default is standard analyzer)
analyzer = NewAnalyzerTask.CreateAnalyzer(config.Get("analyzer",
typeof(Lucene.Net.Analysis.Standard.StandardAnalyzer).AssemblyQualifiedName));
// content source
string sourceClass = config.Get("content.source", typeof(SingleDocSource).AssemblyQualifiedName);
contentSource = (ContentSource)Activator.CreateInstance(Type.GetType(sourceClass)); //Class.forName(sourceClass).asSubclass(typeof(ContentSource)).newInstance();
contentSource.SetConfig(config);
// doc maker
docMaker = (DocMaker)Activator.CreateInstance(Type.GetType(config.Get("doc.maker", typeof(DocMaker).AssemblyQualifiedName))); // "org.apache.lucene.benchmark.byTask.feeds.DocMaker")).asSubclass(DocMaker.class).newInstance();
docMaker.SetConfig(config, contentSource);
// facet source
facetSource = (FacetSource)Activator.CreateInstance(Type.GetType(config.Get("facet.source",
typeof(RandomFacetSource).AssemblyQualifiedName))); // "org.apache.lucene.benchmark.byTask.feeds.RandomFacetSource")).asSubclass(FacetSource.class).newInstance();
facetSource.SetConfig(config);
// query makers
readTaskQueryMaker = new Dictionary <Type, IQueryMaker>();
qmkrClass = Type.GetType(config.Get("query.maker", typeof(SimpleQueryMaker).AssemblyQualifiedName));
// index stuff
Reinit(false);
// statistic points
points = new Points(config);
if (bool.Parse(config.Get("log.queries", "false")))
{
Console.WriteLine("------------> queries:");
Console.WriteLine(GetQueryMaker(new SearchTask(this)).PrintQueries());
}
}
