private void LoadProject(Project project, string outDir = null)
{
_config = new DocConfiguration(project);
outputPath = outDir ?? project.DirectoryPath;
var pathProperty = project.AllEvaluatedProperties.FirstOrDefault(o => o.Name.Equals("OutputPath", StringComparison.InvariantCulture));
if (outDir == null && pathProperty != null)
{
outputPath = Path.Combine(project.DirectoryPath, pathProperty.EvaluatedValue);
}
// Try get path from config
var userPath = _config.Get("OutputPath");
var docPathProperty = project.AllEvaluatedProperties.FirstOrDefault(o => o.Name.Equals("DocOutputPath", StringComparison.InvariantCulture));
if (userPath != null)
{
outputPath = Path.Combine(outputPath, userPath);
}
else if (docPathProperty != null)
{
outputPath = Path.Combine(outputPath, docPathProperty.EvaluatedValue);
}
else
{
outputPath = Path.Combine(outputPath, "Doc");
}
// Ensure path exists
if (!Directory.Exists(outputPath))
{
Directory.CreateDirectory(outputPath);
}
}
public void StoreDocConfiguration(DocConfiguration config)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
var json = JsonConvert.SerializeObject(config);
File.WriteAllText(AbsolutePath("config.json"), json);
}
public void testJsonSerializingConfiguration()
{
Configuration config = new Configuration();
config.ConfigId = null;
config.Name = "test";
config.IsPrimary = true;
config.OneSentence = false;
config.AutoResponse = true;
config.Language = "English";
config.CharsThreshold = 80;
config.Callback = "https://anyapi.anydomain.com/processed/docs.json";
DocConfiguration doc = new DocConfiguration();
doc.ConceptTopicsLimit = 5;
doc.QueryTopicsLimit = 5;
doc.NamedEntitiesLimit = 5;
doc.UserEntitiesLimit = 5;
doc.ThemesLimit = 0;
doc.EntityThemesLimit = 5;
doc.PhrasesLimit = 0;
doc.SummaryLimit = 0;
config.Document = doc;
CollConfiguration coll = new CollConfiguration();
coll.FacetsLimit = 15;
coll.FacetAttsLimit = 20;
coll.ConceptTopicsLimit = 5;
coll.QueryTopicsLimit = 5;
coll.NamedEntitiesLimit = 5;
coll.ThemesLimit = 0;
config.Collection = coll;
Configuration removedConfig = new Configuration();
removedConfig.ConfigId = "45699836";
Configuration clonedConfig = new Configuration();
clonedConfig.ConfigId = "45699836";
clonedConfig.Name = "Cloned Config";
clonedConfig.IsPrimary = false;
clonedConfig.OneSentence = true;
String expectedResult =
"{\"added\":[{\"auto_response\":true,\"callback\":\"https:\\/\\/anyapi.anydomain.com\\/processed\\/docs.json\",\"chars_threshold\":80,\"collection\":{\"concept_topics_limit\":5," +
"\"facet_atts_limit\":20,\"facets_limit\":15,\"named_entities_limit\":5,\"query_topics_limit\":5,\"themes_limit\":0},\"document\":" +
"{\"concept_topics_limit\":5,\"entity_themes_limit\":5,\"named_entities_limit\":5,\"phrases_limit\":0,\"query_topics_limit\":5,\"summary_limit\":0,\"themes_limit\":0," +
"\"user_entities_limit\":5},\"is_primary\":true,\"language\":\"English\",\"name\":\"test\",\"one_sentence\":false},{\"auto_response\":false,\"callback\":null,\"collection\":null,\"document\":null" +
",\"is_primary\":false,\"language\":null,\"name\":\"Cloned Config\",\"one_sentence\":true,\"template\":\"45699836\"}],\"removed\":[\"45699836\"]}";
ISerializer serializer = new JsonSerializer();
IUpdateProxy<Configuration> proxy = Session.CreateSession("", "", serializer).CreateConfigurationsUpdateProxy();
proxy.Add(config);
proxy.Remove(removedConfig);
proxy.Clone(clonedConfig);
ConfigurationManagable cm = (ConfigurationManagable)proxy.Stub;
Regex rgx = new Regex("\\s{2,}");
Assert.AreEqual(expectedResult, rgx.Replace(serializer.Serialize(cm), ""));
}
public void testXmlSerializingConfiguration()
{
Configuration config = new Configuration();
config.Name = "test";
config.OneSentence = false;
config.IsPrimary = true;
config.AutoResponse = true;
config.Language = "English";
config.CharsThreshold = 80;
config.Callback = "https://anyapi.anydomain.com/processed/docs.json";
DocConfiguration doc = new DocConfiguration();
doc.ConceptTopicsLimit = 5;
doc.QueryTopicsLimit = 5;
doc.NamedEntitiesLimit = 5;
doc.UserEntitiesLimit = 5;
doc.ThemesLimit = 0;
doc.EntityThemesLimit = 5;
doc.PhrasesLimit = 0;
doc.SummaryLimit = 0;
config.Document = doc;
CollConfiguration coll = new CollConfiguration();
coll.FacetsLimit = 15;
coll.FacetAttsLimit = 20;
coll.ConceptTopicsLimit = 5;
coll.QueryTopicsLimit = 5;
coll.NamedEntitiesLimit = 5;
coll.ThemesLimit = 0;
config.Collection = coll;
Configuration removedConfig = new Configuration();
removedConfig.ConfigId = "45699836";
Configuration clonedConfig = new Configuration();
clonedConfig.ConfigId = "45699836";
clonedConfig.Name = "Cloned Config";
clonedConfig.IsPrimary = false;
clonedConfig.OneSentence = true;
clonedConfig.Language = "French";
String expectedResult =
"<?xml version=\"1.0\"?>" +
"<configurations xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\">" +
"<added>" +
"<configuration>" +
"<name>" + config.Name + "</name>" +
"<one_sentence>" + config.OneSentence.ToString().ToLower() + "</one_sentence>" +
"<is_primary>" + config.IsPrimary.ToString().ToLower() + "</is_primary>" +
"<auto_response>" + config.AutoResponse.ToString().ToLower() + "</auto_response>" +
"<language>" + config.Language + "</language>" +
"<chars_threshold>" + config.CharsThreshold + "</chars_threshold>" +
"<callback>" + config.Callback + "</callback>" +
"<document>" +
"<concept_topics_limit>" + config.Document.ConceptTopicsLimit + "</concept_topics_limit>" +
"<query_topics_limit>" + config.Document.QueryTopicsLimit + "</query_topics_limit>" +
"<named_entities_limit>" + config.Document.NamedEntitiesLimit + "</named_entities_limit>" +
"<user_entities_limit>" + config.Document.UserEntitiesLimit + "</user_entities_limit>" +
"<entity_themes_limit>" + config.Document.EntityThemesLimit + "</entity_themes_limit>" +
"<themes_limit>" + config.Document.ThemesLimit + "</themes_limit>" +
"<phrases_limit>" + config.Document.PhrasesLimit + "</phrases_limit>" +
"<summary_limit>" + config.Document.SummaryLimit + "</summary_limit>" +
"</document>" +
"<collection>" +
"<facets_limit>" + config.Collection.FacetsLimit+ "</facets_limit>" +
"<facet_atts_limit>" + config.Collection.FacetAttsLimit + "</facet_atts_limit>" +
"<concept_topics_limit>" + config.Collection.ConceptTopicsLimit + "</concept_topics_limit>" +
"<query_topics_limit>" + config.Collection.ConceptTopicsLimit + "</query_topics_limit>" +
"<named_entities_limit>" + config.Collection.NamedEntitiesLimit + "</named_entities_limit>" +
"<themes_limit>" + config.Collection.ThemesLimit + "</themes_limit>" +
"</collection>" +
"</configuration>" +
"<configuration>" +
"<template>" + clonedConfig.ConfigId + "</template>" +
"<name>" + clonedConfig.Name + "</name>" +
"<one_sentence>" + clonedConfig.OneSentence.ToString().ToLower() + "</one_sentence>" +
"<is_primary>" + clonedConfig.IsPrimary.ToString().ToLower() + "</is_primary>" +
"<auto_response>" + clonedConfig.AutoResponse.ToString().ToLower() + "</auto_response>" +
"<language>" + clonedConfig.Language + "</language>" +
"<chars_threshold>" + clonedConfig.CharsThreshold + "</chars_threshold>" +
"</configuration>" +
"</added>" +
"<removed>" +
"<configuration>" + removedConfig.ConfigId + "</configuration>" +
"</removed>" +
"</configurations>";
ISerializer serializer = new XmlSerializer();
IUpdateProxy<Configuration> proxy = Session.CreateSession("", "", serializer).CreateConfigurationsUpdateProxy();
proxy.Add(config);
proxy.Remove(removedConfig);
proxy.Clone(clonedConfig);
ConfigurationManagable cm = (ConfigurationManagable)proxy.Stub;
Regex rgx = new Regex("\\s{2,}");
Assert.AreEqual(expectedResult, rgx.Replace(serializer.Serialize(cm), ""));
}
public async Task ImportAllBranches()
{
var branches = await _githubDataRepository.GetBranches();
var versions = new List <DocVersion>();
DocVersion defaultVersion = null;
foreach (var branch in branches)
{
_dataRepository.DeleteBranch(branch.Name);
var version = new DocVersion(BranchNameToPathString(branch.Name), branch.Name);
versions.Add(version);
if (branch.IsDefault)
{
defaultVersion = version;
}
var archive = await _githubDataRepository.GetArchive(branch);
using (var memoryStream = new MemoryStream(archive))
using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress))
using (var tarReader = new TarReader(gzipStream))
{
string rootDirectory = null;
while (tarReader.Next() is var entry && entry != null)
{
if (entry.Header.Flag == TarHeaderFlag.Directory && rootDirectory == null)
{
rootDirectory = entry.Header.Name;
}
if (entry.Header.Flag == TarHeaderFlag.NormalFile)
{
if (Path.GetExtension(entry.Header.Name) == ".md")
{
using (var streamReader = new StreamReader(entry))
{
var markdown = streamReader.ReadToEnd();
var meta = ParseMeta(ref markdown);
// Parse markdown
var document = Markdig.Markdown.Parse(markdown,
new MarkdownPipelineBuilder()
.UseAdvancedExtensions()
.UsePipeTables()
.Build());
var sw = new StringWriter();
var renderer = new CustomHtmlRenderer(sw);
renderer.Render(document);
sw.Flush();
var html = sw.ToString();
var file = new DocFile
{
Content = html,
Meta = meta
};
var docPath = entry.Header.Name;
if (rootDirectory != null && docPath.StartsWith(rootDirectory))
{
docPath = docPath.Substring(rootDirectory.Length).TrimStart('/', '\\');
}
var fileName = Path.GetFileNameWithoutExtension(docPath);
docPath = Path.Combine(Path.GetDirectoryName(docPath), fileName);
if (fileName == "index")
{
version.DefaultPage = meta.RedirectPage;
}
meta.LastModification = entry.Header.LastModification ?? DateTime.UtcNow;
meta.EditUrl = $"{_githubDataRepository.PublicUrl}/blob/{branch.Name}/{docPath}.md";
meta.Title = meta.Title ?? fileName.Replace('-', ' ');
_dataRepository.StoreDocFile(branch.Name, docPath, file);
}
}
else
{
var assetPath = entry.Header.Name;
if (rootDirectory != null && assetPath.StartsWith(rootDirectory))
{
assetPath = assetPath.Substring(rootDirectory.Length).TrimStart('/', '\\');
}
assetPath = assetPath.ToLower();
// Skip unaccepted asset types
if (_options.Value.AcceptedAssets.Contains(Path.GetExtension(assetPath)))
{
_dataRepository.StoreAsset(branch.Name, assetPath, entry);
}
}
}
}
}
}
var docConfig = new DocConfiguration
{
Versions = versions.ToArray(),
DefaultVersion = defaultVersion
};
_dataRepository.StoreDocConfiguration(docConfig);
}
