private static async Task <ContentObject> LoadPageScript(SiteObject site, Stream stream, FileEntry file, ScriptObject preContent)
{
var evalClock = Stopwatch.StartNew();
// Read the stream
var reader = new StreamReader(stream);
var content = await reader.ReadToEndAsync();
// Early dispose the stream
stream.Dispose();
ContentObject page = null;
// Parse the page, using front-matter mode
var scriptInstance = site.Scripts.ParseScript(content, file.FullName, ScriptMode.FrontMatterAndContent);
if (!scriptInstance.HasErrors)
{
page = new FileContentObject(site, file, scriptInstance, preContent: preContent);
}
evalClock.Stop();
// Update statistics
var contentStat = site.Statistics.GetContentStat(page);
contentStat.LoadingTime += evalClock.Elapsed;
return(page);
}
public override void Terminate()
{
if (_connection == null)
{
return;
}
// Last pass by optimizing the b-tree
using (var command = _connection.CreateCommand())
{
command.CommandText = "INSERT INTO pages(pages) VALUES('optimize');";
command.ExecuteNonQuery();
}
// Commit all changes generated during loading
_currentTransaction.Commit();
// Final compaction of the DB
using (var command = _connection.CreateCommand())
{
command.CommandText = "VACUUM;";
command.ExecuteNonQuery();
}
_connection.Close();
_connection.Dispose();
// Add our dynamic content to the output
var fs = new PhysicalFileSystem();
var srcPath = fs.ConvertPathFromInternal(_dbPathOnDisk);
var content = new FileContentObject(Site, new FileEntry(fs, srcPath), path: OutputUrl.ChangeExtension("sqlite"));
Site.DynamicPages.Add(content);
_currentTransaction = null;
_connection = null;
// TODO: make it configurable by selecting which bundle will receive the search/db
var defaultBundle = Plugin.BundlePlugin.GetOrCreateBundle(null);
if (Plugin.Worker)
{
defaultBundle.InsertLink(0, BundleObjectProperties.ContentType, "/modules/search/sqlite/lunet-sql-wasm.wasm", "/js/lunet-sql-wasm.wasm");
defaultBundle.InsertLink(0, BundleObjectProperties.ContentType, "/modules/search/sqlite/lunet-search-sqlite.js", "/js/lunet-search.js");
defaultBundle.InsertLink(0, BundleObjectProperties.ContentType, "/modules/search/sqlite/lunet-sql-wasm.js", "/js/lunet-sql-wasm.js");
defaultBundle.InsertLink(0, BundleObjectProperties.JsType, "/modules/search/sqlite/lunet-search-ws-client.js");
}
else
{
// Insert content before the others to make sure they are loaded async ASAP
defaultBundle.InsertLink(0, BundleObjectProperties.ContentType, "/modules/search/sqlite/lunet-sql-wasm.wasm", "/js/lunet-sql-wasm.wasm");
defaultBundle.InsertLink(0, BundleObjectProperties.JsType, "/modules/search/sqlite/lunet-search-sqlite.js");
defaultBundle.InsertLink(0, BundleObjectProperties.JsType, "/modules/search/sqlite/lunet-sql-wasm.js");
}
}
private async Task <ContentObject> LoadContent(FileEntry file)
{
ContentObject page = null;
var buffer = new byte[16];
var clock = Stopwatch.StartNew();
var stream = file.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
try
{
var count = await stream.ReadAsync(buffer, 0, buffer.Length);
// Rewind to 0
stream.Position = 0;
bool hasFrontMatter = false;
bool isBinary = false;
int startFrontMatter = 0;
// Does it start with UTF8 BOM? If yes, skip it
// EF BB BF
if (buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF)
{
startFrontMatter = 3;
}
// TODO: avoid the chars alloc
var charBuffer = Encoding.UTF8.GetChars(buffer, startFrontMatter, buffer.Length - startFrontMatter);
foreach (var frontParser in Scripts.FrontMatterParsers)
{
if (frontParser.CanHandle(charBuffer))
{
for (int i = startFrontMatter + 3; i < count; i++)
{
if (buffer[i] == 0)
{
isBinary = true;
break;
}
}
if (!isBinary)
{
hasFrontMatter = true;
}
break;
}
}
// Run pre-content (e.g AttributesPlugin)
ScriptObject preContent = null;
foreach (var preLoadingContentProcessor in BeforeLoadingContentProcessors)
{
preLoadingContentProcessor(file.Path, ref preContent);
}
if (hasFrontMatter)
{
page = await LoadPageScript(Site, stream, file, preContent);
stream = null;
}
else
{
page = new FileContentObject(Site, file, preContent: preContent);
//// Run pre-processing on static content as well
//var pendingPageProcessors = new OrderedList<IContentProcessor>();
//TryProcessPage(page, ContentProcessingStage.AfterLoading, AfterLoadingProcessors, pendingPageProcessors, false);
}
// Initialize the page loaded
page?.Initialize();
}
finally
{
// Dispose stream used
stream?.Dispose();
}
clock.Stop();
Site.Statistics.GetContentStat(page).LoadingParsingTime += clock.Elapsed;
return(page);
}
private void ProcessBundleLinks(BundleObject bundle, Dictionary <UPath, ContentObject> staticFiles)
{
Dictionary <string, ConcatGroup> concatBuilders = null;
if (bundle.Concat)
{
concatBuilders = new Dictionary <string, ConcatGroup>();
foreach (var type in bundle.UrlDestination)
{
if (!concatBuilders.ContainsKey(type.Key))
{
concatBuilders[type.Key] = new ConcatGroup(type.Key);
}
}
}
// Expand wildcard * links
for (int i = 0; i < bundle.Links.Count; i++)
{
var link = bundle.Links[i];
var path = link.Path;
var url = link.Url;
if (!path.Contains("*"))
{
continue;
}
// Always remove the link
bundle.Links.RemoveAt(i);
var upath = (UPath)path;
bool matchingInFileSystem = false;
foreach (var file in Site.FileSystem.EnumerateFileSystemEntries(upath.GetDirectory(), upath.GetName()))
{
var newLink = new BundleLink(bundle, link.Type, (string)file.Path, url + file.Path.GetName(), link.Mode);
bundle.Links.Insert(i++, newLink);
matchingInFileSystem = true;
}
if (!matchingInFileSystem)
{
foreach (var file in Site.MetaFileSystem.EnumerateFileSystemEntries(upath.GetDirectory(), upath.GetName()))
{
var newLink = new BundleLink(bundle, link.Type, (string)file.Path, url + file.Path.GetName(), link.Mode);
bundle.Links.Insert(i++, newLink);
}
}
// Cancel the double i++
i--;
}
// Collect minifier
IContentMinifier minifier = null;
if (bundle.Minify)
{
var minifierName = bundle.Minifier;
foreach (var min in Minifiers)
{
if (minifierName == null || min.Name == minifierName)
{
minifier = min;
break;
}
}
if (minifier == null)
{
Site.Warning($"Minify is setup for bundle [{bundle.Name}] but no minifiers are registered (Minified requested: {minifierName ?? "default"})");
}
}
// Process links
for (int i = 0; i < bundle.Links.Count; i++)
{
var link = bundle.Links[i];
var path = link.Path;
var url = link.Url;
if (url != null)
{
if (!UPath.TryParse(url, out _))
{
Site.Error($"Invalid absolute url [{url}] in bundle [{bundle.Name}]");
}
}
if (path != null)
{
path = ((UPath)path).FullName;
link.Path = path;
ContentObject currentContent;
var isExistingContent = staticFiles.TryGetValue(path, out currentContent);
if (url == null)
{
var outputUrlDirectory = bundle.UrlDestination[link.Type];
// If the file is private or meta, we need to copy to the output
// bool isFilePrivateOrMeta = Site.IsFilePrivateOrMeta(entry.FullName);
url = outputUrlDirectory + Path.GetFileName(path);
link.Url = url;
}
FileEntry entry = null;
// Process file by existing processors
if (currentContent == null)
{
// Check first site, then meta
entry = new FileEntry(Site.FileSystem, path);
if (!entry.Exists)
{
entry = new FileEntry(Site.MetaFileSystem, path);
}
if (entry.Exists)
{
currentContent = new FileContentObject(Site, entry);
}
else
{
Site.Error($"Unable to find content [{path}] in bundle [{bundle.Name}]");
}
}
if (currentContent != null)
{
currentContent.Url = url;
var listTemp = new PageCollection()
{
currentContent
};
Site.Content.ProcessPages(listTemp, false);
link.ContentObject = currentContent;
bool isRawContent = link.Type == BundleObjectProperties.ContentType;
// If we require concat and/or minify, we preload the content of the file
if (!isRawContent && (bundle.Concat || bundle.Minify))
{
try
{
link.Content = currentContent.Content ?? entry.ReadAllText();
// Minify content separately
if (bundle.Minify && minifier != null)
{
Minify(minifier, link, bundle.MinifyExtension);
}
}
catch (Exception ex)
{
Site.Error($"Unable to load content [{path}] while trying to concatenate for bundle [{bundle.Name}]. Reason: {ex.GetReason()}");
}
}
// Remove sourcemaps (TODO: make this configurable)
RemoveSourceMaps(link);
// If we are concatenating
if (!isRawContent && concatBuilders != null)
{
currentContent.Discard = true;
// Remove this link from the list of links, as we are going to squash them after
bundle.Links.RemoveAt(i);
i--;
concatBuilders[link.Type].Pages.Add(currentContent);
concatBuilders[link.Type].AppendContent(link.Mode, link.Content);
}
else if (!isExistingContent)
{
Site.StaticFiles.Add(currentContent);
staticFiles.Add(path, currentContent);
}
}
}
}
// Concatenate files if necessary
if (concatBuilders != null)
{
foreach (var builderGroup in concatBuilders)
{
foreach (var contentPerMode in builderGroup.Value.ModeToBuilders)
{
var mode = contentPerMode.Key;
var builder = contentPerMode.Value;
if (builder.Length > 0)
{
var type = builderGroup.Key;
var outputUrlDirectory = bundle.UrlDestination[type];
// If the file is private or meta, we need to copy to the output
// bool isFilePrivateOrMeta = Site.IsFilePrivateOrMeta(entry.FullName);
var url = outputUrlDirectory + bundle.Name + (string.IsNullOrEmpty(mode) ? $".{type}" : $"-{mode}.{type}");
var newStaticFile = new DynamicContentObject(Site, url)
{
Content = builder.ToString()
};
Site.DynamicPages.Add(newStaticFile);
// Add file dependencies
foreach (var page in builderGroup.Value.Pages)
{
newStaticFile.Dependencies.Add(new PageContentDependency(page));
}
var link = new BundleLink(bundle, type, null, url, mode)
{
Content = newStaticFile.Content,
ContentObject = newStaticFile
};
bundle.Links.Add(link);
}
}
}
}
foreach (var link in bundle.Links)
{
var contentObject = link.ContentObject;
if (contentObject != null)
{
link.Url = contentObject.Url;
}
}
}
