public async ValueTask <(Status, IDictionary <FileType, IList <FileToken> >)> TokenizeAsync(Options options, DocFxConfig config)
{
var dirUri = options.DirectoryUri;
var dir = options.ExplicitScope ? options.Directory : options.DocFxJsonDirectory;
if (dir is null)
{
return(Status.Error, null);
}
var count = 0;
Spinner.Start("Gathering files...", spinner =>
{
spinner.Color = ConsoleColor.Blue;
count =
dir.EnumerateDirectories()
.AsParallel()
.SelectMany(d => d.EnumerateFiles("*.*", SearchOption.AllDirectories))
.Count();
spinner.Succeed();
}, Patterns.Arc);
var tokens = new ConcurrentBag <FileToken>();
var destination = config.Build.Dest;
using (var progressBar = new ProgressBar(count, "Tokenizing files..."))
{
var searchPattern = options.ReportFreshness ? "*.md" : "*.*";
await dir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
.ForEachAsync(
Environment.ProcessorCount,
async fileInfo =>
{
FileToken fileToken = fileInfo;
await fileToken.InitializeAsync(options);
var relyingOnCache = options.EnableCaching ? " (relying on cache)" : string.Empty;
progressBar.Tick($"Materialzing file tokens{relyingOnCache}...{dirUri.ToRelativePath(fileToken.FilePath)}");
tokens.Add(fileToken);
});
progressBar.Tick("Materialization complete...");
}
var cachedCount = FileTokenCacheUtility.CachedCount;
if (cachedCount > 0)
{
ConsoleColor.Green.WriteLine($"Materialized {cachedCount:#,#} file tokens from the local cache rather than re-reading and parsing them.");
}
return(Status.Success, tokens.GroupBy(t => t.FileType).ToDictionary(grp => grp.Key, grp => grp.ToList() as IList <FileToken>));
}
internal static async ValueTask CacheTokenAsync(FileToken token, Options options, string destination)
{
try
{
if (!options.EnableCaching)
{
return;
}
token.WriteToProtoBufFile(await GetTokenCachePathAsync(token, options, destination));
}
catch
{
}
}
static async ValueTask <string> GetTokenCachePathAsync(FileToken token, Options options, string destination)
{
var dest =
string.IsNullOrWhiteSpace(destination)
? (await options.GetConfigAsync()).Build.Dest
: destination;
var destDir = Path.Combine(CacheDir, dest);
if (!Directory.Exists(destDir))
{
Directory.CreateDirectory(destDir);
}
return(Path.Combine(destDir, token.CachedJsonFileName));
}
internal static FileTokenFreshness FromToken(
FileToken token,
int daysOld,
string hostUrl,
string source,
string destination)
{
var header = token.Header;
var tokenFreshness = new FileTokenFreshness
{
DaysOld = daysOld,
NinetyDaysOldDate = header.Date.GetValueOrDefault().AddDays(90),
FileName = Path.GetFileName(token.FilePath),
FolderName = token.DirectoryName,
Author = header.MicrosoftAuthor,
Manager = header.Manager,
Title = header.Title,
Topic = header.Topic,
Description = header.Description,
PubDate = header.Date.GetValueOrDefault(),
Subservice = header.Subservice
};
if (LinkTopics.Contains(header.Topic))
{
var index = token.FilePath.IndexOf(source);
var route =
token.FilePath
.Substring(index)
.Replace(source, destination)
.Replace(".md", "")
.Replace("\\", "/");
tokenFreshness.Link = $"{hostUrl}/{route}";
}
return(tokenFreshness);
}
internal static async ValueTask <bool> TryFindCachedVersionAsync(FileToken token, Options options, string destination)
{
try
{
if (!options.EnableCaching)
{
return(false);
}
var cachedTokenPath = await GetTokenCachePathAsync(token, options, destination);
if (File.Exists(cachedTokenPath))
{
token.MergeWith(cachedTokenPath.ReadFromProtoBufFile <FileToken>());
Interlocked.Increment(ref CachedCount);
return(true);
}
}
catch
{
}
return(false);
}