public async ValueTask <(Status, IDictionary <FileType, IList <FileToken> >)> TokenizeAsync(Options options, DocFxConfig config) { var dirUri = options.DirectoryUri; var dir = options.ExplicitScope ? options.Directory : options.DocFxJsonDirectory; if (dir is null) { return(Status.Error, null); } var count = 0; Spinner.Start("Gathering files...", spinner => { spinner.Color = ConsoleColor.Blue; count = dir.EnumerateDirectories() .AsParallel() .SelectMany(d => d.EnumerateFiles("*.*", SearchOption.AllDirectories)) .Count(); spinner.Succeed(); }, Patterns.Arc); var tokens = new ConcurrentBag <FileToken>(); var destination = config.Build.Dest; using (var progressBar = new ProgressBar(count, "Tokenizing files...")) { var searchPattern = options.ReportFreshness ? "*.md" : "*.*"; await dir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) .ForEachAsync( Environment.ProcessorCount, async fileInfo => { FileToken fileToken = fileInfo; await fileToken.InitializeAsync(options); var relyingOnCache = options.EnableCaching ? " (relying on cache)" : string.Empty; progressBar.Tick($"Materialzing file tokens{relyingOnCache}...{dirUri.ToRelativePath(fileToken.FilePath)}"); tokens.Add(fileToken); }); progressBar.Tick("Materialization complete..."); } var cachedCount = FileTokenCacheUtility.CachedCount; if (cachedCount > 0) { ConsoleColor.Green.WriteLine($"Materialized {cachedCount:#,#} file tokens from the local cache rather than re-reading and parsing them."); } return(Status.Success, tokens.GroupBy(t => t.FileType).ToDictionary(grp => grp.Key, grp => grp.ToList() as IList <FileToken>)); }
internal static async ValueTask CacheTokenAsync(FileToken token, Options options, string destination) { try { if (!options.EnableCaching) { return; } token.WriteToProtoBufFile(await GetTokenCachePathAsync(token, options, destination)); } catch { } }
static async ValueTask <string> GetTokenCachePathAsync(FileToken token, Options options, string destination) { var dest = string.IsNullOrWhiteSpace(destination) ? (await options.GetConfigAsync()).Build.Dest : destination; var destDir = Path.Combine(CacheDir, dest); if (!Directory.Exists(destDir)) { Directory.CreateDirectory(destDir); } return(Path.Combine(destDir, token.CachedJsonFileName)); }
internal static FileTokenFreshness FromToken( FileToken token, int daysOld, string hostUrl, string source, string destination) { var header = token.Header; var tokenFreshness = new FileTokenFreshness { DaysOld = daysOld, NinetyDaysOldDate = header.Date.GetValueOrDefault().AddDays(90), FileName = Path.GetFileName(token.FilePath), FolderName = token.DirectoryName, Author = header.MicrosoftAuthor, Manager = header.Manager, Title = header.Title, Topic = header.Topic, Description = header.Description, PubDate = header.Date.GetValueOrDefault(), Subservice = header.Subservice }; if (LinkTopics.Contains(header.Topic)) { var index = token.FilePath.IndexOf(source); var route = token.FilePath .Substring(index) .Replace(source, destination) .Replace(".md", "") .Replace("\\", "/"); tokenFreshness.Link = $"{hostUrl}/{route}"; } return(tokenFreshness); }
internal static async ValueTask <bool> TryFindCachedVersionAsync(FileToken token, Options options, string destination) { try { if (!options.EnableCaching) { return(false); } var cachedTokenPath = await GetTokenCachePathAsync(token, options, destination); if (File.Exists(cachedTokenPath)) { token.MergeWith(cachedTokenPath.ReadFromProtoBufFile <FileToken>()); Interlocked.Increment(ref CachedCount); return(true); } } catch { } return(false); }