public void Initialize() { root = Helper.GetTestFolder <TestsCacheFileSystem>(); if (Directory.Exists(root)) { Directory.Delete(root, true); } if (File.Exists(root)) { File.Delete(root); } Directory.CreateDirectory(root); cache = new CacheFileSystem(root, IONull.That); zero = Str.Repeat("0", 1000); files = new FileInfo[4]; for (var i = 0; i < 4; i++) { File.WriteAllText(GetCachedFileWithIndex(i), zero); files[i] = new FileInfo(GetCachedFileWithIndex(i)); // Except the first one is outdated. if (i != 0) { files[i].LastAccessTime = DateTime.Now.AddSeconds(-1000); } } }
/// <summary> /// Initializes a new instance of the <see cref="RepositoryBucket"/> class. /// </summary> public RepositoryBucket( ConfigRepositoryBucket configRepository, IIO io, Config config, ITransport transport = null, IEventDispatcher eventDispatcher = null, IVersionParser versionParser = null) { if (!Regex.IsMatch(configRepository.Uri, @"^[\w.]+\??://")) { // assume http as the default protocol configRepository.Uri = $"http://{configRepository.Uri}"; } configRepository.Uri = configRepository.Uri.TrimEnd('/'); if (configRepository.Uri.StartsWith("https?", StringComparison.OrdinalIgnoreCase)) { configRepository.Uri = $"https{configRepository.Uri.Substring(6)}"; uriIsIntelligent = true; } if (!Uri.TryCreate(configRepository.Uri, UriKind.RelativeOrAbsolute, out _)) { throw new ConfigException($"Invalid url given for Bucket repository: {configRepository.Uri}"); } // changes will not be applied to configRepository. uri = configRepository.Uri; // force url for gxpack.org to repo.gxpack.org // without converting other addresses. var match = Regex.Match(configRepository.Uri, @"^(?<proto>https?)://gxpack\.org/?$", RegexOptions.IgnoreCase); if (match.Success) { uri = $"{match.Groups["proto"].Value}://repo.gxpack.org"; } baseUri = Regex.Replace(uri, @"(?:/[^/\\\\]+\.json)?(?:[?#].*)?$", string.Empty).TrimEnd('/'); this.configRepository = configRepository; this.io = io; this.transport = transport ?? new TransportHttp(io, config); this.eventDispatcher = eventDispatcher; this.versionParser = versionParser ?? new BVersionParser(); var cacheDir = Path.Combine(config.Get(Settings.CacheDir), CacheFileSystem.FormatCacheFolder(uri)); cache = new CacheFileSystem(cacheDir, io, "a-z0-9.$~"); loader = new LoaderPackage(versionParser); providersByUid = new Dictionary <int, IPackage>(); }
public void TestInstallWithCache() { var packageMock = new Mock <IPackage>(); packageMock.Setup((o) => o.GetSourceReference()) .Returns("1234567890123456789012345678901234567890"); packageMock.Setup((o) => o.GetSourceUris()) .Returns(new[] { "https://example.com/bucket/bucket" }); packageMock.Setup((o) => o.GetSourceUri()) .Returns("https://example.com/bucket/bucket"); packageMock.Setup((o) => o.GetVersionPretty()) .Returns("dev-master"); var processMock = new Mock <IProcessExecutor>(); var expectedGitCommand = WinCompat("git --version"); processMock.Setup(expectedGitCommand, "git version 2.3.1"); var config = new Config(); SetupConfig(config); var cachePath = config.Get(Settings.CacheVcsDir) + $"/{CacheFileSystem.FormatCacheFolder("https://example.com/bucket/bucket")}/"; expectedGitCommand = WinCompat($"git clone --mirror 'https://example.com/bucket/bucket' '{cachePath}'"); processMock.Setup(expectedGitCommand, returnValue: () => { if (Directory.Exists(cachePath)) { Directory.Delete(cachePath, true); } Directory.CreateDirectory(cachePath); return(0); }); expectedGitCommand = WinCompat($"git clone --no-checkout '{cachePath}' 'bucketPath' --dissociate --reference '{cachePath}' && cd 'bucketPath' && git remote set-url origin 'https://example.com/bucket/bucket' && git remote add bucket 'https://example.com/bucket/bucket'"); processMock.Setup(expectedGitCommand); expectedGitCommand = WinCompat("git branch -r"); processMock.Setup(expectedGitCommand, expectedCwd: bucketPath); expectedGitCommand = WinCompat("git checkout master --"); processMock.Setup(expectedGitCommand, expectedCwd: bucketPath); expectedGitCommand = WinCompat("git reset --hard 1234567890123456789012345678901234567890 --"); processMock.Setup(expectedGitCommand, expectedCwd: bucketPath); var downloader = CreateDownloaderMock(config: config, process: processMock.Object); downloader.Install(packageMock.Object, "bucketPath"); processMock.VerifyAll(); }
public void TestGCSubfolderFiles() { var subfolderCache = new CacheFileSystem(root, IONull.That, null); var path = Path.Combine(root, "foo/bar.zip"); Directory.CreateDirectory(Path.Combine(root, "foo")); File.WriteAllText(path, zero); Assert.AreEqual(zero, subfolderCache.Read("foo/bar.zip").ToText()); var file = new FileInfo(path); file.LastAccessTime = DateTime.Now.AddSeconds(-1000); cache.GC(600, 1024 * 1024 * 1024); Assert.AreEqual(false, subfolderCache.Contains("foo/bar.zip")); }
public async Task Execute(CancellationToken cancellationToken = default) { using var _ = _logger.BeginScope(nameof(Execute)); _logger.LogInformation("Executing pipeline"); await CacheFileSystem.DeleteDir("content"); await CacheFileSystem.GetOrCreateDirectory("content"); await CacheFileSystem.DeleteDir("content-html"); await CacheFileSystem.GetOrCreateDirectory("content-html"); var outputPath = GetRootedPath(CurrentPath, Site.OutputPath); await FileSystem.CleanDirectory(outputPath); await FileSystem.GetOrCreateDirectory(outputPath); RawCache = await CacheFileSystem.Mount("content"); PageCache = await CacheFileSystem.Mount("content-html"); OutputFs = await FileSystem.Mount(outputPath); /* Aggregate content based on branches, tags and etc given in site definition */ var aggregator = new ContentAggregator( Site, GitRoot, FileSystem, new MimeDbClassifier()); var catalog = new Catalog(); /* Add content */ await catalog.Add(aggregator.Aggregate(cancellationToken), cancellationToken); /* Site */ var site = await BuildSite(catalog, Site); /* UI */ var ui = new UiBuilder(PageCache, OutputFs); await ui.BuildSite(site); _logger.LogInformation("Pipeline completed"); }
/// <inheritdoc /> public override void Initialize() { string cacheUri; if (FileSystemLocal.IsLocalPath(Uri)) { Uri = Regex.Replace(Uri, @"[\\/]\.git/?$", string.Empty); repositoryDirectory = Uri; cacheUri = Path.GetFullPath(Uri); } else { var cacheVcsDir = Config.Get(Settings.CacheVcsDir); if (!CacheFileSystem.IsUsable(cacheVcsDir)) { throw new RuntimeException("DriverGit requires a usable cache directory, and it looks like you set it to be disabled."); } if (Regex.IsMatch("^ssh://[^@]+@[^:]+:[^0-9]+", Uri)) { throw new InvalidArgumentException($"The source URL {Uri} is invalid, ssh URLs should have a port number after \":\".{Environment.NewLine}Use ssh://[email protected]:22/path or just [email protected]:path if you do not want to provide a password or custom port."); } Git.CleanEnvironment(); var fileSystem = new FileSystemLocal($"{cacheVcsDir}/{CacheFileSystem.FormatCacheFolder(Uri)}/"); var git = new Git(IO, Config, Process, fileSystem); repositoryDirectory = fileSystem.Root; if (!git.SyncMirror(Uri, repositoryDirectory)) { IO.WriteError($"<error>Failed to update {Uri}, package information from this repository may be outdated</error>"); } cacheUri = Uri; } GetTags(); GetBranches(); cache = new CacheFileSystem($"{Config.Get(Settings.CacheRepoDir)}/{CacheFileSystem.FormatCacheFolder(cacheUri)}", IO); }
private CacheFileSystem CreateCache(string cacheConfigKey) { string cachePath = config.Get(cacheConfigKey); if (string.IsNullOrEmpty(cachePath)) { io.WriteError($"<info>Cache directory does not exist ({cacheConfigKey}): {cachePath}</info>"); return(null); } cachePath = Path.Combine(Environment.CurrentDirectory, cachePath); var cache = new CacheFileSystem(cachePath, io); if (!cache.Enable) { io.WriteError($"<info>Cache is not enabled ({cacheConfigKey}): {cachePath}</info>"); return(null); } return(cache); }
/// <summary> /// Initializes a new instance of the <see cref="DownloaderFile"/> class. /// </summary> public DownloaderFile( IIO io, Config config, ITransport transport, IEventDispatcher eventDispatcher = null, ICache cache = null, IFileSystem fileSystem = null) { this.io = io; this.config = config; this.transport = transport; this.eventDispatcher = eventDispatcher; this.cache = cache; this.fileSystem = fileSystem ?? new FileSystemLocal(); if (CacheFileSystem.GCIsNecessary(cache)) { cache.GC(config.Get(Settings.CacheFilesTTL), config.Get(Settings.CacheFilesMaxSize)); } lastCacheWrites = new Dictionary <string, string>(); }
public void Initialize() { root = Helper.GetTestFolder <TestsExtensionICache>(); cache = new CacheFileSystem(root, IONull.That); }
/// <summary> /// Create a new <see cref="DownloadManager"/> instance. /// </summary> /// <param name="io">The input/output instance.</param> /// <param name="config">The config instance.</param> /// <param name="transport">The transport instance.</param> /// <param name="eventDispatcher">The event dispatcher instance.</param> public static DownloadManager CreateManager(IIO io, Config config, ITransport transport = null, IEventDispatcher eventDispatcher = null) { var manager = new DownloadManager(io); InstallationSource GetInstallationSource(string prefer) { switch (prefer ?? "auto") { case "dist": return(InstallationSource.Dist); case "source": return(InstallationSource.Source); case "auto": default: return(InstallationSource.Auto); } } try { var prefer = GetInstallationSource(config.Get(Settings.PreferredInstall)); switch (prefer) { case InstallationSource.Dist: manager.SetPreferDist(); break; case InstallationSource.Source: manager.SetPreferSource(); break; default: break; } } catch (ConfigException) { // Maybe the developer is using fine configuration. var preferred = config.Get <ConfigPreferred>(Settings.PreferredInstall); manager.SetPreferences(Arr.Map(preferred, (item) => (item.Key, GetInstallationSource(item.Value)))); } transport = transport ?? new TransportHttp(io, config); var process = new BucketProcessExecutor(io); var fileSystem = new FileSystemLocal(process: process); ICache cache = null; if (config.Get(Settings.CacheFilesTTL) > 0) { cache = new CacheFileSystem(config.Get(Settings.CacheFilesDir), io, "a-z0-9_./", fileSystem); } manager.SetDownloader("git", new DownloaderGit(io, config, process, fileSystem)); manager.SetDownloader("zip", new DownloaderZip(io, config, transport, eventDispatcher, cache, fileSystem, process)); manager.SetDownloader("file", new DownloaderFile(io, config, transport, eventDispatcher, cache, fileSystem)); return(manager); }
/// <inheritdoc /> protected override void DoInstall(IPackage package, string cwd, string uri) { Git.CleanEnvironment(); cwd = NormalizePath(cwd); var cachePath = Config.Get(Settings.CacheVcsDir) + $"/{CacheFileSystem.FormatCacheFolder(uri)}/"; var reference = package.GetSourceReference(); var flag = Platform.IsWindows ? "/D " : string.Empty; // --dissociate option is only available since git 2.3.0-rc0 var gitVersion = git.GetVersion(); var message = $"Cloning {GetShortHash(reference)}"; var command = $"git clone --no-checkout %uri% %path% && cd {flag}%path% && git remote add bucket %uri% && git fetch bucket"; if (!string.IsNullOrEmpty(gitVersion) && Comparator.GreaterThanOrEqual(gitVersion, "2.3.0-rc0") && CacheFileSystem.IsUsable(cachePath)) { IO.WriteError(string.Empty, true, Verbosities.Debug); IO.WriteError($" Cloning to cache at {ProcessExecutor.Escape(cachePath)}", true, Verbosities.Debug); try { git.FetchReferenceOrSyncMirror(uri, reference, cachePath); if (FileSystem.Exists(cachePath, FileSystemOptions.Directory)) { command = "git clone --no-checkout %cachePath% %path% --dissociate --reference %cachePath%" + $" && cd {flag}%path%" + " && git remote set-url origin %uri% && git remote add bucket %uri%"; message = $"Cloning {GetShortHash(reference)} from cache."; } } catch (RuntimeException) { // ignore runtime exception because this is an optimization solution. } } IO.WriteError(message); string CommandCallable(string authUri) { var template = command; template = template.Replace("%uri%", ProcessExecutor.Escape(authUri)); template = template.Replace("%path%", ProcessExecutor.Escape(cwd)); template = template.Replace("%cachePath%", ProcessExecutor.Escape(cachePath)); return(template); } git.ExecuteCommand(CommandCallable, uri, cwd, true); if (uri != package.GetSourceUri()) { UpdateOriginUri(package.GetSourceUri(), cwd); } else { SetPushUri(uri, cwd); } ExecuteUpdate(cwd, reference, package); }
public static void SaveCache(CacheFileSystem fileSystem, ExtendedFrameworkVersion frameworkVersion) { string fileName = GetFileName(frameworkVersion); using (MemoryStream stream = new MemoryStream()) { using (BinaryWriter writer = new BinaryWriter(stream)) { bool hasAssemblyCache = assemblyCache.ContainsKey(frameworkVersion); writer.Write(hasAssemblyCache); if (hasAssemblyCache) WriteStrings(writer, assemblyCache[frameworkVersion].ToArray()); WriteStrings(writer, namespaceCache.ToArray()); writer.Flush(); stream.Position = 0; fileSystem.Write(fileName, stream); } } }
public static void LoadCache(CacheFileSystem fileSystem, ExtendedFrameworkVersion frameworkVersion) { if (assemblyCache.ContainsKey(frameworkVersion)) assemblyCache[frameworkVersion].Clear(); else assemblyCache.Add(frameworkVersion, new List<string>()); string fileName = GetFileName(frameworkVersion); Stream stream = fileSystem.Read(fileName); if (stream == null) return; using (BinaryReader reader = new BinaryReader(stream)) { bool hasAssemblyCache = reader.ReadBoolean(); if (hasAssemblyCache) assemblyCache[frameworkVersion].AddRange(ReadStrings(reader)); namespaceCache.Clear(); namespaceCache.AddRange(ReadStrings(reader)); } }