public void Initialize()
{
root = Helper.GetTestFolder <TestsCacheFileSystem>();
if (Directory.Exists(root))
{
Directory.Delete(root, true);
}
if (File.Exists(root))
{
File.Delete(root);
}
Directory.CreateDirectory(root);
cache = new CacheFileSystem(root, IONull.That);
zero = Str.Repeat("0", 1000);
files = new FileInfo[4];
for (var i = 0; i < 4; i++)
{
File.WriteAllText(GetCachedFileWithIndex(i), zero);
files[i] = new FileInfo(GetCachedFileWithIndex(i));
// Except the first one is outdated.
if (i != 0)
{
files[i].LastAccessTime = DateTime.Now.AddSeconds(-1000);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="RepositoryBucket"/> class.
/// </summary>
public RepositoryBucket(
ConfigRepositoryBucket configRepository,
IIO io,
Config config,
ITransport transport = null,
IEventDispatcher eventDispatcher = null,
IVersionParser versionParser = null)
{
if (!Regex.IsMatch(configRepository.Uri, @"^[\w.]+\??://"))
{
// assume http as the default protocol
configRepository.Uri = $"http://{configRepository.Uri}";
}
configRepository.Uri = configRepository.Uri.TrimEnd('/');
if (configRepository.Uri.StartsWith("https?", StringComparison.OrdinalIgnoreCase))
{
configRepository.Uri = $"https{configRepository.Uri.Substring(6)}";
uriIsIntelligent = true;
}
if (!Uri.TryCreate(configRepository.Uri, UriKind.RelativeOrAbsolute, out _))
{
throw new ConfigException($"Invalid url given for Bucket repository: {configRepository.Uri}");
}
// changes will not be applied to configRepository.
uri = configRepository.Uri;
// force url for gxpack.org to repo.gxpack.org
// without converting other addresses.
var match = Regex.Match(configRepository.Uri, @"^(?<proto>https?)://gxpack\.org/?$", RegexOptions.IgnoreCase);
if (match.Success)
{
uri = $"{match.Groups["proto"].Value}://repo.gxpack.org";
}
baseUri = Regex.Replace(uri, @"(?:/[^/\\\\]+\.json)?(?:[?#].*)?$", string.Empty).TrimEnd('/');
this.configRepository = configRepository;
this.io = io;
this.transport = transport ?? new TransportHttp(io, config);
this.eventDispatcher = eventDispatcher;
this.versionParser = versionParser ?? new BVersionParser();
var cacheDir = Path.Combine(config.Get(Settings.CacheDir), CacheFileSystem.FormatCacheFolder(uri));
cache = new CacheFileSystem(cacheDir, io, "a-z0-9.$~");
loader = new LoaderPackage(versionParser);
providersByUid = new Dictionary <int, IPackage>();
}
public void TestInstallWithCache()
{
var packageMock = new Mock <IPackage>();
packageMock.Setup((o) => o.GetSourceReference())
.Returns("1234567890123456789012345678901234567890");
packageMock.Setup((o) => o.GetSourceUris())
.Returns(new[] { "https://example.com/bucket/bucket" });
packageMock.Setup((o) => o.GetSourceUri())
.Returns("https://example.com/bucket/bucket");
packageMock.Setup((o) => o.GetVersionPretty())
.Returns("dev-master");
var processMock = new Mock <IProcessExecutor>();
var expectedGitCommand = WinCompat("git --version");
processMock.Setup(expectedGitCommand, "git version 2.3.1");
var config = new Config();
SetupConfig(config);
var cachePath = config.Get(Settings.CacheVcsDir) + $"/{CacheFileSystem.FormatCacheFolder("https://example.com/bucket/bucket")}/";
expectedGitCommand = WinCompat($"git clone --mirror 'https://example.com/bucket/bucket' '{cachePath}'");
processMock.Setup(expectedGitCommand, returnValue: () =>
{
if (Directory.Exists(cachePath))
{
Directory.Delete(cachePath, true);
}
Directory.CreateDirectory(cachePath);
return(0);
});
expectedGitCommand = WinCompat($"git clone --no-checkout '{cachePath}' 'bucketPath' --dissociate --reference '{cachePath}' && cd 'bucketPath' && git remote set-url origin 'https://example.com/bucket/bucket' && git remote add bucket 'https://example.com/bucket/bucket'");
processMock.Setup(expectedGitCommand);
expectedGitCommand = WinCompat("git branch -r");
processMock.Setup(expectedGitCommand, expectedCwd: bucketPath);
expectedGitCommand = WinCompat("git checkout master --");
processMock.Setup(expectedGitCommand, expectedCwd: bucketPath);
expectedGitCommand = WinCompat("git reset --hard 1234567890123456789012345678901234567890 --");
processMock.Setup(expectedGitCommand, expectedCwd: bucketPath);
var downloader = CreateDownloaderMock(config: config, process: processMock.Object);
downloader.Install(packageMock.Object, "bucketPath");
processMock.VerifyAll();
}
public void TestGCSubfolderFiles()
{
var subfolderCache = new CacheFileSystem(root, IONull.That, null);
var path = Path.Combine(root, "foo/bar.zip");
Directory.CreateDirectory(Path.Combine(root, "foo"));
File.WriteAllText(path, zero);
Assert.AreEqual(zero, subfolderCache.Read("foo/bar.zip").ToText());
var file = new FileInfo(path);
file.LastAccessTime = DateTime.Now.AddSeconds(-1000);
cache.GC(600, 1024 * 1024 * 1024);
Assert.AreEqual(false, subfolderCache.Contains("foo/bar.zip"));
}
public async Task Execute(CancellationToken cancellationToken = default)
{
using var _ = _logger.BeginScope(nameof(Execute));
_logger.LogInformation("Executing pipeline");
await CacheFileSystem.DeleteDir("content");
await CacheFileSystem.GetOrCreateDirectory("content");
await CacheFileSystem.DeleteDir("content-html");
await CacheFileSystem.GetOrCreateDirectory("content-html");
var outputPath = GetRootedPath(CurrentPath, Site.OutputPath);
await FileSystem.CleanDirectory(outputPath);
await FileSystem.GetOrCreateDirectory(outputPath);
RawCache = await CacheFileSystem.Mount("content");
PageCache = await CacheFileSystem.Mount("content-html");
OutputFs = await FileSystem.Mount(outputPath);
/* Aggregate content based on branches, tags and etc given in site definition */
var aggregator = new ContentAggregator(
Site,
GitRoot,
FileSystem,
new MimeDbClassifier());
var catalog = new Catalog();
/* Add content */
await catalog.Add(aggregator.Aggregate(cancellationToken), cancellationToken);
/* Site */
var site = await BuildSite(catalog, Site);
/* UI */
var ui = new UiBuilder(PageCache, OutputFs);
await ui.BuildSite(site);
_logger.LogInformation("Pipeline completed");
}
/// <inheritdoc />
public override void Initialize()
{
string cacheUri;
if (FileSystemLocal.IsLocalPath(Uri))
{
Uri = Regex.Replace(Uri, @"[\\/]\.git/?$", string.Empty);
repositoryDirectory = Uri;
cacheUri = Path.GetFullPath(Uri);
}
else
{
var cacheVcsDir = Config.Get(Settings.CacheVcsDir);
if (!CacheFileSystem.IsUsable(cacheVcsDir))
{
throw new RuntimeException("DriverGit requires a usable cache directory, and it looks like you set it to be disabled.");
}
if (Regex.IsMatch("^ssh://[^@]+@[^:]+:[^0-9]+", Uri))
{
throw new InvalidArgumentException($"The source URL {Uri} is invalid, ssh URLs should have a port number after \":\".{Environment.NewLine}Use ssh://[email protected]:22/path or just [email protected]:path if you do not want to provide a password or custom port.");
}
Git.CleanEnvironment();
var fileSystem = new FileSystemLocal($"{cacheVcsDir}/{CacheFileSystem.FormatCacheFolder(Uri)}/");
var git = new Git(IO, Config, Process, fileSystem);
repositoryDirectory = fileSystem.Root;
if (!git.SyncMirror(Uri, repositoryDirectory))
{
IO.WriteError($"<error>Failed to update {Uri}, package information from this repository may be outdated</error>");
}
cacheUri = Uri;
}
GetTags();
GetBranches();
cache = new CacheFileSystem($"{Config.Get(Settings.CacheRepoDir)}/{CacheFileSystem.FormatCacheFolder(cacheUri)}", IO);
}
private CacheFileSystem CreateCache(string cacheConfigKey)
{
string cachePath = config.Get(cacheConfigKey);
if (string.IsNullOrEmpty(cachePath))
{
io.WriteError($"<info>Cache directory does not exist ({cacheConfigKey}): {cachePath}</info>");
return(null);
}
cachePath = Path.Combine(Environment.CurrentDirectory, cachePath);
var cache = new CacheFileSystem(cachePath, io);
if (!cache.Enable)
{
io.WriteError($"<info>Cache is not enabled ({cacheConfigKey}): {cachePath}</info>");
return(null);
}
return(cache);
}
/// <summary>
/// Initializes a new instance of the <see cref="DownloaderFile"/> class.
/// </summary>
public DownloaderFile(
IIO io,
Config config,
ITransport transport,
IEventDispatcher eventDispatcher = null,
ICache cache = null,
IFileSystem fileSystem = null)
{
this.io = io;
this.config = config;
this.transport = transport;
this.eventDispatcher = eventDispatcher;
this.cache = cache;
this.fileSystem = fileSystem ?? new FileSystemLocal();
if (CacheFileSystem.GCIsNecessary(cache))
{
cache.GC(config.Get(Settings.CacheFilesTTL), config.Get(Settings.CacheFilesMaxSize));
}
lastCacheWrites = new Dictionary <string, string>();
}
public void Initialize()
{
root = Helper.GetTestFolder <TestsExtensionICache>();
cache = new CacheFileSystem(root, IONull.That);
}
/// <summary>
/// Create a new <see cref="DownloadManager"/> instance.
/// </summary>
/// <param name="io">The input/output instance.</param>
/// <param name="config">The config instance.</param>
/// <param name="transport">The transport instance.</param>
/// <param name="eventDispatcher">The event dispatcher instance.</param>
public static DownloadManager CreateManager(IIO io, Config config, ITransport transport = null, IEventDispatcher eventDispatcher = null)
{
var manager = new DownloadManager(io);
InstallationSource GetInstallationSource(string prefer)
{
switch (prefer ?? "auto")
{
case "dist":
return(InstallationSource.Dist);
case "source":
return(InstallationSource.Source);
case "auto":
default:
return(InstallationSource.Auto);
}
}
try
{
var prefer = GetInstallationSource(config.Get(Settings.PreferredInstall));
switch (prefer)
{
case InstallationSource.Dist:
manager.SetPreferDist();
break;
case InstallationSource.Source:
manager.SetPreferSource();
break;
default:
break;
}
}
catch (ConfigException)
{
// Maybe the developer is using fine configuration.
var preferred = config.Get <ConfigPreferred>(Settings.PreferredInstall);
manager.SetPreferences(Arr.Map(preferred, (item) => (item.Key, GetInstallationSource(item.Value))));
}
transport = transport ?? new TransportHttp(io, config);
var process = new BucketProcessExecutor(io);
var fileSystem = new FileSystemLocal(process: process);
ICache cache = null;
if (config.Get(Settings.CacheFilesTTL) > 0)
{
cache = new CacheFileSystem(config.Get(Settings.CacheFilesDir), io, "a-z0-9_./", fileSystem);
}
manager.SetDownloader("git", new DownloaderGit(io, config, process, fileSystem));
manager.SetDownloader("zip", new DownloaderZip(io, config, transport, eventDispatcher, cache, fileSystem, process));
manager.SetDownloader("file", new DownloaderFile(io, config, transport, eventDispatcher, cache, fileSystem));
return(manager);
}
/// <inheritdoc />
protected override void DoInstall(IPackage package, string cwd, string uri)
{
Git.CleanEnvironment();
cwd = NormalizePath(cwd);
var cachePath = Config.Get(Settings.CacheVcsDir) + $"/{CacheFileSystem.FormatCacheFolder(uri)}/";
var reference = package.GetSourceReference();
var flag = Platform.IsWindows ? "/D " : string.Empty;
// --dissociate option is only available since git 2.3.0-rc0
var gitVersion = git.GetVersion();
var message = $"Cloning {GetShortHash(reference)}";
var command = $"git clone --no-checkout %uri% %path% && cd {flag}%path% && git remote add bucket %uri% && git fetch bucket";
if (!string.IsNullOrEmpty(gitVersion) &&
Comparator.GreaterThanOrEqual(gitVersion, "2.3.0-rc0") &&
CacheFileSystem.IsUsable(cachePath))
{
IO.WriteError(string.Empty, true, Verbosities.Debug);
IO.WriteError($" Cloning to cache at {ProcessExecutor.Escape(cachePath)}", true, Verbosities.Debug);
try
{
git.FetchReferenceOrSyncMirror(uri, reference, cachePath);
if (FileSystem.Exists(cachePath, FileSystemOptions.Directory))
{
command = "git clone --no-checkout %cachePath% %path% --dissociate --reference %cachePath%"
+ $" && cd {flag}%path%"
+ " && git remote set-url origin %uri% && git remote add bucket %uri%";
message = $"Cloning {GetShortHash(reference)} from cache.";
}
}
catch (RuntimeException)
{
// ignore runtime exception because this is an optimization solution.
}
}
IO.WriteError(message);
string CommandCallable(string authUri)
{
var template = command;
template = template.Replace("%uri%", ProcessExecutor.Escape(authUri));
template = template.Replace("%path%", ProcessExecutor.Escape(cwd));
template = template.Replace("%cachePath%", ProcessExecutor.Escape(cachePath));
return(template);
}
git.ExecuteCommand(CommandCallable, uri, cwd, true);
if (uri != package.GetSourceUri())
{
UpdateOriginUri(package.GetSourceUri(), cwd);
}
else
{
SetPushUri(uri, cwd);
}
ExecuteUpdate(cwd, reference, package);
}
public static void SaveCache(CacheFileSystem fileSystem, ExtendedFrameworkVersion frameworkVersion)
{
string fileName = GetFileName(frameworkVersion);
using (MemoryStream stream = new MemoryStream())
{
using (BinaryWriter writer = new BinaryWriter(stream))
{
bool hasAssemblyCache = assemblyCache.ContainsKey(frameworkVersion);
writer.Write(hasAssemblyCache);
if (hasAssemblyCache)
WriteStrings(writer, assemblyCache[frameworkVersion].ToArray());
WriteStrings(writer, namespaceCache.ToArray());
writer.Flush();
stream.Position = 0;
fileSystem.Write(fileName, stream);
}
}
}
public static void LoadCache(CacheFileSystem fileSystem, ExtendedFrameworkVersion frameworkVersion)
{
if (assemblyCache.ContainsKey(frameworkVersion))
assemblyCache[frameworkVersion].Clear();
else
assemblyCache.Add(frameworkVersion, new List<string>());
string fileName = GetFileName(frameworkVersion);
Stream stream = fileSystem.Read(fileName);
if (stream == null)
return;
using (BinaryReader reader = new BinaryReader(stream))
{
bool hasAssemblyCache = reader.ReadBoolean();
if (hasAssemblyCache)
assemblyCache[frameworkVersion].AddRange(ReadStrings(reader));
namespaceCache.Clear();
namespaceCache.AddRange(ReadStrings(reader));
}
}
