private static Manifest GenerateManifest(string path, ManifestFormat format, ITaskHandler handler)
{
var generator = new ManifestGenerator(path, format);
handler.RunTask(generator);
return(generator.Manifest);
}
public void TestFromPrefix()
{
Assert.AreSame(ManifestFormat.Sha1, ManifestFormat.FromPrefix("sha1=abc"));
Assert.AreSame(ManifestFormat.Sha1New, ManifestFormat.FromPrefix("sha1new=abc"));
Assert.AreSame(ManifestFormat.Sha256, ManifestFormat.FromPrefix("sha256=abc"));
Assert.AreSame(ManifestFormat.Sha256New, ManifestFormat.FromPrefix("sha256new_abc"));
}
public DedupKey(long size, DateTime lastModified, ManifestFormat format, string digest)
{
Size = size;
LastModified = lastModified;
Format = format;
Digest = digest;
}
public DedupKey(long size, DateTime lastModified, ManifestFormat format, string digest)
{
Size = size;
LastModified = lastModified;
Format = format;
Digest = digest;
}
/// <summary>
/// Prepares to generate a manifest for a directory in the filesystem.
/// </summary>
/// <param name="sourceDirectory">The path of the directory to analyze.</param>
/// <param name="format">The format of the manifest to generate.</param>
public ManifestGenerator([NotNull] string sourceDirectory, [NotNull] ManifestFormat format) : base(sourceDirectory)
{
#region Sanity checks
if (format == null) throw new ArgumentNullException(nameof(format));
#endregion
Format = format;
}
private string DeployPackage(string id, PackageBuilder builder)
{
string path = Path.Combine(_tempDir, id);
builder.WritePackageInto(path);
ManifestTest.CreateDotFile(path, ManifestFormat.FromPrefix(id), _handler);
FileUtils.EnableWriteProtection(path);
return(path);
}
/// <summary>
/// Prepares to generate a manifest for a directory in the filesystem.
/// </summary>
/// <param name="sourceDirectory">The path of the directory to analyze.</param>
/// <param name="format">The format of the manifest to generate.</param>
public ManifestGenerator([NotNull] string sourceDirectory, [NotNull] ManifestFormat format) : base(sourceDirectory)
{
#region Sanity checks
if (format == null)
{
throw new ArgumentNullException(nameof(format));
}
#endregion
Format = format;
}
/// <summary>
/// Creates a manifest node for a directory.
/// </summary>
/// <param name="directory">The directory object to create a node for.</param>
/// <param name="format">The manifest format containing digest rules.</param>
/// <param name="rootPath">The fully qualified path of the root directory the manifest is being generated for.</param>
/// <returns>The node for the list.</returns>
/// <exception cref="IOException">There was an error reading the directory.</exception>
/// <exception cref="UnauthorizedAccessException">You have insufficient rights to read the directory.</exception>
private ManifestNode GetDirectoryNode(DirectoryInfo directory, ManifestFormat format, string rootPath)
{
// Directory symlinks
string symlinkContents;
if (_isUnixFS && FileUtils.IsSymlink(directory.FullName, out symlinkContents))
{
var symlinkData = Encoding.UTF8.GetBytes(symlinkContents);
return(new ManifestSymlink(format.DigestContent(symlinkData), symlinkData.Length, directory.Name));
}
// Remove leading portion of path and use Unix slashes
string trimmedName = directory.FullName.Substring(rootPath.Length).Replace(Path.DirectorySeparatorChar, '/');
return(new ManifestDirectory(directory.LastWriteTime.ToUnixTime(), trimmedName));
}
/// <summary>
/// Creates a new manifest.
/// </summary>
/// <param name="format">The format used for <see cref="Save(Stream)"/>, also specifies the algorithm used in <see cref="ManifestFileBase.Digest"/>.</param>
/// <param name="nodes">A list of all elements in the tree this manifest represents.</param>
public Manifest([NotNull] ManifestFormat format, [NotNull, ItemNotNull, InstantHandle] params ManifestNode[] nodes)
{
#region Sanity checks
if (nodes == null)
{
throw new ArgumentNullException("nodes");
}
if (format == null)
{
throw new ArgumentNullException("format");
}
#endregion
Format = format;
_nodes = nodes;
}
public static Manifest Load([NotNull] string path, [NotNull] ManifestFormat format)
{
#region Sanity checks
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException("path");
}
if (format == null)
{
throw new ArgumentNullException("format");
}
#endregion
using (var stream = File.OpenRead(path))
return(Load(stream, format));
}
/// <summary>
/// Creates a new manifest.
/// </summary>
/// <param name="format">The format used for <see cref="Save(Stream)"/>, also specifies the algorithm used in <see cref="ManifestFileBase.Digest"/>.</param>
/// <param name="nodes">A list of all elements in the tree this manifest represents.</param>
public Manifest([NotNull] ManifestFormat format, [NotNull, ItemNotNull, InstantHandle] IEnumerable <ManifestNode> nodes)
{
#region Sanity checks
if (nodes == null)
{
throw new ArgumentNullException("nodes");
}
if (format == null)
{
throw new ArgumentNullException("format");
}
#endregion
Format = format;
_nodes = nodes.ToArray(); // Make the collection immutable
}
/// <summary>
/// Creates a manifest node for a file.
/// </summary>
/// <param name="file">The file object to create a node for.</param>
/// <param name="format">The manifest format containing digest rules.</param>
/// <param name="externalXbits">A list of fully qualified paths of files that are named in the <see cref="FlagUtils.SymlinkFile"/>.</param>
/// <param name="externalSymlinks">A list of fully qualified paths of files that are named in the <see cref="FlagUtils.SymlinkFile"/>.</param>
/// <returns>The node for the list.</returns>
/// <exception cref="NotSupportedException">The <paramref name="file"/> has illegal properties (e.g. is a device file, has line breaks in the filename, etc.).</exception>
/// <exception cref="IOException">There was an error reading the file.</exception>
/// <exception cref="UnauthorizedAccessException">You have insufficient rights to read the file.</exception>
private ManifestNode GetFileNode(FileInfo file, ManifestFormat format, ICollection <string> externalXbits, ICollection <string> externalSymlinks)
{
if (_isUnixFS)
{
// Symlink
string symlinkContents;
if (FileUtils.IsSymlink(file.FullName, out symlinkContents))
{
var symlinkData = Encoding.UTF8.GetBytes(symlinkContents);
return(new ManifestSymlink(format.DigestContent(symlinkData), symlinkData.Length, file.Name));
}
// Executable file
if (FileUtils.IsExecutable(file.FullName))
{
using (var stream = File.OpenRead(file.FullName))
return(new ManifestExecutableFile(format.DigestContent(stream), file.LastWriteTimeUtc.ToUnixTime(), file.Length, file.Name));
}
// Invalid file type
if (!FileUtils.IsRegularFile(file.FullName))
{
throw new NotSupportedException(string.Format(Resources.IllegalFileType, file.FullName));
}
}
else
{
// External symlink
if (externalSymlinks.Contains(file.FullName))
{
using (var stream = File.OpenRead(file.FullName))
return(new ManifestSymlink(format.DigestContent(stream), file.Length, file.Name));
}
// External executable file
if (externalXbits.Contains(file.FullName))
{
using (var stream = File.OpenRead(file.FullName))
return(new ManifestExecutableFile(format.DigestContent(stream), file.LastWriteTimeUtc.ToUnixTime(), file.Length, file.Name));
}
}
// Normal file
using (var stream = File.OpenRead(file.FullName))
return(new ManifestNormalFile(format.DigestContent(stream), file.LastWriteTimeUtc.ToUnixTime(), file.Length, file.Name));
}
/// <summary>
/// Prepares to generate a manifest for a directory in the filesystem.
/// </summary>
/// <param name="path">The path of the directory to analyze.</param>
/// <param name="format">The format of the manifest to generate.</param>
public ManifestGenerator([NotNull] string path, [NotNull] ManifestFormat format)
{
#region Sanity checks
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException("path");
}
if (format == null)
{
throw new ArgumentNullException("format");
}
#endregion
TargetDir = path.TrimEnd(Path.DirectorySeparatorChar);
Format = format;
_isUnixFS = FlagUtils.IsUnixFS(TargetDir);
}
public static Manifest Load([NotNull] Stream stream, [NotNull] ManifestFormat format)
{
#region Sanity checks
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (format == null)
{
throw new ArgumentNullException("format");
}
#endregion
var nodes = new List <ManifestNode>();
var reader = new StreamReader(stream);
while (!reader.EndOfStream)
{
// Parse each line as a node
string line = reader.ReadLine() ?? "";
if (line.StartsWith("F"))
{
nodes.Add(ManifestNormalFile.FromString(line));
}
else if (line.StartsWith("X"))
{
nodes.Add(ManifestExecutableFile.FromString(line));
}
else if (line.StartsWith("S"))
{
nodes.Add(ManifestSymlink.FromString(line));
}
else if (line.StartsWith("D"))
{
nodes.Add(format.ReadDirectoryNodeFromEntry(line));
}
else
{
throw new FormatException(Resources.InvalidLinesInManifest);
}
}
return(new Manifest(format, nodes));
}
public static Manifest VerifyDirectory(string directory, ManifestDigest expectedDigest, ITaskHandler handler)
{
#region Sanity checks
if (string.IsNullOrEmpty(directory))
{
throw new ArgumentNullException(nameof(directory));
}
if (handler == null)
{
throw new ArgumentNullException(nameof(handler));
}
#endregion
string?expectedDigestValue = expectedDigest.Best;
if (string.IsNullOrEmpty(expectedDigestValue))
{
throw new NotSupportedException(Resources.NoKnownDigestMethod);
}
var format = ManifestFormat.FromPrefix(expectedDigestValue);
var generator = new ManifestGenerator(directory, format)
{
Tag = expectedDigest
};
handler.RunTask(generator);
var manifest = generator.Manifest;
string digest = manifest.CalculateDigest();
if (digest != expectedDigestValue)
{
var offsetManifest = TryFindOffset(manifest, expectedDigestValue);
if (offsetManifest != null)
{
return(offsetManifest);
}
string manifestFilePath = Path.Combine(directory, Manifest.ManifestFile);
var expectedManifest = File.Exists(manifestFilePath) ? Manifest.Load(manifestFilePath, format) : null;
throw new DigestMismatchException(expectedDigestValue, digest, expectedManifest, manifest);
}
return(manifest);
}
public static Manifest VerifyDirectory(string directory, ManifestDigest expectedDigest, ITaskHandler handler)
{
#region Sanity checks
if (string.IsNullOrEmpty(directory))
{
throw new ArgumentNullException("directory");
}
if (handler == null)
{
throw new ArgumentNullException("handler");
}
#endregion
string expectedDigestValue = expectedDigest.Best;
if (string.IsNullOrEmpty(expectedDigestValue))
{
throw new NotSupportedException(Resources.NoKnownDigestMethod);
}
var format = ManifestFormat.FromPrefix(expectedDigestValue);
var generator = new ManifestGenerator(directory, format)
{
Tag = expectedDigest
};
handler.RunTask(generator);
var actualManifest = generator.Result;
string actualDigestValue = actualManifest.CalculateDigest();
string manifestFilePath = Path.Combine(directory, Manifest.ManifestFile);
var expectedManifest = File.Exists(manifestFilePath) ? Manifest.Load(manifestFilePath, format) : null;
if (actualDigestValue != expectedDigestValue)
{
throw new DigestMismatchException(
expectedDigestValue,
actualDigestValue,
// Only log the complete manifests in verbose mode
(handler.Verbosity > 0) ? expectedManifest : null,
(handler.Verbosity > 0) ? actualManifest : null);
}
return(actualManifest);
}
/// <inheritdoc/>
public Digest([NotNull] ICommandHandler handler) : base(handler)
{
Options.Add("manifest", () => Resources.OptionManifest, _ => _printManifest = true);
Options.Add("digest", () => Resources.OptionDigest, _ => _printDigest = true);
Options.Add("algorithm=", () => Resources.OptionAlgorithm + Environment.NewLine + SupportedValues(ManifestFormat.All),
delegate(string algorithm)
{
try
{
_algorithm = ManifestFormat.FromPrefix(algorithm);
}
#region Error handling
catch (ArgumentException ex)
{
// Wrap exception since only certain exception types are allowed
throw new OptionException(ex.Message, algorithm);
}
#endregion
});
}
/// <summary>
/// Generates a manifest for a directory in the filesystem and writes the manifest to a file named Manifest.ManifestFile in that directory.
/// </summary>
/// <param name="path">The path of the directory to analyze.</param>
/// <param name="format">The format of the manifest (which file details are listed, which digest method is used, etc.).</param>
/// <param name="handler">A callback object used when the the user is to be informed about progress.</param>
/// <returns>The manifest digest.</returns>
/// <exception cref="IOException">A problem occurs while writing the file.</exception>
/// <remarks>
/// The exact format is specified here: http://0install.net/manifest-spec.html
/// </remarks>
public static string CreateDotFile([NotNull] string path, [NotNull] ManifestFormat format, [NotNull] ITaskHandler handler)
{
#region Sanity checks
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException(nameof(path));
}
if (format == null)
{
throw new ArgumentNullException(nameof(format));
}
if (handler == null)
{
throw new ArgumentNullException(nameof(handler));
}
#endregion
var generator = new ManifestGenerator(path, format);
handler.RunTask(generator);
return(generator.Manifest.Save(Path.Combine(path, Manifest.ManifestFile)));
}
public void TestFromPrefix()
{
ManifestFormat.FromPrefix("sha1new=abc").Should().BeSameAs(ManifestFormat.Sha1New);
ManifestFormat.FromPrefix("sha256=abc").Should().BeSameAs(ManifestFormat.Sha256);
ManifestFormat.FromPrefix("sha256new_abc").Should().BeSameAs(ManifestFormat.Sha256New);
}
/// <summary>
/// Creates a manifest node for a file.
/// </summary>
/// <param name="file">The file object to create a node for.</param>
/// <param name="format">The manifest format containing digest rules.</param>
/// <param name="externalXbits">A list of fully qualified paths of files that are named in the <see cref="FlagUtils.SymlinkFile"/>.</param>
/// <param name="externalSymlinks">A list of fully qualified paths of files that are named in the <see cref="FlagUtils.SymlinkFile"/>.</param>
/// <returns>The node for the list.</returns>
/// <exception cref="NotSupportedException">The <paramref name="file"/> has illegal properties (e.g. is a device file, has line breaks in the filename, etc.).</exception>
/// <exception cref="IOException">There was an error reading the file.</exception>
/// <exception cref="UnauthorizedAccessException">You have insufficient rights to read the file.</exception>
private ManifestNode GetFileNode(FileInfo file, ManifestFormat format, ICollection<string> externalXbits, ICollection<string> externalSymlinks)
{
if (_isUnixFS)
{
// Symlink
string symlinkContents;
if (FileUtils.IsSymlink(file.FullName, out symlinkContents))
{
var symlinkData = Encoding.UTF8.GetBytes(symlinkContents);
return new ManifestSymlink(format.DigestContent(symlinkData), symlinkData.Length, file.Name);
}
// Executable file
if (FileUtils.IsExecutable(file.FullName))
{
using (var stream = File.OpenRead(file.FullName))
return new ManifestExecutableFile(format.DigestContent(stream), file.LastWriteTimeUtc.ToUnixTime(), file.Length, file.Name);
}
// Invalid file type
if (!FileUtils.IsRegularFile(file.FullName))
throw new NotSupportedException(string.Format(Resources.IllegalFileType, file.FullName));
}
else
{
// External symlink
if (externalSymlinks.Contains(file.FullName))
{
using (var stream = File.OpenRead(file.FullName))
return new ManifestSymlink(format.DigestContent(stream), file.Length, file.Name);
}
// External executable file
if (externalXbits.Contains(file.FullName))
{
using (var stream = File.OpenRead(file.FullName))
return new ManifestExecutableFile(format.DigestContent(stream), file.LastWriteTimeUtc.ToUnixTime(), file.Length, file.Name);
}
}
// Normal file
using (var stream = File.OpenRead(file.FullName))
return new ManifestNormalFile(format.DigestContent(stream), file.LastWriteTimeUtc.ToUnixTime(), file.Length, file.Name);
}
/// <summary>
/// Creates a manifest node for a directory.
/// </summary>
/// <param name="directory">The directory object to create a node for.</param>
/// <param name="format">The manifest format containing digest rules.</param>
/// <param name="rootPath">The fully qualified path of the root directory the manifest is being generated for.</param>
/// <returns>The node for the list.</returns>
/// <exception cref="IOException">There was an error reading the directory.</exception>
/// <exception cref="UnauthorizedAccessException">You have insufficient rights to read the directory.</exception>
private ManifestNode GetDirectoryNode(DirectoryInfo directory, ManifestFormat format, string rootPath)
{
// Directory symlinks
string symlinkContents;
if (_isUnixFS && FileUtils.IsSymlink(directory.FullName, out symlinkContents))
{
var symlinkData = Encoding.UTF8.GetBytes(symlinkContents);
return new ManifestSymlink(format.DigestContent(symlinkData), symlinkData.Length, directory.Name);
}
// Remove leading portion of path and use Unix slashes
string trimmedName = directory.FullName.Substring(rootPath.Length).Replace(Path.DirectorySeparatorChar, '/');
return new ManifestDirectory(directory.LastWriteTime.ToUnixTime(), trimmedName);
}
/// <summary>
/// Executes the work-step for a single implementation.
/// </summary>
public void Work(ManifestDigest manifestDigest)
{
string?digestString = manifestDigest.Best;
if (digestString == null)
{
return;
}
string implementationPath = Path.Combine(_storePath, digestString);
var manifest = Manifest.Load(Path.Combine(implementationPath, Manifest.ManifestFile), ManifestFormat.FromPrefix(digestString));
string currentDirectory = "";
foreach (var node in manifest)
{
switch (node)
{
case ManifestDirectory x:
currentDirectory = FileUtils.UnifySlashes(x.FullPath.TrimStart('/'));
break;
case ManifestFileBase x:
if (x.Size == 0)
{
return;
}
var key = new DedupKey(x.Size, x.ModifiedTime, manifest.Format, x.Digest);
var file = new StoreFile(implementationPath, Path.Combine(currentDirectory, x.Name));
if (_fileHashes.TryGetValue(key, out var existingFile))
{
if (!FileUtils.AreHardlinked(file, existingFile))
{
if (JoinWithHardlink(file, existingFile))
{
SavedBytes += x.Size;
}
}
}
else
{
_fileHashes.Add(key, file);
}
break;
}
}
}
private static Manifest GenerateManifest(string path, ManifestFormat format, ITaskHandler handler)
{
var generator = new ManifestGenerator(path, format);
handler.RunTask(generator);
return generator.Result;
}
/// <summary>
/// Executes the work-step for a single implementation.
/// </summary>
public void Work(ManifestDigest manifestDigest)
{
string digestString = manifestDigest.Best;
if (digestString == null)
{
return;
}
string implementationPath = Path.Combine(_storePath, digestString);
var manifest = Manifest.Load(Path.Combine(implementationPath, Manifest.ManifestFile), ManifestFormat.FromPrefix(digestString));
string currentDirectory = "";
new AggregateDispatcher <ManifestNode>
{
(ManifestDirectory x) => { currentDirectory = FileUtils.UnifySlashes(x.FullPath.TrimStart('/')); },
(ManifestFileBase x) =>
{
if (x.Size == 0)
{
return;
}
var key = new DedupKey(x.Size, x.ModifiedTime, manifest.Format, x.Digest);
var file = new StoreFile(implementationPath, Path.Combine(currentDirectory, x.Name));
StoreFile existingFile;
if (_fileHashes.TryGetValue(key, out existingFile))
{
if (!FileUtils.AreHardlinked(file, existingFile))
{
if (JoinWithHardlink(file, existingFile))
{
SavedBytes += x.Size;
}
}
}
else
{
_fileHashes.Add(key, file);
}
}
}.Dispatch(manifest);
}
