/// <summary>
/// Recursively adds folders and files to archive
/// </summary>
/// <param name="tarArchive"></param>
/// <param name="sourceDirectory"></param>
/// <param name="recurse"></param>
public static void AddDirectoryFilesToTar(TarArchive tarArchive, string sourceDirectory, bool recurse)
{
// Recursively add sub-folders
if (recurse)
{
string[] directories = Directory.GetDirectories(sourceDirectory);
foreach (string directory in directories)
{
AddDirectoryFilesToTar(tarArchive, directory, recurse);
}
}
// Add files
string[] filenames = Directory.GetFiles(sourceDirectory);
foreach (string filename in filenames)
{
TarEntry tarEntry = TarEntry.CreateEntryFromFile(filename);
tarArchive.WriteEntry(tarEntry, true);
}
}
public static void CreateTarGZ(List <string> sourceFileList, string tgzFilename)
{
using (FileStream fs = new FileStream(tgzFilename, FileMode.Create, FileAccess.Write, FileShare.None))
{
using (Stream gzipStream = new GZipOutputStream(fs))
{
using (TarArchive tarArchive = TarArchive.CreateOutputTarArchive(gzipStream))
{
foreach (string filename in sourceFileList)
{
{
TarEntry tarEntry = TarEntry.CreateEntryFromFile(filename);
tarEntry.Name = Path.GetFileName(filename);
tarArchive.WriteEntry(tarEntry, false);
}
}
}
}
}
}
private static void CompressTar(string directory, string archiveFile, Predicate <FileInfo> filter, Func <Stream, Stream> outputStreamFactory)
{
FileSystemTasks.EnsureExistingParentDirectory(archiveFile);
var files = GetFiles(directory, filter);
using (var fileStream = File.Open(archiveFile, FileMode.CreateNew, FileAccess.ReadWrite))
using (var outputStream = outputStreamFactory(fileStream))
using (var tarArchive = TarArchive.CreateOutputTarArchive(outputStream))
{
foreach (var file in files)
{
var entry = TarEntry.CreateEntryFromFile(file);
var relativePath = PathConstruction.GetRelativePath(directory, file);
entry.Name = PathConstruction.NormalizePath(relativePath, separator: '/');
tarArchive.WriteEntry(entry, recurse: false);
}
}
Logger.Log($"Compressed content of '{directory}' to '{Path.GetFileName(archiveFile)}'.");
}
private static void AddDirectoryFilesToTar(TarArchive tarArchive, string sourceDirectory, bool recurse)
{
TarEntry tarEntry;
string[] filenames = Directory.GetFiles(sourceDirectory);
foreach (string filename in filenames)
{
tarEntry = TarEntry.CreateEntryFromFile(filename);
tarEntry.Name = filename.Remove(0, tarArchive.RootPath.Length + 1);
tarArchive.WriteEntry(tarEntry, true);
}
if (recurse)
{
string[] directories = Directory.GetDirectories(sourceDirectory);
foreach (string directory in directories)
{
AddDirectoryFilesToTar(tarArchive, directory, true);
}
}
}
static private void AddDirectoryFilesToTar(TarArchive tarArchive, string sourceDirectory, bool recurse)
{
TarEntry tarEntry = TarEntry.CreateEntryFromFile(sourceDirectory);
tarArchive.WriteEntry(tarEntry, false);
string[] filenames = Directory.GetFiles(sourceDirectory);
foreach (string filename in filenames)
{
tarEntry = TarEntry.CreateEntryFromFile(filename);
tarArchive.WriteEntry(tarEntry, true);
}
if (recurse)
{
string[] directories = Directory.GetDirectories(sourceDirectory);
foreach (string directory in directories)
{
AddDirectoryFilesToTar(tarArchive, directory, recurse);
}
}
}
/// <summary>
/// Adds a file to the archive
/// </summary>
/// <param name="compressIfAble">This parameter is ignored because tar files don't support compression. It is recommended to pass "false" though.</param>
protected override void AddFile(string path, string entryName, bool compressIfAble = true)
{
var fi = new FileInfo(path);
TarEntry newEntry = TarEntry.CreateEntryFromFile(path);
newEntry.TarHeader.Name = entryName;
newEntry.Size = fi.Length;
_tarStream.PutNextEntry(newEntry);
// Add to the archive in buffered chunks
var buffer = new byte[4096];
using (var streamReader = RobustFile.OpenRead(path))
{
StreamUtils.Copy(streamReader, _tarStream, buffer);
}
_tarStream.CloseEntry();
}
private static void AddDirectoryFilesToTar(TarArchive tarArchive, string rootDirectory, string filePath)
{
filePath = filePath.Replace("\\", "/");
string absFilePath = filePath.Replace(rootDirectory, "").Remove(0, 1);
// Debug.Log(absFilePath);
string[] parentDirInfos = absFilePath.Split('/');
string parentDir = "";
string sp = "/";
for (int i = 0; i < parentDirInfos.Length - 1; i++)
{
parentDir += sp + parentDirInfos[i];
TarEntry tarEntryDic = TarEntry.CreateEntryFromFile(rootDirectory + parentDir);
tarArchive.WriteEntry(tarEntryDic, false);
// Debug.Log(rootDirectory + parentDir);
}
TarEntry tarEntry = TarEntry.CreateEntryFromFile(filePath);
tarArchive.WriteEntry(tarEntry, true);
}
public void Update()
{
if (!File.Exists(ArchivePath))
{
throw new Exception("Archive doesn't exist");
}
var tmp = System.IO.Directory.CreateDirectory(Path.Combine(Path.GetDirectoryName(ArchivePath), "tmp"));
using (var fs = File.OpenRead(ArchivePath))
{
using (var decompresed = new GZipInputStream(fs))
{
using (var tararchive = TarArchive.CreateInputTarArchive(decompresed, Encoding.UTF8))
{
tararchive.ExtractContents(tmp.FullName);
}
}
}
File.Create(Path.Combine(tmp.FullName, "foo.c")).Close();
File.Create(Path.Combine(tmp.FullName, "bar.c")).Close();
File.Delete(ArchivePath); //Was possible error
var prepared = GetPreparedFiles(tmp.FullName);
using (var fs = new FileStream(ArchivePath, FileMode.Create))
{
using (var compres = new GZipOutputStream(fs))
{
using (var archive = TarArchive.CreateOutputTarArchive(compres))
{
archive.RootPath = tmp.FullName;
foreach (var p in prepared) //Was error
{
archive.WriteEntry(TarEntry.CreateEntryFromFile(p), true);
}
}
}
}
System.IO.Directory.Delete(tmp.FullName, true); //Was error, recursive
}
/// <summary>
/// 生成tar文件
/// </summary>
/// <param name="listFilesPath">文件的路径：H:\Demo\xxx.txt</param>
/// <param name="tarFileDir">生成的文件路径</param>
/// <param name="tarFileName">生成的文件名称，不带扩展名</param>
/// <param name="encoding">编码</param>
/// <returns>压缩后的文件路径//"20180524" + ".tar"</returns>
public string CreatTarArchive(List <string> listFilesPath, string tarFileDir, string tarFileName = "temp", string encoding = "utf-8")
{
if (!System.IO.Directory.Exists(tarFileDir))
{
System.IO.Directory.CreateDirectory(tarFileDir);
}
tarFileName = Path.Combine(tarFileDir, tarFileName + ".tar");
using Stream outStream = new FileStream(tarFileName, FileMode.OpenOrCreate); //打开.tar文件
using TarArchive archive = TarArchive.CreateOutputTarArchive(outStream, TarBuffer.DefaultBlockFactor, Encoding.GetEncoding(encoding));
archive.RootPath = Path.GetDirectoryName(listFilesPath[0]);
foreach (var fileName in listFilesPath)
{
TarEntry entry = TarEntry.CreateEntryFromFile(fileName);//将文件写到.tar文件中去
archive.WriteEntry(entry, true);
}
return(tarFileName);
}
public string Tar()
{
using (var dockerfileArchiveStream = File.Create(this.DockerfileArchiveFile.FullName))
{
using (var dockerfileArchive = TarArchive.CreateOutputTarArchive(dockerfileArchiveStream))
{
dockerfileArchive.RootPath = this.BaseDirectory.FullName;
void Tar(string baseDirectory)
{
baseDirectory = baseDirectory.Replace('\\', '/');
void WriteEntry(string entry)
{
entry = entry.Replace('\\', '/');
var tarEntry = TarEntry.CreateEntryFromFile(entry);
tarEntry.Name = entry.Replace(dockerfileArchive.RootPath, string.Empty).TrimStart(TrimLeadingChars);
tarEntry.ModTime = DisableModTime;
dockerfileArchive.WriteEntry(tarEntry, File.Exists(entry));
}
if (!dockerfileArchive.RootPath.Equals(baseDirectory))
{
WriteEntry(baseDirectory);
}
Directory.GetFiles(baseDirectory).ToList().ForEach(WriteEntry);
Directory.GetDirectories(baseDirectory).ToList().ForEach(Tar);
}
Tar(this.BaseDirectory.FullName);
}
}
return(this.DockerfileArchiveFile.FullName);
}
public static string CreateTar(string directoryToCompress, string destPath, string tarFile)
{
string destDrive = destPath.Substring(0, destPath.IndexOf(@"\") + 1);
Directory.SetCurrentDirectory(destDrive);
string tarFilePath = Path.Combine(destPath, tarFile);
using (Stream fs = new FileStream(tarFilePath, FileMode.OpenOrCreate))
{
using (TarArchive ta = TarArchive.CreateOutputTarArchive(fs))
{
string[] files = Directory.GetFiles(directoryToCompress);
foreach (string file in files)
{
string entry = file.Substring(file.IndexOf(@"\") + 1);
TarEntry te = TarEntry.CreateEntryFromFile(entry);
ta.WriteEntry(te, false);
}
}
}
return(tarFilePath);
}
public void ExtractingCorruptTarShouldntLeakFiles()
{
using var memoryStream = new MemoryStream();
//Create a tar.gz in the output stream
using (var gzipStream = new GZipOutputStream(memoryStream))
{
gzipStream.IsStreamOwner = false;
using (var tarOut = TarArchive.CreateOutputTarArchive(gzipStream))
using (var dummyFile = Utils.GetDummyFile(size: 32000))
{
tarOut.IsStreamOwner = false;
tarOut.WriteEntry(TarEntry.CreateEntryFromFile(dummyFile), recurse: false);
}
}
// corrupt archive - make sure the file still has more than one block
memoryStream.SetLength(16000);
memoryStream.Seek(0, SeekOrigin.Begin);
// try to extract
using (var gzipStream = new GZipInputStream(memoryStream))
{
gzipStream.IsStreamOwner = false;
using var tempDir = Utils.GetTempDir();
using (var tarIn = TarArchive.CreateInputTarArchive(gzipStream, nameEncoding: null))
{
tarIn.IsStreamOwner = false;
Assert.Throws <SharpZipBaseException>(() => tarIn.ExtractContents(tempDir));
}
// Try to remove the output directory to check if any file handles are still being held
Assert.DoesNotThrow(() => tempDir.Delete());
Assert.That(tempDir.Exists, Is.False, "Temporary folder should have been removed");
}
}
public void EndBlockHandling()
{
int dummySize = 70145;
long outCount, inCount;
using (var ms = new MemoryStream())
{
using (var tarOut = TarArchive.CreateOutputTarArchive(ms))
using (var dummyFile = Utils.GetDummyFile(dummySize))
{
tarOut.IsStreamOwner = false;
tarOut.WriteEntry(TarEntry.CreateEntryFromFile(dummyFile.Filename), false);
}
outCount = ms.Position;
ms.Seek(0, SeekOrigin.Begin);
using (var tarIn = TarArchive.CreateInputTarArchive(ms, null))
using (var tempDir = new Utils.TempDir())
{
tarIn.IsStreamOwner = false;
tarIn.ExtractContents(tempDir.Fullpath);
foreach (var file in Directory.GetFiles(tempDir.Fullpath, "*", SearchOption.AllDirectories))
{
Console.WriteLine($"Extracted \"{file}\"");
}
}
inCount = ms.Position;
Console.WriteLine($"Output count: {outCount}");
Console.WriteLine($"Input count: {inCount}");
Assert.AreEqual(inCount, outCount, "Bytes read and bytes written should be equal");
}
}
/// <summary>
/// create a tar file
/// </summary>
/// <param name="ADirectory"></param>
/// <param name="ATarFileName"></param>
public static void PackTar(String ADirectory, String ATarFileName)
{
TarArchive archive;
FileStream outStream;
TarEntry entry;
if (ATarFileName.Length == 0)
{
ATarFileName = Path.GetFullPath(ADirectory + "/../" + Path.GetFileName(ADirectory) + ".tar");
}
TLogging.Log("Create tar archive " + ATarFileName, TLoggingType.ToConsole);
System.IO.File.Delete(ATarFileName);
outStream = System.IO.File.OpenWrite(ATarFileName);
archive = TarArchive.CreateOutputTarArchive(outStream, TarBuffer.DefaultBlockFactor);
archive.SetUserInfo(-1, "petra", -1, "petra");
archive.RootPath = Path.GetFullPath(ADirectory).Replace("\\", "/");
// just adding the whole directory does not work as expected, it adds an empty directory without a name
string[] directories = System.IO.Directory.GetDirectories(ADirectory);
foreach (string dir in directories)
{
entry = TarEntry.CreateEntryFromFile(dir);
archive.WriteEntry(entry, true);
}
string[] files = System.IO.Directory.GetFiles(ADirectory);
foreach (string filename in files)
{
entry = TarEntry.CreateEntryFromFile(filename);
archive.WriteEntry(entry, false);
}
archive.Close();
}
public static void ToTar(this DirectoryInfo sourceDirectory, Stream output, string[] excludedDirectoryNames)
{
var archive = TarArchive.CreateOutputTarArchive(output);
archive.RootPath = sourceDirectory.FullName.Replace(Path.DirectorySeparatorChar, '/').TrimEnd('/');
var entries = GetFiles(sourceDirectory, excludedDirectoryNames)
.Select(x => TarEntry.CreateEntryFromFile(x.FullName))
.ToList();
var entriesCount = entries.Count();
var progressBar = new MegaByteProgressBar();
for (var i = 0; i < entriesCount; i++)
{
archive.WriteEntry(entries[i], true);
progressBar.Update("Packing files", entries.Take(i + 1).Sum(x => x.Size), entries.Sum(x => x.Size));
}
archive.Close();
}
/// <inheritdoc />
public override ICompressedArchiveEntry CreateEntry(string name)
{
if (IsReadOnly)
{
throw new InvalidOperationException(Resources.Strings.TarArchiveAccess_InvalidModeForCreateEntryError);
}
var root = TarArchive.RootPath;
var isRooted = Path.IsPathRooted(name);
var tarEntry = isRooted ? TarEntry.CreateEntryFromFile(name) : TarEntry.CreateTarEntry(name);
if (isRooted)
{
if (!string.IsNullOrEmpty(RootLocation))
{
var relativeName = PathUtils.GetRelativePath(tarEntry.File, Path.GetDirectoryName(RootLocation));
tarEntry.Name = relativeName;
}
}
var newEntry = new TarArchiveEntry(tarEntry);
_entries.Add(newEntry);
return(newEntry);
}
/// <summary>
/// 打包成Tar包
/// </summary>
/// <param name="strBasePath">压缩文件夹路径</param>
/// <param name="strSourceFolderName">生成tar文件路径</param>
/// <param name="sTarName">生成tar文件名称</param>
/// <returns></returns>
public bool CreatTarArchive(string strBasePath, string strSourceFolderName, string sTarName)
{
if (!System.IO.Directory.Exists(strSourceFolderName))
{
System.IO.Directory.CreateDirectory(strSourceFolderName);//不存在生成Tar文件目录就创建
}
if (string.IsNullOrEmpty(strBasePath) ||
string.IsNullOrEmpty(strSourceFolderName) ||
!System.IO.Directory.Exists(strBasePath)) //Path.Combine(strBasePath, strSourceFolderName)
{
return(false);
}
if (strBasePath.EndsWith("\\"))
{
strBasePath = strSourceFolderName.TrimEnd('\\');
}
Environment.CurrentDirectory = strBasePath; //要压缩的文件夹名称
string strSourceFolderAllPath = strBasePath; // Path.Combine(strBasePath, strSourceFolderName);
string strOupFileAllPath = strSourceFolderName + "\\" + sTarName + ".tar"; //压缩文件名及路径
Stream outStream = new FileStream(strOupFileAllPath, FileMode.OpenOrCreate);
TarArchive archive = TarArchive.CreateOutputTarArchive(outStream, TarBuffer.DefaultBlockFactor);
TarEntry entry = TarEntry.CreateEntryFromFile(strSourceFolderAllPath);
archive.WriteEntry(entry, true);
if (archive != null)
{
archive.Close();
}
outStream.Close();
return(true);
}
static void AddDirToTar(Stream tar, string sourceDirectory, string app, string domain, string basePath, bool writeDirEntry = true)
{
// Optionally, write an entry for the directory itself.
if (writeDirEntry)
{
TarEntry tarEntry = TarEntry.CreateEntryFromFile(sourceDirectory);
tarEntry.Name = basePath;
// tarOutputStream.PutNextEntry(tarEntry);
ABTar.WriteTarFile(app, domain, basePath, null, tar, out var _);
}
// Write each file to the tar.
string[] filenames = Directory.GetFiles(sourceDirectory);
foreach (string filename in filenames)
{
using (FileStream file = File.OpenRead(filename)) {
//AddFileToTarRaw(tarOutputStream, inputStream, new FileInfo(filename),
// PathCombineUnixUnsafe(basePath, Path.GetFileName(filename)));
ABTar.WriteTarFile(app, domain,
PathCombineUnixUnsafe(basePath, Path.GetFileName(filename)),
file, tar, out var _);
}
}
// Recurse.
string[] directories = Directory.GetDirectories(sourceDirectory);
foreach (string directory in directories)
{
//AddDirToTar(tarOutputStream, directory,
// PathCombineUnixUnsafe(basePath, Path.GetFileName(directory)));
ABTar.WriteTarFile(app, domain,
PathCombineUnixUnsafe(basePath, Path.GetFileName(directory)),
null, tar, out var _);
}
}
private void AddFileToTar(TarArchive archive, string filePath, string entry, List <string> addedEntries)
{
if (addedEntries.Contains(entry.ToLower()))
{
return;
}
string[] splitPath = entry.Split('/', '\\');
for (int i = 1; i < splitPath.Length; i++)
{
string path = splitPath[0];
for (int j = 1; j < i; j++)
{
path += Path.DirectorySeparatorChar + splitPath[j];
}
if (addedEntries.Contains(path.ToLower()))
{
continue;
}
TarEntry pathEntry = TarEntry.CreateTarEntry(path);
pathEntry.TarHeader.Mode = 1003;
pathEntry.TarHeader.TypeFlag = TarHeader.LF_DIR;
pathEntry.TarHeader.Size = 0;
archive.WriteEntry(pathEntry, false);
addedEntries.Add(path.ToLower());
}
TarEntry fileEntry = TarEntry.CreateEntryFromFile(filePath);
fileEntry.Name = entry;
archive.WriteEntry(fileEntry, false);
addedEntries.Add(entry.ToLower());
}
private static void AddDirectoryFilesToTar(TarArchive tarArchive, string sourceDirectory, bool recurse)
{
var filenames = Directory.GetFiles(sourceDirectory);
foreach (var filename in filenames)
{
TarEntry tarEntry = TarEntry.CreateEntryFromFile(filename);
string finalName = filename.Remove(0, tarArchive.RootPath.Length + 1).Replace('\\', '/');
tarEntry.Name = finalName;
tarArchive.WriteEntry(tarEntry, true);
}
if (!recurse)
{
return;
}
var directories = Directory.GetDirectories(sourceDirectory);
foreach (var directory in directories)
{
AddDirectoryFilesToTar(tarArchive, directory, true);
}
}
public static MemoryStream DoCreateTarFromDirectory(string sourceDirectory, List <string> fileList, List <string> ignoreList = null, Func <int, bool> progress = null, Logger logger = null, bool isSelectDeploy = false)
{
MemoryStream outputMemStream = new MemoryStream();
TarArchive tarArchive = TarArchive.CreateOutputTarArchive(outputMemStream);
tarArchive.RootPath = sourceDirectory.Replace('\\', '/');
if (tarArchive.RootPath.EndsWith("/"))
{
tarArchive.RootPath = tarArchive.RootPath.Remove(tarArchive.RootPath.Length - 1);
}
TarEntry tarEntry2 = TarEntry.CreateEntryFromFile(sourceDirectory);
tarArchive.WriteEntry(tarEntry2, false);
DirectoryInfo directoryInfo = new DirectoryInfo(sourceDirectory);
string fullName = directoryInfo.FullName;
if (directoryInfo.Parent != null)
{
fullName = directoryInfo.Parent.FullName;
}
var allFile = isSelectDeploy ? GetSelectDeployFiles(fileList) : GetFullFileInfo(fileList, sourceDirectory);// FindFileDir(sourceDirectory);
var allFileLength = allFile.Count();
var index = 0;
var haveFile = false;
foreach (FileSystemInfo enumerateFileSystemInfo in allFile)
{
index++;
var lastProgressNumber = (((long)index * 100 / allFileLength));
if (progress != null)
{
var r = progress.Invoke((int)lastProgressNumber);
if (r)
{
throw new Exception("deploy task was canceled!");
}
}
int length = enumerateFileSystemInfo.FullName.Length - fullName.Length;
string entryName = EntryFromPath(enumerateFileSystemInfo.FullName, fullName.Length, length);
if (ignoreList != null && ignoreList.Count > 0)
{
var mathchEntryName = entryName.Substring(directoryInfo.Name.Length);
var haveMatch = false;
foreach (var ignorRule in ignoreList)
{
try
{
if (ignorRule.StartsWith("*"))
{
var ignorRule2 = ignorRule.Substring(1);
if (mathchEntryName.EndsWith(ignorRule2))
{
haveMatch = true;
break;
}
}
else
{
var isMatch = Regex.Match(mathchEntryName, ignorRule, RegexOptions.IgnoreCase);//忽略大小写
if (isMatch.Success)
{
haveMatch = true;
break;
}
}
}
catch (Exception ex)
{
throw new Exception($"Ignore Rule 【{ignorRule}】 regular error:" + ex.Message);
}
}
if (haveMatch)
{
continue;
}
}
if (enumerateFileSystemInfo is FileInfo)
{
if (entryName.Contains("Dockerfile"))
{
logger?.Info($"Find Dockerfile In Package: {entryName}");
}
haveFile = true;
TarEntry tarEntry = TarEntry.CreateEntryFromFile(enumerateFileSystemInfo.FullName);
tarArchive.WriteEntry(tarEntry, true);
}
else
{
TarEntry tarEntry = TarEntry.CreateEntryFromFile(enumerateFileSystemInfo.FullName);
tarArchive.WriteEntry(tarEntry, false);
}
}
tarArchive.IsStreamOwner = false;
tarArchive.Close();
outputMemStream.Position = 0;
if (!haveFile)
{
throw new Exception("no file was packaged!");
}
return(outputMemStream);
}
/// <summary>
/// Create a tar entry with details obtained from <paramref name="fileName">file</paramref>
/// </summary>
/// <param name="fileName">The name of the file to retrieve details from.</param>
/// <returns>A new <see cref="TarEntry"/></returns>
public TarEntry CreateEntryFromFile(string fileName)
{
return(TarEntry.CreateEntryFromFile(fileName));
}
/// <inheritdoc />
/// <remarks>This is the really tricky clever part. It's built around the assumption that we can, at this point of
/// the disposal, still seek back to the beginning of the stream and commit the update that was started in the
/// constructor. It also assumes that other updates are going to be OK with being committed (if there are any).</remarks>
protected override void Dispose(bool disposing)
{
if (Interlocked.Increment(ref _disposed) == 1)
{
var recurse = false;
var copyToTemporaryFile = true;
var isRooted = Path.IsPathRooted(_tarArchiveEntry.TarEntry.File);
if (isRooted)
{
recurse = Directory.Exists(_tarArchiveEntry.TarEntry.File);
copyToTemporaryFile = !recurse && !File.Exists(_tarArchiveEntry.TarEntry.File);
}
if (copyToTemporaryFile)
{
// operating on something that thus far is in-memory only
using (var temporaryDirectory = new TemporaryDirectory())
{
var subdirectory = Path.GetDirectoryName(_tarArchiveEntry.Name);
if (!string.IsNullOrEmpty(subdirectory))
{
Directory.CreateDirectory(Path.Combine(temporaryDirectory.Path, subdirectory));
}
var entryFilePath = Path.Combine(temporaryDirectory.Path, _tarArchiveEntry.Name);
Seek(0, SeekOrigin.Begin);
using (var fileStream = new FileStream(entryFilePath, FileMode.Create, FileAccess.Write))
{
CopyTo(fileStream);
}
var tarEntry = TarEntry.CreateEntryFromFile(entryFilePath);
tarEntry.Name = _tarArchiveEntry.Name;
recurse = IsDirectoryName(tarEntry.Name);
_tarAccess.TarArchive.WriteEntry(tarEntry, recurse);
_tarArchiveEntry.TarEntry = tarEntry;
}
}
else
{
// already operating on an existing on-disk entity
var tarEntry = _tarArchiveEntry.TarEntry;
var tarArchive = _tarAccess.TarArchive;
var rootPath = tarArchive.RootPath;
if (recurse)
{
if (!string.IsNullOrEmpty(_tarAccess.RootLocation))
{
tarArchive.RootPath = Path.GetDirectoryName(_tarAccess.RootLocation);
}
}
try
{
tarArchive.WriteEntry(tarEntry, recurse);
}
finally
{
if (recurse)
{
tarArchive.RootPath = rootPath;
}
}
}
base.Dispose(disposing);
}
}
/// <summary>
/// Creates the tar file.
/// </summary>
protected override void ExecuteTask()
{
TarArchive archive = null;
Stream outstream = null;
Log(Level.Info, "Tarring {0} files to '{1}'.",
TarFileSets.FileCount, DestFile.FullName);
try {
if (!Directory.Exists(DestFile.DirectoryName))
{
Directory.CreateDirectory(DestFile.DirectoryName);
}
outstream = File.Create(DestFile.FullName);
// wrap outputstream with corresponding compression method
switch (CompressionMethod)
{
case TarCompressionMethod.GZip:
outstream = new GZipOutputStream(outstream);
break;
case TarCompressionMethod.BZip2:
outstream = new BZip2OutputStream(outstream);
break;
}
// create tar archive
archive = TarArchive.CreateOutputTarArchive(outstream,
TarBuffer.DefaultBlockFactor);
// do not use convert line endings of text files to \n, as this
// converts all content to ASCII
archive.AsciiTranslate = false;
// process all filesets
foreach (TarFileSet fileset in TarFileSets)
{
string basePath = fileset.BaseDirectory.FullName;
if (Path.GetPathRoot(basePath) != basePath)
{
basePath = Path.GetDirectoryName(basePath + Path.DirectorySeparatorChar);
}
// add files to tar
foreach (string file in fileset.FileNames)
{
// ensure file exists (in case "asis" was used)
if (!File.Exists(file))
{
throw new BuildException(string.Format(CultureInfo.InvariantCulture,
"File '{0}' does not exist.", file), Location);
}
// the filename of the tar entry
string entryFileName;
// the directory of the tar entry
string entryDirName = string.Empty;
// determine name of the tar entry
if (!Flatten && file.StartsWith(basePath))
{
entryFileName = file.Substring(basePath.Length);
if (entryFileName.Length > 0 && entryFileName[0] == Path.DirectorySeparatorChar)
{
entryFileName = entryFileName.Substring(1);
}
// get directory part of entry
entryDirName = Path.GetDirectoryName(entryFileName);
// ensure directory separators are understood on linux
if (Path.DirectorySeparatorChar == '\\')
{
entryDirName = entryDirName.Replace(@"\", "/");
}
// get filename part of entry
entryFileName = Path.GetFileName(entryFileName);
}
else
{
entryFileName = Path.GetFileName(file);
}
// add prefix if specified
if (fileset.Prefix != null)
{
entryDirName = fileset.Prefix + entryDirName;
}
// ensure directory has trailing slash
if (entryDirName.Length != 0)
{
if (!entryDirName.EndsWith("/"))
{
entryDirName += '/';
}
// create directory entry in archive
CreateDirectoryEntry(archive, entryDirName, fileset);
}
TarEntry entry = TarEntry.CreateEntryFromFile(file);
entry.Name = entryDirName + entryFileName;
entry.GroupId = fileset.Gid;
entry.GroupName = fileset.GroupName;
entry.UserId = fileset.Uid;
entry.UserName = fileset.UserName;
entry.TarHeader.Mode = fileset.FileMode;
// write file to tar file
archive.WriteEntry(entry, true);
}
// add (possibly empty) directories to zip
if (IncludeEmptyDirs)
{
// add (possibly empty) directories to tar
foreach (string directory in fileset.DirectoryNames)
{
// skip directories that are not located beneath the base
// directory
if (!directory.StartsWith(basePath) || directory.Length <= basePath.Length)
{
continue;
}
// determine tar entry name
string entryName = directory.Substring(basePath.Length + 1);
// add prefix if specified
if (fileset.Prefix != null)
{
entryName = fileset.Prefix + entryName;
}
// ensure directory separators are understood on linux
if (Path.DirectorySeparatorChar == '\\')
{
entryName = entryName.Replace(@"\", "/");
}
if (!entryName.EndsWith("/"))
{
// trailing directory signals to #ziplib that we're
// dealing with directory entry
entryName += "/";
}
// create directory entry in archive
CreateDirectoryEntry(archive, entryName, fileset);
}
}
}
// close the tar archive
archive.Close();
} catch (Exception ex) {
// close the tar output stream
if (outstream != null)
{
outstream.Close();
}
// close the tar archive
if (archive != null)
{
archive.Close();
}
// delete the (possibly corrupt) tar file
if (DestFile.Exists)
{
DestFile.Delete();
}
throw new BuildException(string.Format(CultureInfo.InvariantCulture,
"Tar file '{0}' could not be created.", DestFile.FullName),
Location, ex);
}
}
protected override bool _Run()
{
SourcePath = STEM.Sys.IO.Path.AdjustPath(SourcePath);
OutputFile = STEM.Sys.IO.Path.AdjustPath(OutputFile);
string tmpFile = Path.Combine(Path.Combine(STEM.Sys.IO.Path.GetDirectoryName(OutputFile), "TEMP"), STEM.Sys.IO.Path.GetFileName(OutputFile));
try
{
if (!Directory.Exists(STEM.Sys.IO.Path.GetDirectoryName(tmpFile)))
{
Directory.CreateDirectory(STEM.Sys.IO.Path.GetDirectoryName(tmpFile));
}
if (File.Exists(tmpFile))
{
File.Delete(tmpFile);
}
if (File.Exists(OutputFile))
{
switch (OutputFileExists)
{
case Sys.IO.FileExistsAction.Throw:
throw new IOException("The output file already exists.");
case Sys.IO.FileExistsAction.Overwrite:
File.Delete(OutputFile);
break;
case Sys.IO.FileExistsAction.OverwriteIfNewer:
if (File.GetLastWriteTimeUtc(OutputFile) >= Directory.GetLastWriteTimeUtc(SourcePath))
{
return(true);
}
File.Delete(OutputFile);
break;
case Sys.IO.FileExistsAction.Skip:
return(true);
}
}
long inLen = 0;
long outLen = 0;
using (FileStream fs = File.Open(tmpFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None))
{
using (GZipOutputStream zStream = new GZipOutputStream(fs))
{
zStream.IsStreamOwner = false;
using (TarOutputStream tStream = new TarOutputStream(zStream))
{
tStream.IsStreamOwner = false;
foreach (string file in STEM.Sys.IO.Directory.STEM_GetFiles(SourcePath, FileFilter, DirectoryFilter, (RecurseSource ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly), ExpandSource))
{
string name = STEM.Sys.IO.Path.GetFileName(file);
if (RetainDirectoryStructure)
{
name = file.Replace(SourcePath, "");
}
name = name.Trim(Path.DirectorySeparatorChar);
if (InstructionSet.KeyManager.Lock(file))
{
try
{
TarEntry e = TarEntry.CreateEntryFromFile(file);
e.Name = name;
e.ModTime = File.GetLastWriteTimeUtc(file);
using (FileStream s = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read))
{
inLen += s.Length;
tStream.PutNextEntry(e);
s.CopyTo(tStream);
tStream.CloseEntry();
_Files[name] = file;
}
}
finally
{
if (!_Files.ContainsKey(name))
{
InstructionSet.KeyManager.Unlock(file);
}
}
}
}
}
}
outLen = fs.Position;
}
bool goodOutput = (_Files.Count > 0) || AllowEmptyGzResult;
if (goodOutput)
{
if (File.Exists(OutputFile))
{
switch (OutputFileExists)
{
case Sys.IO.FileExistsAction.Throw:
throw new IOException("The output file already exists.");
case Sys.IO.FileExistsAction.Overwrite:
File.Delete(OutputFile);
break;
case Sys.IO.FileExistsAction.OverwriteIfNewer:
if (File.GetLastWriteTimeUtc(OutputFile) >= Directory.GetLastWriteTimeUtc(SourcePath))
{
return(true);
}
File.Delete(OutputFile);
break;
case Sys.IO.FileExistsAction.Skip:
return(true);
}
}
STEM.Sys.IO.File.STEM_Move(tmpFile, OutputFile, OutputFileExists, out _CreatedFile);
if (DeleteSource)
{
foreach (string file in _Files.Values)
{
try
{
File.Delete(file);
}
catch
{
}
}
}
if (PopulatePostMortemMeta)
{
PostMortemMetaData["FilesArchived"] = _Files.Count.ToString();
PostMortemMetaData["OutputFilename"] = _CreatedFile;
PostMortemMetaData["InputBytes"] = inLen.ToString();
PostMortemMetaData["OutputBytes"] = outLen.ToString();
}
}
}
catch (Exception ex)
{
AppendToMessage(ex.ToString());
Exceptions.Add(ex);
}
finally
{
try
{
if (File.Exists(tmpFile))
{
File.Delete(tmpFile);
}
}
catch { }
foreach (string file in _Files.Values)
{
InstructionSet.KeyManager.Unlock(file);
}
}
return(Exceptions.Count == 0);
}
public TarEntry CreateEntryFromFile(string fileName) => TarEntry.CreateEntryFromFile(fileName);
/// <summary>
/// Zips the specified directory files to a zip file.
/// </summary>
/// <param name="zipFilename">The filename and path of the zip file to create.</param>
/// <param name="zipDirectorPath">The directory of the files to zip.</param>
/// <param name="pattern">The directory search pattern.</param>
/// <param name="searchOption">The directory search option.</param>
/// <param name="extensionRegularExpression">The regular expression for excluding files from being compressed.</param>
/// <param name="filesToInclude">The list of files that are only to be compressed.</param>
/// <remarks>Extension Regular Expression should be in the form 'jpg|JPG|gif|GIF|doc|DOC|pdf|PDF'</remarks>
public static void Compress(string zipFilename, string zipDirectorPath, string pattern,
SearchOption searchOption, string extensionRegularExpression, List <string> filesToInclude)
{
// Get the collection of files in the directory
string[] files = Directory.GetFiles(zipDirectorPath.TrimEnd('\\') + "\\", pattern, searchOption);
// Create all the streams
using (Stream zipStream = File.Create(zipFilename))
using (BZip2OutputStream stream = new BZip2OutputStream(zipStream, 9))
using (TarArchive archive = TarArchive.CreateOutputTarArchive(stream, TarBuffer.DefaultBlockFactor))
{
// Assign the archive properties
archive.SetKeepOldFiles(false);
archive.AsciiTranslate = true;
archive.SetUserInfo(0, "anonymous", 0, "None");
// For each file found
foreach (string file in files)
{
bool excludeFile = false;
// If a regular expression has been supplied.
if (!String.IsNullOrEmpty(extensionRegularExpression))
{
excludeFile = Regex.IsMatch(file.Trim(), @".*\.(" + extensionRegularExpression + @")$");
}
// Find all files that need to be included.
if (filesToInclude != null && !excludeFile)
{
// Should the current file be included.
IEnumerable <string> includeFiles = filesToInclude.Where(u => u.ToLower() == file.ToLower());
if (includeFiles.Count() > 0)
{
excludeFile = false;
}
else
{
excludeFile = true;
}
}
// If file should not be excluded
if (!excludeFile)
{
// Get the relative info
string relativePath = Path.GetDirectoryName(file).TrimEnd('\\') + "\\";
relativePath = relativePath.Replace(zipDirectorPath.TrimEnd('\\') + "\\", "").Replace("\\", "/");
// Get the file entry and set the relative
// path as the name of the entry.
TarEntry entry = TarEntry.CreateEntryFromFile(file);
entry.Name = (!String.IsNullOrEmpty(relativePath) ? relativePath.TrimEnd('/') + "/" : "") + Path.GetFileName(file);
// Write to the zip file.
archive.WriteEntry(entry, true);
}
}
// CLose all the streams.
archive.Close();
stream.Close();
zipStream.Close();
}
}
static async Task Main(string[] args)
{
string sourceFolder = null;
string repoName = null;
string blobContainerSasUrl = null;
var options = new OptionSet
{
{ "i=", "The source folder", i => sourceFolder = i },
{ "n=", "The repo name", n => repoName = n },
{ "o=", "The destination blob container url, can also be in the BLOB_CONTAINER_URL environment variable", o => blobContainerSasUrl = o },
};
List <string> extra = options.Parse(args);
if (extra.Any())
{
Fatal($"Unexpected argument {extra.First()}");
}
if (string.IsNullOrEmpty(sourceFolder))
{
Fatal("Missing argument -i");
}
if (string.IsNullOrEmpty(repoName))
{
Fatal("Missing argument -n");
}
if (string.IsNullOrEmpty(blobContainerSasUrl))
{
blobContainerSasUrl = Environment.GetEnvironmentVariable("BLOB_CONTAINER_URL");
}
if (string.IsNullOrEmpty(blobContainerSasUrl))
{
Fatal("Missing argument -o");
}
var containerClient = new BlobContainerClient(new Uri(blobContainerSasUrl));
string newBlobName = $"{repoName}/{DateTime.UtcNow:O}.tar.gz";
BlobClient newBlobClient = containerClient.GetBlobClient(newBlobName);
Console.WriteLine($"Uploading folder {sourceFolder} to blob {new UriBuilder(newBlobClient.Uri) {Fragment = "", Query = ""}.Uri.AbsoluteUri}");
await using (var outputFileStream = new MemoryStream())
{
await using (var gzoStream = new GZipOutputStream(outputFileStream)
{
IsStreamOwner = false
})
using (var tarArchive = TarArchive.CreateOutputTarArchive(gzoStream, Encoding.UTF8))
{
string sourceRoot = Path.GetFullPath(sourceFolder).Replace('\\', '/').TrimEnd('/');
void AddEntry(string path)
{
string normalizedPath = Path.GetFullPath(path).Replace('\\', '/');
var e = TarEntry.CreateEntryFromFile(path);
e.Name = normalizedPath.Substring(sourceRoot.Length).TrimStart('/');
Console.WriteLine($"Adding {path} as {e.Name}");
tarArchive.WriteEntry(e, false);
}
void AddFolder(string path)
{
AddEntry(path);
foreach (string file in Directory.GetFiles(path))
{
AddEntry(file);
}
foreach (string dir in Directory.GetDirectories(path))
{
AddFolder(dir);
}
}
AddFolder(sourceRoot);
}
outputFileStream.Position = 0;
await newBlobClient.UploadAsync(outputFileStream);
}
Console.WriteLine("Cleaning up old blobs");
List <BlobItem> blobs = containerClient.GetBlobs(prefix: repoName + "/").ToList();
List <BlobItem> toDelete = blobs.OrderByDescending(b => b.Name).Skip(10).ToList();
foreach (BlobItem d in toDelete)
{
Console.WriteLine($"Deleting blob {d.Name}");
await containerClient.DeleteBlobAsync(d.Name);
}
Console.WriteLine("Finished.");
}
protected override bool _Run()
{
SourcePath = STEM.Sys.IO.Path.AdjustPath(SourcePath);
OutputFile = STEM.Sys.IO.Path.AdjustPath(OutputFile);
string tmpFile = Path.Combine(Path.Combine(STEM.Sys.IO.Path.GetDirectoryName(OutputFile), "Temp"), STEM.Sys.IO.Path.GetFileName(OutputFile));
try
{
if (!Directory.Exists(STEM.Sys.IO.Path.GetDirectoryName(tmpFile)))
{
Directory.CreateDirectory(STEM.Sys.IO.Path.GetDirectoryName(tmpFile));
}
if (File.Exists(tmpFile))
{
File.Delete(tmpFile);
}
if (File.Exists(OutputFile))
{
switch (OutputFileExists)
{
case Sys.IO.FileExistsAction.Throw:
throw new IOException("The output file already exists.");
case Sys.IO.FileExistsAction.Overwrite:
File.Delete(OutputFile);
break;
case Sys.IO.FileExistsAction.OverwriteIfNewer:
if (File.GetLastWriteTimeUtc(OutputFile) >= Directory.GetLastWriteTimeUtc(SourcePath))
{
return(true);
}
File.Delete(OutputFile);
break;
case Sys.IO.FileExistsAction.Skip:
return(true);
case Sys.IO.FileExistsAction.MakeUnique:
OutputFile = STEM.Sys.IO.File.UniqueFilename(OutputFile);
break;
}
}
using (FileStream fs = File.Open(tmpFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None))
{
using (TarOutputStream tStream = new TarOutputStream(fs))
{
tStream.IsStreamOwner = false;
foreach (string file in STEM.Sys.IO.Directory.STEM_GetFiles(SourcePath, FileFilter, DirectoryFilter, (RecurseSource ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly), ExpandSource))
{
string name = STEM.Sys.IO.Path.GetFileName(file);
if (RetainDirectoryStructure)
{
name = file.Replace(SourcePath, "");
}
name = name.Trim(Path.DirectorySeparatorChar);
TarEntry e = TarEntry.CreateEntryFromFile(file);
e.Name = name;
e.ModTime = File.GetLastWriteTimeUtc(file);
using (FileStream s = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read))
{
tStream.PutNextEntry(e);
s.CopyTo(tStream);
tStream.CloseEntry();
_Files[name] = file;
}
}
}
}
File.Move(tmpFile, OutputFile);
}
catch (Exception ex)
{
AppendToMessage(ex.ToString());
Exceptions.Add(ex);
}
finally
{
try
{
if (File.Exists(tmpFile))
{
File.Delete(tmpFile);
}
}
catch { }
}
return(Exceptions.Count == 0);
}
/// <summary>
/// This is the "real" main. The class main() instantiates a tar object
/// for the application and then calls this method. Process the arguments
/// and perform the requested operation.
/// </summary>
public void InstanceMain(string[] argv)
{
TarArchive archive = null;
int argIdx = this.ProcessArguments(argv);
if (this.archiveName != null && !this.archiveName.Equals("-"))
{
if (operation == Operation.Create)
{
string dirName = Path.GetDirectoryName(archiveName);
if ((dirName.Length > 0) && !Directory.Exists(dirName))
{
Console.Error.WriteLine("Directory for archive doesnt exist");
return;
}
}
else
{
if (File.Exists(this.archiveName) == false)
{
Console.Error.WriteLine("File does not exist " + this.archiveName);
return;
}
}
}
if (operation == Operation.Create)
{ // WRITING
Stream outStream = Console.OpenStandardOutput();
if (this.archiveName != null && !this.archiveName.Equals("-"))
{
outStream = File.Create(archiveName);
}
if (outStream != null)
{
switch (this.compression)
{
case Compression.Compress:
outStream = new DeflaterOutputStream(outStream);
break;
case Compression.Gzip:
outStream = new GZipOutputStream(outStream);
break;
case Compression.Bzip2:
outStream = new BZip2OutputStream(outStream, 9);
break;
}
archive = TarArchive.CreateOutputTarArchive(outStream, this.blockingFactor);
}
}
else
{ // EXTRACTING OR LISTING
Stream inStream = Console.OpenStandardInput();
if (this.archiveName != null && !this.archiveName.Equals("-"))
{
inStream = File.OpenRead(archiveName);
}
if (inStream != null)
{
switch (this.compression)
{
case Compression.Compress:
inStream = new InflaterInputStream(inStream);
break;
case Compression.Gzip:
inStream = new GZipInputStream(inStream);
break;
case Compression.Bzip2:
inStream = new BZip2InputStream(inStream);
break;
}
archive = TarArchive.CreateInputTarArchive(inStream, this.blockingFactor);
}
}
if (archive != null)
{ // SET ARCHIVE OPTIONS
archive.SetKeepOldFiles(this.keepOldFiles);
archive.AsciiTranslate = this.asciiTranslate;
archive.SetUserInfo(this.userId, this.userName, this.groupId, this.groupName);
}
if (archive == null)
{
Console.Error.WriteLine("no processing due to errors");
}
else if (operation == Operation.Create)
{ // WRITING
if (verbose)
{
archive.ProgressMessageEvent += new ProgressMessageHandler(ShowTarProgressMessage);
}
for (; argIdx < argv.Length; ++argIdx)
{
string[] fileNames = GetFilesForSpec(argv[argIdx]);
if (fileNames.Length > 0)
{
foreach (string name in fileNames)
{
TarEntry entry = TarEntry.CreateEntryFromFile(name);
archive.WriteEntry(entry, true);
}
}
else
{
Console.Error.Write("No files for " + argv[argIdx]);
}
}
}
else if (operation == Operation.List)
{ // LISTING
archive.ProgressMessageEvent += new ProgressMessageHandler(ShowTarProgressMessage);
archive.ListContents();
}
else
{ // EXTRACTING
string userDir = Environment.CurrentDirectory;
if (verbose)
{
archive.ProgressMessageEvent += new ProgressMessageHandler(ShowTarProgressMessage);
}
if (userDir != null)
{
archive.ExtractContents(userDir);
}
}
if (archive != null)
{ // CLOSE ARCHIVE
archive.Close();
}
}
