internal static void Extract(System.IO.Stream tarStream, Models.FileTransportInfo fileTransferInfo)
{
using (TarInputStream tarIn = new TarInputStream(tarStream))
{
TarEntry tarEntry = null;
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
string name = tarEntry.Name.Replace('/', System.IO.Path.DirectorySeparatorChar);
if (System.IO.Path.IsPathRooted(name))
{
name = name.Substring(System.IO.Path.GetPathRoot(name).Length);
}
string outName = System.IO.Path.Combine(fileTransferInfo.DestinationFullNameWithBasePath, name);
string directoryName = System.IO.Path.GetDirectoryName(outName);
System.IO.Directory.CreateDirectory(directoryName);
System.IO.FileStream outStr = new System.IO.FileStream(outName, System.IO.FileMode.Create);
tarIn.CopyEntryContents(outStr);
outStr.Close();
DateTime myDt = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc);
System.IO.File.SetLastWriteTimeUtc(outName, myDt);
}
tarIn.Close();
}
}
/// <summary>
/// Gets an enumerable of ticks for the given CoinAPI tar entry
/// </summary>
/// <param name="tar">The tar input stream</param>
/// <param name="entryData">The entry data</param>
/// <returns>An <see cref="IEnumerable{Tick}"/> for the ticks read from the entry</returns>
public IEnumerable <Tick> ProcessCoinApiEntry(TarInputStream tar, CoinApiEntryData entryData)
{
Log.Trace("CoinApiDataReader.ProcessTarEntry(): Processing " +
$"{entryData.Symbol.ID.Market}-{entryData.Symbol.Value}-{entryData.TickType} " +
$"for {entryData.Date:yyyy-MM-dd}");
using (var gzipStream = new MemoryStream())
{
tar.CopyEntryContents(gzipStream);
gzipStream.Seek(0, SeekOrigin.Begin);
using (var innerStream = new GZipStream(gzipStream, CompressionMode.Decompress))
{
using (var reader = new StreamReader(innerStream))
{
var headerLine = reader.ReadLine();
if (headerLine == null)
{
throw new Exception($"CoinApiDataReader.ProcessTarEntry(): CSV header not found for entry name: {entryData.Name}");
}
var headerParts = headerLine.Split(';').ToList();
var ticks = entryData.TickType == TickType.Trade
? ParseTradeData(entryData.Symbol, reader, headerParts)
: ParseQuoteData(entryData.Symbol, reader, headerParts);
foreach (var tick in ticks)
{
yield return(tick);
}
}
}
}
}
internal static void ToFolder(string targetDir, MemoryStream tarStream)
{
var inputTarStream = new TarInputStream(tarStream);
TarEntry tarEntry;
while ((tarEntry = inputTarStream.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
var name = SanitizeName(tarEntry.Name);
if (Path.IsPathRooted(name))
{
name = name.Substring(Path.GetPathRoot(name).Length);
}
var outName = Path.Combine(targetDir, name);
var directoryName = Path.GetDirectoryName(outName);
Directory.CreateDirectory(directoryName);
var outStr = new FileStream(outName, FileMode.Create);
inputTarStream.CopyEntryContents(outStr);
outStr.Close();
}
inputTarStream.Close();
}
private static void ExtractTarStream(Stream inputStream, DirectoryInfo outputDir)
{
using (var tarIn = new TarInputStream(inputStream)) {
TarEntry tarEntry;
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
string entryName = tarEntry.Name;
// Remove any root e.g. '\' because a PathRooted filename defeats Path.Combine
if (Path.IsPathRooted(entryName))
{
entryName = entryName.Substring(Path.GetPathRoot(entryName).Length);
}
string outName = Path.Combine(outputDir.FullName, RemoveInvalidChars(entryName));
if (tarEntry.IsDirectory)
{
Directory.CreateDirectory(outName);
}
else
{
Directory.CreateDirectory(Path.GetDirectoryName(outName));
using (var outStr = new FileStream(outName, FileMode.Create)) {
tarIn.CopyEntryContents(outStr);
}
}
}
}
}
public LuaFunction load(string modulename)
{
string lua_chunk = "";
string filename = modulename + ".lua";
in_stream_.Position = 0; // rewind
Stream gzipStream = new BZip2InputStream(in_stream_);
TarInputStream tar = new TarInputStream(gzipStream);
TarEntry tarEntry;
LuaFunction func = null;
while ((tarEntry = tar.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
if (filename == tarEntry.Name)
{
MemoryStream out_stream = new MemoryStream();
tar.CopyEntryContents(out_stream);
out_stream.Position = 0; // rewind
StreamReader stream_reader = new StreamReader(out_stream);
lua_chunk = stream_reader.ReadToEnd();
func = lua_.LoadString(lua_chunk, modulename);
string dum = func.ToString();
break;
}
}
return func;
}
/// <summary>
/// Tar files only support sequential access!
/// </summary>
/// <param name="tarFileName">The tar file</param>
/// <param name="file">The file to extract</param>
/// <param name="targetDir">The destination directory</param>
public void ExtractTarByEntry(string tarFileName, string file, string targetDir)
{
using (FileStream fsIn = new FileStream(tarFileName, FileMode.Open, FileAccess.Read))
{
TarInputStream tarIn = new TarInputStream(fsIn);
TarEntry tarEntry;
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
if (tarEntry.Name != file)
{
continue;
}
// Apply further name transformations here as necessary
string outName = Path.Combine(targetDir, file);
using (FileStream outStr = new FileStream(outName, FileMode.Create))
{
tarIn.CopyEntryContents(outStr);
}
}
tarIn.Close();
}
}
public static UserInfo Get(string name)
{
string key = "";
Image icon = new Bitmap(10, 10);
DirectoryInfo dir = new DirectoryInfo(SYS.path + "\\User\\" + name);
using (FileStream stream = (new Temp()).LoadFile(dir.FullName + "\\.UserInfo\\data.tar")) {
using (TarInputStream file = new TarInputStream(stream)) {
TarEntry Epasswd = file.GetNextEntry();
byte[] BSpasswd = new byte[Epasswd.Size];
file.Read(BSpasswd, 0, BSpasswd.Length);
key = System.Text.UnicodeEncoding.Unicode.GetString(BSpasswd, 0, BSpasswd.Length);
using (FileStream ico = (new Temp()).CreateFile()) {
file.GetNextEntry();
file.CopyEntryContents(ico);
Bitmap b = new Bitmap(ico);
icon = b.Clone(new Rectangle(0, 0, b.Width, b.Height), b.PixelFormat);
b.Dispose();
}
}
}
UserInfo user = new UserInfo(name, key);
user.Init(icon);
return(user);
}
private void processTask(RestoreTask task)
{
Logger.Debug("StorageThread:processTask:RestoreTask");
Stream inStream = File.OpenRead(task.ArchivePath);
Stream gzipStream = new GZipInputStream(inStream);
TarInputStream tarStream = new TarInputStream(gzipStream);
TarEntry entry;
List <string> list = task.RelativeFilenames();
RecoverResult recover = new RecoverResult(task.OutputDir, true);
while ((entry = tarStream.GetNextEntry()) != null)
{
if (entry.IsDirectory)
{
continue;
}
if (list.IndexOf(entry.Name) != -1)
{
string name = entry.Name.Replace('/', Path.DirectorySeparatorChar);
name = Path.Combine(task.OutputDir, name);
Directory.CreateDirectory(Path.GetDirectoryName(name));
FileStream outStream = new FileStream(name, FileMode.CreateNew);
tarStream.CopyEntryContents(outStream);
outStream.Close();
DateTime myDt = DateTime.SpecifyKind(entry.ModTime, DateTimeKind.Utc);
File.SetLastWriteTime(name, myDt);
}
}
tarStream.Close();
lock (_lock)
{
recoverResults.Enqueue(recover);
}
}
public List <PackageFile> ExtractFiles(byte[] fileData)
{
var packageJsonFile = new PackageJson();
var files = new List <PackageFile>();
Stream inStream = new MemoryStream(fileData);
Stream gzipStream = new GZipInputStream(inStream);
using (var tarInputStream = new TarInputStream(gzipStream, null))
{
TarEntry entry;
while ((entry = tarInputStream.GetNextEntry()) != null)
{
using (var fileContents = new MemoryStream())
{
tarInputStream.CopyEntryContents(fileContents);
var fileName = GetFileName(entry.Name);
if (fileName == "package.json")
{
packageJsonFile = ParsePackageJson(fileContents, entry.Name);
}
files.Add(new PackageFile
{
FileName = entry.Name,
ContentArray = fileContents.GetBuffer()
});
}
}
}
SetFilePath(files, packageJsonFile);
return(files);
}
static void ReadFromBundle()
{
using var inStream = File.OpenRead("bundle-example.tar.gz"); // by default would be bundle.tar.gz
using var gzipStream = new GZipInputStream(inStream);
using var tarStream = new TarInputStream(gzipStream);
TarEntry current = null;
MemoryStream ms = null;
while (null != (current = tarStream.GetNextEntry()))
{
if ("/policy.wasm" == current.Name)
{
ms = new MemoryStream();
tarStream.CopyEntryContents(ms);
break;
}
}
tarStream.Close();
gzipStream.Close();
inStream.Close();
if (null != ms)
{
ms.Position = 0;
var bytes = ms.ToArray();
int length = bytes.Length; // 554984
}
}
private void WriteTarEntry(TarInputStream tar, TarEntry tarEntry)
{
if (tarEntry.IsDirectory)
{
return;
}
// Converts the unix forward slashes in the filenames to windows backslashes
string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);
// Remove any root e.g. '\' because a PathRooted filename defeats Path.Combine
if (Path.IsPathRooted(name))
{
name = name.Substring(Path.GetPathRoot(name).Length);
}
// Apply further name transformations here as necessary
string outName = Path.Combine(recoveryOutputDirectory, name);
string directoryName = Path.GetDirectoryName(outName);
// Does nothing if directory exists
Directory.CreateDirectory(directoryName);
using (FileStream outStr = new FileStream(outName, FileMode.Create))
{
tar.CopyEntryContents(outStr);
outStr.Close();
}
// Set the modification date/time. This approach seems to solve timezone issues.
DateTime myDt = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc);
File.SetLastWriteTime(outName, myDt);
}
public byte[] GetCurrentEntryContent()
{
using (var copy = new MemoryStream())
{
_tar.CopyEntryContents(copy);
return(copy.ToArray());
}
}
/// <summary>
/// Extracts an a Tar archive
/// </summary>
/// <param name="fileEntry"> </param>
/// <returns> </returns>
public async IAsyncEnumerable <FileEntry> ExtractAsync(FileEntry fileEntry, ExtractorOptions options, ResourceGovernor governor)
{
TarEntry tarEntry;
TarInputStream?tarStream = null;
try
{
tarStream = new TarInputStream(fileEntry.Content);
}
catch (Exception e)
{
Logger.Debug(Extractor.DEBUG_STRING, ArchiveFileType.TAR, fileEntry.FullPath, string.Empty, e.GetType());
}
if (tarStream != null)
{
while ((tarEntry = tarStream.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
var fs = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose);
governor.CheckResourceGovernor(tarStream.Length);
try
{
tarStream.CopyEntryContents(fs);
}
catch (Exception e)
{
Logger.Debug(Extractor.DEBUG_STRING, ArchiveFileType.TAR, fileEntry.FullPath, tarEntry.Name, e.GetType());
}
var name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);
var newFileEntry = new FileEntry(name, fs, fileEntry, true);
if (Extractor.IsQuine(newFileEntry))
{
Logger.Info(Extractor.IS_QUINE_STRING, fileEntry.Name, fileEntry.FullPath);
throw new OverflowException();
}
await foreach (var extractedFile in Context.ExtractAsync(newFileEntry, options, governor))
{
yield return(extractedFile);
}
}
tarStream.Dispose();
}
else
{
if (options.ExtractSelfOnFail)
{
yield return(fileEntry);
}
}
}
public override void ExtractCurrentFile(Stream extractedFileContents)
{
if (IsDirectory())
{
return;
}
tarStream.CopyEntryContents(extractedFileContents);
}
private static byte[] ReadBytesEntry(TarInputStream tar, int length)
{
// Pre-allocate storage
var buffer = new byte[length];
using (var stream = new MemoryStream(buffer))
{
tar.CopyEntryContents(stream);
}
return(buffer);
}
public async Task StartAsync(CancellationToken cancellationToken)
{
var run = true;
while (run)
{
Console.WriteLine("Enter something for the cow to say:");
var text = Console.ReadLine();
DockerClient client = new DockerClientConfiguration(
new Uri("http://localhost:2375"))
.CreateClient();
var container = await client.Containers.CreateContainerAsync(new Docker.DotNet.Models.CreateContainerParameters
{
Image = "cowsaid"
});
Console.WriteLine(container.ID);
await client.Containers.StartContainerAsync(container.ID, new Docker.DotNet.Models.ContainerStartParameters {
});
var inspect = await client.Containers.InspectContainerAsync(container.ID);
Console.Write("Container name: " + inspect.Name);
await client.Containers.WaitContainerAsync(container.ID);
var archiveResult = await client.Containers.GetArchiveFromContainerAsync(container.ID, new Docker.DotNet.Models.GetArchiveFromContainerParameters {
Path = "cowsaid.txt"
}, false);
using (var tarStream = new TarInputStream(archiveResult.Stream)) {
for (var entry = tarStream.GetNextEntry(); entry != null; entry = tarStream.GetNextEntry())
{
if (entry.Name == "cowsaid.txt")
{
var tempFile = Path.GetTempFileName();
using (var file = File.OpenWrite(tempFile))
{
tarStream.CopyEntryContents(file);
}
Console.WriteLine("Extracted contents to " + tempFile);
}
}
}
Console.WriteLine("Removing container " + inspect.Name);
await client.Containers.RemoveContainerAsync(container.ID, new Docker.DotNet.Models.ContainerRemoveParameters {
RemoveVolumes = true
});
}
}
void UnpackTar(Stream strm)
{
TarInputStream tar = new TarInputStream(strm);
byte[] tempBuf = new byte[65536];
progressBar2.Style = ProgressBarStyle.Marquee;
int done = 0;
for (; ;)
{
TarEntry entry = tar.GetNextEntry();
if (entry == null)
{
break;
}
string strName = entry.Name;
string firstComponent = strName.Substring(0, strName.IndexOf('/'));
if (firstComponent.ToUpper() == m_NamePrefixToDrop.ToUpper())
{
strName = strName.Substring(m_NamePrefixToDrop.Length + 1);
}
if (strName == "")
{
continue;
}
string fn = m_Dir + "\\" + strName;
if ((_Filter == null) || (_Filter(strName)))
{
if (entry.IsDirectory)
{
if (!Directory.Exists(fn))
{
Directory.CreateDirectory(fn);
}
}
else
{
using (FileStream ostrm = new FileStream(fn, FileMode.Create))
tar.CopyEntryContents(ostrm);
}
}
if ((done += (int)entry.Size) > progressBar2.Maximum)
{
done = progressBar2.Maximum;
}
progressBar2.Value = done;
Application.DoEvents();
}
}
private static string ReadStringEntry(TarInputStream tar, int length)
{
// Pre-allocate storage
var buffer = new byte[length];
using (var stream = new MemoryStream(buffer))
{
tar.CopyEntryContents(stream);
}
return(Encoding.UTF8.GetString(buffer));
}
private static int ExtractTar(string tarFile, string targetDir)
{
// Count up the files inside the tar
var numFiles = 0;
// Ensure the target directory exists
Directory.CreateDirectory(targetDir);
using (FileStream fsIn = new FileStream(tarFile, FileMode.Open, FileAccess.Read))
{
using (TarInputStream tarIn = new TarInputStream(fsIn))
{
TarEntry tarEntry;
try
{
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
string name = Path.GetFileName(tarEntry.Name);
string outName = Path.Combine(targetDir, name);
using (FileStream outStr = new FileStream(outName, FileMode.Create))
{
tarIn.CopyEntryContents(outStr);
}
numFiles++;
}
}
catch (Exception ex)
{
Utils.Log("ERROR - Tar file possibly corrupted: " + tarFile);
Utils.Log(ex.ToString());
// Empty the target directory
var dirInfo = new DirectoryInfo(targetDir);
foreach (FileInfo file in dirInfo.EnumerateFiles())
{
file.Delete();
}
numFiles = 0;
}
}
}
return(numFiles);
}
/// <summary>
/// Extracts the tar by entry.
/// </summary>
/// <param name="tarFileName">
/// Name of the tar file.
/// </param>
/// <param name="targetDir">
/// The target dir.
/// </param>
/// <param name="asciiTranslate">
/// if set to <c>true</c> [ASCII translate].
/// </param>
public static void ExtractTarByEntry(string tarFileName, string targetDir, bool asciiTranslate)
{
using (FileStream fsIn = new FileStream(tarFileName, FileMode.Open, FileAccess.Read))
{
using (TarInputStream tarIn = new TarInputStream(fsIn))
{
TarEntry tarEntry;
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
// Converts the unix forward slashes in the filenames to windows backslashes
string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);
// Remove any root e.g. '\' because a PathRooted filename defeats Path.Combine
if (Path.IsPathRooted(name))
{
name = name.Substring(Path.GetPathRoot(name).Length);
}
// Apply further name transformations here as necessary
string outName = Path.Combine(targetDir, name);
string directoryName = Path.GetDirectoryName(outName);
Directory.CreateDirectory(directoryName); // Does nothing if directory exists
using (FileStream outStr = new FileStream(outName, FileMode.Create))
{
if (asciiTranslate)
{
CopyWithAsciiTranslate(tarIn, outStr);
}
else
{
tarIn.CopyEntryContents(outStr);
}
}
// outStr.Close(); Set the modification date/time. This approach seems to
// solve timezone issues.
DateTime myDt = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc);
File.SetLastWriteTime(outName, myDt);
}
// tarIn.Close();
}
}
}
private void ProcessTarInputStream(TarInputStream tarStream)
{
var entry = tarStream.GetNextEntry();
OutputEpgFile = Path.Combine(OutputDirectoryPath, entry.Name);
var outStream = new FileStream(OutputEpgFile, FileMode.CreateNew);
while (entry != null)
{
Console.WriteLine($"Extracting: {entry.Name} to {OutputEpgFile}");
tarStream.CopyEntryContents(outStream);
entry = tarStream.GetNextEntry();
}
}
private static void ProcessTarGzArchive(FileInfo startFileInfo, Dictionary <string, long> wordFrequencyDictionary, StreamWriter logWriter)
{
HashSet <string> pathSet = new HashSet <string>();
using (var fileStream = startFileInfo.OpenRead())
{
using (var decompressedStream = new GZipStream(fileStream, CompressionMode.Decompress))
{
using (var tarStream = new TarInputStream(decompressedStream))
{
TarEntry tarEntry;
while ((tarEntry = tarStream.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
string filePath = tarEntry.Name;
string dirPath = Path.GetDirectoryName(filePath);
if (pathSet.Contains(dirPath))
{
LogMessage(logWriter, $"Ignore file {tarEntry.Name}");
continue;
}
LogMessage(logWriter, $"Process directory {Path.GetDirectoryName(tarEntry.Name)}");
LogMessage(logWriter, $"Process file {tarEntry.Name}");
Console.WriteLine(tarEntry.Name);
pathSet.Add(dirPath);
using (var stream = new MemoryStream())
{
tarStream.CopyEntryContents(stream);
stream.Position = 0;
try
{
ProcessSubtitleGZ(stream, wordFrequencyDictionary, logWriter);
}
catch { }
}
}
}
}
}
}
/// </summary>
// Iterates through each file entry within the supplied tar,
// extracting them to the nominated folder.
/// </summary>
public static void ExtractTarByEntry(TarInputStream tarIn, string targetDir)
{
TarEntry tarEntry;
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
// Converts the unix forward slashes in the filenames to windows backslashes
string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);
// Remove any root e.g. '\' because a PathRooted filename defeats Path.Combine
if (Path.IsPathRooted(name))
{
name = name.Substring(System.IO.Path.GetPathRoot(name).Length);
}
// Apply further name transformations here as necessary
string outName = Path.Combine(targetDir, name);
string directoryName = Path.GetDirectoryName(outName);
try {
if (tarEntry.IsDirectory)
{
Directory.CreateDirectory(outName);
continue;
}
// Does nothing if directory exists
Directory.CreateDirectory(directoryName);
try {
using (var outStr = File.Open(outName, FileMode.Create)) {
tarIn.CopyEntryContents(outStr);
}
// Set the modification date/time. This approach seems to solve timezone issues.
DateTime myDt = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc);
File.SetLastWriteTime(outName, myDt);
} catch (NotSupportedException) {
Console.WriteLine($"[!] invalid file name: {outName}");
} catch (PathTooLongException) {
Console.WriteLine($"[!] file name too long?! {outName}");
}
} catch (NotSupportedException) {
Console.WriteLine($"[!] invalid directory name: {directoryName}");
}
}
}
public void ExtractFolderFromArchive(FileInfo file, DirectoryInfo targetDirectory)
{
using (var inputStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read))
{
DirectoryInfo singleRootDirectory = GetSingleRootDirectory(inputStream);
inputStream.Position = 0; // Needs resetting after GetSingleRootDirectory()
var tarInputStream = new TarInputStream(inputStream);
TarEntry tarEntry;
while ((tarEntry = tarInputStream.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);
if (singleRootDirectory != null)
{
name = name.Substring(singleRootDirectory.Name.Length);
}
if (Path.IsPathRooted(name))
{
name = name.Substring(Path.GetPathRoot(name).Length);
}
string fullName = Path.Combine(targetDirectory.FullName, name);
string directoryName = Path.GetDirectoryName(fullName);
if (directoryName != null)
{
Directory.CreateDirectory(directoryName);
}
var outputStream = new FileStream(fullName, FileMode.Create);
tarInputStream.CopyEntryContents(outputStream);
outputStream.Close();
DateTime dateTime = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc);
File.SetLastWriteTime(fullName, dateTime);
}
tarInputStream.Close();
}
_log.Debug($"Extracted tar file: {file} to folder {targetDirectory}", file.FullName, targetDirectory.FullName);
}
public static async Task ExtractTar(TarInputStream tarStream, string directory)
{
while (true)
{
var entry = tarStream.GetNextEntry();
if (entry == null)
{
break;
}
EnsurePathDoesNotEscape(directory, entry.Name);
var entryFileName = Path.Combine(directory, entry.Name);
if (entry.TarHeader.TypeFlag == TarHeader.LF_LINK)
{
continue;
}
else if (entry.TarHeader.TypeFlag == TarHeader.LF_SYMLINK)
{
var target = Path.GetDirectoryName(entry.Name) is {} entryDir
? Path.Combine(entryDir, entry.TarHeader.LinkName)
: entry.TarHeader.LinkName;
EnsurePathDoesNotEscape(directory, target);
FileUtil.CreateSymlink(entryFileName, entry.TarHeader.LinkName, entry.IsDirectory);
}
else
{
if (entry.IsDirectory)
{
Directory.CreateDirectory(entryFileName);
}
else
{
if (Path.GetDirectoryName(entryFileName) is {} entryDir)
{
Directory.CreateDirectory(entryDir);
}
await using var outStream = File.Create(entryFileName);
tarStream.CopyEntryContents(outStream);
}
FileUtil.SetUnixMode(entryFileName, entry.TarHeader.Mode);
}
}
}
void ProcessTarGzFile(string filename)
{
WriteOnce.SafeLog(string.Format("Analyzing .tar.gz file: [{0}]", filename), LogLevel.Trace);
_appProfile.MetaData.TotalFiles = GetTarGzFileCount(filename);
using (var fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read))
using (var gzipStream = new GZipInputStream(fileStream))
using (var memoryStream = new MemoryStream())
{
var tarStream = new TarInputStream(gzipStream);
TarEntry tarEntry;
while ((tarEntry = tarStream.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
tarStream.CopyEntryContents(memoryStream);
if (tarEntry.Size > MAX_FILESIZE)
{
_appProfile.MetaData.FilesSkipped++;
WriteOnce.SafeLog(string.Format("{0} in {1} is too large. File skipped", tarEntry.Name, filename), LogLevel.Error);
tarStream.Close();
continue;
}
var mimeType = MimeTypeMap.GetMimeType(Path.GetExtension(tarEntry.Name));
if (IgnoreMimeRegex.IsMatch(mimeType) && new FileInfo(filename).Extension != "ts")
{
_appProfile.MetaData.FilesSkipped++;
WriteOnce.SafeLog(string.Format("Ignoring tar entry [{0}]", tarEntry.Name), LogLevel.Error);
}
else
{
//file name may contain slashes; remove prior to call
byte[] streamByteArray = memoryStream.ToArray();
ProcessInMemory(Path.GetFileName(tarEntry.Name), Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length));
}
memoryStream.SetLength(0); // Clear out the stream
}
tarStream.Close();
}
}
private bool DownloadArchive(string expectedCommit)
{
var tDownload = client.Repository.Content.GetArchive(username, repository);
var bArchive = tDownload.Result;
var stream = new MemoryStream(bArchive);
Stream gzipStream = new GZipInputStream(stream);
Stream gzipStream2 = new GZipInputStream(stream);
TarInputStream tarIn = new TarInputStream(gzipStream);
TarEntry tarEntry;
var strRoot = "";
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);
if (strRoot == "")
{
strRoot = name;
if (!strRoot.Contains(expectedCommit))
{
return(false);
}
}
if (tarEntry.IsDirectory)
{
continue;
}
if (Path.IsPathRooted(name))
{
name = name.Substring(Path.GetPathRoot(name).Length);
}
name = name.Replace(strRoot, "");
string outName = Path.Combine(output, name);
string directoryName = Path.GetDirectoryName(outName);
Directory.CreateDirectory(directoryName);
FileStream outStr = new FileStream(outName, System.IO.FileMode.Create);
tarIn.CopyEntryContents(outStr);
outStr.Close();
}
tarIn.Close();
gzipStream.Close();
stream.Close();
return(true);
}
/// <summary>
/// Enumerate through the files of a TAR and get a list of KVP names-byte arrays
/// </summary>
/// <param name="stream">The input tar stream</param>
/// <param name="isTarGz">True if the input stream is a .tar.gz or .tgz</param>
/// <returns>An enumerable containing each tar entry and it's contents</returns>
public static IEnumerable <KeyValuePair <string, byte[]> > UnTar(Stream stream, bool isTarGz)
{
using (var tar = new TarInputStream(isTarGz ? (Stream) new GZipInputStream(stream) : stream))
{
TarEntry entry;
while ((entry = tar.GetNextEntry()) != null)
{
if (entry.IsDirectory)
{
continue;
}
using (var output = new MemoryStream())
{
tar.CopyEntryContents(output);
yield return(new KeyValuePair <string, byte[]>(entry.Name, output.ToArray()));
}
}
}
}
private static async Task <int> ExtractTarGz(Stream stream, DocumentClient client, ILogger logger, string name)
{
int total = 0, errors = 0;
logger.LogInformation($"Decompressing and extracting files from {name}...");
using (var sourceStream = new GZipInputStream(stream))
{
using (TarInputStream tarIn = new TarInputStream(sourceStream))
{
TarEntry tarEntry;
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
byte[] bytes = null;
var str = new MemoryStream();
tarIn.CopyEntryContents(str);
bytes = str.ToArray();
var rslt = await processMessage(Encoding.UTF8.GetString(bytes), client, logger, tarEntry.Name, name);
total++;
if (!rslt)
{
errors++;
//logger.LogTrace("Unable to process file: " + tarEntry.Name + " un-supported format!");
}
if (total % 1000 == 0)
{
logger.LogTrace($"Processed {total} files with {errors} invalid files from archive {name}");
}
}
}
}
logger.LogInformation($"Processed {total} files with {errors} invalid files from archive {name}");
logger.LogTrace($"Processed {total} files with {errors} invalid files from archive {name}");
return(total);
}
private static Dictionary <String, Byte[]> UnzipToMemory(Stream originalZipStream)
{
var result = new Dictionary <string, Byte[]>();
using (var stream = new MemoryStream())
{
Stream gzipStream = new GZipInputStream(originalZipStream);
TarInputStream tarStream = new TarInputStream(gzipStream);
TarEntry tarEntry;
while ((tarEntry = tarStream.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);
if (!name.EndsWith(".xml"))
{
continue;
}
if (Path.IsPathRooted(name))
{
name = name.Substring(Path.GetPathRoot(name).Length);
}
using (var contentStream = new MemoryStream())
{
tarStream.CopyEntryContents(contentStream);
result.Add(name.ToLower(), contentStream.ToArray());
}
}
tarStream.Close();
}
return(result);
}