internal static void Extract(System.IO.Stream tarStream, Models.FileTransportInfo fileTransferInfo) { using (TarInputStream tarIn = new TarInputStream(tarStream)) { TarEntry tarEntry = null; while ((tarEntry = tarIn.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } string name = tarEntry.Name.Replace('/', System.IO.Path.DirectorySeparatorChar); if (System.IO.Path.IsPathRooted(name)) { name = name.Substring(System.IO.Path.GetPathRoot(name).Length); } string outName = System.IO.Path.Combine(fileTransferInfo.DestinationFullNameWithBasePath, name); string directoryName = System.IO.Path.GetDirectoryName(outName); System.IO.Directory.CreateDirectory(directoryName); System.IO.FileStream outStr = new System.IO.FileStream(outName, System.IO.FileMode.Create); tarIn.CopyEntryContents(outStr); outStr.Close(); DateTime myDt = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc); System.IO.File.SetLastWriteTimeUtc(outName, myDt); } tarIn.Close(); } }
/// <summary> /// Gets an enumerable of ticks for the given CoinAPI tar entry /// </summary> /// <param name="tar">The tar input stream</param> /// <param name="entryData">The entry data</param> /// <returns>An <see cref="IEnumerable{Tick}"/> for the ticks read from the entry</returns> public IEnumerable <Tick> ProcessCoinApiEntry(TarInputStream tar, CoinApiEntryData entryData) { Log.Trace("CoinApiDataReader.ProcessTarEntry(): Processing " + $"{entryData.Symbol.ID.Market}-{entryData.Symbol.Value}-{entryData.TickType} " + $"for {entryData.Date:yyyy-MM-dd}"); using (var gzipStream = new MemoryStream()) { tar.CopyEntryContents(gzipStream); gzipStream.Seek(0, SeekOrigin.Begin); using (var innerStream = new GZipStream(gzipStream, CompressionMode.Decompress)) { using (var reader = new StreamReader(innerStream)) { var headerLine = reader.ReadLine(); if (headerLine == null) { throw new Exception($"CoinApiDataReader.ProcessTarEntry(): CSV header not found for entry name: {entryData.Name}"); } var headerParts = headerLine.Split(';').ToList(); var ticks = entryData.TickType == TickType.Trade ? ParseTradeData(entryData.Symbol, reader, headerParts) : ParseQuoteData(entryData.Symbol, reader, headerParts); foreach (var tick in ticks) { yield return(tick); } } } } }
internal static void ToFolder(string targetDir, MemoryStream tarStream) { var inputTarStream = new TarInputStream(tarStream); TarEntry tarEntry; while ((tarEntry = inputTarStream.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } var name = SanitizeName(tarEntry.Name); if (Path.IsPathRooted(name)) { name = name.Substring(Path.GetPathRoot(name).Length); } var outName = Path.Combine(targetDir, name); var directoryName = Path.GetDirectoryName(outName); Directory.CreateDirectory(directoryName); var outStr = new FileStream(outName, FileMode.Create); inputTarStream.CopyEntryContents(outStr); outStr.Close(); } inputTarStream.Close(); }
private static void ExtractTarStream(Stream inputStream, DirectoryInfo outputDir) { using (var tarIn = new TarInputStream(inputStream)) { TarEntry tarEntry; while ((tarEntry = tarIn.GetNextEntry()) != null) { string entryName = tarEntry.Name; // Remove any root e.g. '\' because a PathRooted filename defeats Path.Combine if (Path.IsPathRooted(entryName)) { entryName = entryName.Substring(Path.GetPathRoot(entryName).Length); } string outName = Path.Combine(outputDir.FullName, RemoveInvalidChars(entryName)); if (tarEntry.IsDirectory) { Directory.CreateDirectory(outName); } else { Directory.CreateDirectory(Path.GetDirectoryName(outName)); using (var outStr = new FileStream(outName, FileMode.Create)) { tarIn.CopyEntryContents(outStr); } } } } }
public LuaFunction load(string modulename) { string lua_chunk = ""; string filename = modulename + ".lua"; in_stream_.Position = 0; // rewind Stream gzipStream = new BZip2InputStream(in_stream_); TarInputStream tar = new TarInputStream(gzipStream); TarEntry tarEntry; LuaFunction func = null; while ((tarEntry = tar.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } if (filename == tarEntry.Name) { MemoryStream out_stream = new MemoryStream(); tar.CopyEntryContents(out_stream); out_stream.Position = 0; // rewind StreamReader stream_reader = new StreamReader(out_stream); lua_chunk = stream_reader.ReadToEnd(); func = lua_.LoadString(lua_chunk, modulename); string dum = func.ToString(); break; } } return func; }
/// <summary> /// Tar files only support sequential access! /// </summary> /// <param name="tarFileName">The tar file</param> /// <param name="file">The file to extract</param> /// <param name="targetDir">The destination directory</param> public void ExtractTarByEntry(string tarFileName, string file, string targetDir) { using (FileStream fsIn = new FileStream(tarFileName, FileMode.Open, FileAccess.Read)) { TarInputStream tarIn = new TarInputStream(fsIn); TarEntry tarEntry; while ((tarEntry = tarIn.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } if (tarEntry.Name != file) { continue; } // Apply further name transformations here as necessary string outName = Path.Combine(targetDir, file); using (FileStream outStr = new FileStream(outName, FileMode.Create)) { tarIn.CopyEntryContents(outStr); } } tarIn.Close(); } }
public static UserInfo Get(string name) { string key = ""; Image icon = new Bitmap(10, 10); DirectoryInfo dir = new DirectoryInfo(SYS.path + "\\User\\" + name); using (FileStream stream = (new Temp()).LoadFile(dir.FullName + "\\.UserInfo\\data.tar")) { using (TarInputStream file = new TarInputStream(stream)) { TarEntry Epasswd = file.GetNextEntry(); byte[] BSpasswd = new byte[Epasswd.Size]; file.Read(BSpasswd, 0, BSpasswd.Length); key = System.Text.UnicodeEncoding.Unicode.GetString(BSpasswd, 0, BSpasswd.Length); using (FileStream ico = (new Temp()).CreateFile()) { file.GetNextEntry(); file.CopyEntryContents(ico); Bitmap b = new Bitmap(ico); icon = b.Clone(new Rectangle(0, 0, b.Width, b.Height), b.PixelFormat); b.Dispose(); } } } UserInfo user = new UserInfo(name, key); user.Init(icon); return(user); }
private void processTask(RestoreTask task) { Logger.Debug("StorageThread:processTask:RestoreTask"); Stream inStream = File.OpenRead(task.ArchivePath); Stream gzipStream = new GZipInputStream(inStream); TarInputStream tarStream = new TarInputStream(gzipStream); TarEntry entry; List <string> list = task.RelativeFilenames(); RecoverResult recover = new RecoverResult(task.OutputDir, true); while ((entry = tarStream.GetNextEntry()) != null) { if (entry.IsDirectory) { continue; } if (list.IndexOf(entry.Name) != -1) { string name = entry.Name.Replace('/', Path.DirectorySeparatorChar); name = Path.Combine(task.OutputDir, name); Directory.CreateDirectory(Path.GetDirectoryName(name)); FileStream outStream = new FileStream(name, FileMode.CreateNew); tarStream.CopyEntryContents(outStream); outStream.Close(); DateTime myDt = DateTime.SpecifyKind(entry.ModTime, DateTimeKind.Utc); File.SetLastWriteTime(name, myDt); } } tarStream.Close(); lock (_lock) { recoverResults.Enqueue(recover); } }
public List <PackageFile> ExtractFiles(byte[] fileData) { var packageJsonFile = new PackageJson(); var files = new List <PackageFile>(); Stream inStream = new MemoryStream(fileData); Stream gzipStream = new GZipInputStream(inStream); using (var tarInputStream = new TarInputStream(gzipStream, null)) { TarEntry entry; while ((entry = tarInputStream.GetNextEntry()) != null) { using (var fileContents = new MemoryStream()) { tarInputStream.CopyEntryContents(fileContents); var fileName = GetFileName(entry.Name); if (fileName == "package.json") { packageJsonFile = ParsePackageJson(fileContents, entry.Name); } files.Add(new PackageFile { FileName = entry.Name, ContentArray = fileContents.GetBuffer() }); } } } SetFilePath(files, packageJsonFile); return(files); }
static void ReadFromBundle() { using var inStream = File.OpenRead("bundle-example.tar.gz"); // by default would be bundle.tar.gz using var gzipStream = new GZipInputStream(inStream); using var tarStream = new TarInputStream(gzipStream); TarEntry current = null; MemoryStream ms = null; while (null != (current = tarStream.GetNextEntry())) { if ("/policy.wasm" == current.Name) { ms = new MemoryStream(); tarStream.CopyEntryContents(ms); break; } } tarStream.Close(); gzipStream.Close(); inStream.Close(); if (null != ms) { ms.Position = 0; var bytes = ms.ToArray(); int length = bytes.Length; // 554984 } }
private void WriteTarEntry(TarInputStream tar, TarEntry tarEntry) { if (tarEntry.IsDirectory) { return; } // Converts the unix forward slashes in the filenames to windows backslashes string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar); // Remove any root e.g. '\' because a PathRooted filename defeats Path.Combine if (Path.IsPathRooted(name)) { name = name.Substring(Path.GetPathRoot(name).Length); } // Apply further name transformations here as necessary string outName = Path.Combine(recoveryOutputDirectory, name); string directoryName = Path.GetDirectoryName(outName); // Does nothing if directory exists Directory.CreateDirectory(directoryName); using (FileStream outStr = new FileStream(outName, FileMode.Create)) { tar.CopyEntryContents(outStr); outStr.Close(); } // Set the modification date/time. This approach seems to solve timezone issues. DateTime myDt = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc); File.SetLastWriteTime(outName, myDt); }
public byte[] GetCurrentEntryContent() { using (var copy = new MemoryStream()) { _tar.CopyEntryContents(copy); return(copy.ToArray()); } }
/// <summary> /// Extracts an a Tar archive /// </summary> /// <param name="fileEntry"> </param> /// <returns> </returns> public async IAsyncEnumerable <FileEntry> ExtractAsync(FileEntry fileEntry, ExtractorOptions options, ResourceGovernor governor) { TarEntry tarEntry; TarInputStream?tarStream = null; try { tarStream = new TarInputStream(fileEntry.Content); } catch (Exception e) { Logger.Debug(Extractor.DEBUG_STRING, ArchiveFileType.TAR, fileEntry.FullPath, string.Empty, e.GetType()); } if (tarStream != null) { while ((tarEntry = tarStream.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } var fs = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose); governor.CheckResourceGovernor(tarStream.Length); try { tarStream.CopyEntryContents(fs); } catch (Exception e) { Logger.Debug(Extractor.DEBUG_STRING, ArchiveFileType.TAR, fileEntry.FullPath, tarEntry.Name, e.GetType()); } var name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar); var newFileEntry = new FileEntry(name, fs, fileEntry, true); if (Extractor.IsQuine(newFileEntry)) { Logger.Info(Extractor.IS_QUINE_STRING, fileEntry.Name, fileEntry.FullPath); throw new OverflowException(); } await foreach (var extractedFile in Context.ExtractAsync(newFileEntry, options, governor)) { yield return(extractedFile); } } tarStream.Dispose(); } else { if (options.ExtractSelfOnFail) { yield return(fileEntry); } } }
public override void ExtractCurrentFile(Stream extractedFileContents) { if (IsDirectory()) { return; } tarStream.CopyEntryContents(extractedFileContents); }
private static byte[] ReadBytesEntry(TarInputStream tar, int length) { // Pre-allocate storage var buffer = new byte[length]; using (var stream = new MemoryStream(buffer)) { tar.CopyEntryContents(stream); } return(buffer); }
public async Task StartAsync(CancellationToken cancellationToken) { var run = true; while (run) { Console.WriteLine("Enter something for the cow to say:"); var text = Console.ReadLine(); DockerClient client = new DockerClientConfiguration( new Uri("http://localhost:2375")) .CreateClient(); var container = await client.Containers.CreateContainerAsync(new Docker.DotNet.Models.CreateContainerParameters { Image = "cowsaid" }); Console.WriteLine(container.ID); await client.Containers.StartContainerAsync(container.ID, new Docker.DotNet.Models.ContainerStartParameters { }); var inspect = await client.Containers.InspectContainerAsync(container.ID); Console.Write("Container name: " + inspect.Name); await client.Containers.WaitContainerAsync(container.ID); var archiveResult = await client.Containers.GetArchiveFromContainerAsync(container.ID, new Docker.DotNet.Models.GetArchiveFromContainerParameters { Path = "cowsaid.txt" }, false); using (var tarStream = new TarInputStream(archiveResult.Stream)) { for (var entry = tarStream.GetNextEntry(); entry != null; entry = tarStream.GetNextEntry()) { if (entry.Name == "cowsaid.txt") { var tempFile = Path.GetTempFileName(); using (var file = File.OpenWrite(tempFile)) { tarStream.CopyEntryContents(file); } Console.WriteLine("Extracted contents to " + tempFile); } } } Console.WriteLine("Removing container " + inspect.Name); await client.Containers.RemoveContainerAsync(container.ID, new Docker.DotNet.Models.ContainerRemoveParameters { RemoveVolumes = true }); } }
void UnpackTar(Stream strm) { TarInputStream tar = new TarInputStream(strm); byte[] tempBuf = new byte[65536]; progressBar2.Style = ProgressBarStyle.Marquee; int done = 0; for (; ;) { TarEntry entry = tar.GetNextEntry(); if (entry == null) { break; } string strName = entry.Name; string firstComponent = strName.Substring(0, strName.IndexOf('/')); if (firstComponent.ToUpper() == m_NamePrefixToDrop.ToUpper()) { strName = strName.Substring(m_NamePrefixToDrop.Length + 1); } if (strName == "") { continue; } string fn = m_Dir + "\\" + strName; if ((_Filter == null) || (_Filter(strName))) { if (entry.IsDirectory) { if (!Directory.Exists(fn)) { Directory.CreateDirectory(fn); } } else { using (FileStream ostrm = new FileStream(fn, FileMode.Create)) tar.CopyEntryContents(ostrm); } } if ((done += (int)entry.Size) > progressBar2.Maximum) { done = progressBar2.Maximum; } progressBar2.Value = done; Application.DoEvents(); } }
private static string ReadStringEntry(TarInputStream tar, int length) { // Pre-allocate storage var buffer = new byte[length]; using (var stream = new MemoryStream(buffer)) { tar.CopyEntryContents(stream); } return(Encoding.UTF8.GetString(buffer)); }
private static int ExtractTar(string tarFile, string targetDir) { // Count up the files inside the tar var numFiles = 0; // Ensure the target directory exists Directory.CreateDirectory(targetDir); using (FileStream fsIn = new FileStream(tarFile, FileMode.Open, FileAccess.Read)) { using (TarInputStream tarIn = new TarInputStream(fsIn)) { TarEntry tarEntry; try { while ((tarEntry = tarIn.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } string name = Path.GetFileName(tarEntry.Name); string outName = Path.Combine(targetDir, name); using (FileStream outStr = new FileStream(outName, FileMode.Create)) { tarIn.CopyEntryContents(outStr); } numFiles++; } } catch (Exception ex) { Utils.Log("ERROR - Tar file possibly corrupted: " + tarFile); Utils.Log(ex.ToString()); // Empty the target directory var dirInfo = new DirectoryInfo(targetDir); foreach (FileInfo file in dirInfo.EnumerateFiles()) { file.Delete(); } numFiles = 0; } } } return(numFiles); }
/// <summary> /// Extracts the tar by entry. /// </summary> /// <param name="tarFileName"> /// Name of the tar file. /// </param> /// <param name="targetDir"> /// The target dir. /// </param> /// <param name="asciiTranslate"> /// if set to <c>true</c> [ASCII translate]. /// </param> public static void ExtractTarByEntry(string tarFileName, string targetDir, bool asciiTranslate) { using (FileStream fsIn = new FileStream(tarFileName, FileMode.Open, FileAccess.Read)) { using (TarInputStream tarIn = new TarInputStream(fsIn)) { TarEntry tarEntry; while ((tarEntry = tarIn.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } // Converts the unix forward slashes in the filenames to windows backslashes string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar); // Remove any root e.g. '\' because a PathRooted filename defeats Path.Combine if (Path.IsPathRooted(name)) { name = name.Substring(Path.GetPathRoot(name).Length); } // Apply further name transformations here as necessary string outName = Path.Combine(targetDir, name); string directoryName = Path.GetDirectoryName(outName); Directory.CreateDirectory(directoryName); // Does nothing if directory exists using (FileStream outStr = new FileStream(outName, FileMode.Create)) { if (asciiTranslate) { CopyWithAsciiTranslate(tarIn, outStr); } else { tarIn.CopyEntryContents(outStr); } } // outStr.Close(); Set the modification date/time. This approach seems to // solve timezone issues. DateTime myDt = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc); File.SetLastWriteTime(outName, myDt); } // tarIn.Close(); } } }
private void ProcessTarInputStream(TarInputStream tarStream) { var entry = tarStream.GetNextEntry(); OutputEpgFile = Path.Combine(OutputDirectoryPath, entry.Name); var outStream = new FileStream(OutputEpgFile, FileMode.CreateNew); while (entry != null) { Console.WriteLine($"Extracting: {entry.Name} to {OutputEpgFile}"); tarStream.CopyEntryContents(outStream); entry = tarStream.GetNextEntry(); } }
private static void ProcessTarGzArchive(FileInfo startFileInfo, Dictionary <string, long> wordFrequencyDictionary, StreamWriter logWriter) { HashSet <string> pathSet = new HashSet <string>(); using (var fileStream = startFileInfo.OpenRead()) { using (var decompressedStream = new GZipStream(fileStream, CompressionMode.Decompress)) { using (var tarStream = new TarInputStream(decompressedStream)) { TarEntry tarEntry; while ((tarEntry = tarStream.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } string filePath = tarEntry.Name; string dirPath = Path.GetDirectoryName(filePath); if (pathSet.Contains(dirPath)) { LogMessage(logWriter, $"Ignore file {tarEntry.Name}"); continue; } LogMessage(logWriter, $"Process directory {Path.GetDirectoryName(tarEntry.Name)}"); LogMessage(logWriter, $"Process file {tarEntry.Name}"); Console.WriteLine(tarEntry.Name); pathSet.Add(dirPath); using (var stream = new MemoryStream()) { tarStream.CopyEntryContents(stream); stream.Position = 0; try { ProcessSubtitleGZ(stream, wordFrequencyDictionary, logWriter); } catch { } } } } } } }
/// </summary> // Iterates through each file entry within the supplied tar, // extracting them to the nominated folder. /// </summary> public static void ExtractTarByEntry(TarInputStream tarIn, string targetDir) { TarEntry tarEntry; while ((tarEntry = tarIn.GetNextEntry()) != null) { // Converts the unix forward slashes in the filenames to windows backslashes string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar); // Remove any root e.g. '\' because a PathRooted filename defeats Path.Combine if (Path.IsPathRooted(name)) { name = name.Substring(System.IO.Path.GetPathRoot(name).Length); } // Apply further name transformations here as necessary string outName = Path.Combine(targetDir, name); string directoryName = Path.GetDirectoryName(outName); try { if (tarEntry.IsDirectory) { Directory.CreateDirectory(outName); continue; } // Does nothing if directory exists Directory.CreateDirectory(directoryName); try { using (var outStr = File.Open(outName, FileMode.Create)) { tarIn.CopyEntryContents(outStr); } // Set the modification date/time. This approach seems to solve timezone issues. DateTime myDt = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc); File.SetLastWriteTime(outName, myDt); } catch (NotSupportedException) { Console.WriteLine($"[!] invalid file name: {outName}"); } catch (PathTooLongException) { Console.WriteLine($"[!] file name too long?! {outName}"); } } catch (NotSupportedException) { Console.WriteLine($"[!] invalid directory name: {directoryName}"); } } }
public void ExtractFolderFromArchive(FileInfo file, DirectoryInfo targetDirectory) { using (var inputStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read)) { DirectoryInfo singleRootDirectory = GetSingleRootDirectory(inputStream); inputStream.Position = 0; // Needs resetting after GetSingleRootDirectory() var tarInputStream = new TarInputStream(inputStream); TarEntry tarEntry; while ((tarEntry = tarInputStream.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar); if (singleRootDirectory != null) { name = name.Substring(singleRootDirectory.Name.Length); } if (Path.IsPathRooted(name)) { name = name.Substring(Path.GetPathRoot(name).Length); } string fullName = Path.Combine(targetDirectory.FullName, name); string directoryName = Path.GetDirectoryName(fullName); if (directoryName != null) { Directory.CreateDirectory(directoryName); } var outputStream = new FileStream(fullName, FileMode.Create); tarInputStream.CopyEntryContents(outputStream); outputStream.Close(); DateTime dateTime = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc); File.SetLastWriteTime(fullName, dateTime); } tarInputStream.Close(); } _log.Debug($"Extracted tar file: {file} to folder {targetDirectory}", file.FullName, targetDirectory.FullName); }
public static async Task ExtractTar(TarInputStream tarStream, string directory) { while (true) { var entry = tarStream.GetNextEntry(); if (entry == null) { break; } EnsurePathDoesNotEscape(directory, entry.Name); var entryFileName = Path.Combine(directory, entry.Name); if (entry.TarHeader.TypeFlag == TarHeader.LF_LINK) { continue; } else if (entry.TarHeader.TypeFlag == TarHeader.LF_SYMLINK) { var target = Path.GetDirectoryName(entry.Name) is {} entryDir ? Path.Combine(entryDir, entry.TarHeader.LinkName) : entry.TarHeader.LinkName; EnsurePathDoesNotEscape(directory, target); FileUtil.CreateSymlink(entryFileName, entry.TarHeader.LinkName, entry.IsDirectory); } else { if (entry.IsDirectory) { Directory.CreateDirectory(entryFileName); } else { if (Path.GetDirectoryName(entryFileName) is {} entryDir) { Directory.CreateDirectory(entryDir); } await using var outStream = File.Create(entryFileName); tarStream.CopyEntryContents(outStream); } FileUtil.SetUnixMode(entryFileName, entry.TarHeader.Mode); } } }
void ProcessTarGzFile(string filename) { WriteOnce.SafeLog(string.Format("Analyzing .tar.gz file: [{0}]", filename), LogLevel.Trace); _appProfile.MetaData.TotalFiles = GetTarGzFileCount(filename); using (var fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read)) using (var gzipStream = new GZipInputStream(fileStream)) using (var memoryStream = new MemoryStream()) { var tarStream = new TarInputStream(gzipStream); TarEntry tarEntry; while ((tarEntry = tarStream.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } tarStream.CopyEntryContents(memoryStream); if (tarEntry.Size > MAX_FILESIZE) { _appProfile.MetaData.FilesSkipped++; WriteOnce.SafeLog(string.Format("{0} in {1} is too large. File skipped", tarEntry.Name, filename), LogLevel.Error); tarStream.Close(); continue; } var mimeType = MimeTypeMap.GetMimeType(Path.GetExtension(tarEntry.Name)); if (IgnoreMimeRegex.IsMatch(mimeType) && new FileInfo(filename).Extension != "ts") { _appProfile.MetaData.FilesSkipped++; WriteOnce.SafeLog(string.Format("Ignoring tar entry [{0}]", tarEntry.Name), LogLevel.Error); } else { //file name may contain slashes; remove prior to call byte[] streamByteArray = memoryStream.ToArray(); ProcessInMemory(Path.GetFileName(tarEntry.Name), Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length)); } memoryStream.SetLength(0); // Clear out the stream } tarStream.Close(); } }
private bool DownloadArchive(string expectedCommit) { var tDownload = client.Repository.Content.GetArchive(username, repository); var bArchive = tDownload.Result; var stream = new MemoryStream(bArchive); Stream gzipStream = new GZipInputStream(stream); Stream gzipStream2 = new GZipInputStream(stream); TarInputStream tarIn = new TarInputStream(gzipStream); TarEntry tarEntry; var strRoot = ""; while ((tarEntry = tarIn.GetNextEntry()) != null) { string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar); if (strRoot == "") { strRoot = name; if (!strRoot.Contains(expectedCommit)) { return(false); } } if (tarEntry.IsDirectory) { continue; } if (Path.IsPathRooted(name)) { name = name.Substring(Path.GetPathRoot(name).Length); } name = name.Replace(strRoot, ""); string outName = Path.Combine(output, name); string directoryName = Path.GetDirectoryName(outName); Directory.CreateDirectory(directoryName); FileStream outStr = new FileStream(outName, System.IO.FileMode.Create); tarIn.CopyEntryContents(outStr); outStr.Close(); } tarIn.Close(); gzipStream.Close(); stream.Close(); return(true); }
/// <summary> /// Enumerate through the files of a TAR and get a list of KVP names-byte arrays /// </summary> /// <param name="stream">The input tar stream</param> /// <param name="isTarGz">True if the input stream is a .tar.gz or .tgz</param> /// <returns>An enumerable containing each tar entry and it's contents</returns> public static IEnumerable <KeyValuePair <string, byte[]> > UnTar(Stream stream, bool isTarGz) { using (var tar = new TarInputStream(isTarGz ? (Stream) new GZipInputStream(stream) : stream)) { TarEntry entry; while ((entry = tar.GetNextEntry()) != null) { if (entry.IsDirectory) { continue; } using (var output = new MemoryStream()) { tar.CopyEntryContents(output); yield return(new KeyValuePair <string, byte[]>(entry.Name, output.ToArray())); } } } }
private static async Task <int> ExtractTarGz(Stream stream, DocumentClient client, ILogger logger, string name) { int total = 0, errors = 0; logger.LogInformation($"Decompressing and extracting files from {name}..."); using (var sourceStream = new GZipInputStream(stream)) { using (TarInputStream tarIn = new TarInputStream(sourceStream)) { TarEntry tarEntry; while ((tarEntry = tarIn.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } byte[] bytes = null; var str = new MemoryStream(); tarIn.CopyEntryContents(str); bytes = str.ToArray(); var rslt = await processMessage(Encoding.UTF8.GetString(bytes), client, logger, tarEntry.Name, name); total++; if (!rslt) { errors++; //logger.LogTrace("Unable to process file: " + tarEntry.Name + " un-supported format!"); } if (total % 1000 == 0) { logger.LogTrace($"Processed {total} files with {errors} invalid files from archive {name}"); } } } } logger.LogInformation($"Processed {total} files with {errors} invalid files from archive {name}"); logger.LogTrace($"Processed {total} files with {errors} invalid files from archive {name}"); return(total); }
private static Dictionary <String, Byte[]> UnzipToMemory(Stream originalZipStream) { var result = new Dictionary <string, Byte[]>(); using (var stream = new MemoryStream()) { Stream gzipStream = new GZipInputStream(originalZipStream); TarInputStream tarStream = new TarInputStream(gzipStream); TarEntry tarEntry; while ((tarEntry = tarStream.GetNextEntry()) != null) { if (tarEntry.IsDirectory) { continue; } string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar); if (!name.EndsWith(".xml")) { continue; } if (Path.IsPathRooted(name)) { name = name.Substring(Path.GetPathRoot(name).Length); } using (var contentStream = new MemoryStream()) { tarStream.CopyEntryContents(contentStream); result.Add(name.ToLower(), contentStream.ToArray()); } } tarStream.Close(); } return(result); }