/// <summary>
/// Enumerate through the files of a TAR and get a list of KVP names-byte arrays
/// </summary>
/// <param name="stream">The input tar stream</param>
/// <param name="isTarGz">True if the input stream is a .tar.gz or .tgz</param>
/// <returns>An enumerable containing each tar entry and it's contents</returns>
public static IEnumerable <KeyValuePair <string, byte[]> > UnTar(Stream stream, bool isTarGz)
{
using (var tar = new TarInputStream(isTarGz ? (Stream) new GZipInputStream(stream) : stream))
{
TarEntry entry;
while ((entry = tar.GetNextEntry()) != null)
{
if (entry.IsDirectory)
{
continue;
}
using (var output = new MemoryStream())
{
tar.CopyEntryContents(output);
yield return(new KeyValuePair <string, byte[]>(entry.Name, output.ToArray()));
}
}
}
}
/** Creates a compressed blob from the TarStreamBuilder and verifies it. */
private async Task VerifyBlobWithCompressionAsync()
{
// Writes the BLOB and captures the output.
MemoryStream tarByteOutputStream = new MemoryStream();
using (Stream compressorStream = new GZipStream(tarByteOutputStream, CompressionMode.Compress))
{
await testTarStreamBuilder.WriteAsTarArchiveToAsync(compressorStream).ConfigureAwait(false);
}
// Rearrange the output into input for verification.
MemoryStream byteArrayInputStream =
new MemoryStream(tarByteOutputStream.ToArray());
Stream tarByteInputStream = new GZipStream(byteArrayInputStream, CompressionMode.Decompress);
using (TarInputStream tarArchiveInputStream = new TarInputStream(tarByteInputStream))
{
VerifyTarArchive(tarArchiveInputStream);
}
}
public async Task TestToBlob_multiByteAsync()
{
testTarStreamBuilder.AddByteEntry(Encoding.UTF8.GetBytes("日本語"), "test");
testTarStreamBuilder.AddByteEntry(Encoding.UTF8.GetBytes("asdf"), "crepecake");
testTarStreamBuilder.AddBlobEntry(
Blobs.From("fib"), Encoding.UTF8.GetBytes("fib").Length, "fib");
// Writes the BLOB and captures the output.
MemoryStream tarByteOutputStream = new MemoryStream();
using (Stream compressorStream = new GZipStream(tarByteOutputStream, CompressionMode.Compress))
{
await testTarStreamBuilder.WriteAsTarArchiveToAsync(compressorStream).ConfigureAwait(false);
}
// Rearrange the output into input for verification.
MemoryStream byteArrayInputStream =
new MemoryStream(tarByteOutputStream.ToArray());
Stream tarByteInputStream = new GZipStream(byteArrayInputStream, CompressionMode.Decompress);
using (TarInputStream tarArchiveInputStream = new TarInputStream(tarByteInputStream))
{
// Verify multi-byte characters are written/read correctly
TarEntry headerFile = tarArchiveInputStream.GetNextEntry();
Assert.AreEqual("test", headerFile.Name);
Assert.AreEqual(
"日本語", Encoding.UTF8.GetString(ByteStreams.ToByteArray(tarArchiveInputStream)));
headerFile = tarArchiveInputStream.GetNextEntry();
Assert.AreEqual("crepecake", headerFile.Name);
Assert.AreEqual(
"asdf", Encoding.UTF8.GetString(ByteStreams.ToByteArray(tarArchiveInputStream)));
headerFile = tarArchiveInputStream.GetNextEntry();
Assert.AreEqual("fib", headerFile.Name);
Assert.AreEqual(
"fib", Encoding.UTF8.GetString(ByteStreams.ToByteArray(tarArchiveInputStream)));
Assert.IsNull(tarArchiveInputStream.GetNextEntry());
}
}
public void Test()
{
var stream = GetArchiveStream();
using var archive = new TarInputStream(stream, Encoding.UTF8);
var normal = archive.GetNextEntry();
normal.Name.Should().Be("normal");
normal.ModTime.Should().Be(TestFile.DefaultLastWrite);
normal.TarHeader.Mode.Should().Be(TarExtractor.DefaultMode);
var executable = archive.GetNextEntry();
executable.Name.Should().Be("executable");
executable.ModTime.Should().Be(TestFile.DefaultLastWrite);
executable.TarHeader.Mode.Should().Be(TarExtractor.DefaultMode | TarExtractor.ExecuteMode);
var symlink = archive.GetNextEntry();
symlink.Name.Should().Be("symlink");
symlink.TarHeader.TypeFlag.Should().Be(TarHeader.LF_SYMLINK);
var hardlink = archive.GetNextEntry();
hardlink.Name.Should().Be("hardlink");
hardlink.TarHeader.TypeFlag.Should().Be(TarHeader.LF_LINK);
hardlink.TarHeader.LinkName.Should().Be("normal");
var directory = archive.GetNextEntry();
directory.Name.Should().Be("dir");
directory.IsDirectory.Should().BeTrue();
directory.TarHeader.Mode.Should().Be(TarExtractor.DefaultMode | TarExtractor.ExecuteMode);
var sub = archive.GetNextEntry();
sub.Name.Should().Be("dir/sub");
sub.ModTime.Should().Be(TestFile.DefaultLastWrite);
sub.TarHeader.Mode.Should().Be(TarExtractor.DefaultMode);
}
public void ExtractTarGzFile()
{
try
{
using (var inputStream = System.IO.File.OpenRead(FullEpgPackagePath))
using (var gzStream = new GZipInputStream(inputStream))
using (var tarStream = new TarInputStream(gzStream, Encoding.UTF8))
{
ProcessTarInputStream(tarStream);
OperationsSuccessful = System.IO.File.Exists(OutputEpgFile);
tarStream.Close();
gzStream.Close();
inputStream.Close();
}
}
catch (Exception etgfException)
{
Console.WriteLine(etgfException);
throw;
}
}
public IEnumerable <string> GetFiles(string repository, Layer layer)
{
var key = $"static:layer:{layer.Digest}:files";
var scope = $"repository:{repository}:pull";
return(GetCached(scope, key, false, () =>
{
var files = new List <string>();
using (var stream = GetLayer(repository, layer))
{
var temp = Path.GetTempFileName();
try
{
using (var gunzipped = new FileStream(temp, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
{
using (var gzipStream = new GZipInputStream(stream)) { gzipStream.CopyTo(gunzipped); }
gunzipped.Seek(0, SeekOrigin.Begin);
using (var tarStream = new TarInputStream(gunzipped))
{
var entry = tarStream.GetNextEntry();
while (entry != null)
{
files.Add(entry.Name);
entry = tarStream.GetNextEntry();
}
}
}
}
catch
{
throw;
}
finally
{
File.Delete(temp);
}
}
return files;
}));
}
private static async Task <int> ExtractTarGz(Stream stream, DocumentClient client, ILogger logger, string name)
{
int total = 0, errors = 0;
logger.LogInformation($"Decompressing and extracting files from {name}...");
using (var sourceStream = new GZipInputStream(stream))
{
using (TarInputStream tarIn = new TarInputStream(sourceStream))
{
TarEntry tarEntry;
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
byte[] bytes = null;
var str = new MemoryStream();
tarIn.CopyEntryContents(str);
bytes = str.ToArray();
var rslt = await processMessage(Encoding.UTF8.GetString(bytes), client, logger, tarEntry.Name, name);
total++;
if (!rslt)
{
errors++;
//logger.LogTrace("Unable to process file: " + tarEntry.Name + " un-supported format!");
}
if (total % 1000 == 0)
{
logger.LogTrace($"Processed {total} files with {errors} invalid files from archive {name}");
}
}
}
}
logger.LogInformation($"Processed {total} files with {errors} invalid files from archive {name}");
logger.LogTrace($"Processed {total} files with {errors} invalid files from archive {name}");
return(total);
}
private static Dictionary <String, Byte[]> UnzipToMemory(Stream originalZipStream)
{
var result = new Dictionary <string, Byte[]>();
using (var stream = new MemoryStream())
{
Stream gzipStream = new GZipInputStream(originalZipStream);
TarInputStream tarStream = new TarInputStream(gzipStream);
TarEntry tarEntry;
while ((tarEntry = tarStream.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);
if (!name.EndsWith(".xml"))
{
continue;
}
if (Path.IsPathRooted(name))
{
name = name.Substring(Path.GetPathRoot(name).Length);
}
using (var contentStream = new MemoryStream())
{
tarStream.CopyEntryContents(contentStream);
result.Add(name.ToLower(), contentStream.ToArray());
}
}
tarStream.Close();
}
return(result);
}
private void ExtractTarByEntry(string tarFileName, string targetDir)
{
using (FileStream fsIn = new FileStream(tarFileName, FileMode.Open, FileAccess.Read))
{
TarInputStream tarIn = new TarInputStream(fsIn);
TarEntry tarEntry;
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
// Converts the unix forward slashes in the filenames to windows backslashes
//
string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);
// Remove any root e.g. '\' because a PathRooted filename defeats Path.Combine
if (Path.IsPathRooted(name))
{
name = name.Substring(Path.GetPathRoot(name).Length);
}
// Apply further name transformations here as necessary
string outName = Path.Combine(targetDir, name);
string directoryName = Path.GetDirectoryName(outName);
Directory.CreateDirectory(directoryName);
FileStream outStr = new FileStream(outName, FileMode.Create);
tarIn.CopyEntryContents(outStr);
outStr.Close();
// Set the modification date/time. This approach seems to solve timezone issues.
DateTime myDt = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc);
File.SetLastWriteTime(outName, myDt);
}
tarIn.Close();
}
}
void TryLongName(string name)
{
var ms = new MemoryStream();
using (TarOutputStream tarOut = new TarOutputStream(ms)) {
DateTime modTime = DateTime.Now;
TarEntry entry = TarEntry.CreateTarEntry(name);
tarOut.PutNextEntry(entry);
}
var ms2 = new MemoryStream();
ms2.Write(ms.GetBuffer(), 0, ms.GetBuffer().Length);
ms2.Seek(0, SeekOrigin.Begin);
using (TarInputStream tarIn = new TarInputStream(ms2)) {
TarEntry nextEntry = tarIn.GetNextEntry();
Assert.AreEqual(nextEntry.Name, name, "Name match failure");
}
}
/// <summary>
/// Verify the archive
/// </summary>
public static async Task <Recipe> VerifyArchiveAsync(Stream stream)
{
using (var gzipStream = new GZipStream(stream, CompressionMode.Decompress, true))
using (var archive = new TarInputStream(gzipStream))
{
Recipe recipe = null;
TarEntry entry = archive.GetNextEntry();
while (entry != null)
{
if (string.Compare(entry.Name, Constants.RecipeFileName, true) == 0)
{
recipe = await RecipeManager.LoadFromStreamAsync(archive);
break;
}
entry = archive.GetNextEntry();
}
return(recipe);
}
}
/// <summary>
/// Extracts a raw list of file entries from a tar'd/gzipped layer
/// </summary>
/// <param name="layerArchive"></param>
/// <returns></returns>
public IEnumerable <string> ExtractFiles(Stream stream)
{
using (var gzipStream = new GZipInputStream(stream)
{
IsStreamOwner = false
})
using (var tarStream = new TarInputStream(gzipStream)
{
IsStreamOwner = false
})
{
var entry = tarStream.GetNextEntry();
while (entry != null)
{
if (!entry.IsDirectory)
{
yield return(entry.Name);
}
entry = tarStream.GetNextEntry();
}
}
}
private void LayerEntriesDo(Action <string, TarEntry> layerConsumer)
{
using (TarInputStream input =
new TarInputStream(Files.NewInputStream(imageTar.ToPath())))
{
TarEntry imageEntry;
while ((imageEntry = input.GetNextEntry()) != null)
{
string imageEntryName = imageEntry.Name;
// assume all .tar.gz files are layers
if (imageEntry.IsFile() && imageEntryName.EndsWith(".tar.gz", StringComparison.Ordinal))
{
TarInputStream layer = new TarInputStream(new GZipStream(input, CompressionMode.Decompress));
TarEntry layerEntry;
while ((layerEntry = layer.GetNextEntry()) != null)
{
layerConsumer(imageEntryName, layerEntry);
}
}
}
}
}
private static DirectoryInfo GetSingleRootDirectory(Stream inputStream)
{
var tarInputStream = new TarInputStream(inputStream);
TarEntry firstEntry = tarInputStream.GetNextEntry();
if (!firstEntry.IsDirectory)
{
return(null);
}
TarEntry tarEntry;
while ((tarEntry = tarInputStream.GetNextEntry()) != null)
{
if (!tarEntry.Name.StartsWith(firstEntry.Name))
{
return(null);
}
}
return(new DirectoryInfo(firstEntry.Name));
}
/// <summary>
/// Prepares to extract a TAR archive contained in a stream.
/// </summary>
/// <param name="stream">The stream containing the archive data to be extracted. Will be disposed when the extractor is disposed.</param>
/// <param name="target">The path to the directory to extract into.</param>
/// <exception cref="IOException">The archive is damaged.</exception>
internal TarExtractor([NotNull] Stream stream, [NotNull] string target)
: base(target)
{
#region Sanity checks
if (stream == null)
{
throw new ArgumentNullException("stream");
}
#endregion
UnitsTotal = stream.Length;
try
{
_tarStream = new TarInputStream(stream);
}
catch (SharpZipBaseException ex)
{
// Wrap exception since only certain exception types are allowed
throw new IOException(Resources.ArchiveInvalid, ex);
}
}
/// <summary>
/// 获取.tar.gz文件列表
/// </summary>
/// <param name="file">.tar.gz文件</param>
/// <param name="encod">编码</param>
/// <returns></returns>
public static List <string> GetTarGZipFiles(string file, Encoding encod)
{
var list = new List <string>();
if (File.Exists(file))
{
using (var fs = File.OpenRead(file))
using (var gz = new GZipInputStream(fs))
using (var tar = new TarInputStream(gz, encod))
{
TarEntry entry;
while (null != (entry = tar.GetNextEntry()))
{
if (string.Empty != entry.Name)
{
list.Add(entry.Name);
}
}
}
}
return(list);
}
private static string tarStreamString(TarInputStream stream, TarEntry entry)
{
// Read each file into a buffer.
int buffer_size;
try
{
buffer_size = Convert.ToInt32(entry.Size);
}
catch (OverflowException)
{
log.ErrorFormat("Error processing {0}: Metadata size too large.", entry.Name);
return(null);
}
byte[] buffer = new byte[buffer_size];
stream.Read(buffer, 0, buffer_size);
// Convert the buffer data to a string.
return(Encoding.ASCII.GetString(buffer));
}
/// <summary>
/// Gets the data for the given CoinAPI tar entry
/// </summary>
/// <param name="tar">The tar input stream</param>
/// <param name="entry">The tar entry</param>
/// <returns>A new instance of type <see cref="CoinApiEntryData"/></returns>
public CoinApiEntryData GetCoinApiEntryData(TarInputStream tar, TarEntry entry)
{
var gzipFileName = entry.Name;
Log.Trace($"CoinApiDataReader.ProcessTarEntry(): Processing entry: {gzipFileName}");
// datatype-exchange-date-symbol/trades/COINBASE/2019/05/07/27781-COINBASE_SPOT_LTC_BTC.csv.gz
var parts = gzipFileName.Split('/');
if (parts.Length != 7)
{
throw new Exception($"CoinApiDataReader.ProcessTarEntry(): Unexpected entry path in tar file: {gzipFileName}");
}
var tickType = parts[1] == "trades" ? TickType.Trade : TickType.Quote;
var market = parts[2] == "COINBASE" ? Market.GDAX : parts[2].ToLower();
var year = Convert.ToInt32(parts[3]);
var month = Convert.ToInt32(parts[4]);
var day = Convert.ToInt32(parts[5]);
var date = new DateTime(year, month, day);
var nameParts = Path.GetFileNameWithoutExtension(parts[6].Substring(0, parts[6].IndexOf('.'))).Split('_');
if (nameParts.Length != 4)
{
throw new Exception($"CoinApiDataReader.ProcessTarEntry(): Unexpected entry name in tar file: {gzipFileName}");
}
var ticker = nameParts[2] + nameParts[3];
var symbol = Symbol.Create(ticker, SecurityType.Crypto, market);
return(new CoinApiEntryData
{
Name = gzipFileName,
Symbol = symbol,
TickType = tickType,
Date = date
});
}
private static void ExtractEntry(TarInputStream tarInputStream, TarEntry entry, string path)
{
Console.WriteLine($"Extracting '{Path.GetFileName(entry.Name)}' into '{path}'...");
EnsureDirExists(Path.GetDirectoryName(path) !);
FileStream?fs = null;
try
{
fs = new FileStream(path, System.IO.FileMode.Create, FileAccess.Write, FileShare.None);
tarInputStream.CopyEntryContents(fs);
}
catch (Exception ex)
{
Console.WriteLine("An exception occurred while trying to extract a file");
Console.WriteLine(ex);
}
finally
{
fs?.Dispose();
}
}
public async Task TestBuild_timestampDefaultAsync()
{
SystemPath file = CreateFile(temporaryFolder.GetRoot().ToPath(), "fileA", "some content", 54321);
IBlob blob =
new ReproducibleLayerBuilder(
ImmutableArray.Create(DefaultLayerEntry(file, AbsoluteUnixPath.Get("/fileA"))))
.Build();
SystemPath tarFile = temporaryFolder.NewFile().ToPath();
using (Stream @out = new BufferedStream(Files.NewOutputStream(tarFile)))
{
await blob.WriteToAsync(@out).ConfigureAwait(false);
}
// Reads the file back.
using (TarInputStream @in = new TarInputStream(Files.NewInputStream(tarFile)))
{
Assert.AreEqual(
(Instant.FromUnixTimeSeconds(0) + Duration.FromSeconds(1)).ToDateTimeUtc(), @in.GetNextEntry().TarHeader.ModTime);
}
}
public static IDictionary <string, byte[]> UntarGz(Stream stream)
{
var result = new Dictionary <string, byte[]>();
using (GZipInputStream gzStream = new GZipInputStream(stream))
using (TarInputStream tarStream = new TarInputStream(gzStream))
{
TarEntry entry;
entry = tarStream.GetNextEntry();
while (entry != null)
{
if (entry.Size != 0)
{
byte[] bytes = new byte[entry.Size];
StreamUtils.ReadFully(tarStream, bytes);
result[entry.Name] = bytes;
}
entry = tarStream.GetNextEntry();
}
}
return(result);
}
private ArchiveEntry GetNextEntryTar()
{
TarInputStream tar_stream = (TarInputStream)archive_stream;
TarEntry tar_entry;
do
{
tar_entry = tar_stream.GetNextEntry();
} while (tar_entry != null && tar_entry.IsDirectory);
// End of the entries;
if (tar_entry == null)
{
return(null);
}
ArchiveEntry entry = new ArchiveEntry();
entry.Name = tar_entry.Name;
entry.Modified = tar_entry.ModTime;
entry.Size = tar_entry.Size;
// Only index smaller subfiles, to avoid filling /tmp
if (entry.Size > MAX_SINGLE_FILE)
{
Log.Debug("Skipping over large file {0} in {1}", entry.Name, Indexable.DisplayUri.ToString());
return(entry);
}
entry.TempFile = StoreStreamInTempFile(archive_stream, GetExtension(entry.Name), entry.Modified);
if (entry.TempFile != null)
{
entry.MimeType = XdgMime.GetMimeType(entry.TempFile);
}
return(entry);
}
private static void JustUntar(string sourceFile, string dstDir)
{
TarArchive archive = null;
try
{
FileStream inStream = File.OpenRead(sourceFile);
TarInputStream tarIn = new TarInputStream(inStream);
archive = TarArchive.CreateInputTarArchive(tarIn);
if (!string.IsNullOrEmpty(dstDir))
{
archive.ExtractContents(dstDir);
}
else
{
archive.ExtractContents("./");
}
archive.Close();
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Error extracting {0} ", sourceFile));
Console.WriteLine(string.Format("Exception message {0}", ex.Message));
Usage();
}
finally
{
if (archive != null)
{
archive.Close();
}
}
}
private static void ExtractEntry(TarInputStream tarInputStream, TarEntry entry, string path)
{
Console.WriteLine($"Extracting '{Path.GetFileName(entry.Name.Replace('/', Path.DirectorySeparatorChar))}' into '{path}'...");
#pragma warning disable SA1009 // Closing parenthesis should be spaced correctly
EnsureDirExists(Path.GetDirectoryName(path) !);
#pragma warning restore SA1009 // Closing parenthesis should be spaced correctly
FileStream?fs = null;
try
{
fs = new FileStream(path, System.IO.FileMode.Create, FileAccess.Write, FileShare.None);
tarInputStream.CopyEntryContents(fs);
}
catch (Exception ex)
{
Console.WriteLine("An exception occurred while trying to extract a file");
Console.WriteLine(ex);
}
finally
{
fs?.Dispose();
}
}
/// <summary>
/// Helper if no other way to get file count for progress meter
/// </summary>
/// <param name="filename"></param>
/// <returns></returns>
int GetTarGzFileCount(string filename)
{
int count = 0;
using (var fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read))
using (var gzipStream = new GZipInputStream(fileStream))
using (var memoryStream = new MemoryStream())
{
var tarStream = new TarInputStream(gzipStream);
TarEntry tarEntry;
while ((tarEntry = tarStream.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
count++;
}
}
Thread.Sleep(300);//give time for OS to close before full processing
return(count);
}
public void Checksum()
{
var ms = new MemoryStream();
using (var tarOut = new TarOutputStream(ms, nameEncoding: null))
{
var entry = TarEntry.CreateTarEntry("TestEntry");
entry.TarHeader.Mode = 12345;
tarOut.PutNextEntry(entry);
}
var ms2 = new MemoryStream();
ms2.Write(ms.GetBuffer(), 0, ms.GetBuffer().Length);
ms2.Seek(0, SeekOrigin.Begin);
TarEntry nextEntry;
using (var tarIn = new TarInputStream(ms2, nameEncoding: null))
{
nextEntry = tarIn.GetNextEntry();
Assert.IsTrue(nextEntry.TarHeader.IsChecksumValid, "Checksum should be valid");
}
var ms3 = new MemoryStream();
ms3.Write(ms.GetBuffer(), 0, ms.GetBuffer().Length);
ms3.Seek(0, SeekOrigin.Begin);
ms3.Write(new byte[] { 34 }, 0, 1);
ms3.Seek(0, SeekOrigin.Begin);
using (var tarIn = new TarInputStream(ms3, nameEncoding: null))
{
Assert.Throws <TarException>(() => tarIn.GetNextEntry(), "Checksum should be invalid");
}
}
/// <summary>
/// Adds a layer around a stream that isolates the <code>data.tar.gz</code> file from a TAR stream.
/// </summary>
/// <param name="stream">The TAR stream.</param>
/// <returns>A stream representing the <code>data.tar.gz</code> data.</returns>
/// <exception cref="IOException">The compressed stream contains invalid data.</exception>
private static Stream GetPartialStream(Stream stream)
{
try
{
var tar = new TarInputStream(stream);
while (true)
{
var entry = tar.GetNextEntry();
if (entry == null)
{
throw new IOException(Resources.RubyGemInvalid);
}
if (entry.Name == "data.tar.gz")
{
return(tar);
}
}
}
#region Error handling
catch (SharpZipBaseException ex)
{
// Wrap exception since only certain exception types are allowed
throw new IOException(Resources.ArchiveInvalid, ex);
}
catch (InvalidDataException ex)
{
// Wrap exception since only certain exception types are allowed
throw new IOException(Resources.ArchiveInvalid, ex);
}
catch (ArgumentOutOfRangeException ex)
{
// Wrap exception since only certain exception types are allowed
throw new IOException(Resources.ArchiveInvalid, ex);
}
#endregion
}
public static long ReadFileOrDirectoryFromStream(Stream stream, string pathToWrite,
bool overwrite = true, bool closeStreamWhenComplete = true)
{
var tar = new TarInputStream(stream);
tar.IsStreamOwner = closeStreamWhenComplete;
long bytesRead = 0;
// we can't use the simple ExtractContents because we need to be overwrite aware,
// so we iterate instead
TarEntry entry;
while ((entry = tar.GetNextEntry()) != null)
{
var extractPath = Path.Combine(pathToWrite, entry.Name);
if (entry.IsDirectory)
{
// we don't have to worry about writing over directories
Directory.CreateDirectory(extractPath);
}
else
{
// if overwrite is on, use Create FileMode. If not, use CreateNew, which will throw
// an IO error if the file already exists.
using (var fs = new FileStream(extractPath, overwrite ? FileMode.Create : FileMode.CreateNew))
{
tar.CopyEntryContents(fs);
bytesRead += entry.Size;
}
}
}
tar.Close();
return(bytesRead);
}
protected override void _Rollback()
{
try
{
if (_Files.Count > 0)
{
using (FileStream fs = File.Open(OutputFile, FileMode.Open, FileAccess.Read, FileShare.None))
{
using (TarInputStream tStream = new TarInputStream(fs))
{
tStream.IsStreamOwner = false;
TarEntry e = null;
while ((e = tStream.GetNextEntry()) != null)
{
if (!e.IsDirectory)
{
try
{
string file = _Files[e.Name];
if (!File.Exists(file))
{
if (!Directory.Exists(STEM.Sys.IO.Path.GetDirectoryName(file)))
{
Directory.CreateDirectory(STEM.Sys.IO.Path.GetDirectoryName(file));
}
using (FileStream s = File.Open(file, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None))
{
tStream.CopyEntryContents(s);
}
File.SetLastWriteTimeUtc(file, e.ModTime);
}
}
catch { }
}
else
{
try
{
if (!Directory.Exists(Path.Combine(SourcePath, e.Name)))
{
Directory.CreateDirectory(Path.Combine(SourcePath, e.Name));
}
}
catch { }
}
}
}
}
if (File.Exists(OutputFile))
{
File.Delete(OutputFile);
}
}
}
catch (Exception ex)
{
AppendToMessage(ex.ToString());
Exceptions.Add(ex);
}
}
public string FetchSnow(DateTime date, bool masked)
{
string url = makeURL(date, masked);
string localFile = url.Substring(url.LastIndexOf("/") + 1);
FtpWebRequest ftpRequest = (FtpWebRequest)FtpWebRequest.Create(url);
FtpWebResponse ftpResponse = null;
try
{
ftpResponse = (FtpWebResponse)ftpRequest.GetResponse();
}
catch
{
throw new WebException("Cannot download file: " + url);
}
Stream ftpStream = null;
int bufferSize = 2048;
ftpStream = ftpResponse.GetResponseStream();
/* Open a File Stream to Write the Downloaded File */
FileStream localFileStream = new FileStream(localFile, FileMode.Create);
/* Buffer for the Downloaded Data */
byte[] byteBuffer = new byte[bufferSize];
int bytesRead = ftpStream.Read(byteBuffer, 0, bufferSize);
/* Download the File by Writing the Buffered Data Until the Transfer is Complete */
try
{
while (bytesRead > 0)
{
localFileStream.Write(byteBuffer, 0, bytesRead);
bytesRead = ftpStream.Read(byteBuffer, 0, bufferSize);
}
}
catch (Exception ex) { Console.WriteLine(ex.ToString()); }
/* Resource Cleanup */
localFileStream.Close();
ftpStream.Close();
ftpResponse.Close();
ftpRequest = null;
string datFileName = null;
string hdrFileName = null;
//now untar the required files
//the mask character indicates whether the data is whole area or u.s
//only.
string mask = "zz";
if (masked)
{
mask = "us";
}
string sweStart = String.Format("{0}_ssmv11034tS__T0001TTNATS{1}{2}", mask, date.Year.ToString(), date.Month.ToString("00"));
using (FileStream fsIn = new FileStream(localFile, FileMode.Open, FileAccess.Read))
{
TarInputStream tarIn = new TarInputStream(fsIn);
TarEntry tarEntry;
while ((tarEntry = tarIn.GetNextEntry()) != null)
{
string tarEntryName = tarEntry.Name;
if (tarEntryName.Contains(sweStart))
{
FileStream outStr = new FileStream(tarEntryName, FileMode.Create);
tarIn.CopyEntryContents(outStr);
outStr.Close();
string unzippedFile = ExtractGZip(tarEntryName);
if (unzippedFile.ToLower().EndsWith("dat"))
{
datFileName = unzippedFile;
}
if (unzippedFile.ToLower().EndsWith("hdr"))
{
hdrFileName = unzippedFile;
}
}
}
}
return(datFileName);
}
