/// <nodoc />
public void SaveFrontEndSnapshot(IWorkspaceBindingSnapshot snapshot)
{
try
{
ExceptionUtilities.HandleRecoverableIOException(
() => DoSaveFrontEndSnapshot(),
e => throw new BuildXLException(string.Empty, e));
}
catch (BuildXLException e)
{
m_logger.SaveFrontEndSnapshotError(m_loggingContext, m_cacheFileName, e.InnerException.ToString());
}
void DoSaveFrontEndSnapshot()
{
var sw = Stopwatch.StartNew();
using (var file = FileUtilities.CreateFileStream(m_cacheFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete))
{
var writer = new BuildXLWriter(debug: false, stream: file, leaveOpen: true, logStats: false);
writer.WriteCompact(Version);
FrontEndSnapshotSerializer.SerializeWorkspaceBindingSnapshot(snapshot, writer, m_pathTable);
}
m_frontEndStatistics.FrontEndSnapshotSavingDuration = sw.Elapsed;
m_logger.SaveFrontEndSnapshot(m_loggingContext, m_cacheFileName, (int)sw.ElapsedMilliseconds);
}
}
/// <summary>
/// Reads a file containing versioning information from the provided store directory.
/// </summary>
private static Possible <int> ReadStoreVersion(string storeDirectory)
{
var versionFile = GetVersionFile(storeDirectory);
if (!FileUtilities.FileExistsNoFollow(versionFile))
{
return(VersionConstants.UnversionedStore);
}
using (var stream = FileUtilities.CreateFileStream(
versionFile,
FileMode.Open,
FileAccess.Read,
FileShare.Delete))
{
try
{
// Check that the accessor version is valid.
s_fileEnvelope.ReadHeader(stream);
// Check that the current store version must match the persisted store's version.
using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: false))
{
// Represents persisted store version
return(reader.ReadInt32());
}
}
catch (Exception e)
{
return(new Failure <string>($"Error reading existing version file: {e.ToStringDemystified()}"));
}
}
}
public async Task ErrorHandling()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var storage = (ISandboxedProcessFileStorage)tempFiles;
var fileName = storage.GetFileName(SandboxedProcessFile.StandardOutput);
SandboxedProcessOutput output;
using (var fileStream = FileUtilities.CreateFileStream(fileName, FileMode.CreateNew, FileAccess.Write, FileShare.None, allowExcludeFileShareDelete: true))
{
var content = new string('S', 100);
var outputBuilder =
new SandboxedProcessOutputBuilder(Encoding.UTF8, content.Length / 2, tempFiles, SandboxedProcessFile.StandardOutput, null);
// NOTE: this only holds on Windows
// The specified content plus a NewLine will exceed the max memory length.
// Thus, the output builder will try to write the content to a file.
// However, the file is not writable (this runs in a using clause that already opened the file).
// Thus, this will internally fail, but not yet throw an exception.
outputBuilder.AppendLine(content);
output = outputBuilder.Freeze();
XAssert.IsTrue(output.HasException);
}
await Assert.ThrowsAsync <BuildXLException>(() => output.ReadValueAsync());
await Assert.ThrowsAsync <BuildXLException>(() => output.SaveAsync());
Assert.Throws <BuildXLException>(() => output.CreateReader());
}
}
/// <summary>
/// Saves a pip cost hash table to a file.
/// </summary>
/// <exception cref="BuildXLException">
/// Thrown if a recoverable error occurs while operating on the file.
/// </exception>
public void Save(string fileName)
{
Contract.Requires(fileName != null);
using (FileStream fileStream = FileUtilities.CreateFileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.Delete))
{
Save(fileStream);
}
}
public WorkspaceBindingSnapshot TryLoadFrontEndSnapshot(int expectedSpecCount)
{
try
{
return(ExceptionUtilities.HandleRecoverableIOException(
() => DoLoadFrontEndSnapshot(),
e => throw new BuildXLException(string.Empty, e)));
}
catch (BuildXLException e)
{
// Recoverable exceptions should not break BuildXL.
m_logger.FailToReuseFrontEndSnapshot(m_loggingContext, I($"IO exception occurred: {e.InnerException}"));
return(null);
}
WorkspaceBindingSnapshot DoLoadFrontEndSnapshot()
{
if (!File.Exists(m_cacheFileName))
{
// Can't reuse the snapshot because the file does not exist.
m_logger.FailToReuseFrontEndSnapshot(m_loggingContext, I($"File '{m_cacheFileName}' does not exist"));
return(null);
}
var sw = Stopwatch.StartNew();
using (var file = FileUtilities.CreateFileStream(m_cacheFileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Delete))
{
var reader = new BuildXLReader(debug: false, stream: file, leaveOpen: true);
var version = reader.ReadInt32Compact();
if (version != Version)
{
// Version mismatch. Can't reuse the file.
m_logger.FailToReuseFrontEndSnapshot(
m_loggingContext,
I($"Cache version '{version}' does not match an expected version '{Version}'"));
return(null);
}
var specCount = reader.ReadInt32Compact();
if (expectedSpecCount != specCount)
{
// Can't use the cache, because it has different amount of specs in there.
m_logger.FailToReuseFrontEndSnapshot(
m_loggingContext,
I($"Cache contains the data for '{specCount}' specs but current execution requires '{expectedSpecCount}' specs"));
return(null);
}
var specs = FrontEndSnapshotSerializer.DeserializeSpecStates(reader, m_pathTable, specCount);
m_frontEndStatistics.FrontEndSnapshotLoadingDuration = sw.Elapsed;
m_logger.LoadFrontEndSnapshot(m_loggingContext, specs.Length, (int)sw.ElapsedMilliseconds);
return(new WorkspaceBindingSnapshot(specs, m_pathTable));
}
}
}
private void Save(FileEnvelopeId atomicSaveToken, string path)
{
Contract.Requires(!string.IsNullOrWhiteSpace(path));
FileUtilities.DeleteFile(path);
FileUtilities.CreateDirectory(Path.GetDirectoryName(path));
using (var stream = FileUtilities.CreateFileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.Delete,
// Do not write the file with SequentialScan since it will be reread in the subsequent build
FileOptions.None))
{
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
using (var pm = BuildXL.Tracing.PerformanceMeasurement.StartWithoutStatistic(
m_loggingContext,
loggingContext => Logger.Log.StartSavingChangeTracker(loggingContext, path),
loggingContext => Logger.Log.EndSavingChangeTracker(loggingContext)))
{
Stopwatch sw = Stopwatch.StartNew();
FileEnvelope.WriteHeader(stream, atomicSaveToken);
using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false))
{
if (IsDisabledOrNullTrackingSet)
{
writer.Write(true);
}
else
{
writer.Write(false);
writer.Write(m_buildEngineFingerprint, (w, s) => w.Write(s));
m_changeTrackingSet.Save(writer);
}
}
FileEnvelope.FixUpHeader(stream, atomicSaveToken);
Logger.Log.SavingChangeTracker(
pm.LoggingContext,
path,
atomicSaveToken.ToString(),
m_changeTrackingSet == null ? "Null" : TrackingState.ToString(),
m_changeTrackingSet == null ? 0 : m_changeTrackingSet.TrackedVolumes.Count(),
sw.ElapsedMilliseconds);
}
},
ex => { throw new BuildXLException("Failed to save file change tracker", ex); });
}
}
private async Task <ContentHash> GetContentHashAsync(string path, HashType hashType)
{
using var fs = FileUtilities.CreateFileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Delete | FileShare.Read,
FileOptions.SequentialScan);
ContentHashingUtilities.SetDefaultHashType(hashType);
return(await ContentHashingUtilities.HashContentStreamAsync(fs, hashType));
}
private TextReader CreateFileReader()
{
// This FileStream is not asynchronous due to an intermittant crash we see on some machines when using
// an asynchronous stream here
FileStream stream = FileUtilities.CreateFileStream(
m_fileName,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete);
return(new StreamReader(stream));
}
/// <summary>
/// Opens the stream at the given path with read access
/// </summary>
public virtual Disposable <Stream> OpenReadStream(string path)
{
return(new Disposable <Stream>(
FileUtilities.CreateFileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete,
// Ok to evict the file from standby since the file will be overwritten and never reread from disk after this point.
FileOptions.SequentialScan),
null));
}
/// <summary>
/// Opens a pip cost hash table from a file.
/// </summary>
/// <exception cref="BuildXLException">
/// Thrown if a recoverable error occurs while operating on the file.
/// </exception>
public static PipRuntimeTimeTable Load(string fileName)
{
Contract.Requires(fileName != null);
using (
FileStream fileStream = FileUtilities.CreateFileStream(
fileName,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete))
{
return Load(fileStream);
}
}
/// <summary>
/// Opens a pip cost hash table from a file.
/// </summary>
/// <exception cref="BuildXLException">
/// Thrown if a recoverable error occurs while operating on the file.
/// </exception>
public static HistoricPerfDataTable Load(LoggingContext loggingContext, string fileName)
{
Contract.Requires(fileName != null);
using (
FileStream fileStream = FileUtilities.CreateFileStream(
fileName,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete))
{
return(Load(loggingContext, fileStream));
}
}
private async Task <ContentHash> GetContentHashAsync(AbsolutePath path, HashType hashType = HashType.Unknown)
{
m_frontEndHost.Engine.RecordFrontEndFile(path, Name);
// We don't call GetFileContentHashAsync() to get the existing hash, since it register the file.
// This has been done in RecordFrontEndFile with the default hasher, re-register it with the specified hasher will cause error.
using (
var fs = FileUtilities.CreateFileStream(
path.ToString(m_context.PathTable),
FileMode.Open,
FileAccess.Read,
FileShare.Delete | FileShare.Read,
FileOptions.SequentialScan))
{
return(await ContentHashingUtilities.HashContentStreamAsync(fs, hashType));
}
}
private VersionedFileIdentityAndContentInfo RecordContentHash(FileContentTable table, AbsolutePath path, ContentHash hash, bool strict = false)
{
Contract.Requires(table != null);
using (
FileStream fs = FileUtilities.CreateFileStream(
path.ToString(m_pathTable),
FileMode.Open,
strict ? FileAccess.ReadWrite : FileAccess.Read,
FileShare.Read | FileShare.Delete,
FileOptions.Asynchronous, true))
{
var info = table.RecordContentHash(fs, hash, strict: strict);
XAssert.AreEqual(hash, info.FileContentInfo.Hash);
XAssert.IsTrue(info.Identity.Kind == VersionedFileIdentity.IdentityKind.StrongUsn || info.Identity.Kind == VersionedFileIdentity.IdentityKind.Anonymous);
return(info);
}
}
public static AppDeployment ReadDeploymentManifest(string baseDir, string fileName, bool skipManifestCheckTestHook = false)
{
List <string> fileNames = new List <string>();
using (FileStream deploymentManifest = FileUtilities.CreateFileStream(
Path.Combine(baseDir, fileName),
FileMode.Open, FileAccess.Read, FileShare.Delete))
{
using (StreamReader reader = new StreamReader(deploymentManifest))
{
while (!reader.EndOfStream)
{
fileNames.Add(reader.ReadLine());
}
}
}
return(new AppDeployment(baseDir, fileNames, skipManifestCheckTestHook));
}
/// <inheritdoc/>
public override async Task <ContentHash> GetFileContentHashAsync(string path, bool trackFile = true, HashType hashType = HashType.Unknown)
{
// If the tracker knows about this file, then we already have the hash
ContentHash hash;
if (trackFile)
{
if (m_inputTracker.TryGetHashForUnchangedFile(path, out hash))
{
return(hash);
}
}
using (
var fs = FileUtilities.CreateFileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Delete | FileShare.Read,
FileOptions.SequentialScan))
{
// Otherwise, check if the file content table already knows about this
var fileContentTable = m_getFileContentTable();
VersionedFileIdentityAndContentInfo?maybeKnownIdentityAndHash = fileContentTable.TryGetKnownContentHash(fs);
if (maybeKnownIdentityAndHash?.FileContentInfo.MatchesHashType(hashType) ?? false)
{
return(maybeKnownIdentityAndHash.Value.FileContentInfo.Hash);
}
// Finally, if all the above failed, compute the hash and record it for next time
hash = await ContentHashingUtilities.HashContentStreamAsync(fs, hashType);
maybeKnownIdentityAndHash = fileContentTable.RecordContentHash(fs, hash);
m_specCache?.AddFile(fs, maybeKnownIdentityAndHash.Value.FileContentInfo.Hash, path);
m_inputTracker.RegisterFileAccess(fs.SafeFileHandle, path, maybeKnownIdentityAndHash.Value);
return(hash);
}
}
/// <inheritdoc />
public override bool TryGetFrontEndFile(AbsolutePath path, string frontEnd, out Stream stream)
{
var physicalPath = path.ToString(m_pathTable);
try
{
stream = FileUtilities.CreateFileStream(
physicalPath,
FileMode.Open,
FileAccess.Read,
FileShare.Delete | FileShare.Read,
FileOptions.SequentialScan);
return(true);
}
catch (BuildXLException)
{
stream = null;
return(false);
}
}
public void StrictModeFlushesDirtyMemoryMappedPages()
{
var table = FileContentTable.CreateNew();
string testFileAExpandedPath = m_testFileA.ToString(m_pathTable);
// Writing a file via a memory mapping leaves 'dirty' pages in cache that get lazily flushed to FS.
// We want to verify that passing strict: true flushes those pages, so that the recorded USN is stable (not invalidated due to lazy flushes).
using (FileStream file = FileUtilities.CreateFileStream(testFileAExpandedPath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete | FileShare.Read))
{
file.SetLength(1024 * 1024);
using (var memoryMappedFile = MemoryMappedFile.CreateFromFile(
file,
mapName: null,
capacity: 0,
access: MemoryMappedFileAccess.ReadWrite,
#if !DISABLE_FEATURE_MEMORYMAP_SECURITY
memoryMappedFileSecurity: null,
#endif
inheritability: HandleInheritability.None,
leaveOpen: false))
{
using (var accessor = memoryMappedFile.CreateViewAccessor())
{
for (int i = 0; i < 1024 * 1024; i += 4)
{
accessor.Write(i, (int)0xAB);
}
}
}
RecordContentHash(table, m_testFileA, s_hashA, strict: true);
}
for (int i = 0; i < 10; i++)
{
Thread.Sleep(10);
ExpectHashKnown(table, m_testFileA, s_hashA);
}
}
/// <summary>
/// Writes a file containing versioning information to the provided store directory.
/// </summary>
private static void WriteVersionFile(string storeDirectory, int storeVersion)
{
var versionFile = GetVersionFile(storeDirectory);
using (var stream = FileUtilities.CreateFileStream(
versionFile,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.Delete))
{
// We don't have anything in particular to correlate this file to,
// so we are simply creating a unique correlation id that is used as part
// of the header consistency check.
var correlationId = FileEnvelopeId.Create();
s_fileEnvelope.WriteHeader(stream, correlationId);
using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false))
{
writer.Write(storeVersion);
}
s_fileEnvelope.FixUpHeader(stream, correlationId);
}
}
/// <summary>
/// Extract files to disk
/// </summary>
/// <remarks>
/// At the point of authoring (Jan 2019) the BCL does not implement tar-files or bz2.
/// https://github.com/dotnet/corefx/issues/3253 has been discussed since Sept 2015
/// Therefore we rely here on 3rd party library: https://github.com/icsharpcode/SharpZipLib
/// </remarks>
private bool TryExtractToDisk(DownloadData downloadData)
{
var archive = downloadData.DownloadedFilePath.ToString(m_context.PathTable);
var target = downloadData.ContentsFolder.Path.ToString(m_context.PathTable);
try
{
FileUtilities.DeleteDirectoryContents(target, false);
FileUtilities.CreateDirectory(target);
}
catch (BuildXLException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
switch (downloadData.Settings.ArchiveType)
{
case DownloadArchiveType.Zip:
try
{
new FastZip().ExtractZip(archive, target, null);
}
catch (ZipException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Gzip:
try
{
var targetFile = Path.Combine(
target,
downloadData.DownloadedFilePath.GetName(m_context.PathTable).RemoveExtension(m_context.StringTable)
.ToString(m_context.StringTable));
using (var reader = m_context.FileSystem.OpenText(downloadData.DownloadedFilePath))
using (var gzipStream = new GZipInputStream(reader.BaseStream))
using (var output = FileUtilities.CreateFileStream(targetFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
byte[] buffer = new byte[4096];
StreamUtils.Copy(gzipStream, output, buffer);
}
}
catch (GZipException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Tar:
try
{
using (var reader = m_context.FileSystem.OpenText(downloadData.DownloadedFilePath))
using (var tar = TarArchive.CreateInputTarArchive(reader.BaseStream))
{
tar.ExtractContents(target);
}
}
catch (TarException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Tgz:
try
{
using (var reader = m_context.FileSystem.OpenText(downloadData.DownloadedFilePath))
using (var gzipStream = new GZipInputStream(reader.BaseStream))
using (var tar = TarArchive.CreateInputTarArchive(gzipStream))
{
tar.ExtractContents(target);
}
}
catch (GZipException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
catch (TarException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
break;
default:
throw Contract.AssertFailure("Unexpected archive type");
}
try
{
if (!FileUtilities.DirectoryExistsNoFollow(target))
{
m_logger.ErrorNothingExtracted(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target);
return(false);
}
}
catch (BuildXLException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
return(true);
}
private async Task <Possible <IReadOnlyList <ContentHash> > > TryGetBuildManifestHashFromLocalFileAsync(string fullFilePath, ContentHash hash, IList <HashType> requestedTypes, int retryAttempt = 0)
{
Contract.Assert(requestedTypes.Count > 0, "Must request at least one hash type");
var result = new List <ContentHash>();
if (retryAttempt >= GetBuildManifestHashFromLocalFileRetryLimit)
{
string message = $"GetBuildManifestHashFromLocalFileRetryLimit exceeded at path '{fullFilePath}'";
Tracing.Logger.Log.ApiServerForwardedIpcServerMessage(m_loggingContext, "BuildManifest", message);
return(new Failure <string>(message));
}
if (retryAttempt > 0)
{
await Task.Delay(TimeSpan.FromMilliseconds(Math.Pow(2, retryAttempt) * GetBuildManifestHashFromLocalFileRetryMultiplierMs));
}
if (!File.Exists(fullFilePath))
{
Tracing.Logger.Log.ApiServerForwardedIpcServerMessage(m_loggingContext, "BuildManifest", $"Local file not found at path '{fullFilePath}' while computing BuildManifest Hash. Trying other methods to obtain hash.");
return(new Failure <string>($"File doesn't exist: '{fullFilePath}'"));
}
var hashers = new HashingStream[requestedTypes.Count - 1];
HashingStream validationStream = null;
try
{
using var fs = FileUtilities.CreateFileStream(fullFilePath, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read, FileOptions.SequentialScan).AssertHasLength();
// If enabled, create a hashing stream for content validation. Using HashType.Unknown uses the default hashtype
validationStream = m_verifyFileContentOnBuildManifestHashComputation ? ContentHashingUtilities.GetContentHasher(HashType.Unknown).CreateReadHashingStream(fs) : null;
// Create a series of nested ReadHashingStream so we compute all the hashes in parallel
StreamWithLength outerStream = validationStream?.AssertHasLength() ?? fs;
for (var i = 0; i < hashers.Length; i++)
{
// Hashers has size (requestedTypes.Count - 1)
// requestedTypes[0] will be used to hash+consume the resulting outerStream (see below)
var hashType = requestedTypes[i + 1];
hashers[i] = ContentHashingUtilities.GetContentHasher(hashType).CreateReadHashingStream(outerStream);
outerStream = hashers[i].AssertHasLength();
}
// Hashing the outermost stream will cause all the nested hashers to also do their processing
var firstManifestHash = await ContentHashingUtilities.HashContentStreamAsync(outerStream, requestedTypes[0]);
result.Add(firstManifestHash);
for (int i = 0; i < hashers.Length; i++)
{
result.Add(hashers[i].GetContentHash());
}
if (m_verifyFileContentOnBuildManifestHashComputation)
{
var actualHash = validationStream.GetContentHash();
if (hash != actualHash)
{
return(new Failure <string>($"Unexpected file content during build manifest hash computation. Path: '{fullFilePath}', expected hash '{hash}', actual hash '{actualHash}'."));
}
}
}
catch (Exception ex) when(ex is BuildXLException || ex is IOException)
{
Tracing.Logger.Log.ApiServerForwardedIpcServerMessage(m_loggingContext, "BuildManifest",
$"Local file found at path '{fullFilePath}' but threw exception while computing BuildManifest Hash. Retry attempt {retryAttempt} out of {GetBuildManifestHashFromLocalFileRetryLimit}. Exception: {ex}");
return(await TryGetBuildManifestHashFromLocalFileAsync(fullFilePath, hash, requestedTypes, retryAttempt + 1));
}
finally
{
validationStream?.Dispose();
for (var i = 0; i < hashers.Length; i++)
{
hashers[i]?.Dispose();
}
}
return(result);
}
private async Task <SerializationResult> SerializeToFileInternalAsync(GraphCacheFile fileType, Action <BuildXLWriter> serializer, string overrideName)
{
// Unblock the caller
await Task.Yield();
FileUtilities.CreateDirectory(m_engineCacheLocation);
string fileName = overrideName ?? GetFileName(fileType);
string path = Path.Combine(m_engineCacheLocation, fileName);
SerializationResult serializationResult = new SerializationResult()
{
Success = false,
FileType = fileType,
FullPath = path,
};
var fileEnvelope = GetFileEnvelope(fileType);
Contract.Assume(m_correlationId is FileEnvelopeId, "EngineSerializer must be initialized with a valid correlation id");
var correlationId = (FileEnvelopeId)m_correlationId;
try
{
// We must delete the existing file in case it was hardlinked from the cache. Opening a filestream
// that truncates the existing file will fail if it is a hardlink.
FileUtilities.DeleteFile(path, tempDirectoryCleaner: m_tempDirectoryCleaner);
using (
FileStream fileStream = FileUtilities.CreateFileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.Delete,
// Do not write the file with SequentialScan since it will be reread in the subsequent build
FileOptions.None))
{
Stopwatch sw = Stopwatch.StartNew();
fileEnvelope.WriteHeader(fileStream, correlationId);
// Write whether the file is compressed or not.
fileStream.WriteByte(m_useCompression ? (byte)1 : (byte)0);
long uncompressedLength = 0;
if (m_useCompression)
{
using (var writer = new BuildXLWriter(m_debug, new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionLevel.Fastest, leaveOpen: true), bufferSize: 64 << 10)), false, false))
{
// TODO: We can improve performance significantly by parallelizing the compression.
// There's no setting to do that, but given you have the entire file content upfront in a memory stream,
// it shouldn't be particularly complicated to split the memory stream into reasonably sized chunks (say 100MB)
// and compress each of them into separate MemoryStream backed DeflateStreams in separate threads.
// Then just write those out to a file. Of course you'll need to write out the position of those streams
// into the header when you write out the actual file.
serializer(writer);
uncompressedLength = writer.BaseStream.Length;
}
}
else
{
using (var writer = new BuildXLWriter(m_debug, fileStream, leaveOpen: true, logStats: false))
{
serializer(writer);
uncompressedLength = writer.BaseStream.Length;
}
}
Interlocked.Add(ref m_bytesSavedDueToCompression, uncompressedLength - fileStream.Length);
fileEnvelope.FixUpHeader(fileStream, correlationId);
Tracing.Logger.Log.SerializedFile(LoggingContext, fileName, sw.ElapsedMilliseconds);
serializationResult.Success = true;
Interlocked.Add(ref m_bytesSerialized, fileStream.Position);
}
}
catch (BuildXLException ex)
{
Tracing.Logger.Log.FailedToSerializePipGraph(LoggingContext, ex.LogEventMessage);
}
catch (IOException ex)
{
Tracing.Logger.Log.FailedToSerializePipGraph(LoggingContext, ex.Message);
}
return(serializationResult);
}
private bool TryExtractToDisk(ExtractorArgs arguments)
{
var archive = arguments.PathToFileToExtract;
var target = arguments.ExtractDirectory;
try
{
FileUtilities.DeleteDirectoryContents(target, false);
FileUtilities.CreateDirectory(target);
}
catch (BuildXLException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
switch (arguments.ArchiveType)
{
case DownloadArchiveType.Zip:
try
{
new FastZip().ExtractZip(archive, target, null);
}
catch (ZipException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Gzip:
try
{
var targetFile = Path.Combine(target, Path.GetFileNameWithoutExtension(arguments.PathToFileToExtract));
using (var reader = new StreamReader(arguments.PathToFileToExtract))
using (var gzipStream = new GZipInputStream(reader.BaseStream))
using (var output = FileUtilities.CreateFileStream(targetFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
byte[] buffer = new byte[4096];
StreamUtils.Copy(gzipStream, output, buffer);
}
}
catch (GZipException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Tar:
try
{
using (var reader = new StreamReader(arguments.PathToFileToExtract))
using (var tar = TarArchive.CreateInputTarArchive(reader.BaseStream, nameEncoding: null))
{
tar.ExtractContents(target);
}
}
catch (TarException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Tgz:
try
{
using (var reader = new StreamReader(arguments.PathToFileToExtract))
using (var gzipStream = new GZipInputStream(reader.BaseStream))
using (var tar = TarArchive.CreateInputTarArchive(gzipStream, nameEncoding: null))
{
tar.ExtractContents(target);
}
}
catch (GZipException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
catch (TarException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.File:
Console.WriteLine("Specified download archive type is 'File'. Nothing to extract.");
return(true);
default:
throw Contract.AssertFailure($"Unexpected archive type '{arguments.ArchiveType}'");
}
try
{
if (!FileUtilities.DirectoryExistsNoFollow(target))
{
ErrorNothingExtracted(archive, target);
return(false);
}
}
catch (BuildXLException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
return(true);
}
