private static Run CreateRun(
IEnumerable <string> analysisTargets,
OptionallyEmittedData dataToInsert,
IEnumerable <string> invocationTokensToRedact,
IEnumerable <string> invocationPropertiesToLog,
string defaultFileEncoding = null)
{
var run = new Run
{
Invocations = new List <Invocation>(),
DefaultFileEncoding = defaultFileEncoding
};
if (analysisTargets != null)
{
run.Files = new Dictionary <string, FileData>();
foreach (string target in analysisTargets)
{
string fileDataKey = UriHelper.MakeValidUri(target);
var fileData = FileData.Create(
new Uri(target, UriKind.RelativeOrAbsolute),
dataToInsert);
run.Files[fileDataKey] = fileData;
}
}
var invocation = Invocation.Create(dataToInsert.Includes(OptionallyEmittedData.EnvironmentVariables), invocationPropertiesToLog);
// TODO we should actually redact across the complete log file context
// by a dedicated rewriting visitor or some other approach.
if (invocationTokensToRedact != null)
{
invocation.CommandLine = Redact(invocation.CommandLine, invocationTokensToRedact);
invocation.Machine = Redact(invocation.Machine, invocationTokensToRedact);
invocation.Account = Redact(invocation.Account, invocationTokensToRedact);
if (invocation.WorkingDirectory != null)
{
invocation.WorkingDirectory.Uri = Redact(invocation.WorkingDirectory.Uri, invocationTokensToRedact);
}
if (invocation.EnvironmentVariables != null)
{
string[] keys = invocation.EnvironmentVariables.Keys.ToArray();
foreach (string key in keys)
{
string value = invocation.EnvironmentVariables[key];
invocation.EnvironmentVariables[key] = Redact(value, invocationTokensToRedact);
}
}
}
run.Invocations.Add(invocation);
return(run);
}
private static void Validate(FileData fileData, OptionallyEmittedData dataToInsert)
{
if (dataToInsert.Includes(OptionallyEmittedData.TextFiles))
{
fileData.Contents.Should().NotBeNull();
}
else
{
fileData.Contents.Should().BeNull();
}
}
public override Run VisitRun(Run node)
{
if (node != null)
{
bool scrapeFileReferences = _dataToInsert.Includes(OptionallyEmittedData.Hashes) ||
_dataToInsert.Includes(OptionallyEmittedData.TextFiles) ||
_dataToInsert.Includes(OptionallyEmittedData.BinaryFiles);
if (scrapeFileReferences)
{
var visitor = new AddFileReferencesVisitor();
visitor.VisitRun(node);
}
if (node.Files != null)
{
var keys = node.Files.Keys.ToArray();
foreach (var key in keys)
{
var value = node.Files[key];
if (value != null)
{
node.Files[key] = VisitDictionaryValueNullChecked(key, value);
}
}
}
}
return(node);
}
public void FileData_PersistBinaryAndTextFileContents(
string fileExtension,
OptionallyEmittedData dataToInsert,
bool shouldBePersisted)
{
string filePath = Path.GetTempFileName() + fileExtension;
string fileContents = Guid.NewGuid().ToString();
Uri uri = new Uri(filePath);
try
{
File.WriteAllText(filePath, fileContents);
FileData fileData = FileData.Create(uri, dataToInsert);
fileData.FileLocation.Should().BeNull();
if (dataToInsert.Includes(OptionallyEmittedData.Hashes))
{
fileData.Hashes.Should().NotBeNull();
}
else
{
fileData.Hashes.Should().BeNull();
}
string encodedFileContents = Convert.ToBase64String(File.ReadAllBytes(filePath));
if (shouldBePersisted)
{
fileData.Contents.Binary.Should().Be(encodedFileContents);
fileData.Contents.Text.Should().BeNull();
}
else
{
fileData.Contents.Should().BeNull();
}
}
finally
{
if (File.Exists(filePath))
{
File.Delete(filePath);
}
}
}
public static FileData Create(
Uri uri,
OptionallyEmittedData dataToInsert = OptionallyEmittedData.None,
string mimeType = null,
Encoding encoding = null,
IFileSystem fileSystem = null)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
mimeType = mimeType ?? SarifWriters.MimeType.DetermineFromFileExtension(uri);
fileSystem = fileSystem ?? new FileSystem();
var fileData = new FileData()
{
MimeType = mimeType
};
// Attempt to persist file contents and/or compute file hash and persist
// this information to the log file. In the event that there is some issue
// accessing the file, for example, due to ACLs applied to a directory,
// we currently swallow these exceptions without populating any requested
// data or putting a notification in the log file that a problem
// occurred. Something to discuss moving forward.
try
{
if (!uri.IsAbsoluteUri || !uri.IsFile || !fileSystem.FileExists(uri.LocalPath))
{
return(fileData);
}
string filePath = uri.LocalPath;
if (dataToInsert.Includes(OptionallyEmittedData.BinaryFiles) &&
SarifWriters.MimeType.IsBinaryMimeType(mimeType))
{
fileData.Contents = GetEncodedFileContents(fileSystem, filePath, mimeType, encoding);
}
if (dataToInsert.Includes(OptionallyEmittedData.TextFiles) &&
SarifWriters.MimeType.IsTextualMimeType(mimeType))
{
fileData.Contents = GetEncodedFileContents(fileSystem, filePath, mimeType, encoding);
}
if (dataToInsert.Includes(OptionallyEmittedData.Hashes))
{
HashData hashes = HashUtilities.ComputeHashes(filePath);
fileData.Hashes = new List <Hash>
{
new Hash()
{
Value = hashes.MD5,
Algorithm = "md5",
},
new Hash()
{
Value = hashes.Sha1,
Algorithm = "sha-1",
},
new Hash()
{
Value = hashes.Sha256,
Algorithm = "sha-256",
},
};
}
}
catch (Exception e) when(e is IOException || e is UnauthorizedAccessException)
{
}
return(fileData);
}
public override Run VisitRun(Run node)
{
_run = node;
if (_originalUriBaseIds != null)
{
_run.OriginalUriBaseIds = _run.OriginalUriBaseIds ?? new Dictionary <string, Uri>();
foreach (string key in _originalUriBaseIds.Keys)
{
_run.OriginalUriBaseIds[key] = _originalUriBaseIds[key];
}
}
if (node == null)
{
return(null);
}
bool scrapeFileReferences = _dataToInsert.Includes(OptionallyEmittedData.Hashes) ||
_dataToInsert.Includes(OptionallyEmittedData.TextFiles) ||
_dataToInsert.Includes(OptionallyEmittedData.BinaryFiles);
if (scrapeFileReferences)
{
var visitor = new AddFileReferencesVisitor();
visitor.VisitRun(node);
}
Run visited = base.VisitRun(node);
return(visited);
}
public override Run VisitRun(Run node)
{
_run = node;
if (node == null)
{
return(null);
}
bool scrapeFileReferences = _dataToInsert.Includes(OptionallyEmittedData.Hashes) ||
_dataToInsert.Includes(OptionallyEmittedData.TextFiles) ||
_dataToInsert.Includes(OptionallyEmittedData.BinaryFiles);
if (scrapeFileReferences)
{
var visitor = new AddFileReferencesVisitor();
visitor.VisitRun(node);
}
if (node.Files != null)
{
// Note, we modify this collection as we enumerate it.
// Hence the need to convert to an array here. Otherwise,
// the standard collection enumerator will throw an
// exception after we touch the collection.
var keys = node.Files.Keys.ToArray();
foreach (string key in keys)
{
var value = node.Files[key];
if (value != null)
{
node.Files[key] = VisitDictionaryValueNullChecked(key, value);
}
}
}
return(base.VisitRun(node));
}