/// <summary>
/// On Windows we can use the .NET API to iterate through all the stores.
/// </summary>
public void ExecuteWindows()
{
foreach (StoreLocation storeLocation in (StoreLocation[])Enum.GetValues(typeof(StoreLocation)))
{
foreach (StoreName storeName in (StoreName[])Enum.GetValues(typeof(StoreName)))
{
try
{
using X509Store store = new X509Store(storeName, storeLocation);
store.Open(OpenFlags.ReadOnly);
foreach (X509Certificate2 certificate in store.Certificates)
{
var obj = new CertificateObject(
StoreLocation: storeLocation.ToString(),
StoreName: storeName.ToString(),
Certificate: new SerializableCertificate(certificate))
{
Pkcs7 = Convert.ToBase64String(certificate.Export(X509ContentType.Cert))
};
Results.Enqueue(obj);
}
store.Close();
}
catch (CryptographicException e)
{
Log.Debug(e, $"Error parsing a certificate in {storeLocation} {storeName}");
}
}
}
}
/// <summary>
/// On linux we check the central trusted root store (a folder), which has symlinks to actual cert locations scattered across the db
/// We list all the certificates and then create a new X509Certificate2 object for each by filename.
/// </summary>
public void ExecuteLinux()
{
try
{
if (ExternalCommandRunner.RunExternalCommand("ls", "/etc/ssl/certs -A", out string result, out string _) == 0)
{
foreach (var _line in result.Split('\n'))
{
Log.Debug("{0}", _line);
try
{
using X509Certificate2 certificate = new X509Certificate2("/etc/ssl/certs/" + _line);
var obj = new CertificateObject(
StoreLocation: StoreLocation.LocalMachine.ToString(),
StoreName: StoreName.Root.ToString(),
Certificate: new SerializableCertificate(certificate))
{
Pkcs7 = Convert.ToBase64String(certificate.Export(X509ContentType.Cert))
};
Results.Enqueue(obj);
}
catch (Exception e)
{
Log.Debug("{0} {1} Issue creating certificate based on /etc/ssl/certs/{2}", e.GetType().ToString(), e.Message, _line);
Log.Debug("{0}", e.StackTrace);
}
}
}
}
catch (Exception e)
{
Log.Error(e, "Failed to dump certificates from 'ls /etc/ssl/certs -A'.");
}
}
/// <summary>
/// On Windows we can use the .NET API to iterate through all the stores.
/// </summary>
public void ExecuteWindows()
{
foreach (StoreLocation storeLocation in (StoreLocation[])Enum.GetValues(typeof(StoreLocation)))
{
foreach (StoreName storeName in (StoreName[])Enum.GetValues(typeof(StoreName)))
{
try
{
X509Store store = new X509Store(storeName, storeLocation);
store.Open(OpenFlags.ReadOnly);
foreach (X509Certificate2 certificate in store.Certificates)
{
var obj = new CertificateObject()
{
StoreLocation = storeLocation.ToString(),
StoreName = storeName.ToString(),
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs7 = certificate.Export(X509ContentType.Cert).ToString()
};
DatabaseManager.Write(obj, this.RunId);
}
store.Close();
}
catch (CryptographicException e)
{
Log.Debug(e, $"Error parsing a certificate in {storeLocation} {storeName}");
}
}
}
}
public void TestInsertAndReadBack()
{
DatabaseManager.Setup("asa.sqlite", new DBSettings()
{
ShardingFactor = 1
});
var co = new CertificateObject("StoreLocation", "StoreName", new SerializableCertificate("Thumbprint", "Subject", "PublicKey", DateTime.Now.AddYears(1), DateTime.Now, "Issuer", "SerialNumber", "CertHashString", "Pkcs7"));
var runId = "TestRun";
var numberOfObjects = 100;
DatabaseManager.Write(co, runId);
while (DatabaseManager.HasElements)
{
Thread.Sleep(1);
}
Assert.IsTrue(DatabaseManager.GetResultsByRunid(runId).Any(x => x.ColObj is CertificateObject co && co.StoreLocation == "StoreLocation"));
for (int i = 0; i < numberOfObjects; i++)
{
DatabaseManager.Write(GetRandomObject(), runId);
}
while (DatabaseManager.HasElements)
{
Thread.Sleep(1);
}
Assert.IsTrue(DatabaseManager.GetResultsByRunid(runId).Count(x => x.ColObj.ResultType == RESULT_TYPE.FILE) == numberOfObjects);
}
/// <summary>
/// On Windows we can use the .NET API to iterate through all the stores.
/// </summary>
public void ExecuteWindows()
{
foreach (StoreLocation storeLocation in (StoreLocation[])Enum.GetValues(typeof(StoreLocation)))
{
foreach (StoreName storeName in (StoreName[])Enum.GetValues(typeof(StoreName)))
{
try
{
X509Store store = new X509Store(storeName, storeLocation);
store.Open(OpenFlags.ReadOnly);
foreach (X509Certificate2 certificate in store.Certificates)
{
var obj = new CertificateObject()
{
StoreLocation = storeLocation.ToString(),
StoreName = storeName.ToString(),
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs12 = certificate.HasPrivateKey ? "redacted" : certificate.Export(X509ContentType.Pkcs12).ToString()
};
DatabaseManager.Write(obj, this.runId);
}
store.Close();
}
catch (Exception e)
{
Logger.DebugException(e);
Telemetry.TrackTrace(Microsoft.ApplicationInsights.DataContracts.SeverityLevel.Error, e);
}
}
}
}
/// <summary>
/// On linux we check the central trusted root store (a folder), which has symlinks to actual cert locations scattered across the db
/// We list all the certificates and then create a new X509Certificate2 object for each by filename.
/// </summary>
public void ExecuteLinux()
{
try
{
var result = ExternalCommandRunner.RunExternalCommand("ls", new string[] { "/etc/ssl/certs", "-A" });
foreach (var _line in result.Split('\n'))
{
Log.Debug("{0}", _line);
try
{
using X509Certificate2 certificate = new X509Certificate2("/etc/ssl/certs/" + _line);
var obj = new CertificateObject(
StoreLocation: StoreLocation.LocalMachine.ToString(),
StoreName: StoreName.Root.ToString(),
Certificate: new SerializableCertificate(certificate),
Pkcs7: certificate.Export(X509ContentType.Cert).ToString());
DatabaseManager.Write(obj, RunId);
}
catch (Exception e)
{
Log.Debug("{0} {1} Issue creating certificate based on /etc/ssl/certs/{2}", e.GetType().ToString(), e.Message, _line);
Log.Debug("{0}", e.StackTrace);
}
}
}
catch (Exception e)
{
Log.Error(e, "Failed to dump certificates from 'ls /etc/ssl/certs -A'.");
}
}
/// <summary>
/// On Windows we can use the .NET API to iterate through all the stores.
/// </summary>
internal void ExecuteWindows(CancellationToken cancellationToken)
{
foreach (StoreLocation storeLocation in (StoreLocation[])Enum.GetValues(typeof(StoreLocation)))
{
foreach (StoreName storeName in (StoreName[])Enum.GetValues(typeof(StoreName)))
{
try
{
using X509Store store = new X509Store(storeName, storeLocation);
store.Open(OpenFlags.ReadOnly);
foreach (X509Certificate2 certificate in store.Certificates)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var obj = new CertificateObject(
StoreLocation: storeLocation.ToString(),
StoreName: storeName.ToString(),
Certificate: new SerializableCertificate(certificate));
HandleChange(obj);
}
store.Close();
}
catch (CryptographicException e)
{
Log.Debug(e, $"Error parsing a certificate in {storeLocation} {storeName}");
}
}
}
}
public void TestSerializeAndDeserializeCertificateObject()
{
var co = new CertificateObject("StoreLocation", "StoreName", new SerializableCertificate("Thumbprint", "Subject", "PublicKey", DateTime.Now.AddYears(1), DateTime.Now, "Issuer", "SerialNumber", "CertHashString", "Pkcs7"));
if (JsonUtils.Hydrate(JsonUtils.Dehydrate(co), RESULT_TYPE.CERTIFICATE) is CertificateObject co2)
{
Assert.IsTrue(co.RowKey.Equals(co2.RowKey));
Assert.IsTrue(co.Certificate.Thumbprint.Equals(co2.Certificate.Thumbprint));
}
else
{
Assert.Fail();
}
}
public void Write(StoreLocation storeLocation, StoreName storeName, X509Certificate2 obj)
{
try
{
recordCounter++;
var cmd = new SqliteCommand(SQL_INSERT, DatabaseManager.Connection, DatabaseManager.Transaction);
cmd.Parameters.AddWithValue("@run_id", runId);
cmd.Parameters.AddWithValue("@store_location", storeLocation.ToString());
cmd.Parameters.AddWithValue("@store_name", storeName.ToString());
cmd.Parameters.AddWithValue("@hash", obj.GetCertHashString());
cmd.Parameters.AddWithValue("@hash_plus_store", obj.GetCertHashString() + storeLocation.ToString() + storeName.ToString());
cmd.Parameters.AddWithValue("@cn", obj.Subject);
if (obj.HasPrivateKey)
{
cmd.Parameters.AddWithValue("@pkcs12", "redacted");
}
else
{
cmd.Parameters.AddWithValue("@pkcs12", obj.Export(X509ContentType.Pfx));
}
cmd.Parameters.AddWithValue("@row_key", CryptoHelpers.CreateHash(runId + recordCounter));
var cert = new CertificateObject()
{
StoreLocation = storeLocation.ToString(),
StoreName = storeName.ToString(),
CertificateHashString = obj.GetCertHashString(),
Subject = obj.Subject
};
cmd.Parameters.AddWithValue("@serialized", JsonConvert.SerializeObject(cert));
cmd.ExecuteNonQuery();
}
catch (NullReferenceException e)
{
Log.Warning(e.StackTrace);
}
catch (Microsoft.Data.Sqlite.SqliteException e)
{
Log.Warning(e.Message);
//This catches duplicate certificates
}
catch (Exception e)
{
Log.Warning(e.GetType().ToString());
Log.Warning(e.StackTrace);
}
}
private void ParseFile(string path)
{
Log.Verbose("Started parsing {0}", path);
FileSystemObject obj = FilePathToFileSystemObject(path);
if (obj != null)
{
HandleChange(obj);
// If we know how to handle this as an archive, and crawling archives is enabled
if (opts.CrawlArchives && MiniMagic.DetectFileType(path) != ArchiveFileType.UNKNOWN)
{
var opts = new ExtractorOptions()
{
ExtractSelfOnFail = false
};
Extractor extractor = new Extractor();
foreach (var fso in extractor.ExtractFile(path, opts).Select(fileEntry => FileEntryToFileSystemObject(fileEntry)))
{
HandleChange(fso);
}
}
// TODO: Also try parse .DER as a key
if (path.EndsWith(".cer", StringComparison.CurrentCulture) ||
path.EndsWith(".der", StringComparison.CurrentCulture) ||
path.EndsWith(".p7b", StringComparison.CurrentCulture) ||
path.EndsWith(".pfx", StringComparison.CurrentCulture))
{
try
{
using var certificate = new X509Certificate2(path);
var certObj = new CertificateObject(
StoreLocation: StoreLocation.LocalMachine.ToString(),
StoreName: StoreName.Root.ToString(),
Certificate: new SerializableCertificate(certificate));
HandleChange(certObj);
}
catch (Exception e)
{
Log.Verbose("Could not parse certificate from file: {0} ({1}:{2})", path, e.GetType(), e.Message);
}
}
}
Log.Verbose("Finished parsing {0}", path);
}
/// <summary>
/// On macos we use the keychain and export the certificates as .pem. However, on macos
/// Certificate2 doesn't support loading from a pem. So first we need pkcs12s instead, we
/// convert using openssl, which requires we set a password we import the pkcs12 with all
/// our certs, delete the temp files and then iterate over it the certs
/// </summary>
internal void ExecuteMacOs(CancellationToken cancellationToken)
{
try
{
if (ExternalCommandRunner.RunExternalCommand("security", "find-certificate -ap /System/Library/Keychains/SystemRootCertificates.keychain", out string result, out string _) == 0)
{
string tmpPath = Path.Combine(Directory.GetCurrentDirectory(), "tmpcert.pem");
string pkPath = Path.Combine(Directory.GetCurrentDirectory(), "tmpcert.pk12");
File.WriteAllText(tmpPath, result);
if (ExternalCommandRunner.RunExternalCommand("openssl", $"pkcs12 -export -nokeys -out {pkPath} -passout pass:pass -in {tmpPath}", out string _, out string _) == 0)
{
X509Certificate2Collection xcert = new X509Certificate2Collection();
xcert.Import(pkPath, "pass", X509KeyStorageFlags.DefaultKeySet); //lgtm [cs/hardcoded-credentials]
File.Delete(tmpPath);
File.Delete(pkPath);
var X509Certificate2Enumerator = xcert.GetEnumerator();
while (X509Certificate2Enumerator.MoveNext())
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var certificate = X509Certificate2Enumerator.Current;
var obj = new CertificateObject(
StoreLocation: StoreLocation.LocalMachine.ToString(),
StoreName: StoreName.Root.ToString(),
Certificate: new SerializableCertificate(certificate));
HandleChange(obj);
}
}
else
{
Log.Debug("Failed to export certificate with OpenSSL."); //DevSkim: ignore DS440000
}
}
/// <summary>
/// On macos we use the keychain and export the certificates as .pem.
/// However, on macos Certificate2 doesn't support loading from a pem.
/// So first we need pkcs12s instead, we convert using openssl, which requires we set a password
/// we import the pkcs12 with all our certs, delete the temp files and then iterate over it the certs
/// </summary>
public void ExecuteMacOs()
{
try
{
if (ExternalCommandRunner.RunExternalCommand("security", "find-certificate -ap /System/Library/Keychains/SystemRootCertificates.keychain", out string result, out string _) == 0)
{
string tmpPath = Path.Combine(Directory.GetCurrentDirectory(), "tmpcert.pem");
string pkPath = Path.Combine(Directory.GetCurrentDirectory(), "tmpcert.pk12");
File.WriteAllText(tmpPath, result);
if (ExternalCommandRunner.RunExternalCommand("openssl", $"pkcs12 -export -nokeys -out {pkPath} -passout pass:pass -in {tmpPath}", out string _, out string _) == 0)
{
X509Certificate2Collection xcert = new X509Certificate2Collection();
xcert.Import(pkPath, "pass", X509KeyStorageFlags.DefaultKeySet); //lgtm [cs/hardcoded-credentials]
File.Delete(tmpPath);
File.Delete(pkPath);
var X509Certificate2Enumerator = xcert.GetEnumerator();
while (X509Certificate2Enumerator.MoveNext())
{
var certificate = X509Certificate2Enumerator.Current;
var obj = new CertificateObject(
StoreLocation: StoreLocation.LocalMachine.ToString(),
StoreName: StoreName.Root.ToString(),
Certificate: new SerializableCertificate(certificate))
{
Pkcs7 = Convert.ToBase64String(certificate.Export(X509ContentType.Cert))
};
DatabaseManager.Write(obj, RunId);
}
}
else
{
Log.Debug("Failed to export certificate with OpenSSL.");
}
}
/// <summary>
/// On macos we use the keychain and export the certificates as .pem.
/// However, on macos Certificate2 doesn't support loading from a pem.
/// So first we need pkcs12s instead, we convert using openssl, which requires we set a password
/// we import the pkcs12 with all our certs, delete the temp files and then iterate over it the certs
/// </summary>
public void ExecuteMacOs()
{
try
{
var result = ExternalCommandRunner.RunExternalCommand("security", new string[] { "find-certificate", "-ap", "/System/Library/Keychains/SystemRootCertificates.keychain" });
string tmpPath = Path.Combine(Directory.GetCurrentDirectory(), "tmpcert.pem");
string pkPath = Path.Combine(Directory.GetCurrentDirectory(), "tmpcert.pk12");
File.WriteAllText(tmpPath, result);
_ = ExternalCommandRunner.RunExternalCommand("openssl", new string[] { "pkcs12", "-export", "-nokeys", "-out", pkPath, "-passout pass:pass", "-in", tmpPath });
X509Certificate2Collection xcert = new X509Certificate2Collection();
xcert.Import(pkPath, "pass", X509KeyStorageFlags.DefaultKeySet);
File.Delete(tmpPath);
File.Delete(pkPath);
var X509Certificate2Enumerator = xcert.GetEnumerator();
while (X509Certificate2Enumerator.MoveNext())
{
var obj = new CertificateObject()
{
StoreLocation = StoreLocation.LocalMachine.ToString(),
StoreName = StoreName.Root.ToString(),
CertificateHashString = X509Certificate2Enumerator.Current.GetCertHashString(),
Subject = X509Certificate2Enumerator.Current.Subject,
Pkcs12 = X509Certificate2Enumerator.Current.GetRawCertDataString()
};
DatabaseManager.Write(obj, this.runId);
}
}
catch (Exception e)
{
Log.Error("Failed to dump certificates from 'security' or 'openssl'.");
Logger.DebugException(e);
}
}
protected override void ProcessRecord()
{
try
{
DenConfigController denConfigController = new DenConfigController(this.Path, this.Key);
DenConfig config = denConfigController.GetConfig();
if (this.ParameterSetName == PKCS12)
{
CertificateObject cert = KeyCertUtils.GetPkcs12CertificateInfo(CertificatePath, this.KeyPassword);
config.DenTraefikConfigObject.Certificate = cert.Certificate;
config.DenTraefikConfigObject.PrivateKey = cert.Privatekey;
}
else
{
config.DenTraefikConfigObject.Certificate = File.ReadAllText(CertificatePath);
config.DenTraefikConfigObject.PrivateKey = File.ReadAllText(PrivateKeyPath);
}
if (string.IsNullOrEmpty(config.DenTraefikConfigObject.Certificate))
{
this.OnError(new Exception("No certificate found"));
}
if (string.IsNullOrEmpty(config.DenTraefikConfigObject.PrivateKey))
{
this.OnError(new Exception("No private key found"));
}
denConfigController.StoreConfig(config);
}
catch (Exception e)
{
this.OnError(e);
}
}
/// <summary>
/// On linux we check the central trusted root store (a folder), which has symlinks to
/// actual cert locations scattered across the db We list all the certificates and then
/// create a new X509Certificate2 object for each by filename.
/// </summary>
internal void ExecuteLinux(CancellationToken cancellationToken)
{
try
{
if (ExternalCommandRunner.RunExternalCommand("ls", "/etc/ssl/certs -A", out string result, out string _) == 0)
{
foreach (var _line in result.Split('\n'))
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
Log.Debug("{0}", _line);
try
{
using X509Certificate2 certificate = new X509Certificate2("/etc/ssl/certs/" + _line);
var obj = new CertificateObject(
StoreLocation: StoreLocation.LocalMachine.ToString(),
StoreName: StoreName.Root.ToString(),
Certificate: new SerializableCertificate(certificate));
HandleChange(obj);
}
catch (Exception e)
{
Log.Debug("{0} {1} Issue creating certificate based on /etc/ssl/certs/{2}", e.GetType().ToString(), e.Message, _line);
Log.Debug("{0}", e.StackTrace);
}
}
}
}
catch (Exception e)
{
Log.Error(e, "Failed to dump certificates from 'ls /etc/ssl/certs -A'.");
}
}
/// <summary>
/// On linux we check the central trusted root store (a folder), which has symlinks to actual cert locations scattered across the db
/// We list all the certificates and then create a new X509Certificate2 object for each by filename.
/// </summary>
public void ExecuteLinux()
{
try
{
var result = ExternalCommandRunner.RunExternalCommand("ls", new string[] { "/etc/ssl/certs", "-A" });
foreach (var _line in result.Split('\n'))
{
Log.Debug("{0}", _line);
try
{
X509Certificate2 certificate = new X509Certificate2("/etc/ssl/certs/" + _line);
var obj = new CertificateObject()
{
StoreLocation = StoreLocation.LocalMachine.ToString(),
StoreName = StoreName.Root.ToString(),
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs12 = certificate.HasPrivateKey ? "redacted" : certificate.Export(X509ContentType.Pkcs12).ToString()
};
DatabaseManager.Write(obj, this.runId);
}
catch (Exception e)
{
Log.Debug("{0} {1} Issue creating certificate based on /etc/ssl/certs/{2}", e.GetType().ToString(), e.Message, _line);
Log.Debug("{0}", e.StackTrace);
}
}
}
catch (Exception e)
{
Log.Error("Failed to dump certificates from 'ls /etc/ssl/certs -A'.");
Logger.DebugException(e);
}
}
public ActionResult Index(CertificateObject certificateObject)
{
foreach (var certificateFile in certificateObject.myfile)
{
if (certificateFile == null)
{
ViewBag.Error = "File cannot be empty.";
return(View());
}
else if (!IsValidContentType(certificateFile.ContentType))
{
ViewBag.Error = "only pdf files allowed.";
return(View());
}
else if (!IsValidFileSize(certificateFile.ContentLength))
{
ViewBag.Error = "File too large";
return(View());
}
}
string result = AddFiletoDB(certificateObject);
if (result != "success")
{
ViewBag.Error = result;
return(View());
}
ViewBag.Message = "Successful";
//if (certificateObject == null)
//{
// ViewBag.Error = "File cannot be empty.";
// //return View(listofFiles);
//}
//else if (!IsValidContentType(certificateObject.myfile.ContentType))
//{
// ViewBag.Error = "only pdf files allowed.";
// //return View(listofFiles);
//}
//else if (!IsValidFileSize(certificateObject.myfile.ContentLength))
//{
// ViewBag.Error = "File too large";
// //return View(listofFiles);
//}
//else
//{
// ViewBag.Message = "Successful";
// string result = AddFiletoDB(certificateObject.myfile);
// if (result != "success")
// {
// ViewBag.Error = result;
// }
//}
return(View());
}
public override void ExecuteInternal()
{
if (!roots.Any())
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (var driveInfo in DriveInfo.GetDrives())
{
if (driveInfo.IsReady && driveInfo.DriveType == DriveType.Fixed)
{
roots.Add(driveInfo.Name);
}
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
roots.Add("/");
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
roots.Add("/");
}
}
Action <string>?IterateOnDirectory = null;
IterateOnDirectory = Path =>
{
Log.Verbose("Started parsing {0}", Path);
// To optimize calls to du on non-windows platforms we run du on the whole directory ahead of time
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var exitCode = ExternalCommandRunner.RunExternalCommand("du", Path, out string StdOut, out string StdErr);
if (exitCode == 0)
{
foreach (var line in StdOut.Split(Environment.NewLine))
{
var fields = line.Split('\t');
if (long.TryParse(fields[0], out long result))
{
sizesOnDisk[fields[1]] = result;
}
}
}
}
var files = Directory.EnumerateFiles(Path, "*", new System.IO.EnumerationOptions()
{
IgnoreInaccessible = true
});
foreach (var file in files)
{
StallIfHighMemoryUsageAndLowMemoryModeEnabled();
Log.Verbose("Started parsing {0}", file);
FileSystemObject obj = FilePathToFileSystemObject(file);
if (obj != null)
{
Results.Push(obj);
// TODO: Also try parse .DER as a key
if (Path.EndsWith(".cer", StringComparison.CurrentCulture) ||
Path.EndsWith(".der", StringComparison.CurrentCulture) ||
Path.EndsWith(".p7b", StringComparison.CurrentCulture) ||
Path.EndsWith(".pfx", StringComparison.CurrentCulture))
{
try
{
using var certificate = new X509Certificate2(Path);
var certObj = new CertificateObject(
StoreLocation: StoreLocation.LocalMachine.ToString(),
StoreName: StoreName.Root.ToString(),
Certificate: new SerializableCertificate(certificate));
Results.Push(certObj);
}
catch (Exception e)
{
Log.Verbose($"Could not parse certificate from file: {file}, {e.GetType().ToString()}");
}
}
}
Log.Verbose("Finished parsing {0}", file);
}
Log.Verbose("Finished parsing {0}", Path);
};
foreach (var root in roots)
{
Log.Information("{0} root {1}", Strings.Get("Scanning"), root);
var directories = Directory.EnumerateDirectories(root, "*", new System.IO.EnumerationOptions()
{
ReturnSpecialDirectories = false,
IgnoreInaccessible = true,
RecurseSubdirectories = true
});
//First do root
IterateOnDirectory?.Invoke(root);
if (!opts.SingleThread == true)
{
Parallel.ForEach(directories, filePath =>
{
IterateOnDirectory?.Invoke(filePath);
});
}
else
{
foreach (var filePath in directories)
{
IterateOnDirectory?.Invoke(filePath);
}
}
}
}
public override void ExecuteInternal()
{
if (roots == null || !roots.Any())
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (var driveInfo in DriveInfo.GetDrives())
{
if (driveInfo.IsReady && driveInfo.DriveType == DriveType.Fixed)
{
roots.Add(driveInfo.Name);
}
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
roots.Add("/");
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
roots.Add("/");
}
}
Action <string> IterateOn = Path =>
{
Log.Verbose("Started parsing {0}", Path);
FileSystemObject obj = FilePathToFileSystemObject(Path, downloadCloud, INCLUDE_CONTENT_HASH);
if (obj != null)
{
DatabaseManager.Write(obj, RunId);
if (examineCertificates &&
Path.EndsWith(".cer", StringComparison.CurrentCulture) ||
Path.EndsWith(".der", StringComparison.CurrentCulture) ||
Path.EndsWith(".p7b", StringComparison.CurrentCulture))
{
try
{
var certificate = X509Certificate.CreateFromCertFile(Path);
var certObj = new CertificateObject()
{
StoreLocation = Path,
StoreName = "Disk",
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs7 = certificate.Export(X509ContentType.Cert).ToString()
};
DatabaseManager.Write(certObj, RunId);
}
catch (Exception e) when(
e is System.Security.Cryptography.CryptographicException ||
e is ArgumentException)
{
Log.Verbose($"Could not parse certificate from file: {Path}");
}
}
}
Log.Verbose("Finished parsing {0}", Path);
};
foreach (var root in roots)
{
Log.Information("{0} root {1}", Strings.Get("Scanning"), root);
var filePathEnumerable = DirectoryWalker.WalkDirectory(root);
if (parallel)
{
Parallel.ForEach(filePathEnumerable,
(filePath =>
{
IterateOn(filePath);
}));
}
else
{
foreach (var filePath in filePathEnumerable)
{
IterateOn(filePath);
}
}
}
}
public override void ExecuteInternal()
{
if (!roots.Any())
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (var driveInfo in DriveInfo.GetDrives())
{
if (driveInfo.IsReady && driveInfo.DriveType == DriveType.Fixed)
{
roots.Add(driveInfo.Name);
}
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
roots.Add("/");
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
roots.Add("/");
}
}
Action <string> IterateOn = Path =>
{
Log.Verbose("Started parsing {0}", Path);
FileSystemObject obj = FilePathToFileSystemObject(Path, downloadCloud, INCLUDE_CONTENT_HASH);
if (obj != null)
{
Results.Enqueue(obj);
if (examineCertificates &&
Path.EndsWith(".cer", StringComparison.CurrentCulture) ||
Path.EndsWith(".der", StringComparison.CurrentCulture) ||
Path.EndsWith(".p7b", StringComparison.CurrentCulture))
{
try
{
using var certificate = new X509Certificate2();
certificate.Import(Path);
var certObj = new CertificateObject(
StoreLocation: StoreLocation.LocalMachine.ToString(),
StoreName: StoreName.Root.ToString(),
Certificate: new SerializableCertificate(certificate))
{
Pkcs7 = Convert.ToBase64String(certificate.Export(X509ContentType.Cert))
};
Results.Enqueue(certObj);
}
catch (Exception e) when(
e is System.Security.Cryptography.CryptographicException ||
e is ArgumentException)
{
Log.Verbose($"Could not parse certificate from file: {Path}");
}
}
}
Log.Verbose("Finished parsing {0}", Path);
};
foreach (var root in roots)
{
Log.Information("{0} root {1}", Strings.Get("Scanning"), root);
var filePathEnumerable = DirectoryWalker.WalkDirectory(root);
if (parallel)
{
Parallel.ForEach(filePathEnumerable,
(filePath =>
{
IterateOn(filePath);
}));
}
else
{
foreach (var filePath in filePathEnumerable)
{
IterateOn(filePath);
}
}
}
}
public override void Execute()
{
if (!CanRunOnPlatform())
{
return;
}
Start();
if (roots == null || !roots.Any())
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (var driveInfo in DriveInfo.GetDrives())
{
if (driveInfo.IsReady && driveInfo.DriveType == DriveType.Fixed)
{
roots.Add(driveInfo.Name);
}
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
roots.Add("/"); // @TODO Improve this
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
roots.Add("/"); // @TODO Improve this
}
}
foreach (var root in roots)
{
Log.Information("{0} root {1}", Strings.Get("Scanning"), root.ToString());
//Ensure the transaction is started to prevent collisions on the multithreaded code ahead
_ = DatabaseManager.Transaction;
try
{
var fileInfoEnumerable = DirectoryWalker.WalkDirectory(root);
Parallel.ForEach(fileInfoEnumerable,
(fileInfo =>
{
try
{
FileSystemObject obj = FileSystemInfoToFileSystemObject(fileInfo, downloadCloud, INCLUDE_CONTENT_HASH);
if (obj != null)
{
DatabaseManager.Write(obj, runId);
if (examineCertificates &&
fileInfo.FullName.EndsWith(".cer", StringComparison.CurrentCulture) ||
fileInfo.FullName.EndsWith(".der", StringComparison.CurrentCulture) ||
fileInfo.FullName.EndsWith(".p7b", StringComparison.CurrentCulture))
{
var certificate = X509Certificate.CreateFromCertFile(fileInfo.FullName);
var certObj = new CertificateObject()
{
StoreLocation = fileInfo.FullName,
StoreName = "Disk",
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs7 = certificate.Export(X509ContentType.Cert).ToString()
};
DatabaseManager.Write(certObj, runId);
}
}
}
catch (Exception e)
{
Logger.DebugException(e);
}
}));
}
catch (Exception e)
{
Log.Warning(e, "Error collecting file system information: {0}", e.Message);
}
DatabaseManager.Commit();
}
Stop();
}
public override void Execute()
{
if (!CanRunOnPlatform())
{
return;
}
Start();
if (gatherFromFiles)
{
var roots = new List <string>();
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (var driveInfo in DriveInfo.GetDrives())
{
if (driveInfo.IsReady && driveInfo.DriveType == DriveType.Fixed)
{
roots.Add(driveInfo.Name);
}
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
roots.Add("/"); // @TODO Improve this
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
roots.Add("/"); // @TODO Improve this
}
foreach (var root in roots)
{
var fileInfoEnumerable = DirectoryWalker.WalkDirectory(root, "Certificate");
Parallel.ForEach(fileInfoEnumerable,
(fileInfo =>
{
try
{
if (fileInfo is DirectoryInfo)
{
return;
}
if (fileInfo.FullName.EndsWith(".cer", StringComparison.CurrentCulture))
{
var certificate = X509Certificate.CreateFromCertFile(fileInfo.FullName);
var obj = new CertificateObject()
{
StoreLocation = fileInfo.FullName,
StoreName = "Disk",
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs12 = certificate.Export(X509ContentType.Pkcs12).ToString()
};
DatabaseManager.Write(obj, this.runId);
}
}
catch (Exception e)
{
Log.Debug("Couldn't parse certificate file {0}", fileInfo.FullName);
Log.Debug("{0} {1}-{2}", e.GetType().ToString(), e.Message, e.StackTrace);
}
}));
}
}
// On Windows we can use the .NET API to iterate through all the stores.
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (StoreLocation storeLocation in (StoreLocation[])Enum.GetValues(typeof(StoreLocation)))
{
foreach (StoreName storeName in (StoreName[])Enum.GetValues(typeof(StoreName)))
{
try
{
X509Store store = new X509Store(storeName, storeLocation);
store.Open(OpenFlags.ReadOnly);
foreach (X509Certificate2 certificate in store.Certificates)
{
var obj = new CertificateObject()
{
StoreLocation = storeLocation.ToString(),
StoreName = storeName.ToString(),
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs12 = certificate.HasPrivateKey ? "redacted" : certificate.Export(X509ContentType.Pkcs12).ToString()
};
DatabaseManager.Write(obj, this.runId);
}
store.Close();
}
catch (Exception e)
{
Logger.DebugException(e);
Telemetry.TrackTrace(Microsoft.ApplicationInsights.DataContracts.SeverityLevel.Error, e);
}
}
}
}
// On linux we check the central trusted root store (a folder), which has symlinks to actual cert locations scattered across the db
// We list all the certificates and then create a new X509Certificate2 object for each by filename.
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
try
{
var result = ExternalCommandRunner.RunExternalCommand("ls", new string[] { "/etc/ssl/certs", "-A" });
foreach (var _line in result.Split('\n'))
{
Log.Debug("{0}", _line);
try
{
X509Certificate2 certificate = new X509Certificate2("/etc/ssl/certs/" + _line);
var obj = new CertificateObject()
{
StoreLocation = StoreLocation.LocalMachine.ToString(),
StoreName = StoreName.Root.ToString(),
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs12 = certificate.HasPrivateKey ? "redacted" : certificate.Export(X509ContentType.Pkcs12).ToString()
};
DatabaseManager.Write(obj, this.runId);
}
catch (Exception e)
{
Log.Debug("{0} {1} Issue creating certificate based on /etc/ssl/certs/{2}", e.GetType().ToString(), e.Message, _line);
Log.Debug("{0}", e.StackTrace);
}
}
}
catch (Exception e)
{
Log.Error("Failed to dump certificates from 'ls /etc/ssl/certs -A'.");
Logger.DebugException(e);
}
}
// On macos we use the keychain and export the certificates as .pem.
// However, on macos Certificate2 doesn't support loading from a pem,
// so first we need pkcs12s instead, we convert using openssl, which requires we set a password
// we import the pkcs12 with all our certs, delete the temp files and then iterate over it the certs
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
try
{
var result = ExternalCommandRunner.RunExternalCommand("security", new string[] { "find-certificate", "-ap", "/System/Library/Keychains/SystemRootCertificates.keychain" });
string tmpPath = Path.Combine(Directory.GetCurrentDirectory(), "tmpcert.pem");
string pkPath = Path.Combine(Directory.GetCurrentDirectory(), "tmpcert.pk12");
File.WriteAllText(tmpPath, result);
_ = ExternalCommandRunner.RunExternalCommand("openssl", new string[] { "pkcs12", "-export", "-nokeys", "-out", pkPath, "-passout pass:pass", "-in", tmpPath });
X509Certificate2Collection xcert = new X509Certificate2Collection();
xcert.Import(pkPath, "pass", X509KeyStorageFlags.DefaultKeySet);
File.Delete(tmpPath);
File.Delete(pkPath);
var X509Certificate2Enumerator = xcert.GetEnumerator();
while (X509Certificate2Enumerator.MoveNext())
{
var obj = new CertificateObject()
{
StoreLocation = StoreLocation.LocalMachine.ToString(),
StoreName = StoreName.Root.ToString(),
CertificateHashString = X509Certificate2Enumerator.Current.GetCertHashString(),
Subject = X509Certificate2Enumerator.Current.Subject,
Pkcs12 = X509Certificate2Enumerator.Current.HasPrivateKey ? "redacted" : X509Certificate2Enumerator.Current.Export(X509ContentType.Pkcs12).ToString()
};
DatabaseManager.Write(obj, this.runId);
}
}
catch (Exception e)
{
Log.Error("Failed to dump certificates from 'security' or 'openssl'.");
Logger.DebugException(e);
}
}
DatabaseManager.Commit();
Stop();
}
public override void ExecuteInternal()
{
if (!roots.Any())
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (var driveInfo in DriveInfo.GetDrives())
{
if (driveInfo.IsReady && driveInfo.DriveType == DriveType.Fixed)
{
roots.Add(driveInfo.Name);
}
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
roots.Add("/");
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
roots.Add("/");
}
}
Action <string> IterateOn = Path =>
{
Log.Verbose("Started parsing {0}", Path);
FileSystemObject obj = FilePathToFileSystemObject(Path, downloadCloud, INCLUDE_CONTENT_HASH);
if (obj != null)
{
Results.Add(obj);
// TODO: Also try parse .DER as a key
if (Path.EndsWith(".cer", StringComparison.CurrentCulture) ||
Path.EndsWith(".der", StringComparison.CurrentCulture) ||
Path.EndsWith(".p7b", StringComparison.CurrentCulture) ||
Path.EndsWith(".pfx", StringComparison.CurrentCulture))
{
try
{
using var certificate = new X509Certificate2(Path);
var certObj = new CertificateObject(
StoreLocation: StoreLocation.LocalMachine.ToString(),
StoreName: StoreName.Root.ToString(),
Certificate: new SerializableCertificate(certificate));
Results.Add(certObj);
}
catch (Exception e)
{
Log.Verbose($"Could not parse certificate from file: {Path}, {e.GetType().ToString()}");
}
}
}
Log.Verbose("Finished parsing {0}", Path);
};
foreach (var root in roots)
{
Log.Information("{0} root {1}", Strings.Get("Scanning"), root);
var filePathEnumerable = DirectoryWalker.WalkDirectory(root);
if (parallel)
{
filePathEnumerable.AsParallel().ForAll(filePath =>
{
IterateOn(filePath);
});
}
else
{
foreach (var filePath in filePathEnumerable)
{
IterateOn(filePath);
}
}
}
}
public override void Execute()
{
if (!CanRunOnPlatform())
{
return;
}
Start();
if (roots == null || !roots.Any())
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (var driveInfo in DriveInfo.GetDrives())
{
if (driveInfo.IsReady && driveInfo.DriveType == DriveType.Fixed)
{
roots.Add(driveInfo.Name);
}
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
roots.Add("/"); // @TODO Improve this
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
roots.Add("/"); // @TODO Improve this
}
}
foreach (var root in roots)
{
Log.Information("{0} root {1}", Strings.Get("Scanning"), root.ToString());
//Ensure the transaction is started to prevent collisions on the multithreaded code ahead
_ = DatabaseManager.Transaction;
try
{
var fileInfoEnumerable = DirectoryWalker.WalkDirectory(root);
Parallel.ForEach(fileInfoEnumerable,
(fileInfo =>
{
try
{
FileSystemObject obj = null;
if (fileInfo is DirectoryInfo)
{
obj = new FileSystemObject()
{
Path = fileInfo.FullName,
Permissions = FileSystemUtils.GetFilePermissions(fileInfo),
IsDirectory = true
};
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
var file = new UnixFileInfo(fileInfo.FullName);
obj.Owner = file.OwnerUser.UserName;
obj.Group = file.OwnerGroup.GroupName;
obj.SetGid = file.IsSetGroup;
obj.SetUid = file.IsSetUser;
}
}
else
{
obj = new FileSystemObject()
{
Path = fileInfo.FullName,
Permissions = FileSystemUtils.GetFilePermissions(fileInfo),
Size = (ulong)(fileInfo as FileInfo).Length,
IsDirectory = false
};
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
var file = new UnixFileInfo(fileInfo.FullName);
obj.Owner = file.OwnerUser.UserName;
obj.Group = file.OwnerGroup.GroupName;
obj.SetGid = file.IsSetGroup;
obj.SetUid = file.IsSetUser;
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
if (WindowsFileSystemUtils.IsLocal(obj.Path) || downloadCloud)
{
if (WindowsFileSystemUtils.NeedsSignature(obj.Path))
{
obj.SignatureStatus = WindowsFileSystemUtils.GetSignatureStatus(fileInfo.FullName);
obj.Characteristics = WindowsFileSystemUtils.GetDllCharacteristics(fileInfo.FullName);
}
else
{
obj.SignatureStatus = "Cloud";
}
}
}
if (INCLUDE_CONTENT_HASH)
{
obj.ContentHash = FileSystemUtils.GetFileHash(fileInfo);
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
if (obj.Permissions.Contains("Execute"))
{
obj.IsExecutable = true;
}
}
else
{
try
{
if (WindowsFileSystemUtils.IsLocal(obj.Path) || downloadCloud)
{
if (WindowsFileSystemUtils.NeedsSignature(obj.Path))
{
obj.IsExecutable = true;
}
}
}
catch (System.UnauthorizedAccessException ex)
{
Log.Verbose(ex, "Couldn't access {0} to check if signature is needed.", fileInfo.FullName);
}
}
}
if (obj != null)
{
DatabaseManager.Write(obj, runId);
if (examineCertificates &&
fileInfo.FullName.EndsWith(".cer", StringComparison.CurrentCulture) ||
fileInfo.FullName.EndsWith(".der", StringComparison.CurrentCulture) ||
fileInfo.FullName.EndsWith(".p7b", StringComparison.CurrentCulture))
{
var certificate = X509Certificate.CreateFromCertFile(fileInfo.FullName);
var certObj = new CertificateObject()
{
StoreLocation = fileInfo.FullName,
StoreName = "Disk",
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs7 = certificate.Export(X509ContentType.Cert).ToString()
};
DatabaseManager.Write(certObj, runId);
}
}
}
catch (System.UnauthorizedAccessException e)
{
Log.Verbose(e, "Access Denied {0}", fileInfo?.FullName);
}
catch (System.IO.IOException e)
{
Log.Verbose(e, "Couldn't parse {0}", fileInfo?.FullName);
}
catch (Exception e)
{
Logger.DebugException(e);
}
}));
}
catch (Exception e)
{
Log.Warning(e, "Error collecting file system information: {0}", e.Message);
}
DatabaseManager.Commit();
}
Stop();
}
public override void ExecuteInternal()
{
if (roots == null || !roots.Any())
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
foreach (var driveInfo in DriveInfo.GetDrives())
{
if (driveInfo.IsReady && driveInfo.DriveType == DriveType.Fixed)
{
roots.Add(driveInfo.Name);
}
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
roots.Add("/");
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
roots.Add("/");
}
}
foreach (var root in roots)
{
Log.Information("{0} root {1}", Strings.Get("Scanning"), root.ToString(CultureInfo.InvariantCulture));
//Ensure the transaction is started to prevent collisions on the multithreaded code ahead
var fileInfoEnumerable = DirectoryWalker.WalkDirectory(root);
Parallel.ForEach(fileInfoEnumerable,
(fileInfo =>
{
FileSystemObject obj = FileSystemInfoToFileSystemObject(fileInfo, downloadCloud, INCLUDE_CONTENT_HASH);
if (obj != null)
{
DatabaseManager.Write(obj, RunId);
if (examineCertificates &&
fileInfo.FullName.EndsWith(".cer", StringComparison.CurrentCulture) ||
fileInfo.FullName.EndsWith(".der", StringComparison.CurrentCulture) ||
fileInfo.FullName.EndsWith(".p7b", StringComparison.CurrentCulture))
{
try
{
var certificate = X509Certificate.CreateFromCertFile(fileInfo.FullName);
var certObj = new CertificateObject()
{
StoreLocation = fileInfo.FullName,
StoreName = "Disk",
CertificateHashString = certificate.GetCertHashString(),
Subject = certificate.Subject,
Pkcs7 = certificate.Export(X509ContentType.Cert).ToString()
};
DatabaseManager.Write(certObj, RunId);
}
catch (Exception e) when(
e is System.Security.Cryptography.CryptographicException ||
e is ArgumentException)
{
Log.Debug($"Could not parse certificate from file: {fileInfo.FullName}");
}
}
}
}));
}
}
private string AddFiletoDB(CertificateObject fileList)
{
#region Old
/*
* HttpFileCollection filesColl = Request.Files;
* using (CustomersEntities db = new CustomersEntities())
* {
* foreach (string uploader in filesColl)
* {
* HttpPostedFile file = filesColl[uploader];
*
* if (file.ContentLength > 0)
* {
* BinaryReader br = new BinaryReader(file.InputStream);
*
* byte[] buffer = br.ReadBytes(file.ContentLength);
*
* db.tblFilestoDbs.Add(new tblFilestoDb
* {
* FileName = file.FileName,
* ContentType = file.ContentType,
* FileExtension = Path.GetExtension(file.FileName),
* FileSize = file.ContentLength,
* FileContent = buffer
* });
* }
* }
* db.SaveChanges();
* }
*/
#endregion
List <FilestoDb> filestoAddtoDb = new List <FilestoDb>();
string response = string.Empty;
try
{
using (CustomersEntities db = new CustomersEntities())
{
foreach (var fileObject in fileList.myfile)
{
if (fileObject.ContentLength > 0)
{
BinaryReader br = new BinaryReader(fileObject.InputStream);
byte[] buffer = br.ReadBytes(fileObject.ContentLength);
filestoAddtoDb.Add(new FilestoDb()
{
FileName = fileObject.FileName,
ContentType = fileObject.ContentType,
FileExtension = Path.GetExtension(fileObject.FileName),
FileSize = fileObject.ContentLength,
FileContent = buffer
});
}
}
db.FilestoDbs.AddRange(filestoAddtoDb);
db.SaveChanges();
response = "success";
return(response);
}
}
catch (Exception ex)
{
response = ex.Message;
return(response);
}
}