private void DeleteFilesAndDirsExcept(string fileToKeep, string dirToKeep, ITracer tracer)
{
// Best effort. Using the "Safe" variants does retries and swallows exceptions but
// we may catch something non-obvious.
try
{
var files = FileSystemHelpers.GetFiles(_environment.ZipTempPath, "*")
.Where(p => !PathUtilityFactory.Instance.PathsEquals(p, fileToKeep));
foreach (var file in files)
{
FileSystemHelpers.DeleteFileSafe(file);
}
var dirs = FileSystemHelpers.GetDirectories(_environment.ZipTempPath)
.Where(p => !PathUtilityFactory.Instance.PathsEquals(p, dirToKeep));
foreach (var dir in dirs)
{
FileSystemHelpers.DeleteDirectorySafe(dir);
}
}
catch (Exception ex)
{
tracer.TraceError(ex, "Exception encountered during zip folder cleanup");
throw;
}
}
public void TracerMaxXmlFilesTest()
{
// Mock
var path = @"x:\kudu\trace";
var tracer = new XmlTracer(path, TraceLevel.Verbose);
FileSystemHelpers.Instance = GetMockFileSystem();
var total = XmlTracer.MaxXmlFiles + 10;
for (int i = 0; i < total; ++i)
{
tracer.Trace(Guid.NewGuid().ToString(), new Dictionary <string, string>());
}
var files = FileSystemHelpers.GetFiles(path, "*.xml");
Assert.Equal(total, files.Length);
// wait till interval and write another trace
typeof(XmlTracer).GetField("_lastCleanup", BindingFlags.Static | BindingFlags.NonPublic).SetValue(null, DateTime.MinValue);
tracer.Trace(Guid.NewGuid().ToString(), new Dictionary <string, string>());
files = FileSystemHelpers.GetFiles(path, "*.xml");
Assert.True(files.Length < XmlTracer.MaxXmlFiles);
}
private void EnsureMaxXmlFiles()
{
var now = DateTime.UtcNow;
if (_lastCleanup.AddSeconds(CleanUpIntervalSecs) > now)
{
return;
}
_lastCleanup = now;
try
{
var files = FileSystemHelpers.GetFiles(_path, "*.xml");
if (files.Length < MaxXmlFiles)
{
return;
}
foreach (var file in files.OrderBy(n => n).Take(files.Length - (MaxXmlFiles * 80) / 100))
{
FileSystemHelpers.DeleteFileSafe(file);
}
}
catch
{
// no-op
}
}
public static void PurgeBuildArtifactsIfNecessary(string sitePackagesPath, BuildArtifactType fileExtension, ITracer tracer, int totalAllowedFiles)
{
string extension = fileExtension.ToString().ToLowerInvariant();
IEnumerable <string> fileNames = FileSystemHelpers.GetFiles(sitePackagesPath, $"*.{extension}");
if (fileNames.Count() > totalAllowedFiles)
{
// Order the files in descending order of the modified date and remove the last (N - allowed zip files).
var fileNamesToDelete = fileNames.OrderByDescending(fileName => FileSystemHelpers.GetLastWriteTimeUtc(fileName)).Skip(totalAllowedFiles);
foreach (var fileName in fileNamesToDelete)
{
using (tracer.Step("Deleting outdated zip file {0}", fileName))
{
try
{
File.Delete(fileName);
}
catch (Exception ex)
{
tracer.TraceError(ex, "Unable to delete zip file {0}", fileName);
}
}
}
}
}
public static async Task BuildDotnetProject(string outputPath, string dotnetCliParams)
{
EnsureDotnet();
var csProjFiles = FileSystemHelpers.GetFiles(Environment.CurrentDirectory, searchPattern: "*.csproj").ToList();
if (csProjFiles.Count == 1)
{
if (FileSystemHelpers.DirectoryExists(outputPath))
{
FileSystemHelpers.DeleteDirectorySafe(outputPath);
}
var exe = new Executable("dotnet", $"build --output {outputPath} {dotnetCliParams}");
var exitCode = await exe.RunAsync(o => ColoredConsole.WriteLine(o), e => ColoredConsole.Error.WriteLine(e));
if (exitCode != 0)
{
throw new CliException("Error building project");
}
}
else
{
throw new CliException($"Can't determine Project to build. Expected 1 .csproj but found {csProjFiles.Count}");
}
}
private void RunDeterminePrimaryScriptFileFunc(string expect, string jObjectStr, string dir)
{
JObject functionConfig = JObject.Parse(jObjectStr);
var traceFactoryMock = new Mock <ITraceFactory>();
traceFactoryMock.Setup(tf => tf.GetTracer()).Returns(NullTracer.Instance);
var functionManager = new FunctionManager(new Mock <IEnvironment>().Object, traceFactoryMock.Object);
string[] functionFiles = FileSystemHelpers.GetFiles(dir, "*.*", SearchOption.TopDirectoryOnly)
.Where(p => !String.Equals(Path.GetFileName(p), Constants.FunctionsConfigFile, StringComparison.OrdinalIgnoreCase))
.ToArray();
string scriptFile = (string)functionConfig["scriptFile"];
string scriptPath = string.IsNullOrEmpty(scriptFile) ? null : Path.Combine(dir, scriptFile);
if ((string.IsNullOrEmpty(scriptFile) && functionFiles.Length == 0) || expect == null)
{
Assert.Null(functionManager.DeterminePrimaryScriptFile((string)functionConfig["scriptFile"], dir));
return;
}
if (!string.IsNullOrEmpty(scriptPath) && !FileSystemHelpers.FileExists(scriptPath))
{
Assert.Throws <ConfigurationErrorsException>(() => functionManager.DeterminePrimaryScriptFile((string)functionConfig["scriptFile"], dir));
}
else
{
Assert.Equal(expect, functionManager.DeterminePrimaryScriptFile((string)functionConfig["scriptFile"], dir), StringComparer.OrdinalIgnoreCase);
}
}
/// <summary>
/// <para>Loop thru all site extension settings files under 'siteExtensionStatusRoot', check if there is any successful installation</para>
/// <para>if not install to webroot, will trigger restart; if install to webroot and with applicationHost.xdt file, trigger restart.</para>
/// </summary>
/// <param name="siteExtensionStatusRoot">should be $ROOT\site\siteextensions</param>
/// <param name="siteExtensionRoot">should be $ROOT\SiteExtensions</param>
public static bool IsAnyInstallationRequireRestart(string siteExtensionStatusRoot, string siteExtensionRoot, ITracer tracer, IAnalytics analytics)
{
try
{
string[] packageDirs = FileSystemHelpers.GetDirectories(siteExtensionStatusRoot);
foreach (var dir in packageDirs)
{
try
{
DirectoryInfo dirInfo = new DirectoryInfo(dir);
string[] settingFile = FileSystemHelpers.GetFiles(dir, _statusSettingsFileName);
foreach (var file in settingFile)
{
var statusSettings = new SiteExtensionStatus(siteExtensionStatusRoot, dirInfo.Name, tracer);
if (statusSettings.IsSiteExtensionRequireRestart(siteExtensionRoot))
{
return(true);
}
}
}
catch (Exception ex)
{
analytics.UnexpectedException(ex, trace: false);
tracer.TraceError(ex, "Failed to query {0} under {1}, continus to check others ...", _statusSettingsFileName, dir);
}
}
}
catch (Exception ex)
{
analytics.UnexpectedException(ex, trace: false);
tracer.TraceError(ex, "Not able to query directory under {0}", siteExtensionStatusRoot);
}
return(false);
}
private void EnsureMaxXmlFiles()
{
// _lastCleanup ensures we only check (iterate) the log directory every certain period.
// _cleanupPending ensures there is one cleanup thread at a time.
lock (_cleanupLock)
{
if (_cleanupPending)
{
return;
}
var now = DateTime.UtcNow;
if (_lastCleanup.AddSeconds(CleanUpIntervalSecs) > now)
{
return;
}
_lastCleanup = now;
_cleanupPending = true;
}
try
{
ITracer tracer = this;
using (tracer.Step("Cleanup Xml Logs"))
{
try
{
var files = FileSystemHelpers.GetFiles(_path, "*.xml");
if (files.Length < MaxXmlFiles)
{
return;
}
var toCleanup = files.Length - (MaxXmlFiles * 80) / 100;
var attribs = new Dictionary <string, string>
{
{ "totalFiles", files.Length.ToString() },
{ "totalCleanup", toCleanup.ToString() }
};
using (tracer.Step("Cleanup Infos", attribs))
{
foreach (var file in files.OrderBy(n => n).Take(toCleanup))
{
FileSystemHelpers.DeleteFileSafe(file);
}
}
}
catch (Exception ex)
{
tracer.TraceError(ex);
}
}
}
finally
{
_cleanupPending = false;
}
}
private string GetFunctionScriptPath(string functionName, JObject functionConfig)
{
var functionPath = GetFunctionPath(functionName);
var functionFiles = FileSystemHelpers.GetFiles(functionPath, "*.*", SearchOption.TopDirectoryOnly)
.Where(p => Path.GetFileName(p).ToLowerInvariant() != "function.json").ToArray();
return(DeterminePrimaryScriptFile(functionConfig, functionFiles));
}
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, BuildOption buildOption, string additionalPackages)
{
var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt);
if (!FileSystemHelpers.FileExists(reqTxtFile))
{
throw new CliException($"{Constants.RequirementsTxt} is not found. " +
$"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing.");
}
var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages);
if (FileSystemHelpers.DirectoryExists(packagesLocation))
{
// Only update packages if checksum of requirements.txt does not match
// If build option is remote, we don't need to verify if packages are in sync, as we need to delete them regardless
if (buildOption != BuildOption.Remote && await ArePackagesInSync(reqTxtFile, packagesLocation))
{
ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies..."));
return(await ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory");
FileSystemHelpers.DeleteDirectorySafe(packagesLocation);
}
FileSystemHelpers.EnsureDirectory(packagesLocation);
// Only one of the remote build or build-native-deps flag can be chosen
if (buildNativeDeps)
{
if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess())
{
await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages);
}
else
{
throw new CliException("Docker is required to build native dependencies for python function apps");
}
}
else if (buildOption == BuildOption.Remote)
{
// No-ops, python packages will be resolved on the server side
}
else
{
await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation);
}
// No need to generate and compare .md5 when using remote build
if (buildOption != BuildOption.Remote)
{
// Store a checksum of requirements.txt
var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5");
await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile));
}
return(await ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
private Boolean IsFolderModified(JObject fileObj, string directoryPath)
{
//Fetch all files in this directory
string[] filePaths = FileSystemHelpers.GetFiles(directoryPath, "*");
if (filePaths != null)
{
foreach (string filePath in filePaths)
{
Console.WriteLine(filePath);
//If manifest does not contain an entry for this file
//It means the file was newly added
//We need to scan as this is a modification
JToken val = null;
if (!fileObj.TryGetValue(filePath, out val))
{
return(true);
}
//Modified time in manifest
String lastModTime = (string)fileObj[filePath];
//Current modified time
String currModTime = FileSystemHelpers.GetDirectoryLastWriteTimeUtc(filePath).ToString();
//If they are different
//It means file has been modified after last scan
if (!currModTime.Equals(lastModTime))
{
return(true);
}
}
}
//Fetch all the child directories of this directory
string[] direcPaths = FileSystemHelpers.GetDirectories(directoryPath);
if (direcPaths != null)
{
//Do recursive comparison of all files in the child directories
foreach (string direcPath in direcPaths)
{
if (IsFolderModified(fileObj, direcPath))
{
return(true);
}
}
}
//No modifications found
return(false);
}
// Logic for this function is copied from here:
// https://github.com/Azure/azure-webjobs-sdk-script/blob/dev/src/WebJobs.Script/Host/ScriptHost.cs
// These two implementations must stay in sync!
/// <summary>
/// Determines which script should be considered the "primary" entry point script.
/// </summary>
/// <exception cref="ConfigurationErrorsException">Thrown if the function metadata points to an invalid script file, or no script files are present.</exception>
internal string DeterminePrimaryScriptFile(JObject functionConfig, string scriptDirectory)
{
// First see if there is an explicit primary file indicated
// in config. If so use that.
string functionPrimary = null;
string scriptFile = (string)functionConfig["scriptFile"];
if (!string.IsNullOrEmpty(scriptFile))
{
string scriptPath = Path.Combine(scriptDirectory, scriptFile);
if (!FileSystemHelpers.FileExists(scriptPath))
{
TraceAndThrowError(new ConfigurationErrorsException("Invalid script file name configuration. The 'scriptFile' property is set to a file that does not exist."));
}
functionPrimary = scriptPath;
}
else
{
string[] functionFiles = FileSystemHelpers.GetFiles(scriptDirectory, "*.*", SearchOption.TopDirectoryOnly)
.Where(p => !String.Equals(Path.GetFileName(p), "function.json", StringComparison.OrdinalIgnoreCase))
.ToArray();
if (functionFiles.Length == 0)
{
TraceAndThrowError(new ConfigurationErrorsException("No function script files present."));
}
if (functionFiles.Length == 1)
{
// if there is only a single file, that file is primary
functionPrimary = functionFiles[0];
}
else
{
// if there is a "run" file, that file is primary,
// for Node, any index.js file is primary
functionPrimary = functionFiles.FirstOrDefault(p =>
String.Equals(Path.GetFileNameWithoutExtension(p), "run", StringComparison.OrdinalIgnoreCase) ||
String.Equals(Path.GetFileName(p), "index.js", StringComparison.OrdinalIgnoreCase));
}
}
if (string.IsNullOrEmpty(functionPrimary))
{
TraceAndThrowError(new ConfigurationErrorsException("Unable to determine the primary function script. Try renaming your entry point script to 'run' (or 'index' in the case of Node), " +
"or alternatively you can specify the name of the entry point script explicitly by adding a 'scriptFile' property to your function metadata."));
}
return(Path.GetFullPath(functionPrimary));
}
public void TracerRequestsTest(TraceLevel traceLevel, RequestInfo[] requests)
{
// Mock
var path = @"x:\kudu\trace";
var tracer = new XmlTracer(path, traceLevel);
FileSystemHelpers.Instance = GetMockFileSystem();
try
{
foreach (var request in requests)
{
Dictionary <string, string> attribs = new Dictionary <string, string>
{
{ "type", "request" },
{ "url", request.Url },
{ "method", "GET" },
{ "statusCode", ((int)request.StatusCode).ToString() },
};
using (tracer.Step("Incoming Request", attribs))
{
tracer.Trace("Outgoing response", attribs);
}
}
var traces = new List <RequestInfo>();
foreach (var file in FileSystemHelpers.GetFiles(path, "*s.xml").OrderBy(n => n))
{
var document = XDocument.Load(FileSystemHelpers.OpenRead(file));
var trace = new RequestInfo {
Url = document.Root.Attribute("url").Value
};
var elapsed = document.Root.Nodes().Last();
// Assert
Assert.Equal(XmlNodeType.Comment, elapsed.NodeType);
Assert.Contains("duration:", ((XComment)elapsed).Value);
traces.Add(trace);
}
// Assert
Assert.Equal(requests.Where(r => r.Traced), traces, RequestInfoComparer.Instance);
}
catch (Exception)
{
FileSystemHelpers.Instance = null;
throw;
}
}
public static string GetCsproj()
{
EnsureDotnet();
var csProjFiles = FileSystemHelpers.GetFiles(Environment.CurrentDirectory, searchPattern: "*.csproj").ToList();
if (csProjFiles.Count == 1)
{
return(csProjFiles.First());
}
else
{
throw new CliException($"Can't determin Project to build. Expected 1 .csproj but found {csProjFiles.Count}");
}
}
// Logic for this function is copied from here:
// https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Host/FunctionMetadataProvider.cs
// These two implementations must stay in sync!
/// <summary>
/// Determines which script should be considered the "primary" entry point script. Returns null if Primary script file cannot be determined
/// </summary>
internal string DeterminePrimaryScriptFile(string scriptFile, string scriptDirectory)
{
// First see if there is an explicit primary file indicated
// in config. If so use that.
string functionPrimary = null;
if (!string.IsNullOrEmpty(scriptFile))
{
string scriptPath = Path.Combine(scriptDirectory, scriptFile);
if (!FileSystemHelpers.FileExists(scriptPath))
{
TraceAndThrowError(new ConfigurationErrorsException("Invalid script file name configuration. The 'scriptFile' property is set to a file that does not exist."));
}
functionPrimary = scriptPath;
}
else
{
string[] functionFiles = FileSystemHelpers.GetFiles(scriptDirectory, "*.*", SearchOption.TopDirectoryOnly)
.Where(p => !String.Equals(Path.GetFileName(p), Constants.FunctionsConfigFile, StringComparison.OrdinalIgnoreCase))
.ToArray();
if (functionFiles.Length == 0)
{
return(null);
}
if (functionFiles.Length == 1)
{
// if there is only a single file, that file is primary
functionPrimary = functionFiles[0];
}
else
{
// if there is a "run" file, that file is primary,
// for Node, any index.js file is primary
functionPrimary = functionFiles.FirstOrDefault(p =>
String.Equals(Path.GetFileNameWithoutExtension(p), "run", StringComparison.OrdinalIgnoreCase) ||
String.Equals(Path.GetFileName(p), "index.js", StringComparison.OrdinalIgnoreCase));
}
}
// Primary script file is not required for HttpWorker or any custom language worker
if (string.IsNullOrEmpty(functionPrimary))
{
return(null);
}
return(Path.GetFullPath(functionPrimary));
}
public static bool LooksLikePython(string siteFolder)
{
if (!OSDetector.IsOnWindows())
{
// For Linux web apps: Rely on WEBSITE_PYTHON_VERSION environment variable to
// detect if this is a python app
string pythonVersion = System.Environment.GetEnvironmentVariable("WEBSITE_PYTHON_VERSION");
if (!string.IsNullOrEmpty(pythonVersion) && pythonVersion.StartsWith("3", StringComparison.OrdinalIgnoreCase))
{
return(true);
}
return(false);
}
var reqsFilePath = Path.Combine(siteFolder, RequirementsFileName);
if (!FileSystemHelpers.FileExists(reqsFilePath))
{
return(false);
}
var pythonFiles = FileSystemHelpers.GetFiles(siteFolder, PythonFileExtension);
if (pythonFiles.Length > 0)
{
return(true);
}
// Most Python sites will have at least a .py file in the root, but
// some may not. In that case, let them opt in with the runtime.txt
// file, which is used to specify the version of Python.
var runtimeFilePath = Path.Combine(siteFolder, RuntimeFileName);
if (FileSystemHelpers.FileExists(runtimeFilePath))
{
try
{
var text = FileSystemHelpers.ReadAllTextFromFile(runtimeFilePath);
return(text.IndexOf("python", StringComparison.OrdinalIgnoreCase) >= 0);
}
catch (IOException)
{
return(false);
}
}
return(false);
}
private static async Task DotnetTemplatesAction(string action)
{
var templatesLocation = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "templates");
if (!FileSystemHelpers.DirectoryExists(templatesLocation))
{
throw new CliException($"Can't find templates location. Looked under '{templatesLocation}'");
}
foreach (var nupkg in FileSystemHelpers.GetFiles(templatesLocation, null, null, "*.nupkg"))
{
var exe = new Executable("dotnet", $"new --{action} \"{nupkg}\"");
await exe.RunAsync();
}
}
public static bool IsAnyPendingLock(string rootPath, ITracer tracer)
{
bool hasPendingLock = false;
try
{
using (tracer.Step("Checking if there is other pending installation ..."))
{
string[] packageDirs = FileSystemHelpers.GetDirectories(rootPath);
foreach (var dir in packageDirs)
{
string[] lockFiles = FileSystemHelpers.GetFiles(dir, string.Format("*{0}", LockNameSuffix));
foreach (var file in lockFiles)
{
// If there's no file then there's no process holding onto it
if (!FileSystemHelpers.FileExists(file))
{
continue;
}
try
{
// If there is a file, lets see if someone has an open handle to it, or if it's
// just hanging there for no reason
using (FileSystemHelpers.OpenFile(file, FileMode.Open, FileAccess.Write, FileShare.Read)) { }
}
catch
{
hasPendingLock = true;
break;
}
}
if (hasPendingLock)
{
break;
}
}
}
}
catch (Exception ex)
{
// no-op
tracer.TraceError(ex, "Failed to check pending installation lock. Assume there is no pending lock.");
}
return(hasPendingLock);
}
public static bool CanDotnetBuild()
{
EnsureDotnet();
var csProjFiles = FileSystemHelpers.GetFiles(Environment.CurrentDirectory, searchPattern: "*.csproj").ToList();
// If the project name is extensions only then is extensions.csproj a valid csproj file
if (!Path.GetFileName(Environment.CurrentDirectory).Equals("extensions"))
{
csProjFiles.Remove("extensions.csproj");
}
if (csProjFiles.Count > 1)
{
throw new CliException($"Can't determine Project to build. Expected 1 .csproj but found {csProjFiles.Count}");
}
return(csProjFiles.Count == 1);
}
private static IEnumerable <string> GetBindings()
{
var functionJsonfiles = FileSystemHelpers.GetFiles(Environment.CurrentDirectory, searchPattern: Constants.FunctionJsonFileName);
var bindings = new HashSet <string>();
foreach (var functionJson in functionJsonfiles)
{
string functionJsonContents = FileSystemHelpers.ReadAllTextFromFile(functionJson);
var functionMetadata = JsonConvert.DeserializeObject <FunctionMetadata>(functionJsonContents);
foreach (var binding in functionMetadata.Bindings)
{
bindings.Add(binding.Type.ToLower());
}
}
return(bindings);
}
protected override void OnLockRelease()
{
// if installed failed, when release lock, we should also remove empty folder as well
base.OnLockRelease();
try
{
string folder = Path.GetDirectoryName(_path);
if (FileSystemHelpers.DirectoryExists(folder) && FileSystemHelpers.GetFiles(folder, "*").Length == 0)
{
FileSystemHelpers.DeleteDirectorySafe(folder);
}
}
catch
{
// no-op
}
}
/// <summary>
/// Update the status and the detailed status of the job.
/// The status is the status of one of the instances,
/// The detailed status contains status for all instances and looks like:
///
/// aabbcc - Running
/// 112233 - PendingRestart
///
/// </summary>
private void UpdateDetailedStatus(ContinuousJob job, string jobsSpecificDataPath)
{
string instanceId = InstanceIdUtility.GetShortInstanceId();
string[] statusFiles = FileSystemHelpers.GetFiles(jobsSpecificDataPath, StatusFilesSearchPattern);
if (statusFiles.Length <= 0)
{
// If no status files exist update to default values
string status = ContinuousJobStatus.Initializing.Status;
job.DetailedStatus = instanceId + " - " + status;
job.Status = status;
return;
}
string lastStatus = null;
var stringBuilder = new StringBuilder();
foreach (string statusFile in statusFiles)
{
// Extract instance id from the file name
int statusFileNameIndex = statusFile.IndexOf(ContinuousJobStatus.FileNamePrefix, StringComparison.OrdinalIgnoreCase);
string statusFileInstanceId = statusFile.Substring(statusFileNameIndex + ContinuousJobStatus.FileNamePrefix.Length);
// Try to delete file, it'll be deleted if no one holds a lock on it
// That way we know the status file is obsolete
if (String.Equals(statusFileInstanceId, instanceId, StringComparison.OrdinalIgnoreCase) ||
!TryDelete(statusFile))
{
// If we couldn't delete the file, we know it holds the status of an actual instance holding it
var continuousJobStatus = GetStatus <ContinuousJobStatus>(statusFile) ?? ContinuousJobStatus.Initializing;
stringBuilder.AppendLine(statusFileInstanceId + " - " + continuousJobStatus.Status);
if (lastStatus == null ||
!ContinuousJobStatus.InactiveInstance.Equals(continuousJobStatus))
{
lastStatus = continuousJobStatus.Status;
}
}
}
// Job status will only show one of the statuses
job.Status = lastStatus;
job.DetailedStatus = stringBuilder.ToString();
}
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, string additionalPackages)
{
var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt);
if (!FileSystemHelpers.FileExists(reqTxtFile))
{
throw new CliException($"{Constants.RequirementsTxt} is not found. " +
$"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing.");
}
var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages);
if (FileSystemHelpers.DirectoryExists(packagesLocation))
{
// Only update packages if checksum of requirements.txt does not match or a sync is forced
if (await ArePackagesInSync(reqTxtFile, packagesLocation))
{
ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies..."));
return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory");
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(functionAppRoot, Constants.ExternalPythonPackages));
}
FileSystemHelpers.EnsureDirectory(packagesLocation);
if (buildNativeDeps)
{
if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess())
{
await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages);
}
else
{
throw new CliException("Docker is required to build native dependencies for python function apps");
}
}
else
{
await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation);
}
// Store a checksum of requirements.txt
var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5");
await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile));
return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
private static StreamContent CreateZip(string path)
{
var memoryStream = new MemoryStream();
using (var zip = new ZipArchive(memoryStream, ZipArchiveMode.Create, leaveOpen: true))
{
foreach (var fileName in FileSystemHelpers.GetFiles(path, new[] { ".git", ".vscode" }, new[] { ".gitignore", "appsettings.json", "local.settings.json", "project.lock.json" }))
{
zip.AddFile(fileName, fileName, path);
}
}
memoryStream.Seek(0, SeekOrigin.Begin);
var content = new StreamContent(memoryStream);
content.Headers.ContentType = new MediaTypeHeaderValue("application/zip");
return(content);
}
// Logic for this function is copied from here
// https://github.com/Azure/azure-webjobs-sdk-script/blob/e0a783e882dd8680bf23e3c8818fb9638071c21d/src/WebJobs.Script/Config/ScriptHost.cs#L113-L150
private string GetFunctionScriptPath(string functionName, JObject functionInfo)
{
var functionPath = GetFunctionPath(functionName);
var functionFiles = FileSystemHelpers.GetFiles(functionPath, "*.*", SearchOption.TopDirectoryOnly)
.Where(p => Path.GetFileName(p).ToLowerInvariant() != "function.json").ToArray();
if (functionFiles.Length == 0)
{
return(functionPath);
}
else if (functionFiles.Length == 1)
{
// if there is only a single file, that file is primary
return(functionFiles[0]);
}
else
{
// if there is a "run" file, that file is primary
string functionPrimary = null;
functionPrimary = functionFiles.FirstOrDefault(p => Path.GetFileNameWithoutExtension(p).ToLowerInvariant() == "run");
if (string.IsNullOrEmpty(functionPrimary))
{
// for Node, any index.js file is primary
functionPrimary = functionFiles.FirstOrDefault(p => Path.GetFileName(p).ToLowerInvariant() == "index.js");
if (string.IsNullOrEmpty(functionPrimary))
{
// finally, if there is an explicit primary file indicated
// in config, use it
JToken token = functionInfo["source"];
if (token != null)
{
string sourceFileName = (string)token;
functionPrimary = Path.Combine(functionPath, sourceFileName);
}
}
}
if (string.IsNullOrEmpty(functionPrimary))
{
// TODO: should this be an error?
return(functionPath);
}
return(functionPrimary);
}
}
public static bool IsAnyPendingLock(string rootPath)
{
bool hasPendingLock = false;
try
{
string[] packageDirs = FileSystemHelpers.GetDirectories(rootPath);
foreach (var dir in packageDirs)
{
string[] lockFiles = FileSystemHelpers.GetFiles(dir, string.Format("*{0}", LockNameSuffix));
foreach (var file in lockFiles)
{
// If there's no file then there's no process holding onto it
if (!FileSystemHelpers.FileExists(file))
{
continue;
}
try
{
// If there is a file, lets see if someone has an open handle to it, or if it's
// just hanging there for no reason
using (FileSystemHelpers.OpenFile(file, FileMode.Open, FileAccess.Write, FileShare.Read)) { }
}
catch
{
hasPendingLock = true;
break;
}
}
if (hasPendingLock)
{
break;
}
}
}
catch
{
// no-op
}
return(hasPendingLock);
}
public static IEnumerable <string> GetLocalFiles(string path, GitIgnoreParser ignoreParser = null, bool returnIgnored = false)
{
var ignoredDirectories = new[] { ".git", ".vscode" };
var ignoredFiles = new[] { ".funcignore", ".gitignore", "appsettings.json", "local.settings.json", "project.lock.json" };
foreach (var file in FileSystemHelpers.GetFiles(path, ignoredDirectories, ignoredFiles))
{
if (preCondition(file))
{
yield return(file);
}
}
bool preCondition(string file)
{
var fileName = file.Replace(path, string.Empty).Trim(Path.DirectorySeparatorChar).Replace("\\", "/");
return((returnIgnored ? ignoreParser?.Denies(fileName) : ignoreParser?.Accepts(fileName)) ?? true);
}
}
private void ModifyManifestFile(JObject fileObj, string directoryPath)
{
//Get all files in this directory
string[] filePaths = FileSystemHelpers.GetFiles(directoryPath, "*");
foreach (string filePath in filePaths)
{
//Get last modified timestamp of this file
String timeString = FileSystemHelpers.GetDirectoryLastWriteTimeUtc(filePath).ToString();
//Add it as an entry into the manifest
fileObj.Add(filePath, timeString);
}
//Get all child directories of this directory
string[] direcPaths = FileSystemHelpers.GetDirectories(directoryPath);
//Do a recursive call to add files of child directories to manifest
foreach (string direcPath in direcPaths)
{
ModifyManifestFile(fileObj, direcPath);
}
}
public static bool LooksLikePython(string siteFolder)
{
var reqsFilePath = Path.Combine(siteFolder, RequirementsFileName);
if (!FileSystemHelpers.FileExists(reqsFilePath))
{
return(false);
}
var pythonFiles = FileSystemHelpers.GetFiles(siteFolder, PythonFileExtension);
if (pythonFiles.Length > 0)
{
return(true);
}
// Most Python sites will have at least a .py file in the root, but
// some may not. In that case, let them opt in with the runtime.txt
// file, which is used to specify the version of Python.
var runtimeFilePath = Path.Combine(siteFolder, RuntimeFileName);
if (FileSystemHelpers.FileExists(runtimeFilePath))
{
try
{
var text = FileSystemHelpers.ReadAllTextFromFile(runtimeFilePath);
return(text.IndexOf("python", StringComparison.OrdinalIgnoreCase) >= 0);
}
catch (IOException)
{
return(false);
}
}
return(false);
}
public static void PurgeZipsIfNecessary(string sitePackagesPath, ITracer tracer, int totalAllowedZips)
{
IEnumerable <string> zipFiles = FileSystemHelpers.GetFiles(sitePackagesPath, "*.zip");
if (zipFiles.Count() > totalAllowedZips)
{
// Order the files in descending order of the modified date and remove the last (N - allowed zip files).
var fileNamesToDelete = zipFiles.OrderByDescending(fileName => FileSystemHelpers.GetLastWriteTimeUtc(fileName)).Skip(totalAllowedZips);
foreach (var fileName in fileNamesToDelete)
{
using (tracer.Step("Deleting outdated zip file {0}", fileName))
{
try
{
File.Delete(fileName);
}
catch (Exception ex)
{
tracer.TraceError(ex, "Unable to delete zip file {0}", fileName);
}
}
}
}
}
